feat(templates): refactor template processing and GitHub template fetching

- Implement new template processing utility in `processors.ts`
- Simplify GitHub template fetching with a more lightweight approach
- Add comprehensive test suite for template processing
- Improve type safety and modularity of template-related functions

apps/dokploy/__test__/templates/config.template.test.ts

@@ -0,0 +1,341 @@
+import { describe, expect, it } from "vitest";
+import type { CompleteTemplate } from "@dokploy/server/templates/utils/processors";
+import { processTemplate } from "@dokploy/server/templates/utils/processors";
+import type { Schema } from "@dokploy/server/templates/utils";
+
+describe("processTemplate", () => {
+	// Mock schema for testing
+	const mockSchema: Schema = {
+		projectName: "test",
+		serverIp: "127.0.0.1",
+	};
+
+	describe("variables processing", () => {
+		it("should process basic variables with utility functions", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+					secret_base: "${base64:64}",
+					totp_key: "${base64:32}",
+					password: "${password:32}",
+					hash: "${hash:16}",
+				},
+				config: {
+					domains: [],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.envs).toHaveLength(0);
+			expect(result.domains).toHaveLength(0);
+			expect(result.mounts).toHaveLength(0);
+		});
+
+		it("should allow referencing variables in other variables", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+					api_domain: "api.${main_domain}",
+				},
+				config: {
+					domains: [],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.envs).toHaveLength(0);
+			expect(result.domains).toHaveLength(0);
+			expect(result.mounts).toHaveLength(0);
+		});
+	});
+
+	describe("domains processing", () => {
+		it("should process domains with explicit host", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+				},
+				config: {
+					domains: [
+						{
+							serviceName: "plausible",
+							port: 8000,
+							host: "${main_domain}",
+						},
+					],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.domains).toHaveLength(1);
+			const domain = result.domains[0];
+			expect(domain).toBeDefined();
+			if (!domain) return;
+			expect(domain).toMatchObject({
+				serviceName: "plausible",
+				port: 8000,
+			});
+			expect(domain.host).toBeDefined();
+			expect(domain.host).toContain(mockSchema.projectName);
+		});
+
+		it("should generate random domain if host is not specified", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {},
+				config: {
+					domains: [
+						{
+							serviceName: "plausible",
+							port: 8000,
+						},
+					],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.domains).toHaveLength(1);
+			const domain = result.domains[0];
+			expect(domain).toBeDefined();
+			if (!domain || !domain.host) return;
+			expect(domain.host).toBeDefined();
+			expect(domain.host).toContain(mockSchema.projectName);
+		});
+
+		it("should allow using ${randomDomain} directly in host", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {},
+				config: {
+					domains: [
+						{
+							serviceName: "plausible",
+							port: 8000,
+							host: "${randomDomain}",
+						},
+					],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.domains).toHaveLength(1);
+			const domain = result.domains[0];
+			expect(domain).toBeDefined();
+			if (!domain || !domain.host) return;
+			expect(domain.host).toBeDefined();
+			expect(domain.host).toContain(mockSchema.projectName);
+		});
+	});
+
+	describe("environment variables processing", () => {
+		it("should process env vars with variable references", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+					secret_base: "${base64:64}",
+				},
+				config: {
+					domains: [],
+					env: {
+						BASE_URL: "http://${main_domain}",
+						SECRET_KEY_BASE: "${secret_base}",
+					},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.envs).toHaveLength(2);
+			const baseUrl = result.envs.find((env: string) =>
+				env.startsWith("BASE_URL="),
+			);
+			const secretKey = result.envs.find((env: string) =>
+				env.startsWith("SECRET_KEY_BASE="),
+			);
+
+			expect(baseUrl).toBeDefined();
+			expect(secretKey).toBeDefined();
+			if (!baseUrl || !secretKey) return;
+
+			expect(baseUrl).toContain(mockSchema.projectName);
+			expect(secretKey.split("=")[1]).toHaveLength(64);
+		});
+
+		it("should allow using utility functions directly in env vars", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {},
+				config: {
+					domains: [],
+					env: {
+						RANDOM_DOMAIN: "${randomDomain}",
+						SECRET_KEY: "${base64:32}",
+					},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.envs).toHaveLength(2);
+			const randomDomainEnv = result.envs.find((env: string) =>
+				env.startsWith("RANDOM_DOMAIN="),
+			);
+			const secretKeyEnv = result.envs.find((env: string) =>
+				env.startsWith("SECRET_KEY="),
+			);
+			expect(randomDomainEnv).toBeDefined();
+			expect(secretKeyEnv).toBeDefined();
+			if (!randomDomainEnv || !secretKeyEnv) return;
+
+			expect(randomDomainEnv).toContain(mockSchema.projectName);
+			expect(secretKeyEnv.split("=")[1]).toHaveLength(32);
+		});
+	});
+
+	describe("mounts processing", () => {
+		it("should process mounts with variable references", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					config_path: "/etc/config",
+					secret_key: "${base64:32}",
+				},
+				config: {
+					domains: [],
+					env: {},
+					mounts: [
+						{
+							filePath: "${config_path}/config.xml",
+							content: "secret_key=${secret_key}",
+						},
+					],
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.mounts).toHaveLength(1);
+			const mount = result.mounts[0];
+			expect(mount).toBeDefined();
+			if (!mount) return;
+			expect(mount.filePath).toContain("/etc/config");
+			expect(mount.content).toMatch(/secret_key=[A-Za-z0-9+/]{32}/);
+		});
+
+		it("should allow using utility functions directly in mount content", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {},
+				config: {
+					domains: [],
+					env: {},
+					mounts: [
+						{
+							filePath: "/config/secrets.txt",
+							content: "random_domain=${randomDomain}\nsecret=${base64:32}",
+						},
+					],
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.mounts).toHaveLength(1);
+			const mount = result.mounts[0];
+			expect(mount).toBeDefined();
+			if (!mount) return;
+			expect(mount.content).toContain(mockSchema.projectName);
+			expect(mount.content).toMatch(/secret=[A-Za-z0-9+/]{32}/);
+		});
+	});
+
+	describe("complex template processing", () => {
+		it("should process a complete template with all features", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+					secret_base: "${base64:64}",
+					totp_key: "${base64:32}",
+				},
+				config: {
+					domains: [
+						{
+							serviceName: "plausible",
+							port: 8000,
+							host: "${main_domain}",
+						},
+						{
+							serviceName: "api",
+							port: 3000,
+							host: "api.${main_domain}",
+						},
+					],
+					env: {
+						BASE_URL: "http://${main_domain}",
+						SECRET_KEY_BASE: "${secret_base}",
+						TOTP_VAULT_KEY: "${totp_key}",
+					},
+					mounts: [
+						{
+							filePath: "/config/app.conf",
+							content: `
+                domain=\${main_domain}
+                secret=\${secret_base}
+                totp=\${totp_key}
+              `,
+						},
+					],
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+
+			// Check domains
+			expect(result.domains).toHaveLength(2);
+			const [domain1, domain2] = result.domains;
+			expect(domain1).toBeDefined();
+			expect(domain2).toBeDefined();
+			if (!domain1 || !domain2) return;
+			expect(domain1.host).toBeDefined();
+			expect(domain1.host).toContain(mockSchema.projectName);
+			expect(domain2.host).toContain("api.");
+			expect(domain2.host).toContain(mockSchema.projectName);
+
+			// Check env vars
+			expect(result.envs).toHaveLength(3);
+			const baseUrl = result.envs.find((env: string) =>
+				env.startsWith("BASE_URL="),
+			);
+			const secretKey = result.envs.find((env: string) =>
+				env.startsWith("SECRET_KEY_BASE="),
+			);
+			const totpKey = result.envs.find((env: string) =>
+				env.startsWith("TOTP_VAULT_KEY="),
+			);
+
+			expect(baseUrl).toBeDefined();
+			expect(secretKey).toBeDefined();
+			expect(totpKey).toBeDefined();
+			if (!baseUrl || !secretKey || !totpKey) return;
+
+			expect(baseUrl).toContain(mockSchema.projectName);
+			expect(secretKey.split("=")[1]).toHaveLength(64);
+			expect(totpKey.split("=")[1]).toHaveLength(32);
+
+			// Check mounts
+			expect(result.mounts).toHaveLength(1);
+			const mount = result.mounts[0];
+			expect(mount).toBeDefined();
+			if (!mount) return;
+			expect(mount.content).toContain(mockSchema.projectName);
+			expect(mount.content).toMatch(/secret=[A-Za-z0-9+/]{64}/);
+			expect(mount.content).toMatch(/totp=[A-Za-z0-9+/]{32}/);
+		});
+	});
+});

apps/dokploy/server/api/routers/compose.ts

@@ -2,7 +2,6 @@ import { slugify } from "@/lib/slug";
 import { db } from "@/server/db";
 import {
 	apiCreateCompose,
-	apiCreateComposeByTemplate,
 	apiDeleteCompose,
 	apiFetchServices,
 	apiFindCompose,
@@ -12,14 +11,12 @@ import {
 } from "@/server/db/schema";
 import { cleanQueuesByCompose, myQueue } from "@/server/queues/queueSetup";
 import { templates } from "@/templates/templates";
-import type { TemplatesKeys } from "@/templates/types/templates-data.type";
 import { generatePassword } from "@/templates/utils";
 import {
 	fetchTemplateFiles,
 	fetchTemplatesList,
-	processTemplate,
 } from "@dokploy/server/templates/utils/github";
-import { readTemplateComposeFile } from "@dokploy/server/templates/utils";
+import { processTemplate } from "@dokploy/server/templates/utils/processors";
 import { TRPCError } from "@trpc/server";
 import { eq } from "drizzle-orm";
 import { dump } from "js-yaml";
@@ -27,7 +24,6 @@ import _ from "lodash";
 import { nanoid } from "nanoid";
 import { createTRPCRouter, protectedProcedure, publicProcedure } from "../trpc";
 import { z } from "zod";
-
 import type { DeploymentJob } from "@/server/queues/queue-types";
 import { deploy } from "@/server/utils/deploy";
 import {
@@ -58,29 +54,6 @@ import {
 	updateCompose,
 } from "@dokploy/server";
 
-import { z } from "zod";
-import type { TemplateConfig } from "@dokploy/server/types/template";
-
-// Add the template config schema
-const templateConfigSchema = z.object({
-	variables: z.record(z.string()),
-	domains: z.array(
-		z.object({
-			serviceName: z.string(),
-			port: z.number(),
-			path: z.string().optional(),
-			host: z.string().optional(),
-		}),
-	),
-	env: z.record(z.string()),
-	mounts: z.array(
-		z.object({
-			filePath: z.string(),
-			content: z.string(),
-		}),
-	),
-}) satisfies z.ZodType<TemplateConfig>;
-
 export const composeRouter = createTRPCRouter({
 	create: protectedProcedure
 		.input(apiCreateCompose)
@@ -466,62 +439,59 @@ export const composeRouter = createTRPCRouter({
 				projectName: project.name,
 			});
 
-			console.log(generate.domains);
-			console.log(generate.envs);
-			console.log(generate.mounts);
+			const projectName = slugify(`${project.name} ${input.id}`);
 
-			// const projectName = slugify(`${project.name} ${input.id}`);
-			// const { envs, mounts, domains } = await generate({
-			// 	serverIp: serverIp || "",
-			// 	projectName: projectName,
-			// });
+			const compose = await createComposeByTemplate({
+				...input,
+				composeFile: template.dockerCompose,
+				env: generate.envs?.join("\n"),
+				serverId: input.serverId,
+				name: input.id,
+				sourceType: "raw",
+				appName: `${projectName}-${generatePassword(6)}`,
+				isolatedDeployment: true,
+			});
 
-			// const compose = await createComposeByTemplate({
-			// 	...input,
-			// 	composeFile: composeFile,
-			// 	env: envs?.join("\n"),
-			// 	serverId: input.serverId,
-			// 	name: input.id,
-			// 	sourceType: "raw",
-			// 	appName: `${projectName}-${generatePassword(6)}`,
-			// 	isolatedDeployment: true,
-			// });
+			if (ctx.user.rol === "member") {
+				await addNewService(
+					ctx.user.id,
+					compose.composeId,
+					ctx.session.activeOrganizationId,
+				);
+			}
 
-			// if (ctx.user.rol === "user") {
-			// 	await addNewService(ctx.user.authId, compose.composeId);
-			// }
+			if (generate.mounts && generate.mounts?.length > 0) {
+				for (const mount of generate.mounts) {
+					await createMount({
+						filePath: mount.filePath,
+						mountPath: "",
+						content: mount.content,
+						serviceId: compose.composeId,
+						serviceType: "compose",
+						type: "file",
+					});
+				}
+			}
 
-			// if (mounts && mounts?.length > 0) {
-			// 	for (const mount of mounts) {
-			// 		await createMount({
-			// 			filePath: mount.filePath,
-			// 			mountPath: "",
-			// 			content: mount.content,
-			// 			serviceId: compose.composeId,
-			// 			serviceType: "compose",
-			// 			type: "file",
-			// 		});
-			// 	}
-			// }
-
-			// if (domains && domains?.length > 0) {
-			// 	for (const domain of domains) {
-			// 		await createDomain({
-			// 			...domain,
-			// 			domainType: "compose",
-			// 			certificateType: "none",
-			// 			composeId: compose.composeId,
-			// 		});
-			// 	}
-			// }
+			if (generate.domains && generate.domains?.length > 0) {
+				for (const domain of generate.domains) {
+					await createDomain({
+						...domain,
+						domainType: "compose",
+						certificateType: "none",
+						composeId: compose.composeId,
+						host: domain.host || "",
+					});
+				}
+			}
 
 			return null;
 		}),
 
 	templates: publicProcedure.query(async () => {
-		// First try to fetch templates from GitHub
 		try {
 			const githubTemplates = await fetchTemplatesList();
+
 			if (githubTemplates.length > 0) {
 				return githubTemplates;
 			}
@@ -531,8 +501,6 @@ export const composeRouter = createTRPCRouter({
 				error,
 			);
 		}
-
-		// Fall back to local templates if GitHub fetch fails
 		return [];
 	}),