feat(templates): refactor template processing and GitHub template fetching

- Implement new template processing utility in `processors.ts`
- Simplify GitHub template fetching with a more lightweight approach
- Add comprehensive test suite for template processing
- Improve type safety and modularity of template-related functions

apps/dokploy/__test__/templates/config.template.test.ts

@@ -0,0 +1,341 @@
+import { describe, expect, it } from "vitest";
+import type { CompleteTemplate } from "@dokploy/server/templates/utils/processors";
+import { processTemplate } from "@dokploy/server/templates/utils/processors";
+import type { Schema } from "@dokploy/server/templates/utils";
+
+describe("processTemplate", () => {
+	// Mock schema for testing
+	const mockSchema: Schema = {
+		projectName: "test",
+		serverIp: "127.0.0.1",
+	};
+
+	describe("variables processing", () => {
+		it("should process basic variables with utility functions", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+					secret_base: "${base64:64}",
+					totp_key: "${base64:32}",
+					password: "${password:32}",
+					hash: "${hash:16}",
+				},
+				config: {
+					domains: [],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.envs).toHaveLength(0);
+			expect(result.domains).toHaveLength(0);
+			expect(result.mounts).toHaveLength(0);
+		});
+
+		it("should allow referencing variables in other variables", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+					api_domain: "api.${main_domain}",
+				},
+				config: {
+					domains: [],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.envs).toHaveLength(0);
+			expect(result.domains).toHaveLength(0);
+			expect(result.mounts).toHaveLength(0);
+		});
+	});
+
+	describe("domains processing", () => {
+		it("should process domains with explicit host", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+				},
+				config: {
+					domains: [
+						{
+							serviceName: "plausible",
+							port: 8000,
+							host: "${main_domain}",
+						},
+					],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.domains).toHaveLength(1);
+			const domain = result.domains[0];
+			expect(domain).toBeDefined();
+			if (!domain) return;
+			expect(domain).toMatchObject({
+				serviceName: "plausible",
+				port: 8000,
+			});
+			expect(domain.host).toBeDefined();
+			expect(domain.host).toContain(mockSchema.projectName);
+		});
+
+		it("should generate random domain if host is not specified", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {},
+				config: {
+					domains: [
+						{
+							serviceName: "plausible",
+							port: 8000,
+						},
+					],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.domains).toHaveLength(1);
+			const domain = result.domains[0];
+			expect(domain).toBeDefined();
+			if (!domain || !domain.host) return;
+			expect(domain.host).toBeDefined();
+			expect(domain.host).toContain(mockSchema.projectName);
+		});
+
+		it("should allow using ${randomDomain} directly in host", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {},
+				config: {
+					domains: [
+						{
+							serviceName: "plausible",
+							port: 8000,
+							host: "${randomDomain}",
+						},
+					],
+					env: {},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.domains).toHaveLength(1);
+			const domain = result.domains[0];
+			expect(domain).toBeDefined();
+			if (!domain || !domain.host) return;
+			expect(domain.host).toBeDefined();
+			expect(domain.host).toContain(mockSchema.projectName);
+		});
+	});
+
+	describe("environment variables processing", () => {
+		it("should process env vars with variable references", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+					secret_base: "${base64:64}",
+				},
+				config: {
+					domains: [],
+					env: {
+						BASE_URL: "http://${main_domain}",
+						SECRET_KEY_BASE: "${secret_base}",
+					},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.envs).toHaveLength(2);
+			const baseUrl = result.envs.find((env: string) =>
+				env.startsWith("BASE_URL="),
+			);
+			const secretKey = result.envs.find((env: string) =>
+				env.startsWith("SECRET_KEY_BASE="),
+			);
+
+			expect(baseUrl).toBeDefined();
+			expect(secretKey).toBeDefined();
+			if (!baseUrl || !secretKey) return;
+
+			expect(baseUrl).toContain(mockSchema.projectName);
+			expect(secretKey.split("=")[1]).toHaveLength(64);
+		});
+
+		it("should allow using utility functions directly in env vars", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {},
+				config: {
+					domains: [],
+					env: {
+						RANDOM_DOMAIN: "${randomDomain}",
+						SECRET_KEY: "${base64:32}",
+					},
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.envs).toHaveLength(2);
+			const randomDomainEnv = result.envs.find((env: string) =>
+				env.startsWith("RANDOM_DOMAIN="),
+			);
+			const secretKeyEnv = result.envs.find((env: string) =>
+				env.startsWith("SECRET_KEY="),
+			);
+			expect(randomDomainEnv).toBeDefined();
+			expect(secretKeyEnv).toBeDefined();
+			if (!randomDomainEnv || !secretKeyEnv) return;
+
+			expect(randomDomainEnv).toContain(mockSchema.projectName);
+			expect(secretKeyEnv.split("=")[1]).toHaveLength(32);
+		});
+	});
+
+	describe("mounts processing", () => {
+		it("should process mounts with variable references", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					config_path: "/etc/config",
+					secret_key: "${base64:32}",
+				},
+				config: {
+					domains: [],
+					env: {},
+					mounts: [
+						{
+							filePath: "${config_path}/config.xml",
+							content: "secret_key=${secret_key}",
+						},
+					],
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.mounts).toHaveLength(1);
+			const mount = result.mounts[0];
+			expect(mount).toBeDefined();
+			if (!mount) return;
+			expect(mount.filePath).toContain("/etc/config");
+			expect(mount.content).toMatch(/secret_key=[A-Za-z0-9+/]{32}/);
+		});
+
+		it("should allow using utility functions directly in mount content", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {},
+				config: {
+					domains: [],
+					env: {},
+					mounts: [
+						{
+							filePath: "/config/secrets.txt",
+							content: "random_domain=${randomDomain}\nsecret=${base64:32}",
+						},
+					],
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+			expect(result.mounts).toHaveLength(1);
+			const mount = result.mounts[0];
+			expect(mount).toBeDefined();
+			if (!mount) return;
+			expect(mount.content).toContain(mockSchema.projectName);
+			expect(mount.content).toMatch(/secret=[A-Za-z0-9+/]{32}/);
+		});
+	});
+
+	describe("complex template processing", () => {
+		it("should process a complete template with all features", () => {
+			const template: CompleteTemplate = {
+				metadata: {} as any,
+				variables: {
+					main_domain: "${randomDomain}",
+					secret_base: "${base64:64}",
+					totp_key: "${base64:32}",
+				},
+				config: {
+					domains: [
+						{
+							serviceName: "plausible",
+							port: 8000,
+							host: "${main_domain}",
+						},
+						{
+							serviceName: "api",
+							port: 3000,
+							host: "api.${main_domain}",
+						},
+					],
+					env: {
+						BASE_URL: "http://${main_domain}",
+						SECRET_KEY_BASE: "${secret_base}",
+						TOTP_VAULT_KEY: "${totp_key}",
+					},
+					mounts: [
+						{
+							filePath: "/config/app.conf",
+							content: `
+                domain=\${main_domain}
+                secret=\${secret_base}
+                totp=\${totp_key}
+              `,
+						},
+					],
+				},
+			};
+
+			const result = processTemplate(template, mockSchema);
+
+			// Check domains
+			expect(result.domains).toHaveLength(2);
+			const [domain1, domain2] = result.domains;
+			expect(domain1).toBeDefined();
+			expect(domain2).toBeDefined();
+			if (!domain1 || !domain2) return;
+			expect(domain1.host).toBeDefined();
+			expect(domain1.host).toContain(mockSchema.projectName);
+			expect(domain2.host).toContain("api.");
+			expect(domain2.host).toContain(mockSchema.projectName);
+
+			// Check env vars
+			expect(result.envs).toHaveLength(3);
+			const baseUrl = result.envs.find((env: string) =>
+				env.startsWith("BASE_URL="),
+			);
+			const secretKey = result.envs.find((env: string) =>
+				env.startsWith("SECRET_KEY_BASE="),
+			);
+			const totpKey = result.envs.find((env: string) =>
+				env.startsWith("TOTP_VAULT_KEY="),
+			);
+
+			expect(baseUrl).toBeDefined();
+			expect(secretKey).toBeDefined();
+			expect(totpKey).toBeDefined();
+			if (!baseUrl || !secretKey || !totpKey) return;
+
+			expect(baseUrl).toContain(mockSchema.projectName);
+			expect(secretKey.split("=")[1]).toHaveLength(64);
+			expect(totpKey.split("=")[1]).toHaveLength(32);
+
+			// Check mounts
+			expect(result.mounts).toHaveLength(1);
+			const mount = result.mounts[0];
+			expect(mount).toBeDefined();
+			if (!mount) return;
+			expect(mount.content).toContain(mockSchema.projectName);
+			expect(mount.content).toMatch(/secret=[A-Za-z0-9+/]{64}/);
+			expect(mount.content).toMatch(/totp=[A-Za-z0-9+/]{32}/);
+		});
+	});
+});

apps/dokploy/server/api/routers/compose.ts

@@ -2,7 +2,6 @@ import { slugify } from "@/lib/slug";
 import { db } from "@/server/db";
 import {
 	apiCreateCompose,
-	apiCreateComposeByTemplate,
 	apiDeleteCompose,
 	apiFetchServices,
 	apiFindCompose,
@@ -12,14 +11,12 @@ import {
 } from "@/server/db/schema";
 import { cleanQueuesByCompose, myQueue } from "@/server/queues/queueSetup";
 import { templates } from "@/templates/templates";
-import type { TemplatesKeys } from "@/templates/types/templates-data.type";
 import { generatePassword } from "@/templates/utils";
 import {
 	fetchTemplateFiles,
 	fetchTemplatesList,
-	processTemplate,
 } from "@dokploy/server/templates/utils/github";
-import { readTemplateComposeFile } from "@dokploy/server/templates/utils";
+import { processTemplate } from "@dokploy/server/templates/utils/processors";
 import { TRPCError } from "@trpc/server";
 import { eq } from "drizzle-orm";
 import { dump } from "js-yaml";
@@ -27,7 +24,6 @@ import _ from "lodash";
 import { nanoid } from "nanoid";
 import { createTRPCRouter, protectedProcedure, publicProcedure } from "../trpc";
 import { z } from "zod";
-
 import type { DeploymentJob } from "@/server/queues/queue-types";
 import { deploy } from "@/server/utils/deploy";
 import {
@@ -58,29 +54,6 @@ import {
 	updateCompose,
 } from "@dokploy/server";
 
-import { z } from "zod";
-import type { TemplateConfig } from "@dokploy/server/types/template";
-
-// Add the template config schema
-const templateConfigSchema = z.object({
-	variables: z.record(z.string()),
-	domains: z.array(
-		z.object({
-			serviceName: z.string(),
-			port: z.number(),
-			path: z.string().optional(),
-			host: z.string().optional(),
-		}),
-	),
-	env: z.record(z.string()),
-	mounts: z.array(
-		z.object({
-			filePath: z.string(),
-			content: z.string(),
-		}),
-	),
-}) satisfies z.ZodType<TemplateConfig>;
-
 export const composeRouter = createTRPCRouter({
 	create: protectedProcedure
 		.input(apiCreateCompose)
@@ -466,62 +439,59 @@ export const composeRouter = createTRPCRouter({
 				projectName: project.name,
 			});
 
-			console.log(generate.domains);
-			console.log(generate.envs);
-			console.log(generate.mounts);
+			const projectName = slugify(`${project.name} ${input.id}`);
 
-			// const projectName = slugify(`${project.name} ${input.id}`);
-			// const { envs, mounts, domains } = await generate({
-			// 	serverIp: serverIp || "",
-			// 	projectName: projectName,
-			// });
+			const compose = await createComposeByTemplate({
+				...input,
+				composeFile: template.dockerCompose,
+				env: generate.envs?.join("\n"),
+				serverId: input.serverId,
+				name: input.id,
+				sourceType: "raw",
+				appName: `${projectName}-${generatePassword(6)}`,
+				isolatedDeployment: true,
+			});
 
-			// const compose = await createComposeByTemplate({
-			// 	...input,
-			// 	composeFile: composeFile,
-			// 	env: envs?.join("\n"),
-			// 	serverId: input.serverId,
-			// 	name: input.id,
-			// 	sourceType: "raw",
-			// 	appName: `${projectName}-${generatePassword(6)}`,
-			// 	isolatedDeployment: true,
-			// });
+			if (ctx.user.rol === "member") {
+				await addNewService(
+					ctx.user.id,
+					compose.composeId,
+					ctx.session.activeOrganizationId,
+				);
+			}
 
-			// if (ctx.user.rol === "user") {
-			// 	await addNewService(ctx.user.authId, compose.composeId);
-			// }
+			if (generate.mounts && generate.mounts?.length > 0) {
+				for (const mount of generate.mounts) {
+					await createMount({
+						filePath: mount.filePath,
+						mountPath: "",
+						content: mount.content,
+						serviceId: compose.composeId,
+						serviceType: "compose",
+						type: "file",
+					});
+				}
+			}
 
-			// if (mounts && mounts?.length > 0) {
-			// 	for (const mount of mounts) {
-			// 		await createMount({
-			// 			filePath: mount.filePath,
-			// 			mountPath: "",
-			// 			content: mount.content,
-			// 			serviceId: compose.composeId,
-			// 			serviceType: "compose",
-			// 			type: "file",
-			// 		});
-			// 	}
-			// }
-
-			// if (domains && domains?.length > 0) {
-			// 	for (const domain of domains) {
-			// 		await createDomain({
-			// 			...domain,
-			// 			domainType: "compose",
-			// 			certificateType: "none",
-			// 			composeId: compose.composeId,
-			// 		});
-			// 	}
-			// }
+			if (generate.domains && generate.domains?.length > 0) {
+				for (const domain of generate.domains) {
+					await createDomain({
+						...domain,
+						domainType: "compose",
+						certificateType: "none",
+						composeId: compose.composeId,
+						host: domain.host || "",
+					});
+				}
+			}
 
 			return null;
 		}),
 
 	templates: publicProcedure.query(async () => {
-		// First try to fetch templates from GitHub
 		try {
 			const githubTemplates = await fetchTemplatesList();
+
 			if (githubTemplates.length > 0) {
 				return githubTemplates;
 			}
@@ -531,8 +501,6 @@ export const composeRouter = createTRPCRouter({
 				error,
 			);
 		}
-
-		// Fall back to local templates if GitHub fetch fails
 		return [];
 	}),
 

packages/server/src/index.ts

@@ -47,6 +47,7 @@ export * from "./utils/backups/mongo";
 export * from "./utils/backups/mysql";
 export * from "./utils/backups/postgres";
 export * from "./utils/backups/utils";
+export * from "./templates/utils/processors";
 
 export * from "./utils/notifications/build-error";
 export * from "./utils/notifications/build-success";

packages/server/src/templates/utils/github.ts

@@ -1,22 +1,4 @@
-import { randomBytes } from "node:crypto";
-import { existsSync } from "node:fs";
-import { mkdir, readFile, writeFile, rm } from "node:fs/promises";
-import { join } from "node:path";
-import { Octokit } from "@octokit/rest";
 import { load } from "js-yaml";
-import { templateConfig } from "../config";
-import type { Schema, Template, DomainSchema } from "./index";
-import {
-	generateBase64,
-	generateHash,
-	generatePassword,
-	generateRandomDomain,
-} from "./index";
-
-// GitHub API client
-const octokit = new Octokit({
-	auth: templateConfig.token,
-});
 
 /**
  * Complete template interface that includes both metadata and configuration
@@ -53,202 +35,50 @@ export interface CompleteTemplate {
 	};
 }
 
-/**
- * Utility functions that can be used in template values
- */
-const TEMPLATE_FUNCTIONS = {
-	$randomDomain: () => true,
-	$password: (length = 16) => `$password(${length})`,
-	$base64: (bytes = 32) => `$base64(${bytes})`,
-	$base32: (bytes = 32) => `$base32(${bytes})`,
-	$hash: (length = 8) => `$hash(${length})`,
-} as const;
-
-/**
- * Process a string value and replace variables
- */
-function processValue(
-	value: string,
-	variables: Record<string, string>,
-	schema: Schema,
-): string {
-	// First replace utility functions
-	let processedValue = value.replace(/\${([^}]+)}/g, (match, varName) => {
-		// Handle utility functions
-		if (varName === "randomDomain") {
-			return generateRandomDomain(schema);
-		}
-		if (varName.startsWith("base64:")) {
-			const length = Number.parseInt(varName.split(":")[1], 10) || 32;
-			return generateBase64(length);
-		}
-		if (varName.startsWith("base32:")) {
-			const length = Number.parseInt(varName.split(":")[1], 10) || 32;
-			return Buffer.from(randomBytes(length))
-				.toString("base64")
-				.substring(0, length);
-		}
-		if (varName.startsWith("password:")) {
-			const length = Number.parseInt(varName.split(":")[1], 10) || 16;
-			return generatePassword(length);
-		}
-		if (varName.startsWith("hash:")) {
-			const length = Number.parseInt(varName.split(":")[1], 10) || 8;
-			return generateHash(length);
-		}
-		// If not a utility function, try to get from variables
-		return variables[varName] || match;
-	});
-
-	// Then replace any remaining ${var} with their values from variables
-	processedValue = processedValue.replace(/\${([^}]+)}/g, (match, varName) => {
-		return variables[varName] || match;
-	});
-
-	return processedValue;
-}
-
-/**
- * Processes a template configuration and returns the generated template
- */
-export function processTemplate(
-	config: CompleteTemplate,
-	schema: Schema,
-): Template {
-	const result: Template = {
-		envs: [],
-		domains: [],
-		mounts: [],
+interface TemplateMetadata {
+	id: string;
+	name: string;
+	description: string;
+	version: string;
+	logo: string;
+	links: {
+		github: string;
+		website?: string;
+		docs?: string;
 	};
-
-	// First pass: Process variables that don't depend on domains
-	const variables: Record<string, string> = {};
-	for (const [key, value] of Object.entries(config.variables)) {
-		if (value === "${randomDomain}") {
-			variables[key] = generateRandomDomain(schema);
-		} else if (value.startsWith("${base64:")) {
-			const match = value.match(/\${base64:(\d+)}/);
-			const length = match?.[1] ? Number.parseInt(match[1], 10) : 32;
-			variables[key] = generateBase64(length);
-		} else if (value.startsWith("${base32:")) {
-			const match = value.match(/\${base32:(\d+)}/);
-			const length = match?.[1] ? Number.parseInt(match[1], 10) : 32;
-			variables[key] = Buffer.from(randomBytes(length))
-				.toString("base64")
-				.substring(0, length);
-		} else if (value.startsWith("${password:")) {
-			const match = value.match(/\${password:(\d+)}/);
-			const length = match?.[1] ? Number.parseInt(match[1], 10) : 16;
-			variables[key] = generatePassword(length);
-		} else if (value.startsWith("${hash:")) {
-			const match = value.match(/\${hash:(\d+)}/);
-			const length = match?.[1] ? Number.parseInt(match[1], 10) : 8;
-			variables[key] = generateHash(length);
-		} else {
-			variables[key] = value;
-		}
-	}
-
-	console.log(variables);
-
-	// Process domains and add them to variables
-	for (const domain of config.config.domains) {
-		// If host is specified, process it with variables, otherwise generate random domain
-		const host = domain.host
-			? processValue(domain.host, variables, schema)
-			: generateRandomDomain(schema);
-
-		result.domains.push({
-			host,
-			...domain,
-		});
-		// Add domain to variables for reference
-		variables[`domain:${domain.serviceName}`] = host;
-	}
-
-	// Process environment variables with access to all variables
-	for (const [key, value] of Object.entries(config.config.env)) {
-		const processedValue = processValue(value, variables, schema);
-		result.envs.push(`${key}=${processedValue}`);
-	}
-
-	// Process mounts with access to all variables
-	if (config.config.mounts) {
-		for (const mount of config.config.mounts) {
-			result.mounts.push({
-				filePath: mount.filePath,
-				content: processValue(mount.content, variables, schema),
-			});
-		}
-	}
-
-	return result;
+	tags: string[];
 }
 
 /**
- * GitHub tree item with required fields
- */
-interface GitTreeItem {
-	path: string;
-	type: string;
-	sha: string;
-}
-
-/**
- * Fetches the list of available templates from GitHub
+ * Fetches the list of available templates from meta.json
  */
 export async function fetchTemplatesList(
-	owner = templateConfig.owner,
-	repo = templateConfig.repo,
-	branch = templateConfig.branch,
+	baseUrl = "https://dokploy.github.io/templates",
 ): Promise<CompleteTemplate[]> {
 	try {
-		// First get the tree SHA for the branch
-		const { data: ref } = await octokit.git.getRef({
-			owner,
-			repo,
-			ref: `heads/${branch}`,
-		});
-
-		// Get the full tree recursively
-		const { data: tree } = await octokit.git.getTree({
-			owner,
-			repo,
-			tree_sha: ref.object.sha,
-			recursive: "true",
-		});
-
-		// Filter for template.yml files in the templates directory
-		const templateFiles = tree.tree.filter((item): item is GitTreeItem => {
-			return (
-				item.type === "blob" &&
-				typeof item.path === "string" &&
-				typeof item.sha === "string" &&
-				item.path.startsWith("templates/") &&
-				item.path.endsWith("/template.yml")
-			);
-		});
-
-		// Fetch and parse each template.yml
-		const templates = await Promise.all(
-			templateFiles.map(async (file) => {
-				try {
-					const { data: content } = await octokit.git.getBlob({
-						owner,
-						repo,
-						file_sha: file.sha,
-					});
-
-					const decoded = Buffer.from(content.content, "base64").toString();
-					return load(decoded) as CompleteTemplate;
-				} catch (error) {
-					console.warn(`Failed to load template from ${file.path}:`, error);
-					return null;
-				}
-			}),
-		);
-
-		return templates.filter(Boolean) as CompleteTemplate[];
+		const response = await fetch(`${baseUrl}/meta.json`);
+		if (!response.ok) {
+			throw new Error(`Failed to fetch templates: ${response.statusText}`);
+		}
+		const templates = (await response.json()) as TemplateMetadata[];
+		return templates.map((template) => ({
+			metadata: {
+				id: template.id,
+				name: template.name,
+				description: template.description,
+				version: template.version,
+				logo: template.logo,
+				links: template.links,
+				tags: template.tags,
+			},
+			// These will be populated when fetching individual templates
+			variables: {},
+			config: {
+				domains: [],
+				env: {},
+				mounts: [],
+			},
+		}));
 	} catch (error) {
 		console.error("Error fetching templates list:", error);
 		throw error;
@@ -256,78 +86,29 @@ export async function fetchTemplatesList(
 }
 
 /**
- * Fetches a specific template's files from GitHub
+ * Fetches a specific template's files
  */
 export async function fetchTemplateFiles(
 	templateId: string,
-	owner = templateConfig.owner,
-	repo = templateConfig.repo,
-	branch = templateConfig.branch,
+	baseUrl = "https://dokploy.github.io/templates",
 ): Promise<{ config: CompleteTemplate; dockerCompose: string }> {
 	try {
-		// Get the tree SHA for the branch
-		const { data: ref } = await octokit.git.getRef({
-			owner,
-			repo,
-			ref: `heads/${branch}`,
-		});
+		// Fetch both files in parallel
+		const [templateYmlResponse, dockerComposeResponse] = await Promise.all([
+			fetch(`${baseUrl}/templates/${templateId}/template.yml`),
+			fetch(`${baseUrl}/templates/${templateId}/docker-compose.yml`),
+		]);
 
-		// Get the full tree recursively
-		const { data: tree } = await octokit.git.getTree({
-			owner,
-			repo,
-			tree_sha: ref.object.sha,
-			recursive: "true",
-		});
-
-		// Find the template.yml and docker-compose.yml files
-		const templateYml = tree.tree
-			.filter((item): item is GitTreeItem => {
-				return (
-					item.type === "blob" &&
-					typeof item.path === "string" &&
-					typeof item.sha === "string"
-				);
-			})
-			.find((item) => item.path === `templates/${templateId}/template.yml`);
-
-		const dockerComposeYml = tree.tree
-			.filter((item): item is GitTreeItem => {
-				return (
-					item.type === "blob" &&
-					typeof item.path === "string" &&
-					typeof item.sha === "string"
-				);
-			})
-			.find(
-				(item) => item.path === `templates/${templateId}/docker-compose.yml`,
-			);
-
-		if (!templateYml || !dockerComposeYml) {
+		if (!templateYmlResponse.ok || !dockerComposeResponse.ok) {
 			throw new Error("Template files not found");
 		}
 
-		// Fetch both files in parallel
-		const [templateContent, composeContent] = await Promise.all([
-			octokit.git.getBlob({
-				owner,
-				repo,
-				file_sha: templateYml.sha,
-			}),
-			octokit.git.getBlob({
-				owner,
-				repo,
-				file_sha: dockerComposeYml.sha,
-			}),
+		const [templateYml, dockerCompose] = await Promise.all([
+			templateYmlResponse.text(),
+			dockerComposeResponse.text(),
 		]);
 
-		const config = load(
-			Buffer.from(templateContent.data.content, "base64").toString(),
-		) as CompleteTemplate;
-		const dockerCompose = Buffer.from(
-			composeContent.data.content,
-			"base64",
-		).toString();
+		const config = load(templateYml) as CompleteTemplate;
 
 		return { config, dockerCompose };
 	} catch (error) {
@@ -335,13 +116,3 @@ export async function fetchTemplateFiles(
 		throw error;
 	}
 }
-
-/**
- * Loads and processes a template
- */
-export async function loadTemplateModule(
-	id: string,
-): Promise<(schema: Schema) => Promise<Template>> {
-	const { config } = await fetchTemplateFiles(id);
-	return async (schema: Schema) => processTemplate(config, schema);
-}

packages/server/src/templates/utils/index.ts

@@ -1,10 +1,9 @@
 import { randomBytes } from "node:crypto";
 import { existsSync } from "node:fs";
-import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import type { Domain } from "@dokploy/server/services/domain";
 import { TRPCError } from "@trpc/server";
-import { templateConfig } from "../config";
 import { fetchTemplateFiles } from "./github";
 
 export interface Schema {
@@ -53,26 +52,15 @@ export const generatePassword = (quantity = 16): string => {
 	return password.toLowerCase();
 };
 
-export const generateBase64 = (bytes = 32): string => {
-	return randomBytes(bytes).toString("base64");
-};
-
 /**
- * Checks if a cached file is still valid based on its modification time
+ * Generate a random base64 string of specified length
  */
-async function isCacheValid(filePath: string): Promise<boolean> {
-	try {
-		if (!existsSync(filePath)) return false;
-
-		const fileStats = await stat(filePath);
-		const modifiedTime = fileStats.mtime.getTime();
-		const currentTime = Date.now();
-
-		// Check if the file is older than the cache duration
-		return currentTime - modifiedTime < templateConfig.cacheDuration;
-	} catch (error) {
-		return false;
-	}
+export function generateBase64(length: number): string {
+	// To get N characters in base64, we need to generate N * 3/4 bytes
+	const bytesNeeded = Math.ceil((length * 3) / 4);
+	return Buffer.from(randomBytes(bytesNeeded))
+		.toString("base64")
+		.substring(0, length);
 }
 
 /**

packages/server/src/templates/utils/processors.ts

@@ -0,0 +1,202 @@
+import type { Schema } from "./index";
+import {
+	generateBase64,
+	generateHash,
+	generatePassword,
+	generateRandomDomain,
+} from "./index";
+
+/**
+ * Domain configuration
+ */
+interface DomainConfig {
+	serviceName: string;
+	port: number;
+	path?: string;
+	host?: string;
+}
+
+/**
+ * Mount configuration
+ */
+interface MountConfig {
+	filePath: string;
+	content: string;
+}
+
+/**
+ * Complete template interface that includes both metadata and configuration
+ */
+export interface CompleteTemplate {
+	metadata: {
+		id: string;
+		name: string;
+		description: string;
+		tags: string[];
+		version: string;
+		logo: string;
+		links: {
+			github: string;
+			website?: string;
+			docs?: string;
+		};
+	};
+	variables: Record<string, string>;
+	config: {
+		domains: DomainConfig[];
+		env: Record<string, string>;
+		mounts?: MountConfig[];
+	};
+}
+
+/**
+ * Processed template output
+ */
+export interface Template {
+	domains: Array<DomainConfig>;
+	envs: string[];
+	mounts: MountConfig[];
+}
+
+/**
+ * Process a string value and replace variables
+ */
+function processValue(
+	value: string,
+	variables: Record<string, string>,
+	schema: Schema,
+): string {
+	// First replace utility functions
+	let processedValue = value.replace(/\${([^}]+)}/g, (match, varName) => {
+		// Handle utility functions
+		if (varName === "randomDomain") {
+			return generateRandomDomain(schema);
+		}
+		if (varName.startsWith("base64:")) {
+			const length = Number.parseInt(varName.split(":")[1], 10) || 32;
+			return generateBase64(length);
+		}
+		if (varName.startsWith("password:")) {
+			const length = Number.parseInt(varName.split(":")[1], 10) || 16;
+			return generatePassword(length);
+		}
+		if (varName.startsWith("hash:")) {
+			const length = Number.parseInt(varName.split(":")[1], 10) || 8;
+			return generateHash(length);
+		}
+		// If not a utility function, try to get from variables
+		return variables[varName] || match;
+	});
+
+	// Then replace any remaining ${var} with their values from variables
+	processedValue = processedValue.replace(/\${([^}]+)}/g, (match, varName) => {
+		return variables[varName] || match;
+	});
+
+	return processedValue;
+}
+
+/**
+ * Process variables in a template
+ */
+export function processVariables(
+	template: CompleteTemplate,
+	schema: Schema,
+): Record<string, string> {
+	const variables: Record<string, string> = {};
+
+	// First pass: Process variables that don't depend on other variables
+	for (const [key, value] of Object.entries(template.variables)) {
+		if (typeof value !== "string") continue;
+
+		if (value === "${randomDomain}") {
+			variables[key] = generateRandomDomain(schema);
+		} else if (value.startsWith("${base64:")) {
+			const match = value.match(/\${base64:(\d+)}/);
+			const length = match?.[1] ? Number.parseInt(match[1], 10) : 32;
+			variables[key] = generateBase64(length);
+		} else if (value.startsWith("${password:")) {
+			const match = value.match(/\${password:(\d+)}/);
+			const length = match?.[1] ? Number.parseInt(match[1], 10) : 16;
+			variables[key] = generatePassword(length);
+		} else if (value.startsWith("${hash:")) {
+			const match = value.match(/\${hash:(\d+)}/);
+			const length = match?.[1] ? Number.parseInt(match[1], 10) : 8;
+			variables[key] = generateHash(length);
+		} else {
+			variables[key] = value;
+		}
+	}
+
+	// Second pass: Process variables that reference other variables
+	for (const [key, value] of Object.entries(variables)) {
+		variables[key] = processValue(value, variables, schema);
+	}
+
+	return variables;
+}
+
+/**
+ * Process domains in a template
+ */
+export function processDomains(
+	template: CompleteTemplate,
+	variables: Record<string, string>,
+	schema: Schema,
+): Template["domains"] {
+	return template.config.domains.map((domain: DomainConfig) => ({
+		...domain,
+		host: domain.host
+			? processValue(domain.host, variables, schema)
+			: generateRandomDomain(schema),
+	}));
+}
+
+/**
+ * Process environment variables in a template
+ */
+export function processEnvVars(
+	template: CompleteTemplate,
+	variables: Record<string, string>,
+	schema: Schema,
+): Template["envs"] {
+	return Object.entries(template.config.env).map(
+		([key, value]: [string, string]) => {
+			const processedValue = processValue(value, variables, schema);
+			return `${key}=${processedValue}`;
+		},
+	);
+}
+
+/**
+ * Process mounts in a template
+ */
+export function processMounts(
+	template: CompleteTemplate,
+	variables: Record<string, string>,
+	schema: Schema,
+): Template["mounts"] {
+	if (!template.config.mounts) return [];
+
+	return template.config.mounts.map((mount: MountConfig) => ({
+		filePath: processValue(mount.filePath, variables, schema),
+		content: processValue(mount.content, variables, schema),
+	}));
+}
+
+/**
+ * Process a complete template
+ */
+export function processTemplate(
+	template: CompleteTemplate,
+	schema: Schema,
+): Template {
+	// First process variables as they might be referenced by other sections
+	const variables = processVariables(template, schema);
+
+	return {
+		domains: processDomains(template, variables, schema),
+		envs: processEnvVars(template, variables, schema),
+		mounts: processMounts(template, variables, schema),
+	};
+}