feat: add GitHub-based template fetching and caching mechanism

This commit is contained in:
Mauricio Siu 2025-03-01 00:57:32 -06:00
parent 0a6382a731
commit 49b37d531a
6 changed files with 553 additions and 21 deletions

View File

@ -18,12 +18,13 @@ import {
loadTemplateModule,
readTemplateComposeFile,
} from "@/templates/utils";
import { fetchTemplatesList } from "@dokploy/server/templates/utils/github";
import { TRPCError } from "@trpc/server";
import { eq } from "drizzle-orm";
import { dump } from "js-yaml";
import _ from "lodash";
import { nanoid } from "nanoid";
import { createTRPCRouter, protectedProcedure } from "../trpc";
import { createTRPCRouter, protectedProcedure, publicProcedure } from "../trpc";
import type { DeploymentJob } from "@/server/queues/queue-types";
import { deploy } from "@/server/utils/deploy";
@ -388,7 +389,7 @@ export const composeRouter = createTRPCRouter({
const composeFile = await readTemplateComposeFile(input.id);
const generate = await loadTemplateModule(input.id as TemplatesKeys);
const generate = await loadTemplateModule(input.id);
const admin = await findAdminById(ctx.user.adminId);
let serverIp = admin.serverIp || "127.0.0.1";
@ -402,7 +403,7 @@ export const composeRouter = createTRPCRouter({
serverIp = "127.0.0.1";
}
const projectName = slugify(`${project.name} ${input.id}`);
const { envs, mounts, domains } = generate({
const { envs, mounts, domains } = await generate({
serverIp: serverIp || "",
projectName: projectName,
});
@ -449,18 +450,22 @@ export const composeRouter = createTRPCRouter({
return null;
}),
templates: protectedProcedure.query(async () => {
const templatesData = templates.map((t) => ({
name: t.name,
description: t.description,
id: t.id,
links: t.links,
tags: t.tags,
logo: t.logo,
version: t.version,
}));
templates: publicProcedure.query(async () => {
// First try to fetch templates from GitHub
try {
const githubTemplates = await fetchTemplatesList();
if (githubTemplates.length > 0) {
return githubTemplates;
}
} catch (error) {
console.warn(
"Failed to fetch templates from GitHub, falling back to local templates:",
error,
);
}
return templatesData;
// Fall back to local templates if GitHub fetch fails
return templates;
}),
getTags: protectedProcedure.query(async ({ input }) => {

View File

@ -62,7 +62,8 @@
"slugify": "^1.6.6",
"ws": "8.16.0",
"zod": "^3.23.4",
"ssh2": "1.15.0"
"ssh2": "1.15.0",
"@octokit/rest": "^20.0.2"
},
"devDependencies": {
"esbuild-plugin-alias": "0.2.1",

View File

@ -0,0 +1,38 @@
/**
* Configuration for the GitHub template repository
*/
export const templateConfig = {
/**
* GitHub repository owner
* @default "dokploy"
*/
owner: process.env.TEMPLATE_REPO_OWNER || "dokploy",
/**
* GitHub repository name
* @default "templates"
*/
repo: process.env.TEMPLATE_REPO_NAME || "templates",
/**
* GitHub repository branch
* @default "main"
*/
branch: process.env.TEMPLATE_REPO_BRANCH || "main",
/**
* Cache duration in milliseconds
* How long to cache templates before checking for updates
* @default 3600000 (1 hour)
*/
cacheDuration: Number.parseInt(
process.env.TEMPLATE_CACHE_DURATION || "3600000",
10,
),
/**
* GitHub API token (optional)
* Used for higher rate limits
*/
token: process.env.GITHUB_TOKEN,
};

View File

@ -0,0 +1,325 @@
import { execSync } from "node:child_process";
import { randomBytes } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { Octokit } from "@octokit/rest";
import * as esbuild from "esbuild";
import { load } from "js-yaml";
import { templateConfig } from "../config";
import type { Template } from "./index";
import {
generateBase64,
generateHash,
generatePassword,
generateRandomDomain,
} from "./index";
// GitHub API client
const octokit = new Octokit({
auth: templateConfig.token,
});
/**
* Interface for template metadata
*/
export interface TemplateMetadata {
id: string;
name: string;
version: string;
description: string;
logo: string;
links: {
github?: string;
website?: string;
docs?: string;
};
tags: string[];
}
/**
* Fetches the list of available templates from GitHub
*/
export async function fetchTemplatesList(
owner = templateConfig.owner,
repo = templateConfig.repo,
branch = templateConfig.branch,
): Promise<TemplateMetadata[]> {
try {
// Fetch templates directory content
const { data: dirContent } = await octokit.repos.getContent({
owner,
repo,
path: "templates",
ref: branch,
});
console.log("DIR CONTENT", dirContent);
if (!Array.isArray(dirContent)) {
throw new Error("Templates directory not found or is not a directory");
}
// Filter for directories only (each directory is a template)
const templateDirs = dirContent.filter((item) => item.type === "dir");
// Fetch metadata for each template
const templates = await Promise.all(
templateDirs.map(async (dir) => {
try {
// Try to fetch metadata.json for each template
const { data: metadataFile } = await octokit.repos.getContent({
owner,
repo,
path: `templates/${dir.name}/metadata.json`,
ref: branch,
});
if ("content" in metadataFile && metadataFile.encoding === "base64") {
const content = Buffer.from(
metadataFile.content,
"base64",
).toString();
return JSON.parse(content) as TemplateMetadata;
}
} catch (error) {
// If metadata.json doesn't exist, create a basic metadata object
return {
id: dir.name,
name: dir.name.charAt(0).toUpperCase() + dir.name.slice(1),
version: "latest",
description: `${dir.name} template`,
logo: "default.svg",
links: {},
tags: [],
};
}
return null;
}),
);
return templates.filter(Boolean) as TemplateMetadata[];
} catch (error) {
console.error("Error fetching templates list:", error);
throw error;
}
}
/**
* Fetches a specific template's files from GitHub
*/
export async function fetchTemplateFiles(
templateId: string,
owner = templateConfig.owner,
repo = templateConfig.repo,
branch = templateConfig.branch,
): Promise<{ indexTs: string; dockerCompose: string }> {
try {
// Fetch index.ts
const { data: indexFile } = await octokit.repos.getContent({
owner,
repo,
path: `templates/${templateId}/index.ts`,
ref: branch,
});
// Fetch docker-compose.yml
const { data: composeFile } = await octokit.repos.getContent({
owner,
repo,
path: `templates/${templateId}/docker-compose.yml`,
ref: branch,
});
if (!("content" in indexFile) || !("content" in composeFile)) {
throw new Error("Template files not found");
}
const indexTs = Buffer.from(indexFile.content, "base64").toString();
const dockerCompose = Buffer.from(composeFile.content, "base64").toString();
return { indexTs, dockerCompose };
} catch (error) {
console.error(`Error fetching template ${templateId}:`, error);
throw error;
}
}
/**
* Executes the template's index.ts code dynamically
* Uses a template-based approach that's safer and more efficient
*/
export async function executeTemplateCode(
indexTsCode: string,
schema: { serverIp: string; projectName: string },
): Promise<Template> {
try {
// Create a temporary directory for the template
const cwd = process.cwd();
const tempId = randomBytes(8).toString("hex");
const tempDir = join(cwd, ".next", "temp", tempId);
if (!existsSync(tempDir)) {
await mkdir(tempDir, { recursive: true });
}
// Extract the generate function body
// This approach assumes templates follow a standard structure with a generate function
const generateFunctionMatch = indexTsCode.match(
/export\s+function\s+generate\s*\([^)]*\)\s*{([\s\S]*?)return\s+{([\s\S]*?)};?\s*}/,
);
if (!generateFunctionMatch) {
throw new Error("Could not extract generate function from template");
}
const functionBody = generateFunctionMatch[1];
const returnStatement = generateFunctionMatch[2];
// Create a simplified template that doesn't require imports
const templateCode = `
// Utility functions provided to the template
function generateRandomDomain(schema) {
const hash = Math.random().toString(36).substring(2, 8);
const slugIp = schema.serverIp.replaceAll(".", "-");
return \`\${schema.projectName}-\${hash}\${slugIp === "" ? "" : \`-\${slugIp}\`}.traefik.me\`;
}
function generateHash(projectName, quantity = 3) {
const hash = Math.random().toString(36).substring(2, 2 + quantity);
return \`\${projectName}-\${hash}\`;
}
function generatePassword(quantity = 16) {
const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let password = "";
for (let i = 0; i < quantity; i++) {
password += characters.charAt(Math.floor(Math.random() * characters.length));
}
return password.toLowerCase();
}
function generateBase64(bytes = 32) {
return Math.random().toString(36).substring(2, 2 + bytes);
}
// Template execution
function execute(schema) {
${functionBody}
return {
${returnStatement}
};
}
// Run with the provided schema and output the result
const result = execute(${JSON.stringify(schema)});
console.log(JSON.stringify(result));
`;
// Write the template code to a file
const templatePath = join(tempDir, "template.js");
await writeFile(templatePath, templateCode, "utf8");
// Execute the template using Node.js
const output = execSync(`node ${templatePath}`, {
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
// Parse the output as JSON
return JSON.parse(output);
} catch (error) {
console.error("Error executing template code:", error);
// Fallback to a simpler approach if the template extraction fails
return fallbackExecuteTemplate(indexTsCode, schema);
}
}
/**
* Fallback method to execute templates that don't follow the standard structure
*/
async function fallbackExecuteTemplate(
indexTsCode: string,
schema: { serverIp: string; projectName: string },
): Promise<Template> {
try {
// Create a temporary directory
const cwd = process.cwd();
const tempId = randomBytes(8).toString("hex");
const tempDir = join(cwd, ".next", "temp", tempId);
if (!existsSync(tempDir)) {
await mkdir(tempDir, { recursive: true });
}
// Create a simplified version of the template code
// Remove TypeScript types and imports
const simplifiedCode = indexTsCode
.replace(/import\s+.*?from\s+['"].*?['"]\s*;?/g, "")
.replace(/export\s+interface\s+.*?{[\s\S]*?}/g, "")
.replace(/:\s*Schema/g, "")
.replace(/:\s*DomainSchema/g, "")
.replace(/:\s*Template/g, "")
.replace(/:\s*string/g, "")
.replace(/:\s*number/g, "")
.replace(/:\s*boolean/g, "")
.replace(/:\s*any/g, "")
.replace(/:\s*unknown/g, "")
.replace(/<.*?>/g, "");
// Create a wrapper with all necessary utilities
const wrapperCode = `
// Utility functions
function generateRandomDomain(schema) {
const hash = Math.random().toString(36).substring(2, 8);
const slugIp = schema.serverIp.replaceAll(".", "-");
return \`\${schema.projectName}-\${hash}\${slugIp === "" ? "" : \`-\${slugIp}\`}.traefik.me\`;
}
function generateHash(projectName, quantity = 3) {
const hash = Math.random().toString(36).substring(2, 2 + quantity);
return \`\${projectName}-\${hash}\`;
}
function generatePassword(quantity = 16) {
const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let password = "";
for (let i = 0; i < quantity; i++) {
password += characters.charAt(Math.floor(Math.random() * characters.length));
}
return password.toLowerCase();
}
function generateBase64(bytes = 32) {
return Math.random().toString(36).substring(2, 2 + bytes);
}
// Simplified template code
${simplifiedCode}
// Execute the template
const result = generate(${JSON.stringify(schema)});
console.log(JSON.stringify(result));
`;
// Write the wrapper code to a file
const wrapperPath = join(tempDir, "wrapper.js");
await writeFile(wrapperPath, wrapperCode, "utf8");
// Execute the code using Node.js
const output = execSync(`node ${wrapperPath}`, {
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
// Parse the output as JSON
return JSON.parse(output);
} catch (error) {
console.error("Error in fallback template execution:", error);
throw new Error(
`Failed to execute template: ${error instanceof Error ? error.message : String(error)}`,
);
}
}

View File

@ -1,7 +1,11 @@
import { randomBytes } from "node:crypto";
import { readFile } from "node:fs/promises";
import { existsSync } from "node:fs";
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { Domain } from "@dokploy/server/services/domain";
import { TRPCError } from "@trpc/server";
import { templateConfig } from "../config";
import { executeTemplateCode, fetchTemplateFiles } from "./github";
export interface Schema {
serverIp: string;
@ -50,12 +54,113 @@ export const generateBase64 = (bytes = 32): string => {
return randomBytes(bytes).toString("base64");
};
/**
* Checks if a cached file is still valid based on its modification time
*/
async function isCacheValid(filePath: string): Promise<boolean> {
try {
if (!existsSync(filePath)) return false;
const fileStats = await stat(filePath);
const modifiedTime = fileStats.mtime.getTime();
const currentTime = Date.now();
// Check if the file is older than the cache duration
return currentTime - modifiedTime < templateConfig.cacheDuration;
} catch (error) {
return false;
}
}
/**
* Reads a template's docker-compose.yml file
* First tries to read from the local cache, if not found or expired, fetches from GitHub
*/
export const readTemplateComposeFile = async (id: string) => {
const cwd = process.cwd();
const composeFile = await readFile(
join(cwd, ".next", "templates", id, "docker-compose.yml"),
"utf8",
);
const templatePath = join(cwd, ".next", "templates", id);
const composeFilePath = join(templatePath, "docker-compose.yml");
return composeFile;
// Check if the file exists in the local cache and is still valid
if (await isCacheValid(composeFilePath)) {
return await readFile(composeFilePath, "utf8");
}
// If not in cache or expired, fetch from GitHub and cache it
try {
const { dockerCompose } = await fetchTemplateFiles(id);
// Ensure the template directory exists
if (!existsSync(templatePath)) {
await mkdir(templatePath, { recursive: true });
}
// Cache the file for future use
await writeFile(composeFilePath, dockerCompose, "utf8");
return dockerCompose;
} catch (error) {
// If fetch fails but we have a cached version, use it as fallback
if (existsSync(composeFilePath)) {
console.warn(
`Using cached version of template ${id} due to fetch error:`,
error,
);
return await readFile(composeFilePath, "utf8");
}
console.error(`Error fetching template ${id}:`, error);
throw new TRPCError({
code: "NOT_FOUND",
message: `Template ${id} not found or could not be fetched`,
});
}
};
/**
* Loads a template module and returns its generate function
* First tries to execute from local cache, if not found or expired, fetches from GitHub
*/
export const loadTemplateModule = async (id: string) => {
const cwd = process.cwd();
const templatePath = join(cwd, ".next", "templates", id);
const indexFilePath = join(templatePath, "index.ts");
// Check if we have the template cached locally and it's still valid
if (await isCacheValid(indexFilePath)) {
const indexTs = await readFile(indexFilePath, "utf8");
return (schema: Schema) => executeTemplateCode(indexTs, schema);
}
// If not in cache or expired, fetch from GitHub and cache it
try {
const { indexTs } = await fetchTemplateFiles(id);
// Ensure the template directory exists
if (!existsSync(templatePath)) {
await mkdir(templatePath, { recursive: true });
}
// Cache the file for future use
await writeFile(indexFilePath, indexTs, "utf8");
// Return a function that will execute the template code
return (schema: Schema) => executeTemplateCode(indexTs, schema);
} catch (error) {
// If fetch fails but we have a cached version, use it as fallback
if (existsSync(indexFilePath)) {
console.warn(
`Using cached version of template ${id} due to fetch error:`,
error,
);
const indexTs = await readFile(indexFilePath, "utf8");
return (schema: Schema) => executeTemplateCode(indexTs, schema);
}
console.error(`Error loading template module ${id}:`, error);
throw new TRPCError({
code: "NOT_FOUND",
message: `Template ${id} not found or could not be loaded`,
});
}
};

View File

@ -555,6 +555,9 @@ importers:
'@octokit/auth-app':
specifier: ^6.0.4
version: 6.1.1
'@octokit/rest':
specifier: ^20.0.2
version: 20.1.2
'@react-email/components':
specifier: ^0.0.21
version: 0.0.21(@types/react@18.3.5)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
@ -2063,6 +2066,9 @@ packages:
'@octokit/openapi-types@22.2.0':
resolution: {integrity: sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==}
'@octokit/openapi-types@23.0.1':
resolution: {integrity: sha512-izFjMJ1sir0jn0ldEKhZ7xegCTj/ObmEDlEfpFrx4k/JyZSMRHbO3/rBwgE7f3m2DHt+RrNGIVw4wSmwnm3t/g==}
'@octokit/openapi-webhooks-types@8.3.0':
resolution: {integrity: sha512-vKLsoR4xQxg4Z+6rU/F65ItTUz/EXbD+j/d4mlq2GW8TsA4Tc8Kdma2JTAAJ5hrKWUQzkR/Esn2fjsqiVRYaQg==}
@ -2072,18 +2078,36 @@ packages:
peerDependencies:
'@octokit/core': '>=5'
'@octokit/plugin-paginate-rest@11.4.4-cjs.2':
resolution: {integrity: sha512-2dK6z8fhs8lla5PaOTgqfCGBxgAv/le+EhPs27KklPhm1bKObpu6lXzwfUEQ16ajXzqNrKMujsFyo9K2eaoISw==}
engines: {node: '>= 18'}
peerDependencies:
'@octokit/core': '5'
'@octokit/plugin-paginate-rest@9.2.1':
resolution: {integrity: sha512-wfGhE/TAkXZRLjksFXuDZdmGnJQHvtU/joFQdweXUgzo1XwvBCD4o4+75NtFfjfLK5IwLf9vHTfSiU3sLRYpRw==}
engines: {node: '>= 18'}
peerDependencies:
'@octokit/core': '5'
'@octokit/plugin-request-log@4.0.1':
resolution: {integrity: sha512-GihNqNpGHorUrO7Qa9JbAl0dbLnqJVrV8OXe2Zm5/Y4wFkZQDfTreBzVmiRfJVfE4mClXdihHnbpyyO9FSX4HA==}
engines: {node: '>= 18'}
peerDependencies:
'@octokit/core': '5'
'@octokit/plugin-rest-endpoint-methods@10.4.1':
resolution: {integrity: sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==}
engines: {node: '>= 18'}
peerDependencies:
'@octokit/core': '5'
'@octokit/plugin-rest-endpoint-methods@13.3.2-cjs.1':
resolution: {integrity: sha512-VUjIjOOvF2oELQmiFpWA1aOPdawpyaCUqcEBc/UOUnj3Xp6DJGrJ1+bjUIIDzdHjnFNO6q57ODMfdEZnoBkCwQ==}
engines: {node: '>= 18'}
peerDependencies:
'@octokit/core': ^5
'@octokit/plugin-retry@6.0.1':
resolution: {integrity: sha512-SKs+Tz9oj0g4p28qkZwl/topGcb0k0qPNX/i7vBKmDsjoeqnVfFUquqrE/O9oJY7+oLzdCtkiWSXLpLjvl6uog==}
engines: {node: '>= 18'}
@ -2108,12 +2132,19 @@ packages:
resolution: {integrity: sha512-9Bb014e+m2TgBeEJGEbdplMVWwPmL1FPtggHQRkV+WVsMggPtEkLKPlcVYm/o8xKLkpJ7B+6N8WfQMtDLX2Dpw==}
engines: {node: '>= 18'}
'@octokit/rest@20.1.2':
resolution: {integrity: sha512-GmYiltypkHHtihFwPRxlaorG5R9VAHuk/vbszVoRTGXnAsY60wYLkh/E2XiFmdZmqrisw+9FaazS1i5SbdWYgA==}
engines: {node: '>= 18'}
'@octokit/types@12.6.0':
resolution: {integrity: sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==}
'@octokit/types@13.5.0':
resolution: {integrity: sha512-HdqWTf5Z3qwDVlzCrP8UJquMwunpDiMPt5er+QjGzL4hqr/vBVY/MauQgS1xWxCDT1oMx1EULyqxncdCY/NVSQ==}
'@octokit/types@13.8.0':
resolution: {integrity: sha512-x7DjTIbEpEWXK99DMd01QfWy0hd5h4EN+Q7shkdKds3otGQP+oWE/y0A76i1OvH9fygo4ddvNf7ZvF0t78P98A==}
'@octokit/webhooks-methods@4.1.0':
resolution: {integrity: sha512-zoQyKw8h9STNPqtm28UGOYFE7O6D4Il8VJwhAtMHFt2C4L0VQT1qGKLeefUOqHNs1mNRYSadVv7x0z8U2yyeWQ==}
engines: {node: '>= 18'}
@ -8326,22 +8357,38 @@ snapshots:
'@octokit/openapi-types@22.2.0': {}
'@octokit/openapi-types@23.0.1': {}
'@octokit/openapi-webhooks-types@8.3.0': {}
'@octokit/plugin-paginate-graphql@4.0.1(@octokit/core@5.2.0)':
dependencies:
'@octokit/core': 5.2.0
'@octokit/plugin-paginate-rest@11.4.4-cjs.2(@octokit/core@5.2.0)':
dependencies:
'@octokit/core': 5.2.0
'@octokit/types': 13.8.0
'@octokit/plugin-paginate-rest@9.2.1(@octokit/core@5.2.0)':
dependencies:
'@octokit/core': 5.2.0
'@octokit/types': 12.6.0
'@octokit/plugin-request-log@4.0.1(@octokit/core@5.2.0)':
dependencies:
'@octokit/core': 5.2.0
'@octokit/plugin-rest-endpoint-methods@10.4.1(@octokit/core@5.2.0)':
dependencies:
'@octokit/core': 5.2.0
'@octokit/types': 12.6.0
'@octokit/plugin-rest-endpoint-methods@13.3.2-cjs.1(@octokit/core@5.2.0)':
dependencies:
'@octokit/core': 5.2.0
'@octokit/types': 13.8.0
'@octokit/plugin-retry@6.0.1(@octokit/core@5.2.0)':
dependencies:
'@octokit/core': 5.2.0
@ -8372,6 +8419,13 @@ snapshots:
'@octokit/types': 13.5.0
universal-user-agent: 6.0.1
'@octokit/rest@20.1.2':
dependencies:
'@octokit/core': 5.2.0
'@octokit/plugin-paginate-rest': 11.4.4-cjs.2(@octokit/core@5.2.0)
'@octokit/plugin-request-log': 4.0.1(@octokit/core@5.2.0)
'@octokit/plugin-rest-endpoint-methods': 13.3.2-cjs.1(@octokit/core@5.2.0)
'@octokit/types@12.6.0':
dependencies:
'@octokit/openapi-types': 20.0.0
@ -8380,6 +8434,10 @@ snapshots:
dependencies:
'@octokit/openapi-types': 22.2.0
'@octokit/types@13.8.0':
dependencies:
'@octokit/openapi-types': 23.0.1
'@octokit/webhooks-methods@4.1.0': {}
'@octokit/webhooks-methods@5.1.0': {}