From 49b37d531a45c10956068e0a0fcf5c5fc3c7f696 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sat, 1 Mar 2025 00:57:32 -0600 Subject: [PATCH] feat: add GitHub-based template fetching and caching mechanism --- apps/dokploy/server/api/routers/compose.ts | 33 +- packages/server/package.json | 3 +- packages/server/src/templates/config.ts | 38 ++ packages/server/src/templates/utils/github.ts | 325 ++++++++++++++++++ packages/server/src/templates/utils/index.ts | 117 ++++++- pnpm-lock.yaml | 58 ++++ 6 files changed, 553 insertions(+), 21 deletions(-) create mode 100644 packages/server/src/templates/config.ts create mode 100644 packages/server/src/templates/utils/github.ts diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index 463d0398..f8deea0d 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -18,12 +18,13 @@ import { loadTemplateModule, readTemplateComposeFile, } from "@/templates/utils"; +import { fetchTemplatesList } from "@dokploy/server/templates/utils/github"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; import { dump } from "js-yaml"; import _ from "lodash"; import { nanoid } from "nanoid"; -import { createTRPCRouter, protectedProcedure } from "../trpc"; +import { createTRPCRouter, protectedProcedure, publicProcedure } from "../trpc"; import type { DeploymentJob } from "@/server/queues/queue-types"; import { deploy } from "@/server/utils/deploy"; @@ -388,7 +389,7 @@ export const composeRouter = createTRPCRouter({ const composeFile = await readTemplateComposeFile(input.id); - const generate = await loadTemplateModule(input.id as TemplatesKeys); + const generate = await loadTemplateModule(input.id); const admin = await findAdminById(ctx.user.adminId); let serverIp = admin.serverIp || "127.0.0.1"; @@ -402,7 +403,7 @@ export const composeRouter = createTRPCRouter({ serverIp = "127.0.0.1"; } const projectName = slugify(`${project.name} ${input.id}`); - const { envs, mounts, domains } = generate({ + const { envs, mounts, domains } = await generate({ serverIp: serverIp || "", projectName: projectName, }); @@ -449,18 +450,22 @@ export const composeRouter = createTRPCRouter({ return null; }), - templates: protectedProcedure.query(async () => { - const templatesData = templates.map((t) => ({ - name: t.name, - description: t.description, - id: t.id, - links: t.links, - tags: t.tags, - logo: t.logo, - version: t.version, - })); + templates: publicProcedure.query(async () => { + // First try to fetch templates from GitHub + try { + const githubTemplates = await fetchTemplatesList(); + if (githubTemplates.length > 0) { + return githubTemplates; + } + } catch (error) { + console.warn( + "Failed to fetch templates from GitHub, falling back to local templates:", + error, + ); + } - return templatesData; + // Fall back to local templates if GitHub fetch fails + return templates; }), getTags: protectedProcedure.query(async ({ input }) => { diff --git a/packages/server/package.json b/packages/server/package.json index cfff36fe..fd4c9394 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -62,7 +62,8 @@ "slugify": "^1.6.6", "ws": "8.16.0", "zod": "^3.23.4", - "ssh2": "1.15.0" + "ssh2": "1.15.0", + "@octokit/rest": "^20.0.2" }, "devDependencies": { "esbuild-plugin-alias": "0.2.1", diff --git a/packages/server/src/templates/config.ts b/packages/server/src/templates/config.ts new file mode 100644 index 00000000..c161b440 --- /dev/null +++ b/packages/server/src/templates/config.ts @@ -0,0 +1,38 @@ +/** + * Configuration for the GitHub template repository + */ +export const templateConfig = { + /** + * GitHub repository owner + * @default "dokploy" + */ + owner: process.env.TEMPLATE_REPO_OWNER || "dokploy", + + /** + * GitHub repository name + * @default "templates" + */ + repo: process.env.TEMPLATE_REPO_NAME || "templates", + + /** + * GitHub repository branch + * @default "main" + */ + branch: process.env.TEMPLATE_REPO_BRANCH || "main", + + /** + * Cache duration in milliseconds + * How long to cache templates before checking for updates + * @default 3600000 (1 hour) + */ + cacheDuration: Number.parseInt( + process.env.TEMPLATE_CACHE_DURATION || "3600000", + 10, + ), + + /** + * GitHub API token (optional) + * Used for higher rate limits + */ + token: process.env.GITHUB_TOKEN, +}; diff --git a/packages/server/src/templates/utils/github.ts b/packages/server/src/templates/utils/github.ts new file mode 100644 index 00000000..ebbbbdb2 --- /dev/null +++ b/packages/server/src/templates/utils/github.ts @@ -0,0 +1,325 @@ +import { execSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Octokit } from "@octokit/rest"; +import * as esbuild from "esbuild"; +import { load } from "js-yaml"; +import { templateConfig } from "../config"; +import type { Template } from "./index"; +import { + generateBase64, + generateHash, + generatePassword, + generateRandomDomain, +} from "./index"; + +// GitHub API client +const octokit = new Octokit({ + auth: templateConfig.token, +}); + +/** + * Interface for template metadata + */ +export interface TemplateMetadata { + id: string; + name: string; + version: string; + description: string; + logo: string; + links: { + github?: string; + website?: string; + docs?: string; + }; + tags: string[]; +} + +/** + * Fetches the list of available templates from GitHub + */ +export async function fetchTemplatesList( + owner = templateConfig.owner, + repo = templateConfig.repo, + branch = templateConfig.branch, +): Promise { + try { + // Fetch templates directory content + const { data: dirContent } = await octokit.repos.getContent({ + owner, + repo, + path: "templates", + ref: branch, + }); + + console.log("DIR CONTENT", dirContent); + + if (!Array.isArray(dirContent)) { + throw new Error("Templates directory not found or is not a directory"); + } + + // Filter for directories only (each directory is a template) + const templateDirs = dirContent.filter((item) => item.type === "dir"); + + // Fetch metadata for each template + const templates = await Promise.all( + templateDirs.map(async (dir) => { + try { + // Try to fetch metadata.json for each template + const { data: metadataFile } = await octokit.repos.getContent({ + owner, + repo, + path: `templates/${dir.name}/metadata.json`, + ref: branch, + }); + + if ("content" in metadataFile && metadataFile.encoding === "base64") { + const content = Buffer.from( + metadataFile.content, + "base64", + ).toString(); + return JSON.parse(content) as TemplateMetadata; + } + } catch (error) { + // If metadata.json doesn't exist, create a basic metadata object + return { + id: dir.name, + name: dir.name.charAt(0).toUpperCase() + dir.name.slice(1), + version: "latest", + description: `${dir.name} template`, + logo: "default.svg", + links: {}, + tags: [], + }; + } + + return null; + }), + ); + + return templates.filter(Boolean) as TemplateMetadata[]; + } catch (error) { + console.error("Error fetching templates list:", error); + throw error; + } +} + +/** + * Fetches a specific template's files from GitHub + */ +export async function fetchTemplateFiles( + templateId: string, + owner = templateConfig.owner, + repo = templateConfig.repo, + branch = templateConfig.branch, +): Promise<{ indexTs: string; dockerCompose: string }> { + try { + // Fetch index.ts + const { data: indexFile } = await octokit.repos.getContent({ + owner, + repo, + path: `templates/${templateId}/index.ts`, + ref: branch, + }); + + // Fetch docker-compose.yml + const { data: composeFile } = await octokit.repos.getContent({ + owner, + repo, + path: `templates/${templateId}/docker-compose.yml`, + ref: branch, + }); + + if (!("content" in indexFile) || !("content" in composeFile)) { + throw new Error("Template files not found"); + } + + const indexTs = Buffer.from(indexFile.content, "base64").toString(); + const dockerCompose = Buffer.from(composeFile.content, "base64").toString(); + + return { indexTs, dockerCompose }; + } catch (error) { + console.error(`Error fetching template ${templateId}:`, error); + throw error; + } +} + +/** + * Executes the template's index.ts code dynamically + * Uses a template-based approach that's safer and more efficient + */ +export async function executeTemplateCode( + indexTsCode: string, + schema: { serverIp: string; projectName: string }, +): Promise