From 203da1a8fe25a5c2ab66738cf07a0c7e3b768427 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Tue, 25 Feb 2025 22:51:02 -0600 Subject: [PATCH 001/100] refactor(traefik): migrate from Docker Swarm service to standalone container --- packages/server/src/setup/traefik-setup.ts | 143 +++++++++------------ 1 file changed, 59 insertions(+), 84 deletions(-) diff --git a/packages/server/src/setup/traefik-setup.ts b/packages/server/src/setup/traefik-setup.ts index 21caa5cf..5a904eb8 100644 --- a/packages/server/src/setup/traefik-setup.ts +++ b/packages/server/src/setup/traefik-setup.ts @@ -1,6 +1,6 @@ import { chmodSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; -import type { ContainerTaskSpec, CreateServiceOptions } from "dockerode"; +import type { ContainerCreateOptions } from "dockerode"; import { dump } from "js-yaml"; import { paths } from "../constants"; import { pullImage, pullRemoteImage } from "../utils/docker/utils"; @@ -34,68 +34,53 @@ export const initializeTraefik = async ({ const { MAIN_TRAEFIK_PATH, DYNAMIC_TRAEFIK_PATH } = paths(!!serverId); const imageName = `traefik:v${TRAEFIK_VERSION}`; const containerName = "dokploy-traefik"; - const settings: CreateServiceOptions = { - Name: containerName, - TaskTemplate: { - ContainerSpec: { - Image: imageName, - Env: env, - Mounts: [ + const settings: ContainerCreateOptions = { + name: containerName, + Image: imageName, + NetworkingConfig: { + EndpointsConfig: { + "dokploy-network": {}, + }, + }, + HostConfig: { + RestartPolicy: { + Name: "always", + }, + Binds: [ + `${MAIN_TRAEFIK_PATH}/traefik.yml:/etc/traefik/traefik.yml`, + `${DYNAMIC_TRAEFIK_PATH}:/etc/dokploy/traefik/dynamic`, + "/var/run/docker.sock:/var/run/docker.sock", + ], + PortBindings: { + [`${TRAEFIK_SSL_PORT}/tcp`]: [ { - Type: "bind", - Source: `${MAIN_TRAEFIK_PATH}/traefik.yml`, - Target: "/etc/traefik/traefik.yml", - }, - { - Type: "bind", - Source: DYNAMIC_TRAEFIK_PATH, - Target: "/etc/dokploy/traefik/dynamic", - }, - { - Type: "bind", - Source: "/var/run/docker.sock", - Target: "/var/run/docker.sock", + HostPort: TRAEFIK_SSL_PORT.toString(), }, ], - }, - Networks: [{ Target: "dokploy-network" }], - Placement: { - Constraints: ["node.role==manager"], - }, - }, - Mode: { - Replicated: { - Replicas: 1, - }, - }, - EndpointSpec: { - Ports: [ - { - TargetPort: 443, - PublishedPort: TRAEFIK_SSL_PORT, - PublishMode: "host", - }, - { - TargetPort: 80, - PublishedPort: TRAEFIK_PORT, - PublishMode: "host", - }, - ...(enableDashboard - ? [ + [`${TRAEFIK_PORT}/tcp`]: [ + { + HostPort: TRAEFIK_PORT.toString(), + }, + ], + ...(enableDashboard && { + [`${8080}/tcp`]: [ + { + HostPort: "8080", + }, + ], + }), + ...additionalPorts.map((port) => { + return { + [`${port.targetPort}/tcp`]: [ { - TargetPort: 8080, - PublishedPort: 8080, - PublishMode: "host" as const, + HostPort: port.publishedPort.toString(), }, - ] - : []), - ...additionalPorts.map((port) => ({ - TargetPort: port.targetPort, - PublishedPort: port.publishedPort, - PublishMode: port.publishMode || ("host" as const), - })), - ], + ], + }; + }), + }, }, + Env: env, }; const docker = await getRemoteDocker(serverId); try { @@ -105,38 +90,28 @@ export const initializeTraefik = async ({ await pullImage(imageName); } - const service = docker.getService(containerName); - const inspect = await service.inspect(); + const container = docker.getContainer(containerName); + try { + const inspect = await container.inspect(); + if (inspect.State.Status === "running") { + console.log("Traefik already running"); + return; + } - const existingEnv = inspect.Spec.TaskTemplate.ContainerSpec.Env || []; - const updatedEnv = !env ? existingEnv : env; + await container.remove({ force: true }); + console.log("Removed existing container"); + } catch (error) { + console.log("Traefik Not Found: Starting1 ✅"); + console.log(error); + } - const updatedSettings = { - ...settings, - TaskTemplate: { - ...settings.TaskTemplate, - ContainerSpec: { - ...(settings?.TaskTemplate as ContainerTaskSpec).ContainerSpec, - Env: updatedEnv, - }, - }, - }; - await service.update({ - version: Number.parseInt(inspect.Version.Index), - ...updatedSettings, - }); + await docker.createContainer(settings); + const newContainer = docker.getContainer(containerName); + await newContainer.start(); console.log("Traefik Started ✅"); } catch (error) { - try { - await docker.createService(settings); - } catch (error: any) { - if (error?.statusCode !== 409) { - throw error; - } - console.log("Traefik service already exists, continuing..."); - } - console.log("Traefik Not Found: Starting ✅"); + console.log("Traefik Not Found: Starting2 ✅", error); } }; From 29c1e4691ea9ed41b01c20ba8981224ea147ba72 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Tue, 25 Feb 2025 23:04:19 -0600 Subject: [PATCH 002/100] feat(docker): add support for standalone container log retrieval --- .../settings/servers/actions/show-traefik-actions.tsx | 6 +++++- .../dashboard/settings/web-server/show-modal-logs.tsx | 9 ++++++++- apps/dokploy/server/api/routers/docker.ts | 7 ++++++- packages/server/src/services/docker.ts | 8 +++++++- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/apps/dokploy/components/dashboard/settings/servers/actions/show-traefik-actions.tsx b/apps/dokploy/components/dashboard/settings/servers/actions/show-traefik-actions.tsx index 17a6ae75..f9b79947 100644 --- a/apps/dokploy/components/dashboard/settings/servers/actions/show-traefik-actions.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/actions/show-traefik-actions.tsx @@ -79,7 +79,11 @@ export const ShowTraefikActions = ({ serverId }: Props) => { > {t("settings.server.webServer.reload")} - + e.preventDefault()} className="cursor-pointer" diff --git a/apps/dokploy/components/dashboard/settings/web-server/show-modal-logs.tsx b/apps/dokploy/components/dashboard/settings/web-server/show-modal-logs.tsx index 12e7b670..f6d17f9b 100644 --- a/apps/dokploy/components/dashboard/settings/web-server/show-modal-logs.tsx +++ b/apps/dokploy/components/dashboard/settings/web-server/show-modal-logs.tsx @@ -37,13 +37,20 @@ interface Props { appName: string; children?: React.ReactNode; serverId?: string; + type: "standalone" | "swarm"; } -export const ShowModalLogs = ({ appName, children, serverId }: Props) => { +export const ShowModalLogs = ({ + appName, + children, + serverId, + type = "swarm", +}: Props) => { const { data, isLoading } = api.docker.getContainersByAppLabel.useQuery( { appName, serverId, + type, }, { enabled: !!appName, diff --git a/apps/dokploy/server/api/routers/docker.ts b/apps/dokploy/server/api/routers/docker.ts index f6972e16..3de59058 100644 --- a/apps/dokploy/server/api/routers/docker.ts +++ b/apps/dokploy/server/api/routers/docker.ts @@ -65,10 +65,15 @@ export const dockerRouter = createTRPCRouter({ z.object({ appName: z.string().min(1), serverId: z.string().optional(), + type: z.enum(["standalone", "swarm"]), }), ) .query(async ({ input }) => { - return await getContainersByAppLabel(input.appName, input.serverId); + return await getContainersByAppLabel( + input.appName, + input.type, + input.serverId, + ); }), getStackContainersByAppName: protectedProcedure diff --git a/packages/server/src/services/docker.ts b/packages/server/src/services/docker.ts index 597c03fa..fbb51192 100644 --- a/packages/server/src/services/docker.ts +++ b/packages/server/src/services/docker.ts @@ -281,13 +281,19 @@ export const getServiceContainersByAppName = async ( export const getContainersByAppLabel = async ( appName: string, + type: "standalone" | "swarm", serverId?: string, ) => { try { let stdout = ""; let stderr = ""; - const command = `docker ps --filter "label=com.docker.swarm.service.name=${appName}" --format 'CONTAINER ID : {{.ID}} | Name: {{.Names}} | State: {{.State}}'`; + const command = + type === "swarm" + ? `docker ps --filter "label=com.docker.swarm.service.name=${appName}" --format 'CONTAINER ID : {{.ID}} | Name: {{.Names}} | State: {{.State}}'` + : type === "standalone" + ? `docker ps --filter "name=${appName}" --format 'CONTAINER ID : {{.ID}} | Name: {{.Names}} | State: {{.State}}'` + : `docker ps --filter "label=com.docker.compose.project=${appName}" --format 'CONTAINER ID : {{.ID}} | Name: {{.Names}} | State: {{.State}}'`; if (serverId) { const result = await execAsyncRemote(serverId, command); stdout = result.stdout; From 49b37d531a45c10956068e0a0fcf5c5fc3c7f696 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Sat, 1 Mar 2025 00:57:32 -0600 Subject: [PATCH 003/100] feat: add GitHub-based template fetching and caching mechanism --- apps/dokploy/server/api/routers/compose.ts | 33 +- packages/server/package.json | 3 +- packages/server/src/templates/config.ts | 38 ++ packages/server/src/templates/utils/github.ts | 325 ++++++++++++++++++ packages/server/src/templates/utils/index.ts | 117 ++++++- pnpm-lock.yaml | 58 ++++ 6 files changed, 553 insertions(+), 21 deletions(-) create mode 100644 packages/server/src/templates/config.ts create mode 100644 packages/server/src/templates/utils/github.ts diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index 463d0398..f8deea0d 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -18,12 +18,13 @@ import { loadTemplateModule, readTemplateComposeFile, } from "@/templates/utils"; +import { fetchTemplatesList } from "@dokploy/server/templates/utils/github"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; import { dump } from "js-yaml"; import _ from "lodash"; import { nanoid } from "nanoid"; -import { createTRPCRouter, protectedProcedure } from "../trpc"; +import { createTRPCRouter, protectedProcedure, publicProcedure } from "../trpc"; import type { DeploymentJob } from "@/server/queues/queue-types"; import { deploy } from "@/server/utils/deploy"; @@ -388,7 +389,7 @@ export const composeRouter = createTRPCRouter({ const composeFile = await readTemplateComposeFile(input.id); - const generate = await loadTemplateModule(input.id as TemplatesKeys); + const generate = await loadTemplateModule(input.id); const admin = await findAdminById(ctx.user.adminId); let serverIp = admin.serverIp || "127.0.0.1"; @@ -402,7 +403,7 @@ export const composeRouter = createTRPCRouter({ serverIp = "127.0.0.1"; } const projectName = slugify(`${project.name} ${input.id}`); - const { envs, mounts, domains } = generate({ + const { envs, mounts, domains } = await generate({ serverIp: serverIp || "", projectName: projectName, }); @@ -449,18 +450,22 @@ export const composeRouter = createTRPCRouter({ return null; }), - templates: protectedProcedure.query(async () => { - const templatesData = templates.map((t) => ({ - name: t.name, - description: t.description, - id: t.id, - links: t.links, - tags: t.tags, - logo: t.logo, - version: t.version, - })); + templates: publicProcedure.query(async () => { + // First try to fetch templates from GitHub + try { + const githubTemplates = await fetchTemplatesList(); + if (githubTemplates.length > 0) { + return githubTemplates; + } + } catch (error) { + console.warn( + "Failed to fetch templates from GitHub, falling back to local templates:", + error, + ); + } - return templatesData; + // Fall back to local templates if GitHub fetch fails + return templates; }), getTags: protectedProcedure.query(async ({ input }) => { diff --git a/packages/server/package.json b/packages/server/package.json index cfff36fe..fd4c9394 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -62,7 +62,8 @@ "slugify": "^1.6.6", "ws": "8.16.0", "zod": "^3.23.4", - "ssh2": "1.15.0" + "ssh2": "1.15.0", + "@octokit/rest": "^20.0.2" }, "devDependencies": { "esbuild-plugin-alias": "0.2.1", diff --git a/packages/server/src/templates/config.ts b/packages/server/src/templates/config.ts new file mode 100644 index 00000000..c161b440 --- /dev/null +++ b/packages/server/src/templates/config.ts @@ -0,0 +1,38 @@ +/** + * Configuration for the GitHub template repository + */ +export const templateConfig = { + /** + * GitHub repository owner + * @default "dokploy" + */ + owner: process.env.TEMPLATE_REPO_OWNER || "dokploy", + + /** + * GitHub repository name + * @default "templates" + */ + repo: process.env.TEMPLATE_REPO_NAME || "templates", + + /** + * GitHub repository branch + * @default "main" + */ + branch: process.env.TEMPLATE_REPO_BRANCH || "main", + + /** + * Cache duration in milliseconds + * How long to cache templates before checking for updates + * @default 3600000 (1 hour) + */ + cacheDuration: Number.parseInt( + process.env.TEMPLATE_CACHE_DURATION || "3600000", + 10, + ), + + /** + * GitHub API token (optional) + * Used for higher rate limits + */ + token: process.env.GITHUB_TOKEN, +}; diff --git a/packages/server/src/templates/utils/github.ts b/packages/server/src/templates/utils/github.ts new file mode 100644 index 00000000..ebbbbdb2 --- /dev/null +++ b/packages/server/src/templates/utils/github.ts @@ -0,0 +1,325 @@ +import { execSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Octokit } from "@octokit/rest"; +import * as esbuild from "esbuild"; +import { load } from "js-yaml"; +import { templateConfig } from "../config"; +import type { Template } from "./index"; +import { + generateBase64, + generateHash, + generatePassword, + generateRandomDomain, +} from "./index"; + +// GitHub API client +const octokit = new Octokit({ + auth: templateConfig.token, +}); + +/** + * Interface for template metadata + */ +export interface TemplateMetadata { + id: string; + name: string; + version: string; + description: string; + logo: string; + links: { + github?: string; + website?: string; + docs?: string; + }; + tags: string[]; +} + +/** + * Fetches the list of available templates from GitHub + */ +export async function fetchTemplatesList( + owner = templateConfig.owner, + repo = templateConfig.repo, + branch = templateConfig.branch, +): Promise { + try { + // Fetch templates directory content + const { data: dirContent } = await octokit.repos.getContent({ + owner, + repo, + path: "templates", + ref: branch, + }); + + console.log("DIR CONTENT", dirContent); + + if (!Array.isArray(dirContent)) { + throw new Error("Templates directory not found or is not a directory"); + } + + // Filter for directories only (each directory is a template) + const templateDirs = dirContent.filter((item) => item.type === "dir"); + + // Fetch metadata for each template + const templates = await Promise.all( + templateDirs.map(async (dir) => { + try { + // Try to fetch metadata.json for each template + const { data: metadataFile } = await octokit.repos.getContent({ + owner, + repo, + path: `templates/${dir.name}/metadata.json`, + ref: branch, + }); + + if ("content" in metadataFile && metadataFile.encoding === "base64") { + const content = Buffer.from( + metadataFile.content, + "base64", + ).toString(); + return JSON.parse(content) as TemplateMetadata; + } + } catch (error) { + // If metadata.json doesn't exist, create a basic metadata object + return { + id: dir.name, + name: dir.name.charAt(0).toUpperCase() + dir.name.slice(1), + version: "latest", + description: `${dir.name} template`, + logo: "default.svg", + links: {}, + tags: [], + }; + } + + return null; + }), + ); + + return templates.filter(Boolean) as TemplateMetadata[]; + } catch (error) { + console.error("Error fetching templates list:", error); + throw error; + } +} + +/** + * Fetches a specific template's files from GitHub + */ +export async function fetchTemplateFiles( + templateId: string, + owner = templateConfig.owner, + repo = templateConfig.repo, + branch = templateConfig.branch, +): Promise<{ indexTs: string; dockerCompose: string }> { + try { + // Fetch index.ts + const { data: indexFile } = await octokit.repos.getContent({ + owner, + repo, + path: `templates/${templateId}/index.ts`, + ref: branch, + }); + + // Fetch docker-compose.yml + const { data: composeFile } = await octokit.repos.getContent({ + owner, + repo, + path: `templates/${templateId}/docker-compose.yml`, + ref: branch, + }); + + if (!("content" in indexFile) || !("content" in composeFile)) { + throw new Error("Template files not found"); + } + + const indexTs = Buffer.from(indexFile.content, "base64").toString(); + const dockerCompose = Buffer.from(composeFile.content, "base64").toString(); + + return { indexTs, dockerCompose }; + } catch (error) { + console.error(`Error fetching template ${templateId}:`, error); + throw error; + } +} + +/** + * Executes the template's index.ts code dynamically + * Uses a template-based approach that's safer and more efficient + */ +export async function executeTemplateCode( + indexTsCode: string, + schema: { serverIp: string; projectName: string }, +): Promise