mirror of
https://github.com/Dokploy/dokploy
synced 2025-06-26 18:27:59 +00:00
feat: add GitHub-based template fetching and caching mechanism
This commit is contained in:
Mauricio Siu
38
packages/server/src/templates/config.ts
Normal file
38
packages/server/src/templates/config.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Configuration for the GitHub template repository
|
||||
*/
|
||||
export const templateConfig = {
|
||||
/**
|
||||
* GitHub repository owner
|
||||
* @default "dokploy"
|
||||
*/
|
||||
owner: process.env.TEMPLATE_REPO_OWNER || "dokploy",
|
||||
|
||||
/**
|
||||
* GitHub repository name
|
||||
* @default "templates"
|
||||
*/
|
||||
repo: process.env.TEMPLATE_REPO_NAME || "templates",
|
||||
|
||||
/**
|
||||
* GitHub repository branch
|
||||
* @default "main"
|
||||
*/
|
||||
branch: process.env.TEMPLATE_REPO_BRANCH || "main",
|
||||
|
||||
/**
|
||||
* Cache duration in milliseconds
|
||||
* How long to cache templates before checking for updates
|
||||
* @default 3600000 (1 hour)
|
||||
*/
|
||||
cacheDuration: Number.parseInt(
|
||||
process.env.TEMPLATE_CACHE_DURATION || "3600000",
|
||||
10,
|
||||
),
|
||||
|
||||
/**
|
||||
* GitHub API token (optional)
|
||||
* Used for higher rate limits
|
||||
*/
|
||||
token: process.env.GITHUB_TOKEN,
|
||||
};
|
||||
325
packages/server/src/templates/utils/github.ts
Normal file
325
packages/server/src/templates/utils/github.ts
Normal file
@@ -0,0 +1,325 @@
|
||||
import { execSync } from "node:child_process";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Octokit } from "@octokit/rest";
|
||||
import * as esbuild from "esbuild";
|
||||
import { load } from "js-yaml";
|
||||
import { templateConfig } from "../config";
|
||||
import type { Template } from "./index";
|
||||
import {
|
||||
generateBase64,
|
||||
generateHash,
|
||||
generatePassword,
|
||||
generateRandomDomain,
|
||||
} from "./index";
|
||||
|
||||
// GitHub API client
|
||||
const octokit = new Octokit({
|
||||
auth: templateConfig.token,
|
||||
});
|
||||
|
||||
/**
|
||||
* Interface for template metadata
|
||||
*/
|
||||
export interface TemplateMetadata {
|
||||
id: string;
|
||||
name: string;
|
||||
version: string;
|
||||
description: string;
|
||||
logo: string;
|
||||
links: {
|
||||
github?: string;
|
||||
website?: string;
|
||||
docs?: string;
|
||||
};
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the list of available templates from GitHub
|
||||
*/
|
||||
export async function fetchTemplatesList(
|
||||
owner = templateConfig.owner,
|
||||
repo = templateConfig.repo,
|
||||
branch = templateConfig.branch,
|
||||
): Promise<TemplateMetadata[]> {
|
||||
try {
|
||||
// Fetch templates directory content
|
||||
const { data: dirContent } = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: "templates",
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
console.log("DIR CONTENT", dirContent);
|
||||
|
||||
if (!Array.isArray(dirContent)) {
|
||||
throw new Error("Templates directory not found or is not a directory");
|
||||
}
|
||||
|
||||
// Filter for directories only (each directory is a template)
|
||||
const templateDirs = dirContent.filter((item) => item.type === "dir");
|
||||
|
||||
// Fetch metadata for each template
|
||||
const templates = await Promise.all(
|
||||
templateDirs.map(async (dir) => {
|
||||
try {
|
||||
// Try to fetch metadata.json for each template
|
||||
const { data: metadataFile } = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: `templates/${dir.name}/metadata.json`,
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
if ("content" in metadataFile && metadataFile.encoding === "base64") {
|
||||
const content = Buffer.from(
|
||||
metadataFile.content,
|
||||
"base64",
|
||||
).toString();
|
||||
return JSON.parse(content) as TemplateMetadata;
|
||||
}
|
||||
} catch (error) {
|
||||
// If metadata.json doesn't exist, create a basic metadata object
|
||||
return {
|
||||
id: dir.name,
|
||||
name: dir.name.charAt(0).toUpperCase() + dir.name.slice(1),
|
||||
version: "latest",
|
||||
description: `${dir.name} template`,
|
||||
logo: "default.svg",
|
||||
links: {},
|
||||
tags: [],
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}),
|
||||
);
|
||||
|
||||
return templates.filter(Boolean) as TemplateMetadata[];
|
||||
} catch (error) {
|
||||
console.error("Error fetching templates list:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a specific template's files from GitHub
|
||||
*/
|
||||
export async function fetchTemplateFiles(
|
||||
templateId: string,
|
||||
owner = templateConfig.owner,
|
||||
repo = templateConfig.repo,
|
||||
branch = templateConfig.branch,
|
||||
): Promise<{ indexTs: string; dockerCompose: string }> {
|
||||
try {
|
||||
// Fetch index.ts
|
||||
const { data: indexFile } = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: `templates/${templateId}/index.ts`,
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
// Fetch docker-compose.yml
|
||||
const { data: composeFile } = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: `templates/${templateId}/docker-compose.yml`,
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
if (!("content" in indexFile) || !("content" in composeFile)) {
|
||||
throw new Error("Template files not found");
|
||||
}
|
||||
|
||||
const indexTs = Buffer.from(indexFile.content, "base64").toString();
|
||||
const dockerCompose = Buffer.from(composeFile.content, "base64").toString();
|
||||
|
||||
return { indexTs, dockerCompose };
|
||||
} catch (error) {
|
||||
console.error(`Error fetching template ${templateId}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the template's index.ts code dynamically
|
||||
* Uses a template-based approach that's safer and more efficient
|
||||
*/
|
||||
export async function executeTemplateCode(
|
||||
indexTsCode: string,
|
||||
schema: { serverIp: string; projectName: string },
|
||||
): Promise<Template> {
|
||||
try {
|
||||
// Create a temporary directory for the template
|
||||
const cwd = process.cwd();
|
||||
const tempId = randomBytes(8).toString("hex");
|
||||
const tempDir = join(cwd, ".next", "temp", tempId);
|
||||
|
||||
if (!existsSync(tempDir)) {
|
||||
await mkdir(tempDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Extract the generate function body
|
||||
// This approach assumes templates follow a standard structure with a generate function
|
||||
const generateFunctionMatch = indexTsCode.match(
|
||||
/export\s+function\s+generate\s*\([^)]*\)\s*{([\s\S]*?)return\s+{([\s\S]*?)};?\s*}/,
|
||||
);
|
||||
|
||||
if (!generateFunctionMatch) {
|
||||
throw new Error("Could not extract generate function from template");
|
||||
}
|
||||
|
||||
const functionBody = generateFunctionMatch[1];
|
||||
const returnStatement = generateFunctionMatch[2];
|
||||
|
||||
// Create a simplified template that doesn't require imports
|
||||
const templateCode = `
|
||||
// Utility functions provided to the template
|
||||
function generateRandomDomain(schema) {
|
||||
const hash = Math.random().toString(36).substring(2, 8);
|
||||
const slugIp = schema.serverIp.replaceAll(".", "-");
|
||||
return \`\${schema.projectName}-\${hash}\${slugIp === "" ? "" : \`-\${slugIp}\`}.traefik.me\`;
|
||||
}
|
||||
|
||||
function generateHash(projectName, quantity = 3) {
|
||||
const hash = Math.random().toString(36).substring(2, 2 + quantity);
|
||||
return \`\${projectName}-\${hash}\`;
|
||||
}
|
||||
|
||||
function generatePassword(quantity = 16) {
|
||||
const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
||||
let password = "";
|
||||
for (let i = 0; i < quantity; i++) {
|
||||
password += characters.charAt(Math.floor(Math.random() * characters.length));
|
||||
}
|
||||
return password.toLowerCase();
|
||||
}
|
||||
|
||||
function generateBase64(bytes = 32) {
|
||||
return Math.random().toString(36).substring(2, 2 + bytes);
|
||||
}
|
||||
|
||||
// Template execution
|
||||
function execute(schema) {
|
||||
${functionBody}
|
||||
return {
|
||||
${returnStatement}
|
||||
};
|
||||
}
|
||||
|
||||
// Run with the provided schema and output the result
|
||||
const result = execute(${JSON.stringify(schema)});
|
||||
console.log(JSON.stringify(result));
|
||||
`;
|
||||
|
||||
// Write the template code to a file
|
||||
const templatePath = join(tempDir, "template.js");
|
||||
await writeFile(templatePath, templateCode, "utf8");
|
||||
|
||||
// Execute the template using Node.js
|
||||
const output = execSync(`node ${templatePath}`, {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
// Parse the output as JSON
|
||||
return JSON.parse(output);
|
||||
} catch (error) {
|
||||
console.error("Error executing template code:", error);
|
||||
|
||||
// Fallback to a simpler approach if the template extraction fails
|
||||
return fallbackExecuteTemplate(indexTsCode, schema);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback method to execute templates that don't follow the standard structure
|
||||
*/
|
||||
async function fallbackExecuteTemplate(
|
||||
indexTsCode: string,
|
||||
schema: { serverIp: string; projectName: string },
|
||||
): Promise<Template> {
|
||||
try {
|
||||
// Create a temporary directory
|
||||
const cwd = process.cwd();
|
||||
const tempId = randomBytes(8).toString("hex");
|
||||
const tempDir = join(cwd, ".next", "temp", tempId);
|
||||
|
||||
if (!existsSync(tempDir)) {
|
||||
await mkdir(tempDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Create a simplified version of the template code
|
||||
// Remove TypeScript types and imports
|
||||
const simplifiedCode = indexTsCode
|
||||
.replace(/import\s+.*?from\s+['"].*?['"]\s*;?/g, "")
|
||||
.replace(/export\s+interface\s+.*?{[\s\S]*?}/g, "")
|
||||
.replace(/:\s*Schema/g, "")
|
||||
.replace(/:\s*DomainSchema/g, "")
|
||||
.replace(/:\s*Template/g, "")
|
||||
.replace(/:\s*string/g, "")
|
||||
.replace(/:\s*number/g, "")
|
||||
.replace(/:\s*boolean/g, "")
|
||||
.replace(/:\s*any/g, "")
|
||||
.replace(/:\s*unknown/g, "")
|
||||
.replace(/<.*?>/g, "");
|
||||
|
||||
// Create a wrapper with all necessary utilities
|
||||
const wrapperCode = `
|
||||
// Utility functions
|
||||
function generateRandomDomain(schema) {
|
||||
const hash = Math.random().toString(36).substring(2, 8);
|
||||
const slugIp = schema.serverIp.replaceAll(".", "-");
|
||||
return \`\${schema.projectName}-\${hash}\${slugIp === "" ? "" : \`-\${slugIp}\`}.traefik.me\`;
|
||||
}
|
||||
|
||||
function generateHash(projectName, quantity = 3) {
|
||||
const hash = Math.random().toString(36).substring(2, 2 + quantity);
|
||||
return \`\${projectName}-\${hash}\`;
|
||||
}
|
||||
|
||||
function generatePassword(quantity = 16) {
|
||||
const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
||||
let password = "";
|
||||
for (let i = 0; i < quantity; i++) {
|
||||
password += characters.charAt(Math.floor(Math.random() * characters.length));
|
||||
}
|
||||
return password.toLowerCase();
|
||||
}
|
||||
|
||||
function generateBase64(bytes = 32) {
|
||||
return Math.random().toString(36).substring(2, 2 + bytes);
|
||||
}
|
||||
|
||||
// Simplified template code
|
||||
${simplifiedCode}
|
||||
|
||||
// Execute the template
|
||||
const result = generate(${JSON.stringify(schema)});
|
||||
console.log(JSON.stringify(result));
|
||||
`;
|
||||
|
||||
// Write the wrapper code to a file
|
||||
const wrapperPath = join(tempDir, "wrapper.js");
|
||||
await writeFile(wrapperPath, wrapperCode, "utf8");
|
||||
|
||||
// Execute the code using Node.js
|
||||
const output = execSync(`node ${wrapperPath}`, {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
// Parse the output as JSON
|
||||
return JSON.parse(output);
|
||||
} catch (error) {
|
||||
console.error("Error in fallback template execution:", error);
|
||||
throw new Error(
|
||||
`Failed to execute template: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,11 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { Domain } from "@dokploy/server/services/domain";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { templateConfig } from "../config";
|
||||
import { executeTemplateCode, fetchTemplateFiles } from "./github";
|
||||
|
||||
export interface Schema {
|
||||
serverIp: string;
|
||||
@@ -50,12 +54,113 @@ export const generateBase64 = (bytes = 32): string => {
|
||||
return randomBytes(bytes).toString("base64");
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if a cached file is still valid based on its modification time
|
||||
*/
|
||||
async function isCacheValid(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
if (!existsSync(filePath)) return false;
|
||||
|
||||
const fileStats = await stat(filePath);
|
||||
const modifiedTime = fileStats.mtime.getTime();
|
||||
const currentTime = Date.now();
|
||||
|
||||
// Check if the file is older than the cache duration
|
||||
return currentTime - modifiedTime < templateConfig.cacheDuration;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a template's docker-compose.yml file
|
||||
* First tries to read from the local cache, if not found or expired, fetches from GitHub
|
||||
*/
|
||||
export const readTemplateComposeFile = async (id: string) => {
|
||||
const cwd = process.cwd();
|
||||
const composeFile = await readFile(
|
||||
join(cwd, ".next", "templates", id, "docker-compose.yml"),
|
||||
"utf8",
|
||||
);
|
||||
const templatePath = join(cwd, ".next", "templates", id);
|
||||
const composeFilePath = join(templatePath, "docker-compose.yml");
|
||||
|
||||
return composeFile;
|
||||
// Check if the file exists in the local cache and is still valid
|
||||
if (await isCacheValid(composeFilePath)) {
|
||||
return await readFile(composeFilePath, "utf8");
|
||||
}
|
||||
|
||||
// If not in cache or expired, fetch from GitHub and cache it
|
||||
try {
|
||||
const { dockerCompose } = await fetchTemplateFiles(id);
|
||||
|
||||
// Ensure the template directory exists
|
||||
if (!existsSync(templatePath)) {
|
||||
await mkdir(templatePath, { recursive: true });
|
||||
}
|
||||
|
||||
// Cache the file for future use
|
||||
await writeFile(composeFilePath, dockerCompose, "utf8");
|
||||
|
||||
return dockerCompose;
|
||||
} catch (error) {
|
||||
// If fetch fails but we have a cached version, use it as fallback
|
||||
if (existsSync(composeFilePath)) {
|
||||
console.warn(
|
||||
`Using cached version of template ${id} due to fetch error:`,
|
||||
error,
|
||||
);
|
||||
return await readFile(composeFilePath, "utf8");
|
||||
}
|
||||
|
||||
console.error(`Error fetching template ${id}:`, error);
|
||||
throw new TRPCError({
|
||||
code: "NOT_FOUND",
|
||||
message: `Template ${id} not found or could not be fetched`,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Loads a template module and returns its generate function
|
||||
* First tries to execute from local cache, if not found or expired, fetches from GitHub
|
||||
*/
|
||||
export const loadTemplateModule = async (id: string) => {
|
||||
const cwd = process.cwd();
|
||||
const templatePath = join(cwd, ".next", "templates", id);
|
||||
const indexFilePath = join(templatePath, "index.ts");
|
||||
|
||||
// Check if we have the template cached locally and it's still valid
|
||||
if (await isCacheValid(indexFilePath)) {
|
||||
const indexTs = await readFile(indexFilePath, "utf8");
|
||||
return (schema: Schema) => executeTemplateCode(indexTs, schema);
|
||||
}
|
||||
|
||||
// If not in cache or expired, fetch from GitHub and cache it
|
||||
try {
|
||||
const { indexTs } = await fetchTemplateFiles(id);
|
||||
|
||||
// Ensure the template directory exists
|
||||
if (!existsSync(templatePath)) {
|
||||
await mkdir(templatePath, { recursive: true });
|
||||
}
|
||||
|
||||
// Cache the file for future use
|
||||
await writeFile(indexFilePath, indexTs, "utf8");
|
||||
|
||||
// Return a function that will execute the template code
|
||||
return (schema: Schema) => executeTemplateCode(indexTs, schema);
|
||||
} catch (error) {
|
||||
// If fetch fails but we have a cached version, use it as fallback
|
||||
if (existsSync(indexFilePath)) {
|
||||
console.warn(
|
||||
`Using cached version of template ${id} due to fetch error:`,
|
||||
error,
|
||||
);
|
||||
const indexTs = await readFile(indexFilePath, "utf8");
|
||||
return (schema: Schema) => executeTemplateCode(indexTs, schema);
|
||||
}
|
||||
|
||||
console.error(`Error loading template module ${id}:`, error);
|
||||
throw new TRPCError({
|
||||
code: "NOT_FOUND",
|
||||
message: `Template ${id} not found or could not be loaded`,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user