refactor: update template system with new configuration structure and processing

This commit is contained in:
Mauricio Siu
2025-03-01 03:11:29 -06:00
parent 49b37d531a
commit 9aff4bc10b
11 changed files with 585 additions and 434 deletions

View File

@@ -1,13 +1,11 @@
import { execSync } from "node:child_process";
import { randomBytes } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { mkdir, readFile, writeFile, rm } from "node:fs/promises";
import { join } from "node:path";
import { Octokit } from "@octokit/rest";
import * as esbuild from "esbuild";
import { load } from "js-yaml";
import { templateConfig } from "../config";
import type { Template } from "./index";
import type { Schema, Template, DomainSchema } from "./index";
import {
generateBase64,
generateHash,
@@ -21,20 +19,179 @@ const octokit = new Octokit({
});
/**
* Interface for template metadata
* Complete template interface that includes both metadata and configuration
*/
export interface TemplateMetadata {
id: string;
name: string;
version: string;
description: string;
logo: string;
links: {
github?: string;
website?: string;
docs?: string;
export interface CompleteTemplate {
metadata: {
id: string;
name: string;
description: string;
tags: string[];
version: string;
logo: string;
links: {
github: string;
website?: string;
docs?: string;
};
};
tags: string[];
variables: {
[key: string]: string;
};
config: {
domains: Array<{
serviceName: string;
port: number;
path?: string;
host?: string;
}>;
env: Record<string, string>;
mounts?: Array<{
filePath: string;
content: string;
}>;
};
}
/**
* Utility functions that can be used in template values
*/
const TEMPLATE_FUNCTIONS = {
$randomDomain: () => true,
$password: (length = 16) => `$password(${length})`,
$base64: (bytes = 32) => `$base64(${bytes})`,
$base32: (bytes = 32) => `$base32(${bytes})`,
$hash: (length = 8) => `$hash(${length})`,
} as const;
/**
* Process a string value and replace variables
*/
function processValue(
value: string,
variables: Record<string, string>,
schema: Schema,
): string {
// First replace utility functions
let processedValue = value.replace(/\${([^}]+)}/g, (match, varName) => {
// Handle utility functions
if (varName === "randomDomain") {
return generateRandomDomain(schema);
}
if (varName.startsWith("base64:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 32;
return generateBase64(length);
}
if (varName.startsWith("base32:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 32;
return Buffer.from(randomBytes(length))
.toString("base64")
.substring(0, length);
}
if (varName.startsWith("password:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 16;
return generatePassword(length);
}
if (varName.startsWith("hash:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 8;
return generateHash(length);
}
// If not a utility function, try to get from variables
return variables[varName] || match;
});
// Then replace any remaining ${var} with their values from variables
processedValue = processedValue.replace(/\${([^}]+)}/g, (match, varName) => {
return variables[varName] || match;
});
return processedValue;
}
/**
* Processes a template configuration and returns the generated template
*/
export function processTemplate(
config: CompleteTemplate,
schema: Schema,
): Template {
const result: Template = {
envs: [],
domains: [],
mounts: [],
};
// First pass: Process variables that don't depend on domains
const variables: Record<string, string> = {};
for (const [key, value] of Object.entries(config.variables)) {
if (value === "${randomDomain}") {
variables[key] = generateRandomDomain(schema);
} else if (value.startsWith("${base64:")) {
const match = value.match(/\${base64:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 32;
variables[key] = generateBase64(length);
} else if (value.startsWith("${base32:")) {
const match = value.match(/\${base32:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 32;
variables[key] = Buffer.from(randomBytes(length))
.toString("base64")
.substring(0, length);
} else if (value.startsWith("${password:")) {
const match = value.match(/\${password:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 16;
variables[key] = generatePassword(length);
} else if (value.startsWith("${hash:")) {
const match = value.match(/\${hash:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 8;
variables[key] = generateHash(length);
} else {
variables[key] = value;
}
}
console.log(variables);
// Process domains and add them to variables
for (const domain of config.config.domains) {
// If host is specified, process it with variables, otherwise generate random domain
const host = domain.host
? processValue(domain.host, variables, schema)
: generateRandomDomain(schema);
result.domains.push({
host,
...domain,
});
// Add domain to variables for reference
variables[`domain:${domain.serviceName}`] = host;
}
// Process environment variables with access to all variables
for (const [key, value] of Object.entries(config.config.env)) {
const processedValue = processValue(value, variables, schema);
result.envs.push(`${key}=${processedValue}`);
}
// Process mounts with access to all variables
if (config.config.mounts) {
for (const mount of config.config.mounts) {
result.mounts.push({
filePath: mount.filePath,
content: processValue(mount.content, variables, schema),
});
}
}
return result;
}
/**
* GitHub tree item with required fields
*/
interface GitTreeItem {
path: string;
type: string;
sha: string;
}
/**
@@ -44,62 +201,54 @@ export async function fetchTemplatesList(
owner = templateConfig.owner,
repo = templateConfig.repo,
branch = templateConfig.branch,
): Promise<TemplateMetadata[]> {
): Promise<CompleteTemplate[]> {
try {
// Fetch templates directory content
const { data: dirContent } = await octokit.repos.getContent({
// First get the tree SHA for the branch
const { data: ref } = await octokit.git.getRef({
owner,
repo,
path: "templates",
ref: branch,
ref: `heads/${branch}`,
});
console.log("DIR CONTENT", dirContent);
// Get the full tree recursively
const { data: tree } = await octokit.git.getTree({
owner,
repo,
tree_sha: ref.object.sha,
recursive: "true",
});
if (!Array.isArray(dirContent)) {
throw new Error("Templates directory not found or is not a directory");
}
// Filter for template.yml files in the templates directory
const templateFiles = tree.tree.filter((item): item is GitTreeItem => {
return (
item.type === "blob" &&
typeof item.path === "string" &&
typeof item.sha === "string" &&
item.path.startsWith("templates/") &&
item.path.endsWith("/template.yml")
);
});
// Filter for directories only (each directory is a template)
const templateDirs = dirContent.filter((item) => item.type === "dir");
// Fetch metadata for each template
// Fetch and parse each template.yml
const templates = await Promise.all(
templateDirs.map(async (dir) => {
templateFiles.map(async (file) => {
try {
// Try to fetch metadata.json for each template
const { data: metadataFile } = await octokit.repos.getContent({
const { data: content } = await octokit.git.getBlob({
owner,
repo,
path: `templates/${dir.name}/metadata.json`,
ref: branch,
file_sha: file.sha,
});
if ("content" in metadataFile && metadataFile.encoding === "base64") {
const content = Buffer.from(
metadataFile.content,
"base64",
).toString();
return JSON.parse(content) as TemplateMetadata;
}
const decoded = Buffer.from(content.content, "base64").toString();
return load(decoded) as CompleteTemplate;
} catch (error) {
// If metadata.json doesn't exist, create a basic metadata object
return {
id: dir.name,
name: dir.name.charAt(0).toUpperCase() + dir.name.slice(1),
version: "latest",
description: `${dir.name} template`,
logo: "default.svg",
links: {},
tags: [],
};
console.warn(`Failed to load template from ${file.path}:`, error);
return null;
}
return null;
}),
);
return templates.filter(Boolean) as TemplateMetadata[];
return templates.filter(Boolean) as CompleteTemplate[];
} catch (error) {
console.error("Error fetching templates list:", error);
throw error;
@@ -114,32 +263,73 @@ export async function fetchTemplateFiles(
owner = templateConfig.owner,
repo = templateConfig.repo,
branch = templateConfig.branch,
): Promise<{ indexTs: string; dockerCompose: string }> {
): Promise<{ config: CompleteTemplate; dockerCompose: string }> {
try {
// Fetch index.ts
const { data: indexFile } = await octokit.repos.getContent({
// Get the tree SHA for the branch
const { data: ref } = await octokit.git.getRef({
owner,
repo,
path: `templates/${templateId}/index.ts`,
ref: branch,
ref: `heads/${branch}`,
});
// Fetch docker-compose.yml
const { data: composeFile } = await octokit.repos.getContent({
// Get the full tree recursively
const { data: tree } = await octokit.git.getTree({
owner,
repo,
path: `templates/${templateId}/docker-compose.yml`,
ref: branch,
tree_sha: ref.object.sha,
recursive: "true",
});
if (!("content" in indexFile) || !("content" in composeFile)) {
// Find the template.yml and docker-compose.yml files
const templateYml = tree.tree
.filter((item): item is GitTreeItem => {
return (
item.type === "blob" &&
typeof item.path === "string" &&
typeof item.sha === "string"
);
})
.find((item) => item.path === `templates/${templateId}/template.yml`);
const dockerComposeYml = tree.tree
.filter((item): item is GitTreeItem => {
return (
item.type === "blob" &&
typeof item.path === "string" &&
typeof item.sha === "string"
);
})
.find(
(item) => item.path === `templates/${templateId}/docker-compose.yml`,
);
if (!templateYml || !dockerComposeYml) {
throw new Error("Template files not found");
}
const indexTs = Buffer.from(indexFile.content, "base64").toString();
const dockerCompose = Buffer.from(composeFile.content, "base64").toString();
// Fetch both files in parallel
const [templateContent, composeContent] = await Promise.all([
octokit.git.getBlob({
owner,
repo,
file_sha: templateYml.sha,
}),
octokit.git.getBlob({
owner,
repo,
file_sha: dockerComposeYml.sha,
}),
]);
return { indexTs, dockerCompose };
const config = load(
Buffer.from(templateContent.data.content, "base64").toString(),
) as CompleteTemplate;
const dockerCompose = Buffer.from(
composeContent.data.content,
"base64",
).toString();
return { config, dockerCompose };
} catch (error) {
console.error(`Error fetching template ${templateId}:`, error);
throw error;
@@ -147,179 +337,11 @@ export async function fetchTemplateFiles(
}
/**
* Executes the template's index.ts code dynamically
* Uses a template-based approach that's safer and more efficient
* Loads and processes a template
*/
export async function executeTemplateCode(
indexTsCode: string,
schema: { serverIp: string; projectName: string },
): Promise<Template> {
try {
// Create a temporary directory for the template
const cwd = process.cwd();
const tempId = randomBytes(8).toString("hex");
const tempDir = join(cwd, ".next", "temp", tempId);
if (!existsSync(tempDir)) {
await mkdir(tempDir, { recursive: true });
}
// Extract the generate function body
// This approach assumes templates follow a standard structure with a generate function
const generateFunctionMatch = indexTsCode.match(
/export\s+function\s+generate\s*\([^)]*\)\s*{([\s\S]*?)return\s+{([\s\S]*?)};?\s*}/,
);
if (!generateFunctionMatch) {
throw new Error("Could not extract generate function from template");
}
const functionBody = generateFunctionMatch[1];
const returnStatement = generateFunctionMatch[2];
// Create a simplified template that doesn't require imports
const templateCode = `
// Utility functions provided to the template
function generateRandomDomain(schema) {
const hash = Math.random().toString(36).substring(2, 8);
const slugIp = schema.serverIp.replaceAll(".", "-");
return \`\${schema.projectName}-\${hash}\${slugIp === "" ? "" : \`-\${slugIp}\`}.traefik.me\`;
}
function generateHash(projectName, quantity = 3) {
const hash = Math.random().toString(36).substring(2, 2 + quantity);
return \`\${projectName}-\${hash}\`;
}
function generatePassword(quantity = 16) {
const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let password = "";
for (let i = 0; i < quantity; i++) {
password += characters.charAt(Math.floor(Math.random() * characters.length));
}
return password.toLowerCase();
}
function generateBase64(bytes = 32) {
return Math.random().toString(36).substring(2, 2 + bytes);
}
// Template execution
function execute(schema) {
${functionBody}
return {
${returnStatement}
};
}
// Run with the provided schema and output the result
const result = execute(${JSON.stringify(schema)});
console.log(JSON.stringify(result));
`;
// Write the template code to a file
const templatePath = join(tempDir, "template.js");
await writeFile(templatePath, templateCode, "utf8");
// Execute the template using Node.js
const output = execSync(`node ${templatePath}`, {
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
// Parse the output as JSON
return JSON.parse(output);
} catch (error) {
console.error("Error executing template code:", error);
// Fallback to a simpler approach if the template extraction fails
return fallbackExecuteTemplate(indexTsCode, schema);
}
}
/**
* Fallback method to execute templates that don't follow the standard structure
*/
async function fallbackExecuteTemplate(
indexTsCode: string,
schema: { serverIp: string; projectName: string },
): Promise<Template> {
try {
// Create a temporary directory
const cwd = process.cwd();
const tempId = randomBytes(8).toString("hex");
const tempDir = join(cwd, ".next", "temp", tempId);
if (!existsSync(tempDir)) {
await mkdir(tempDir, { recursive: true });
}
// Create a simplified version of the template code
// Remove TypeScript types and imports
const simplifiedCode = indexTsCode
.replace(/import\s+.*?from\s+['"].*?['"]\s*;?/g, "")
.replace(/export\s+interface\s+.*?{[\s\S]*?}/g, "")
.replace(/:\s*Schema/g, "")
.replace(/:\s*DomainSchema/g, "")
.replace(/:\s*Template/g, "")
.replace(/:\s*string/g, "")
.replace(/:\s*number/g, "")
.replace(/:\s*boolean/g, "")
.replace(/:\s*any/g, "")
.replace(/:\s*unknown/g, "")
.replace(/<.*?>/g, "");
// Create a wrapper with all necessary utilities
const wrapperCode = `
// Utility functions
function generateRandomDomain(schema) {
const hash = Math.random().toString(36).substring(2, 8);
const slugIp = schema.serverIp.replaceAll(".", "-");
return \`\${schema.projectName}-\${hash}\${slugIp === "" ? "" : \`-\${slugIp}\`}.traefik.me\`;
}
function generateHash(projectName, quantity = 3) {
const hash = Math.random().toString(36).substring(2, 2 + quantity);
return \`\${projectName}-\${hash}\`;
}
function generatePassword(quantity = 16) {
const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let password = "";
for (let i = 0; i < quantity; i++) {
password += characters.charAt(Math.floor(Math.random() * characters.length));
}
return password.toLowerCase();
}
function generateBase64(bytes = 32) {
return Math.random().toString(36).substring(2, 2 + bytes);
}
// Simplified template code
${simplifiedCode}
// Execute the template
const result = generate(${JSON.stringify(schema)});
console.log(JSON.stringify(result));
`;
// Write the wrapper code to a file
const wrapperPath = join(tempDir, "wrapper.js");
await writeFile(wrapperPath, wrapperCode, "utf8");
// Execute the code using Node.js
const output = execSync(`node ${wrapperPath}`, {
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
// Parse the output as JSON
return JSON.parse(output);
} catch (error) {
console.error("Error in fallback template execution:", error);
throw new Error(
`Failed to execute template: ${error instanceof Error ? error.message : String(error)}`,
);
}
export async function loadTemplateModule(
id: string,
): Promise<(schema: Schema) => Promise<Template>> {
const { config } = await fetchTemplateFiles(id);
return async (schema: Schema) => processTemplate(config, schema);
}

View File

@@ -5,22 +5,24 @@ import { join } from "node:path";
import type { Domain } from "@dokploy/server/services/domain";
import { TRPCError } from "@trpc/server";
import { templateConfig } from "../config";
import { executeTemplateCode, fetchTemplateFiles } from "./github";
import { fetchTemplateFiles } from "./github";
export interface Schema {
serverIp: string;
projectName: string;
}
export type DomainSchema = Pick<Domain, "host" | "port" | "serviceName">;
export type DomainSchema = Pick<Domain, "host" | "port" | "serviceName"> & {
path?: string;
};
export interface Template {
envs?: string[];
mounts?: {
envs: string[];
mounts: Array<{
filePath: string;
content?: string;
}[];
domains?: DomainSchema[];
content: string;
}>;
domains: DomainSchema[];
}
export const generateRandomDomain = ({
@@ -33,9 +35,10 @@ export const generateRandomDomain = ({
return `${projectName}-${hash}${slugIp === "" ? "" : `-${slugIp}`}.traefik.me`;
};
export const generateHash = (projectName: string, quantity = 3): string => {
const hash = randomBytes(quantity).toString("hex");
return `${projectName}-${hash}`;
export const generateHash = (length = 8): string => {
return randomBytes(Math.ceil(length / 2))
.toString("hex")
.substring(0, length);
};
export const generatePassword = (quantity = 16): string => {
@@ -74,22 +77,18 @@ async function isCacheValid(filePath: string): Promise<boolean> {
/**
* Reads a template's docker-compose.yml file
* First tries to read from the local cache, if not found or expired, fetches from GitHub
* First tries to fetch from GitHub, falls back to local cache if fetch fails
*/
export const readTemplateComposeFile = async (id: string) => {
const cwd = process.cwd();
const templatePath = join(cwd, ".next", "templates", id);
const composeFilePath = join(templatePath, "docker-compose.yml");
// Check if the file exists in the local cache and is still valid
if (await isCacheValid(composeFilePath)) {
return await readFile(composeFilePath, "utf8");
}
// If not in cache or expired, fetch from GitHub and cache it
// First try to fetch from GitHub
try {
const { dockerCompose } = await fetchTemplateFiles(id);
// Cache the file for future use
const cwd = process.cwd();
const templatePath = join(cwd, ".next", "templates", id);
const composeFilePath = join(templatePath, "docker-compose.yml");
// Ensure the template directory exists
if (!existsSync(templatePath)) {
await mkdir(templatePath, { recursive: true });
@@ -100,16 +99,24 @@ export const readTemplateComposeFile = async (id: string) => {
return dockerCompose;
} catch (error) {
// If fetch fails but we have a cached version, use it as fallback
console.warn(`Failed to fetch template ${id} from GitHub:`, error);
// Try to use cached version as fallback
const cwd = process.cwd();
const composeFilePath = join(
cwd,
".next",
"templates",
id,
"docker-compose.yml",
);
if (existsSync(composeFilePath)) {
console.warn(
`Using cached version of template ${id} due to fetch error:`,
error,
);
console.warn(`Using cached version of template ${id}`);
return await readFile(composeFilePath, "utf8");
}
console.error(`Error fetching template ${id}:`, error);
console.error(`Error: Template ${id} not found in GitHub or cache`);
throw new TRPCError({
code: "NOT_FOUND",
message: `Template ${id} not found or could not be fetched`,
@@ -118,49 +125,6 @@ export const readTemplateComposeFile = async (id: string) => {
};
/**
* Loads a template module and returns its generate function
* First tries to execute from local cache, if not found or expired, fetches from GitHub
* Loads a template module from GitHub or local cache
* First tries to fetch from GitHub, falls back to local cache if fetch fails
*/
export const loadTemplateModule = async (id: string) => {
const cwd = process.cwd();
const templatePath = join(cwd, ".next", "templates", id);
const indexFilePath = join(templatePath, "index.ts");
// Check if we have the template cached locally and it's still valid
if (await isCacheValid(indexFilePath)) {
const indexTs = await readFile(indexFilePath, "utf8");
return (schema: Schema) => executeTemplateCode(indexTs, schema);
}
// If not in cache or expired, fetch from GitHub and cache it
try {
const { indexTs } = await fetchTemplateFiles(id);
// Ensure the template directory exists
if (!existsSync(templatePath)) {
await mkdir(templatePath, { recursive: true });
}
// Cache the file for future use
await writeFile(indexFilePath, indexTs, "utf8");
// Return a function that will execute the template code
return (schema: Schema) => executeTemplateCode(indexTs, schema);
} catch (error) {
// If fetch fails but we have a cached version, use it as fallback
if (existsSync(indexFilePath)) {
console.warn(
`Using cached version of template ${id} due to fetch error:`,
error,
);
const indexTs = await readFile(indexFilePath, "utf8");
return (schema: Schema) => executeTemplateCode(indexTs, schema);
}
console.error(`Error loading template module ${id}:`, error);
throw new TRPCError({
code: "NOT_FOUND",
message: `Template ${id} not found or could not be loaded`,
});
}
};

View File

@@ -0,0 +1,44 @@
metadata:
id: plausible
name: Plausible Analytics
description: Privacy-focused Google Analytics alternative
tags:
- analytics
- privacy
- statistics
variables:
db_password: ${password:32}
admin_password: ${password:16}
secret_key: ${base64:64}
main_domain: ${randomDomain}
config:
domains:
- serviceName: plausible
port: 8000
path: /
host: plausible-${main_domain}
- serviceName: admin
port: 8001
path: /admin
host: admin-${main_domain}
env:
ADMIN_USER_EMAIL: admin@example.com
ADMIN_USER_NAME: admin
ADMIN_USER_PWD: ${admin_password}
SECRET_KEY_BASE: ${secret_key}
DB: plausible
DB_USER: plausible
DB_PASSWORD: ${db_password}
CLICKHOUSE_USER: default
CLICKHOUSE_PASSWORD: ${password:32}
DOMAIN: ${domain:plausible}
ADMIN_DOMAIN: ${domain:admin}
mounts:
- filePath: nginx.conf
content: |
server_name ${domain:plausible};
# resto de la configuración...

View File

@@ -0,0 +1,42 @@
{
"metadata": {
"id": "plausible",
"name": "Plausible Analytics",
"version": "latest",
"description": "Privacy-focused Google Analytics alternative",
"logo": "plausible.svg",
"links": {
"website": "https://plausible.io",
"docs": "https://plausible.io/docs",
"github": "https://github.com/plausible/analytics"
},
"tags": ["analytics", "privacy", "statistics"]
},
"config": {
"domains": {
"plausible": {
"port": 8000,
"path": "/",
"serviceName": "plausible",
"host": "plausible.com"
}
},
"env": {
"ADMIN_USER_EMAIL": "admin@example.com",
"ADMIN_USER_NAME": "admin",
"ADMIN_USER_PWD": "plausible",
"SECRET_KEY_BASE": "plausible",
"DB": "plausible",
"DB_USER": "plausible",
"DB_PASSWORD": "plausible",
"CLICKHOUSE_USER": "default",
"CLICKHOUSE_PASSWORD": "default"
},
"mounts": [
{
"filePath": "custom.css",
"content": "/* Add your custom CSS here */"
}
]
}
}