Dokploy/dokploy
1
0
Fork 0
mirror of https://github.com/Dokploy/dokploy synced 2025-06-26 18:27:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(templates): refactor template processing and GitHub template fetching

Browse Source 
- Implement new template processing utility in `processors.ts`
- Simplify GitHub template fetching with a more lightweight approach
- Add comprehensive test suite for template processing
- Improve type safety and modularity of template-related functions
This commit is contained in:
Mauricio Siu
2025-03-09 13:50:34 -06:00
parent 466fdf20b8
commit 6e7e7b3f9a
6 changed files with 643 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/server/src/index.ts
View File

@@ -47,6 +47,7 @@ export * from "./utils/backups/mongo";
export * from "./utils/backups/mysql";
export * from "./utils/backups/postgres";
export * from "./utils/backups/utils";
export * from "./templates/utils/processors";
export * from "./utils/notifications/build-error";
export * from "./utils/notifications/build-success";

325
packages/server/src/templates/utils/github.ts
View File

@@ -1,22 +1,4 @@
import { randomBytes } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, writeFile, rm } from "node:fs/promises";
import { join } from "node:path";
import { Octokit } from "@octokit/rest";
import { load } from "js-yaml";
import { templateConfig } from "../config";
import type { Schema, Template, DomainSchema } from "./index";
import {
generateBase64,
generateHash,
generatePassword,
generateRandomDomain,
} from "./index";
// GitHub API client
const octokit = new Octokit({
auth: templateConfig.token,
});
/**
* Complete template interface that includes both metadata and configuration
@@ -53,202 +35,50 @@ export interface CompleteTemplate {
};
}
/**
* Utility functions that can be used in template values
*/
const TEMPLATE_FUNCTIONS = {
$randomDomain: () => true,
$password: (length = 16) => `$password(${length})`,
$base64: (bytes = 32) => `$base64(${bytes})`,
$base32: (bytes = 32) => `$base32(${bytes})`,
$hash: (length = 8) => `$hash(${length})`,
} as const;
/**
* Process a string value and replace variables
*/
function processValue(
value: string,
variables: Record<string, string>,
schema: Schema,
): string {
// First replace utility functions
let processedValue = value.replace(/\${([^}]+)}/g, (match, varName) => {
// Handle utility functions
if (varName === "randomDomain") {
return generateRandomDomain(schema);
}
if (varName.startsWith("base64:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 32;
return generateBase64(length);
}
if (varName.startsWith("base32:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 32;
return Buffer.from(randomBytes(length))
.toString("base64")
.substring(0, length);
}
if (varName.startsWith("password:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 16;
return generatePassword(length);
}
if (varName.startsWith("hash:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 8;
return generateHash(length);
}
// If not a utility function, try to get from variables
return variables[varName] || match;
});
// Then replace any remaining ${var} with their values from variables
processedValue = processedValue.replace(/\${([^}]+)}/g, (match, varName) => {
return variables[varName] || match;
});
return processedValue;
}
/**
* Processes a template configuration and returns the generated template
*/
export function processTemplate(
config: CompleteTemplate,
schema: Schema,
): Template {
const result: Template = {
envs: [],
domains: [],
mounts: [],
interface TemplateMetadata {
id: string;
name: string;
description: string;
version: string;
logo: string;
links: {
github: string;
website?: string;
docs?: string;
};
// First pass: Process variables that don't depend on domains
const variables: Record<string, string> = {};
for (const [key, value] of Object.entries(config.variables)) {
if (value === "${randomDomain}") {
variables[key] = generateRandomDomain(schema);
} else if (value.startsWith("${base64:")) {
const match = value.match(/\${base64:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 32;
variables[key] = generateBase64(length);
} else if (value.startsWith("${base32:")) {
const match = value.match(/\${base32:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 32;
variables[key] = Buffer.from(randomBytes(length))
.toString("base64")
.substring(0, length);
} else if (value.startsWith("${password:")) {
const match = value.match(/\${password:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 16;
variables[key] = generatePassword(length);
} else if (value.startsWith("${hash:")) {
const match = value.match(/\${hash:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 8;
variables[key] = generateHash(length);
} else {
variables[key] = value;
}
}
console.log(variables);
// Process domains and add them to variables
for (const domain of config.config.domains) {
// If host is specified, process it with variables, otherwise generate random domain
const host = domain.host
? processValue(domain.host, variables, schema)
: generateRandomDomain(schema);
result.domains.push({
host,
...domain,
});
// Add domain to variables for reference
variables[`domain:${domain.serviceName}`] = host;
}
// Process environment variables with access to all variables
for (const [key, value] of Object.entries(config.config.env)) {
const processedValue = processValue(value, variables, schema);
result.envs.push(`${key}=${processedValue}`);
}
// Process mounts with access to all variables
if (config.config.mounts) {
for (const mount of config.config.mounts) {
result.mounts.push({
filePath: mount.filePath,
content: processValue(mount.content, variables, schema),
});
}
}
return result;
tags: string[];
}
/**
* GitHub tree item with required fields
*/
interface GitTreeItem {
path: string;
type: string;
sha: string;
}
/**
* Fetches the list of available templates from GitHub
* Fetches the list of available templates from meta.json
*/
export async function fetchTemplatesList(
owner = templateConfig.owner,
repo = templateConfig.repo,
branch = templateConfig.branch,
baseUrl = "https://dokploy.github.io/templates",
): Promise<CompleteTemplate[]> {
try {
// First get the tree SHA for the branch
const { data: ref } = await octokit.git.getRef({
owner,
repo,
ref: `heads/${branch}`,
});
// Get the full tree recursively
const { data: tree } = await octokit.git.getTree({
owner,
repo,
tree_sha: ref.object.sha,
recursive: "true",
});
// Filter for template.yml files in the templates directory
const templateFiles = tree.tree.filter((item): item is GitTreeItem => {
return (
item.type === "blob" &&
typeof item.path === "string" &&
typeof item.sha === "string" &&
item.path.startsWith("templates/") &&
item.path.endsWith("/template.yml")
);
});
// Fetch and parse each template.yml
const templates = await Promise.all(
templateFiles.map(async (file) => {
try {
const { data: content } = await octokit.git.getBlob({
owner,
repo,
file_sha: file.sha,
});
const decoded = Buffer.from(content.content, "base64").toString();
return load(decoded) as CompleteTemplate;
} catch (error) {
console.warn(`Failed to load template from ${file.path}:`, error);
return null;
}
}),
);
return templates.filter(Boolean) as CompleteTemplate[];
const response = await fetch(`${baseUrl}/meta.json`);
if (!response.ok) {
throw new Error(`Failed to fetch templates: ${response.statusText}`);
}
const templates = (await response.json()) as TemplateMetadata[];
return templates.map((template) => ({
metadata: {
id: template.id,
name: template.name,
description: template.description,
version: template.version,
logo: template.logo,
links: template.links,
tags: template.tags,
},
// These will be populated when fetching individual templates
variables: {},
config: {
domains: [],
env: {},
mounts: [],
},
}));
} catch (error) {
console.error("Error fetching templates list:", error);
throw error;
@@ -256,78 +86,29 @@ export async function fetchTemplatesList(
}
/**
* Fetches a specific template's files from GitHub
* Fetches a specific template's files
*/
export async function fetchTemplateFiles(
templateId: string,
owner = templateConfig.owner,
repo = templateConfig.repo,
branch = templateConfig.branch,
baseUrl = "https://dokploy.github.io/templates",
): Promise<{ config: CompleteTemplate; dockerCompose: string }> {
try {
// Get the tree SHA for the branch
const { data: ref } = await octokit.git.getRef({
owner,
repo,
ref: `heads/${branch}`,
});
// Fetch both files in parallel
const [templateYmlResponse, dockerComposeResponse] = await Promise.all([
fetch(`${baseUrl}/templates/${templateId}/template.yml`),
fetch(`${baseUrl}/templates/${templateId}/docker-compose.yml`),
]);
// Get the full tree recursively
const { data: tree } = await octokit.git.getTree({
owner,
repo,
tree_sha: ref.object.sha,
recursive: "true",
});
// Find the template.yml and docker-compose.yml files
const templateYml = tree.tree
.filter((item): item is GitTreeItem => {
return (
item.type === "blob" &&
typeof item.path === "string" &&
typeof item.sha === "string"
);
})
.find((item) => item.path === `templates/${templateId}/template.yml`);
const dockerComposeYml = tree.tree
.filter((item): item is GitTreeItem => {
return (
item.type === "blob" &&
typeof item.path === "string" &&
typeof item.sha === "string"
);
})
.find(
(item) => item.path === `templates/${templateId}/docker-compose.yml`,
);
if (!templateYml || !dockerComposeYml) {
if (!templateYmlResponse.ok || !dockerComposeResponse.ok) {
throw new Error("Template files not found");
}
// Fetch both files in parallel
const [templateContent, composeContent] = await Promise.all([
octokit.git.getBlob({
owner,
repo,
file_sha: templateYml.sha,
}),
octokit.git.getBlob({
owner,
repo,
file_sha: dockerComposeYml.sha,
}),
const [templateYml, dockerCompose] = await Promise.all([
templateYmlResponse.text(),
dockerComposeResponse.text(),
]);
const config = load(
Buffer.from(templateContent.data.content, "base64").toString(),
) as CompleteTemplate;
const dockerCompose = Buffer.from(
composeContent.data.content,
"base64",
).toString();
const config = load(templateYml) as CompleteTemplate;
return { config, dockerCompose };
} catch (error) {
@@ -335,13 +116,3 @@ export async function fetchTemplateFiles(
throw error;
}
}
/**
* Loads and processes a template
*/
export async function loadTemplateModule(
id: string,
): Promise<(schema: Schema) => Promise<Template>> {
const { config } = await fetchTemplateFiles(id);
return async (schema: Schema) => processTemplate(config, schema);
}

28
packages/server/src/templates/utils/index.ts
View File

@@ -1,10 +1,9 @@
import { randomBytes } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { Domain } from "@dokploy/server/services/domain";
import { TRPCError } from "@trpc/server";
import { templateConfig } from "../config";
import { fetchTemplateFiles } from "./github";
export interface Schema {
@@ -53,26 +52,15 @@ export const generatePassword = (quantity = 16): string => {
return password.toLowerCase();
};
export const generateBase64 = (bytes = 32): string => {
return randomBytes(bytes).toString("base64");
};
/**
* Checks if a cached file is still valid based on its modification time
* Generate a random base64 string of specified length
*/
async function isCacheValid(filePath: string): Promise<boolean> {
try {
if (!existsSync(filePath)) return false;
const fileStats = await stat(filePath);
const modifiedTime = fileStats.mtime.getTime();
const currentTime = Date.now();
// Check if the file is older than the cache duration
return currentTime - modifiedTime < templateConfig.cacheDuration;
} catch (error) {
return false;
}
export function generateBase64(length: number): string {
// To get N characters in base64, we need to generate N * 3/4 bytes
const bytesNeeded = Math.ceil((length * 3) / 4);
return Buffer.from(randomBytes(bytesNeeded))
.toString("base64")
.substring(0, length);
}
/**

202
packages/server/src/templates/utils/processors.ts Normal file
View File

@@ -0,0 +1,202 @@
import type { Schema } from "./index";
import {
generateBase64,
generateHash,
generatePassword,
generateRandomDomain,
} from "./index";
/**
* Domain configuration
*/
interface DomainConfig {
serviceName: string;
port: number;
path?: string;
host?: string;
}
/**
* Mount configuration
*/
interface MountConfig {
filePath: string;
content: string;
}
/**
* Complete template interface that includes both metadata and configuration
*/
export interface CompleteTemplate {
metadata: {
id: string;
name: string;
description: string;
tags: string[];
version: string;
logo: string;
links: {
github: string;
website?: string;
docs?: string;
};
};
variables: Record<string, string>;
config: {
domains: DomainConfig[];
env: Record<string, string>;
mounts?: MountConfig[];
};
}
/**
* Processed template output
*/
export interface Template {
domains: Array<DomainConfig>;
envs: string[];
mounts: MountConfig[];
}
/**
* Process a string value and replace variables
*/
function processValue(
value: string,
variables: Record<string, string>,
schema: Schema,
): string {
// First replace utility functions
let processedValue = value.replace(/\${([^}]+)}/g, (match, varName) => {
// Handle utility functions
if (varName === "randomDomain") {
return generateRandomDomain(schema);
}
if (varName.startsWith("base64:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 32;
return generateBase64(length);
}
if (varName.startsWith("password:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 16;
return generatePassword(length);
}
if (varName.startsWith("hash:")) {
const length = Number.parseInt(varName.split(":")[1], 10) || 8;
return generateHash(length);
}
// If not a utility function, try to get from variables
return variables[varName] || match;
});
// Then replace any remaining ${var} with their values from variables
processedValue = processedValue.replace(/\${([^}]+)}/g, (match, varName) => {
return variables[varName] || match;
});
return processedValue;
}
/**
* Process variables in a template
*/
export function processVariables(
template: CompleteTemplate,
schema: Schema,
): Record<string, string> {
const variables: Record<string, string> = {};
// First pass: Process variables that don't depend on other variables
for (const [key, value] of Object.entries(template.variables)) {
if (typeof value !== "string") continue;
if (value === "${randomDomain}") {
variables[key] = generateRandomDomain(schema);
} else if (value.startsWith("${base64:")) {
const match = value.match(/\${base64:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 32;
variables[key] = generateBase64(length);
} else if (value.startsWith("${password:")) {
const match = value.match(/\${password:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 16;
variables[key] = generatePassword(length);
} else if (value.startsWith("${hash:")) {
const match = value.match(/\${hash:(\d+)}/);
const length = match?.[1] ? Number.parseInt(match[1], 10) : 8;
variables[key] = generateHash(length);
} else {
variables[key] = value;
}
}
// Second pass: Process variables that reference other variables
for (const [key, value] of Object.entries(variables)) {
variables[key] = processValue(value, variables, schema);
}
return variables;
}
/**
* Process domains in a template
*/
export function processDomains(
template: CompleteTemplate,
variables: Record<string, string>,
schema: Schema,
): Template["domains"] {
return template.config.domains.map((domain: DomainConfig) => ({
...domain,
host: domain.host
? processValue(domain.host, variables, schema)
: generateRandomDomain(schema),
}));
}
/**
* Process environment variables in a template
*/
export function processEnvVars(
template: CompleteTemplate,
variables: Record<string, string>,
schema: Schema,
): Template["envs"] {
return Object.entries(template.config.env).map(
([key, value]: [string, string]) => {
const processedValue = processValue(value, variables, schema);
return `${key}=${processedValue}`;
},
);
}
/**
* Process mounts in a template
*/
export function processMounts(
template: CompleteTemplate,
variables: Record<string, string>,
schema: Schema,
): Template["mounts"] {
if (!template.config.mounts) return [];
return template.config.mounts.map((mount: MountConfig) => ({
filePath: processValue(mount.filePath, variables, schema),
content: processValue(mount.content, variables, schema),
}));
}
/**
* Process a complete template
*/
export function processTemplate(
template: CompleteTemplate,
schema: Schema,
): Template {
// First process variables as they might be referenced by other sections
const variables = processVariables(template, schema);
return {
domains: processDomains(template, variables, schema),
envs: processEnvVars(template, variables, schema),
mounts: processMounts(template, variables, schema),
};
}