Dokploy/dokploy
1
0
Fork 0
mirror of https://github.com/Dokploy/dokploy synced 2025-06-26 18:27:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'Dokploy:canary' into add-disable-recurse-submodules-option

Browse Source
This commit is contained in:
Yusoof Moh
2025-03-30 21:11:25 +07:00
committed by GitHub
parent 67e85cabcb ffc85b04a8
commit 96e9799afb
36 changed files with 11964 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
packages/server/package.json
View File

@@ -28,7 +28,7 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"micromatch":"4.0.8",
"micromatch": "4.0.8",
"@ai-sdk/anthropic": "^1.0.6",
"@ai-sdk/azure": "^1.0.15",
"@ai-sdk/cohere": "^1.0.6",
@@ -36,11 +36,11 @@
"@ai-sdk/mistral": "^1.0.6",
"@ai-sdk/openai": "^1.0.12",
"@ai-sdk/openai-compatible": "^0.0.13",
"@better-auth/utils":"0.2.3",
"@oslojs/encoding":"1.1.0",
"@oslojs/crypto":"1.0.1",
"drizzle-dbml-generator":"0.10.0",
"better-auth":"1.2.4",
"@better-auth/utils": "0.2.3",
"@oslojs/encoding": "1.1.0",
"@oslojs/crypto": "1.0.1",
"drizzle-dbml-generator": "0.10.0",
"better-auth": "1.2.4",
"@faker-js/faker": "^8.4.1",
"@octokit/auth-app": "^6.0.4",
"@react-email/components": "^0.0.21",
@@ -76,7 +76,8 @@
"ws": "8.16.0",
"zod": "^3.23.4",
"ssh2": "1.15.0",
"@octokit/rest": "^20.0.2"
"@octokit/rest": "^20.0.2",
"toml": "3.0.0"
},
"devDependencies": {
"@types/micromatch": "4.0.9",

12
packages/server/src/db/schema/backups.ts
View File

@@ -15,12 +15,13 @@ import { mariadb } from "./mariadb";
import { mongo } from "./mongo";
import { mysql } from "./mysql";
import { postgres } from "./postgres";
import { users_temp } from "./user";
export const databaseType = pgEnum("databaseType", [
"postgres",
"mariadb",
"mysql",
"mongo",
"web-server",
]);
export const backups = pgTable("backup", {
@@ -58,6 +59,7 @@ export const backups = pgTable("backup", {
mongoId: text("mongoId").references((): AnyPgColumn => mongo.mongoId, {
onDelete: "cascade",
}),
userId: text("userId").references(() => users_temp.id),
});
export const backupsRelations = relations(backups, ({ one }) => ({
@@ -81,6 +83,10 @@ export const backupsRelations = relations(backups, ({ one }) => ({
fields: [backups.mongoId],
references: [mongo.mongoId],
}),
user: one(users_temp, {
fields: [backups.userId],
references: [users_temp.id],
}),
}));
const createSchema = createInsertSchema(backups, {
@@ -91,11 +97,12 @@ const createSchema = createInsertSchema(backups, {
database: z.string().min(1),
schedule: z.string(),
keepLatestCount: z.number().optional(),
databaseType: z.enum(["postgres", "mariadb", "mysql", "mongo"]),
databaseType: z.enum(["postgres", "mariadb", "mysql", "mongo", "web-server"]),
postgresId: z.string().optional(),
mariadbId: z.string().optional(),
mysqlId: z.string().optional(),
mongoId: z.string().optional(),
userId: z.string().optional(),
});
export const apiCreateBackup = createSchema.pick({
@@ -110,6 +117,7 @@ export const apiCreateBackup = createSchema.pick({
postgresId: true,
mongoId: true,
databaseType: true,
userId: true,
});
export const apiFindOneBackup = createSchema

3
packages/server/src/db/schema/compose.ts
View File

@@ -139,7 +139,7 @@ const createSchema = createInsertSchema(compose, {
name: z.string().min(1),
description: z.string(),
env: z.string().optional(),
composeFile: z.string().min(1),
composeFile: z.string().optional(),
projectId: z.string(),
customGitSSHKeyId: z.string().optional(),
command: z.string().optional(),
@@ -155,6 +155,7 @@ export const apiCreateCompose = createSchema.pick({
composeType: true,
appName: true,
serverId: true,
composeFile: true,
});
export const apiCreateComposeByTemplate = createSchema

1
packages/server/src/db/schema/project.ts
View File

@@ -52,6 +52,7 @@ const createSchema = createInsertSchema(projects, {
export const apiCreateProject = createSchema.pick({
name: true,
description: true,
env: true,
});
export const apiFindOneProject = createSchema

2
packages/server/src/db/schema/user.ts
View File

@@ -13,6 +13,7 @@ import { z } from "zod";
import { account, apikey, organization } from "./account";
import { projects } from "./project";
import { certificateType } from "./shared";
import { backups } from "./backups";
/**
* This is an example of how to use the multi-project schema feature of Drizzle ORM. Use the same
* database instance for multiple projects.
@@ -124,6 +125,7 @@ export const usersRelations = relations(users_temp, ({ one, many }) => ({
organizations: many(organization),
projects: many(projects),
apiKeys: many(apikey),
backups: many(backups),
}));
const createSchema = createInsertSchema(users_temp, {

1
packages/server/src/index.ts
View File

@@ -48,6 +48,7 @@ export * from "./utils/backups/mongo";
export * from "./utils/backups/mysql";
export * from "./utils/backups/postgres";
export * from "./utils/backups/utils";
export * from "./utils/backups/web-server";
export * from "./templates/processors";
export * from "./utils/notifications/build-error";

2
packages/server/src/services/compose.ts
View File

@@ -69,7 +69,7 @@ export const createCompose = async (input: typeof apiCreateCompose._type) => {
.insert(compose)
.values({
...input,
composeFile: "",
composeFile: input.composeFile || "",
appName,
})
.returning()

6
packages/server/src/templates/github.ts
View File

@@ -1,4 +1,4 @@
import { load } from "js-yaml";
import { parse } from "toml";
/**
* Complete template interface that includes both metadata and configuration
@@ -86,7 +86,7 @@ export async function fetchTemplateFiles(
try {
// Fetch both files in parallel
const [templateYmlResponse, dockerComposeResponse] = await Promise.all([
fetch(`${baseUrl}/blueprints/${templateId}/template.yml`),
fetch(`${baseUrl}/blueprints/${templateId}/template.toml`),
fetch(`${baseUrl}/blueprints/${templateId}/docker-compose.yml`),
]);
@@ -99,7 +99,7 @@ export async function fetchTemplateFiles(
dockerComposeResponse.text(),
]);
const config = load(templateYml) as CompleteTemplate;
const config = parse(templateYml) as CompleteTemplate;
return { config, dockerCompose };
} catch (error) {

24
packages/server/src/templates/processors.ts
View File

@@ -45,7 +45,9 @@ export interface CompleteTemplate {
variables: Record<string, string>;
config: {
domains: DomainConfig[];
env: Record<string, string> | string[];
env:
| Record<string, string | boolean | number>
| (string | Record<string, string | boolean | number>)[];
mounts?: MountConfig[];
};
}
@@ -200,17 +202,27 @@ export function processEnvVars(
if (typeof env === "string") {
return processValue(env, variables, schema);
}
return env;
// Si es un objeto, asumimos que es un par clave-valor
if (typeof env === "object" && env !== null) {
const keys = Object.keys(env);
if (keys.length > 0) {
const key = keys[0];
return `${key}=${env[key as keyof typeof env]}`;
}
}
// Para valores primitivos (boolean, number)
return String(env);
});
}
// Handle object of env vars
return Object.entries(template.config.env).map(
([key, value]: [string, string]) => {
return Object.entries(template.config.env).map(([key, value]) => {
if (typeof value === "string") {
const processedValue = processValue(value, variables, schema);
return `${key}=${processedValue}`;
},
);
}
return `${key}=${value}`;
});
}
/**

137
packages/server/src/utils/backups/index.ts
View File

@@ -10,11 +10,7 @@ import {
} from "../docker/utils";
import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { runMariadbBackup } from "./mariadb";
import { runMongoBackup } from "./mongo";
import { runMySqlBackup } from "./mysql";
import { runPostgresBackup } from "./postgres";
import { getS3Credentials } from "./utils";
import { getS3Credentials, scheduleBackup } from "./utils";
import type { BackupSchedule } from "@dokploy/server/services/backup";
import { startLogCleanup } from "../access-log/handler";
@@ -56,126 +52,27 @@ export const initCronJobs = async () => {
}
}
const pgs = await db.query.postgres.findMany({
const backups = await db.query.backups.findMany({
with: {
backups: {
with: {
destination: true,
postgres: true,
mariadb: true,
mysql: true,
mongo: true,
},
},
},
});
for (const pg of pgs) {
for (const backup of pg.backups) {
const { schedule, backupId, enabled, database } = backup;
if (enabled) {
console.log(
`[Backup] Postgres DB ${pg.name} for ${database} Activated`,
);
scheduleJob(backupId, schedule, async () => {
console.log(
`PG-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runPostgresBackup(pg, backup);
await keepLatestNBackups(backup, pg.serverId);
});
}
}
}
const mariadbs = await db.query.mariadb.findMany({
with: {
backups: {
with: {
destination: true,
postgres: true,
mariadb: true,
mysql: true,
mongo: true,
},
},
destination: true,
postgres: true,
mariadb: true,
mysql: true,
mongo: true,
user: true,
},
});
for (const maria of mariadbs) {
for (const backup of maria.backups) {
const { schedule, backupId, enabled, database } = backup;
if (enabled) {
console.log(
`[Backup] MariaDB DB ${maria.name} for ${database} Activated`,
);
scheduleJob(backupId, schedule, async () => {
console.log(
`MARIADB-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMariadbBackup(maria, backup);
await keepLatestNBackups(backup, maria.serverId);
});
}
}
}
const mongodbs = await db.query.mongo.findMany({
with: {
backups: {
with: {
destination: true,
postgres: true,
mariadb: true,
mysql: true,
mongo: true,
},
},
},
});
for (const mongo of mongodbs) {
for (const backup of mongo.backups) {
const { schedule, backupId, enabled } = backup;
if (enabled) {
console.log(`[Backup] MongoDB DB ${mongo.name} Activated`);
scheduleJob(backupId, schedule, async () => {
console.log(
`MONGO-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMongoBackup(mongo, backup);
await keepLatestNBackups(backup, mongo.serverId);
});
}
}
}
const mysqls = await db.query.mysql.findMany({
with: {
backups: {
with: {
destination: true,
postgres: true,
mariadb: true,
mysql: true,
mongo: true,
},
},
},
});
for (const mysql of mysqls) {
for (const backup of mysql.backups) {
const { schedule, backupId, enabled } = backup;
if (enabled) {
console.log(`[Backup] MySQL DB ${mysql.name} Activated`);
scheduleJob(backupId, schedule, async () => {
console.log(
`MYSQL-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMySqlBackup(mysql, backup);
await keepLatestNBackups(backup, mysql.serverId);
});
for (const backup of backups) {
try {
if (backup.enabled) {
scheduleBackup(backup);
}
console.log(
`[Backup] ${backup.databaseType} Enabled with cron: [${backup.schedule}]`,
);
} catch (error) {
console.error(`[Backup] ${backup.databaseType} Error`, error);
}
}

4
packages/server/src/utils/backups/utils.ts
View File

@@ -6,6 +6,7 @@ import { runMariadbBackup } from "./mariadb";
import { runMongoBackup } from "./mongo";
import { runMySqlBackup } from "./mysql";
import { runPostgresBackup } from "./postgres";
import { runWebServerBackup } from "./web-server";
export const scheduleBackup = (backup: BackupSchedule) => {
const { schedule, backupId, databaseType, postgres, mysql, mongo, mariadb } =
@@ -23,6 +24,9 @@ export const scheduleBackup = (backup: BackupSchedule) => {
} else if (databaseType === "mariadb" && mariadb) {
await runMariadbBackup(mariadb, backup);
await keepLatestNBackups(backup, mariadb.serverId);
} else if (databaseType === "web-server") {
await runWebServerBackup(backup);
await keepLatestNBackups(backup);
}
});
};

45
packages/server/src/utils/backups/web-server.ts Normal file
View File

@@ -0,0 +1,45 @@
import type { BackupSchedule } from "@dokploy/server/services/backup";
import { execAsync } from "../process/execAsync";
import { getS3Credentials } from "./utils";
import { findDestinationById } from "@dokploy/server/services/destination";
import { IS_CLOUD, paths } from "@dokploy/server/constants";
import { mkdtemp } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
export const runWebServerBackup = async (backup: BackupSchedule) => {
try {
if (IS_CLOUD) {
return;
}
const destination = await findDestinationById(backup.destinationId);
const rcloneFlags = getS3Credentials(destination);
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
const { BASE_PATH } = paths();
const tempDir = await mkdtemp(join(tmpdir(), "dokploy-backup-"));
const backupFileName = `webserver-backup-${timestamp}.zip`;
const s3Path = `:s3:${destination.bucket}/${backup.prefix}${backupFileName}`;
try {
await execAsync(`mkdir -p ${tempDir}/filesystem`);
const postgresCommand = `docker exec $(docker ps --filter "name=dokploy-postgres" -q) pg_dump -v -Fc -U dokploy -d dokploy > ${tempDir}/database.sql`;
await execAsync(postgresCommand);
await execAsync(`cp -r ${BASE_PATH}/* ${tempDir}/filesystem/`);
await execAsync(
`cd ${tempDir} && zip -r ${backupFileName} database.sql filesystem/`,
);
const uploadCommand = `rclone copyto ${rcloneFlags.join(" ")} "${tempDir}/${backupFileName}" "${s3Path}"`;
await execAsync(uploadCommand);
return true;
} finally {
await execAsync(`rm -rf ${tempDir}`);
}
} catch (error) {
console.error("Backup error:", error);
throw error;
}
};

16
packages/server/src/utils/docker/compose/volume.ts
View File

@@ -30,12 +30,22 @@ export const addSuffixToVolumesInServices = (
// skip bind mounts and variables (e.g. $PWD)
if (
volumeName?.startsWith(".") ||
volumeName?.startsWith("/") ||
volumeName?.startsWith("$")
!volumeName ||
volumeName.startsWith(".") ||
volumeName.startsWith("/") ||
volumeName.startsWith("$")
) {
return volume;
}
// Handle volume paths with subdirectories
const parts = volumeName.split("/");
if (parts.length > 1) {
const baseName = parts[0];
const rest = parts.slice(1).join("/");
return `${baseName}-${suffix}/${rest}:${path}`;
}
return `${volumeName}-${suffix}:${path}`;
}
if (_.isObject(volume) && volume.type === "volume" && volume.source) {

1
packages/server/src/utils/restore/index.ts
View File

@@ -2,3 +2,4 @@ export { restorePostgresBackup } from "./postgres";
export { restoreMySqlBackup } from "./mysql";
export { restoreMariadbBackup } from "./mariadb";
export { restoreMongoBackup } from "./mongo";
export { restoreWebServerBackup } from "./web-server";

143
packages/server/src/utils/restore/web-server.ts Normal file
View File

@@ -0,0 +1,143 @@
import type { Destination } from "@dokploy/server/services/destination";
import { getS3Credentials } from "../backups/utils";
import { execAsync } from "../process/execAsync";
import { paths, IS_CLOUD } from "@dokploy/server/constants";
import { mkdtemp } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
export const restoreWebServerBackup = async (
destination: Destination,
backupFile: string,
emit: (log: string) => void,
) => {
if (IS_CLOUD) {
return;
}
try {
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupFile}`;
const { BASE_PATH } = paths();
// Create a temporary directory outside of BASE_PATH
const tempDir = await mkdtemp(join(tmpdir(), "dokploy-restore-"));
try {
emit("Starting restore...");
emit(`Backup path: ${backupPath}`);
emit(`Temp directory: ${tempDir}`);
// Create temp directory
emit("Creating temporary directory...");
await execAsync(`mkdir -p ${tempDir}`);
// Download backup from S3
emit("Downloading backup from S3...");
await execAsync(
`rclone copyto ${rcloneFlags.join(" ")} "${backupPath}" "${tempDir}/${backupFile}"`,
);
// List files before extraction
emit("Listing files before extraction...");
const { stdout: beforeFiles } = await execAsync(`ls -la ${tempDir}`);
emit(`Files before extraction: ${beforeFiles}`);
// Extract backup
emit("Extracting backup...");
await execAsync(`cd ${tempDir} && unzip ${backupFile}`);
// Restore filesystem first
emit("Restoring filesystem...");
emit(`Copying from ${tempDir}/filesystem/* to ${BASE_PATH}/`);
// First clean the target directory
emit("Cleaning target directory...");
await execAsync(`rm -rf "${BASE_PATH}/"*`);
// Ensure the target directory exists
emit("Setting up target directory...");
await execAsync(`mkdir -p "${BASE_PATH}"`);
// Copy files preserving permissions
emit("Copying files...");
await execAsync(`cp -rp "${tempDir}/filesystem/"* "${BASE_PATH}/"`);
// Now handle database restore
emit("Starting database restore...");
// Check if database.sql.gz exists and decompress it
const { stdout: hasGzFile } = await execAsync(
`ls ${tempDir}/database.sql.gz || true`,
);
if (hasGzFile.includes("database.sql.gz")) {
emit("Found compressed database file, decompressing...");
await execAsync(`cd ${tempDir} && gunzip database.sql.gz`);
}
// Verify database file exists
const { stdout: hasSqlFile } = await execAsync(
`ls ${tempDir}/database.sql || true`,
);
if (!hasSqlFile.includes("database.sql")) {
throw new Error("Database file not found after extraction");
}
// Drop and recreate database
emit("Disconnecting all users from database...");
await execAsync(
`docker exec $(docker ps --filter "name=dokploy-postgres" -q) psql -U dokploy postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'dokploy' AND pid <> pg_backend_pid();"`,
);
emit("Dropping existing database...");
await execAsync(
`docker exec $(docker ps --filter "name=dokploy-postgres" -q) psql -U dokploy postgres -c "DROP DATABASE IF EXISTS dokploy;"`,
);
emit("Creating fresh database...");
await execAsync(
`docker exec $(docker ps --filter "name=dokploy-postgres" -q) psql -U dokploy postgres -c "CREATE DATABASE dokploy;"`,
);
// Copy the backup file into the container
emit("Copying backup file into container...");
await execAsync(
`docker cp ${tempDir}/database.sql $(docker ps --filter "name=dokploy-postgres" -q):/tmp/database.sql`,
);
// Verify file in container
emit("Verifying file in container...");
await execAsync(
`docker exec $(docker ps --filter "name=dokploy-postgres" -q) ls -l /tmp/database.sql`,
);
// Restore from the copied file
emit("Running database restore...");
await execAsync(
`docker exec $(docker ps --filter "name=dokploy-postgres" -q) pg_restore -v -U dokploy -d dokploy /tmp/database.sql`,
);
// Cleanup the temporary file in the container
emit("Cleaning up container temp file...");
await execAsync(
`docker exec $(docker ps --filter "name=dokploy-postgres" -q) rm /tmp/database.sql`,
);
emit("Restore completed successfully!");
} finally {
// Cleanup
emit("Cleaning up temporary files...");
await execAsync(`rm -rf ${tempDir}`);
}
} catch (error) {
console.error(error);
emit(
`Error: ${
error instanceof Error
? error.message
: "Error restoring web server backup"
}`,
);
throw error;
}
};