Merge branch '187-backups-for-docker-compose' into feat/introduce-monitoring-self-hosted-pay

packages/server/package.json

@@ -1,21 +1,32 @@
 {
   "name": "@dokploy/server",
   "version": "1.0.0",
-  "main": "./src/index.ts",
+  "main": "./dist/index.js",
   "type": "module",
   "exports": {
-    ".": "./src/index.ts",
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs.js"
+    },
     "./db": {
-      "import": "./src/db/index.ts",
+      "import": "./dist/db/index.js",
       "require": "./dist/db/index.cjs.js"
     },
-    "./setup/*": {
-      "import": "./src/setup/*.ts",
-      "require": "./dist/setup/index.cjs.js"
+    "./*": {
+      "import": "./dist/*",
+      "require": "./dist/*.cjs"
     },
-    "./constants": {
-      "import": "./src/constants/index.ts",
-      "require": "./dist/constants.cjs.js"
+    "./dist": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs.js"
+    },
+    "./dist/db": {
+      "import": "./dist/db/index.js",
+      "require": "./dist/db/index.cjs.js"
+    },
+    "./dist/db/schema": {
+      "import": "./dist/db/schema/index.js",
+      "require": "./dist/db/schema/index.cjs.js"
     }
   },
   "scripts": {

packages/server/src/db/schema/backups.ts

@@ -3,6 +3,7 @@ import {
 	type AnyPgColumn,
 	boolean,
 	integer,
+	jsonb,
 	pgEnum,
 	pgTable,
 	text,
@@ -16,6 +17,8 @@ import { mongo } from "./mongo";
 import { mysql } from "./mysql";
 import { postgres } from "./postgres";
 import { users_temp } from "./user";
+import { compose } from "./compose";
+
 export const databaseType = pgEnum("databaseType", [
 	"postgres",
 	"mariadb",
@@ -24,6 +27,8 @@ export const databaseType = pgEnum("databaseType", [
 	"web-server",
 ]);
 
+export const backupType = pgEnum("backupType", ["database", "compose"]);
+
 export const backups = pgTable("backup", {
 	backupId: text("backupId")
 		.notNull()
@@ -33,14 +38,19 @@ export const backups = pgTable("backup", {
 	enabled: boolean("enabled"),
 	database: text("database").notNull(),
 	prefix: text("prefix").notNull(),
-
+	serviceName: text("serviceName"),
 	destinationId: text("destinationId")
 		.notNull()
 		.references(() => destinations.destinationId, { onDelete: "cascade" }),
-
 	keepLatestCount: integer("keepLatestCount"),
-
+	backupType: backupType("backupType").notNull().default("database"),
 	databaseType: databaseType("databaseType").notNull(),
+	composeId: text("composeId").references(
+		(): AnyPgColumn => compose.composeId,
+		{
+			onDelete: "cascade",
+		},
+	),
 	postgresId: text("postgresId").references(
 		(): AnyPgColumn => postgres.postgresId,
 		{
@@ -60,6 +70,26 @@ export const backups = pgTable("backup", {
 		onDelete: "cascade",
 	}),
 	userId: text("userId").references(() => users_temp.id),
+	// Only for compose backups
+	metadata: jsonb("metadata").$type<
+		| {
+				postgres?: {
+					databaseUser: string;
+				};
+				mariadb?: {
+					databaseUser: string;
+					databasePassword: string;
+				};
+				mongo?: {
+					databaseUser: string;
+					databasePassword: string;
+				};
+				mysql?: {
+					databaseRootPassword: string;
+				};
+		  }
+		| undefined
+	>(),
 });
 
 export const backupsRelations = relations(backups, ({ one }) => ({
@@ -87,6 +117,10 @@ export const backupsRelations = relations(backups, ({ one }) => ({
 		fields: [backups.userId],
 		references: [users_temp.id],
 	}),
+	compose: one(compose, {
+		fields: [backups.composeId],
+		references: [compose.composeId],
+	}),
 }));
 
 const createSchema = createInsertSchema(backups, {
@@ -103,6 +137,7 @@ const createSchema = createInsertSchema(backups, {
 	mysqlId: z.string().optional(),
 	mongoId: z.string().optional(),
 	userId: z.string().optional(),
+	metadata: z.any().optional(),
 });
 
 export const apiCreateBackup = createSchema.pick({
@@ -118,6 +153,10 @@ export const apiCreateBackup = createSchema.pick({
 	mongoId: true,
 	databaseType: true,
 	userId: true,
+	backupType: true,
+	composeId: true,
+	serviceName: true,
+	metadata: true,
 });
 
 export const apiFindOneBackup = createSchema
@@ -141,5 +180,8 @@ export const apiUpdateBackup = createSchema
 		destinationId: true,
 		database: true,
 		keepLatestCount: true,
+		serviceName: true,
+		metadata: true,
+		databaseType: true,
 	})
 	.required();

packages/server/src/db/schema/compose.ts

@@ -15,6 +15,7 @@ import { server } from "./server";
 import { applicationStatus, triggerType } from "./shared";
 import { sshKeys } from "./ssh-key";
 import { generateAppName } from "./utils";
+import { backups } from "./backups";
 
 export const sourceTypeCompose = pgEnum("sourceTypeCompose", [
 	"git",
@@ -135,6 +136,7 @@ export const composeRelations = relations(compose, ({ one, many }) => ({
 		fields: [compose.serverId],
 		references: [server.serverId],
 	}),
+	backups: many(backups),
 }));
 
 const createSchema = createInsertSchema(compose, {

packages/server/src/index.ts

@@ -49,6 +49,7 @@ export * from "./utils/backups/mysql";
 export * from "./utils/backups/postgres";
 export * from "./utils/backups/utils";
 export * from "./utils/backups/web-server";
+export * from "./utils/backups/compose";
 export * from "./templates/processors";
 
 export * from "./utils/notifications/build-error";

packages/server/src/services/backup.ts

@@ -35,6 +35,7 @@ export const findBackupById = async (backupId: string) => {
 			mariadb: true,
 			mongo: true,
 			destination: true,
+			compose: true,
 		},
 	});
 	if (!backup) {

packages/server/src/services/compose.ts

@@ -131,6 +131,11 @@ export const findComposeById = async (composeId: string) => {
 			bitbucket: true,
 			gitea: true,
 			server: true,
+			backups: {
+				with: {
+					destination: true,
+				},
+			},
 		},
 	});
 	if (!result) {

packages/server/src/services/domain.ts

@@ -7,6 +7,8 @@ import { type apiCreateDomain, domains } from "../db/schema";
 import { findUserById } from "./admin";
 import { findApplicationById } from "./application";
 import { findServerById } from "./server";
+import dns from "node:dns";
+import { promisify } from "node:util";
 
 export type Domain = typeof domains.$inferSelect;
 
@@ -137,3 +139,84 @@ export const removeDomainById = async (domainId: string) => {
 export const getDomainHost = (domain: Domain) => {
 	return `${domain.https ? "https" : "http"}://${domain.host}`;
 };
+
+const resolveDns = promisify(dns.resolve4);
+
+// Cloudflare IP ranges (simplified - these are some common ones)
+const CLOUDFLARE_IPS = [
+	"172.67.",
+	"104.21.",
+	"104.16.",
+	"104.17.",
+	"104.18.",
+	"104.19.",
+	"104.20.",
+	"104.22.",
+	"104.23.",
+	"104.24.",
+	"104.25.",
+	"104.26.",
+	"104.27.",
+	"104.28.",
+];
+
+const isCloudflareIp = (ip: string) => {
+	return CLOUDFLARE_IPS.some((range) => ip.startsWith(range));
+};
+
+export const validateDomain = async (
+	domain: string,
+	expectedIp?: string,
+): Promise<{
+	isValid: boolean;
+	resolvedIp?: string;
+	error?: string;
+	isCloudflare?: boolean;
+}> => {
+	try {
+		// Remove protocol and path if present
+		const cleanDomain = domain.replace(/^https?:\/\//, "").split("/")[0];
+
+		// Resolve the domain to get its IP
+		const ips = await resolveDns(cleanDomain || "");
+
+		const resolvedIps = ips.map((ip) => ip.toString());
+
+		// Check if it's a Cloudflare IP
+		const behindCloudflare = ips.some((ip) => isCloudflareIp(ip));
+
+		// If behind Cloudflare, we consider it valid but inform the user
+		if (behindCloudflare) {
+			return {
+				isValid: true,
+				resolvedIp: resolvedIps.join(", "),
+				isCloudflare: true,
+				error:
+					"Domain is behind Cloudflare - actual IP is masked by Cloudflare proxy",
+			};
+		}
+
+		// If we have an expected IP, validate against it
+		if (expectedIp) {
+			return {
+				isValid: resolvedIps.includes(expectedIp),
+				resolvedIp: resolvedIps.join(", "),
+				error: !resolvedIps.includes(expectedIp)
+					? `Domain resolves to ${resolvedIps.join(", ")} but should point to ${expectedIp}`
+					: undefined,
+			};
+		}
+
+		// If no expected IP, just return the resolved IP
+		return {
+			isValid: true,
+			resolvedIp: resolvedIps.join(", "),
+		};
+	} catch (error) {
+		return {
+			isValid: false,
+			error:
+				error instanceof Error ? error.message : "Failed to resolve domain",
+		};
+	}
+};

packages/server/src/services/mongo.ts

@@ -1,5 +1,10 @@
 import { db } from "@dokploy/server/db";
-import { type apiCreateMongo, backups, mongo } from "@dokploy/server/db/schema";
+import {
+	type apiCreateMongo,
+	backups,
+	compose,
+	mongo,
+} from "@dokploy/server/db/schema";
 import { buildAppName } from "@dokploy/server/db/schema";
 import { generatePassword } from "@dokploy/server/templates";
 import { buildMongo } from "@dokploy/server/utils/databases/mongo";
@@ -103,6 +108,25 @@ export const findMongoByBackupId = async (backupId: string) => {
 	return result[0];
 };
 
+export const findComposeByBackupId = async (backupId: string) => {
+	const result = await db
+		.select({
+			...getTableColumns(compose),
+		})
+		.from(compose)
+		.innerJoin(backups, eq(compose.composeId, backups.composeId))
+		.where(eq(backups.backupId, backupId))
+		.limit(1);
+
+	if (!result || !result[0]) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Compose not found",
+		});
+	}
+	return result[0];
+};
+
 export const removeMongoById = async (mongoId: string) => {
 	const result = await db
 		.delete(mongo)

packages/server/src/utils/backups/compose.ts

@@ -0,0 +1,113 @@
+import type { BackupSchedule } from "@dokploy/server/services/backup";
+import type { Compose } from "@dokploy/server/services/compose";
+import { findProjectById } from "@dokploy/server/services/project";
+import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+import { getS3Credentials, normalizeS3Path } from "./utils";
+
+export const runComposeBackup = async (
+	compose: Compose,
+	backup: BackupSchedule,
+) => {
+	const { projectId, name } = compose;
+	const project = await findProjectById(projectId);
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.dump.gz`;
+	const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
+
+	try {
+		const rcloneFlags = getS3Credentials(destination);
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
+
+		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+		const command = getFindContainerCommand(compose, backup.serviceName || "");
+		if (compose.serverId) {
+			const { stdout } = await execAsyncRemote(compose.serverId, command);
+			if (!stdout) {
+				throw new Error("Container not found");
+			}
+			const containerId = stdout.trim();
+
+			let backupCommand = "";
+
+			if (backup.databaseType === "postgres") {
+				backupCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${backup.metadata?.postgres?.databaseUser} --no-password '${database}' | gzip"`;
+			} else if (backup.databaseType === "mariadb") {
+				backupCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${backup.metadata?.mariadb?.databaseUser}' --password='${backup.metadata?.mariadb?.databasePassword}' --databases ${database} | gzip"`;
+			} else if (backup.databaseType === "mysql") {
+				backupCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${backup.metadata?.mysql?.databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
+			} else if (backup.databaseType === "mongo") {
+				backupCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${backup.metadata?.mongo?.databaseUser}' -p '${backup.metadata?.mongo?.databasePassword}' --archive --authenticationDatabase admin --gzip"`;
+			}
+
+			await execAsyncRemote(
+				compose.serverId,
+				`${backupCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { stdout } = await execAsync(command);
+			if (!stdout) {
+				throw new Error("Container not found");
+			}
+			const containerId = stdout.trim();
+
+			let backupCommand = "";
+
+			if (backup.databaseType === "postgres") {
+				backupCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${backup.metadata?.postgres?.databaseUser} --no-password '${database}' | gzip"`;
+			} else if (backup.databaseType === "mariadb") {
+				backupCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${backup.metadata?.mariadb?.databaseUser}' --password='${backup.metadata?.mariadb?.databasePassword}' --databases ${database} | gzip"`;
+			} else if (backup.databaseType === "mysql") {
+				backupCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${backup.metadata?.mysql?.databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
+			} else if (backup.databaseType === "mongo") {
+				backupCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${backup.metadata?.mongo?.databaseUser}' -p '${backup.metadata?.mongo?.databasePassword}' --archive --authenticationDatabase admin --gzip"`;
+			}
+
+			await execAsync(`${backupCommand} | ${rcloneCommand}`);
+		}
+
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "mongodb",
+			type: "success",
+			organizationId: project.organizationId,
+		});
+	} catch (error) {
+		console.log(error);
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "mongodb",
+			type: "error",
+			// @ts-ignore
+			errorMessage: error?.message || "Error message not provided",
+			organizationId: project.organizationId,
+		});
+		throw error;
+	}
+};
+
+export const getFindContainerCommand = (
+	compose: Compose,
+	serviceName: string,
+) => {
+	const { appName, composeType } = compose;
+	const labels =
+		composeType === "stack"
+			? {
+					namespace: `label=com.docker.stack.namespace=${appName}`,
+					service: `label=com.docker.swarm.service.name=${appName}_${serviceName}`,
+				}
+			: {
+					project: `label=com.docker.compose.project=${appName}`,
+					service: `label=com.docker.compose.service=${serviceName}`,
+				};
+
+	const command = `docker ps --filter "status=running" \
+	  --filter "${Object.values(labels).join('" --filter "')}" \
+	  --format "{{.ID}}" | head -n 1`;
+
+	return command.trim();
+};

packages/server/src/utils/backups/index.ts

@@ -70,6 +70,7 @@ export const initCronJobs = async () => {
 			mysql: true,
 			mongo: true,
 			user: true,
+			compose: true,
 		},
 	});
 
@@ -77,10 +78,10 @@ export const initCronJobs = async () => {
 		try {
 			if (backup.enabled) {
 				scheduleBackup(backup);
+				console.log(
+					`[Backup] ${backup.databaseType} Enabled with cron: [${backup.schedule}]`,
+				);
 			}
-			console.log(
-				`[Backup] ${backup.databaseType} Enabled with cron: [${backup.schedule}]`,
-			);
 		} catch (error) {
 			console.error(`[Backup] ${backup.databaseType} Error`, error);
 		}

packages/server/src/utils/backups/utils.ts

@@ -7,26 +7,40 @@ import { runMongoBackup } from "./mongo";
 import { runMySqlBackup } from "./mysql";
 import { runPostgresBackup } from "./postgres";
 import { runWebServerBackup } from "./web-server";
+import { runComposeBackup } from "./compose";
 
 export const scheduleBackup = (backup: BackupSchedule) => {
-	const { schedule, backupId, databaseType, postgres, mysql, mongo, mariadb } =
-		backup;
+	const {
+		schedule,
+		backupId,
+		databaseType,
+		postgres,
+		mysql,
+		mongo,
+		mariadb,
+		compose,
+	} = backup;
 	scheduleJob(backupId, schedule, async () => {
-		if (databaseType === "postgres" && postgres) {
-			await runPostgresBackup(postgres, backup);
-			await keepLatestNBackups(backup, postgres.serverId);
-		} else if (databaseType === "mysql" && mysql) {
-			await runMySqlBackup(mysql, backup);
-			await keepLatestNBackups(backup, mysql.serverId);
-		} else if (databaseType === "mongo" && mongo) {
-			await runMongoBackup(mongo, backup);
-			await keepLatestNBackups(backup, mongo.serverId);
-		} else if (databaseType === "mariadb" && mariadb) {
-			await runMariadbBackup(mariadb, backup);
-			await keepLatestNBackups(backup, mariadb.serverId);
-		} else if (databaseType === "web-server") {
-			await runWebServerBackup(backup);
-			await keepLatestNBackups(backup);
+		if (backup.backupType === "database") {
+			if (databaseType === "postgres" && postgres) {
+				await runPostgresBackup(postgres, backup);
+				await keepLatestNBackups(backup, postgres.serverId);
+			} else if (databaseType === "mysql" && mysql) {
+				await runMySqlBackup(mysql, backup);
+				await keepLatestNBackups(backup, mysql.serverId);
+			} else if (databaseType === "mongo" && mongo) {
+				await runMongoBackup(mongo, backup);
+				await keepLatestNBackups(backup, mongo.serverId);
+			} else if (databaseType === "mariadb" && mariadb) {
+				await runMariadbBackup(mariadb, backup);
+				await keepLatestNBackups(backup, mariadb.serverId);
+			} else if (databaseType === "web-server") {
+				await runWebServerBackup(backup);
+				await keepLatestNBackups(backup);
+			}
+		} else if (backup.backupType === "compose" && compose) {
+			await runComposeBackup(compose, backup);
+			await keepLatestNBackups(backup, compose.serverId);
 		}
 	});
 };

packages/server/src/utils/docker/backup.ts

@@ -0,0 +1,4 @@
+export const createBackupLabels = (backupId: string) => {
+	const labels = [`dokploy.backup.id=${backupId}`];
+	return labels;
+};

packages/server/src/utils/restore/compose.ts

@@ -0,0 +1,93 @@
+import type { Destination } from "@dokploy/server/services/destination";
+import type { Compose } from "@dokploy/server/services/compose";
+import { getS3Credentials } from "../backups/utils";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+import type { Backup } from "@dokploy/server/services/backup";
+import { getFindContainerCommand } from "../backups/compose";
+
+export const restoreComposeBackup = async (
+	compose: Compose,
+	destination: Destination,
+	database: string,
+	backupFile: string,
+	metadata: Backup["metadata"] & { serviceName: string },
+	emit: (log: string) => void,
+) => {
+	try {
+		const { serverId } = compose;
+
+		const rcloneFlags = getS3Credentials(destination);
+		const bucketPath = `:s3:${destination.bucket}`;
+		const backupPath = `${bucketPath}/${backupFile}`;
+
+		const command = getFindContainerCommand(compose, metadata.serviceName);
+
+		let containerId = "";
+		if (serverId) {
+			const { stdout, stderr } = await execAsyncRemote(serverId, command);
+			emit(stdout);
+			emit(stderr);
+			containerId = stdout.trim();
+		} else {
+			const { stdout, stderr } = await execAsync(command);
+			emit(stdout);
+			emit(stderr);
+			containerId = stdout.trim();
+		}
+		let restoreCommand = "";
+
+		if (metadata.postgres) {
+			restoreCommand = `rclone cat ${rcloneFlags.join(" ")} "${backupPath}" | gunzip | docker exec -i ${containerId} pg_restore -U ${metadata.postgres.databaseUser} -d ${database} --clean --if-exists`;
+		} else if (metadata.mariadb) {
+			restoreCommand = `
+			rclone cat ${rcloneFlags.join(" ")} "${backupPath}" | gunzip | docker exec -i ${containerId} mariadb -u ${metadata.mariadb.databaseUser} -p${metadata.mariadb.databasePassword} ${database}
+		  `;
+		} else if (metadata.mysql) {
+			restoreCommand = `
+			rclone cat ${rcloneFlags.join(" ")} "${backupPath}" | gunzip | docker exec -i ${containerId} mysql -u root -p${metadata.mysql.databaseRootPassword} ${database}
+		  `;
+		} else if (metadata.mongo) {
+			const tempDir = "/tmp/dokploy-restore";
+			const fileName = backupFile.split("/").pop() || "backup.dump.gz";
+			const decompressedName = fileName.replace(".gz", "");
+			restoreCommand = `\
+			rm -rf ${tempDir} && \
+			mkdir -p ${tempDir} && \
+			rclone copy ${rcloneFlags.join(" ")} "${backupPath}" ${tempDir} && \
+			cd ${tempDir} && \
+			gunzip -f "${fileName}" && \
+			docker exec -i ${containerId} mongorestore --username ${metadata.mongo.databaseUser} --password ${metadata.mongo.databasePassword} --authenticationDatabase admin --db ${database} --archive < "${decompressedName}" && \
+			rm -rf ${tempDir}`;
+		}
+
+		emit("Starting restore...");
+		emit(`Backup path: ${backupPath}`);
+
+		emit(`Executing command: ${restoreCommand}`);
+
+		if (serverId) {
+			const { stdout, stderr } = await execAsyncRemote(
+				serverId,
+				restoreCommand,
+			);
+			emit(stdout);
+			emit(stderr);
+		} else {
+			const { stdout, stderr } = await execAsync(restoreCommand);
+			emit(stdout);
+			emit(stderr);
+		}
+
+		emit("Restore completed successfully!");
+	} catch (error) {
+		console.error(error);
+		emit(
+			`Error: ${
+				error instanceof Error ? error.message : "Error restoring mongo backup"
+			}`,
+		);
+		throw new Error(
+			error instanceof Error ? error.message : "Error restoring mongo backup",
+		);
+	}
+};

packages/server/src/utils/restore/index.ts

@@ -3,3 +3,4 @@ export { restoreMySqlBackup } from "./mysql";
 export { restoreMariadbBackup } from "./mariadb";
 export { restoreMongoBackup } from "./mongo";
 export { restoreWebServerBackup } from "./web-server";
+export { restoreComposeBackup } from "./compose";

packages/server/src/utils/restore/postgres.ts

@@ -40,8 +40,6 @@ rclone cat ${rcloneFlags.join(" ")} "${backupPath}" | gunzip | docker exec -i ${
 			emit(stderr);
 		} else {
 			const { stdout, stderr } = await execAsync(command);
-			console.log("stdout", stdout);
-			console.log("stderr", stderr);
 			emit(stdout);
 			emit(stderr);
 		}