Enhance backup and deployment features

- Updated the RestoreBackupSchema to require serviceName for compose backups, improving validation and user feedback. - Refactored the ShowBackups component to include deployment information, enhancing the user interface and experience. - Introduced new SQL migration files to add backupId to the deployment table and appName to the backup table, improving data relationships and integrity. - Enhanced deployment creation logic to support backup deployments, ensuring better tracking and management of backup processes. - Improved backup and restore utility functions to streamline command execution and error handling during backup operations.
2025-06-26 18:27:59 +00:00 · 2025-05-03 12:39:52 -06:00
parent 50aeeb2fb8
commit e437903ef8
23 changed files with 11785 additions and 92 deletions
--- a/packages/server/src/utils/backups/compose.ts
+++ b/packages/server/src/utils/backups/compose.ts
@@ -2,8 +2,21 @@ import type { BackupSchedule } from "@dokploy/server/services/backup";
 import type { Compose } from "@dokploy/server/services/compose";
 import { findProjectById } from "@dokploy/server/services/project";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
-import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { getS3Credentials, normalizeS3Path } from "./utils";
+import { execAsyncRemote, execAsyncStream } from "../process/execAsync";
+import {
+	getMariadbBackupCommand,
+	getMysqlBackupCommand,
+	getMongoBackupCommand,
+	getPostgresBackupCommand,
+	getS3Credentials,
+	normalizeS3Path,
+} from "./utils";
+import {
+	createDeploymentBackup,
+	updateDeploymentStatus,
+} from "@dokploy/server/services/deployment";
+import { createWriteStream } from "node:fs";
+import { getComposeContainer } from "../docker/utils";

 export const runComposeBackup = async (
 	compose: Compose,
@@ -15,56 +28,81 @@ export const runComposeBackup = async (
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.dump.gz`;
 	const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
-
+	const deployment = await createDeploymentBackup({
+		backupId: backup.backupId,
+		title: "Compose Backup",
+		description: "Compose Backup",
+	});
 	try {
 		const rcloneFlags = getS3Credentials(destination);
 		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;

+		const { Id: containerId } = await getComposeContainer(
+			compose,
+			backup.serviceName || "",
+		);
+
 		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
-		const command = getFindContainerCommand(compose, backup.serviceName || "");
+		let backupCommand = "";
+
+		if (backup.databaseType === "postgres") {
+			backupCommand = getPostgresBackupCommand(
+				containerId,
+				database,
+				backup.metadata?.postgres?.databaseUser || "",
+			);
+		} else if (backup.databaseType === "mariadb") {
+			backupCommand = getMariadbBackupCommand(
+				containerId,
+				database,
+				backup.metadata?.mariadb?.databaseUser || "",
+				backup.metadata?.mariadb?.databasePassword || "",
+			);
+		} else if (backup.databaseType === "mysql") {
+			backupCommand = getMysqlBackupCommand(
+				containerId,
+				database,
+				backup.metadata?.mysql?.databaseRootPassword || "",
+			);
+		} else if (backup.databaseType === "mongo") {
+			backupCommand = getMongoBackupCommand(
+				containerId,
+				database,
+				backup.metadata?.mongo?.databaseUser || "",
+				backup.metadata?.mongo?.databasePassword || "",
+			);
+		}
 		if (compose.serverId) {
-			const { stdout } = await execAsyncRemote(compose.serverId, command);
-			if (!stdout) {
-				throw new Error("Container not found");
-			}
-			const containerId = stdout.trim();
-
-			let backupCommand = "";
-
-			if (backup.databaseType === "postgres") {
-				backupCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${backup.metadata?.postgres?.databaseUser} --no-password '${database}' | gzip"`;
-			} else if (backup.databaseType === "mariadb") {
-				backupCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${backup.metadata?.mariadb?.databaseUser}' --password='${backup.metadata?.mariadb?.databasePassword}' --databases ${database} | gzip"`;
-			} else if (backup.databaseType === "mysql") {
-				backupCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${backup.metadata?.mysql?.databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
-			} else if (backup.databaseType === "mongo") {
-				backupCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${backup.metadata?.mongo?.databaseUser}' -p '${backup.metadata?.mongo?.databasePassword}' --archive --authenticationDatabase admin --gzip"`;
-			}
-
 			await execAsyncRemote(
 				compose.serverId,
-				`${backupCommand} | ${rcloneCommand}`,
+				`
+				 set -e;
+				 Running command.
+				${backupCommand} | ${rcloneCommand} >> ${deployment.logPath} 2>> ${deployment.logPath} || {
+					echo "❌ Command failed" >> ${deployment.logPath};
+					exit 1;
+				}
+				echo "✅ Command executed successfully" >> ${deployment.logPath};
+				`,
 			);
 		} else {
-			const { stdout } = await execAsync(command);
-			if (!stdout) {
-				throw new Error("Container not found");
-			}
-			const containerId = stdout.trim();
-
-			let backupCommand = "";
-
-			if (backup.databaseType === "postgres") {
-				backupCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${backup.metadata?.postgres?.databaseUser} --no-password '${database}' | gzip"`;
-			} else if (backup.databaseType === "mariadb") {
-				backupCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${backup.metadata?.mariadb?.databaseUser}' --password='${backup.metadata?.mariadb?.databasePassword}' --databases ${database} | gzip"`;
-			} else if (backup.databaseType === "mysql") {
-				backupCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${backup.metadata?.mysql?.databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
-			} else if (backup.databaseType === "mongo") {
-				backupCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${backup.metadata?.mongo?.databaseUser}' -p '${backup.metadata?.mongo?.databasePassword}' --archive --authenticationDatabase admin --gzip"`;
-			}
-
-			await execAsync(`${backupCommand} | ${rcloneCommand}`);
+			const writeStream = createWriteStream(deployment.logPath, { flags: "a" });
+			await execAsyncStream(
+				`${backupCommand} | ${rcloneCommand}`,
+				(data) => {
+					if (writeStream.write(data)) {
+						console.log(data);
+					}
+				},
+				{
+					env: {
+						...process.env,
+						RCLONE_LOG_LEVEL: "DEBUG",
+					},
+				},
+			);
+			writeStream.write("Backup done✅");
+			writeStream.end();
 		}

 		await sendDatabaseBackupNotifications({
@@ -74,6 +112,8 @@ export const runComposeBackup = async (
 			type: "success",
 			organizationId: project.organizationId,
 		});
+
+		await updateDeploymentStatus(deployment.deploymentId, "done");
 	} catch (error) {
 		console.log(error);
 		await sendDatabaseBackupNotifications({
@@ -85,29 +125,8 @@ export const runComposeBackup = async (
 			errorMessage: error?.message || "Error message not provided",
 			organizationId: project.organizationId,
 		});
+
+		await updateDeploymentStatus(deployment.deploymentId, "error");
 		throw error;
 	}
 };
-
-export const getFindContainerCommand = (
-	compose: Compose,
-	serviceName: string,
-) => {
-	const { appName, composeType } = compose;
-	const labels =
-		composeType === "stack"
-			? {
-					namespace: `label=com.docker.stack.namespace=${appName}`,
-					service: `label=com.docker.swarm.service.name=${appName}_${serviceName}`,
-				}
-			: {
-					project: `label=com.docker.compose.project=${appName}`,
-					service: `label=com.docker.compose.service=${serviceName}`,
-				};
-
-	const command = `docker ps --filter "status=running" \
-	  --filter "${Object.values(labels).join('" --filter "')}" \
-	  --format "{{.ID}}" | head -n 1`;
-
-	return command.trim();
-};
--- a/packages/server/src/utils/backups/postgres.ts
+++ b/packages/server/src/utils/backups/postgres.ts
@@ -8,6 +8,7 @@ import {
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
 import { getS3Credentials, normalizeS3Path } from "./utils";
+import { createDeploymentBackup } from "@dokploy/server/services/deployment";

 export const runPostgresBackup = async (
 	postgres: Postgres,
@@ -16,6 +17,11 @@ export const runPostgresBackup = async (
 	const { appName, databaseUser, name, projectId } = postgres;
 	const project = await findProjectById(projectId);

+	const deployment = await createDeploymentBackup({
+		backupId: backup.backupId,
+		title: "Postgres Backup",
+		description: "Postgres Backup",
+	});
 	const { prefix, database } = backup;
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.sql.gz`;
@@ -40,7 +46,11 @@ export const runPostgresBackup = async (
 			const { Id: containerId } = await getServiceContainer(appName);

 			const pgDumpCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip"`;
-			await execAsync(`${pgDumpCommand} | ${rcloneCommand}`);
+
+			await execAsync(`${pgDumpCommand} | ${rcloneCommand}`, (data) => {
+				console.log(data);
+			});
+			// await execAsync(`${pgDumpCommand} | ${rcloneCommand}`);
 		}

 		await sendDatabaseBackupNotifications({
--- a/packages/server/src/utils/backups/utils.ts
+++ b/packages/server/src/utils/backups/utils.ts
@@ -75,3 +75,37 @@ export const getS3Credentials = (destination: Destination) => {

 	return rcloneFlags;
 };
+
+export const getPostgresBackupCommand = (
+	containerId: string,
+	database: string,
+	databaseUser: string,
+) => {
+	return `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip"`;
+};
+
+export const getMariadbBackupCommand = (
+	containerId: string,
+	database: string,
+	databaseUser: string,
+	databasePassword: string,
+) => {
+	return `docker exec ${containerId} sh -c "mariadb-dump --user='${databaseUser}' --password='${databasePassword}' --databases ${database} | gzip"`;
+};
+
+export const getMysqlBackupCommand = (
+	containerId: string,
+	database: string,
+	databasePassword: string,
+) => {
+	return `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${databasePassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
+};
+
+export const getMongoBackupCommand = (
+	containerId: string,
+	database: string,
+	databaseUser: string,
+	databasePassword: string,
+) => {
+	return `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${databaseUser}' -p '${databasePassword}' --archive --authenticationDatabase admin --gzip"`;
+};