refactor(multi-server): use rclone for databases backup local and external server

Dockerfile

@@ -42,7 +42,7 @@ COPY --from=build /prod/dokploy/node_modules ./node_modules
 
 
 # Install docker
-RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && rm get-docker.sh
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && rm get-docker.sh && curl https://rclone.org/install.sh | sudo bash
 
 # Install Nixpacks and tsx
 # | VERBOSE=1 VERSION=1.21.0 bash

apps/dokploy/components/dashboard/application/general/show.tsx

@@ -13,6 +13,7 @@ import { StartApplication } from "../start-application";
 import { StopApplication } from "../stop-application";
 import { DeployApplication } from "./deploy-application";
 import { ResetApplication } from "./reset-application";
+import { Badge } from "@/components/ui/badge";
 interface Props {
 	applicationId: string;
 }
@@ -75,7 +76,8 @@ export const ShowGeneralApplication = ({ applicationId }: Props) => {
 							Open Terminal
 						</Button>
 					</DockerTerminalModal>
-					{data?.server?.name || "No Server"}
+
+					{/* {data?.server?.name || "No Server"} */}
 				</CardContent>
 			</Card>
 			<ShowProviderForm applicationId={applicationId} />

apps/dokploy/package.json

@@ -35,7 +35,6 @@
 	},
 	"dependencies": {
 		"rotating-file-stream": "3.2.3",
-		"@aws-sdk/client-s3": "3.515.0",
 		"@codemirror/lang-json": "^6.0.1",
 		"@codemirror/lang-yaml": "^6.1.1",
 		"@codemirror/language": "^6.10.1",

apps/dokploy/pages/dashboard/project/[projectId]/services/application/[applicationId].tsx

@@ -16,6 +16,7 @@ import { UpdateApplication } from "@/components/dashboard/application/update-app
 import { DockerMonitoring } from "@/components/dashboard/monitoring/docker/show";
 import { ProjectLayout } from "@/components/layouts/project-layout";
 import { StatusTooltip } from "@/components/shared/status-tooltip";
+import { Badge } from "@/components/ui/badge";
 import {
 	Breadcrumb,
 	BreadcrumbItem,
@@ -99,6 +100,9 @@ const Service = (
 							</h1>
 							<span className="text-sm">{data?.appName}</span>
 						</div>
+						<div>
+							<Badge>{data?.server?.name || "Dokploy Server"}</Badge>
+						</div>
 
 						{data?.description && (
 							<p className="text-sm text-muted-foreground max-w-6xl">

apps/dokploy/pages/dashboard/project/[projectId]/services/compose/[composeId].tsx

@@ -10,6 +10,7 @@ import { ShowMonitoringCompose } from "@/components/dashboard/compose/monitoring
 import { UpdateCompose } from "@/components/dashboard/compose/update-compose";
 import { ProjectLayout } from "@/components/layouts/project-layout";
 import { StatusTooltip } from "@/components/shared/status-tooltip";
+import { Badge } from "@/components/ui/badge";
 import {
 	Breadcrumb,
 	BreadcrumbItem,
@@ -93,13 +94,16 @@ const Service = (
 							</h1>
 							<span className="text-sm">{data?.appName}</span>
 						</div>
-
+						<div>
+							<Badge>{data?.server?.name || "Dokploy Server"}</Badge>
+						</div>
 						{data?.description && (
 							<p className="text-sm text-muted-foreground max-w-6xl">
 								{data?.description}
 							</p>
 						)}
 					</div>
+
 					<div className="relative flex flex-row gap-4">
 						<div className="absolute -right-1  -top-2">
 							<StatusTooltip status={data?.composeStatus} />

apps/dokploy/pages/dashboard/project/[projectId]/services/mariadb/[mariadbId].tsx

@@ -11,6 +11,7 @@ import { DockerMonitoring } from "@/components/dashboard/monitoring/docker/show"
 import { MariadbIcon } from "@/components/icons/data-tools-icons";
 import { ProjectLayout } from "@/components/layouts/project-layout";
 import { StatusTooltip } from "@/components/shared/status-tooltip";
+import { Badge } from "@/components/ui/badge";
 import {
 	Breadcrumb,
 	BreadcrumbItem,
@@ -81,7 +82,9 @@ const Mariadb = (
 							</h1>
 							<span className="text-sm">{data?.appName}</span>
 						</div>
-
+						<div>
+							<Badge>{data?.server?.name || "Dokploy Server"}</Badge>
+						</div>
 						{data?.description && (
 							<p className="text-sm text-muted-foreground  max-w-6xl">
 								{data?.description}

apps/dokploy/pages/dashboard/project/[projectId]/services/mongo/[mongoId].tsx

@@ -11,6 +11,7 @@ import { DockerMonitoring } from "@/components/dashboard/monitoring/docker/show"
 import { MongodbIcon } from "@/components/icons/data-tools-icons";
 import { ProjectLayout } from "@/components/layouts/project-layout";
 import { StatusTooltip } from "@/components/shared/status-tooltip";
+import { Badge } from "@/components/ui/badge";
 import {
 	Breadcrumb,
 	BreadcrumbItem,
@@ -82,7 +83,9 @@ const Mongo = (
 							</h1>
 							<span className="text-sm">{data?.appName}</span>
 						</div>
-
+						<div>
+							<Badge>{data?.server?.name || "Dokploy Server"}</Badge>
+						</div>
 						{data?.description && (
 							<p className="text-sm text-muted-foreground  max-w-6xl">
 								{data?.description}

apps/dokploy/pages/dashboard/project/[projectId]/services/mysql/[mysqlId].tsx

@@ -9,9 +9,9 @@ import { ShowGeneralMysql } from "@/components/dashboard/mysql/general/show-gene
 import { ShowInternalMysqlCredentials } from "@/components/dashboard/mysql/general/show-internal-mysql-credentials";
 import { UpdateMysql } from "@/components/dashboard/mysql/update-mysql";
 import { MysqlIcon } from "@/components/icons/data-tools-icons";
-import { DashboardLayout } from "@/components/layouts/dashboard-layout";
 import { ProjectLayout } from "@/components/layouts/project-layout";
 import { StatusTooltip } from "@/components/shared/status-tooltip";
+import { Badge } from "@/components/ui/badge";
 import {
 	Breadcrumb,
 	BreadcrumbItem,
@@ -81,7 +81,9 @@ const MySql = (
 							</h1>
 							<span className="text-sm">{data?.appName}</span>
 						</div>
-
+						<div>
+							<Badge>{data?.server?.name || "Dokploy Server"}</Badge>
+						</div>
 						{data?.description && (
 							<p className="text-sm text-muted-foreground  max-w-6xl">
 								{data?.description}

apps/dokploy/pages/dashboard/project/[projectId]/services/postgres/[postgresId].tsx

@@ -11,6 +11,7 @@ import { UpdatePostgres } from "@/components/dashboard/postgres/update-postgres"
 import { PostgresqlIcon } from "@/components/icons/data-tools-icons";
 import { ProjectLayout } from "@/components/layouts/project-layout";
 import { StatusTooltip } from "@/components/shared/status-tooltip";
+import { Badge } from "@/components/ui/badge";
 import {
 	Breadcrumb,
 	BreadcrumbItem,
@@ -81,7 +82,9 @@ const Postgresql = (
 							</h1>
 							<span className="text-sm">{data?.appName}</span>
 						</div>
-
+						<div>
+							<Badge>{data?.server?.name || "Dokploy Server"}</Badge>
+						</div>
 						{data?.description && (
 							<p className="text-sm text-muted-foreground  max-w-6xl">
 								{data?.description}

apps/dokploy/pages/dashboard/project/[projectId]/services/redis/[redisId].tsx

@@ -10,6 +10,7 @@ import { UpdateRedis } from "@/components/dashboard/redis/update-redis";
 import { RedisIcon } from "@/components/icons/data-tools-icons";
 import { ProjectLayout } from "@/components/layouts/project-layout";
 import { StatusTooltip } from "@/components/shared/status-tooltip";
+import { Badge } from "@/components/ui/badge";
 import {
 	Breadcrumb,
 	BreadcrumbItem,
@@ -80,7 +81,9 @@ const Redis = (
 							</h1>
 							<span className="text-sm">{data?.appName}</span>
 						</div>
-
+						<div>
+							<Badge>{data?.server?.name || "Dokploy Server"}</Badge>
+						</div>
 						{data?.description && (
 							<p className="text-sm text-muted-foreground  max-w-6xl">
 								{data?.description}

apps/dokploy/server/api/routers/backup.ts

@@ -5,22 +5,10 @@ import {
 	apiRemoveBackup,
 	apiUpdateBackup,
 } from "@/server/db/schema";
-import {
+import { runMariadbBackup } from "@/server/utils/backups/mariadb";
-	runMariadbBackup,
+import { runMongoBackup } from "@/server/utils/backups/mongo";
-	runRemoteMariadbBackup,
+import { runMySqlBackup } from "@/server/utils/backups/mysql";
-} from "@/server/utils/backups/mariadb";
+import { runPostgresBackup } from "@/server/utils/backups/postgres";
-import {
-	runMongoBackup,
-	runRemoteMongoBackup,
-} from "@/server/utils/backups/mongo";
-import {
-	runMySqlBackup,
-	runRemoteMySqlBackup,
-} from "@/server/utils/backups/mysql";
-import {
-	runPostgresBackup,
-	runRemotePostgresBackup,
-} from "@/server/utils/backups/postgres";
 import {
 	removeScheduleBackup,
 	scheduleBackup,
@@ -101,13 +89,7 @@ export const backupRouter = createTRPCRouter({
 			try {
 				const backup = await findBackupById(input.backupId);
 				const postgres = await findPostgresByBackupId(backup.backupId);
-
+				await runPostgresBackup(postgres, backup);
-				if (postgres.serverId) {
-					await runRemotePostgresBackup(postgres, backup);
-				} else {
-					await runPostgresBackup(postgres, backup);
-				}
-
 				return true;
 			} catch (error) {
 				console.log(error);
@@ -125,11 +107,7 @@ export const backupRouter = createTRPCRouter({
 			try {
 				const backup = await findBackupById(input.backupId);
 				const mysql = await findMySqlByBackupId(backup.backupId);
-				if (mysql.serverId) {
+				await runMySqlBackup(mysql, backup);
-					await runRemoteMySqlBackup(mysql, backup);
-				} else {
-					await runMySqlBackup(mysql, backup);
-				}
 				return true;
 			} catch (error) {
 				throw new TRPCError({
@@ -145,12 +123,7 @@ export const backupRouter = createTRPCRouter({
 			try {
 				const backup = await findBackupById(input.backupId);
 				const mariadb = await findMariadbByBackupId(backup.backupId);
-
+				await runMariadbBackup(mariadb, backup);
-				if (mariadb.serverId) {
-					await runRemoteMariadbBackup(mariadb, backup);
-				} else {
-					await runMariadbBackup(mariadb, backup);
-				}
 				return true;
 			} catch (error) {
 				throw new TRPCError({
@@ -166,12 +139,7 @@ export const backupRouter = createTRPCRouter({
 			try {
 				const backup = await findBackupById(input.backupId);
 				const mongo = await findMongoByBackupId(backup.backupId);
-
+				await runMongoBackup(mongo, backup);
-				if (mongo.serverId) {
-					await runRemoteMongoBackup(mongo, backup);
-				} else {
-					await runMongoBackup(mongo, backup);
-				}
 				return true;
 			} catch (error) {
 				throw new TRPCError({

apps/dokploy/server/api/routers/destination.ts

@@ -10,7 +10,6 @@ import {
 	apiRemoveDestination,
 	apiUpdateDestination,
 } from "@/server/db/schema";
-import { HeadBucketCommand, S3Client } from "@aws-sdk/client-s3";
 import { TRPCError } from "@trpc/server";
 import { findAdmin } from "../services/admin";
 import {
@@ -19,6 +18,7 @@ import {
 	removeDestinationById,
 	updateDestinationById,
 } from "../services/destination";
+import { execAsync } from "@/server/utils/process/execAsync";
 
 export const destinationRouter = createTRPCRouter({
 	create: adminProcedure
@@ -39,22 +39,22 @@ export const destinationRouter = createTRPCRouter({
 		.input(apiCreateDestination)
 		.mutation(async ({ input }) => {
 			const { secretAccessKey, bucket, region, endpoint, accessKey } = input;
-			const s3Client = new S3Client({
-				region: region,
-				...(endpoint && {
-					endpoint: endpoint,
-				}),
-				credentials: {
-					accessKeyId: accessKey,
-					secretAccessKey: secretAccessKey,
-				},
-				forcePathStyle: true,
-			});
-			const headBucketCommand = new HeadBucketCommand({ Bucket: bucket });
 
 			try {
-				await s3Client.send(headBucketCommand);
+				const rcloneFlags = [
+					// `--s3-provider=Cloudflare`,
+					`--s3-access-key-id=${accessKey}`,
+					`--s3-secret-access-key=${secretAccessKey}`,
+					`--s3-region=${region}`,
+					`--s3-endpoint=${endpoint}`,
+					"--s3-no-check-bucket",
+					"--s3-force-path-style",
+				];
+				const rcloneDestination = `:s3:${bucket}`;
+				const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+				await execAsync(rcloneCommand);
 			} catch (error) {
+				console.log(error);
 				throw new TRPCError({
 					code: "BAD_REQUEST",
 					message: "Error to connect to bucket",

apps/dokploy/server/utils/backups/mariadb.ts

@@ -1,4 +1,3 @@
-import { unlink } from "node:fs/promises";
 import path from "node:path";
 import type { BackupSchedule } from "@/server/api/services/backup";
 import type { Mariadb } from "@/server/api/services/mariadb";
@@ -9,7 +8,7 @@ import {
 } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { uploadToS3 } from "./utils";
+import { getS3Credentials } from "./utils";
 
 export const runMariadbBackup = async (
 	mariadb: Mariadb,
@@ -21,81 +20,30 @@ export const runMariadbBackup = async (
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.sql.gz`;
 	const bucketDestination = path.join(prefix, backupFileName);
-	const containerPath = `/backup/${backupFileName}`;
-	const hostPath = `./${backupFileName}`;
 
 	try {
-		const { Id: containerId } = await getServiceContainer(appName);
+		const rcloneFlags = getS3Credentials(destination);
-		await execAsync(
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
-			`docker exec ${containerId} sh -c "rm -rf /backup && mkdir -p /backup"`,
-		);
 
-		await execAsync(
-			`docker exec ${containerId} sh -c "mariadb-dump --user='${databaseUser}' --password='${databasePassword}' --databases ${database} | gzip  > ${containerPath}"`,
-		);
-		await execAsync(
-			`docker cp ${containerId}:/backup/${backupFileName} ${hostPath}`,
-		);
-		await uploadToS3(destination, bucketDestination, hostPath);
-
-		await sendDatabaseBackupNotifications({
-			applicationName: name,
-			projectName: project.name,
-			databaseType: "mariadb",
-			type: "success",
-		});
-	} catch (error) {
-		console.log(error);
-		await sendDatabaseBackupNotifications({
-			applicationName: name,
-			projectName: project.name,
-			databaseType: "mariadb",
-			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
-		});
-		throw error;
-	} finally {
-		await unlink(hostPath);
-	}
-};
-
-export const runRemoteMariadbBackup = async (
-	mariadb: Mariadb,
-	backup: BackupSchedule,
-) => {
-	const { appName, databasePassword, databaseUser, projectId, name, serverId } =
-		mariadb;
-
-	if (!serverId) {
-		throw new Error("Server ID not provided");
-	}
-	const project = await findProjectById(projectId);
-	const { prefix, database } = backup;
-	const destination = backup.destination;
-	const backupFileName = `${new Date().toISOString()}.sql.gz`;
-	const bucketDestination = path.join(prefix, backupFileName);
-	const { accessKey, secretAccessKey, bucket, region, endpoint } = destination;
-
-	try {
-		const { Id: containerId } = await getRemoteServiceContainer(
-			serverId,
-			appName,
-		);
-		const mariadbDumpCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${databaseUser}' --password='${databasePassword}' --databases ${database} | gzip"`;
-		const rcloneFlags = [
-			`--s3-access-key-id=${accessKey}`,
-			`--s3-secret-access-key=${secretAccessKey}`,
-			`--s3-region=${region}`,
-			`--s3-endpoint=${endpoint}`,
-			"--s3-no-check-bucket",
-			"--s3-force-path-style",
-		];
-
-		const rcloneDestination = `:s3:${bucket}/${bucketDestination}`;
 		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+		if (mariadb.serverId) {
+			const { Id: containerId } = await getRemoteServiceContainer(
+				mariadb.serverId,
+				appName,
+			);
+			const mariadbDumpCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${databaseUser}' --password='${databasePassword}' --databases ${database} | gzip"`;
+
+			await execAsyncRemote(
+				mariadb.serverId,
+				`${mariadbDumpCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { Id: containerId } = await getServiceContainer(appName);
+			const mariadbDumpCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${databaseUser}' --password='${databasePassword}' --databases ${database} | gzip"`;
+
+			await execAsync(`${mariadbDumpCommand} | ${rcloneCommand}`);
+		}
 
-		await execAsyncRemote(serverId, `${mariadbDumpCommand} | ${rcloneCommand}`);
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
 			projectName: project.name,

apps/dokploy/server/utils/backups/mongo.ts

@@ -1,4 +1,3 @@
-import { unlink } from "node:fs/promises";
 import path from "node:path";
 import type { BackupSchedule } from "@/server/api/services/backup";
 import type { Mongo } from "@/server/api/services/mongo";
@@ -9,7 +8,7 @@ import {
 } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { uploadToS3 } from "./utils";
+import { getS3Credentials } from "./utils";
 
 // mongodb://mongo:Bqh7AQl-PRbnBu@localhost:27017/?tls=false&directConnection=true
 export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
@@ -19,20 +18,28 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.dump.gz`;
 	const bucketDestination = path.join(prefix, backupFileName);
-	const containerPath = `/backup/${backupFileName}`;
-	const hostPath = `./${backupFileName}`;
 
 	try {
-		const { Id: containerId } = await getServiceContainer(appName);
+		const rcloneFlags = getS3Credentials(destination);
-		await execAsync(
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
-			`docker exec ${containerId} sh -c "rm -rf /backup && mkdir -p /backup"`,
-		);
 
-		await execAsync(
+		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
-			`docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${databaseUser}' -p '${databasePassword}' --authenticationDatabase=admin --archive=${containerPath} --gzip"`,
+		if (mongo.serverId) {
-		);
+			const { Id: containerId } = await getRemoteServiceContainer(
-		await execAsync(`docker cp ${containerId}:${containerPath} ${hostPath}`);
+				mongo.serverId,
-		await uploadToS3(destination, bucketDestination, hostPath);
+				appName,
+			);
+			const mongoDumpCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${databaseUser}' -p '${databasePassword}' --authenticationDatabase=admin --gzip"`;
+
+			await execAsyncRemote(
+				mongo.serverId,
+				`${mongoDumpCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { Id: containerId } = await getServiceContainer(appName);
+			const mongoDumpCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${databaseUser}' -p '${databasePassword}' --authenticationDatabase=admin --gzip"`;
+			await execAsync(`${mongoDumpCommand} | ${rcloneCommand}`);
+		}
 
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
@@ -51,64 +58,6 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
 			errorMessage: error?.message || "Error message not provided",
 		});
 		throw error;
-	} finally {
-		await unlink(hostPath);
 	}
 };
 // mongorestore -d monguito -u mongo -p Bqh7AQl-PRbnBu --authenticationDatabase admin --gzip --archive=2024-04-13T05:03:58.937Z.dump.gz
-
-export const runRemoteMongoBackup = async (
-	mongo: Mongo,
-	backup: BackupSchedule,
-) => {
-	const { appName, databasePassword, databaseUser, projectId, name, serverId } =
-		mongo;
-
-	if (!serverId) {
-		throw new Error("Server ID not provided");
-	}
-	const project = await findProjectById(projectId);
-	const { prefix, database } = backup;
-	const destination = backup.destination;
-	const backupFileName = `${new Date().toISOString()}.dump.gz`;
-	const bucketDestination = path.join(prefix, backupFileName);
-	const { accessKey, secretAccessKey, bucket, region, endpoint } = destination;
-
-	try {
-		const { Id: containerId } = await getRemoteServiceContainer(
-			serverId,
-			appName,
-		);
-		const mongoDumpCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${databaseUser}' -p '${databasePassword}' --authenticationDatabase=admin --gzip"`;
-		const rcloneFlags = [
-			`--s3-access-key-id=${accessKey}`,
-			`--s3-secret-access-key=${secretAccessKey}`,
-			`--s3-region=${region}`,
-			`--s3-endpoint=${endpoint}`,
-			"--s3-no-check-bucket",
-			"--s3-force-path-style",
-		];
-
-		const rcloneDestination = `:s3:${bucket}/${bucketDestination}`;
-		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
-
-		await execAsyncRemote(serverId, `${mongoDumpCommand} | ${rcloneCommand}`);
-		await sendDatabaseBackupNotifications({
-			applicationName: name,
-			projectName: project.name,
-			databaseType: "mongodb",
-			type: "success",
-		});
-	} catch (error) {
-		console.log(error);
-		await sendDatabaseBackupNotifications({
-			applicationName: name,
-			projectName: project.name,
-			databaseType: "mongodb",
-			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
-		});
-		throw error;
-	}
-};

apps/dokploy/server/utils/backups/mysql.ts

@@ -9,7 +9,7 @@ import {
 } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { uploadToS3 } from "./utils";
+import { getS3Credentials } from "./utils";
 
 export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
 	const { appName, databaseRootPassword, projectId, name } = mysql;
@@ -18,81 +18,29 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.sql.gz`;
 	const bucketDestination = path.join(prefix, backupFileName);
-	const containerPath = `/backup/${backupFileName}`;
-	const hostPath = `./${backupFileName}`;
 
 	try {
-		const { Id: containerId } = await getServiceContainer(appName);
+		const rcloneFlags = getS3Credentials(destination);
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
 
-		await execAsync(
-			`docker exec ${containerId} sh -c "rm -rf /backup && mkdir -p /backup"`,
-		);
-
-		await execAsync(
-			`docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}'  | gzip > ${containerPath}"`,
-		);
-		await execAsync(
-			`docker cp ${containerId}:/backup/${backupFileName} ${hostPath}`,
-		);
-		await uploadToS3(destination, bucketDestination, hostPath);
-		await sendDatabaseBackupNotifications({
-			applicationName: name,
-			projectName: project.name,
-			databaseType: "mysql",
-			type: "success",
-		});
-	} catch (error) {
-		console.log(error);
-		await sendDatabaseBackupNotifications({
-			applicationName: name,
-			projectName: project.name,
-			databaseType: "mysql",
-			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
-		});
-		throw error;
-	} finally {
-		await unlink(hostPath);
-	}
-};
-
-export const runRemoteMySqlBackup = async (
-	mysql: MySql,
-	backup: BackupSchedule,
-) => {
-	const { appName, databaseRootPassword, projectId, name, serverId } = mysql;
-
-	if (!serverId) {
-		throw new Error("Server ID not provided");
-	}
-	const project = await findProjectById(projectId);
-	const { prefix, database } = backup;
-	const destination = backup.destination;
-	const backupFileName = `${new Date().toISOString()}.sql.gz`;
-	const bucketDestination = path.join(prefix, backupFileName);
-	const { accessKey, secretAccessKey, bucket, region, endpoint } = destination;
-
-	try {
-		const { Id: containerId } = await getRemoteServiceContainer(
-			serverId,
-			appName,
-		);
-
-		const mysqlDumpCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
-		const rcloneFlags = [
-			`--s3-access-key-id=${accessKey}`,
-			`--s3-secret-access-key=${secretAccessKey}`,
-			`--s3-region=${region}`,
-			`--s3-endpoint=${endpoint}`,
-			"--s3-no-check-bucket",
-			"--s3-force-path-style",
-		];
-
-		const rcloneDestination = `:s3:${bucket}/${bucketDestination}`;
 		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+		if (mysql.serverId) {
+			const { Id: containerId } = await getRemoteServiceContainer(
+				mysql.serverId,
+				appName,
+			);
+			const mysqlDumpCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
 
-		await execAsyncRemote(serverId, `${mysqlDumpCommand} | ${rcloneCommand}`);
+			await execAsyncRemote(
+				mysql.serverId,
+				`${mysqlDumpCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { Id: containerId } = await getServiceContainer(appName);
+			const mysqlDumpCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
+
+			await execAsync(`${mysqlDumpCommand} | ${rcloneCommand}`);
+		}
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
 			projectName: project.name,

apps/dokploy/server/utils/backups/postgres.ts

@@ -1,4 +1,3 @@
-import { unlink } from "node:fs/promises";
 import path from "node:path";
 import type { BackupSchedule } from "@/server/api/services/backup";
 import type { Postgres } from "@/server/api/services/postgres";
@@ -9,7 +8,7 @@ import {
 } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { uploadToS3 } from "./utils";
+import { getS3Credentials } from "./utils";
 
 export const runPostgresBackup = async (
 	postgres: Postgres,
@@ -22,20 +21,29 @@ export const runPostgresBackup = async (
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.sql.gz`;
 	const bucketDestination = path.join(prefix, backupFileName);
-	const containerPath = `/backup/${backupFileName}`;
-	const hostPath = `./${backupFileName}`;
 	try {
-		const { Id: containerId } = await getServiceContainer(appName);
+		const rcloneFlags = getS3Credentials(destination);
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
 
-		await execAsync(
+		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
-			`docker exec ${containerId} /bin/bash -c "rm -rf /backup && mkdir -p /backup"`,
+		if (postgres.serverId) {
-		);
+			const { Id: containerId } = await getRemoteServiceContainer(
-		await execAsync(
+				postgres.serverId,
-			`docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password  '${database}' | gzip > ${containerPath}"`,
+				appName,
-		);
+			);
-		await execAsync(`docker cp ${containerId}:${containerPath} ${hostPath}`);
+			const pgDumpCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip"`;
+
+			await execAsyncRemote(
+				postgres.serverId,
+				`${pgDumpCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { Id: containerId } = await getServiceContainer(appName);
+
+			const pgDumpCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip"`;
+			await execAsync(`${pgDumpCommand} | ${rcloneCommand}`);
+		}
 
-		await uploadToS3(destination, bucketDestination, hostPath);
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
 			projectName: project.name,
@@ -54,72 +62,8 @@ export const runPostgresBackup = async (
 
 		throw error;
 	} finally {
-		await unlink(hostPath);
 	}
 };
 
 // Restore
 // /Applications/pgAdmin 4.app/Contents/SharedSupport/pg_restore --host "localhost" --port "5432" --username "mauricio" --no-password --dbname "postgres" --verbose "/Users/mauricio/Downloads/_databases_2024-04-12T07_02_05.234Z.sql"
-
-export const runRemotePostgresBackup = async (
-	postgres: Postgres,
-	backup: BackupSchedule,
-) => {
-	const { appName, databaseUser, name, projectId, serverId } = postgres;
-
-	if (!serverId) {
-		throw new Error("Server ID not provided");
-	}
-	const project = await findProjectById(projectId);
-
-	const { prefix, database } = backup;
-	const destination = backup.destination;
-	const backupFileName = `${new Date().toISOString()}.sql.gz`;
-	const bucketDestination = path.join(prefix, backupFileName);
-	const { accessKey, secretAccessKey, bucket, region, endpoint } = destination;
-
-	try {
-		const { Id: containerId } = await getRemoteServiceContainer(
-			serverId,
-			appName,
-		);
-		const pgDumpCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip"`;
-		const rcloneFlags = [
-			// `--s3-provider=Cloudflare`,
-			`--s3-access-key-id=${accessKey}`,
-			`--s3-secret-access-key=${secretAccessKey}`,
-			`--s3-region=${region}`,
-			`--s3-endpoint=${endpoint}`,
-			"--s3-no-check-bucket",
-			"--s3-force-path-style",
-		];
-
-		const rcloneDestination = `:s3:${bucket}/${bucketDestination}`;
-		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
-
-		console.log(`${pgDumpCommand} | ${rcloneCommand}`);
-		await execAsyncRemote(
-			postgres.serverId,
-			`${pgDumpCommand} | ${rcloneCommand}`,
-		);
-		await sendDatabaseBackupNotifications({
-			applicationName: name,
-			projectName: project.name,
-			databaseType: "postgres",
-			type: "success",
-		});
-	} catch (error) {
-		console.log(error);
-		await sendDatabaseBackupNotifications({
-			applicationName: name,
-			projectName: project.name,
-			databaseType: "postgres",
-			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
-		});
-
-		throw error;
-	} finally {
-	}
-};

apps/dokploy/server/utils/backups/utils.ts

@@ -1,67 +1,23 @@
-import { readFile } from "node:fs/promises";
 import type { BackupSchedule } from "@/server/api/services/backup";
 import type { Destination } from "@/server/api/services/destination";
-import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
 import { scheduleJob, scheduledJobs } from "node-schedule";
-import { runMariadbBackup, runRemoteMariadbBackup } from "./mariadb";
+import { runMariadbBackup } from "./mariadb";
-import { runMongoBackup, runRemoteMongoBackup } from "./mongo";
+import { runMySqlBackup } from "./mysql";
-import { runMySqlBackup, runRemoteMySqlBackup } from "./mysql";
+import { runPostgresBackup } from "./postgres";
-import { runPostgresBackup, runRemotePostgresBackup } from "./postgres";
+import { runMongoBackup } from "./mongo";
 
-export const uploadToS3 = async (
-	destination: Destination,
-	destinationBucketPath: string,
-	filePath: string,
-) => {
-	const { accessKey, secretAccessKey, bucket, region, endpoint } = destination;
-
-	const s3Client = new S3Client({
-		region: region,
-		endpoint: endpoint,
-		credentials: {
-			accessKeyId: accessKey,
-			secretAccessKey: secretAccessKey,
-		},
-		forcePathStyle: true,
-	});
-
-	const fileContent = await readFile(filePath);
-	const command = new PutObjectCommand({
-		Bucket: bucket,
-		Key: destinationBucketPath,
-		Body: fileContent,
-	});
-
-	await s3Client.send(command);
-};
 export const scheduleBackup = (backup: BackupSchedule) => {
 	const { schedule, backupId, databaseType, postgres, mysql, mongo, mariadb } =
 		backup;
 	scheduleJob(backupId, schedule, async () => {
 		if (databaseType === "postgres" && postgres) {
-			if (postgres.serverId) {
+			await runPostgresBackup(postgres, backup);
-				await runRemotePostgresBackup(postgres, backup);
-			} else {
-				await runPostgresBackup(postgres, backup);
-			}
 		} else if (databaseType === "mysql" && mysql) {
-			if (mysql.serverId) {
+			await runMySqlBackup(mysql, backup);
-				await runRemoteMySqlBackup(mysql, backup);
-			} else {
-				await runMySqlBackup(mysql, backup);
-			}
 		} else if (databaseType === "mongo" && mongo) {
-			if (mongo.serverId) {
+			await runMongoBackup(mongo, backup);
-				await runRemoteMongoBackup(mongo, backup);
-			} else {
-				await runMongoBackup(mongo, backup);
-			}
 		} else if (databaseType === "mariadb" && mariadb) {
-			if (mariadb.serverId) {
+			await runMariadbBackup(mariadb, backup);
-				await runRemoteMariadbBackup(mariadb, backup);
-			} else {
-				await runMariadbBackup(mariadb, backup);
-			}
 		}
 	});
 };
@@ -70,3 +26,18 @@ export const removeScheduleBackup = (backupId: string) => {
 	const currentJob = scheduledJobs[backupId];
 	currentJob?.cancel();
 };
+
+export const getS3Credentials = (destination: Destination) => {
+	const { accessKey, secretAccessKey, bucket, region, endpoint } = destination;
+	const rcloneFlags = [
+		// `--s3-provider=Cloudflare`,
+		`--s3-access-key-id=${accessKey}`,
+		`--s3-secret-access-key=${secretAccessKey}`,
+		`--s3-region=${region}`,
+		`--s3-endpoint=${endpoint}`,
+		"--s3-no-check-bucket",
+		"--s3-force-path-style",
+	];
+
+	return rcloneFlags;
+};