Merge pull request #1446 from Dokploy/feat/latest-n-backups

Feat/latest n backups

apps/dokploy/components/dashboard/database/backups/add-backup.tsx

@@ -54,6 +54,7 @@ const AddPostgresBackup1Schema = z.object({
 	prefix: z.string().min(1, "Prefix required"),
 	enabled: z.boolean(),
 	database: z.string().min(1, "Database required"),
+	keepLatestCount: z.coerce.number().optional(),
 });
 
 type AddPostgresBackup = z.infer<typeof AddPostgresBackup1Schema>;
@@ -77,6 +78,7 @@ export const AddBackup = ({ databaseId, databaseType, refetch }: Props) => {
 			enabled: true,
 			prefix: "/",
 			schedule: "",
+			keepLatestCount: undefined,
 		},
 		resolver: zodResolver(AddPostgresBackup1Schema),
 	});
@@ -88,6 +90,7 @@ export const AddBackup = ({ databaseId, databaseType, refetch }: Props) => {
 			enabled: true,
 			prefix: "/",
 			schedule: "",
+			keepLatestCount: undefined,
 		});
 	}, [form, form.reset, form.formState.isSubmitSuccessful]);
 
@@ -117,6 +120,7 @@ export const AddBackup = ({ databaseId, databaseType, refetch }: Props) => {
 			schedule: data.schedule,
 			enabled: data.enabled,
 			database: data.database,
+			keepLatestCount: data.keepLatestCount,
 			databaseType,
 			...getDatabaseId,
 		})
@@ -265,7 +269,7 @@ export const AddBackup = ({ databaseId, databaseType, refetch }: Props) => {
 												<Input placeholder={"dokploy/"} {...field} />
 											</FormControl>
 											<FormDescription>
-												Use if you want to storage in a specific path of your
+												Use if you want to back up in a specific path of your
 												destination/bucket
 											</FormDescription>
 
@@ -274,6 +278,24 @@ export const AddBackup = ({ databaseId, databaseType, refetch }: Props) => {
 									);
 								}}
 							/>
+							<FormField
+								control={form.control}
+								name="keepLatestCount"
+								render={({ field }) => {
+									return (
+										<FormItem>
+											<FormLabel>Keep the latest</FormLabel>
+											<FormControl>
+												<Input type="number" placeholder={"keeps all the backups if left empty"} {...field} />
+											</FormControl>
+											<FormDescription>
+												Optional. If provided, only keeps the latest N backups in the cloud.
+											</FormDescription>
+											<FormMessage />
+										</FormItem>
+									);
+								}}
+							/>							
 							<FormField
 								control={form.control}
 								name="enabled"

apps/dokploy/components/dashboard/database/backups/show-backups.tsx

@@ -20,12 +20,14 @@ import { toast } from "sonner";
 import type { ServiceType } from "../../application/advanced/show-resources";
 import { AddBackup } from "./add-backup";
 import { UpdateBackup } from "./update-backup";
+import { useState } from "react";
 
 interface Props {
 	id: string;
 	type: Exclude<ServiceType, "application" | "redis">;
 }
 export const ShowBackups = ({ id, type }: Props) => {
+	const [activeManualBackup, setActiveManualBackup] = useState<string | undefined>();
 	const queryMap = {
 		postgres: () =>
 			api.postgres.one.useQuery({ postgresId: id }, { enabled: !!id }),
@@ -106,7 +108,7 @@ export const ShowBackups = ({ id, type }: Props) => {
 									{postgres?.backups.map((backup) => (
 										<div key={backup.backupId}>
 											<div className="flex w-full flex-col md:flex-row md:items-center justify-between gap-4 md:gap-10 border rounded-lg p-4">
-												<div className="grid grid-cols-1 md:grid-cols-3 xl:grid-cols-5 flex-col gap-8">
+												<div className="grid grid-cols-1 md:grid-cols-3 xl:grid-cols-6 flex-col gap-8">
 													<div className="flex flex-col gap-1">
 														<span className="font-medium">Destination</span>
 														<span className="text-sm text-muted-foreground">
@@ -137,6 +139,12 @@ export const ShowBackups = ({ id, type }: Props) => {
 															{backup.enabled ? "Yes" : "No"}
 														</span>
 													</div>
+													<div className="flex flex-col gap-1">
+														<span className="font-medium">Keep Latest</span>
+														<span className="text-sm text-muted-foreground">
+															{backup.keepLatestCount || 'All'}
+														</span>
+													</div>
 												</div>
 												<div className="flex flex-row gap-4">
 													<TooltipProvider delayDuration={0}>
@@ -145,8 +153,9 @@ export const ShowBackups = ({ id, type }: Props) => {
 																<Button
 																	type="button"
 																	variant="ghost"
-																	isLoading={isManualBackup}
+																	isLoading={isManualBackup && activeManualBackup === backup.backupId}
 																	onClick={async () => {
+																		setActiveManualBackup(backup.backupId);
 																		await manualBackup({
 																			backupId: backup.backupId as string,
 																		})
@@ -160,6 +169,7 @@ export const ShowBackups = ({ id, type }: Props) => {
 																					"Error creating the manual backup",
 																				);
 																			});
+																		setActiveManualBackup(undefined);
 																	}}
 																>
 																	<Play className="size-5  text-muted-foreground" />

apps/dokploy/components/dashboard/database/backups/update-backup.tsx

@@ -47,6 +47,7 @@ const UpdateBackupSchema = z.object({
 	prefix: z.string().min(1, "Prefix required"),
 	enabled: z.boolean(),
 	database: z.string().min(1, "Database required"),
+	keepLatestCount: z.coerce.number().optional(),
 });
 
 type UpdateBackup = z.infer<typeof UpdateBackupSchema>;
@@ -78,6 +79,7 @@ export const UpdateBackup = ({ backupId, refetch }: Props) => {
 			enabled: true,
 			prefix: "/",
 			schedule: "",
+			keepLatestCount: undefined,
 		},
 		resolver: zodResolver(UpdateBackupSchema),
 	});
@@ -90,6 +92,7 @@ export const UpdateBackup = ({ backupId, refetch }: Props) => {
 				enabled: backup.enabled || false,
 				prefix: backup.prefix,
 				schedule: backup.schedule,
+				keepLatestCount: backup.keepLatestCount ? Number(backup.keepLatestCount) : undefined,
 			});
 		}
 	}, [form, form.reset, backup]);
@@ -102,6 +105,7 @@ export const UpdateBackup = ({ backupId, refetch }: Props) => {
 			schedule: data.schedule,
 			enabled: data.enabled,
 			database: data.database,
+			keepLatestCount: data.keepLatestCount as number | null,
 		})
 			.then(async () => {
 				toast.success("Backup Updated");
@@ -253,7 +257,7 @@ export const UpdateBackup = ({ backupId, refetch }: Props) => {
 												<Input placeholder={"dokploy/"} {...field} />
 											</FormControl>
 											<FormDescription>
-												Use if you want to storage in a specific path of your
+												Use if you want to back up in a specific path of your
 												destination/bucket
 											</FormDescription>
 
@@ -262,6 +266,24 @@ export const UpdateBackup = ({ backupId, refetch }: Props) => {
 									);
 								}}
 							/>
+							<FormField
+								control={form.control}
+								name="keepLatestCount"
+								render={({ field }) => {
+									return (
+										<FormItem>
+											<FormLabel>Keep the latest</FormLabel>
+											<FormControl>
+												<Input type="number" placeholder={"keeps all the backups if left empty"} {...field} />
+											</FormControl>
+											<FormDescription>
+												Optional. If provided, only keeps the latest N backups in the cloud.
+											</FormDescription>
+											<FormMessage />
+										</FormItem>
+									);
+								}}
+							/>
 							<FormField
 								control={form.control}
 								name="enabled"

apps/dokploy/drizzle/0077_chemical_dreadnoughts.sql

@@ -0,0 +1 @@
+ALTER TABLE "backup" ADD COLUMN "keepLatestCount" integer;

apps/dokploy/drizzle/meta/_journal.json

@@ -540,6 +540,13 @@
       "when": 1741493754270,
       "tag": "0076_young_sharon_ventura",
       "breakpoints": true
+    },
+    {
+      "idx": 77,
+      "version": "7",
+      "when": 1741510086231,
+      "tag": "0077_chemical_dreadnoughts",
+      "breakpoints": true
     }
   ]
 }

apps/schedules/src/utils.ts

@@ -4,6 +4,7 @@ import {
 	cleanUpUnusedImages,
 	findBackupById,
 	findServerById,
+	keepLatestNBackups,
 	runMariadbBackup,
 	runMongoBackup,
 	runMySqlBackup,
@@ -30,6 +31,7 @@ export const runJobs = async (job: QueueJob) => {
 					return;
 				}
 				await runPostgresBackup(postgres, backup);
+				await keepLatestNBackups(backup, server.serverId);
 			} else if (databaseType === "mysql" && mysql) {
 				const server = await findServerById(mysql.serverId as string);
 				if (server.serverStatus === "inactive") {
@@ -37,6 +39,7 @@ export const runJobs = async (job: QueueJob) => {
 					return;
 				}
 				await runMySqlBackup(mysql, backup);
+				await keepLatestNBackups(backup, server.serverId);
 			} else if (databaseType === "mongo" && mongo) {
 				const server = await findServerById(mongo.serverId as string);
 				if (server.serverStatus === "inactive") {
@@ -44,6 +47,7 @@ export const runJobs = async (job: QueueJob) => {
 					return;
 				}
 				await runMongoBackup(mongo, backup);
+				await keepLatestNBackups(backup, server.serverId);
 			} else if (databaseType === "mariadb" && mariadb) {
 				const server = await findServerById(mariadb.serverId as string);
 				if (server.serverStatus === "inactive") {
@@ -51,6 +55,7 @@ export const runJobs = async (job: QueueJob) => {
 					return;
 				}
 				await runMariadbBackup(mariadb, backup);
+				await keepLatestNBackups(backup, server.serverId);
 			}
 		}
 		if (job.type === "server") {

packages/server/src/db/schema/backups.ts

@@ -2,6 +2,7 @@ import { relations } from "drizzle-orm";
 import {
 	type AnyPgColumn,
 	boolean,
+	integer,
 	pgEnum,
 	pgTable,
 	text,
@@ -36,6 +37,8 @@ export const backups = pgTable("backup", {
 		.notNull()
 		.references(() => destinations.destinationId, { onDelete: "cascade" }),
 
+	keepLatestCount: integer("keepLatestCount"),
+
 	databaseType: databaseType("databaseType").notNull(),
 	postgresId: text("postgresId").references(
 		(): AnyPgColumn => postgres.postgresId,
@@ -87,6 +90,7 @@ const createSchema = createInsertSchema(backups, {
 	prefix: z.string().min(1),
 	database: z.string().min(1),
 	schedule: z.string(),
+	keepLatestCount: z.number().optional(),
 	databaseType: z.enum(["postgres", "mariadb", "mysql", "mongo"]),
 	postgresId: z.string().optional(),
 	mariadbId: z.string().optional(),
@@ -99,6 +103,7 @@ export const apiCreateBackup = createSchema.pick({
 	enabled: true,
 	prefix: true,
 	destinationId: true,
+	keepLatestCount: true,
 	database: true,
 	mariadbId: true,
 	mysqlId: true,
@@ -127,5 +132,6 @@ export const apiUpdateBackup = createSchema
 		backupId: true,
 		destinationId: true,
 		database: true,
+		keepLatestCount: true,
 	})
 	.required();

packages/server/src/utils/backups/index.ts

@@ -1,3 +1,4 @@
+import path from "node:path";
 import { getAllServers } from "@dokploy/server/services/server";
 import { scheduleJob } from "node-schedule";
 import { db } from "../../db/index";
@@ -12,6 +13,10 @@ import { runMongoBackup } from "./mongo";
 import { runMySqlBackup } from "./mysql";
 import { runPostgresBackup } from "./postgres";
 import { findAdmin } from "../../services/admin";
+import { getS3Credentials } from "./utils";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+
+import type { BackupSchedule } from "@dokploy/server/services/backup";
 import { startLogCleanup } from "../access-log/handler";
 
 export const initCronJobs = async () => {
@@ -174,3 +179,39 @@ export const initCronJobs = async () => {
 		await startLogCleanup(admin.user.logCleanupCron);
 	}
 };
+
+export const keepLatestNBackups = async (
+	backup: BackupSchedule,
+	serverId?: string | null,
+) => {
+	// 0 also immediately returns which is good as the empty "keep latest" field in the UI
+	// is saved as 0 in the database
+	if (!backup.keepLatestCount) return;
+
+	try {
+		const rcloneFlags = getS3Credentials(backup.destination);
+		const backupFilesPath = path.join(
+			`:s3:${backup.destination.bucket}`,
+			backup.prefix,
+		);
+
+		// --include "*.sql.gz" ensures nothing else other than the db backup files are touched by rclone
+		const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*.sql.gz" ${backupFilesPath}`;
+		// when we pipe the above command with this one, we only get the list of files we want to delete
+		const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`;
+		// this command deletes the files
+		// to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}/{}
+		const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}/{}`;
+
+		const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`;
+
+		if (serverId) {
+			await execAsyncRemote(serverId, rcloneCommand);
+		} else {
+			await execAsync(rcloneCommand);
+		}
+	} catch (error) {
+		console.error(error);
+		throw error;
+	}
+};

packages/server/src/utils/backups/utils.ts

@@ -5,6 +5,7 @@ import { runMariadbBackup } from "./mariadb";
 import { runMongoBackup } from "./mongo";
 import { runMySqlBackup } from "./mysql";
 import { runPostgresBackup } from "./postgres";
+import { keepLatestNBackups } from ".";
 
 export const scheduleBackup = (backup: BackupSchedule) => {
 	const { schedule, backupId, databaseType, postgres, mysql, mongo, mariadb } =
@@ -12,12 +13,16 @@ export const scheduleBackup = (backup: BackupSchedule) => {
 	scheduleJob(backupId, schedule, async () => {
 		if (databaseType === "postgres" && postgres) {
 			await runPostgresBackup(postgres, backup);
+			await keepLatestNBackups(backup, postgres.serverId);
 		} else if (databaseType === "mysql" && mysql) {
 			await runMySqlBackup(mysql, backup);
+			await keepLatestNBackups(backup, mysql.serverId);
 		} else if (databaseType === "mongo" && mongo) {
 			await runMongoBackup(mongo, backup);
+			await keepLatestNBackups(backup, mongo.serverId);
 		} else if (databaseType === "mariadb" && mariadb) {
 			await runMariadbBackup(mariadb, backup);
+			await keepLatestNBackups(backup, mariadb.serverId);
 		}
 	});
 };