feat(server): function to keep only the latest N backups

apps/schedules/src/utils.ts

@@ -4,10 +4,11 @@ import {
 	cleanUpUnusedImages,
 	findBackupById,
 	findServerById,
+	keepLatestNBackups,
 	runMariadbBackup,
 	runMongoBackup,
 	runMySqlBackup,
-	runPostgresBackup,
+	runPostgresBackup
 } from "@dokploy/server";
 import { db } from "@dokploy/server/dist/db";
 import { backups, server } from "@dokploy/server/dist/db/schema";
@@ -52,6 +53,8 @@ export const runJobs = async (job: QueueJob) => {
 				}
 				await runMariadbBackup(mariadb, backup);
 			}
+
+			await keepLatestNBackups(backup);
 		}
 		if (job.type === "server") {
 			const { serverId } = job;

packages/server/src/utils/backups/index.ts

@@ -1,3 +1,4 @@
+import path from "node:path";
 import { getAllServers } from "@dokploy/server/services/server";
 import { scheduleJob } from "node-schedule";
 import { db } from "../../db/index";
@@ -12,6 +13,10 @@ import { runMongoBackup } from "./mongo";
 import { runMySqlBackup } from "./mysql";
 import { runPostgresBackup } from "./postgres";
 import { findAdmin } from "../../services/admin";
+import { getS3Credentials } from "./utils";
+import { execAsync } from "../process/execAsync";
+
+import type { BackupSchedule } from "@dokploy/server/services/backup";
 
 export const initCronJobs = async () => {
 	console.log("Setting up cron jobs....");
@@ -169,3 +174,30 @@ export const initCronJobs = async () => {
 		}
 	}
 };
+
+export const keepLatestNBackups = async (backup: BackupSchedule) => {
+	// 0 also immediately returns which is good as the empty "keep latest" field in the UI
+	// is saved as 0 in the database
+	if (!backup.keepLatestCount) return;
+
+	try {
+		const rcloneFlags = getS3Credentials(backup.destination);
+		const backupFilesPath = path.join(`:s3:${backup.destination.bucket}`, backup.prefix);
+
+		// --include "*.sql.gz" ensures nothing else other than the db backup files are touched by rclone
+		const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*.sql.gz" ${backupFilesPath}`;
+		// when we pipe the above command with this one, we only get the list of files we want to delete
+		const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`;
+		// this command deletes the files
+		// to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}/{}
+		const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}/{}`;
+
+		const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`;
+
+		// we can execute this command on any server it doesn't matter
+		await execAsync(rcloneCommand);
+	} catch (error) {
+		console.error(error);
+		throw error;
+	}
+};

packages/server/src/utils/backups/utils.ts

@@ -5,6 +5,7 @@ import { runMariadbBackup } from "./mariadb";
 import { runMongoBackup } from "./mongo";
 import { runMySqlBackup } from "./mysql";
 import { runPostgresBackup } from "./postgres";
+import { keepLatestNBackups } from ".";
 
 export const scheduleBackup = (backup: BackupSchedule) => {
 	const { schedule, backupId, databaseType, postgres, mysql, mongo, mariadb } =
@@ -19,6 +20,8 @@ export const scheduleBackup = (backup: BackupSchedule) => {
 		} else if (databaseType === "mariadb" && mariadb) {
 			await runMariadbBackup(mariadb, backup);
 		}
+
+		await keepLatestNBackups(backup);
 	});
 };