feat: initial commit

server/utils/backups/index.ts

@@ -0,0 +1,140 @@
+import { scheduleJob } from "node-schedule";
+import { db } from "../../db/index";
+import { runMariadbBackup } from "./mariadb";
+import { runMongoBackup } from "./mongo";
+import { runMySqlBackup } from "./mysql";
+import { runPostgresBackup } from "./postgres";
+import {
+	cleanUpDockerBuilder,
+	cleanUpSystemPrune,
+	cleanUpUnusedImages,
+} from "../docker/utils";
+import { findAdmin } from "@/server/api/services/admin";
+
+export const initCronJobs = async () => {
+	console.log("Setting up cron jobs....");
+
+	const admin = await findAdmin();
+
+	if (admin?.enableDockerCleanup) {
+		scheduleJob("docker-cleanup", "0 0 * * *", async () => {
+			console.log(
+				`Docker Cleanup ${new Date().toLocaleString()}]  Running docker cleanup`,
+			);
+			await cleanUpUnusedImages();
+			await cleanUpDockerBuilder();
+			await cleanUpSystemPrune();
+		});
+	}
+
+	const pgs = await db.query.postgres.findMany({
+		with: {
+			backups: {
+				with: {
+					destination: true,
+					postgres: true,
+					mariadb: true,
+					mysql: true,
+					mongo: true,
+				},
+			},
+		},
+	});
+	for (const pg of pgs) {
+		for (const backup of pg.backups) {
+			const { schedule, backupId, enabled } = backup;
+			if (enabled) {
+				scheduleJob(backupId, schedule, async () => {
+					console.log(
+						`PG-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
+					);
+					runPostgresBackup(pg, backup);
+				});
+			}
+		}
+	}
+
+	const mariadbs = await db.query.mariadb.findMany({
+		with: {
+			backups: {
+				with: {
+					destination: true,
+					postgres: true,
+					mariadb: true,
+					mysql: true,
+					mongo: true,
+				},
+			},
+		},
+	});
+
+	for (const maria of mariadbs) {
+		for (const backup of maria.backups) {
+			const { schedule, backupId, enabled } = backup;
+			if (enabled) {
+				scheduleJob(backupId, schedule, async () => {
+					console.log(
+						`MARIADB-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
+					);
+					await runMariadbBackup(maria, backup);
+				});
+			}
+		}
+	}
+
+	const mongodbs = await db.query.mongo.findMany({
+		with: {
+			backups: {
+				with: {
+					destination: true,
+					postgres: true,
+					mariadb: true,
+					mysql: true,
+					mongo: true,
+				},
+			},
+		},
+	});
+
+	for (const mongo of mongodbs) {
+		for (const backup of mongo.backups) {
+			const { schedule, backupId, enabled } = backup;
+			if (enabled) {
+				scheduleJob(backupId, schedule, async () => {
+					console.log(
+						`MONGO-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
+					);
+					await runMongoBackup(mongo, backup);
+				});
+			}
+		}
+	}
+
+	const mysqls = await db.query.mysql.findMany({
+		with: {
+			backups: {
+				with: {
+					destination: true,
+					postgres: true,
+					mariadb: true,
+					mysql: true,
+					mongo: true,
+				},
+			},
+		},
+	});
+
+	for (const mysql of mysqls) {
+		for (const backup of mysql.backups) {
+			const { schedule, backupId, enabled } = backup;
+			if (enabled) {
+				scheduleJob(backupId, schedule, async () => {
+					console.log(
+						`MYSQL-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
+					);
+					await runMySqlBackup(mysql, backup);
+				});
+			}
+		}
+	}
+};

server/utils/backups/mariadb.ts

@@ -0,0 +1,40 @@
+import { unlink } from "node:fs/promises";
+import path from "node:path";
+import { execAsync } from "../process/execAsync";
+import { uploadToS3 } from "./utils";
+import type { BackupSchedule } from "@/server/api/services/backup";
+import type { Mariadb } from "@/server/api/services/mariadb";
+import { getServiceContainer } from "../docker/utils";
+
+export const runMariadbBackup = async (
+	mariadb: Mariadb,
+	backup: BackupSchedule,
+) => {
+	const { appName, databasePassword, databaseUser } = mariadb;
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.sql.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+	const containerPath = `/backup/${backupFileName}`;
+	const hostPath = `./${backupFileName}`;
+
+	try {
+		const { Id: containerId } = await getServiceContainer(appName);
+		await execAsync(
+			`docker exec ${containerId} sh -c "rm -rf /backup && mkdir -p /backup"`,
+		);
+
+		await execAsync(
+			`docker exec ${containerId} sh -c "mariadb-dump --user='${databaseUser}' --password='${databasePassword}' --databases ${database} | gzip  > ${containerPath}"`,
+		);
+		await execAsync(
+			`docker cp ${containerId}:/backup/${backupFileName} ${hostPath}`,
+		);
+		await uploadToS3(destination, bucketDestination, hostPath);
+	} catch (error) {
+		console.log(error);
+		throw error;
+	} finally {
+		await unlink(hostPath);
+	}
+};

server/utils/backups/mongo.ts

@@ -0,0 +1,37 @@
+import { unlink } from "node:fs/promises";
+import path from "node:path";
+import { execAsync } from "../process/execAsync";
+import { uploadToS3 } from "./utils";
+import type { BackupSchedule } from "@/server/api/services/backup";
+import type { Mongo } from "@/server/api/services/mongo";
+import { getServiceContainer } from "../docker/utils";
+
+// mongodb://mongo:Bqh7AQl-PRbnBu@localhost:27017/?tls=false&directConnection=true
+export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
+	const { appName, databasePassword } = mongo;
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.dump.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+	const containerPath = `/backup/${backupFileName}`;
+	const hostPath = `./${backupFileName}`;
+
+	try {
+		const { Id: containerId } = await getServiceContainer(appName);
+		await execAsync(
+			`docker exec ${containerId} sh -c "rm -rf /backup && mkdir -p /backup"`,
+		);
+
+		await execAsync(
+			`docker exec ${containerId} sh -c "mongodump -d '${database}' -u 'mongo' -p '${databasePassword}' --authenticationDatabase=admin --archive=${containerPath} --gzip"`,
+		);
+		await execAsync(`docker cp ${containerId}:${containerPath} ${hostPath}`);
+		await uploadToS3(destination, bucketDestination, hostPath);
+	} catch (error) {
+		console.log(error);
+		throw error;
+	} finally {
+		await unlink(hostPath);
+	}
+};
+// mongorestore -d monguito -u mongo -p Bqh7AQl-PRbnBu --authenticationDatabase admin --gzip --archive=2024-04-13T05:03:58.937Z.dump.gz

server/utils/backups/mysql.ts

@@ -0,0 +1,38 @@
+import path from "node:path";
+import { execAsync } from "../process/execAsync";
+import { unlink } from "node:fs/promises";
+import { uploadToS3 } from "./utils";
+import type { BackupSchedule } from "@/server/api/services/backup";
+import type { MySql } from "@/server/api/services/mysql";
+import { getServiceContainer } from "../docker/utils";
+
+export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
+	const { appName, databaseRootPassword } = mysql;
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.sql.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+	const containerPath = `/backup/${backupFileName}`;
+	const hostPath = `./${backupFileName}`;
+
+	try {
+		const { Id: containerId } = await getServiceContainer(appName);
+
+		await execAsync(
+			`docker exec ${containerId} sh -c "rm -rf /backup && mkdir -p /backup"`,
+		);
+
+		await execAsync(
+			`docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}'  | gzip > ${containerPath}"`,
+		);
+		await execAsync(
+			`docker cp ${containerId}:/backup/${backupFileName} ${hostPath}`,
+		);
+		await uploadToS3(destination, bucketDestination, hostPath);
+	} catch (error) {
+		console.log(error);
+		throw error;
+	} finally {
+		await unlink(hostPath);
+	}
+};

server/utils/backups/postgres.ts

@@ -0,0 +1,41 @@
+import path from "node:path";
+import { execAsync } from "../process/execAsync";
+import { unlink } from "node:fs/promises";
+import { uploadToS3 } from "./utils";
+import type { BackupSchedule } from "@/server/api/services/backup";
+import type { Postgres } from "@/server/api/services/postgres";
+import { getServiceContainer } from "../docker/utils";
+
+export const runPostgresBackup = async (
+	postgres: Postgres,
+	backup: BackupSchedule,
+) => {
+	const { appName, databaseUser } = postgres;
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.sql.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+	const containerPath = `/backup/${backupFileName}`;
+	const hostPath = `./${backupFileName}`;
+	try {
+		const { Id: containerId } = await getServiceContainer(appName);
+
+		await execAsync(
+			`docker exec ${containerId} /bin/bash -c "rm -rf /backup && mkdir -p /backup"`,
+		);
+		await execAsync(
+			`docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password  '${database}' | gzip > ${containerPath}"`,
+		);
+		await execAsync(`docker cp ${containerId}:${containerPath} ${hostPath}`);
+
+		await uploadToS3(destination, bucketDestination, hostPath);
+	} catch (error) {
+		console.log(error);
+		throw error;
+	} finally {
+		await unlink(hostPath);
+	}
+};
+
+// Restore
+// /Applications/pgAdmin 4.app/Contents/SharedSupport/pg_restore --host "localhost" --port "5432" --username "mauricio" --no-password --dbname "postgres" --verbose "/Users/mauricio/Downloads/_databases_2024-04-12T07_02_05.234Z.sql"

server/utils/backups/utils.ts

@@ -0,0 +1,59 @@
+import type { BackupSchedule } from "@/server/api/services/backup";
+import type { Destination } from "@/server/api/services/destination";
+import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
+import { scheduleJob, scheduledJobs } from "node-schedule";
+import { readFile } from "node:fs/promises";
+import { runPostgresBackup } from "./postgres";
+import { runMySqlBackup } from "./mysql";
+import { runMongoBackup } from "./mongo";
+import { runMariadbBackup } from "./mariadb";
+
+export const uploadToS3 = async (
+	destination: Destination,
+	destinationBucketPath: string,
+	filePath: string,
+) => {
+	const { accessKey, secretAccessKey, bucket, region, endpoint } = destination;
+
+	const s3Client = new S3Client({
+		region: region,
+		...(endpoint && {
+			endpoint: endpoint,
+		}),
+		credentials: {
+			accessKeyId: accessKey,
+			secretAccessKey: secretAccessKey,
+		},
+		forcePathStyle: true,
+	});
+
+	const fileContent = await readFile(filePath);
+	const command = new PutObjectCommand({
+		Bucket: bucket,
+		Key: destinationBucketPath,
+		Body: fileContent,
+	});
+
+	await s3Client.send(command);
+};
+
+export const scheduleBackup = (backup: BackupSchedule) => {
+	const { schedule, backupId, databaseType, postgres, mysql, mongo, mariadb } =
+		backup;
+	scheduleJob(backupId, schedule, async () => {
+		if (databaseType === "postgres" && postgres) {
+			await runPostgresBackup(postgres, backup);
+		} else if (databaseType === "mysql" && mysql) {
+			await runMySqlBackup(mysql, backup);
+		} else if (databaseType === "mongo" && mongo) {
+			await runMongoBackup(mongo, backup);
+		} else if (databaseType === "mariadb" && mariadb) {
+			await runMariadbBackup(mariadb, backup);
+		}
+	});
+};
+
+export const removeScheduleBackup = (backupId: string) => {
+	const currentJob = scheduledJobs[backupId];
+	currentJob?.cancel();
+};