refactor: rename builders to server

packages/server/src/utils/backups/index.ts

@@ -0,0 +1,157 @@
+import { findAdmin } from "@/server/services/admin";
+import { getAllServers } from "@/server/services/server";
+import { scheduleJob } from "node-schedule";
+import { db } from "../../db/index";
+import {
+	cleanUpDockerBuilder,
+	cleanUpSystemPrune,
+	cleanUpUnusedImages,
+} from "../docker/utils";
+import { runMariadbBackup } from "./mariadb";
+import { runMongoBackup } from "./mongo";
+import { runMySqlBackup } from "./mysql";
+import { runPostgresBackup } from "./postgres";
+
+export const initCronJobs = async () => {
+	console.log("Setting up cron jobs....");
+
+	const admin = await findAdmin();
+
+	if (admin?.enableDockerCleanup) {
+		scheduleJob("docker-cleanup", "0 0 * * *", async () => {
+			console.log(
+				`Docker Cleanup ${new Date().toLocaleString()}]  Running docker cleanup`,
+			);
+			await cleanUpUnusedImages();
+			await cleanUpDockerBuilder();
+			await cleanUpSystemPrune();
+		});
+	}
+
+	const servers = await getAllServers();
+
+	for (const server of servers) {
+		const { appName, serverId } = server;
+		if (serverId) {
+			scheduleJob(serverId, "0 0 * * *", async () => {
+				console.log(
+					`SERVER-BACKUP[${new Date().toLocaleString()}] Running Cleanup ${appName}`,
+				);
+				await cleanUpUnusedImages(serverId);
+				await cleanUpDockerBuilder(serverId);
+				await cleanUpSystemPrune(serverId);
+			});
+		}
+	}
+
+	const pgs = await db.query.postgres.findMany({
+		with: {
+			backups: {
+				with: {
+					destination: true,
+					postgres: true,
+					mariadb: true,
+					mysql: true,
+					mongo: true,
+				},
+			},
+		},
+	});
+	for (const pg of pgs) {
+		for (const backup of pg.backups) {
+			const { schedule, backupId, enabled } = backup;
+			if (enabled) {
+				scheduleJob(backupId, schedule, async () => {
+					console.log(
+						`PG-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
+					);
+					runPostgresBackup(pg, backup);
+				});
+			}
+		}
+	}
+
+	const mariadbs = await db.query.mariadb.findMany({
+		with: {
+			backups: {
+				with: {
+					destination: true,
+					postgres: true,
+					mariadb: true,
+					mysql: true,
+					mongo: true,
+				},
+			},
+		},
+	});
+
+	for (const maria of mariadbs) {
+		for (const backup of maria.backups) {
+			const { schedule, backupId, enabled } = backup;
+			if (enabled) {
+				scheduleJob(backupId, schedule, async () => {
+					console.log(
+						`MARIADB-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
+					);
+					await runMariadbBackup(maria, backup);
+				});
+			}
+		}
+	}
+
+	const mongodbs = await db.query.mongo.findMany({
+		with: {
+			backups: {
+				with: {
+					destination: true,
+					postgres: true,
+					mariadb: true,
+					mysql: true,
+					mongo: true,
+				},
+			},
+		},
+	});
+
+	for (const mongo of mongodbs) {
+		for (const backup of mongo.backups) {
+			const { schedule, backupId, enabled } = backup;
+			if (enabled) {
+				scheduleJob(backupId, schedule, async () => {
+					console.log(
+						`MONGO-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
+					);
+					await runMongoBackup(mongo, backup);
+				});
+			}
+		}
+	}
+
+	const mysqls = await db.query.mysql.findMany({
+		with: {
+			backups: {
+				with: {
+					destination: true,
+					postgres: true,
+					mariadb: true,
+					mysql: true,
+					mongo: true,
+				},
+			},
+		},
+	});
+
+	for (const mysql of mysqls) {
+		for (const backup of mysql.backups) {
+			const { schedule, backupId, enabled } = backup;
+			if (enabled) {
+				scheduleJob(backupId, schedule, async () => {
+					console.log(
+						`MYSQL-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
+					);
+					await runMySqlBackup(mysql, backup);
+				});
+			}
+		}
+	}
+};

packages/server/src/utils/backups/mariadb.ts

@@ -0,0 +1,65 @@
+import path from "node:path";
+import type { BackupSchedule } from "@/server/services/backup";
+import type { Mariadb } from "@/server/services/mariadb";
+import { findProjectById } from "@/server/services/project";
+import {
+	getRemoteServiceContainer,
+	getServiceContainer,
+} from "../docker/utils";
+import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+import { getS3Credentials } from "./utils";
+
+export const runMariadbBackup = async (
+	mariadb: Mariadb,
+	backup: BackupSchedule,
+) => {
+	const { appName, databasePassword, databaseUser, projectId, name } = mariadb;
+	const project = await findProjectById(projectId);
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.sql.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+
+	try {
+		const rcloneFlags = getS3Credentials(destination);
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
+
+		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+		if (mariadb.serverId) {
+			const { Id: containerId } = await getRemoteServiceContainer(
+				mariadb.serverId,
+				appName,
+			);
+			const mariadbDumpCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${databaseUser}' --password='${databasePassword}' --databases ${database} | gzip"`;
+
+			await execAsyncRemote(
+				mariadb.serverId,
+				`${mariadbDumpCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { Id: containerId } = await getServiceContainer(appName);
+			const mariadbDumpCommand = `docker exec ${containerId} sh -c "mariadb-dump --user='${databaseUser}' --password='${databasePassword}' --databases ${database} | gzip"`;
+
+			await execAsync(`${mariadbDumpCommand} | ${rcloneCommand}`);
+		}
+
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "mariadb",
+			type: "success",
+		});
+	} catch (error) {
+		console.log(error);
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "mariadb",
+			type: "error",
+			// @ts-ignore
+			errorMessage: error?.message || "Error message not provided",
+		});
+		throw error;
+	}
+};

packages/server/src/utils/backups/mongo.ts

@@ -0,0 +1,63 @@
+import path from "node:path";
+import type { BackupSchedule } from "@/server/services/backup";
+import type { Mongo } from "@/server/services/mongo";
+import { findProjectById } from "@/server/services/project";
+import {
+	getRemoteServiceContainer,
+	getServiceContainer,
+} from "../docker/utils";
+import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+import { getS3Credentials } from "./utils";
+
+// mongodb://mongo:Bqh7AQl-PRbnBu@localhost:27017/?tls=false&directConnection=true
+export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
+	const { appName, databasePassword, databaseUser, projectId, name } = mongo;
+	const project = await findProjectById(projectId);
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.dump.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+
+	try {
+		const rcloneFlags = getS3Credentials(destination);
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
+
+		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+		if (mongo.serverId) {
+			const { Id: containerId } = await getRemoteServiceContainer(
+				mongo.serverId,
+				appName,
+			);
+			const mongoDumpCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${databaseUser}' -p '${databasePassword}' --authenticationDatabase=admin --gzip"`;
+
+			await execAsyncRemote(
+				mongo.serverId,
+				`${mongoDumpCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { Id: containerId } = await getServiceContainer(appName);
+			const mongoDumpCommand = `docker exec ${containerId} sh -c "mongodump -d '${database}' -u '${databaseUser}' -p '${databasePassword}' --authenticationDatabase=admin --gzip"`;
+			await execAsync(`${mongoDumpCommand} | ${rcloneCommand}`);
+		}
+
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "mongodb",
+			type: "success",
+		});
+	} catch (error) {
+		console.log(error);
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "mongodb",
+			type: "error",
+			// @ts-ignore
+			errorMessage: error?.message || "Error message not provided",
+		});
+		throw error;
+	}
+};
+// mongorestore -d monguito -u mongo -p Bqh7AQl-PRbnBu --authenticationDatabase admin --gzip --archive=2024-04-13T05:03:58.937Z.dump.gz

packages/server/src/utils/backups/mysql.ts

@@ -0,0 +1,62 @@
+import { unlink } from "node:fs/promises";
+import path from "node:path";
+import type { BackupSchedule } from "@/server/services/backup";
+import type { MySql } from "@/server/services/mysql";
+import { findProjectById } from "@/server/services/project";
+import {
+	getRemoteServiceContainer,
+	getServiceContainer,
+} from "../docker/utils";
+import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+import { getS3Credentials } from "./utils";
+
+export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
+	const { appName, databaseRootPassword, projectId, name } = mysql;
+	const project = await findProjectById(projectId);
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.sql.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+
+	try {
+		const rcloneFlags = getS3Credentials(destination);
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
+
+		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+		if (mysql.serverId) {
+			const { Id: containerId } = await getRemoteServiceContainer(
+				mysql.serverId,
+				appName,
+			);
+			const mysqlDumpCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
+
+			await execAsyncRemote(
+				mysql.serverId,
+				`${mysqlDumpCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { Id: containerId } = await getServiceContainer(appName);
+			const mysqlDumpCommand = `docker exec ${containerId} sh -c "mysqldump --default-character-set=utf8mb4 -u 'root' --password='${databaseRootPassword}' --single-transaction --no-tablespaces --quick '${database}' | gzip"`;
+
+			await execAsync(`${mysqlDumpCommand} | ${rcloneCommand}`);
+		}
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "mysql",
+			type: "success",
+		});
+	} catch (error) {
+		console.log(error);
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "mysql",
+			type: "error",
+			// @ts-ignore
+			errorMessage: error?.message || "Error message not provided",
+		});
+		throw error;
+	}
+};

packages/server/src/utils/backups/postgres.ts

@@ -0,0 +1,69 @@
+import path from "node:path";
+import type { BackupSchedule } from "@/server/services/backup";
+import type { Postgres } from "@/server/services/postgres";
+import { findProjectById } from "@/server/services/project";
+import {
+	getRemoteServiceContainer,
+	getServiceContainer,
+} from "../docker/utils";
+import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
+import { getS3Credentials } from "./utils";
+
+export const runPostgresBackup = async (
+	postgres: Postgres,
+	backup: BackupSchedule,
+) => {
+	const { appName, databaseUser, name, projectId } = postgres;
+	const project = await findProjectById(projectId);
+
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.sql.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+	try {
+		const rcloneFlags = getS3Credentials(destination);
+		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
+
+		const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+		if (postgres.serverId) {
+			const { Id: containerId } = await getRemoteServiceContainer(
+				postgres.serverId,
+				appName,
+			);
+			const pgDumpCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip"`;
+
+			await execAsyncRemote(
+				postgres.serverId,
+				`${pgDumpCommand} | ${rcloneCommand}`,
+			);
+		} else {
+			const { Id: containerId } = await getServiceContainer(appName);
+
+			const pgDumpCommand = `docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip"`;
+			await execAsync(`${pgDumpCommand} | ${rcloneCommand}`);
+		}
+
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "postgres",
+			type: "success",
+		});
+	} catch (error) {
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "postgres",
+			type: "error",
+			// @ts-ignore
+			errorMessage: error?.message || "Error message not provided",
+		});
+
+		throw error;
+	} finally {
+	}
+};
+
+// Restore
+// /Applications/pgAdmin 4.app/Contents/SharedSupport/pg_restore --host "localhost" --port "5432" --username "mauricio" --no-password --dbname "postgres" --verbose "/Users/mauricio/Downloads/_databases_2024-04-12T07_02_05.234Z.sql"

packages/server/src/utils/backups/utils.ts

@@ -0,0 +1,43 @@
+import type { BackupSchedule } from "@/server/services/backup";
+import type { Destination } from "@/server/services/destination";
+import { scheduleJob, scheduledJobs } from "node-schedule";
+import { runMariadbBackup } from "./mariadb";
+import { runMongoBackup } from "./mongo";
+import { runMySqlBackup } from "./mysql";
+import { runPostgresBackup } from "./postgres";
+
+export const scheduleBackup = (backup: BackupSchedule) => {
+	const { schedule, backupId, databaseType, postgres, mysql, mongo, mariadb } =
+		backup;
+	scheduleJob(backupId, schedule, async () => {
+		if (databaseType === "postgres" && postgres) {
+			await runPostgresBackup(postgres, backup);
+		} else if (databaseType === "mysql" && mysql) {
+			await runMySqlBackup(mysql, backup);
+		} else if (databaseType === "mongo" && mongo) {
+			await runMongoBackup(mongo, backup);
+		} else if (databaseType === "mariadb" && mariadb) {
+			await runMariadbBackup(mariadb, backup);
+		}
+	});
+};
+
+export const removeScheduleBackup = (backupId: string) => {
+	const currentJob = scheduledJobs[backupId];
+	currentJob?.cancel();
+};
+
+export const getS3Credentials = (destination: Destination) => {
+	const { accessKey, secretAccessKey, bucket, region, endpoint } = destination;
+	const rcloneFlags = [
+		// `--s3-provider=Cloudflare`,
+		`--s3-access-key-id=${accessKey}`,
+		`--s3-secret-access-key=${secretAccessKey}`,
+		`--s3-region=${region}`,
+		`--s3-endpoint=${endpoint}`,
+		"--s3-no-check-bucket",
+		"--s3-force-path-style",
+	];
+
+	return rcloneFlags;
+};