feat(schedules): add schedules server

apps/schedules/.gitignore

@@ -0,0 +1,28 @@
+# dev
+.yarn/
+!.yarn/releases
+.vscode/*
+!.vscode/launch.json
+!.vscode/*.code-snippets
+.idea/workspace.xml
+.idea/usage.statistics.xml
+.idea/shelf
+
+# deps
+node_modules/
+
+# env
+.env
+.env.production
+
+# logs
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+# misc
+.DS_Store

apps/schedules/README.md

@@ -0,0 +1,8 @@
+```
+npm install
+npm run dev
+```
+
+```
+open http://localhost:3000
+```

apps/schedules/package.json

@@ -0,0 +1,30 @@
+{
+	"name": "@dokploy/schedules",
+	"scripts": {
+		"dev": "PORT=4001 tsx watch src/index.ts",
+		"build": "tsc --project tsconfig.json",
+		"start": "node --experimental-specifier-resolution=node dist/index.js"
+	},
+	"dependencies": {
+		"bullmq": "5.4.2",
+		"@hono/zod-validator": "0.3.0",
+		"zod": "^3.23.4",
+		"react": "18.2.0",
+		"react-dom": "18.2.0",
+		"@dokploy/builders": "workspace:*",
+		"@hono/node-server": "^1.12.1",
+		"hono": "^4.5.8",
+		"dotenv": "^16.3.1",
+		"pino": "9.4.0",
+		"pino-pretty": "11.2.2",
+		"redis": "4.7.0"
+	},
+	"devDependencies": {
+		"typescript": "^5.4.2",
+		"@types/react": "^18.2.37",
+		"@types/react-dom": "^18.2.15",
+		"@types/node": "^20.11.17",
+		"tsx": "^4.7.1"
+	},
+	"packageManager": "pnpm@9.5.0"
+}

apps/schedules/src/index.ts

@@ -0,0 +1,55 @@
+import { serve } from "@hono/node-server";
+import { Hono } from "hono";
+import "dotenv/config";
+import { zValidator } from "@hono/zod-validator";
+import { jobQueueSchema } from "./schema";
+import { firstWorker, secondWorker } from "./workers";
+import { logger } from "./logger";
+import { cleanQueue, removeJob, scheduleJob } from "./queue";
+
+const app = new Hono();
+
+cleanQueue();
+
+app.post("/create-backup", zValidator("json", jobQueueSchema), (c) => {
+	const data = c.req.valid("json");
+	scheduleJob(data);
+
+	logger.info("Backup created successfully", data);
+	return c.json({ message: "Backup created successfully" });
+});
+
+app.post("/remove-job", zValidator("json", jobQueueSchema), async (c) => {
+	const data = c.req.valid("json");
+	const result = await removeJob(data);
+	logger.info("Job removed successfully", data);
+	return c.json({ message: "Job removed successfully", result });
+});
+
+app.get("/health", async (c) => {
+	return c.json({ status: "ok" });
+});
+
+export const gracefulShutdown = async (signal: string) => {
+	logger.warn(`Received ${signal}, closing server...`);
+	await firstWorker.close();
+	await secondWorker.close();
+	process.exit(0);
+};
+
+process.on("SIGINT", () => gracefulShutdown("SIGINT"));
+
+process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
+
+process.on("uncaughtException", (err) => {
+	logger.error(err, "Uncaught exception");
+});
+
+process.on("unhandledRejection", (reason, promise) => {
+	logger.error({ promise, reason }, "Unhandled Rejection at: Promise");
+});
+
+const port = Number.parseInt(process.env.PORT || "3000");
+
+logger.info("Starting Schedules Server ✅", port);
+serve({ fetch: app.fetch, port });

apps/schedules/src/logger.ts

@@ -0,0 +1,10 @@
+import pino from "pino";
+
+export const logger = pino({
+	transport: {
+		target: "pino-pretty",
+		options: {
+			colorize: true,
+		},
+	},
+});

apps/schedules/src/queue.ts

@@ -0,0 +1,55 @@
+import { Queue } from "bullmq";
+import type { QueueJob } from "./schema";
+import { logger } from "./logger";
+
+export const jobQueue = new Queue("backupQueue", {
+	connection: {
+		host: process.env.REDIS_URL,
+	},
+	defaultJobOptions: {
+		removeOnComplete: true,
+		removeOnFail: true,
+	},
+});
+
+export const cleanQueue = async () => {
+	try {
+		await jobQueue.obliterate({ force: true });
+		logger.info("Queue Cleaned");
+	} catch (error) {
+		logger.error("Error cleaning queue:", error);
+	}
+};
+
+export const scheduleJob = (job: QueueJob) => {
+	if (job.type === "backup") {
+		jobQueue.add(job.backupId, job, {
+			repeat: {
+				pattern: job.cronSchedule,
+			},
+		});
+	} else if (job.type === "server") {
+		jobQueue.add(`${job.serverId}-cleanup`, job, {
+			repeat: {
+				pattern: job.cronSchedule,
+			},
+		});
+	}
+};
+
+export const removeJob = async (data: QueueJob) => {
+	if (data.type === "backup") {
+		const { backupId, cronSchedule } = data;
+		const result = await jobQueue.removeRepeatable(backupId, {
+			pattern: cronSchedule,
+		});
+		return result;
+	}
+	if (data.type === "server") {
+		const { serverId, cronSchedule } = data;
+		const result = await jobQueue.removeRepeatable(`${serverId}-cleanup`, {
+			pattern: cronSchedule,
+		});
+		return result;
+	}
+};

apps/schedules/src/schema.ts

@@ -0,0 +1,16 @@
+import { z } from "zod";
+
+export const jobQueueSchema = z.discriminatedUnion("type", [
+	z.object({
+		cronSchedule: z.string(),
+		type: z.literal("backup"),
+		backupId: z.string(),
+	}),
+	z.object({
+		cronSchedule: z.string(),
+		type: z.literal("server"),
+		serverId: z.string(),
+	}),
+]);
+
+export type QueueJob = z.infer<typeof jobQueueSchema>;

apps/schedules/src/utils.ts

@@ -0,0 +1,42 @@
+import {
+	cleanUpDockerBuilder,
+	cleanUpSystemPrune,
+	cleanUpUnusedImages,
+	findBackupById,
+	runMariadbBackup,
+	runMongoBackup,
+	runMySqlBackup,
+	runPostgresBackup,
+} from "@dokploy/builders";
+import type { QueueJob } from "./schema";
+import { logger } from "./logger";
+
+export const runJobs = async (job: QueueJob) => {
+	try {
+		if (job.type === "backup") {
+			const { backupId } = job;
+			const backup = await findBackupById(backupId);
+			const { databaseType, postgres, mysql, mongo, mariadb } = backup;
+			if (databaseType === "postgres" && postgres) {
+				await runPostgresBackup(postgres, backup);
+			} else if (databaseType === "mysql" && mysql) {
+				await runMySqlBackup(mysql, backup);
+			} else if (databaseType === "mongo" && mongo) {
+				await runMongoBackup(mongo, backup);
+			} else if (databaseType === "mariadb" && mariadb) {
+				await runMariadbBackup(mariadb, backup);
+			}
+		}
+		if (job.type === "server") {
+			const { serverId } = job;
+			await cleanUpUnusedImages(serverId);
+			await cleanUpDockerBuilder(serverId);
+			await cleanUpSystemPrune(serverId);
+			// await sendDockerCleanupNotifications();
+		}
+	} catch (error) {
+		logger.error(error);
+	}
+
+	return true;
+};

apps/schedules/src/workers.ts

@@ -0,0 +1,28 @@
+import type { QueueJob } from "./schema";
+import { type Job, Worker } from "bullmq";
+import { runJobs } from "./utils";
+
+export const firstWorker = new Worker(
+	"backupQueue",
+	async (job: Job<QueueJob>) => {
+		await runJobs(job.data);
+	},
+	{
+		concurrency: 50,
+		connection: {
+			host: process.env.REDIS_URL,
+		},
+	},
+);
+export const secondWorker = new Worker(
+	"backupQueue",
+	async (job: Job<QueueJob>) => {
+		await runJobs(job.data);
+	},
+	{
+		concurrency: 50,
+		connection: {
+			host: process.env.REDIS_URL,
+		},
+	},
+);

apps/schedules/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "strict": true,
+    "skipLibCheck": true,
+    "outDir": "dist",
+    "jsx": "react-jsx",
+    "jsxImportSource": "hono/jsx"
+  },
+  "exclude": ["node_modules", "dist"]
+}