Feat/monitoring (#1267) Cloud Version

* feat: add start monitoring remote servers

* reafctor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor:

* refactor: add metrics

* feat: add disk monitoring

* refactor: translate to english

* refacotor: add stats

* refactor: remove color

* feat: add log server metrics

* refactor: remove unused deps

* refactor: add origin

* refactor: add logs

* refactor: update

* feat: add series monitoring

* refactor: add system monitoring

* feat: add benchmark to optimize data

* refactor: update fn

* refactor: remove comments

* refactor: update

* refactor: exclude items

* feat: add refresh rate

* feat: add monitoring remote servers

* refactor: update

* refactor: remove unsued volumes

* refactor: update monitoring

* refactor: add more presets

* feat: add container metrics

* feat: add docker monitoring

* refactor: update conversion

* refactor: remove unused code

* refactor: update

* refactor: add docker compose logs

* refactor: add docker cli

* refactor: add install curl

* refactor: add get update

* refactor: add monitoring remote servers

* refactor: add containers config

* feat: add container specification

* refactor: update path

* refactor: add server filter

* refactor: simplify logic

* fix: verify if file exist before get stats

* refactor: update

* refactor: remove unused deps

* test: add test for containers

* refactor: update

* refactor add memory collector

* refactor: update

* refactor: update

* refactor: update

* refactor: remove

* refactor: add memory

* refactor: add server memory usage

* refactor: change memory

* refactor: update

* refactor: update

* refactor: add container metrics

* refactor: comment code

* refactor: mount proc bind

* refactor: change interval with node cron

* refactor: remove opening file

* refactor: use streams

* refactor: remove unused ws

* refactor: disable live when is all

* refactor: add sqlite

* refactor: update

* feat: add golang benchmark

* refactor: update go

* refactor: update dockerfile

* refactor: update db

* refactor: add env

* refactor: separate logic

* refactor: split logic

* refactor: update logs

* refactor: update dockerfile

* refactor: hide .env

* refactor: update

* chore: hide ,.ebnv

* refactor: add end angle

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update

* refactor: update monitoring

* refactor: add mount db

* refactor: add metrics and url callback

* refactor: add middleware

* refactor: add threshold property

* feat: add memory and cpu threshold notification

* feat: send notifications to the server

* feat: add metrics for dokploy server

* refactor: add dokploy server to monitoring

* refactor: update methods

* refactor: add admin to useeffect

* refactor: stop monitoring containers if elements are 0

* refactor: cancel request if appName is empty

* refactor: reuse methods

* chore; add feat monitoring

* refactor: set base url

* refactor: adjust monitoring

* refactor: delete migrations

* feat: add columns

* fix: add missing flag

* refactor: add free metrics

* refactor: add paid monitoring

* refactor: update methods

* feat: improve ui

* feat: add container stats

* refactor: add all container metrics

* refactor: add color primary

* refactor: change default rate limiting refresher

* refactor: update retention days

* refactor: use json instead of individual properties

* refactor: lint

* refactor: pass json env

* refactor: update

* refactor: delete

* refactor: update

* refactor: fix types

* refactor: add retention days

* chore: add license

* refactor: create db

* refactor: update path

* refactor: update setup

* refactor: update

* refactor: create files

* refactor: update

* refactor: delete

* refactor: update

* refactor: update token metrics

* fix: typechecks

* refactor: setup web server

* refactor: update error handling and add monitoring

* refactor: add local storage save

* refactor: add spacing

* refactor: update

* refactor: upgrade drizzle

* refactor: delete

* refactor: uppgrade drizzle kit

* refactor: update search with jsonB

* chore: upgrade drizzle

* chore: update packages

* refactor: add missing type

* refactor: add serverType

* refactor: update url

* refactor: update

* refactor: update

* refactor: hide monitoring on self hosted

* refactor: update server

* refactor: update

* refactor: update

* refactor: pin node version

packages/server/package.json

@@ -41,7 +41,7 @@
     "date-fns": "3.6.0",
     "dockerode": "4.0.2",
     "dotenv": "16.4.5",
-    "drizzle-orm": "^0.30.8",
+   "drizzle-orm": "^0.39.1",
     "drizzle-zod": "0.5.1",
     "hi-base32": "^0.5.1",
     "js-yaml": "4.1.0",
@@ -82,7 +82,7 @@
     "@types/react": "^18.2.37",
     "@types/react-dom": "^18.2.15",
     "@types/ws": "8.5.10",
-    "drizzle-kit": "^0.21.1",
+    "drizzle-kit": "^0.30.4",
     "esbuild": "0.20.2",
     "postcss": "^8.4.31",
     "typescript": "^5.4.2",

packages/server/src/db/schema/admin.ts

@@ -1,5 +1,12 @@
 import { relations } from "drizzle-orm";
-import { boolean, integer, pgTable, text } from "drizzle-orm/pg-core";
+import {
+	boolean,
+	integer,
+	json,
+	jsonb,
+	pgTable,
+	text,
+} from "drizzle-orm/pg-core";
 import { createInsertSchema } from "drizzle-zod";
 import { nanoid } from "nanoid";
 import { z } from "zod";
@@ -31,6 +38,55 @@ export const admins = pgTable("admin", {
 	stripeCustomerId: text("stripeCustomerId"),
 	stripeSubscriptionId: text("stripeSubscriptionId"),
 	serversQuantity: integer("serversQuantity").notNull().default(0),
+
+	// Metrics
+	enablePaidFeatures: boolean("enablePaidFeatures").notNull().default(false),
+	metricsConfig: jsonb("metricsConfig")
+		.$type<{
+			server: {
+				type: "Dokploy" | "Remote";
+				refreshRate: number;
+				port: number;
+				token: string;
+				urlCallback: string;
+				retentionDays: number;
+				cronJob: string;
+				thresholds: {
+					cpu: number;
+					memory: number;
+				};
+			};
+			containers: {
+				refreshRate: number;
+				services: {
+					include: string[];
+					exclude: string[];
+				};
+			};
+		}>()
+		.notNull()
+		.default({
+			server: {
+				type: "Dokploy",
+				refreshRate: 60,
+				port: 4500,
+				token: "",
+				retentionDays: 2,
+				cronJob: "",
+				urlCallback: "",
+				thresholds: {
+					cpu: 0,
+					memory: 0,
+				},
+			},
+			containers: {
+				refreshRate: 60,
+				services: {
+					include: [],
+					exclude: [],
+				},
+			},
+		}),
 	cleanupCacheApplications: boolean("cleanupCacheApplications")
 		.notNull()
 		.default(false),
@@ -126,3 +182,29 @@ export const apiReadStatsLogs = z.object({
 	search: z.string().optional(),
 	sort: z.object({ id: z.string(), desc: z.boolean() }).optional(),
 });
+
+export const apiUpdateWebServerMonitoring = z.object({
+	metricsConfig: z
+		.object({
+			server: z.object({
+				refreshRate: z.number().min(2),
+				port: z.number().min(1),
+				token: z.string(),
+				urlCallback: z.string().url(),
+				retentionDays: z.number().min(1),
+				cronJob: z.string().min(1),
+				thresholds: z.object({
+					cpu: z.number().min(0),
+					memory: z.number().min(0),
+				}),
+			}),
+			containers: z.object({
+				refreshRate: z.number().min(2),
+				services: z.object({
+					include: z.array(z.string()).optional(),
+					exclude: z.array(z.string()).optional(),
+				}),
+			}),
+		})
+		.required(),
+});

packages/server/src/db/schema/notification.ts

@@ -24,6 +24,7 @@ export const notifications = pgTable("notification", {
 	databaseBackup: boolean("databaseBackup").notNull().default(false),
 	dokployRestart: boolean("dokployRestart").notNull().default(false),
 	dockerCleanup: boolean("dockerCleanup").notNull().default(false),
+	serverThreshold: boolean("serverThreshold").notNull().default(false),
 	notificationType: notificationType("notificationType").notNull(),
 	createdAt: text("createdAt")
 		.notNull()
@@ -136,6 +137,7 @@ export const apiCreateSlack = notificationsSchema
 		name: true,
 		appDeploy: true,
 		dockerCleanup: true,
+		serverThreshold: true,
 	})
 	.extend({
 		webhookUrl: z.string().min(1),
@@ -162,6 +164,7 @@ export const apiCreateTelegram = notificationsSchema
 		name: true,
 		appDeploy: true,
 		dockerCleanup: true,
+		serverThreshold: true,
 	})
 	.extend({
 		botToken: z.string().min(1),
@@ -188,6 +191,7 @@ export const apiCreateDiscord = notificationsSchema
 		name: true,
 		appDeploy: true,
 		dockerCleanup: true,
+		serverThreshold: true,
 	})
 	.extend({
 		webhookUrl: z.string().min(1),
@@ -217,6 +221,7 @@ export const apiCreateEmail = notificationsSchema
 		name: true,
 		appDeploy: true,
 		dockerCleanup: true,
+		serverThreshold: true,
 	})
 	.extend({
 		smtpServer: z.string().min(1),

packages/server/src/db/schema/server.ts

@@ -1,5 +1,12 @@
 import { relations } from "drizzle-orm";
-import { boolean, integer, pgEnum, pgTable, text } from "drizzle-orm/pg-core";
+import {
+	boolean,
+	integer,
+	jsonb,
+	pgEnum,
+	pgTable,
+	text,
+} from "drizzle-orm/pg-core";
 import { createInsertSchema } from "drizzle-zod";
 import { nanoid } from "nanoid";
 import { z } from "zod";
@@ -44,6 +51,52 @@ export const server = pgTable("server", {
 	sshKeyId: text("sshKeyId").references(() => sshKeys.sshKeyId, {
 		onDelete: "set null",
 	}),
+	metricsConfig: jsonb("metricsConfig")
+		.$type<{
+			server: {
+				type: "Dokploy" | "Remote";
+				refreshRate: number;
+				port: number;
+				token: string;
+				urlCallback: string;
+				retentionDays: number;
+				cronJob: string;
+				thresholds: {
+					cpu: number;
+					memory: number;
+				};
+			};
+			containers: {
+				refreshRate: number;
+				services: {
+					include: string[];
+					exclude: string[];
+				};
+			};
+		}>()
+		.notNull()
+		.default({
+			server: {
+				type: "Remote",
+				refreshRate: 60,
+				port: 4500,
+				token: "",
+				urlCallback: "",
+				cronJob: "",
+				retentionDays: 2,
+				thresholds: {
+					cpu: 0,
+					memory: 0,
+				},
+			},
+			containers: {
+				refreshRate: 60,
+				services: {
+					include: [],
+					exclude: [],
+				},
+			},
+		}),
 });
 
 export const serverRelations = relations(server, ({ one, many }) => ({
@@ -109,3 +162,34 @@ export const apiUpdateServer = createSchema
 	.extend({
 		command: z.string().optional(),
 	});
+
+export const apiUpdateServerMonitoring = createSchema
+	.pick({
+		serverId: true,
+	})
+	.required()
+	.extend({
+		metricsConfig: z
+			.object({
+				server: z.object({
+					refreshRate: z.number().min(2),
+					port: z.number().min(1),
+					token: z.string(),
+					urlCallback: z.string().url(),
+					retentionDays: z.number().min(1),
+					cronJob: z.string().min(1),
+					thresholds: z.object({
+						cpu: z.number().min(0),
+						memory: z.number().min(0),
+					}),
+				}),
+				containers: z.object({
+					refreshRate: z.number().min(2),
+					services: z.object({
+						include: z.array(z.string()).optional(),
+						exclude: z.array(z.string()).optional(),
+					}),
+				}),
+			})
+			.required(),
+	});

packages/server/src/index.ts

@@ -39,6 +39,7 @@ export * from "./setup/config-paths";
 export * from "./setup/postgres-setup";
 export * from "./setup/redis-setup";
 export * from "./setup/server-setup";
+export * from "./setup/monitoring-setup";
 export * from "./setup/setup";
 export * from "./setup/traefik-setup";
 export * from "./setup/server-validate";
@@ -57,6 +58,7 @@ export * from "./utils/notifications/database-backup";
 export * from "./utils/notifications/dokploy-restart";
 export * from "./utils/notifications/utils";
 export * from "./utils/notifications/docker-cleanup";
+export * from "./utils/notifications/server-threshold";
 
 export * from "./utils/builders/index";
 export * from "./utils/builders/compose";

packages/server/src/services/admin.ts

@@ -77,6 +77,22 @@ export const updateAdmin = async (
 	return admin;
 };
 
+export const updateAdminById = async (
+	adminId: string,
+	adminData: Partial<Admin>,
+) => {
+	const admin = await db
+		.update(admins)
+		.set({
+			...adminData,
+		})
+		.where(eq(admins.adminId, adminId))
+		.returning()
+		.then((res) => res[0]);
+
+	return admin;
+};
+
 export const isAdminPresent = async () => {
 	const admin = await db.query.admins.findFirst();
 	if (!admin) {

packages/server/src/services/notification.ts

@@ -55,6 +55,7 @@ export const createSlackNotification = async (
 				dockerCleanup: input.dockerCleanup,
 				notificationType: "slack",
 				adminId: adminId,
+				serverThreshold: input.serverThreshold,
 			})
 			.returning()
 			.then((value) => value[0]);
@@ -84,6 +85,7 @@ export const updateSlackNotification = async (
 				dokployRestart: input.dokployRestart,
 				dockerCleanup: input.dockerCleanup,
 				adminId: input.adminId,
+				serverThreshold: input.serverThreshold,
 			})
 			.where(eq(notifications.notificationId, input.notificationId))
 			.returning()
@@ -143,6 +145,7 @@ export const createTelegramNotification = async (
 				dockerCleanup: input.dockerCleanup,
 				notificationType: "telegram",
 				adminId: adminId,
+				serverThreshold: input.serverThreshold,
 			})
 			.returning()
 			.then((value) => value[0]);
@@ -172,6 +175,7 @@ export const updateTelegramNotification = async (
 				dokployRestart: input.dokployRestart,
 				dockerCleanup: input.dockerCleanup,
 				adminId: input.adminId,
+				serverThreshold: input.serverThreshold,
 			})
 			.where(eq(notifications.notificationId, input.notificationId))
 			.returning()
@@ -231,6 +235,7 @@ export const createDiscordNotification = async (
 				dockerCleanup: input.dockerCleanup,
 				notificationType: "discord",
 				adminId: adminId,
+				serverThreshold: input.serverThreshold,
 			})
 			.returning()
 			.then((value) => value[0]);
@@ -260,6 +265,7 @@ export const updateDiscordNotification = async (
 				dokployRestart: input.dokployRestart,
 				dockerCleanup: input.dockerCleanup,
 				adminId: input.adminId,
+				serverThreshold: input.serverThreshold,
 			})
 			.where(eq(notifications.notificationId, input.notificationId))
 			.returning()
@@ -323,6 +329,7 @@ export const createEmailNotification = async (
 				dockerCleanup: input.dockerCleanup,
 				notificationType: "email",
 				adminId: adminId,
+				serverThreshold: input.serverThreshold,
 			})
 			.returning()
 			.then((value) => value[0]);
@@ -352,6 +359,7 @@ export const updateEmailNotification = async (
 				dokployRestart: input.dokployRestart,
 				dockerCleanup: input.dockerCleanup,
 				adminId: input.adminId,
+				serverThreshold: input.serverThreshold,
 			})
 			.where(eq(notifications.notificationId, input.notificationId))
 			.returning()

packages/server/src/services/settings.ts

@@ -219,8 +219,8 @@ export const cleanupFullDocker = async (serverId?: string | null) => {
 	const cleanupImages = "docker image prune --force";
 	const cleanupVolumes = "docker volume prune --force";
 	const cleanupContainers = "docker container prune --force";
-	const cleanupSystem = "docker system prune --all --force --volumes";
-	const cleanupBuilder = "docker builder prune --all --force";
+	const cleanupSystem = "docker system prune  --force --volumes";
+	const cleanupBuilder = "docker builder prune  --force";
 
 	try {
 		if (serverId) {

packages/server/src/setup/monitoring-setup.ts

@@ -0,0 +1,130 @@
+import { findServerById } from "@dokploy/server/services/server";
+import type { ContainerCreateOptions } from "dockerode";
+import { findAdminById } from "../services/admin";
+import { pullImage, pullRemoteImage } from "../utils/docker/utils";
+import { execAsync, execAsyncRemote } from "../utils/process/execAsync";
+import { getRemoteDocker } from "../utils/servers/remote-docker";
+
+export const setupMonitoring = async (serverId: string) => {
+	const server = await findServerById(serverId);
+
+	const containerName = "mauricio-monitoring";
+	const imageName = "siumauricio/monitoring:canary";
+
+	const settings: ContainerCreateOptions = {
+		name: containerName,
+		Env: [`METRICS_CONFIG=${JSON.stringify(server?.metricsConfig)}`],
+		Image: imageName,
+		HostConfig: {
+			// Memory: 100 * 1024 * 1024, // 100MB en bytes
+			// PidMode: "host",
+			// CapAdd: ["NET_ADMIN", "SYS_ADMIN"],
+			// Privileged: true,
+			PortBindings: {
+				[`${server.metricsConfig.server.port}/tcp`]: [
+					{
+						HostPort: server.metricsConfig.server.port.toString(),
+					},
+				],
+			},
+			Binds: [
+				"/var/run/docker.sock:/var/run/docker.sock:ro",
+				"/sys:/host/sys:ro",
+				"/etc/os-release:/etc/os-release:ro",
+				"/proc:/host/proc:ro",
+				"/etc/dokploy/monitoring/monitoring.db:/app/monitoring.db",
+			],
+			NetworkMode: "host",
+		},
+		ExposedPorts: {
+			[`${server.metricsConfig.server.port}/tcp`]: {},
+		},
+	};
+	const docker = await getRemoteDocker(serverId);
+	try {
+		await execAsyncRemote(
+			serverId,
+			"mkdir -p /etc/dokploy/monitoring && touch /etc/dokploy/monitoring/monitoring.db",
+		);
+		if (serverId) {
+			await pullRemoteImage(imageName, serverId);
+		}
+
+		// Check if container exists
+		const container = docker.getContainer(containerName);
+		try {
+			await container.inspect();
+			await container.remove({ force: true });
+			console.log("Removed existing container");
+		} catch (error) {
+			// Container doesn't exist, continue
+		}
+
+		await docker.createContainer(settings);
+		const newContainer = docker.getContainer(containerName);
+		await newContainer.start();
+
+		console.log("Monitoring Started ");
+	} catch (error) {
+		console.log("Monitoring Not Found: Starting ", error);
+	}
+};
+
+export const setupWebMonitoring = async (adminId: string) => {
+	const admin = await findAdminById(adminId);
+
+	const containerName = "mauricio-monitoring";
+	const imageName = "siumauricio/monitoring:canary";
+
+	const settings: ContainerCreateOptions = {
+		name: containerName,
+		Env: [`METRICS_CONFIG=${JSON.stringify(admin?.metricsConfig)}`],
+		Image: imageName,
+		HostConfig: {
+			// Memory: 100 * 1024 * 1024, // 100MB en bytes
+			// PidMode: "host",
+			// CapAdd: ["NET_ADMIN", "SYS_ADMIN"],
+			// Privileged: true,
+			PortBindings: {
+				[`${admin.metricsConfig.server.port}/tcp`]: [
+					{
+						HostPort: admin.metricsConfig.server.port.toString(),
+					},
+				],
+			},
+			Binds: [
+				"/var/run/docker.sock:/var/run/docker.sock:ro",
+				"/sys:/host/sys:ro",
+				"/etc/os-release:/etc/os-release:ro",
+				"/proc:/host/proc:ro",
+				"/etc/dokploy/monitoring/monitoring.db:/app/monitoring.db",
+			],
+			// NetworkMode: "host",
+		},
+		ExposedPorts: {
+			[`${admin.metricsConfig.server.port}/tcp`]: {},
+		},
+	};
+	const docker = await getRemoteDocker();
+	try {
+		await execAsync(
+			"mkdir -p /etc/dokploy/monitoring && touch /etc/dokploy/monitoring/monitoring.db",
+		);
+		await pullImage(imageName);
+
+		const container = docker.getContainer(containerName);
+		try {
+			await container.inspect();
+			await container.remove({ force: true });
+			console.log("Removed existing container");
+		} catch (error) {}
+
+		await docker.createContainer(settings);
+		const newContainer = docker.getContainer(containerName);
+		await newContainer.start();
+
+		console.log("Monitoring Started ");
+	} catch (error) {
+		console.log("Monitoring Not Found: Starting ", error);
+	}
+};

packages/server/src/utils/notifications/server-threshold.ts

@@ -0,0 +1,155 @@
+import { and, eq } from "drizzle-orm";
+import { db } from "../../db";
+import { notifications } from "../../db/schema";
+import {
+	sendDiscordNotification,
+	sendSlackNotification,
+	sendTelegramNotification,
+} from "./utils";
+
+interface ServerThresholdPayload {
+	Type: "CPU" | "Memory";
+	Value: number;
+	Threshold: number;
+	Message: string;
+	Timestamp: string;
+	Token: string;
+	ServerName: string;
+}
+
+export const sendServerThresholdNotifications = async (
+	adminId: string,
+	payload: ServerThresholdPayload,
+) => {
+	const date = new Date(payload.Timestamp);
+	const unixDate = ~~(Number(date) / 1000);
+
+	const notificationList = await db.query.notifications.findMany({
+		where: and(
+			eq(notifications.serverThreshold, true),
+			eq(notifications.adminId, adminId),
+		),
+		with: {
+			email: true,
+			discord: true,
+			telegram: true,
+			slack: true,
+		},
+	});
+
+	const typeEmoji = payload.Type === "CPU" ? "🔲" : "💾";
+	const typeColor = 0xff0000; // Rojo para indicar alerta
+
+	for (const notification of notificationList) {
+		const { discord, telegram, slack } = notification;
+
+		if (discord) {
+			const decorate = (decoration: string, text: string) =>
+				`${discord.decoration ? decoration : ""} ${text}`.trim();
+
+			await sendDiscordNotification(discord, {
+				title: decorate(">", `\`⚠️\` Server ${payload.Type} Alert`),
+				color: typeColor,
+				fields: [
+					{
+						name: decorate("`🏷️`", "Server Name"),
+						value: payload.ServerName,
+						inline: true,
+					},
+					{
+						name: decorate("`📅`", "Date"),
+						value: `<t:${unixDate}:D>`,
+						inline: true,
+					},
+					{
+						name: decorate("`⌚`", "Time"),
+						value: `<t:${unixDate}:t>`,
+						inline: true,
+					},
+					{
+						name: decorate(typeEmoji, "Type"),
+						value: payload.Type,
+						inline: true,
+					},
+					{
+						name: decorate("📊", "Current Value"),
+						value: `${payload.Value.toFixed(2)}%`,
+						inline: true,
+					},
+					{
+						name: decorate("⚠️", "Threshold"),
+						value: `${payload.Threshold.toFixed(2)}%`,
+						inline: true,
+					},
+					{
+						name: decorate("`📜`", "Message"),
+						value: `\`\`\`${payload.Message}\`\`\``,
+					},
+				],
+				timestamp: date.toISOString(),
+				footer: {
+					text: "Dokploy Server Monitoring Alert",
+				},
+			});
+		}
+
+		if (telegram) {
+			await sendTelegramNotification(
+				telegram,
+				`
+				<b>⚠️ Server ${payload.Type} Alert</b>
+                <b>Server Name:</b> ${payload.ServerName}
+				<b>Type:</b> ${payload.Type}
+				<b>Current Value:</b> ${payload.Value.toFixed(2)}%
+				<b>Threshold:</b> ${payload.Threshold.toFixed(2)}%
+				<b>Message:</b> ${payload.Message}
+				<b>Time:</b> ${date.toLocaleString()}
+			`,
+			);
+		}
+
+		if (slack) {
+			const { channel } = slack;
+			await sendSlackNotification(slack, {
+				channel: channel,
+				attachments: [
+					{
+						color: "#FF0000",
+						pretext: `:warning: *Server ${payload.Type} Alert*`,
+						fields: [
+							{
+								title: "Server Name",
+								value: payload.ServerName,
+								short: true,
+							},
+							{
+								title: "Type",
+								value: payload.Type,
+								short: true,
+							},
+							{
+								title: "Current Value",
+								value: `${payload.Value.toFixed(2)}%`,
+								short: true,
+							},
+							{
+								title: "Threshold",
+								value: `${payload.Threshold.toFixed(2)}%`,
+								short: true,
+							},
+							{
+								title: "Message",
+								value: payload.Message,
+							},
+							{
+								title: "Time",
+								value: date.toLocaleString(),
+								short: true,
+							},
+						],
+					},
+				],
+			});
+		}
+	}
+};