Merge remote-tracking branch 'upstream/canary' into add-disable-recurse-submodules-option

packages/server/src/db/schema/user.ts

@@ -50,6 +50,7 @@ export const users_temp = pgTable("user_temp", {
 	// Admin
 	serverIp: text("serverIp"),
 	certificateType: certificateType("certificateType").notNull().default("none"),
+	https: boolean("https").notNull().default(false),
 	host: text("host"),
 	letsEncryptEmail: text("letsEncryptEmail"),
 	sshPrivateKey: text("sshPrivateKey"),
@@ -202,10 +203,12 @@ export const apiAssignDomain = createSchema
 		host: true,
 		certificateType: true,
 		letsEncryptEmail: true,
+		https: true,
 	})
 	.required()
 	.partial({
 		letsEncryptEmail: true,
+		https: true,
 	});
 
 export const apiUpdateDockerCleanup = createSchema

packages/server/src/templates/processors.ts

@@ -1,3 +1,4 @@
+import { faker } from "@faker-js/faker";
 import type { Schema } from "./index";
 import {
 	generateBase64,
@@ -70,7 +71,7 @@ function processValue(
 	schema: Schema,
 ): string {
 	// First replace utility functions
-	let processedValue = value.replace(/\${([^}]+)}/g, (match, varName) => {
+	let processedValue = value?.replace(/\${([^}]+)}/g, (match, varName) => {
 		// Handle utility functions
 		if (varName === "domain") {
 			return generateRandomDomain(schema);
@@ -117,6 +118,14 @@ function processValue(
 			return generateJwt(length);
 		}
 
+		if (varName === "username") {
+			return faker.internet.userName().toLowerCase();
+		}
+
+		if (varName === "email") {
+			return faker.internet.email().toLowerCase();
+		}
+
 		// If not a utility function, try to get from variables
 		return variables[varName] || match;
 	});
@@ -177,7 +186,14 @@ export function processDomains(
 	variables: Record<string, string>,
 	schema: Schema,
 ): Template["domains"] {
-	if (!template?.config?.domains) return [];
+	if (
+		!template?.config?.domains ||
+		template.config.domains.length === 0 ||
+		template.config.domains.every((domain) => !domain.serviceName)
+	) {
+		return [];
+	}
+
 	return template?.config?.domains?.map((domain: DomainConfig) => ({
 		...domain,
 		host: domain.host
@@ -194,7 +210,9 @@ export function processEnvVars(
 	variables: Record<string, string>,
 	schema: Schema,
 ): Template["envs"] {
-	if (!template?.config?.env) return [];
+	if (!template?.config?.env || Object.keys(template.config.env).length === 0) {
+		return [];
+	}
 
 	// Handle array of env vars
 	if (Array.isArray(template.config.env)) {
@@ -233,7 +251,13 @@ export function processMounts(
 	variables: Record<string, string>,
 	schema: Schema,
 ): Template["mounts"] {
-	if (!template?.config?.mounts) return [];
+	if (
+		!template?.config?.mounts ||
+		template.config.mounts.length === 0 ||
+		template.config.mounts.every((mount) => !mount.filePath && !mount.content)
+	) {
+		return [];
+	}
 
 	return template?.config?.mounts?.map((mount: MountConfig) => ({
 		filePath: processValue(mount.filePath, variables, schema),

packages/server/src/utils/backups/index.ts

@@ -2,7 +2,6 @@ import path from "node:path";
 import { getAllServers } from "@dokploy/server/services/server";
 import { scheduleJob } from "node-schedule";
 import { db } from "../../db/index";
-import { findAdmin } from "../../services/admin";
 import {
 	cleanUpDockerBuilder,
 	cleanUpSystemPrune,
@@ -14,13 +13,24 @@ import { getS3Credentials, scheduleBackup } from "./utils";
 
 import type { BackupSchedule } from "@dokploy/server/services/backup";
 import { startLogCleanup } from "../access-log/handler";
+import { member } from "@dokploy/server/db/schema";
+import { eq } from "drizzle-orm";
 
 export const initCronJobs = async () => {
 	console.log("Setting up cron jobs....");
 
-	const admin = await findAdmin();
+	const admin = await db.query.member.findFirst({
+		where: eq(member.role, "owner"),
+		with: {
+			user: true,
+		},
+	});
 
-	if (admin?.user.enableDockerCleanup) {
+	if (!admin) {
+		return;
+	}
+
+	if (admin.user.enableDockerCleanup) {
 		scheduleJob("docker-cleanup", "0 0 * * *", async () => {
 			console.log(
 				`Docker Cleanup ${new Date().toLocaleString()}]  Running docker cleanup`,
@@ -96,8 +106,8 @@ export const keepLatestNBackups = async (
 			backup.prefix,
 		);
 
-		// --include "*.sql.gz" ensures nothing else other than the db backup files are touched by rclone
-		const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*.sql.gz" ${backupFilesPath}`;
+		// --include "*.sql.gz" or "*.zip" ensures nothing else other than the dokploy backup files are touched by rclone
+		const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".sql.gz"}" ${backupFilesPath}`;
 		// when we pipe the above command with this one, we only get the list of files we want to delete
 		const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`;
 		// this command deletes the files

packages/server/src/utils/backups/mariadb.ts

@@ -1,4 +1,3 @@
-import path from "node:path";
 import type { BackupSchedule } from "@dokploy/server/services/backup";
 import type { Mariadb } from "@dokploy/server/services/mariadb";
 import { findProjectById } from "@dokploy/server/services/project";
@@ -8,7 +7,7 @@ import {
 } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { getS3Credentials } from "./utils";
+import { getS3Credentials, normalizeS3Path } from "./utils";
 
 export const runMariadbBackup = async (
 	mariadb: Mariadb,
@@ -19,7 +18,7 @@ export const runMariadbBackup = async (
 	const { prefix, database } = backup;
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.sql.gz`;
-	const bucketDestination = path.join(prefix, backupFileName);
+	const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
 
 	try {
 		const rcloneFlags = getS3Credentials(destination);

packages/server/src/utils/backups/mongo.ts

@@ -1,4 +1,3 @@
-import path from "node:path";
 import type { BackupSchedule } from "@dokploy/server/services/backup";
 import type { Mongo } from "@dokploy/server/services/mongo";
 import { findProjectById } from "@dokploy/server/services/project";
@@ -8,7 +7,7 @@ import {
 } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { getS3Credentials } from "./utils";
+import { getS3Credentials, normalizeS3Path } from "./utils";
 
 // mongodb://mongo:Bqh7AQl-PRbnBu@localhost:27017/?tls=false&directConnection=true
 export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
@@ -17,7 +16,7 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
 	const { prefix, database } = backup;
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.dump.gz`;
-	const bucketDestination = path.join(prefix, backupFileName);
+	const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
 
 	try {
 		const rcloneFlags = getS3Credentials(destination);

packages/server/src/utils/backups/mysql.ts

@@ -1,4 +1,3 @@
-import path from "node:path";
 import type { BackupSchedule } from "@dokploy/server/services/backup";
 import type { MySql } from "@dokploy/server/services/mysql";
 import { findProjectById } from "@dokploy/server/services/project";
@@ -8,7 +7,7 @@ import {
 } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { getS3Credentials } from "./utils";
+import { getS3Credentials, normalizeS3Path } from "./utils";
 
 export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
 	const { appName, databaseRootPassword, projectId, name } = mysql;
@@ -16,7 +15,7 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
 	const { prefix, database } = backup;
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.sql.gz`;
-	const bucketDestination = path.join(prefix, backupFileName);
+	const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
 
 	try {
 		const rcloneFlags = getS3Credentials(destination);

packages/server/src/utils/backups/postgres.ts

@@ -1,4 +1,3 @@
-import path from "node:path";
 import type { BackupSchedule } from "@dokploy/server/services/backup";
 import type { Postgres } from "@dokploy/server/services/postgres";
 import { findProjectById } from "@dokploy/server/services/project";
@@ -8,7 +7,7 @@ import {
 } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
 import { execAsync, execAsyncRemote } from "../process/execAsync";
-import { getS3Credentials } from "./utils";
+import { getS3Credentials, normalizeS3Path } from "./utils";
 
 export const runPostgresBackup = async (
 	postgres: Postgres,
@@ -20,7 +19,7 @@ export const runPostgresBackup = async (
 	const { prefix, database } = backup;
 	const destination = backup.destination;
 	const backupFileName = `${new Date().toISOString()}.sql.gz`;
-	const bucketDestination = path.join(prefix, backupFileName);
+	const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
 	try {
 		const rcloneFlags = getS3Credentials(destination);
 		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;

packages/server/src/utils/backups/utils.ts

@@ -36,6 +36,13 @@ export const removeScheduleBackup = (backupId: string) => {
 	currentJob?.cancel();
 };
 
+export const normalizeS3Path = (prefix: string) => {
+	// Trim whitespace and remove leading/trailing slashes
+	const normalizedPrefix = prefix.trim().replace(/^\/+|\/+$/g, "");
+	// Return empty string if prefix is empty, otherwise append trailing slash
+	return normalizedPrefix ? `${normalizedPrefix}/` : "";
+};
+
 export const getS3Credentials = (destination: Destination) => {
 	const { accessKey, secretAccessKey, region, endpoint, provider } =
 		destination;

packages/server/src/utils/backups/web-server.ts

@@ -1,6 +1,6 @@
 import type { BackupSchedule } from "@dokploy/server/services/backup";
 import { execAsync } from "../process/execAsync";
-import { getS3Credentials } from "./utils";
+import { getS3Credentials, normalizeS3Path } from "./utils";
 import { findDestinationById } from "@dokploy/server/services/destination";
 import { IS_CLOUD, paths } from "@dokploy/server/constants";
 import { mkdtemp } from "node:fs/promises";
@@ -18,18 +18,28 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
 		const { BASE_PATH } = paths();
 		const tempDir = await mkdtemp(join(tmpdir(), "dokploy-backup-"));
 		const backupFileName = `webserver-backup-${timestamp}.zip`;
-		const s3Path = `:s3:${destination.bucket}/${backup.prefix}${backupFileName}`;
+		const s3Path = `:s3:${destination.bucket}/${normalizeS3Path(backup.prefix)}${backupFileName}`;
 
 		try {
 			await execAsync(`mkdir -p ${tempDir}/filesystem`);
 
-			const postgresCommand = `docker exec $(docker ps --filter "name=dokploy-postgres" -q) pg_dump -v -Fc -U dokploy -d dokploy > ${tempDir}/database.sql`;
+			// First get the container ID
+			const { stdout: containerId } = await execAsync(
+				"docker ps --filter 'name=dokploy-postgres' -q",
+			);
+
+			if (!containerId) {
+				throw new Error("PostgreSQL container not found");
+			}
+
+			// Then run pg_dump with the container ID
+			const postgresCommand = `docker exec ${containerId.trim()} pg_dump -v -Fc -U dokploy -d dokploy > '${tempDir}/database.sql'`;
 			await execAsync(postgresCommand);
 
 			await execAsync(`cp -r ${BASE_PATH}/* ${tempDir}/filesystem/`);
 
 			await execAsync(
-				`cd ${tempDir} && zip -r ${backupFileName} database.sql filesystem/`,
+				`cd ${tempDir} && zip -r ${backupFileName} database.sql filesystem/ > /dev/null 2>&1`,
 			);
 
 			const uploadCommand = `rclone copyto ${rcloneFlags.join(" ")} "${tempDir}/${backupFileName}" "${s3Path}"`;

packages/server/src/utils/builders/railpack.ts

@@ -84,7 +84,7 @@ export const buildRailpack = async (
 		for (const envVar of envVariables) {
 			const [key, value] = envVar.split("=");
 			if (key && value) {
-				buildArgs.push("--secret", `id=${key},env=${key}`);
+				buildArgs.push("--secret", `id=${key},env='${key}'`);
 				env[key] = value;
 			}
 		}
@@ -132,7 +132,7 @@ export const getRailpackCommand = (
 	];
 
 	for (const env of envVariables) {
-		prepareArgs.push("--env", env);
+		prepareArgs.push("--env", `'${env}'`);
 	}
 
 	// Calculate secrets hash for layer invalidation
@@ -164,7 +164,7 @@ export const getRailpackCommand = (
 	for (const envVar of envVariables) {
 		const [key, value] = envVar.split("=");
 		if (key && value) {
-			buildArgs.push("--secret", `id=${key},env=${key}`);
+			buildArgs.push("--secret", `id=${key},env='${key}'`);
 			exportEnvs.push(`export ${key}=${value}`);
 		}
 	}

packages/server/src/utils/builders/static.ts

@@ -25,6 +25,12 @@ export const buildStatic = async (
 			].join("\n"),
 		);
 
+		createFile(
+			buildAppDirectory,
+			".dockerignore",
+			[".git", ".env", "Dockerfile", ".dockerignore"].join("\n"),
+		);
+
 		await buildCustomDocker(
 			{
 				...application,

packages/server/src/utils/docker/domain.ts

@@ -249,6 +249,11 @@ export const addDomainToCompose = async (
 				labels.unshift("traefik.enable=true");
 			}
 			labels.unshift(...httpLabels);
+			if (!compose.isolatedDeployment) {
+				if (!labels.includes("traefik.docker.network=dokploy-network")) {
+					labels.unshift("traefik.docker.network=dokploy-network");
+				}
+			}
 		}
 
 		if (!compose.isolatedDeployment) {

packages/server/src/utils/process/execAsync.ts

@@ -1,9 +1,48 @@
-import { exec } from "node:child_process";
+import { exec, execFile } from "node:child_process";
 import util from "node:util";
 import { findServerById } from "@dokploy/server/services/server";
 import { Client } from "ssh2";
+
 export const execAsync = util.promisify(exec);
 
+export const execFileAsync = async (
+	command: string,
+	args: string[],
+	options: { input?: string } = {},
+): Promise<{ stdout: string; stderr: string }> => {
+	const child = execFile(command, args);
+
+	if (options.input && child.stdin) {
+		child.stdin.write(options.input);
+		child.stdin.end();
+	}
+
+	return new Promise((resolve, reject) => {
+		let stdout = "";
+		let stderr = "";
+
+		child.stdout?.on("data", (data) => {
+			stdout += data.toString();
+		});
+
+		child.stderr?.on("data", (data) => {
+			stderr += data.toString();
+		});
+
+		child.on("close", (code) => {
+			if (code === 0) {
+				resolve({ stdout, stderr });
+			} else {
+				reject(
+					new Error(`Command failed with code ${code}. Stderr: ${stderr}`),
+				);
+			}
+		});
+
+		child.on("error", reject);
+	});
+};
+
 export const execAsyncRemote = async (
 	serverId: string | null,
 	command: string,

packages/server/src/utils/providers/gitlab.ts

@@ -270,7 +270,11 @@ export const getGitlabRepositories = async (gitlabId?: string) => {
 		const groupName = gitlabProvider.groupName?.toLowerCase();
 
 		if (groupName) {
-			return full_path.toLowerCase().includes(groupName) && kind === "group";
+			const isIncluded = groupName
+				.split(",")
+				.some((name) => full_path.toLowerCase().includes(name));
+
+			return isIncluded && kind === "group";
 		}
 		return kind === "user";
 	});
@@ -453,7 +457,9 @@ export const testGitlabConnection = async (
 		const { full_path, kind } = repo.namespace;
 
 		if (groupName) {
-			return full_path.toLowerCase().includes(groupName) && kind === "group";
+			return groupName
+				.split(",")
+				.some((name) => full_path.toLowerCase().includes(name));
 		}
 		return kind === "user";
 	});

packages/server/src/utils/restore/web-server.ts

@@ -45,7 +45,7 @@ export const restoreWebServerBackup = async (
 
 			// Extract backup
 			emit("Extracting backup...");
-			await execAsync(`cd ${tempDir} && unzip ${backupFile}`);
+			await execAsync(`cd ${tempDir} && unzip ${backupFile} > /dev/null 2>&1`);
 
 			// Restore filesystem first
 			emit("Restoring filesystem...");

packages/server/src/utils/traefik/web-server.ts

@@ -3,7 +3,11 @@ import { join } from "node:path";
 import { paths } from "@dokploy/server/constants";
 import type { User } from "@dokploy/server/services/user";
 import { dump, load } from "js-yaml";
-import { loadOrCreateConfig, writeTraefikConfig } from "./application";
+import {
+	loadOrCreateConfig,
+	removeTraefikConfig,
+	writeTraefikConfig,
+} from "./application";
 import type { FileConfig } from "./file-types";
 import type { MainTraefikConfig } from "./types";
 
@@ -11,32 +15,62 @@ export const updateServerTraefik = (
 	user: User | null,
 	newHost: string | null,
 ) => {
+	const { https, certificateType } = user || {};
 	const appName = "dokploy";
 	const config: FileConfig = loadOrCreateConfig(appName);
 
 	config.http = config.http || { routers: {}, services: {} };
 	config.http.routers = config.http.routers || {};
+	config.http.services = config.http.services || {};
 
-	const currentRouterConfig = config.http.routers[`${appName}-router-app`];
+	const currentRouterConfig = config.http.routers[`${appName}-router-app`] || {
+		rule: `Host(\`${newHost}\`)`,
+		service: `${appName}-service-app`,
+		entryPoints: ["web"],
+	};
+	config.http.routers[`${appName}-router-app`] = currentRouterConfig;
 
-	if (currentRouterConfig && newHost) {
-		currentRouterConfig.rule = `Host(\`${newHost}\`)`;
+	config.http.services = {
+		...config.http.services,
+		[`${appName}-service-app`]: {
+			loadBalancer: {
+				servers: [
+					{
+						url: `http://dokploy:${process.env.PORT || 3000}`,
+					},
+				],
+				passHostHeader: true,
+			},
+		},
+	};
 
-		if (user?.certificateType === "letsencrypt") {
+	if (https) {
+		currentRouterConfig.middlewares = ["redirect-to-https"];
+
+		if (certificateType === "letsencrypt") {
 			config.http.routers[`${appName}-router-app-secure`] = {
-				...currentRouterConfig,
+				rule: `Host(\`${newHost}\`)`,
+				service: `${appName}-service-app`,
 				entryPoints: ["websecure"],
 				tls: { certResolver: "letsencrypt" },
 			};
-
-			currentRouterConfig.middlewares = ["redirect-to-https"];
 		} else {
-			delete config.http.routers[`${appName}-router-app-secure`];
-			currentRouterConfig.middlewares = [];
+			config.http.routers[`${appName}-router-app-secure`] = {
+				rule: `Host(\`${newHost}\`)`,
+				service: `${appName}-service-app`,
+				entryPoints: ["websecure"],
+			};
 		}
+	} else {
+		delete config.http.routers[`${appName}-router-app-secure`];
+		currentRouterConfig.middlewares = [];
 	}
 
-	writeTraefikConfig(config, appName);
+	if (newHost) {
+		writeTraefikConfig(config, appName);
+	} else {
+		removeTraefikConfig(appName);
+	}
 };
 
 export const updateLetsEncryptEmail = (newEmail: string | null) => {