refactor(multi-server): add rclone to multi server

apps/dokploy/components/dashboard/project/add-database.tsx

@@ -80,9 +80,7 @@ const baseDatabaseSchema = z.object({
 	databasePassword: z.string(),
 	dockerImage: z.string(),
 	description: z.string().nullable(),
-	serverId: z.string().min(1, {
+	serverId: z.string().nullable(),
-		message: "Server is required",
-	}),
 });
 
 const mySchema = z.discriminatedUnion("type", [
@@ -174,6 +172,7 @@ export const AddDatabase = ({ projectId, projectName }: Props) => {
 			description: "",
 			databaseName: "",
 			databaseUser: "",
+			serverId: null,
 		},
 		resolver: zodResolver(mySchema),
 	});
@@ -222,6 +221,7 @@ export const AddDatabase = ({ projectId, projectName }: Props) => {
 			promise = redisMutation.mutateAsync({
 				...commonParams,
 				databasePassword: data.databasePassword,
+				serverId: data.serverId,
 				projectId,
 			});
 		} else if (data.type === "mariadb") {
@@ -379,7 +379,7 @@ export const AddDatabase = ({ projectId, projectName }: Props) => {
 											<FormLabel>Select a Server</FormLabel>
 											<Select
 												onValueChange={field.onChange}
-												defaultValue={field.value}
+												defaultValue={field.value || ""}
 											>
 												<SelectTrigger>
 													<SelectValue placeholder="Select a Server" />

apps/dokploy/components/dashboard/settings/servers/show-servers.tsx

@@ -25,6 +25,7 @@ import { toast } from "sonner";
 import { TerminalModal } from "../web-server/terminal-modal";
 import { AddServer } from "./add-server";
 import { SetupServer } from "./setup-server";
+import { UpdateServer } from "./update-server";
 export const ShowServers = () => {
 	const { data, refetch } = api.server.all.useQuery();
 	const { mutateAsync } = api.server.remove.useMutation();
@@ -83,6 +84,7 @@ export const ShowServers = () => {
 									<TableHead className="text-center">IP Address</TableHead>
 									<TableHead className="text-center">Port</TableHead>
 									<TableHead className="text-center">Username</TableHead>
+									<TableHead className="text-center">SSH Key</TableHead>
 									<TableHead className="text-center">Created</TableHead>
 									<TableHead className="text-right">Actions</TableHead>
 								</TableRow>
@@ -101,6 +103,11 @@ export const ShowServers = () => {
 											<TableCell className="text-center">
 												{server.username}
 											</TableCell>
+											<TableCell className="text-right">
+												<span className="text-sm text-muted-foreground">
+													{server.sshKeyId ? "Yes" : "No"}
+												</span>
+											</TableCell>
 											<TableCell className="text-right">
 												<span className="text-sm text-muted-foreground">
 													{format(new Date(server.createdAt), "PPpp")}
@@ -121,7 +128,7 @@ export const ShowServers = () => {
 															<span>Enter the terminal</span>
 														</TerminalModal>
 														<SetupServer serverId={server.serverId} />
-
+														<UpdateServer serverId={server.serverId} />
 														<DialogAction
 															title={"Delete Server"}
 															description="This will delete the server and all associated data"

apps/dokploy/components/dashboard/settings/servers/update-server.tsx

@@ -0,0 +1,269 @@
+import { AlertBlock } from "@/components/shared/alert-block";
+import { Button } from "@/components/ui/button";
+import {
+	Dialog,
+	DialogContent,
+	DialogDescription,
+	DialogFooter,
+	DialogHeader,
+	DialogTitle,
+	DialogTrigger,
+} from "@/components/ui/dialog";
+import { DropdownMenuItem } from "@/components/ui/dropdown-menu";
+import {
+	Form,
+	FormControl,
+	FormField,
+	FormItem,
+	FormLabel,
+	FormMessage,
+} from "@/components/ui/form";
+import { Input } from "@/components/ui/input";
+import {
+	Select,
+	SelectContent,
+	SelectGroup,
+	SelectItem,
+	SelectLabel,
+	SelectTrigger,
+	SelectValue,
+} from "@/components/ui/select";
+import { Textarea } from "@/components/ui/textarea";
+import { api } from "@/utils/api";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { PlusIcon } from "lucide-react";
+import { useRouter } from "next/router";
+import { useEffect, useState } from "react";
+import { useForm } from "react-hook-form";
+import { toast } from "sonner";
+import { z } from "zod";
+
+const Schema = z.object({
+	name: z.string().min(1, {
+		message: "Name is required",
+	}),
+	description: z.string().optional(),
+	ipAddress: z.string().min(1, {
+		message: "IP Address is required",
+	}),
+	port: z.number().optional(),
+	username: z.string().optional(),
+	sshKeyId: z.string().min(1, {
+		message: "SSH Key is required",
+	}),
+});
+
+type Schema = z.infer<typeof Schema>;
+
+interface Props {
+	serverId: string;
+}
+
+export const UpdateServer = ({ serverId }: Props) => {
+	const utils = api.useUtils();
+	const [isOpen, setIsOpen] = useState(false);
+	const { data, isLoading } = api.server.one.useQuery(
+		{
+			serverId,
+		},
+		{
+			enabled: !!serverId,
+		},
+	);
+	const { data: sshKeys } = api.sshKey.all.useQuery();
+	const { mutateAsync, error, isError } = api.server.update.useMutation();
+	const form = useForm<Schema>({
+		defaultValues: {
+			description: "",
+			name: "",
+			ipAddress: "",
+			port: 22,
+			username: "root",
+			sshKeyId: "",
+		},
+		resolver: zodResolver(Schema),
+	});
+
+	useEffect(() => {
+		form.reset({
+			description: data?.description || "",
+			name: data?.name || "",
+			ipAddress: data?.ipAddress || "",
+			port: data?.port || 22,
+			username: data?.username || "root",
+			sshKeyId: data?.sshKeyId || "",
+		});
+	}, [form, form.reset, form.formState.isSubmitSuccessful, data]);
+
+	const onSubmit = async (formData: Schema) => {
+		await mutateAsync({
+			name: formData.name,
+			description: formData.description || "",
+			ipAddress: formData.ipAddress || "",
+			port: formData.port || 22,
+			username: formData.username || "root",
+			sshKeyId: formData.sshKeyId || "",
+			serverId: serverId,
+		})
+			.then(async (data) => {
+				await utils.server.all.invalidate();
+				toast.success("Server Updated");
+				setIsOpen(false);
+			})
+			.catch(() => {
+				toast.error("Error to update a server");
+			});
+	};
+
+	return (
+		<Dialog open={isOpen} onOpenChange={setIsOpen}>
+			<DialogTrigger asChild>
+				<DropdownMenuItem
+					className="w-full cursor-pointer "
+					onSelect={(e) => e.preventDefault()}
+				>
+					Edit Server
+				</DropdownMenuItem>
+			</DialogTrigger>
+			<DialogContent className="sm:max-w-3xl ">
+				<DialogHeader>
+					<DialogTitle>Update Server</DialogTitle>
+					<DialogDescription>
+						Update a server to deploy your applications remotely.
+					</DialogDescription>
+				</DialogHeader>
+				{isError && <AlertBlock type="error">{error?.message}</AlertBlock>}
+				<Form {...form}>
+					<form
+						id="hook-form-update-server"
+						onSubmit={form.handleSubmit(onSubmit)}
+						className="grid w-full gap-4"
+					>
+						<div className="flex flex-col gap-4 ">
+							<FormField
+								control={form.control}
+								name="name"
+								render={({ field }) => (
+									<FormItem>
+										<FormLabel>Name</FormLabel>
+										<FormControl>
+											<Input placeholder="Hostinger Server" {...field} />
+										</FormControl>
+
+										<FormMessage />
+									</FormItem>
+								)}
+							/>
+						</div>
+						<FormField
+							control={form.control}
+							name="description"
+							render={({ field }) => (
+								<FormItem>
+									<FormLabel>Description</FormLabel>
+									<FormControl>
+										<Textarea
+											placeholder="This server is for databases..."
+											className="resize-none"
+											{...field}
+										/>
+									</FormControl>
+
+									<FormMessage />
+								</FormItem>
+							)}
+						/>
+						<FormField
+							control={form.control}
+							name="sshKeyId"
+							render={({ field }) => (
+								<FormItem>
+									<FormLabel>Select a SSH Key</FormLabel>
+									<Select
+										onValueChange={field.onChange}
+										defaultValue={field.value}
+									>
+										<SelectTrigger>
+											<SelectValue placeholder="Select a SSH Key" />
+										</SelectTrigger>
+										<SelectContent>
+											<SelectGroup>
+												{sshKeys?.map((sshKey) => (
+													<SelectItem
+														key={sshKey.sshKeyId}
+														value={sshKey.sshKeyId}
+													>
+														{sshKey.name}
+													</SelectItem>
+												))}
+												<SelectLabel>
+													Registries ({sshKeys?.length})
+												</SelectLabel>
+											</SelectGroup>
+										</SelectContent>
+									</Select>
+									<FormMessage />
+								</FormItem>
+							)}
+						/>
+						<div className="grid grid-cols-2 gap-4">
+							<FormField
+								control={form.control}
+								name="ipAddress"
+								render={({ field }) => (
+									<FormItem>
+										<FormLabel>IP Address</FormLabel>
+										<FormControl>
+											<Input placeholder="192.168.1.100" {...field} />
+										</FormControl>
+
+										<FormMessage />
+									</FormItem>
+								)}
+							/>
+							<FormField
+								control={form.control}
+								name="port"
+								render={({ field }) => (
+									<FormItem>
+										<FormLabel>Port</FormLabel>
+										<FormControl>
+											<Input placeholder="22" {...field} />
+										</FormControl>
+
+										<FormMessage />
+									</FormItem>
+								)}
+							/>
+						</div>
+
+						<FormField
+							control={form.control}
+							name="username"
+							render={({ field }) => (
+								<FormItem>
+									<FormLabel>Username</FormLabel>
+									<FormControl>
+										<Input placeholder="root" {...field} />
+									</FormControl>
+
+									<FormMessage />
+								</FormItem>
+							)}
+						/>
+					</form>
+
+					<DialogFooter>
+						<Button
+							isLoading={form.formState.isSubmitting}
+							form="hook-form-update-server"
+							type="submit"
+						>
+							Update
+						</Button>
+					</DialogFooter>
+				</Form>
+			</DialogContent>
+		</Dialog>
+	);
+};

apps/dokploy/server/api/routers/compose.ts

@@ -15,10 +15,7 @@ import {
 } from "@/server/queues/deployments-queue";
 import { myQueue } from "@/server/queues/queueSetup";
 import { createCommand } from "@/server/utils/builders/compose";
-import {
+import { randomizeComposeFile } from "@/server/utils/docker/compose";
-	randomizeComposeFile,
-	randomizeSpecificationFile,
-} from "@/server/utils/docker/compose";
 import {
 	addDomainToCompose,
 	cloneCompose,
@@ -252,8 +249,9 @@ export const composeRouter = createTRPCRouter({
 			if (input.serverId) {
 				const server = await findServerById(input.serverId);
 				serverIp = server.ipAddress;
+			} else if (process.env.NODE_ENV === "development") {
+				serverIp = "127.0.0.1";
 			}
-
 			const projectName = slugify(`${project.name} ${input.id}`);
 			const { envs, mounts, domains } = generate({
 				serverIp: serverIp || "",

apps/dokploy/server/api/services/compose.ts

@@ -335,15 +335,8 @@ export const deployRemoteCompose = async ({
 				command += getCreateComposeFileCommand(compose, deployment.logPath);
 			}
 
-			async function* sequentialSteps() {
+			await execAsyncRemote(compose.serverId, command);
-				yield execAsyncRemote(compose.serverId, command);
+			await getBuildComposeCommand(compose, deployment.logPath);
-				yield getBuildComposeCommand(compose, deployment.logPath);
-			}
-
-			const steps = sequentialSteps();
-			for await (const step of steps) {
-				step;
-			}
 
 			console.log(" ---- done ----");
 		}

apps/dokploy/server/api/services/postgres.ts

@@ -116,8 +116,9 @@ export const removePostgresById = async (postgresId: string) => {
 export const deployPostgres = async (postgresId: string) => {
 	const postgres = await findPostgresById(postgresId);
 	try {
+		const promises = [];
 		if (postgres.serverId) {
-			await execAsyncRemote(
+			const result = await execAsyncRemote(
 				postgres.serverId,
 				`docker pull ${postgres.dockerImage}`,
 			);

apps/dokploy/server/db/schema/mariadb.ts

@@ -83,6 +83,7 @@ const createSchema = createInsertSchema(mariadb, {
 	applicationStatus: z.enum(["idle", "running", "done", "error"]),
 	externalPort: z.number(),
 	description: z.string().optional(),
+	serverId: z.string().optional(),
 });
 
 export const apiCreateMariaDB = createSchema

apps/dokploy/server/db/schema/mongo.ts

@@ -77,6 +77,7 @@ const createSchema = createInsertSchema(mongo, {
 	applicationStatus: z.enum(["idle", "running", "done", "error"]),
 	externalPort: z.number(),
 	description: z.string().optional(),
+	serverId: z.string().optional(),
 });
 
 export const apiCreateMongo = createSchema

apps/dokploy/server/db/schema/mysql.ts

@@ -80,6 +80,7 @@ const createSchema = createInsertSchema(mysql, {
 	applicationStatus: z.enum(["idle", "running", "done", "error"]),
 	externalPort: z.number(),
 	description: z.string().optional(),
+	serverId: z.string().optional(),
 });
 
 export const apiCreateMySql = createSchema

apps/dokploy/server/db/schema/postgres.ts

@@ -78,6 +78,7 @@ const createSchema = createInsertSchema(postgres, {
 	externalPort: z.number(),
 	createdAt: z.string(),
 	description: z.string().optional(),
+	serverId: z.string().optional(),
 });
 
 export const apiCreatePostgres = createSchema

apps/dokploy/server/db/schema/redis.ts

@@ -72,6 +72,7 @@ const createSchema = createInsertSchema(redis, {
 	applicationStatus: z.enum(["idle", "running", "done", "error"]),
 	externalPort: z.number(),
 	description: z.string().optional(),
+	serverId: z.string().optional(),
 });
 
 export const apiCreateRedis = createSchema

apps/dokploy/server/db/schema/server.ts

@@ -95,5 +95,9 @@ export const apiUpdateServer = createSchema
 		name: true,
 		description: true,
 		serverId: true,
+		ipAddress: true,
+		port: true,
+		username: true,
+		sshKeyId: true,
 	})
 	.required();

apps/dokploy/server/utils/backups/postgres.ts

@@ -5,7 +5,7 @@ import type { Postgres } from "@/server/api/services/postgres";
 import { findProjectById } from "@/server/api/services/project";
 import { getServiceContainer } from "../docker/utils";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
-import { execAsync } from "../process/execAsync";
+import { execAsync, execAsyncRemote } from "../process/execAsync";
 import { uploadToS3 } from "./utils";
 
 export const runPostgresBackup = async (
@@ -57,3 +57,57 @@ export const runPostgresBackup = async (
 
 // Restore
 // /Applications/pgAdmin 4.app/Contents/SharedSupport/pg_restore --host "localhost" --port "5432" --username "mauricio" --no-password --dbname "postgres" --verbose "/Users/mauricio/Downloads/_databases_2024-04-12T07_02_05.234Z.sql"
+
+export const runRemotePostgresBackup = async (
+	postgres: Postgres,
+	backup: BackupSchedule,
+) => {
+	const { appName, databaseUser, name, projectId } = postgres;
+	const project = await findProjectById(projectId);
+
+	const { prefix, database } = backup;
+	const destination = backup.destination;
+	const backupFileName = `${new Date().toISOString()}.sql.gz`;
+	const bucketDestination = path.join(prefix, backupFileName);
+	const containerPath = `/backup/${backupFileName}`;
+	const hostPath = `./${backupFileName}`;
+	try {
+		const { Id: containerId } = await getServiceContainer(appName);
+		const pgDumpCommand = `pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password '${database}' | gzip`;
+		// const rcloneCommand = `rclone rcat --buffer-size 16M ${rcloneDestination}`;
+
+		// const command = `
+		// 	// docker exec ${containerId} /bin/bash -c "${pgDumpCommand} | ${rcloneCommand}"
+		// `;
+
+		await execAsyncRemote(
+			postgres.serverId,
+			`docker exec ${containerId} /bin/bash -c "rm -rf /backup && mkdir -p /backup"`,
+		);
+		// await execAsync(
+		// 	`docker exec ${containerId} sh -c "pg_dump -Fc --no-acl --no-owner -h localhost -U ${databaseUser} --no-password  '${database}' | gzip > ${containerPath}"`,
+		// );
+		await execAsync(`docker cp ${containerId}:${containerPath} ${hostPath}`);
+
+		await uploadToS3(destination, bucketDestination, hostPath);
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "postgres",
+			type: "success",
+		});
+	} catch (error) {
+		await sendDatabaseBackupNotifications({
+			applicationName: name,
+			projectName: project.name,
+			databaseType: "postgres",
+			type: "error",
+			// @ts-ignore
+			errorMessage: error?.message || "Error message not provided",
+		});
+
+		throw error;
+	} finally {
+		await unlink(hostPath);
+	}
+};

apps/dokploy/server/utils/databases/mongo.ts

@@ -1,5 +1,3 @@
-import type { Mongo } from "@/server/api/services/mongo";
-import type { Mount } from "@/server/api/services/mount";
 import type { CreateServiceOptions } from "dockerode";
 import {
 	calculateResources,

apps/dokploy/server/utils/process/execAsync.ts

@@ -15,10 +15,12 @@ export const execAsyncRemote = async (
 
 	const keys = await readSSHKey(server.sshKeyId);
 
-	const conn = new Client();
 	let stdout = "";
 	let stderr = "";
 	return new Promise((resolve, reject) => {
+		const conn = new Client();
+
+		sleep(1000);
 		conn
 			.once("ready", () => {
 				console.log("Client :: ready");
@@ -57,3 +59,7 @@ export const execAsyncRemote = async (
 			});
 	});
 };
+
+export const sleep = (ms: number) => {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+};

apps/dokploy/server/utils/servers/setup-server.ts

@@ -74,6 +74,7 @@ const connectToServer = async (serverId: string, logPath: string) => {
 				command_exists() {
 					command -v "$@" > /dev/null 2>&1
 				}
+				${installRClone()}
 				${installDocker()}
 				${setupSwarm()}
 				${setupNetwork()}
@@ -235,6 +236,10 @@ export const createDefaultMiddlewares = () => {
 	return command;
 };
 
+export const installRClone = () => `
+curl https://rclone.org/install.sh | sudo bash
+`;
+
 export const createTraefikInstance = () => {
 	const command = `
 	    # Check if dokpyloy-traefik exists

apps/dokploy/server/wss/listen-deployment.ts

@@ -45,9 +45,7 @@ export const setupDeploymentLogsWebSocketServer = (
 		}
 
 		try {
-			console.log(serverId);
 			if (serverId) {
-				console.log("Entre aca");
 				const server = await findServerById(serverId);
 
 				if (!server.sshKeyId) return;
@@ -90,8 +88,6 @@ export const setupDeploymentLogsWebSocketServer = (
 						});
 				});
 			} else {
-				console.log("Entre aca2");
-				console.log(logPath);
 				const tail = spawn("tail", ["-n", "+1", "-f", logPath]);
 
 				tail.stdout.on("data", (data) => {
@@ -105,7 +101,7 @@ export const setupDeploymentLogsWebSocketServer = (
 		} catch (error) {
 			// @ts-ignore
 			//   const errorMessage = error?.message as unknown as string;
-			//   ws.send(errorMessage);
+			ws.send(errorMessage);
 		}
 	});
 };

pnpm-lock.yaml

@@ -14110,7 +14110,7 @@ snapshots:
       eslint: 8.45.0
       eslint-import-resolver-node: 0.3.9
       eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0))(eslint@8.45.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1)(eslint@8.45.0)
       eslint-plugin-jsx-a11y: 6.9.0(eslint@8.45.0)
       eslint-plugin-react: 7.35.0(eslint@8.45.0)
       eslint-plugin-react-hooks: 5.0.0-canary-7118f5dd7-20230705(eslint@8.45.0)
@@ -14134,7 +14134,7 @@ snapshots:
       enhanced-resolve: 5.17.1
       eslint: 8.45.0
       eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0))(eslint@8.45.0)
-      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0))(eslint@8.45.0)
+      eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1)(eslint@8.45.0)
       fast-glob: 3.3.2
       get-tsconfig: 4.7.5
       is-core-module: 2.15.0
@@ -14156,7 +14156,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.45.0))(eslint@8.45.0))(eslint@8.45.0):
+  eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.45.0)(typescript@5.1.6))(eslint-import-resolver-typescript@3.6.1)(eslint@8.45.0):
     dependencies:
       array-includes: 3.1.8
       array.prototype.findlastindex: 1.2.5