Dokploy/dokploy
1
0
Fork 0
mirror of https://github.com/Dokploy/dokploy synced 2025-06-26 18:27:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: streamline cloud storage backup and restore processes by simplifying path construction and enhancing logging

Browse Source
This commit is contained in:
vishalkadam47
2025-05-31 22:03:30 +05:30
parent 2ee40f28ed
commit 5c3e7ed87f
3 changed files with 69 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
apps/dokploy/components/dashboard/database/backups/cloud-storage/restore-cloud-backup.tsx
View File

@@ -280,23 +280,25 @@ export const RestoreCloudBackup = ({ databaseId, databaseType }: Props) => {
const remoteName = (() => { const remoteName = (() => {
switch (selectedBackup.cloudStorageDestination?.provider) { switch (selectedBackup.cloudStorageDestination?.provider) {
case "drive": case "drive":
return "dokploy-drive"; return "drive";
case "dropbox": case "dropbox":
return "dokploy-dropbox"; return "dropbox";
case "box": case "box":
return "dokploy-box"; return "box";
case "ftp":
return "ftp";
case "sftp":
return "sftp";
default: default:
return selectedBackup.cloudStorageDestination?.provider; return selectedBackup.cloudStorageDestination?.provider;
} }
})(); })();
const prefix = selectedBackup.prefix.startsWith("/") const backupPath = `${remoteName}:${values.backupFile}`;
? selectedBackup.prefix.slice(1)
: selectedBackup.prefix;
await restoreBackup({ await restoreBackup({
destinationId: selectedBackup.cloudStorageDestinationId, destinationId: selectedBackup.cloudStorageDestinationId,
backupFile: `${remoteName}:${prefix}/${values.backupFile}`, backupFile: backupPath,
databaseType, databaseType,
databaseName: values.databaseName, databaseName: values.databaseName,
metadata: values.metadata, metadata: values.metadata,
@@ -524,6 +526,7 @@ export const RestoreCloudBackup = ({ databaseId, databaseType }: Props) => {
value={file.path} value={file.path}
key={file.path} key={file.path}
onSelect={() => { onSelect={() => {
console.log("Selected file:", file.path);
form.setValue("backupFile", file.path); form.setValue("backupFile", file.path);
setSearch(""); setSearch("");
setDebouncedSearchTerm(""); setDebouncedSearchTerm("");

6
apps/dokploy/components/dashboard/database/backups/cloud-storage/show-cloud-backups.tsx
View File

@@ -178,9 +178,9 @@ export const ShowCloudBackups = ({ databaseId, databaseType }: Props) => {
<Button <Button
type="button" type="button"
variant="ghost" variant="ghost"
isLoading={ // isLoading={
isManualBackup && activeManualBackup === backup.id // isManualBackup && activeManualBackup === backup.id
} // }
onClick={async () => { onClick={async () => {
setActiveManualBackup(backup.id); setActiveManualBackup(backup.id);
await manualBackup({ await manualBackup({

104
packages/server/src/utils/backups/cloud-storage.ts
View File

@@ -26,26 +26,8 @@ export const normalizeCloudPath = (path: string): string => {
return cleaned ? `${cleaned}/` : ""; return cleaned ? `${cleaned}/` : "";
}; };
export const constructCloudStoragePath = ( export const isCloudStorage = (provider?: string): boolean => {
provider: CloudStorageProvider, return ["drive", "dropbox", "box", "ftp", "sftp"].includes(provider || "");
prefix: string | null,
fileName: string,
): string => {
const normalizedPrefix = normalizeCloudPath(prefix || "");
switch (provider) {
case "drive":
case "dropbox":
case "box":
case "ftp":
return `${provider}:${normalizedPrefix}${fileName}`;
case "sftp": {
const sftpPath = normalizedPrefix.replace(/^\/+/, "");
return `${provider}:${sftpPath}${fileName}`;
}
default:
throw new Error(`Unsupported provider: ${provider}`);
}
}; };
export const getCloudStorageCredentials = async ( export const getCloudStorageCredentials = async (
@@ -157,10 +139,6 @@ encoding = Slash,Del,Ctl,RightSpace,Dot`
return credentials; return credentials;
}; };
export const isCloudStorage = (provider?: string): boolean => {
return ["drive", "dropbox", "box", "ftp", "sftp"].includes(provider || "");
};
export const executeCloudStorageBackup = async ( export const executeCloudStorageBackup = async (
backup: CloudStorageBackup, backup: CloudStorageBackup,
destination: CloudStorageDestination, destination: CloudStorageDestination,
@@ -179,19 +157,19 @@ export const executeCloudStorageBackup = async (
case "postgres": case "postgres":
if (!backup.postgres) if (!backup.postgres)
throw new Error("PostgreSQL configuration missing"); throw new Error("PostgreSQL configuration missing");
backupCommand = `PGPASSWORD="${backup.postgres.password}" pg_dump -h ${backup.postgres.host} -p ${backup.postgres.port} -U ${backup.postgres.user} -d ${backup.database} | gzip | rclone rcat ${(await getCloudStorageCredentials(destination)).join(" ")} "${constructCloudStoragePath(destination.provider as CloudStorageProvider, backup.prefix || "", filename)}"`; backupCommand = `PGPASSWORD="${backup.postgres.password}" pg_dump -h ${backup.postgres.host} -p ${backup.postgres.port} -U ${backup.postgres.user} -d ${backup.database} | gzip > ${tempDir}/${filename}`;
break; break;
case "mysql": case "mysql":
if (!backup.mysql) throw new Error("MySQL configuration missing"); if (!backup.mysql) throw new Error("MySQL configuration missing");
backupCommand = `mysqldump -h ${backup.mysql.host} -P ${backup.mysql.port} -u ${backup.mysql.user} -p${backup.mysql.password} ${backup.database} | gzip | rclone rcat ${(await getCloudStorageCredentials(destination)).join(" ")} "${constructCloudStoragePath(destination.provider as CloudStorageProvider, backup.prefix || "", filename)}"`; backupCommand = `mysqldump -h ${backup.mysql.host} -P ${backup.mysql.port} -u ${backup.mysql.user} -p${backup.mysql.password} ${backup.database} | gzip > ${tempDir}/${filename}`;
break; break;
case "mariadb": case "mariadb":
if (!backup.mariadb) throw new Error("MariaDB configuration missing"); if (!backup.mariadb) throw new Error("MariaDB configuration missing");
backupCommand = `mysqldump -h ${backup.mariadb.host} -P ${backup.mariadb.port} -u ${backup.mariadb.user} -p${backup.mariadb.password} ${backup.database} | gzip | rclone rcat ${(await getCloudStorageCredentials(destination)).join(" ")} "${constructCloudStoragePath(destination.provider as CloudStorageProvider, backup.prefix || "", filename)}"`; backupCommand = `mysqldump -h ${backup.mariadb.host} -P ${backup.mariadb.port} -u ${backup.mariadb.user} -p${backup.mariadb.password} ${backup.database} | gzip > ${tempDir}/${filename}`;
break; break;
case "mongo": case "mongo":
if (!backup.mongo) throw new Error("MongoDB configuration missing"); if (!backup.mongo) throw new Error("MongoDB configuration missing");
backupCommand = `mongodump --host ${backup.mongo.host} --port ${backup.mongo.port} --username ${backup.mongo.user} --password ${backup.mongo.password} --db ${backup.database} --archive --gzip | rclone rcat ${(await getCloudStorageCredentials(destination)).join(" ")} "${constructCloudStoragePath(destination.provider as CloudStorageProvider, backup.prefix || "", filename)}"`; backupCommand = `mongodump --host ${backup.mongo.host} --port ${backup.mongo.port} --username ${backup.mongo.user} --password ${backup.mongo.password} --db ${backup.database} --archive --gzip > ${tempDir}/${filename}`;
break; break;
case "web-server": { case "web-server": {
const { BASE_PATH } = paths(); const { BASE_PATH } = paths();
@@ -231,14 +209,11 @@ export const executeCloudStorageBackup = async (
`cd ${tempDir} && zip -r ${webServerFilename} *.sql filesystem/ > /dev/null 2>&1`, `cd ${tempDir} && zip -r ${webServerFilename} *.sql filesystem/ > /dev/null 2>&1`,
); );
const backupPath = constructCloudStoragePath( const normalizedPrefix = normalizeCloudPath(backup.prefix || "");
destination.provider as CloudStorageProvider, const backupPath = `${destination.provider}:${normalizedPrefix}${webServerFilename}`;
backup.prefix || "",
webServerFilename,
);
const credentials = await getCloudStorageCredentials(destination); const credentials = await getCloudStorageCredentials(destination);
const rcloneCommand = `rclone copyto ${tempDir}/${webServerFilename} "${backupPath}" ${credentials.join(" ")}`; const rcloneCommand = `rclone copyto ${credentials.join(" ")} "${tempDir}/${webServerFilename}" "${backupPath}"`;
await execAsync(rcloneCommand); await execAsync(rcloneCommand);
console.log(`Cloud storage backup completed: ${backupPath}`); console.log(`Cloud storage backup completed: ${backupPath}`);
@@ -253,7 +228,14 @@ export const executeCloudStorageBackup = async (
if (backupCommand) { if (backupCommand) {
await execAsync(backupCommand); await execAsync(backupCommand);
console.log(`Cloud storage backup completed: ${filename}`); const normalizedPrefix = normalizeCloudPath(backup.prefix || "");
const backupPath = `${destination.provider}:${normalizedPrefix}${filename}`;
const credentials = await getCloudStorageCredentials(destination);
const rcloneCommand = `rclone copyto ${credentials.join(" ")} "${tempDir}/${filename}" "${backupPath}"`;
console.log(`Executing rclone command: ${rcloneCommand}`);
await execAsync(rcloneCommand);
console.log(`Cloud storage backup completed: ${backupPath}`);
} }
if (backup.keepLatestCount) { if (backup.keepLatestCount) {
@@ -283,11 +265,26 @@ export const executeCloudStorageRestore = async (
await execAsync(`mkdir -p ${tempDir}`); await execAsync(`mkdir -p ${tempDir}`);
try { try {
const configPath = getRcloneConfigPath( emit("Starting restore...");
destination.organizationId, emit(`Backup path: ${backupPath}`);
destination.id,
); const fileName = backupPath.split("/").pop();
const rcloneCommand = `rclone copy ${backupPath} ${tempDir} --config=${configPath}`; if (!fileName) {
throw new Error("Invalid backup file path");
}
emit("Downloading backup file...");
const credentials = await getCloudStorageCredentials(destination);
// Get just the filename without any provider prefix
const cleanFileName = fileName.replace(/^[^:]+:/, '');
// Construct the full path for the cloud storage provider
const normalizedPrefix = normalizeCloudPath(backup.prefix || "");
const fullPath = `${destination.provider}:${normalizedPrefix}${cleanFileName}`;
const rcloneCommand = `rclone copyto ${credentials.join(" ")} "${fullPath}" "${tempDir}/${cleanFileName}"`;
emit(`Executing rclone command: ${rcloneCommand}`);
await execAsync(rcloneCommand); await execAsync(rcloneCommand);
let restoreCommand = ""; let restoreCommand = "";
@@ -295,19 +292,23 @@ export const executeCloudStorageRestore = async (
case "postgres": case "postgres":
if (!backup.postgres) if (!backup.postgres)
throw new Error("PostgreSQL configuration missing"); throw new Error("PostgreSQL configuration missing");
restoreCommand = `psql -h ${backup.postgres.host} -p ${backup.postgres.port} -U ${backup.postgres.user} -d ${backup.database} < ${tempDir}/${backupPath.split("/").pop()}`; emit("Restoring PostgreSQL database...");
restoreCommand = `gunzip -c ${tempDir}/${cleanFileName} | psql -h ${backup.postgres.host} -p ${backup.postgres.port} -U ${backup.postgres.user} -d ${backup.database}`;
break; break;
case "mysql": case "mysql":
if (!backup.mysql) throw new Error("MySQL configuration missing"); if (!backup.mysql) throw new Error("MySQL configuration missing");
restoreCommand = `mysql -h ${backup.mysql.host} -P ${backup.mysql.port} -u ${backup.mysql.user} -p${backup.mysql.password} ${backup.database} < ${tempDir}/${backupPath.split("/").pop()}`; emit("Restoring MySQL database...");
restoreCommand = `gunzip -c ${tempDir}/${cleanFileName} | mysql -h ${backup.mysql.host} -P ${backup.mysql.port} -u ${backup.mysql.user} -p${backup.mysql.password} ${backup.database}`;
break; break;
case "mariadb": case "mariadb":
if (!backup.mariadb) throw new Error("MariaDB configuration missing"); if (!backup.mariadb) throw new Error("MariaDB configuration missing");
restoreCommand = `mysql -h ${backup.mariadb.host} -P ${backup.mariadb.port} -u ${backup.mariadb.user} -p${backup.mariadb.password} ${backup.database} < ${tempDir}/${backupPath.split("/").pop()}`; emit("Restoring MariaDB database...");
restoreCommand = `gunzip -c ${tempDir}/${cleanFileName} | mysql -h ${backup.mariadb.host} -P ${backup.mariadb.port} -u ${backup.mariadb.user} -p${backup.mariadb.password} ${backup.database}`;
break; break;
case "mongo": case "mongo":
if (!backup.mongo) throw new Error("MongoDB configuration missing"); if (!backup.mongo) throw new Error("MongoDB configuration missing");
restoreCommand = `mongorestore --host ${backup.mongo.host} --port ${backup.mongo.port} --username ${backup.mongo.user} --password ${backup.mongo.password} --db ${backup.database} ${tempDir}/${backup.database}`; emit("Restoring MongoDB database...");
restoreCommand = `mongorestore --host ${backup.mongo.host} --port ${backup.mongo.port} --username ${backup.mongo.user} --password ${backup.mongo.password} --db ${backup.database} --archive=${tempDir}/${cleanFileName} --gzip`;
break; break;
case "web-server": { case "web-server": {
if (!backup.webServer) if (!backup.webServer)
@@ -316,7 +317,7 @@ export const executeCloudStorageRestore = async (
emit("Extracting backup..."); emit("Extracting backup...");
await execAsync( await execAsync(
`cd ${tempDir} && unzip ${backupPath.split("/").pop()} > /dev/null 2>&1`, `cd ${tempDir} && unzip ${cleanFileName} > /dev/null 2>&1`,
); );
emit("Restoring filesystem..."); emit("Restoring filesystem...");
@@ -385,14 +386,19 @@ export const executeCloudStorageRestore = async (
throw new Error(`Unsupported database type: ${backup.databaseType}`); throw new Error(`Unsupported database type: ${backup.databaseType}`);
} }
if (restoreCommand) {
emit(`Executing restore command...`);
await execAsync(restoreCommand); await execAsync(restoreCommand);
await execAsync(`rm -rf ${tempDir}`); }
emit(`Cloud storage restore completed: ${backupPath}`);
emit("Restore completed successfully!");
} finally { } finally {
emit("Cleaning up temporary files...");
await execAsync(`rm -rf ${tempDir}`); await execAsync(`rm -rf ${tempDir}`);
} }
} catch (error) { } catch (error) {
console.error(`Error executing cloud storage restore: ${error}`); console.error("Error during cloud storage restore:", error);
emit(`Error: ${error instanceof Error ? error.message : "Unknown error"}`);
throw error; throw error;
} }
}; };
@@ -470,7 +476,6 @@ export function getRcloneConfigPath(
} }
return join(configDir, `${destinationId}.conf`); return join(configDir, `${destinationId}.conf`);
} }
export const keepLatestNCloudStorageBackups = async ( export const keepLatestNCloudStorageBackups = async (
backup: CloudStorageBackup, backup: CloudStorageBackup,
destination: CloudStorageDestination, destination: CloudStorageDestination,
@@ -494,3 +499,4 @@ export const keepLatestNCloudStorageBackups = async (
console.error("Error keeping latest backups:", error); console.error("Error keeping latest backups:", error);
} }
}; };