Dokploy/dokploy
1
0
Fork 0
mirror of https://github.com/Dokploy/dokploy synced 2025-06-26 18:27:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1638 from Dokploy/1588-invalid-behavior-when-using-s3-prefix-destination-for-backups

Browse Source 
feat(backups): implement normalizeS3Path utility and integrate into b…
This commit is contained in:
Mauricio Siu 2025-04-06 02:15:37 -06:00 committed by GitHub
commit 4b51744d0d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 84 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
apps/dokploy/__test__/utils/backups.test.ts Normal file
View File

@ -0,0 +1,61 @@
import { describe, expect, test } from "vitest";
import { normalizeS3Path } from "@dokploy/server/utils/backups/utils";
describe("normalizeS3Path", () => {
test("should handle empty and whitespace-only prefix", () => {
expect(normalizeS3Path("")).toBe("");
expect(normalizeS3Path("/")).toBe("");
expect(normalizeS3Path(" ")).toBe("");
expect(normalizeS3Path("\t")).toBe("");
expect(normalizeS3Path("\n")).toBe("");
expect(normalizeS3Path(" \n \t ")).toBe("");
});
test("should trim whitespace from prefix", () => {
expect(normalizeS3Path(" prefix")).toBe("prefix/");
expect(normalizeS3Path("prefix ")).toBe("prefix/");
expect(normalizeS3Path(" prefix ")).toBe("prefix/");
expect(normalizeS3Path("\tprefix\t")).toBe("prefix/");
expect(normalizeS3Path(" prefix/nested ")).toBe("prefix/nested/");
});
test("should remove leading slashes", () => {
expect(normalizeS3Path("/prefix")).toBe("prefix/");
expect(normalizeS3Path("///prefix")).toBe("prefix/");
});
test("should remove trailing slashes", () => {
expect(normalizeS3Path("prefix/")).toBe("prefix/");
expect(normalizeS3Path("prefix///")).toBe("prefix/");
});
test("should remove both leading and trailing slashes", () => {
expect(normalizeS3Path("/prefix/")).toBe("prefix/");
expect(normalizeS3Path("///prefix///")).toBe("prefix/");
});
test("should handle nested paths", () => {
expect(normalizeS3Path("prefix/nested")).toBe("prefix/nested/");
expect(normalizeS3Path("/prefix/nested/")).toBe("prefix/nested/");
expect(normalizeS3Path("///prefix/nested///")).toBe("prefix/nested/");
});
test("should preserve middle slashes", () => {
expect(normalizeS3Path("prefix/nested/deep")).toBe("prefix/nested/deep/");
expect(normalizeS3Path("/prefix/nested/deep/")).toBe("prefix/nested/deep/");
});
test("should handle special characters", () => {
expect(normalizeS3Path("prefix-with-dashes")).toBe("prefix-with-dashes/");
expect(normalizeS3Path("prefix_with_underscores")).toBe(
"prefix_with_underscores/",
);
expect(normalizeS3Path("prefix.with.dots")).toBe("prefix.with.dots/");
});
test("should handle the cases from the bug report", () => {
expect(normalizeS3Path("instance-backups/")).toBe("instance-backups/");
expect(normalizeS3Path("/instance-backups/")).toBe("instance-backups/");
expect(normalizeS3Path("instance-backups")).toBe("instance-backups/");
});
});

9
apps/dokploy/server/api/routers/backup.ts
View File

@ -31,7 +31,10 @@ import {
} from "@dokploy/server";
import { findDestinationById } from "@dokploy/server/services/destination";
import { getS3Credentials } from "@dokploy/server/utils/backups/utils";
import {
getS3Credentials,
normalizeS3Path,
} from "@dokploy/server/utils/backups/utils";
import {
execAsync,
execAsyncRemote,
@ -257,7 +260,7 @@ export const backupRouter = createTRPCRouter({
const lastSlashIndex = input.search.lastIndexOf("/");
const baseDir =
lastSlashIndex !== -1
? input.search.slice(0, lastSlashIndex + 1)
? normalizeS3Path(input.search.slice(0, lastSlashIndex + 1))
: "";
const searchTerm =
lastSlashIndex !== -1
@ -270,7 +273,7 @@ export const backupRouter = createTRPCRouter({
let stdout = "";
if (input.serverId) {
const result = await execAsyncRemote(listCommand, input.serverId);
const result = await execAsyncRemote(input.serverId, listCommand);
stdout = result.stdout;
} else {
const result = await execAsync(listCommand);

5
packages/server/src/utils/backups/mariadb.ts
View File

@ -1,4 +1,3 @@
import path from "node:path";
import type { BackupSchedule } from "@dokploy/server/services/backup";
import type { Mariadb } from "@dokploy/server/services/mariadb";
import { findProjectById } from "@dokploy/server/services/project";
@ -8,7 +7,7 @@ import {
} from "../docker/utils";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getS3Credentials } from "./utils";
import { getS3Credentials, normalizeS3Path } from "./utils";
export const runMariadbBackup = async (
mariadb: Mariadb,
@ -19,7 +18,7 @@ export const runMariadbBackup = async (
const { prefix, database } = backup;
const destination = backup.destination;
const backupFileName = `${new Date().toISOString()}.sql.gz`;
const bucketDestination = path.join(prefix, backupFileName);
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
try {
const rcloneFlags = getS3Credentials(destination);

5
packages/server/src/utils/backups/mongo.ts
View File

@ -1,4 +1,3 @@
import path from "node:path";
import type { BackupSchedule } from "@dokploy/server/services/backup";
import type { Mongo } from "@dokploy/server/services/mongo";
import { findProjectById } from "@dokploy/server/services/project";
@ -8,7 +7,7 @@ import {
} from "../docker/utils";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getS3Credentials } from "./utils";
import { getS3Credentials, normalizeS3Path } from "./utils";
// mongodb://mongo:Bqh7AQl-PRbnBu@localhost:27017/?tls=false&directConnection=true
export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
@ -17,7 +16,7 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
const { prefix, database } = backup;
const destination = backup.destination;
const backupFileName = `${new Date().toISOString()}.dump.gz`;
const bucketDestination = path.join(prefix, backupFileName);
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
try {
const rcloneFlags = getS3Credentials(destination);

5
packages/server/src/utils/backups/mysql.ts
View File

@ -1,4 +1,3 @@
import path from "node:path";
import type { BackupSchedule } from "@dokploy/server/services/backup";
import type { MySql } from "@dokploy/server/services/mysql";
import { findProjectById } from "@dokploy/server/services/project";
@ -8,7 +7,7 @@ import {
} from "../docker/utils";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getS3Credentials } from "./utils";
import { getS3Credentials, normalizeS3Path } from "./utils";
export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
const { appName, databaseRootPassword, projectId, name } = mysql;
@ -16,7 +15,7 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
const { prefix, database } = backup;
const destination = backup.destination;
const backupFileName = `${new Date().toISOString()}.sql.gz`;
const bucketDestination = path.join(prefix, backupFileName);
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
try {
const rcloneFlags = getS3Credentials(destination);

5
packages/server/src/utils/backups/postgres.ts
View File

@ -1,4 +1,3 @@
import path from "node:path";
import type { BackupSchedule } from "@dokploy/server/services/backup";
import type { Postgres } from "@dokploy/server/services/postgres";
import { findProjectById } from "@dokploy/server/services/project";
@ -8,7 +7,7 @@ import {
} from "../docker/utils";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getS3Credentials } from "./utils";
import { getS3Credentials, normalizeS3Path } from "./utils";
export const runPostgresBackup = async (
postgres: Postgres,
@ -20,7 +19,7 @@ export const runPostgresBackup = async (
const { prefix, database } = backup;
const destination = backup.destination;
const backupFileName = `${new Date().toISOString()}.sql.gz`;
const bucketDestination = path.join(prefix, backupFileName);
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;

7
packages/server/src/utils/backups/utils.ts
View File

@ -36,6 +36,13 @@ export const removeScheduleBackup = (backupId: string) => {
currentJob?.cancel();
};
export const normalizeS3Path = (prefix: string) => {
// Trim whitespace and remove leading/trailing slashes
const normalizedPrefix = prefix.trim().replace(/^\/+|\/+$/g, "");
// Return empty string if prefix is empty, otherwise append trailing slash
return normalizedPrefix ? `${normalizedPrefix}/` : "";
};
export const getS3Credentials = (destination: Destination) => {
const { accessKey, secretAccessKey, region, endpoint, provider } =
destination;

4
packages/server/src/utils/backups/web-server.ts
View File

@ -1,6 +1,6 @@
import type { BackupSchedule } from "@dokploy/server/services/backup";
import { execAsync } from "../process/execAsync";
import { getS3Credentials } from "./utils";
import { getS3Credentials, normalizeS3Path } from "./utils";
import { findDestinationById } from "@dokploy/server/services/destination";
import { IS_CLOUD, paths } from "@dokploy/server/constants";
import { mkdtemp } from "node:fs/promises";
@ -18,7 +18,7 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
const { BASE_PATH } = paths();
const tempDir = await mkdtemp(join(tmpdir(), "dokploy-backup-"));
const backupFileName = `webserver-backup-${timestamp}.zip`;
const s3Path = `:s3:${destination.bucket}/${backup.prefix}${backupFileName}`;
const s3Path = `:s3:${destination.bucket}/${normalizeS3Path(backup.prefix)}${backupFileName}`;
try {
await execAsync(`mkdir -p ${tempDir}/filesystem`);