mirror of
https://github.com/Dokploy/dokploy
synced 2025-06-26 18:27:59 +00:00
feat(backups): implement normalizeS3Path utility and integrate into backup processes
- Added normalizeS3Path function to standardize S3 path formatting by trimming whitespace and removing leading/trailing slashes. - Updated backup-related modules (MySQL, MongoDB, PostgreSQL, MariaDB, and web server) to utilize normalizeS3Path for consistent S3 path handling. - Introduced unit tests for normalizeS3Path to ensure correct functionality across various input scenarios.
This commit is contained in:
parent
14bc26e065
commit
42fa4008ab
61
apps/dokploy/__test__/utils/backups.test.ts
Normal file
61
apps/dokploy/__test__/utils/backups.test.ts
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
import { describe, expect, test } from "vitest";
|
||||||
|
import { normalizeS3Path } from "@dokploy/server/utils/backups/utils";
|
||||||
|
|
||||||
|
describe("normalizeS3Path", () => {
|
||||||
|
test("should handle empty and whitespace-only prefix", () => {
|
||||||
|
expect(normalizeS3Path("")).toBe("");
|
||||||
|
expect(normalizeS3Path("/")).toBe("");
|
||||||
|
expect(normalizeS3Path(" ")).toBe("");
|
||||||
|
expect(normalizeS3Path("\t")).toBe("");
|
||||||
|
expect(normalizeS3Path("\n")).toBe("");
|
||||||
|
expect(normalizeS3Path(" \n \t ")).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should trim whitespace from prefix", () => {
|
||||||
|
expect(normalizeS3Path(" prefix")).toBe("prefix/");
|
||||||
|
expect(normalizeS3Path("prefix ")).toBe("prefix/");
|
||||||
|
expect(normalizeS3Path(" prefix ")).toBe("prefix/");
|
||||||
|
expect(normalizeS3Path("\tprefix\t")).toBe("prefix/");
|
||||||
|
expect(normalizeS3Path(" prefix/nested ")).toBe("prefix/nested/");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should remove leading slashes", () => {
|
||||||
|
expect(normalizeS3Path("/prefix")).toBe("prefix/");
|
||||||
|
expect(normalizeS3Path("///prefix")).toBe("prefix/");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should remove trailing slashes", () => {
|
||||||
|
expect(normalizeS3Path("prefix/")).toBe("prefix/");
|
||||||
|
expect(normalizeS3Path("prefix///")).toBe("prefix/");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should remove both leading and trailing slashes", () => {
|
||||||
|
expect(normalizeS3Path("/prefix/")).toBe("prefix/");
|
||||||
|
expect(normalizeS3Path("///prefix///")).toBe("prefix/");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle nested paths", () => {
|
||||||
|
expect(normalizeS3Path("prefix/nested")).toBe("prefix/nested/");
|
||||||
|
expect(normalizeS3Path("/prefix/nested/")).toBe("prefix/nested/");
|
||||||
|
expect(normalizeS3Path("///prefix/nested///")).toBe("prefix/nested/");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should preserve middle slashes", () => {
|
||||||
|
expect(normalizeS3Path("prefix/nested/deep")).toBe("prefix/nested/deep/");
|
||||||
|
expect(normalizeS3Path("/prefix/nested/deep/")).toBe("prefix/nested/deep/");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle special characters", () => {
|
||||||
|
expect(normalizeS3Path("prefix-with-dashes")).toBe("prefix-with-dashes/");
|
||||||
|
expect(normalizeS3Path("prefix_with_underscores")).toBe(
|
||||||
|
"prefix_with_underscores/",
|
||||||
|
);
|
||||||
|
expect(normalizeS3Path("prefix.with.dots")).toBe("prefix.with.dots/");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle the cases from the bug report", () => {
|
||||||
|
expect(normalizeS3Path("instance-backups/")).toBe("instance-backups/");
|
||||||
|
expect(normalizeS3Path("/instance-backups/")).toBe("instance-backups/");
|
||||||
|
expect(normalizeS3Path("instance-backups")).toBe("instance-backups/");
|
||||||
|
});
|
||||||
|
});
|
@ -31,7 +31,10 @@ import {
|
|||||||
} from "@dokploy/server";
|
} from "@dokploy/server";
|
||||||
|
|
||||||
import { findDestinationById } from "@dokploy/server/services/destination";
|
import { findDestinationById } from "@dokploy/server/services/destination";
|
||||||
import { getS3Credentials } from "@dokploy/server/utils/backups/utils";
|
import {
|
||||||
|
getS3Credentials,
|
||||||
|
normalizeS3Path,
|
||||||
|
} from "@dokploy/server/utils/backups/utils";
|
||||||
import {
|
import {
|
||||||
execAsync,
|
execAsync,
|
||||||
execAsyncRemote,
|
execAsyncRemote,
|
||||||
@ -257,7 +260,7 @@ export const backupRouter = createTRPCRouter({
|
|||||||
const lastSlashIndex = input.search.lastIndexOf("/");
|
const lastSlashIndex = input.search.lastIndexOf("/");
|
||||||
const baseDir =
|
const baseDir =
|
||||||
lastSlashIndex !== -1
|
lastSlashIndex !== -1
|
||||||
? input.search.slice(0, lastSlashIndex + 1)
|
? normalizeS3Path(input.search.slice(0, lastSlashIndex + 1))
|
||||||
: "";
|
: "";
|
||||||
const searchTerm =
|
const searchTerm =
|
||||||
lastSlashIndex !== -1
|
lastSlashIndex !== -1
|
||||||
@ -270,7 +273,7 @@ export const backupRouter = createTRPCRouter({
|
|||||||
let stdout = "";
|
let stdout = "";
|
||||||
|
|
||||||
if (input.serverId) {
|
if (input.serverId) {
|
||||||
const result = await execAsyncRemote(listCommand, input.serverId);
|
const result = await execAsyncRemote(input.serverId, listCommand);
|
||||||
stdout = result.stdout;
|
stdout = result.stdout;
|
||||||
} else {
|
} else {
|
||||||
const result = await execAsync(listCommand);
|
const result = await execAsync(listCommand);
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
import path from "node:path";
|
|
||||||
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
||||||
import type { Mariadb } from "@dokploy/server/services/mariadb";
|
import type { Mariadb } from "@dokploy/server/services/mariadb";
|
||||||
import { findProjectById } from "@dokploy/server/services/project";
|
import { findProjectById } from "@dokploy/server/services/project";
|
||||||
@ -8,7 +7,7 @@ import {
|
|||||||
} from "../docker/utils";
|
} from "../docker/utils";
|
||||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||||
import { getS3Credentials } from "./utils";
|
import { getS3Credentials, normalizeS3Path } from "./utils";
|
||||||
|
|
||||||
export const runMariadbBackup = async (
|
export const runMariadbBackup = async (
|
||||||
mariadb: Mariadb,
|
mariadb: Mariadb,
|
||||||
@ -19,7 +18,7 @@ export const runMariadbBackup = async (
|
|||||||
const { prefix, database } = backup;
|
const { prefix, database } = backup;
|
||||||
const destination = backup.destination;
|
const destination = backup.destination;
|
||||||
const backupFileName = `${new Date().toISOString()}.sql.gz`;
|
const backupFileName = `${new Date().toISOString()}.sql.gz`;
|
||||||
const bucketDestination = path.join(prefix, backupFileName);
|
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const rcloneFlags = getS3Credentials(destination);
|
const rcloneFlags = getS3Credentials(destination);
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
import path from "node:path";
|
|
||||||
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
||||||
import type { Mongo } from "@dokploy/server/services/mongo";
|
import type { Mongo } from "@dokploy/server/services/mongo";
|
||||||
import { findProjectById } from "@dokploy/server/services/project";
|
import { findProjectById } from "@dokploy/server/services/project";
|
||||||
@ -8,7 +7,7 @@ import {
|
|||||||
} from "../docker/utils";
|
} from "../docker/utils";
|
||||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||||
import { getS3Credentials } from "./utils";
|
import { getS3Credentials, normalizeS3Path } from "./utils";
|
||||||
|
|
||||||
// mongodb://mongo:Bqh7AQl-PRbnBu@localhost:27017/?tls=false&directConnection=true
|
// mongodb://mongo:Bqh7AQl-PRbnBu@localhost:27017/?tls=false&directConnection=true
|
||||||
export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
|
export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
|
||||||
@ -17,7 +16,7 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
|
|||||||
const { prefix, database } = backup;
|
const { prefix, database } = backup;
|
||||||
const destination = backup.destination;
|
const destination = backup.destination;
|
||||||
const backupFileName = `${new Date().toISOString()}.dump.gz`;
|
const backupFileName = `${new Date().toISOString()}.dump.gz`;
|
||||||
const bucketDestination = path.join(prefix, backupFileName);
|
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const rcloneFlags = getS3Credentials(destination);
|
const rcloneFlags = getS3Credentials(destination);
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
import path from "node:path";
|
|
||||||
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
||||||
import type { MySql } from "@dokploy/server/services/mysql";
|
import type { MySql } from "@dokploy/server/services/mysql";
|
||||||
import { findProjectById } from "@dokploy/server/services/project";
|
import { findProjectById } from "@dokploy/server/services/project";
|
||||||
@ -8,7 +7,7 @@ import {
|
|||||||
} from "../docker/utils";
|
} from "../docker/utils";
|
||||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||||
import { getS3Credentials } from "./utils";
|
import { getS3Credentials, normalizeS3Path } from "./utils";
|
||||||
|
|
||||||
export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
|
export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
|
||||||
const { appName, databaseRootPassword, projectId, name } = mysql;
|
const { appName, databaseRootPassword, projectId, name } = mysql;
|
||||||
@ -16,7 +15,7 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
|
|||||||
const { prefix, database } = backup;
|
const { prefix, database } = backup;
|
||||||
const destination = backup.destination;
|
const destination = backup.destination;
|
||||||
const backupFileName = `${new Date().toISOString()}.sql.gz`;
|
const backupFileName = `${new Date().toISOString()}.sql.gz`;
|
||||||
const bucketDestination = path.join(prefix, backupFileName);
|
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const rcloneFlags = getS3Credentials(destination);
|
const rcloneFlags = getS3Credentials(destination);
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
import path from "node:path";
|
|
||||||
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
||||||
import type { Postgres } from "@dokploy/server/services/postgres";
|
import type { Postgres } from "@dokploy/server/services/postgres";
|
||||||
import { findProjectById } from "@dokploy/server/services/project";
|
import { findProjectById } from "@dokploy/server/services/project";
|
||||||
@ -8,7 +7,7 @@ import {
|
|||||||
} from "../docker/utils";
|
} from "../docker/utils";
|
||||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||||
import { getS3Credentials } from "./utils";
|
import { getS3Credentials, normalizeS3Path } from "./utils";
|
||||||
|
|
||||||
export const runPostgresBackup = async (
|
export const runPostgresBackup = async (
|
||||||
postgres: Postgres,
|
postgres: Postgres,
|
||||||
@ -20,7 +19,7 @@ export const runPostgresBackup = async (
|
|||||||
const { prefix, database } = backup;
|
const { prefix, database } = backup;
|
||||||
const destination = backup.destination;
|
const destination = backup.destination;
|
||||||
const backupFileName = `${new Date().toISOString()}.sql.gz`;
|
const backupFileName = `${new Date().toISOString()}.sql.gz`;
|
||||||
const bucketDestination = path.join(prefix, backupFileName);
|
const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
|
||||||
try {
|
try {
|
||||||
const rcloneFlags = getS3Credentials(destination);
|
const rcloneFlags = getS3Credentials(destination);
|
||||||
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
||||||
|
@ -36,6 +36,13 @@ export const removeScheduleBackup = (backupId: string) => {
|
|||||||
currentJob?.cancel();
|
currentJob?.cancel();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const normalizeS3Path = (prefix: string) => {
|
||||||
|
// Trim whitespace and remove leading/trailing slashes
|
||||||
|
const normalizedPrefix = prefix.trim().replace(/^\/+|\/+$/g, "");
|
||||||
|
// Return empty string if prefix is empty, otherwise append trailing slash
|
||||||
|
return normalizedPrefix ? `${normalizedPrefix}/` : "";
|
||||||
|
};
|
||||||
|
|
||||||
export const getS3Credentials = (destination: Destination) => {
|
export const getS3Credentials = (destination: Destination) => {
|
||||||
const { accessKey, secretAccessKey, region, endpoint, provider } =
|
const { accessKey, secretAccessKey, region, endpoint, provider } =
|
||||||
destination;
|
destination;
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
||||||
import { execAsync } from "../process/execAsync";
|
import { execAsync } from "../process/execAsync";
|
||||||
import { getS3Credentials } from "./utils";
|
import { getS3Credentials, normalizeS3Path } from "./utils";
|
||||||
import { findDestinationById } from "@dokploy/server/services/destination";
|
import { findDestinationById } from "@dokploy/server/services/destination";
|
||||||
import { IS_CLOUD, paths } from "@dokploy/server/constants";
|
import { IS_CLOUD, paths } from "@dokploy/server/constants";
|
||||||
import { mkdtemp } from "node:fs/promises";
|
import { mkdtemp } from "node:fs/promises";
|
||||||
@ -18,7 +18,7 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
|
|||||||
const { BASE_PATH } = paths();
|
const { BASE_PATH } = paths();
|
||||||
const tempDir = await mkdtemp(join(tmpdir(), "dokploy-backup-"));
|
const tempDir = await mkdtemp(join(tmpdir(), "dokploy-backup-"));
|
||||||
const backupFileName = `webserver-backup-${timestamp}.zip`;
|
const backupFileName = `webserver-backup-${timestamp}.zip`;
|
||||||
const s3Path = `:s3:${destination.bucket}/${backup.prefix}${backupFileName}`;
|
const s3Path = `:s3:${destination.bucket}/${normalizeS3Path(backup.prefix)}${backupFileName}`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await execAsync(`mkdir -p ${tempDir}/filesystem`);
|
await execAsync(`mkdir -p ${tempDir}/filesystem`);
|
||||||
|
Loading…
Reference in New Issue
Block a user