feat: Implement cloud storage provider backups system with OAuth support

This commit is contained in:
vishalkadam47
2025-05-31 17:01:47 +05:30
parent 2619cb49d1
commit 2ee40f28ed
22 changed files with 10598 additions and 278 deletions

View File

@@ -0,0 +1,222 @@
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { api } from "@/utils/api";
import { zodResolver } from "@hookform/resolvers/zod";
import { PlusIcon } from "lucide-react";
import { useEffect } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import { z } from "zod";
const AddCloudBackupSchema = z.object({
destinationId: z.string().min(1, "Destination required"),
schedule: z.string().min(1, "Schedule (Cron) required"),
prefix: z.string().min(1, "Prefix required"),
enabled: z.boolean(),
database: z.string().min(1, "Database required"),
keepLatestCount: z.coerce.number().optional(),
});
type AddCloudBackup = z.infer<typeof AddCloudBackupSchema>;
interface Props {
databaseId: string;
databaseType: "postgres" | "mariadb" | "mysql" | "mongo" | "web-server";
refetch: () => void;
}
export const AddCloudBackup = ({
databaseId,
databaseType,
refetch,
}: Props) => {
const { data: cloudDestinations, isLoading: isLoadingDestinations } =
api.cloudStorageDestination.all.useQuery();
const { mutateAsync: createBackup, isLoading: isCreating } =
api.cloudStorageBackup.create.useMutation();
const form = useForm<AddCloudBackup>({
defaultValues: {
database: "",
destinationId: "",
enabled: true,
prefix: "/",
schedule: "",
keepLatestCount: undefined,
},
resolver: zodResolver(AddCloudBackupSchema),
});
useEffect(() => {
form.reset({
database: databaseType === "web-server" ? "dokploy" : "",
destinationId: "",
enabled: true,
prefix: "/",
schedule: "",
keepLatestCount: undefined,
});
}, [form, form.reset, form.formState.isSubmitSuccessful]);
const onSubmit = async (data: AddCloudBackup) => {
try {
await createBackup({
...data,
databaseType,
cloudStorageDestinationId: data.destinationId,
postgresId: databaseType === "postgres" ? databaseId : undefined,
mysqlId: databaseType === "mysql" ? databaseId : undefined,
mariadbId: databaseType === "mariadb" ? databaseId : undefined,
mongoId: databaseType === "mongo" ? databaseId : undefined,
});
toast.success("Cloud backup created successfully");
refetch();
} catch (error) {
toast.error("Failed to create cloud backup");
console.error(error);
}
};
return (
<Dialog>
<DialogTrigger asChild>
<Button>
<PlusIcon className="mr-2 h-4 w-4" />
Add Cloud Backup
</Button>
</DialogTrigger>
<DialogContent className="sm:max-w-[425px]">
<DialogHeader>
<DialogTitle>Add Cloud Backup</DialogTitle>
<DialogDescription>
Configure a new backup to your cloud storage destination.
</DialogDescription>
</DialogHeader>
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
<FormField
control={form.control}
name="destinationId"
render={({ field }) => (
<FormItem>
<FormLabel>Destination</FormLabel>
<Select
onValueChange={field.onChange}
defaultValue={field.value}
disabled={isLoadingDestinations}
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select a destination" />
</SelectTrigger>
</FormControl>
<SelectContent>
{cloudDestinations?.map((destination) => (
<SelectItem key={destination.id} value={destination.id}>
{destination.name} ({destination.provider})
</SelectItem>
))}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="schedule"
render={({ field }) => (
<FormItem>
<FormLabel>Schedule (Cron)</FormLabel>
<FormControl>
<Input placeholder="0 0 * * *" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="prefix"
render={({ field }) => (
<FormItem>
<FormLabel>Prefix</FormLabel>
<FormControl>
<Input placeholder="/backups/" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="database"
render={({ field }) => (
<FormItem>
<FormLabel>Database</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="keepLatestCount"
render={({ field }) => (
<FormItem>
<FormLabel>Keep Latest Count</FormLabel>
<FormControl>
<Input
type="number"
placeholder="Number of backups to keep"
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<DialogFooter>
<Button type="submit" isLoading={isCreating}>
Create Backup
</Button>
</DialogFooter>
</form>
</Form>
</DialogContent>
</Dialog>
);
};

View File

@@ -0,0 +1,738 @@
import {
type LogLine,
parseLogs,
} from "@/components/dashboard/docker/logs/utils";
import { DrawerLogs } from "@/components/shared/drawer-logs";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
Command,
CommandEmpty,
CommandGroup,
CommandInput,
CommandItem,
} from "@/components/ui/command";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import {
Popover,
PopoverContent,
PopoverTrigger,
} from "@/components/ui/popover";
import { ScrollArea } from "@/components/ui/scroll-area";
import { cn } from "@/lib/utils";
import { api } from "@/utils/api";
import { zodResolver } from "@hookform/resolvers/zod";
import { debounce } from "lodash";
import { CheckIcon, ChevronsUpDown, RotateCcw } from "lucide-react";
import { useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import { z } from "zod";
interface BackupFile {
path: string;
name: string;
size: number;
isDir: boolean;
hashes?: {
MD5: string;
};
lastModified: string;
}
type DatabaseType = "postgres" | "mariadb" | "mysql" | "mongo" | "web-server";
interface Props {
databaseId: string;
databaseType: DatabaseType;
}
const restoreBackupSchema = z
.object({
backupId: z.string().min(1, "Backup is required"),
backupFile: z.string().min(1, "Backup file is required"),
databaseName: z.string().min(1, "Database name is required"),
metadata: z
.object({
postgres: z
.object({
databaseUser: z.string(),
})
.optional(),
mariadb: z
.object({
databaseUser: z.string(),
databasePassword: z.string(),
})
.optional(),
mongo: z
.object({
databaseUser: z.string(),
databasePassword: z.string(),
})
.optional(),
mysql: z
.object({
databaseRootPassword: z.string(),
})
.optional(),
})
.optional(),
})
.superRefine((data, ctx) => {
if (data.metadata?.postgres && !data.metadata.postgres.databaseUser) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Database user is required for PostgreSQL",
path: ["metadata", "postgres", "databaseUser"],
});
}
if (data.metadata?.mariadb) {
if (!data.metadata.mariadb.databaseUser) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Database user is required for MariaDB",
path: ["metadata", "mariadb", "databaseUser"],
});
}
if (!data.metadata.mariadb.databasePassword) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Database password is required for MariaDB",
path: ["metadata", "mariadb", "databasePassword"],
});
}
}
if (data.metadata?.mongo) {
if (!data.metadata.mongo.databaseUser) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Database user is required for MongoDB",
path: ["metadata", "mongo", "databaseUser"],
});
}
if (!data.metadata.mongo.databasePassword) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Database password is required for MongoDB",
path: ["metadata", "mongo", "databasePassword"],
});
}
}
if (data.metadata?.mysql && !data.metadata.mysql.databaseRootPassword) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Root password is required for MySQL",
path: ["metadata", "mysql", "databaseRootPassword"],
});
}
});
const formatBytes = (bytes: number): string => {
if (bytes === 0) return "0 Bytes";
const k = 1024;
const sizes = ["Bytes", "KB", "MB", "GB", "TB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${Number.parseFloat((bytes / k ** i).toFixed(2))} ${sizes[i]}`;
};
export const RestoreCloudBackup = ({ databaseId, databaseType }: Props) => {
const [open, setOpen] = useState(false);
const [search, setSearch] = useState("");
const [debouncedSearchTerm, setDebouncedSearchTerm] = useState("");
const [selectedBackup, setSelectedBackup] = useState<any>(null);
const [isDrawerOpen, setIsDrawerOpen] = useState(false);
const [filteredLogs, setFilteredLogs] = useState<LogLine[]>([]);
const [isDeploying, setIsDeploying] = useState(false);
const [isFilePopoverOpen, setIsFilePopoverOpen] = useState(false);
const { data: cloudBackups } = api.cloudStorageBackup.list.useQuery();
const { data: backupFiles, isLoading: isLoadingFiles } =
api.cloudStorageBackup.listBackupFiles.useQuery(
{
destinationId: selectedBackup?.cloudStorageDestinationId || "",
prefix: selectedBackup?.prefix || "",
searchTerm: debouncedSearchTerm,
},
{
enabled: !!selectedBackup?.cloudStorageDestinationId,
onSuccess: (data) => {
console.log("Backup files loaded:", data);
},
onError: (error) => {
console.error("Error loading backup files:", error);
},
},
);
const { mutateAsync: restoreBackup } =
api.cloudStorageBackup.restore.useMutation();
const form = useForm<z.infer<typeof restoreBackupSchema>>({
resolver: zodResolver(restoreBackupSchema),
defaultValues: {
backupId: "",
backupFile: "",
databaseName: databaseType === "web-server" ? "dokploy" : "",
metadata: {},
},
});
const debouncedSetSearch = debounce((value: string) => {
setDebouncedSearchTerm(value);
}, 350);
const handleSearchChange = (value: string) => {
setSearch(value);
debouncedSetSearch(value);
};
const handleBackupSelect = (backup: any) => {
console.log("Selected backup:", backup);
form.setValue("backupId", backup.id);
setSelectedBackup(backup);
setSearch("");
setDebouncedSearchTerm("");
};
api.cloudStorageBackup.restoreBackupWithLogs.useSubscription(
{
databaseId,
databaseType,
databaseName: form.watch("databaseName"),
backupFile: form.watch("backupFile"),
destinationId: selectedBackup?.cloudStorageDestinationId || "",
metadata: form.watch("metadata"),
},
{
enabled: isDeploying,
onData(log) {
if (!isDrawerOpen) {
setIsDrawerOpen(true);
}
if (log === "Restore completed successfully!") {
setIsDeploying(false);
}
const parsedLogs = parseLogs(log);
setFilteredLogs((prev) => [...prev, ...parsedLogs]);
},
onError(error) {
console.error("Restore logs error:", error);
setIsDeploying(false);
},
},
);
const filteredBackups = cloudBackups?.filter((backup) => {
const matchesType = (() => {
switch (databaseType) {
case "postgres":
return backup.postgresId === databaseId;
case "mysql":
return backup.mysqlId === databaseId;
case "mariadb":
return backup.mariadbId === databaseId;
case "mongo":
return backup.mongoId === databaseId;
case "web-server":
return backup.databaseType === "web-server";
default:
return false;
}
})();
const matchesSearch = search
? backup.database?.toLowerCase().includes(search.toLowerCase()) ||
backup.cloudStorageDestination?.name
?.toLowerCase()
.includes(search.toLowerCase())
: true;
return matchesType && matchesSearch;
});
const onSubmit = async (values: z.infer<typeof restoreBackupSchema>) => {
if (!selectedBackup) {
toast.error("Please select a valid backup");
return;
}
setIsDeploying(true);
try {
const remoteName = (() => {
switch (selectedBackup.cloudStorageDestination?.provider) {
case "drive":
return "dokploy-drive";
case "dropbox":
return "dokploy-dropbox";
case "box":
return "dokploy-box";
default:
return selectedBackup.cloudStorageDestination?.provider;
}
})();
const prefix = selectedBackup.prefix.startsWith("/")
? selectedBackup.prefix.slice(1)
: selectedBackup.prefix;
await restoreBackup({
destinationId: selectedBackup.cloudStorageDestinationId,
backupFile: `${remoteName}:${prefix}/${values.backupFile}`,
databaseType,
databaseName: values.databaseName,
metadata: values.metadata,
});
toast.success("Backup restore initiated successfully");
setOpen(false);
} catch (error) {
console.error("Restore error:", error);
toast.error(
error instanceof Error ? error.message : "Failed to restore backup",
);
} finally {
setIsDeploying(false);
}
};
return (
<Dialog
open={open}
onOpenChange={(newOpen) => {
setOpen(newOpen);
if (!newOpen) {
// Reset form and state when dialog is closed
form.reset();
setSelectedBackup(null);
setSearch("");
setDebouncedSearchTerm("");
setIsFilePopoverOpen(false);
setFilteredLogs([]);
setIsDeploying(false);
}
}}
>
<DialogTrigger asChild>
<Button variant="outline">
<RotateCcw className="mr-2 size-4" />
Restore Backup
</Button>
</DialogTrigger>
<DialogContent className="max-h-screen overflow-y-auto sm:max-w-lg">
<DialogHeader>
<DialogTitle className="flex items-center">
<Button
variant="secondary"
size="icon"
className="mr-2"
onClick={() => {
form.reset();
setSelectedBackup(null);
setSearch("");
setDebouncedSearchTerm("");
}}
>
<RotateCcw className="size-4" />
</Button>
Restore Cloud Backup
</DialogTitle>
<DialogDescription>
Select a backup to restore from your cloud storage destination.
</DialogDescription>
</DialogHeader>
<Form {...form}>
<form
id="hook-form-restore-cloud-backup"
onSubmit={form.handleSubmit(onSubmit)}
className="grid w-full gap-4"
>
<FormField
control={form.control}
name="backupId"
render={({ field }) => (
<FormItem className="">
<FormLabel>Backup</FormLabel>
<Popover>
<PopoverTrigger asChild>
<FormControl>
<Button
variant="outline"
className={cn(
"w-full justify-between !bg-input",
!field.value && "text-muted-foreground",
)}
>
{field.value
? `${
filteredBackups?.find(
(backup) => backup.id === field.value,
)?.cloudStorageDestination?.name
} - ${
filteredBackups?.find(
(backup) => backup.id === field.value,
)?.database
}`
: "Select backup"}
<ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
</Button>
</FormControl>
</PopoverTrigger>
<PopoverContent className="p-0" align="start">
<Command>
<CommandInput
placeholder="Search backups..."
value={search}
onValueChange={handleSearchChange}
className="h-9"
/>
<CommandEmpty>No backups found.</CommandEmpty>
<ScrollArea className="h-64">
<CommandGroup>
{filteredBackups?.map((backup) => (
<CommandItem
value={backup.id}
key={backup.id}
onSelect={() => handleBackupSelect(backup)}
>
<CheckIcon
className={cn(
"mr-2 h-4 w-4",
backup.id === field.value
? "opacity-100"
: "opacity-0",
)}
/>
{backup.cloudStorageDestination?.name} -{" "}
{backup.database}
</CommandItem>
))}
</CommandGroup>
</ScrollArea>
</Command>
</PopoverContent>
</Popover>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="backupFile"
render={({ field }) => (
<FormItem className="">
<FormLabel className="flex items-center justify-between">
<span>Search Backup Files</span>
</FormLabel>
<Popover
modal
open={isFilePopoverOpen}
onOpenChange={setIsFilePopoverOpen}
>
<PopoverTrigger asChild>
<FormControl>
<Button
variant="outline"
className={cn(
"w-full justify-between !bg-input",
!field.value && "text-muted-foreground",
)}
disabled={!selectedBackup}
onClick={() => {
setSearch("");
setDebouncedSearchTerm("");
}}
>
{field.value || "Select a backup destination first"}
<ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
</Button>
</FormControl>
</PopoverTrigger>
<PopoverContent className="p-0 w-[500px]" align="start">
<Command>
<CommandInput
placeholder="Search backup files..."
value={search}
onValueChange={handleSearchChange}
className="h-9"
/>
{isLoadingFiles ? (
<div className="flex items-center justify-center py-6">
<div className="flex flex-col items-center gap-2">
<div className="size-6 animate-spin rounded-full border-2 border-primary border-t-transparent" />
<span className="text-sm text-muted-foreground">
Loading backup files...
</span>
</div>
</div>
) : backupFiles?.length === 0 && search ? (
<div className="flex flex-col items-center justify-center py-6 text-center">
<span className="text-sm text-muted-foreground">
No backup files found for "{search}"
</span>
<Button
variant="ghost"
size="sm"
className="mt-2"
onClick={() => {
setSearch("");
setDebouncedSearchTerm("");
}}
>
Clear search
</Button>
</div>
) : backupFiles?.length === 0 ? (
<div className="flex flex-col items-center justify-center py-6 text-center">
<span className="text-sm text-muted-foreground">
No backup files available
</span>
<span className="mt-1 text-xs text-muted-foreground">
Make sure you have backup files in the selected
destination
</span>
</div>
) : (
<ScrollArea className="h-64">
<CommandGroup>
{backupFiles
?.sort(
(a: BackupFile, b: BackupFile) =>
new Date(b.lastModified).getTime() -
new Date(a.lastModified).getTime(),
)
.map((file: BackupFile) => (
<CommandItem
value={file.path}
key={file.path}
onSelect={() => {
form.setValue("backupFile", file.path);
setSearch("");
setDebouncedSearchTerm("");
setIsFilePopoverOpen(false);
}}
className="flex flex-col items-start gap-1 p-2"
>
<div className="flex w-full items-center justify-between">
<span className="font-medium truncate max-w-[250px]">
{file.name}
</span>
<div className="flex flex-col items-end">
<Badge variant="outline">
{new Date(
file.lastModified,
).toLocaleDateString()}
</Badge>
<span className="text-xs text-muted-foreground mt-1">
{new Date(
file.lastModified,
).toLocaleTimeString()}
</span>
</div>
</div>
<div className="flex items-center gap-4 text-xs text-muted-foreground">
<span className="flex items-center gap-1">
<span className="font-medium">
Size:
</span>
{formatBytes(file.size)}
</span>
{file.isDir && (
<span className="flex items-center gap-1 text-blue-500">
<span className="font-medium">
Type:
</span>
Directory
</span>
)}
{file.hashes?.MD5 && (
<span className="flex items-center gap-1">
<span className="font-medium">
MD5:
</span>
<span className="font-mono">
{file.hashes.MD5}
</span>
</span>
)}
</div>
</CommandItem>
))}
</CommandGroup>
</ScrollArea>
)}
</Command>
</PopoverContent>
</Popover>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="databaseName"
render={({ field }) => (
<FormItem>
<FormLabel>Database Name</FormLabel>
<FormControl>
<Input
placeholder="Enter database name"
{...field}
disabled={databaseType === "web-server"}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
{databaseType === "postgres" && (
<FormField
control={form.control}
name="metadata.postgres.databaseUser"
render={({ field }) => (
<FormItem>
<FormLabel>Database User</FormLabel>
<FormControl>
<Input placeholder="Enter database user" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}
{databaseType === "mariadb" && (
<>
<FormField
control={form.control}
name="metadata.mariadb.databaseUser"
render={({ field }) => (
<FormItem>
<FormLabel>Database User</FormLabel>
<FormControl>
<Input placeholder="Enter database user" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="metadata.mariadb.databasePassword"
render={({ field }) => (
<FormItem>
<FormLabel>Database Password</FormLabel>
<FormControl>
<Input
type="password"
placeholder="Enter database password"
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
</>
)}
{databaseType === "mongo" && (
<>
<FormField
control={form.control}
name="metadata.mongo.databaseUser"
render={({ field }) => (
<FormItem>
<FormLabel>Database User</FormLabel>
<FormControl>
<Input placeholder="Enter database user" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="metadata.mongo.databasePassword"
render={({ field }) => (
<FormItem>
<FormLabel>Database Password</FormLabel>
<FormControl>
<Input
type="password"
placeholder="Enter database password"
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
</>
)}
{databaseType === "mysql" && (
<FormField
control={form.control}
name="metadata.mysql.databaseRootPassword"
render={({ field }) => (
<FormItem>
<FormLabel>Root Password</FormLabel>
<FormControl>
<Input
type="password"
placeholder="Enter root password"
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}
<DialogFooter>
<Button
isLoading={isDeploying}
form="hook-form-restore-cloud-backup"
type="submit"
>
Restore
</Button>
</DialogFooter>
</form>
</Form>
<DrawerLogs
isOpen={isDrawerOpen}
onClose={() => {
setIsDrawerOpen(false);
setFilteredLogs([]);
setIsDeploying(false);
}}
filteredLogs={filteredLogs}
/>
</DialogContent>
</Dialog>
);
};

View File

@@ -0,0 +1,285 @@
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { api } from "@/utils/api";
import { Database, Play, RefreshCw, Trash2 } from "lucide-react";
import { useState } from "react";
import { toast } from "sonner";
import { AddCloudBackup } from "./add-cloud-backup";
import { RestoreCloudBackup } from "./restore-cloud-backup";
import { UpdateCloudBackup } from "./update-cloud-backup";
interface Props {
databaseId: string;
databaseType: "postgres" | "mariadb" | "mysql" | "mongo" | "web-server";
}
export const ShowCloudBackups = ({ databaseId, databaseType }: Props) => {
const [activeManualBackup, setActiveManualBackup] = useState<
string | undefined
>();
const [reconnectingBackupId, setReconnectingBackupId] = useState<
string | undefined
>();
const {
data: cloudBackups,
isLoading: isLoadingBackups,
refetch,
} = api.cloudStorageBackup.list.useQuery();
const { mutateAsync: manualBackup, isLoading: isManualBackup } =
api.cloudStorageBackup.manualBackup.useMutation();
const { mutateAsync: deleteBackup, isLoading: isRemoving } =
api.cloudStorageBackup.remove.useMutation();
const { mutateAsync: reconnectMutation } =
api.cloudStorageDestination.reconnect.useMutation();
const filteredBackups = cloudBackups?.filter((backup) => {
switch (databaseType) {
case "postgres":
return backup.postgresId === databaseId;
case "mysql":
return backup.mysqlId === databaseId;
case "mariadb":
return backup.mariadbId === databaseId;
case "mongo":
return backup.mongoId === databaseId;
case "web-server":
return backup.databaseType === "web-server";
default:
return false;
}
});
const handleReconnect = async (destinationId: string, backupId: string) => {
try {
setReconnectingBackupId(backupId);
const result = await reconnectMutation({ destinationId });
if (result.silent === false) {
toast.info(
"Please complete the authentication in the opened browser window.",
);
}
toast.success("Reconnected successfully! Please try your backup again.");
await refetch();
} catch (_err) {
toast.error("Reconnect failed or was cancelled. Please try again.");
} finally {
setReconnectingBackupId(undefined);
}
};
return (
<Card className="bg-background">
<CardHeader className="flex flex-row justify-between gap-4 flex-wrap">
<div className="flex flex-col gap-0.5">
<CardTitle className="text-xl flex flex-row gap-2">
<Database className="size-6 text-muted-foreground" />
Cloud Backups
</CardTitle>
<CardDescription>
Manage your cloud storage backups and restore data when needed.
</CardDescription>
</div>
{filteredBackups && filteredBackups.length > 0 && (
<div className="flex flex-col lg:flex-row gap-4 w-full lg:w-auto">
<AddCloudBackup
databaseId={databaseId}
databaseType={databaseType}
refetch={refetch}
/>
<RestoreCloudBackup
databaseId={databaseId}
databaseType={databaseType}
/>
</div>
)}
</CardHeader>
<CardContent className="flex flex-col gap-4">
{isLoadingBackups ? (
<div className="flex items-center justify-center py-8">
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary" />
</div>
) : filteredBackups?.length === 0 ? (
<div className="flex flex-col items-center gap-3 pt-10">
<Database className="size-8 text-muted-foreground" />
<span className="text-base text-muted-foreground">
No cloud backups configured
</span>
<div className="flex flex-col sm:flex-row gap-4 w-full sm:w-auto">
<AddCloudBackup
databaseId={databaseId}
databaseType={databaseType}
refetch={refetch}
/>
<RestoreCloudBackup
databaseId={databaseId}
databaseType={databaseType}
/>
</div>
</div>
) : (
<div className="flex flex-col gap-6">
{filteredBackups?.map((backup) => (
<div
key={backup.id}
className="flex w-full flex-col md:flex-row md:items-center justify-between gap-4 md:gap-10 border rounded-lg p-4"
>
<div className="grid grid-cols-1 md:grid-cols-3 xl:grid-cols-6 flex-col gap-8">
<div className="flex flex-col gap-1">
<span className="font-medium">Destination</span>
<span className="text-sm text-muted-foreground">
{backup.cloudStorageDestination?.name || "Unknown"}
</span>
</div>
<div className="flex flex-col gap-1">
<span className="font-medium">Database</span>
<span className="text-sm text-muted-foreground">
{backup.database}
</span>
</div>
<div className="flex flex-col gap-1">
<span className="font-medium">Scheduled</span>
<span className="text-sm text-muted-foreground">
{backup.schedule}
</span>
</div>
<div className="flex flex-col gap-1">
<span className="font-medium">Prefix Storage</span>
<span className="text-sm text-muted-foreground">
{backup.prefix}
</span>
</div>
<div className="flex flex-col gap-1">
<span className="font-medium">Enabled</span>
<span className="text-sm text-muted-foreground">
{backup.enabled ? "Yes" : "No"}
</span>
</div>
</div>
<div className="flex flex-row gap-4">
<TooltipProvider delayDuration={0}>
<Tooltip>
<TooltipTrigger asChild>
<Button
type="button"
variant="ghost"
isLoading={
isManualBackup && activeManualBackup === backup.id
}
onClick={async () => {
setActiveManualBackup(backup.id);
await manualBackup({
backupId: backup.id,
})
.then(() => {
toast.success("Manual Backup Successful");
})
.catch((err) => {
if (
err?.message
?.toLowerCase()
.includes("token expired") ||
err?.message
?.toLowerCase()
.includes("invalid_grant")
) {
toast.error(
"Error creating the manual backup: token expired, please refresh the token.",
);
} else {
toast.error(
"Error creating the manual backup",
);
}
});
setActiveManualBackup(undefined);
}}
>
{isManualBackup && activeManualBackup === backup.id ? (
<div className="animate-spin rounded-full h-5 w-5 border-b-2 border-primary" />
) : (
<Play className="size-5 text-muted-foreground" />
)}
</Button>
</TooltipTrigger>
<TooltipContent>Run Manual Backup</TooltipContent>
</Tooltip>
{/* Refresh Token for OAuth providers */}
{backup.cloudStorageDestination?.provider &&
["drive", "dropbox", "box"].includes(
backup.cloudStorageDestination.provider,
) && (
<Tooltip>
<TooltipTrigger asChild>
<Button
size="icon"
variant="ghost"
onClick={() =>
handleReconnect(
backup.cloudStorageDestination.id,
backup.id,
)
}
>
<RefreshCw
className={`size-4 text-primary group-hover:text-lime-500 ${reconnectingBackupId === backup.id ? "animate-spin" : ""}`}
/>
</Button>
</TooltipTrigger>
<TooltipContent>Refresh Token</TooltipContent>
</Tooltip>
)}
<UpdateCloudBackup backupId={backup.id} refetch={refetch} />
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="ghost"
size="icon"
className="group hover:bg-red-500/10"
isLoading={isRemoving}
onClick={async () => {
await deleteBackup({
backupId: backup.id,
})
.then(() => {
refetch();
toast.success("Backup deleted successfully");
})
.catch(() => {
toast.error("Error deleting backup");
});
}}
>
<Trash2 className="size-4 text-primary group-hover:text-red-500" />
</Button>
</TooltipTrigger>
<TooltipContent>Delete Backup</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>
</div>
))}
</div>
)}
</CardContent>
</Card>
);
};

View File

@@ -0,0 +1,183 @@
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import { Switch } from "@/components/ui/switch";
import { api } from "@/utils/api";
import { zodResolver } from "@hookform/resolvers/zod";
import { Pencil } from "lucide-react";
import { useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import { z } from "zod";
const updateBackupSchema = z.object({
schedule: z.string().min(1, "Schedule is required"),
prefix: z.string().min(1, "Prefix is required"),
enabled: z.boolean(),
keepLatestCount: z.string().optional(),
});
interface Props {
backupId: string;
refetch: () => void;
}
export const UpdateCloudBackup = ({ backupId, refetch }: Props) => {
const [open, setOpen] = useState(false);
const { data: backup, isLoading: isLoadingBackup } =
api.cloudStorageBackup.get.useQuery({ backupId });
const { mutateAsync: updateBackup, isLoading: isUpdating } =
api.cloudStorageBackup.update.useMutation();
const form = useForm<z.infer<typeof updateBackupSchema>>({
resolver: zodResolver(updateBackupSchema),
defaultValues: {
schedule: backup?.schedule || "",
prefix: backup?.prefix || "",
enabled: backup?.enabled || false,
keepLatestCount: backup?.keepLatestCount?.toString() || "",
},
});
const onSubmit = async (values: z.infer<typeof updateBackupSchema>) => {
await updateBackup({
backupId,
schedule: values.schedule,
prefix: values.prefix,
enabled: values.enabled,
keepLatestCount: values.keepLatestCount
? Number.parseInt(values.keepLatestCount)
: undefined,
})
.then(() => {
toast.success("Backup updated successfully");
refetch();
setOpen(false);
})
.catch(() => {
toast.error("Error updating backup");
});
};
return (
<Dialog open={open} onOpenChange={setOpen}>
<DialogTrigger asChild>
<Button variant="ghost" size="icon">
<Pencil className="size-4 text-muted-foreground" />
</Button>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>Update Cloud Backup</DialogTitle>
<DialogDescription>
Modify the settings for your cloud storage backup.
</DialogDescription>
</DialogHeader>
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
<FormField
control={form.control}
name="schedule"
render={({ field }) => (
<FormItem>
<FormLabel>Schedule (Cron Expression)</FormLabel>
<FormControl>
<Input
placeholder="0 0 * * *"
{...field}
disabled={isLoadingBackup || isUpdating}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="prefix"
render={({ field }) => (
<FormItem>
<FormLabel>Storage Prefix</FormLabel>
<FormControl>
<Input
placeholder="backups/database"
{...field}
disabled={isLoadingBackup || isUpdating}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="enabled"
render={({ field }) => (
<FormItem className="flex flex-row items-center justify-between rounded-lg border p-4">
<div className="space-y-0.5">
<FormLabel className="text-base">Enabled</FormLabel>
</div>
<FormControl>
<Switch
checked={field.value}
onCheckedChange={field.onChange}
disabled={isLoadingBackup || isUpdating}
/>
</FormControl>
</FormItem>
)}
/>
<FormField
control={form.control}
name="keepLatestCount"
render={({ field }) => (
<FormItem>
<FormLabel>Keep Latest Count (Optional)</FormLabel>
<FormControl>
<Input
type="number"
placeholder="10"
{...field}
disabled={isLoadingBackup || isUpdating}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<div className="flex justify-end gap-4">
<Button
type="button"
variant="outline"
onClick={() => setOpen(false)}
disabled={isUpdating}
>
Cancel
</Button>
<Button type="submit" isLoading={isUpdating}>
Update
</Button>
</div>
</form>
</Form>
</DialogContent>
</Dialog>
);
};

View File

@@ -34,6 +34,7 @@ import { useState } from "react";
import { toast } from "sonner";
import type { ServiceType } from "../../application/advanced/show-resources";
import { ShowDeploymentsModal } from "../../application/deployments/show-deployments-modal";
import { ShowCloudBackups } from "./cloud-storage/show-cloud-backups";
import { HandleBackup } from "./handle-backup";
import { RestoreBackup } from "./restore-backup";
@@ -102,10 +103,10 @@ export const ShowBackups = ({
<div className="flex flex-col gap-0.5">
<CardTitle className="text-xl flex flex-row gap-2">
<Database className="size-6 text-muted-foreground" />
Backups
S3 Backups
</CardTitle>
<CardDescription>
Add backups to your database to save the data to a different
Add S3 backups to your database to save the data to a different
provider.
</CardDescription>
</div>
@@ -151,7 +152,7 @@ export const ShowBackups = ({
<div className="flex w-full flex-col items-center justify-center gap-3 pt-10">
<DatabaseBackup className="size-8 text-muted-foreground" />
<span className="text-base text-muted-foreground">
No backups configured
No S3 backups configured
</span>
<div className="flex flex-col sm:flex-row gap-4 w-full sm:w-auto">
<HandleBackup
@@ -376,6 +377,9 @@ export const ShowBackups = ({
</div>
)}
</CardContent>
{databaseType && (
<ShowCloudBackups databaseId={id} databaseType={databaseType} />
)}
</Card>
);
};

View File

@@ -0,0 +1,562 @@
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { api } from "@/utils/api";
import {
type CloudStorageDestinationFormValues,
cloudStorageDestinationSchema,
} from "@dokploy/server/db/schema/cloud-storage-destination";
import { zodResolver } from "@hookform/resolvers/zod";
import {
Check,
CheckCircle2,
Copy,
Eye,
EyeOff,
Loader2,
XCircle,
} from "lucide-react";
import { useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import { CLOUD_STORAGE_PROVIDERS } from "./constants";
export const CLOUD_STORAGE_PROVIDER_TYPES = CLOUD_STORAGE_PROVIDERS.map(
(p) => p.key as string,
);
export function isCloudStorageProvider(provider: string): boolean {
return CLOUD_STORAGE_PROVIDER_TYPES.includes(provider);
}
interface Props {
destinationId?: string;
inTabContent?: boolean;
commonName?: string;
}
type CloudStorageProviderType = "drive" | "dropbox" | "box" | "ftp" | "sftp";
export const CloudStorageDestinations = ({
destinationId,
inTabContent = false,
commonName,
}: Props) => {
const [open, setOpen] = useState(false);
const utils = api.useUtils();
const [connectionTested, setConnectionTested] = useState(false);
const { mutateAsync: createMutation, isLoading: isCreating } =
api.cloudStorageDestination.create.useMutation();
const { mutateAsync: updateMutation, isLoading: isUpdating } =
api.cloudStorageDestination.update.useMutation();
const { mutateAsync: testConnection, isLoading: isLoadingConnection } =
api.cloudStorageDestination.testConnection.useMutation();
const isLoading = isCreating || isUpdating;
const { data: destinations } = api.cloudStorageDestination.all.useQuery();
const destination = destinationId
? destinations?.find((d) => d.id === destinationId)
: undefined;
const form = useForm<CloudStorageDestinationFormValues>({
defaultValues: {
providerType: undefined,
host: "",
username: "",
password: "",
port: "",
token: "",
},
resolver: zodResolver(cloudStorageDestinationSchema),
});
const providerType = form.watch("providerType") as CloudStorageProviderType;
useEffect(() => {
setConnectionTested(false);
form.setValue("token", "");
}, [providerType, form]);
useEffect(() => {
if (!open) {
form.reset();
setConnectionTested(false);
}
}, [open, form]);
useEffect(() => {
if (destination) {
form.reset({
providerType: destination.provider as any,
host: destination.host || "",
username: destination.username || "",
password: destination.password || "",
port: destination.port || "",
token: destination.config || "",
});
} else {
form.reset({
providerType: undefined,
host: "",
username: "",
password: "",
port: "",
token: "",
});
}
}, [form, destination]);
// Handle form submission
const onSubmit = async (data: CloudStorageDestinationFormValues) => {
try {
if (!connectionTested && !destinationId) {
toast.error("Please test the connection first");
return;
}
const credentials = {
...(data.providerType === "ftp" || data.providerType === "sftp"
? {
host: data.host,
username: data.username,
password: data.password,
port: data.port,
}
: {}),
...(["drive", "dropbox", "box"].includes(data.providerType)
? {
token: data.token,
}
: {}),
};
if (destinationId) {
await updateMutation({
destinationId,
name: commonName || data.providerType,
provider: data.providerType,
config: JSON.stringify(credentials),
});
toast.success("Destination updated");
} else {
await createMutation({
name: commonName || data.providerType,
provider: data.providerType,
config: JSON.stringify(credentials),
});
toast.success("Destination created");
}
await utils.cloudStorageDestination.all.invalidate();
form.reset();
setConnectionTested(false);
setOpen(false);
} catch (err: any) {
toast.error("Error saving destination", {
description: err.message,
});
}
};
// Handle test connection
const handleTestConnection = async () => {
const data = form.getValues();
try {
// For FTP/SFTP, ensure password is base64 encoded
if (
(data.providerType === "ftp" || data.providerType === "sftp") &&
data.password
) {
try {
const cleanPassword = data.password.trim();
if (!cleanPassword || cleanPassword.includes(" ")) {
throw new Error("Invalid password format");
}
form.setValue("password", cleanPassword);
} catch (_e) {
toast.error("Invalid password format", {
description:
"Please make sure you copied the entire output from 'rclone obscure' command without any extra spaces.",
});
return;
}
}
const result = await testConnection({
provider: data.providerType,
credentials: {
host: data.host,
username: data.username,
password: data.password,
port: data.port,
token: data.token,
},
});
// Store the token if it exists
if (result.token) {
form.setValue("token", result.token);
const providerName = (() => {
switch (data.providerType as CloudStorageProviderType) {
case "drive":
return "Google Drive";
case "dropbox":
return "Dropbox";
case "box":
return "Box";
case "ftp":
return "FTP";
case "sftp":
return "SFTP";
default:
return data.providerType;
}
})();
toast.success(`${providerName} authentication successful`, {
description: "Your account has been connected successfully.",
});
}
setConnectionTested(true);
if (!result.token) {
toast.success("Connection test successful");
}
} catch (err: any) {
setConnectionTested(false);
form.setValue("token", "");
toast.error("Connection test failed", {
description: "Connection failed, please try again.",
});
}
};
// Render provider-specific fields
const renderProviderFields = () => {
if (!providerType) return null;
switch (providerType) {
case "ftp":
case "sftp":
return (
<>
<FormField
control={form.control}
name="host"
render={({ field }) => (
<FormItem>
<FormLabel>Host</FormLabel>
<FormControl>
<Input placeholder="ftp.example.com" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="username"
render={({ field }) => (
<FormItem>
<FormLabel>Username</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="password"
render={({ field }) => (
<FormItem>
<FormLabel>Password</FormLabel>
<FormControl>
<Input
type="password"
{...field}
placeholder="Enter base64 encoded password"
/>
</FormControl>
<FormMessage />
<div className="text-xs text-muted-foreground mt-2 space-y-1">
<p>To get the base64 encoded password:</p>
<ol className="list-decimal list-inside space-y-1">
<li>
Open terminal and run:{" "}
<code className="px-1 py-0.5 bg-muted rounded">
rclone obscure your_password
</code>
</li>
<li>Copy the output and paste it here</li>
</ol>
<p className="text-yellow-600 dark:text-yellow-500 mt-1">
Note: Do not enter your plain password. It must be base64
encoded using rclone obscure.
</p>
</div>
</FormItem>
)}
/>
<FormField
control={form.control}
name="port"
render={({ field }) => (
<FormItem>
<FormLabel>Port (Optional)</FormLabel>
<FormControl>
<Input
{...field}
placeholder={providerType === "ftp" ? "21" : "22"}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
</>
);
case "drive":
case "dropbox":
case "box":
return (
<FormField
control={form.control}
name="token"
render={({ field: { value } }) => {
const [copied, setCopied] = useState(false);
const [showToken, setShowToken] = useState(false);
const handleCopy = async (text: string) => {
await navigator.clipboard.writeText(text);
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
const providerName = (() => {
switch (providerType as CloudStorageProviderType) {
case "drive":
return "Google Drive";
case "dropbox":
return "Dropbox";
case "box":
return "Box";
case "ftp":
return "FTP";
case "sftp":
return "SFTP";
default:
return providerType;
}
})();
return (
<FormItem>
<FormLabel>{providerName}</FormLabel>
<FormControl>
<div className="flex flex-col gap-3 p-4 border rounded-lg">
<div className="flex items-center gap-2">
{value ? (
<>
<CheckCircle2 className="h-5 w-5 text-green-500" />
<span className="text-sm font-medium">
Authentication Successful
</span>
</>
) : (
<>
<XCircle className="h-5 w-5 text-muted-foreground" />
<span className="text-sm font-medium text-muted-foreground">
Not Authenticated
</span>
</>
)}
</div>
{value && (
<div className="mt-2 space-y-2">
<div className="flex items-center justify-between">
<div className="text-xs font-medium text-muted-foreground">
OAuth Token:
</div>
<div className="flex items-center gap-2">
<Button
type="button"
variant="ghost"
size="sm"
className="h-8 px-2"
onClick={() => setShowToken(!showToken)}
>
{showToken ? (
<EyeOff className="h-4 w-4" />
) : (
<Eye className="h-4 w-4" />
)}
<span className="ml-2 text-xs">
{showToken ? "Hide Token" : "Show Token"}
</span>
</Button>
<Button
type="button"
variant="ghost"
size="sm"
className="h-8 px-2"
onClick={() => handleCopy(value)}
>
{copied ? (
<Check className="h-4 w-4 text-green-500" />
) : (
<Copy className="h-4 w-4" />
)}
<span className="ml-2 text-xs">
{copied ? "Copied!" : "Copy Token"}
</span>
</Button>
</div>
</div>
{showToken && (
<div className="p-3 bg-muted rounded-md">
<code className="font-mono text-xs break-all whitespace-pre-wrap">
{JSON.stringify(JSON.parse(value), null, 2)}
</code>
</div>
)}
</div>
)}
<div className="text-xs text-muted-foreground">
{value
? "Token obtained successfully. You can now create the destination."
: `Click 'Test Connection' to start the OAuth flow with ${providerName}`}
</div>
</div>
</FormControl>
<FormMessage />
</FormItem>
);
}}
/>
);
}
};
// Render the form content
const formContent = (
<>
<Form {...form}>
<form
id="cloud-storage-form"
onSubmit={form.handleSubmit(onSubmit)}
className="grid w-full gap-4"
>
<FormField
control={form.control}
name="providerType"
render={({ field }) => (
<FormItem>
<FormLabel>Provider Type</FormLabel>
<Select onValueChange={field.onChange} value={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select a storage provider" />
</SelectTrigger>
</FormControl>
<SelectContent>
{CLOUD_STORAGE_PROVIDERS.map((provider) => (
<SelectItem key={provider.key} value={provider.key}>
{provider.name}
</SelectItem>
))}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
{renderProviderFields()}
<DialogFooter className="flex flex-col sm:flex-row sm:justify-between items-center gap-4 mt-2">
<Button
type="button"
variant="secondary"
disabled={!providerType || isLoadingConnection}
onClick={handleTestConnection}
className="w-full sm:w-auto"
>
{isLoadingConnection ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Testing...
</>
) : (
"Test Connection"
)}
</Button>
<Button
type="submit"
isLoading={isLoading}
disabled={!providerType || (!connectionTested && !destinationId)}
className="w-full sm:w-auto order-first sm:order-last"
>
{destinationId ? "Update" : "Create"} Destination
</Button>
</DialogFooter>
</form>
</Form>
</>
);
if (inTabContent) {
return formContent;
}
return (
<Dialog open={open} onOpenChange={setOpen}>
<DialogTrigger className="" asChild>
<Button variant="outline">Add Cloud Storage</Button>
</DialogTrigger>
<DialogContent className="max-h-screen overflow-y-auto sm:max-w-2xl">
<DialogHeader>
<DialogTitle>
{destinationId ? "Update" : "Add"} Cloud Storage Destination
</DialogTitle>
<DialogDescription>
Configure cloud storage providers like Google Drive, FTP, or SFTP.
</DialogDescription>
</DialogHeader>
{formContent}
</DialogContent>
</Dialog>
);
};

View File

@@ -1,7 +1,15 @@
export const S3_PROVIDERS: Array<{
export interface CloudStorageProvider {
key: string;
name: string;
}> = [
oauthRequired?: boolean;
}
export interface S3Provider {
key: string;
name: string;
}
export const S3_PROVIDERS: S3Provider[] = [
{
key: "AWS",
name: "Amazon Web Services (AWS) S3",
@@ -131,3 +139,29 @@ export const S3_PROVIDERS: Array<{
name: "Any other S3 compatible provider",
},
];
export const CLOUD_STORAGE_PROVIDERS: CloudStorageProvider[] = [
{
key: "drive",
name: "Google Drive",
oauthRequired: true,
},
{
key: "dropbox",
name: "Dropbox",
oauthRequired: true,
},
{
key: "box",
name: "Box",
oauthRequired: true,
},
{
key: "ftp",
name: "FTP",
},
{
key: "sftp",
name: "SFTP",
},
] as const;

View File

@@ -27,6 +27,7 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { cn } from "@/lib/utils";
import { api } from "@/utils/api";
import { zodResolver } from "@hookform/resolvers/zod";
@@ -35,6 +36,7 @@ import { useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import { z } from "zod";
import { CloudStorageDestinations } from "./cloud-storage-destinations";
import { S3_PROVIDERS } from "./constants";
const addDestination = z.object({
@@ -52,13 +54,20 @@ type AddDestination = z.infer<typeof addDestination>;
interface Props {
destinationId?: string;
cloudDestinationId?: string;
type?: "s3" | "cloud";
}
export const HandleDestinations = ({ destinationId }: Props) => {
export const HandleDestinations = ({
destinationId,
cloudDestinationId,
type,
}: Props) => {
const [open, setOpen] = useState(false);
const utils = api.useUtils();
const { data: servers } = api.server.withSSHKey.useQuery();
const { data: isCloud } = api.settings.isCloud.useQuery();
const [activeTab, setActiveTab] = useState<string>(type || "s3");
const { mutateAsync, isError, error, isLoading } = destinationId
? api.destination.update.useMutation()
@@ -119,13 +128,15 @@ export const HandleDestinations = ({ destinationId }: Props) => {
destinationId: destinationId || "",
})
.then(async () => {
toast.success(`Destination ${destinationId ? "Updated" : "Created"}`);
toast.success(
`S3 Destination ${destinationId ? "Updated" : "Created"}`,
);
await utils.destination.all.invalidate();
setOpen(false);
})
.catch(() => {
toast.error(
`Error ${destinationId ? "Updating" : "Creating"} the Destination`,
`Error ${destinationId ? "Updating" : "Creating"} the S3 Destination`,
);
});
};
@@ -189,7 +200,7 @@ export const HandleDestinations = ({ destinationId }: Props) => {
return (
<Dialog open={open} onOpenChange={setOpen}>
<DialogTrigger className="" asChild>
{destinationId ? (
{destinationId || cloudDestinationId ? (
<Button
variant="ghost"
size="icon"
@@ -207,12 +218,11 @@ export const HandleDestinations = ({ destinationId }: Props) => {
<DialogContent className="max-h-screen overflow-y-auto sm:max-w-2xl">
<DialogHeader>
<DialogTitle>
{destinationId ? "Update" : "Add"} Destination
{destinationId || cloudDestinationId ? "Update" : "Add"} Destination
</DialogTitle>
<DialogDescription>
In this section, you can configure and add new destinations for your
backups. Please ensure that you provide the correct information to
guarantee secure and efficient storage.
Configure storage providers for your backups including S3-compatible
services, Google Drive, Dropbox, Box, and more.
</DialogDescription>
</DialogHeader>
{(isError || isErrorConnection) && (
@@ -221,6 +231,13 @@ export const HandleDestinations = ({ destinationId }: Props) => {
</AlertBlock>
)}
<Tabs
defaultValue={activeTab}
value={activeTab}
onValueChange={setActiveTab}
className="w-full"
>
{/* Common Name Field */}
<Form {...form}>
<form
id="hook-form-destination-add"
@@ -235,13 +252,23 @@ export const HandleDestinations = ({ destinationId }: Props) => {
<FormItem>
<FormLabel>Name</FormLabel>
<FormControl>
<Input placeholder={"S3 Bucket"} {...field} />
<Input
placeholder={"Backup Destination Name"}
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
);
}}
/>
<TabsList className="grid w-full grid-cols-2">
<TabsTrigger value="s3">S3 Storage</TabsTrigger>
<TabsTrigger value="cloud">Cloud Storage</TabsTrigger>
</TabsList>
<TabsContent value="s3" className="space-y-4 pt-2">
<FormField
control={form.control}
name="provider"
@@ -354,7 +381,6 @@ export const HandleDestinations = ({ destinationId }: Props) => {
</FormItem>
)}
/>
</form>
<DialogFooter
className={cn(
@@ -365,8 +391,8 @@ export const HandleDestinations = ({ destinationId }: Props) => {
{isCloud ? (
<div className="flex flex-col gap-4 border p-2 rounded-lg">
<span className="text-sm text-muted-foreground">
Select a server to test the destination. If you don't have a
server choose the default one.
Select a server to test the destination. If you don't
have a server choose the default one.
</span>
<FormField
control={form.control}
@@ -408,7 +434,9 @@ export const HandleDestinations = ({ destinationId }: Props) => {
variant={"secondary"}
isLoading={isLoadingConnection}
onClick={async () => {
await handleTestConnection(form.getValues("serverId"));
await handleTestConnection(
form.getValues("serverId"),
);
}}
>
Test Connection
@@ -435,7 +463,19 @@ export const HandleDestinations = ({ destinationId }: Props) => {
{destinationId ? "Update" : "Create"}
</Button>
</DialogFooter>
</TabsContent>
</form>
</Form>
{/* Cloud Storage Tab Content */}
<TabsContent value="cloud">
<CloudStorageDestinations
destinationId={cloudDestinationId}
inTabContent={true}
commonName={form.watch("name")}
/>
</TabsContent>
</Tabs>
</DialogContent>
</Dialog>
);

View File

@@ -10,24 +10,74 @@ import {
import { api } from "@/utils/api";
import { Database, FolderUp, Loader2, Trash2 } from "lucide-react";
import { toast } from "sonner";
import { isCloudStorageProvider } from "./cloud-storage-destinations";
import { HandleDestinations } from "./handle-destinations";
export const ShowDestinations = () => {
const { data, isLoading, refetch } = api.destination.all.useQuery();
const { mutateAsync, isLoading: isRemoving } =
const {
data: s3Destinations,
isLoading: isLoadingS3,
refetch: refetchS3,
} = api.destination.all.useQuery();
const {
data: cloudDestinations,
isLoading: isLoadingCloud,
refetch: refetchCloud,
} = api.cloudStorageDestination.all.useQuery();
const destinations = [
...(s3Destinations || []).map((s3) => ({
...s3,
isCloudStorage: false,
})),
...(cloudDestinations || []).map((cloud) => ({
destinationId: cloud.id,
name: cloud.name,
provider: cloud.provider,
createdAt: cloud.createdAt,
isCloudStorage: true,
})),
];
const isLoading = isLoadingS3 || isLoadingCloud;
const { mutateAsync: removeS3Destination, isLoading: isRemovingS3 } =
api.destination.remove.useMutation();
const { mutateAsync: removeCloudDestination, isLoading: isRemovingCloud } =
api.cloudStorageDestination.delete.useMutation();
const handleDelete = async (
destinationId: string,
isCloudStorage: boolean,
) => {
try {
if (isCloudStorage) {
await removeCloudDestination({ destinationId });
} else {
await removeS3Destination({ destinationId });
}
toast.success("Destination deleted successfully");
await refetchS3();
await refetchCloud();
} catch (_err: any) {
toast.error("Cannot delete destination", {
description:
"You must delete all backups associated with this destination before deleting the destination.",
});
}
};
return (
<div className="w-full">
<Card className="h-full bg-sidebar p-2.5 rounded-xl max-w-5xl mx-auto">
<div className="rounded-xl bg-background shadow-md ">
<CardHeader className="">
<CardHeader>
<CardTitle className="text-xl flex flex-row gap-2">
<Database className="size-6 text-muted-foreground self-center" />
S3 Destinations
Backup Destinations
</CardTitle>
<CardDescription>
Add your providers like AWS S3, Cloudflare R2, Wasabi,
DigitalOcean Spaces etc.
Configure storage providers for your backups including
S3-compatible services, Google Drive, Dropbox, Box and more.
</CardDescription>
</CardHeader>
<CardContent className="space-y-2 py-8 border-t">
@@ -38,63 +88,81 @@ export const ShowDestinations = () => {
</div>
) : (
<>
{data?.length === 0 ? (
{destinations?.length === 0 ? (
<div className="flex flex-col items-center gap-3 min-h-[25vh] justify-center">
<FolderUp className="size-8 self-center text-muted-foreground" />
<span className="text-base text-muted-foreground">
To create a backup it is required to set at least 1
provider.
storage provider.
</span>
<div className="flex flex-row gap-2">
<HandleDestinations />
</div>
</div>
) : (
<div className="flex flex-col gap-4 min-h-[25vh]">
<div className="flex flex-col gap-4 rounded-lg">
{data?.map((destination, index) => (
{destinations?.map((destination, index) => {
const isCloud = isCloudStorageProvider(
destination.provider || "",
);
return (
<div
key={destination.destinationId}
className="flex items-center justify-between bg-sidebar p-1 w-full rounded-lg"
className="flex flex-col bg-sidebar p-1 w-full rounded-lg"
>
<div className="flex items-center justify-between p-3.5 rounded-lg bg-background border w-full">
<div className="flex flex-col gap-1">
<div className="flex items-center gap-2">
<span className="text-sm">
{index + 1}. {destination.name}
{destination.provider && (
<span className="ml-2 text-xs text-muted-foreground">
({destination.provider})
</span>
)}
</span>
</div>
<span className="text-xs text-muted-foreground">
Created at:{" "}
{new Date(
{destination.createdAt
? new Date(
destination.createdAt,
).toLocaleDateString()}
).toLocaleDateString()
: "N/A"}
</span>
</div>
<div className="flex flex-row gap-1">
{isCloud ? (
<HandleDestinations
cloudDestinationId={
destination.destinationId
}
type="cloud"
/>
) : (
<HandleDestinations
destinationId={destination.destinationId}
type="s3"
/>
)}
<DialogAction
title="Delete Destination"
description="Are you sure you want to delete this destination?"
type="destructive"
onClick={async () => {
await mutateAsync({
destinationId: destination.destinationId,
})
.then(() => {
toast.success(
"Destination deleted successfully",
await handleDelete(
destination.destinationId,
isCloud,
);
refetch();
})
.catch(() => {
toast.error("Error deleting destination");
});
}}
>
<Button
variant="ghost"
size="icon"
className="group hover:bg-red-500/10"
isLoading={isRemoving}
isLoading={isRemovingS3 || isRemovingCloud}
>
<Trash2 className="size-4 text-primary group-hover:text-red-500" />
</Button>
@@ -102,7 +170,8 @@ export const ShowDestinations = () => {
</div>
</div>
</div>
))}
);
})}
</div>
<div className="flex flex-row gap-2 flex-wrap w-full justify-end mr-4">

View File

@@ -330,7 +330,7 @@ const MENU: Menu = {
},
{
isSingle: true,
title: "S3 Destinations",
title: "Backup Destinations",
url: "/dashboard/settings/destinations",
icon: Database,
// Only enabled for admins

View File

@@ -0,0 +1,33 @@
CREATE TABLE "cloud_storage_destination" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"name" text NOT NULL,
"provider" text NOT NULL,
"username" text,
"password" text,
"host" text,
"port" text,
"config" text,
"organization_id" text NOT NULL,
"created_at" timestamp DEFAULT now(),
"updated_at" timestamp DEFAULT now()
);
--> statement-breakpoint
CREATE TABLE "cloud_storage_backup" (
"id" text PRIMARY KEY NOT NULL,
"organization_id" text NOT NULL,
"cloud_storage_destination_id" uuid NOT NULL,
"schedule" text NOT NULL,
"enabled" boolean DEFAULT true NOT NULL,
"database_type" text NOT NULL,
"prefix" text,
"database" text,
"postgres_id" text,
"mysql_id" text,
"mariadb_id" text,
"mongo_id" text,
"keep_latest_count" integer,
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL
);
--> statement-breakpoint
ALTER TABLE "cloud_storage_backup" ADD CONSTRAINT "cloud_storage_backup_cloud_storage_destination_id_cloud_storage_destination_id_fk" FOREIGN KEY ("cloud_storage_destination_id") REFERENCES "public"."cloud_storage_destination"("id") ON DELETE no action ON UPDATE no action;

File diff suppressed because it is too large Load Diff

View File

@@ -652,6 +652,13 @@
"when": 1747713229160,
"tag": "0092_stiff_the_watchers",
"breakpoints": true
},
{
"idx": 93,
"version": "7",
"when": 1748419547200,
"tag": "0093_pale_guardsmen",
"breakpoints": true
}
]
}

View File

@@ -5,6 +5,8 @@ import { applicationRouter } from "./routers/application";
import { backupRouter } from "./routers/backup";
import { bitbucketRouter } from "./routers/bitbucket";
import { certificateRouter } from "./routers/certificate";
import { cloudStorageBackupRouter } from "./routers/cloud-storage-backup";
import { cloudStorageDestinationRouter } from "./routers/cloud-storage-destination";
import { clusterRouter } from "./routers/cluster";
import { composeRouter } from "./routers/compose";
import { deploymentRouter } from "./routers/deployment";
@@ -57,6 +59,7 @@ export const appRouter = createTRPCRouter({
domain: domainRouter,
destination: destinationRouter,
backup: backupRouter,
cloudStorageBackup: cloudStorageBackupRouter,
deployment: deploymentRouter,
previewDeployment: previewDeploymentRouter,
mounts: mountRouter,
@@ -80,6 +83,7 @@ export const appRouter = createTRPCRouter({
ai: aiRouter,
organization: organizationRouter,
schedule: scheduleRouter,
cloudStorageDestination: cloudStorageDestinationRouter,
});
// export type definition of API

View File

@@ -0,0 +1,626 @@
import { writeFileSync } from "node:fs";
import { createTRPCRouter, protectedProcedure } from "@/server/api/trpc";
import { db } from "@/server/db";
import { cloudStorageBackup } from "@dokploy/server/db/schema/cloud-storage-backup";
import { cloudStorageDestination } from "@dokploy/server/db/schema/cloud-storage-destination";
import { generateProviderConfig } from "@dokploy/server/db/schema/cloud-storage-destination";
import { cloudStorageBackupService } from "@dokploy/server/services/cloud-storage-backup";
import {
type CloudStorageProvider,
getRcloneConfigPath,
normalizeCloudPath,
} from "@dokploy/server/utils/backups/cloud-storage";
import {
executeCloudStorageBackup,
executeCloudStorageRestore,
} from "@dokploy/server/utils/backups/cloud-storage";
import {
execAsync,
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { TRPCError } from "@trpc/server";
import { observable } from "@trpc/server/observable";
import { eq } from "drizzle-orm";
import { z } from "zod";
const listBackupFilesInput = z.object({
destinationId: z.string(),
prefix: z.string().optional(),
searchTerm: z.string().optional(),
serverId: z.string().optional(),
});
const createBackupInput = z.object({
schedule: z.string(),
enabled: z.boolean(),
databaseType: z.enum(["postgres", "mariadb", "mysql", "mongo", "web-server"]),
cloudStorageDestinationId: z.string(),
prefix: z.string().optional(),
database: z.string().optional(),
postgresId: z.string().optional(),
mysqlId: z.string().optional(),
mariadbId: z.string().optional(),
mongoId: z.string().optional(),
});
const updateBackupInput = z.object({
backupId: z.string(),
schedule: z.string().optional(),
enabled: z.boolean().optional(),
databaseType: z
.enum(["postgres", "mariadb", "mysql", "mongo", "web-server"])
.optional(),
cloudStorageDestinationId: z.string().optional(),
prefix: z.string().optional(),
database: z.string().optional(),
postgresId: z.string().optional(),
mysqlId: z.string().optional(),
mariadbId: z.string().optional(),
mongoId: z.string().optional(),
keepLatestCount: z.number().optional(),
});
const manualBackupInput = z.object({
backupId: z.string(),
});
// Helper to perform an operation with silent token refresh for OAuth providers
async function withSilentTokenRefresh(params: {
ctx: any;
destination: any;
operationFn: (destination: any) => Promise<any>;
updateConfig?: boolean;
}): Promise<any> {
const { ctx, destination, operationFn, updateConfig = true } = params;
const provider = destination.provider;
if (["drive", "dropbox", "box"].includes(provider)) {
let credentials: Record<string, unknown> = {};
try {
credentials = destination.config ? JSON.parse(destination.config) : {};
} catch {}
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
destination.id,
);
// Use existing token if available
if (credentials.token) {
try {
const configContent = generateProviderConfig(provider, credentials);
writeFileSync(configPath, configContent);
return await operationFn(destination);
} catch (err: any) {
if (
err.stderr &&
(err.stderr.includes("token expired") ||
err.stderr.includes("invalid_grant") ||
err.stderr.includes("unauthorized") ||
err.stderr.includes("401") ||
err.stderr.includes("403"))
) {
console.log("Token expired or invalid, attempting refresh");
credentials.token = undefined;
} else {
throw err;
}
}
}
console.log("Getting new token through OAuth flow");
const {
testCloudStorageConnection,
} = require("./cloud-storage-destination");
const result = await testCloudStorageConnection({
provider,
credentials,
destinationId: destination.id,
ctx,
});
if (result.token) {
credentials.token = result.token;
if (updateConfig) {
await db
.update(cloudStorageDestination)
.set({ config: JSON.stringify(credentials), updatedAt: new Date() })
.where(eq(cloudStorageDestination.id, destination.id));
}
const configContent = generateProviderConfig(provider, credentials);
writeFileSync(configPath, configContent);
return await operationFn({
...destination,
config: JSON.stringify(credentials),
});
}
throw new TRPCError({
code: "UNAUTHORIZED",
message: "Failed to get new token. Please reconnect this destination.",
});
}
return await operationFn(destination);
}
export const cloudStorageBackupRouter = createTRPCRouter({
listBackupFiles: protectedProcedure
.input(listBackupFilesInput)
.query(async ({ input, ctx }) => {
const { destinationId, prefix, searchTerm, serverId } = input;
const [destination] = await db
.select()
.from(cloudStorageDestination)
.where(eq(cloudStorageDestination.id, destinationId))
.limit(1);
if (!destination) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Cloud storage destination not found",
});
}
if (destination.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not allowed to access this destination",
});
}
if (!destination.provider) {
throw new TRPCError({
code: "BAD_REQUEST",
message: "Destination is not a valid cloud storage provider",
});
}
const provider = destination.provider as CloudStorageProvider;
// Use withSilentTokenRefresh to handle OAuth token refresh
return await withSilentTokenRefresh({
ctx,
destination,
operationFn: async (dest) => {
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
dest.id,
);
let credentials: Record<string, unknown>;
try {
credentials = dest.config ? JSON.parse(dest.config) : {};
} catch (_e) {
credentials = {};
}
const configContent = generateProviderConfig(provider, credentials);
writeFileSync(configPath, configContent);
const normalizedPrefix =
prefix === "/" ? "" : normalizeCloudPath(prefix || "");
const rcloneRemote = provider;
const command = `rclone lsjson --config=${configPath} "${rcloneRemote}:${normalizedPrefix}"`;
console.log("Executing rclone command:", command);
try {
const { stdout } = serverId
? await execAsyncRemote(serverId, command)
: await execAsync(command);
console.log("Rclone output:", stdout);
const files = JSON.parse(stdout) as Array<{
Path: string;
Name: string;
Size: number;
ModTime: string;
IsDir: boolean;
Hashes: { MD5: string };
}>;
console.log("Parsed files:", files);
const backupFiles = files
.filter((file) =>
file.Name.endsWith(".zip") ||
file.Name.endsWith(".sql.gz")
)
.map((file) => ({
path: file.Path,
name: file.Name,
size: file.Size,
lastModified: new Date(file.ModTime),
isDir: file.IsDir,
hashes: file.Hashes,
}));
console.log("Filtered backup files:", backupFiles);
if (searchTerm) {
const filteredFiles = backupFiles.filter((file) =>
file.name.toLowerCase().includes(searchTerm.toLowerCase()),
);
console.log("Search filtered files:", filteredFiles);
return filteredFiles;
}
return backupFiles;
} catch (error: any) {
if (error.stderr?.includes("directory not found")) {
console.log("Directory not found, returning empty array");
return [];
}
console.error("Error listing cloud storage backup files:", error);
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: "Failed to list cloud storage backup files",
cause: error,
});
}
},
});
}),
create: protectedProcedure
.input(createBackupInput)
.mutation(async ({ input, ctx }) => {
const { cloudStorageDestinationId, ...data } = input;
const [destination] = await db
.select()
.from(cloudStorageDestination)
.where(eq(cloudStorageDestination.id, cloudStorageDestinationId))
.limit(1);
if (!destination) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Cloud storage destination not found",
});
}
if (destination.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not allowed to access this destination",
});
}
const backup = await cloudStorageBackupService.createBackup({
...data,
cloudStorageDestinationId,
organizationId: ctx.session.activeOrganizationId,
});
return backup;
}),
update: protectedProcedure
.input(updateBackupInput)
.mutation(async ({ input, ctx }) => {
const { backupId, cloudStorageDestinationId, ...data } = input;
if (cloudStorageDestinationId) {
const [destination] = await db
.select()
.from(cloudStorageDestination)
.where(eq(cloudStorageDestination.id, cloudStorageDestinationId))
.limit(1);
if (!destination) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Cloud storage destination not found",
});
}
if (destination.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not allowed to access this destination",
});
}
}
return await cloudStorageBackupService.updateBackup(backupId, {
...data,
cloudStorageDestinationId,
});
}),
remove: protectedProcedure
.input(z.object({ backupId: z.string() }))
.mutation(async ({ input }) => {
return await cloudStorageBackupService.removeBackup(input.backupId);
}),
list: protectedProcedure.query(async () => {
const backups = await db.query.cloudStorageBackup.findMany({
with: {
cloudStorageDestination: true,
},
});
return backups;
}),
get: protectedProcedure
.input(z.object({ backupId: z.string() }))
.query(async ({ input }) => {
const backup = await db.query.cloudStorageBackup.findFirst({
where: eq(cloudStorageBackup.id, input.backupId),
with: {
cloudStorageDestination: true,
},
});
if (!backup) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Backup not found",
});
}
return backup;
}),
manualBackup: protectedProcedure
.input(manualBackupInput)
.mutation(async ({ input, ctx }) => {
const { backupId } = input;
const backup = await cloudStorageBackupService.findBackupById(backupId);
if (backup.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not allowed to access this backup",
});
}
const [destination] = await db
.select()
.from(cloudStorageDestination)
.where(eq(cloudStorageDestination.id, backup.cloudStorageDestinationId))
.limit(1);
if (!destination) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Cloud storage destination not found",
});
}
if (destination.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not allowed to access this destination",
});
}
const doBackup = async (dest: any) => {
switch (backup.databaseType) {
case "web-server":
return await executeCloudStorageBackup(
{ ...backup, webServer: { path: backup.database || "" } },
dest,
);
case "postgres":
return await executeCloudStorageBackup(
{
...backup,
postgres: {
host: "localhost",
port: 5432,
user: "postgres",
password: "",
},
},
dest,
);
case "mysql":
return await executeCloudStorageBackup(
{
...backup,
mysql: {
host: "localhost",
port: 3306,
user: "root",
password: "",
},
},
dest,
);
case "mariadb":
return await executeCloudStorageBackup(
{
...backup,
mariadb: {
host: "localhost",
port: 3306,
user: "root",
password: "",
},
},
dest,
);
case "mongo":
return await executeCloudStorageBackup(
{
...backup,
mongo: {
host: "localhost",
port: 27017,
user: "",
password: "",
},
},
dest,
);
default:
throw new TRPCError({
code: "BAD_REQUEST",
message: "Unsupported database type for manual backup",
});
}
};
try {
await withSilentTokenRefresh({
ctx,
destination,
operationFn: doBackup,
});
return { success: true };
} catch (error) {
console.error("Manual backup error:", error);
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message:
error instanceof Error
? error.message
: "Failed to execute manual backup",
cause: error,
});
}
}),
restore: protectedProcedure
.input(
z.object({
destinationId: z.string(),
backupFile: z.string(),
databaseType: z
.enum(["postgres", "mariadb", "mysql", "mongo", "web-server"])
.optional(),
databaseName: z.string().optional(),
metadata: z.any().optional(),
}),
)
.mutation(async ({ input, ctx }) => {
const destination = await db.query.cloudStorageDestination.findFirst({
where: eq(cloudStorageDestination.id, input.destinationId),
});
if (!destination) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Destination not found",
});
}
const backup = await db.query.cloudStorageBackup.findFirst({
where: eq(
cloudStorageBackup.cloudStorageDestinationId,
input.destinationId,
),
});
if (!backup) {
throw new TRPCError({
code: "NOT_FOUND",
message: "No backup found for this destination",
});
}
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
destination.id,
);
let credentials: Record<string, unknown>;
try {
credentials = destination.config ? JSON.parse(destination.config) : {};
} catch (_e) {
credentials = {};
}
const configContent = generateProviderConfig(
destination.provider as CloudStorageProvider,
credentials,
);
writeFileSync(configPath, configContent);
const doRestore = async (dest: any) => {
const backupWithConfig = {
...backup,
webServer:
input.databaseType === "web-server"
? { path: input.databaseName || "/var/www/html" }
: undefined,
postgres:
input.databaseType === "postgres"
? {
host: "localhost",
port: 5432,
user: input.metadata?.postgres?.databaseUser || "postgres",
password: "",
}
: undefined,
mysql:
input.databaseType === "mysql"
? {
host: "localhost",
port: 3306,
user: "root",
password: input.metadata?.mysql?.databaseRootPassword || "",
}
: undefined,
mariadb:
input.databaseType === "mariadb"
? {
host: "localhost",
port: 3306,
user: input.metadata?.mariadb?.databaseUser || "root",
password: input.metadata?.mariadb?.databasePassword || "",
}
: undefined,
mongo:
input.databaseType === "mongo"
? {
host: "localhost",
port: 27017,
user: input.metadata?.mongo?.databaseUser || "",
password: input.metadata?.mongo?.databasePassword || "",
}
: undefined,
};
return await executeCloudStorageRestore(
backupWithConfig,
dest,
input.backupFile,
() => {},
);
};
try {
await withSilentTokenRefresh({
ctx,
destination,
operationFn: doRestore,
});
return { success: true };
} catch (error) {
console.error("Restore error:", error);
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message:
error instanceof Error ? error.message : "Failed to restore backup",
cause: error,
});
}
}),
restoreBackupWithLogs: protectedProcedure
.input(
z.object({
databaseId: z.string(),
databaseType: z.enum([
"postgres",
"mariadb",
"mysql",
"mongo",
"web-server",
]),
databaseName: z.string(),
backupFile: z.string(),
destinationId: z.string(),
metadata: z.any().optional(),
}),
)
.subscription(async () => {
return observable<string>((_emit) => {
// Implementation will be added
return () => {};
});
}),
});

View File

@@ -0,0 +1,642 @@
import { spawn } from "node:child_process";
import { existsSync, mkdirSync, unlinkSync, writeFileSync } from "node:fs";
import os from "node:os";
import { join } from "node:path";
import { createTRPCRouter, protectedProcedure } from "@/server/api/trpc";
import { db } from "@/server/db";
import { cloudStorageBackup } from "@dokploy/server/db/schema/cloud-storage-backup";
import {
cloudStorageDestination,
generateProviderConfig,
storeCredentials,
} from "@dokploy/server/db/schema/cloud-storage-destination";
import {
CloudStorageProviderEnum,
apiCreateCloudStorageDestination,
apiDeleteCloudStorageDestination,
apiUpdateCloudStorageDestination,
} from "@dokploy/server/db/schema/cloud-storage-destination";
import { execAsync } from "@dokploy/server/utils/process/execAsync";
import { TRPCError } from "@trpc/server";
import { observable } from "@trpc/server/observable";
import { and, desc, eq } from "drizzle-orm";
import { z } from "zod";
// Function to kill any existing rclone processes
async function killExistingRcloneProcesses() {
try {
const { stdout } = await execAsync("pgrep -f rclone");
const pids = stdout.trim().split("\n").filter(Boolean);
for (const pid of pids) {
try {
await execAsync(`kill -9 ${pid}`);
} catch (error) {
console.error(`Failed to kill process ${pid}:`, error);
}
}
} catch (_error) {
console.log("No existing rclone processes found");
}
}
// Function to create rclone config path
function getRcloneConfigPath(
organizationId: string,
destinationId: string,
): string {
const configDir = join(
os.homedir(),
".dokploy",
"rclone",
"cloud",
organizationId,
);
if (!existsSync(configDir)) {
mkdirSync(configDir, { recursive: true });
}
return join(configDir, `${destinationId}.conf`);
}
// Helper function to run rclone authorize command
async function runRcloneAuthorize(providerType: string): Promise<string> {
try {
const tempConfigPath = join(os.tmpdir(), `rclone-auth-${Date.now()}.conf`);
const configContent = `[${providerType}]
no_browser = true`;
writeFileSync(tempConfigPath, configContent);
try {
const { stdout } = await execAsync(
`rclone --config ${tempConfigPath} authorize "${providerType}"`,
);
console.log("Rclone output:", stdout);
const tokenMatch = stdout.match(/\{[\s\S]*?\}/);
if (tokenMatch) {
return tokenMatch[0];
}
throw new Error("Could not find token in rclone output");
} finally {
if (existsSync(tempConfigPath)) {
unlinkSync(tempConfigPath);
}
}
} catch (error) {
console.error("Failed to run rclone authorize:", error);
throw error;
}
}
// Add this helper function near the top of the file
async function testCloudStorageConnection({
provider,
credentials,
}: { provider: string; credentials: any }) {
if (["drive", "dropbox", "box"].includes(provider)) {
if (!credentials.token) {
const providerType =
provider === "drive"
? "drive"
: provider === "dropbox"
? "dropbox"
: "box";
try {
console.log("Starting OAuth flow for provider:", providerType);
await killExistingRcloneProcesses();
let lastError: Error | undefined;
for (let attempt = 0; attempt < 3; attempt++) {
try {
const token = await runRcloneAuthorize(providerType);
console.log("Got token from OAuth flow");
try {
const parsedToken = JSON.parse(token);
if (!parsedToken.access_token) {
throw new Error("Invalid token format: missing access_token");
}
credentials.token = token;
console.log("Token validated successfully");
break;
} catch (parseError) {
console.error("Token parse error:", parseError);
console.error("Token string:", token);
throw new Error(
"Failed to parse the OAuth token. Please try authenticating again.",
);
}
} catch (error: unknown) {
lastError =
error instanceof Error ? error : new Error(String(error));
console.error(`Attempt ${attempt + 1} failed:`, error);
if (
error instanceof Error &&
error.message.includes("address already in use")
) {
await killExistingRcloneProcesses();
await new Promise((resolve) => setTimeout(resolve, 1000));
continue;
}
throw error;
}
}
if (lastError) {
throw lastError;
}
} catch (authError) {
console.error("Authentication error:", authError);
throw new Error(
authError instanceof Error
? authError.message
: "Authentication failed",
);
}
}
}
// Create temporary config file
const configPath = join(os.tmpdir(), `test-${Date.now()}.conf`);
try {
const configContent = generateProviderConfig(provider, credentials);
writeFileSync(configPath, configContent);
await execAsync(`rclone lsd --config=${configPath} "${provider}:"`);
return {
success: true,
token: ["drive", "dropbox", "box"].includes(provider)
? credentials.token
: undefined,
};
} finally {
if (existsSync(configPath)) {
unlinkSync(configPath);
}
}
}
export const cloudStorageDestinationRouter = createTRPCRouter({
all: protectedProcedure.query(async ({ ctx }) => {
try {
const destinations = await db
.select()
.from(cloudStorageDestination)
.where(
eq(
cloudStorageDestination.organizationId,
ctx.session.activeOrganizationId,
),
)
.orderBy(desc(cloudStorageDestination.createdAt));
return destinations;
} catch (error) {
console.error("Error fetching destinations:", error);
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: "Error fetching cloud storage destinations",
});
}
}),
create: protectedProcedure
.input(apiCreateCloudStorageDestination)
.mutation(async ({ input, ctx }) => {
try {
const { provider, name, config } = input;
const credentials = JSON.parse(config);
const storedConfig = storeCredentials(provider, credentials);
const [newDestination] = await db
.insert(cloudStorageDestination)
.values({
name,
provider,
organizationId: ctx.session.activeOrganizationId,
config: storedConfig,
})
.returning();
if (!newDestination) {
throw new Error("Failed to create cloud storage destination");
}
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
newDestination.id,
);
const configContent = generateProviderConfig(provider, credentials);
writeFileSync(configPath, configContent);
return newDestination;
} catch (error) {
console.error("Create error:", error);
throw new TRPCError({
code: "BAD_REQUEST",
message:
error instanceof Error
? error.message
: "Error creating destination",
});
}
}),
update: protectedProcedure
.input(apiUpdateCloudStorageDestination)
.mutation(async ({ input, ctx }) => {
try {
const { destinationId, provider, name, config } = input;
let credentials: any = {};
try {
credentials = JSON.parse(config);
} catch {}
const [updatedDestination] = await db
.update(cloudStorageDestination)
.set({
name,
provider,
config,
host: credentials.host || null,
port: credentials.port || null,
username: credentials.username || null,
password: credentials.password || null,
updatedAt: new Date(),
})
.where(
and(
eq(cloudStorageDestination.id, destinationId),
eq(
cloudStorageDestination.organizationId,
ctx.session.activeOrganizationId,
),
),
)
.returning();
if (!updatedDestination) {
throw new Error("Cloud storage destination not found");
}
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
updatedDestination.id,
);
const configContent = generateProviderConfig(
updatedDestination.provider,
credentials,
);
writeFileSync(configPath, configContent);
return updatedDestination;
} catch (error) {
console.error("Update error:", error);
throw new TRPCError({
code: "BAD_REQUEST",
message:
error instanceof Error
? error.message
: "Error updating destination",
});
}
}),
// Delete a cloud storage destination
delete: protectedProcedure
.input(apiDeleteCloudStorageDestination)
.mutation(async ({ input, ctx }) => {
try {
const associatedBackups = await db
.select()
.from(cloudStorageBackup)
.where(
and(
eq(
cloudStorageBackup.cloudStorageDestinationId,
input.destinationId,
),
eq(
cloudStorageBackup.organizationId,
ctx.session.activeOrganizationId,
),
),
);
if (associatedBackups.length > 0) {
throw new TRPCError({
code: "BAD_REQUEST",
message:
"Cannot delete destination with associated backups. Please delete the backups first.",
});
}
const deletedDestination = await db
.delete(cloudStorageDestination)
.where(
and(
eq(cloudStorageDestination.id, input.destinationId),
eq(
cloudStorageDestination.organizationId,
ctx.session.activeOrganizationId,
),
),
)
.returning();
if (!deletedDestination.length) {
throw new Error("Cloud storage destination not found");
}
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
input.destinationId,
);
if (existsSync(configPath)) {
unlinkSync(configPath);
}
return deletedDestination[0];
} catch (error) {
console.error("Delete error:", error);
throw new TRPCError({
code: "BAD_REQUEST",
message:
error instanceof Error
? error.message
: "Error deleting destination",
});
}
}),
getStorageMetrics: protectedProcedure
.input(z.object({ destinationId: z.string() }))
.query(async ({ input, ctx }) => {
try {
const destinations = await db
.select()
.from(cloudStorageDestination)
.where(
and(
eq(cloudStorageDestination.id, input.destinationId),
eq(
cloudStorageDestination.organizationId,
ctx.session.activeOrganizationId,
),
),
)
.limit(1);
const destination = destinations[0];
if (!destination) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Cloud storage destination not found",
});
}
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
input.destinationId,
);
const result = await execAsync(
`rclone about --config=${configPath} "${destination.provider}:" --json`,
);
return JSON.parse(result.stdout);
} catch (error) {
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: "Failed to get storage metrics",
cause: error,
});
}
}),
authorizeOAuth: protectedProcedure
.input(
z.object({
provider: CloudStorageProviderEnum,
}),
)
.mutation(async ({ input }) => {
try {
if (input.provider !== "drive") {
throw new Error(
"OAuth authorization is only supported for Google Drive",
);
}
const result = await execAsync('rclone authorize "drive" | cat');
const tokenMatch = result.stdout.match(/\{.*\}/);
if (!tokenMatch) {
throw new Error("Failed to extract OAuth token from rclone output");
}
const token = tokenMatch[0];
try {
const parsedToken = JSON.parse(token);
if (!parsedToken.access_token || !parsedToken.refresh_token) {
throw new Error("Invalid token format");
}
} catch {
throw new Error("Invalid token JSON");
}
return { token };
} catch (error) {
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message:
error instanceof Error
? error.message
: "Failed to authorize with provider",
});
}
}),
testConnection: protectedProcedure
.input(
z.object({
provider: CloudStorageProviderEnum,
credentials: z.record(z.any()),
destinationId: z.string().optional(),
}),
)
.mutation(async ({ input }) => {
try {
return await testCloudStorageConnection({
provider: input.provider,
credentials: input.credentials,
});
} catch (error) {
console.error("Test connection error:", error);
const errorMessage =
error instanceof Error ? error.message : "Connection test failed";
throw new TRPCError({
code: "BAD_REQUEST",
message: errorMessage,
});
}
}),
getTransferStatus: protectedProcedure
.input(z.object({ destinationId: z.string() }))
.subscription(async ({ input, ctx }) => {
const destinations = await db
.select()
.from(cloudStorageDestination)
.where(
and(
eq(cloudStorageDestination.id, input.destinationId),
eq(
cloudStorageDestination.organizationId,
ctx.session.activeOrganizationId,
),
),
)
.limit(1);
const destination = destinations[0];
if (!destination) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Cloud storage destination not found",
});
}
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
input.destinationId,
);
return observable<string>((emit) => {
const process = spawn("rclone", [
"rc",
"core/stats",
`--config=${configPath}`,
]);
process.stdout.on("data", (data: Buffer) => {
emit.next(data.toString());
});
process.stderr.on("data", (data: Buffer) => {
console.error(`rclone error: ${data.toString()}`);
});
process.on("error", (error) => {
console.error("Failed to start rclone process:", error);
emit.error(error);
});
return () => {
process.kill();
};
});
}),
reconnect: protectedProcedure
.input(z.object({ destinationId: z.string() }))
.mutation(async ({ input, ctx }) => {
const destination = await db
.select()
.from(cloudStorageDestination)
.where(
and(
eq(cloudStorageDestination.id, input.destinationId),
eq(
cloudStorageDestination.organizationId,
ctx.session.activeOrganizationId,
),
),
);
const dest = destination[0];
if (!dest) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Destination not found",
});
}
let credentials: any = {};
try {
credentials = dest.config ? JSON.parse(dest.config) : {};
} catch {}
const provider = dest.provider;
if (["drive", "dropbox", "box"].includes(provider)) {
const configPath = getRcloneConfigPath(
ctx.session.activeOrganizationId,
dest.id,
);
// If there is a token, try silent refresh first
if (credentials.token) {
try {
const configContent = generateProviderConfig(provider, credentials);
writeFileSync(configPath, configContent);
await execAsync(`rclone lsd --config=${configPath} "${provider}:"`);
return { success: true, silent: true };
} catch (err: any) {
if (
err.stderr &&
(err.stderr.includes("token expired") ||
err.stderr.includes("invalid_grant") ||
err.stderr.includes("unauthorized") ||
err.stderr.includes("401") ||
err.stderr.includes("403"))
) {
console.log("Token expired or invalid, starting OAuth flow");
credentials.token = undefined;
} else {
console.error("Reconnect error:", err);
throw new TRPCError({
code: "BAD_REQUEST",
message: err.message || "Reconnect failed",
});
}
}
}
// If there is no token, start the OAuth flow
console.log("Starting full OAuth flow");
const result = await testCloudStorageConnection({
provider,
credentials,
});
if (result.token) {
credentials.token = result.token;
await db
.update(cloudStorageDestination)
.set({ config: JSON.stringify(credentials), updatedAt: new Date() })
.where(
and(
eq(cloudStorageDestination.id, input.destinationId),
eq(
cloudStorageDestination.organizationId,
ctx.session.activeOrganizationId,
),
),
);
const configContent = generateProviderConfig(provider, credentials);
writeFileSync(configPath, configContent);
return { success: true, silent: false };
}
throw new TRPCError({
code: "BAD_REQUEST",
message: "Failed to get new token",
});
}
throw new TRPCError({
code: "BAD_REQUEST",
message: "Reconnect is only supported for OAuth providers.",
});
}),
});

View File

@@ -0,0 +1,40 @@
import { relations } from "drizzle-orm";
import {
boolean,
integer,
pgTable,
text,
timestamp,
uuid,
} from "drizzle-orm/pg-core";
import { cloudStorageDestination } from "./cloud-storage-destination";
export const cloudStorageBackup = pgTable("cloud_storage_backup", {
id: text("id").primaryKey(),
organizationId: text("organization_id").notNull(),
cloudStorageDestinationId: uuid("cloud_storage_destination_id")
.notNull()
.references(() => cloudStorageDestination.id),
schedule: text("schedule").notNull(),
enabled: boolean("enabled").notNull().default(true),
databaseType: text("database_type").notNull(),
prefix: text("prefix"),
database: text("database"),
postgresId: text("postgres_id"),
mysqlId: text("mysql_id"),
mariadbId: text("mariadb_id"),
mongoId: text("mongo_id"),
keepLatestCount: integer("keep_latest_count"),
createdAt: timestamp("created_at").defaultNow().notNull(),
updatedAt: timestamp("updated_at").defaultNow().notNull(),
});
export const cloudStorageBackupRelations = relations(
cloudStorageBackup,
({ one }) => ({
cloudStorageDestination: one(cloudStorageDestination, {
fields: [cloudStorageBackup.cloudStorageDestinationId],
references: [cloudStorageDestination.id],
}),
}),
);

View File

@@ -0,0 +1,125 @@
import { pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core";
import { createInsertSchema, createSelectSchema } from "drizzle-zod";
import { z } from "zod";
// Define cloud storage provider types
export const CloudStorageProviderEnum = z.enum([
"drive", // Google Drive
"dropbox", // Dropbox
"box", // Box
"ftp", // FTP
"sftp", // SFTP
]);
export type CloudStorageProvider = z.infer<typeof CloudStorageProviderEnum>;
// All providers use direct credentials
export function isCredentialProvider(provider: string): boolean {
return ["ftp", "sftp"].includes(provider);
}
export function isOAuthProvider(provider: string): boolean {
return ["drive", "dropbox", "box"].includes(provider);
}
// Define the cloud storage destination table
export const cloudStorageDestination = pgTable("cloud_storage_destination", {
id: uuid("id").primaryKey().defaultRandom(),
name: text("name").notNull(),
provider: text("provider").notNull(),
// Direct credential fields for FTP/SFTP
username: text("username"), // For FTP/SFTP
password: text("password"), // For FTP/SFTP
host: text("host"), // For FTP/SFTP
port: text("port"), // For FTP/SFTP
config: text("config"), // For OAuth token (Drive, Dropbox, Box)
organizationId: text("organization_id").notNull(),
createdAt: timestamp("created_at").defaultNow(),
updatedAt: timestamp("updated_at").defaultNow(),
});
// Create Zod schemas for validation
export const insertCloudStorageDestinationSchema = createInsertSchema(
cloudStorageDestination,
);
export const selectCloudStorageDestinationSchema = createSelectSchema(
cloudStorageDestination,
);
// API schemas
export const apiCreateCloudStorageDestination = z.object({
name: z.string().min(1, "Name is required"),
provider: CloudStorageProviderEnum,
config: z.string(),
});
export const apiUpdateCloudStorageDestination =
apiCreateCloudStorageDestination.extend({
destinationId: z.string(),
});
export const apiFindOneCloudStorageDestination = z.object({
destinationId: z.string(),
});
export const apiDeleteCloudStorageDestination = z.object({
destinationId: z.string(),
});
// Form validation schema
export const cloudStorageDestinationSchema = z.object({
providerType: CloudStorageProviderEnum,
host: z.string().optional(),
username: z.string().optional(),
password: z.string().optional(),
port: z.string().optional(),
token: z.string().optional(),
});
export type CloudStorageDestinationFormValues = z.infer<
typeof cloudStorageDestinationSchema
>;
// Store raw credentials
export function storeCredentials(
provider: string,
credentials?: Record<string, any>,
): string {
// For OAuth, just store the token
if (provider === "drive") {
return credentials?.token || "";
}
// For other providers, store credentials as is
return JSON.stringify(credentials || {});
}
// Generate rclone config
export function generateProviderConfig(
provider: string,
credentials?: Record<string, any>,
): string {
switch (provider) {
case "ftp":
case "sftp":
return `[${provider}]
type = ${provider}
host = ${credentials?.host || ""}
user = ${credentials?.username || ""}
pass = ${credentials?.password || ""}
${credentials?.port ? `port = ${credentials?.port}` : ""}`;
case "drive":
case "dropbox":
case "box":
if (!credentials?.token) {
throw new Error("OAuth token is required");
}
return `[${provider}]
type = ${provider}
${provider === "drive" ? "scope = drive.file\n" : ""}token = ${credentials.token}`;
default:
throw new Error(`Unsupported provider: ${provider}`);
}
}

View File

@@ -32,3 +32,5 @@ export * from "./preview-deployments";
export * from "./ai";
export * from "./account";
export * from "./schedule";
export * from "./cloud-storage-destination";
export * from "./cloud-storage-backup";

View File

@@ -0,0 +1,104 @@
import { randomUUID } from "node:crypto";
import { db } from "@dokploy/server/db";
import { cloudStorageBackup } from "@dokploy/server/db/schema/cloud-storage-backup";
import { TRPCError } from "@trpc/server";
import { eq } from "drizzle-orm";
export type CloudStorageBackup = typeof cloudStorageBackup.$inferSelect;
export const cloudStorageBackupService = {
createBackup: async (data: {
organizationId: string;
schedule: string;
enabled: boolean;
databaseType: string;
cloudStorageDestinationId: string;
prefix?: string;
database?: string;
postgresId?: string;
mysqlId?: string;
mariadbId?: string;
mongoId?: string;
}) => {
console.log("Creating backup with data:", data);
const backupData = {
...data,
id: randomUUID(),
};
console.log("Backup data to insert:", backupData);
try {
const [backup] = await db
.insert(cloudStorageBackup)
.values(backupData)
.returning();
console.log("Successfully created backup:", backup);
return backup;
} catch (error) {
console.error("Error creating backup:", error);
throw error;
}
},
updateBackup: async (
id: string,
data: {
schedule?: string;
enabled?: boolean;
databaseType?: string;
cloudStorageDestinationId?: string;
prefix?: string;
database?: string;
postgresId?: string;
mysqlId?: string;
mariadbId?: string;
mongoId?: string;
},
) => {
const [backup] = await db
.update(cloudStorageBackup)
.set(data)
.where(eq(cloudStorageBackup.id, id))
.returning();
return backup;
},
removeBackup: async (id: string) => {
const [backup] = await db
.delete(cloudStorageBackup)
.where(eq(cloudStorageBackup.id, id))
.returning();
return backup;
},
findBackupById: async (id: string) => {
const backup = await db.query.cloudStorageBackup.findFirst({
where: eq(cloudStorageBackup.id, id),
with: {
cloudStorageDestination: true,
},
});
if (!backup) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Backup not found",
});
}
return backup;
},
listBackups: async (organizationId: string) => {
console.log("Listing backups for organization:", organizationId);
const backups = await db
.select()
.from(cloudStorageBackup)
.where(eq(cloudStorageBackup.organizationId, organizationId));
console.log("Found backups:", backups);
return backups;
},
};

View File

@@ -0,0 +1,185 @@
import { eq } from "drizzle-orm";
import { db } from "../db";
import {
type CloudStorageProvider,
cloudStorageDestination,
isCredentialProvider,
} from "../db/schema/cloud-storage-destination";
interface Credential {
user: string;
pass: string;
host: string;
}
/**
* Service for handling cloud storage destinations
*/
export class CloudStorageDestinationService {
/**
* Create a new cloud storage destination
*/
async create({
name,
provider,
config,
organizationId,
}: {
name: string;
provider: CloudStorageProvider;
config: string;
organizationId: string;
}) {
try {
return await db
.insert(cloudStorageDestination)
.values({
name,
provider,
config,
organizationId,
})
.returning();
} catch (error) {
console.error("Error creating cloud storage destination", {
error,
name,
provider,
});
throw error;
}
}
/**
* Update an existing cloud storage destination
*/
async update({
destinationId,
name,
provider,
config,
}: {
destinationId: string;
name: string;
provider: CloudStorageProvider;
config: string;
}) {
try {
return await db
.update(cloudStorageDestination)
.set({
name,
provider,
config,
})
.where(eq(cloudStorageDestination.id, destinationId))
.returning();
} catch (error) {
console.error("Error updating cloud storage destination", {
error,
destinationId,
});
throw error;
}
}
/**
* Delete a cloud storage destination
*/
async delete(destinationId: string) {
try {
await db
.delete(cloudStorageDestination)
.where(eq(cloudStorageDestination.id, destinationId));
return { success: true };
} catch (error) {
console.error("Error deleting cloud storage destination", {
error,
destinationId,
});
throw error;
}
}
/**
* Test connection to a cloud storage destination
*/
async testConnection({
provider,
customConfig,
}: {
provider: CloudStorageProvider;
customConfig: string;
}) {
try {
// For credential-based providers
if (isCredentialProvider(provider)) {
if (provider === "drive") {
try {
const config = JSON.parse(customConfig);
if (!config.clientId || !config.clientSecret) {
throw new Error(
"Invalid OAuth configuration: missing client credentials",
);
}
return {
success: true,
message: "Google Drive OAuth configuration is valid",
};
} catch (_error) {
throw new Error("Invalid OAuth configuration format");
}
}
const credentials = this.parseCredentials(customConfig);
if (!credentials) {
throw new Error(
`Invalid ${provider.toUpperCase()} configuration: missing credentials`,
);
}
return {
success: true,
message: `${provider.toUpperCase()} configuration is valid`,
};
}
return { success: true, message: "Configuration validated" };
} catch (error) {
console.error("Error testing cloud storage connection", {
error,
provider,
});
throw error;
}
}
/**
* Parse credentials from configuration string for FTP/SFTP
*/
private parseCredentials(configString: string): Credential | null {
try {
const lines = configString.split("\n");
const hostLine = lines.find((line) => line.startsWith("host="));
const userLine = lines.find((line) => line.startsWith("user="));
const passLine = lines.find((line) => line.startsWith("pass="));
if (!hostLine || !userLine || !passLine) {
return null;
}
return {
host: hostLine.replace("host=", "").trim(),
user: userLine.replace("user=", "").trim(),
pass: passLine.replace("pass=", "").trim(),
};
} catch (error) {
console.error("Error parsing credentials from config string", { error });
return null;
}
}
}
// Export a singleton instance
export const cloudStorageDestinationService =
new CloudStorageDestinationService();

View File

@@ -0,0 +1,496 @@
import { existsSync, mkdirSync } from "node:fs";
import { mkdir, writeFile } from "node:fs/promises";
import { homedir } from "node:os";
import { join } from "node:path";
import { paths } from "@dokploy/server/constants";
import type {
cloudStorageBackup,
cloudStorageDestination,
} from "@dokploy/server/db/schema";
import type { InferSelectModel } from "drizzle-orm";
import { execAsync } from "../process/execAsync";
type CloudStorageDestination = InferSelectModel<typeof cloudStorageDestination>;
type CloudStorageBackup = InferSelectModel<typeof cloudStorageBackup> & {
postgres?: { host: string; port: number; user: string; password: string };
mysql?: { host: string; port: number; user: string; password: string };
mariadb?: { host: string; port: number; user: string; password: string };
mongo?: { host: string; port: number; user: string; password: string };
webServer?: { path: string };
};
export type CloudStorageProvider = "drive" | "dropbox" | "box" | "ftp" | "sftp";
export const normalizeCloudPath = (path: string): string => {
const normalized = path.replace(/\/+/g, "/").trim();
const cleaned = normalized.replace(/^\/+|\/+$/g, "");
return cleaned ? `${cleaned}/` : "";
};
export const constructCloudStoragePath = (
provider: CloudStorageProvider,
prefix: string | null,
fileName: string,
): string => {
const normalizedPrefix = normalizeCloudPath(prefix || "");
switch (provider) {
case "drive":
case "dropbox":
case "box":
case "ftp":
return `${provider}:${normalizedPrefix}${fileName}`;
case "sftp": {
const sftpPath = normalizedPrefix.replace(/^\/+/, "");
return `${provider}:${sftpPath}${fileName}`;
}
default:
throw new Error(`Unsupported provider: ${provider}`);
}
};
export const getCloudStorageCredentials = async (
destination: CloudStorageDestination,
): Promise<string[]> => {
const { provider, config } = destination;
const credentials: string[] = [];
try {
switch (provider) {
case "drive":
case "dropbox":
case "box": {
const configDir = join(homedir(), ".config", "rclone");
await mkdir(configDir, { recursive: true });
const configFile = join(configDir, "rclone.conf");
let configContent = "";
const parsedOAuthConfig = JSON.parse(config || "{}");
switch (provider) {
case "drive":
if (!parsedOAuthConfig.token) {
throw new Error("Google Drive OAuth token is missing");
}
configContent = `[drive]
type = drive
token = ${parsedOAuthConfig.token}
`;
break;
case "dropbox":
if (!parsedOAuthConfig.token) {
throw new Error("Dropbox OAuth token is missing");
}
configContent = `[dropbox]
type = dropbox
token = ${parsedOAuthConfig.token}
`;
break;
case "box":
if (!parsedOAuthConfig.token) {
throw new Error("Box OAuth token is missing");
}
configContent = `[box]
type = box
token = ${parsedOAuthConfig.token}
`;
break;
}
await writeFile(configFile, configContent, "utf8");
credentials.push(`--config=${configFile}`);
break;
}
case "ftp":
case "sftp": {
const ftpConfigDir = join(homedir(), ".config", "rclone");
await mkdir(ftpConfigDir, { recursive: true });
const ftpConfigFile = join(ftpConfigDir, "rclone.conf");
let ftpConfigContent = "";
const parsedFtpConfig = JSON.parse(config || "{}");
const { host, username, password, port } = parsedFtpConfig;
if (!username || !password || !host) {
throw new Error(`${provider.toUpperCase()} credentials are missing`);
}
ftpConfigContent = `[${provider}]
type = ${provider}
host = ${host}
user = ${username}
pass = ${password}
${port ? `port = ${port}` : ""}
mkdir = true
dir_perms = 0755
file_perms = 0644
concurrency = 4
transfers = 4
checkers = 4
low_level_retries = 10
retries = 3
retries_sleep = 10s
${
provider === "ftp"
? `
allow_writeable_chroot = true
passive = true
encoding = Slash,Del,Ctl,RightSpace,Dot`
: ""
}`;
await writeFile(ftpConfigFile, ftpConfigContent, "utf8");
credentials.push(`--config=${ftpConfigFile}`);
break;
}
default:
throw new Error(`Unsupported cloud storage provider: ${provider}`);
}
} catch (error) {
console.error("Error parsing cloud storage config:", error);
throw new Error("Invalid cloud storage configuration");
}
return credentials;
};
export const isCloudStorage = (provider?: string): boolean => {
return ["drive", "dropbox", "box", "ftp", "sftp"].includes(provider || "");
};
export const executeCloudStorageBackup = async (
backup: CloudStorageBackup,
destination: CloudStorageDestination,
): Promise<void> => {
try {
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
const filename = `${backup.database || "backup"}-${timestamp}.sql.gz`;
const tempDir = `/tmp/dokploy-backup-${Date.now()}`;
await execAsync(`rm -rf ${tempDir}`);
await execAsync(`mkdir -p ${tempDir}`);
try {
let backupCommand = "";
switch (backup.databaseType) {
case "postgres":
if (!backup.postgres)
throw new Error("PostgreSQL configuration missing");
backupCommand = `PGPASSWORD="${backup.postgres.password}" pg_dump -h ${backup.postgres.host} -p ${backup.postgres.port} -U ${backup.postgres.user} -d ${backup.database} | gzip | rclone rcat ${(await getCloudStorageCredentials(destination)).join(" ")} "${constructCloudStoragePath(destination.provider as CloudStorageProvider, backup.prefix || "", filename)}"`;
break;
case "mysql":
if (!backup.mysql) throw new Error("MySQL configuration missing");
backupCommand = `mysqldump -h ${backup.mysql.host} -P ${backup.mysql.port} -u ${backup.mysql.user} -p${backup.mysql.password} ${backup.database} | gzip | rclone rcat ${(await getCloudStorageCredentials(destination)).join(" ")} "${constructCloudStoragePath(destination.provider as CloudStorageProvider, backup.prefix || "", filename)}"`;
break;
case "mariadb":
if (!backup.mariadb) throw new Error("MariaDB configuration missing");
backupCommand = `mysqldump -h ${backup.mariadb.host} -P ${backup.mariadb.port} -u ${backup.mariadb.user} -p${backup.mariadb.password} ${backup.database} | gzip | rclone rcat ${(await getCloudStorageCredentials(destination)).join(" ")} "${constructCloudStoragePath(destination.provider as CloudStorageProvider, backup.prefix || "", filename)}"`;
break;
case "mongo":
if (!backup.mongo) throw new Error("MongoDB configuration missing");
backupCommand = `mongodump --host ${backup.mongo.host} --port ${backup.mongo.port} --username ${backup.mongo.user} --password ${backup.mongo.password} --db ${backup.database} --archive --gzip | rclone rcat ${(await getCloudStorageCredentials(destination)).join(" ")} "${constructCloudStoragePath(destination.provider as CloudStorageProvider, backup.prefix || "", filename)}"`;
break;
case "web-server": {
const { BASE_PATH } = paths();
const webServerFilename = `webserver-backup-${timestamp}.zip`;
try {
const { stdout: dirExists } = await execAsync(
`test -d "${BASE_PATH}" && echo "exists" || echo "not exists"`,
);
if (dirExists.trim() === "not exists") {
throw new Error(`Directory ${BASE_PATH} does not exist`);
}
await execAsync(`mkdir -p ${tempDir}/filesystem`);
await execAsync(
`rsync -a --ignore-errors ${BASE_PATH}/ ${tempDir}/filesystem/`,
);
const { stdout: containerId } = await execAsync(
`docker ps --filter "name=dokploy-postgres" --filter "status=running" -q | head -n 1`,
);
if (containerId) {
const postgresContainerId = containerId.trim();
await execAsync(
`docker exec ${postgresContainerId} pg_dump -v -Fc -U dokploy -d dokploy -f /tmp/database.sql`,
);
await execAsync(
`docker cp ${postgresContainerId}:/tmp/database.sql ${tempDir}/database.sql`,
);
await execAsync(
`docker exec ${postgresContainerId} rm -f /tmp/database.sql`,
);
}
await execAsync(
`cd ${tempDir} && zip -r ${webServerFilename} *.sql filesystem/ > /dev/null 2>&1`,
);
const backupPath = constructCloudStoragePath(
destination.provider as CloudStorageProvider,
backup.prefix || "",
webServerFilename,
);
const credentials = await getCloudStorageCredentials(destination);
const rcloneCommand = `rclone copyto ${tempDir}/${webServerFilename} "${backupPath}" ${credentials.join(" ")}`;
await execAsync(rcloneCommand);
console.log(`Cloud storage backup completed: ${backupPath}`);
return;
} finally {
await execAsync(`rm -rf ${tempDir}`);
}
}
default:
throw new Error(`Unsupported database type: ${backup.databaseType}`);
}
if (backupCommand) {
await execAsync(backupCommand);
console.log(`Cloud storage backup completed: ${filename}`);
}
if (backup.keepLatestCount) {
await keepLatestNCloudStorageBackups(
backup,
destination,
backup.keepLatestCount,
);
}
} finally {
await execAsync(`rm -rf ${tempDir}`);
}
} catch (error) {
console.error("Error during cloud storage backup:", error);
throw error;
}
};
export const executeCloudStorageRestore = async (
backup: CloudStorageBackup,
destination: CloudStorageDestination,
backupPath: string,
emit: (log: string) => void,
): Promise<void> => {
try {
const tempDir = `/tmp/dokploy-restore-${Date.now()}`;
await execAsync(`mkdir -p ${tempDir}`);
try {
const configPath = getRcloneConfigPath(
destination.organizationId,
destination.id,
);
const rcloneCommand = `rclone copy ${backupPath} ${tempDir} --config=${configPath}`;
await execAsync(rcloneCommand);
let restoreCommand = "";
switch (backup.databaseType) {
case "postgres":
if (!backup.postgres)
throw new Error("PostgreSQL configuration missing");
restoreCommand = `psql -h ${backup.postgres.host} -p ${backup.postgres.port} -U ${backup.postgres.user} -d ${backup.database} < ${tempDir}/${backupPath.split("/").pop()}`;
break;
case "mysql":
if (!backup.mysql) throw new Error("MySQL configuration missing");
restoreCommand = `mysql -h ${backup.mysql.host} -P ${backup.mysql.port} -u ${backup.mysql.user} -p${backup.mysql.password} ${backup.database} < ${tempDir}/${backupPath.split("/").pop()}`;
break;
case "mariadb":
if (!backup.mariadb) throw new Error("MariaDB configuration missing");
restoreCommand = `mysql -h ${backup.mariadb.host} -P ${backup.mariadb.port} -u ${backup.mariadb.user} -p${backup.mariadb.password} ${backup.database} < ${tempDir}/${backupPath.split("/").pop()}`;
break;
case "mongo":
if (!backup.mongo) throw new Error("MongoDB configuration missing");
restoreCommand = `mongorestore --host ${backup.mongo.host} --port ${backup.mongo.port} --username ${backup.mongo.user} --password ${backup.mongo.password} --db ${backup.database} ${tempDir}/${backup.database}`;
break;
case "web-server": {
if (!backup.webServer)
throw new Error("Web server configuration missing");
await execAsync(`mkdir -p ${backup.webServer.path}`);
emit("Extracting backup...");
await execAsync(
`cd ${tempDir} && unzip ${backupPath.split("/").pop()} > /dev/null 2>&1`,
);
emit("Restoring filesystem...");
emit(
`Copying from ${tempDir}/filesystem/* to ${backup.webServer.path}/`,
);
emit("Cleaning target directory...");
await execAsync(`rm -rf "${backup.webServer.path}/"*`);
emit("Setting up target directory...");
await execAsync(`mkdir -p "${backup.webServer.path}"`);
emit("Copying files...");
await execAsync(
`cp -rp "${tempDir}/filesystem/"* "${backup.webServer.path}/"`,
);
emit("Starting database restore...");
const { stdout: hasSqlFile } = await execAsync(
`ls ${tempDir}/database.sql || true`,
);
if (hasSqlFile.includes("database.sql")) {
const { stdout: postgresContainer } = await execAsync(
`docker ps --filter "name=dokploy-postgres" --filter "status=running" -q | head -n 1`,
);
if (postgresContainer) {
const postgresContainerId = postgresContainer.trim();
emit("Disconnecting all users from database...");
await execAsync(
`docker exec ${postgresContainerId} psql -U dokploy postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'dokploy' AND pid <> pg_backend_pid();"`,
);
emit("Dropping existing database...");
await execAsync(
`docker exec ${postgresContainerId} psql -U dokploy postgres -c "DROP DATABASE IF EXISTS dokploy;"`,
);
emit("Creating fresh database...");
await execAsync(
`docker exec ${postgresContainerId} psql -U dokploy postgres -c "CREATE DATABASE dokploy;"`,
);
emit("Copying backup file into container...");
await execAsync(
`docker cp ${tempDir}/database.sql ${postgresContainerId}:/tmp/database.sql`,
);
emit("Running database restore...");
await execAsync(
`docker exec ${postgresContainerId} pg_restore -v -U dokploy -d dokploy /tmp/database.sql`,
);
emit("Cleaning up container temp file...");
await execAsync(
`docker exec ${postgresContainerId} rm /tmp/database.sql`,
);
}
}
break;
}
default:
throw new Error(`Unsupported database type: ${backup.databaseType}`);
}
await execAsync(restoreCommand);
await execAsync(`rm -rf ${tempDir}`);
emit(`Cloud storage restore completed: ${backupPath}`);
} finally {
await execAsync(`rm -rf ${tempDir}`);
}
} catch (error) {
console.error(`Error executing cloud storage restore: ${error}`);
throw error;
}
};
export const listCloudStorageFiles = async (
destination: CloudStorageDestination,
prefix: string | null,
searchTerm?: string,
): Promise<
Array<{
path: string;
size: number;
isDir: boolean;
hashes?: { MD5?: string };
}>
> => {
try {
const credentials = await getCloudStorageCredentials(destination);
const normalizedPrefix = normalizeCloudPath(prefix || "");
const searchPattern = searchTerm ? `*${searchTerm}*` : "*";
const rclonePath = (() => {
switch (destination.provider) {
case "drive":
return `drive:${normalizedPrefix}${searchPattern}`;
case "dropbox":
return `dropbox:${normalizedPrefix}${searchPattern}`;
case "box":
return `box:${normalizedPrefix}${searchPattern}`;
case "ftp":
case "sftp":
return `${destination.provider}:${normalizedPrefix}${searchPattern}`;
default:
throw new Error(
`Unsupported cloud storage provider: ${destination.provider}`,
);
}
})();
const command = `rclone lsjson ${credentials.join(" ")} "${rclonePath}"`;
const { stdout } = await execAsync(command);
const files = JSON.parse(stdout) as Array<{
Path: string;
Size: number;
IsDir: boolean;
Hashes?: { MD5?: string };
}>;
return files.map((file) => ({
path: file.Path,
size: file.Size,
isDir: file.IsDir,
hashes: file.Hashes,
}));
} catch (error) {
console.error("Error listing cloud storage files:", error);
throw new Error("Failed to list backup files from cloud storage");
}
};
export function getRcloneConfigPath(
organizationId: string,
destinationId: string,
): string {
const configDir = join(
homedir(),
".dokploy",
"rclone",
"cloud",
organizationId,
);
if (!existsSync(configDir)) {
mkdirSync(configDir, { recursive: true });
}
return join(configDir, `${destinationId}.conf`);
}
export const keepLatestNCloudStorageBackups = async (
backup: CloudStorageBackup,
destination: CloudStorageDestination,
keepLatestCount: number,
) => {
if (!keepLatestCount) return;
try {
const credentials = await getCloudStorageCredentials(destination);
const normalizedPrefix = normalizeCloudPath(backup.prefix || "");
const backupFilesPath = `${destination.provider}:${normalizedPrefix}`;
const rcloneList = `rclone lsf ${credentials.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".sql.gz"}" ${backupFilesPath}`;
const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${keepLatestCount}+1)) | xargs -I{}`;
const rcloneDelete = `rclone delete ${credentials.join(" ")} ${backupFilesPath}/{}`;
const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`;
await execAsync(rcloneCommand);
} catch (error) {
console.error("Error keeping latest backups:", error);
}
};