feat: add optional Provider attribute to S3 Destinations

This commit is contained in:
chuyun
2024-11-27 02:14:45 +08:00
parent fabe946526
commit 0477329db7
11 changed files with 4224 additions and 14 deletions

View File

@@ -34,9 +34,11 @@ import { useEffect } from "react";
import { useForm } from "react-hook-form"; import { useForm } from "react-hook-form";
import { toast } from "sonner"; import { toast } from "sonner";
import { z } from "zod"; import { z } from "zod";
import { S3_PROVIDERS } from "./constants";
const addDestination = z.object({ const addDestination = z.object({
name: z.string().min(1, "Name is required"), name: z.string().min(1, "Name is required"),
provider: z.string().optional(),
accessKeyId: z.string(), accessKeyId: z.string(),
secretAccessKey: z.string(), secretAccessKey: z.string(),
bucket: z.string(), bucket: z.string(),
@@ -58,6 +60,7 @@ export const AddDestination = () => {
api.destination.testConnection.useMutation(); api.destination.testConnection.useMutation();
const form = useForm<AddDestination>({ const form = useForm<AddDestination>({
defaultValues: { defaultValues: {
provider: "",
accessKeyId: "", accessKeyId: "",
bucket: "", bucket: "",
name: "", name: "",
@@ -73,6 +76,7 @@ export const AddDestination = () => {
const onSubmit = async (data: AddDestination) => { const onSubmit = async (data: AddDestination) => {
await mutateAsync({ await mutateAsync({
provider: data.provider,
accessKey: data.accessKeyId, accessKey: data.accessKeyId,
bucket: data.bucket, bucket: data.bucket,
endpoint: data.endpoint, endpoint: data.endpoint,
@@ -123,6 +127,40 @@ export const AddDestination = () => {
); );
}} }}
/> />
<FormField
control={form.control}
name="provider"
render={({ field }) => {
return (
<FormItem>
<FormLabel>Provider</FormLabel>
<FormControl>
<Select
onValueChange={field.onChange}
defaultValue={field.value}
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select a S3 Provider" />
</SelectTrigger>
</FormControl>
<SelectContent>
{S3_PROVIDERS.map((s3Provider) => (
<SelectItem
key={s3Provider.key}
value={s3Provider.key}
>
{s3Provider.name}
</SelectItem>
))}
</SelectContent>
</Select>
</FormControl>
<FormMessage />
</FormItem>
);
}}
/>
<FormField <FormField
control={form.control} control={form.control}
@@ -255,6 +293,7 @@ export const AddDestination = () => {
isLoading={isLoading} isLoading={isLoading}
onClick={async () => { onClick={async () => {
await testConnection({ await testConnection({
provider: form.getValues("provider"),
accessKey: form.getValues("accessKeyId"), accessKey: form.getValues("accessKeyId"),
bucket: form.getValues("bucket"), bucket: form.getValues("bucket"),
endpoint: form.getValues("endpoint"), endpoint: form.getValues("endpoint"),
@@ -283,6 +322,7 @@ export const AddDestination = () => {
variant="secondary" variant="secondary"
onClick={async () => { onClick={async () => {
await testConnection({ await testConnection({
provider: form.getValues("provider"),
accessKey: form.getValues("accessKeyId"), accessKey: form.getValues("accessKeyId"),
bucket: form.getValues("bucket"), bucket: form.getValues("bucket"),
endpoint: form.getValues("endpoint"), endpoint: form.getValues("endpoint"),

View File

@@ -0,0 +1,133 @@
export const S3_PROVIDERS: Array<{
key: string;
name: string;
}> = [
{
key: "AWS",
name: "Amazon Web Services (AWS) S3",
},
{
key: "Alibaba",
name: "Alibaba Cloud Object Storage System (OSS) formerly Aliyun",
},
{
key: "ArvanCloud",
name: "Arvan Cloud Object Storage (AOS)",
},
{
key: "Ceph",
name: "Ceph Object Storage",
},
{
key: "ChinaMobile",
name: "China Mobile Ecloud Elastic Object Storage (EOS)",
},
{
key: "Cloudflare",
name: "Cloudflare R2 Storage",
},
{
key: "DigitalOcean",
name: "DigitalOcean Spaces",
},
{
key: "Dreamhost",
name: "Dreamhost DreamObjects",
},
{
key: "GCS",
name: "Google Cloud Storage",
},
{
key: "HuaweiOBS",
name: "Huawei Object Storage Service",
},
{
key: "IBMCOS",
name: "IBM COS S3",
},
{
key: "IDrive",
name: "IDrive e2",
},
{
key: "IONOS",
name: "IONOS Cloud",
},
{
key: "LyveCloud",
name: "Seagate Lyve Cloud",
},
{
key: "Leviia",
name: "Leviia Object Storage",
},
{
key: "Liara",
name: "Liara Object Storage",
},
{
key: "Linode",
name: "Linode Object Storage",
},
{
key: "Magalu",
name: "Magalu Object Storage",
},
{
key: "Minio",
name: "Minio Object Storage",
},
{
key: "Netease",
name: "Netease Object Storage (NOS)",
},
{
key: "Petabox",
name: "Petabox Object Storage",
},
{
key: "RackCorp",
name: "RackCorp Object Storage",
},
{
key: "Rclone",
name: "Rclone S3 Server",
},
{
key: "Scaleway",
name: "Scaleway Object Storage",
},
{
key: "SeaweedFS",
name: "SeaweedFS S3",
},
{
key: "StackPath",
name: "StackPath Object Storage",
},
{
key: "Storj",
name: "Storj (S3 Compatible Gateway)",
},
{
key: "Synology",
name: "Synology C2 Object Storage",
},
{
key: "TencentCOS",
name: "Tencent Cloud Object Storage (COS)",
},
{
key: "Wasabi",
name: "Wasabi Object Storage",
},
{
key: "Qiniu",
name: "Qiniu Object Storage (Kodo)",
},
{
key: "Other",
name: "Any other S3 compatible provider",
},
];

View File

@@ -35,9 +35,11 @@ import { useEffect, useState } from "react";
import { useForm } from "react-hook-form"; import { useForm } from "react-hook-form";
import { toast } from "sonner"; import { toast } from "sonner";
import { z } from "zod"; import { z } from "zod";
import { S3_PROVIDERS } from "./constants";
const updateDestination = z.object({ const updateDestination = z.object({
name: z.string().min(1, "Name is required"), name: z.string().min(1, "Name is required"),
provider: z.string().optional(),
accessKeyId: z.string(), accessKeyId: z.string(),
secretAccessKey: z.string(), secretAccessKey: z.string(),
bucket: z.string(), bucket: z.string(),
@@ -70,6 +72,7 @@ export const UpdateDestination = ({ destinationId }: Props) => {
api.destination.testConnection.useMutation(); api.destination.testConnection.useMutation();
const form = useForm<UpdateDestination>({ const form = useForm<UpdateDestination>({
defaultValues: { defaultValues: {
provider: "",
accessKeyId: "", accessKeyId: "",
bucket: "", bucket: "",
name: "", name: "",
@@ -152,6 +155,40 @@ export const UpdateDestination = ({ destinationId }: Props) => {
); );
}} }}
/> />
<FormField
control={form.control}
name="provider"
render={({ field }) => {
return (
<FormItem>
<FormLabel>Provider</FormLabel>
<FormControl>
<Select
onValueChange={field.onChange}
defaultValue={field.value}
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select a S3 Provider" />
</SelectTrigger>
</FormControl>
<SelectContent>
{S3_PROVIDERS.map((s3Provider) => (
<SelectItem
key={s3Provider.key}
value={s3Provider.key}
>
{s3Provider.name}
</SelectItem>
))}
</SelectContent>
</Select>
</FormControl>
<FormMessage />
</FormItem>
);
}}
/>
<FormField <FormField
control={form.control} control={form.control}
@@ -285,6 +322,7 @@ export const UpdateDestination = ({ destinationId }: Props) => {
variant={"secondary"} variant={"secondary"}
onClick={async () => { onClick={async () => {
await testConnection({ await testConnection({
provider: form.getValues("provider"),
accessKey: form.getValues("accessKeyId"), accessKey: form.getValues("accessKeyId"),
bucket: form.getValues("bucket"), bucket: form.getValues("bucket"),
endpoint: form.getValues("endpoint"), endpoint: form.getValues("endpoint"),
@@ -311,6 +349,7 @@ export const UpdateDestination = ({ destinationId }: Props) => {
variant="secondary" variant="secondary"
onClick={async () => { onClick={async () => {
await testConnection({ await testConnection({
provider: form.getValues("provider"),
accessKey: form.getValues("accessKeyId"), accessKey: form.getValues("accessKeyId"),
bucket: form.getValues("bucket"), bucket: form.getValues("bucket"),
endpoint: form.getValues("endpoint"), endpoint: form.getValues("endpoint"),

View File

@@ -0,0 +1 @@
ALTER TABLE "destination" ADD COLUMN "provider" text;

File diff suppressed because it is too large Load Diff

View File

@@ -316,6 +316,13 @@
"when": 1731875539532, "when": 1731875539532,
"tag": "0044_sour_true_believers", "tag": "0044_sour_true_believers",
"breakpoints": true "breakpoints": true
},
{
"idx": 45,
"version": "6",
"when": 1732644181718,
"tag": "0045_smiling_blur",
"breakpoints": true
} }
] ]
} }

View File

@@ -40,11 +40,10 @@ export const destinationRouter = createTRPCRouter({
testConnection: adminProcedure testConnection: adminProcedure
.input(apiCreateDestination) .input(apiCreateDestination)
.mutation(async ({ input }) => { .mutation(async ({ input }) => {
const { secretAccessKey, bucket, region, endpoint, accessKey } = input; const { secretAccessKey, bucket, region, endpoint, accessKey, provider } =
input;
try { try {
const rcloneFlags = [ const rcloneFlags = [
// `--s3-provider=Cloudflare`,
`--s3-access-key-id=${accessKey}`, `--s3-access-key-id=${accessKey}`,
`--s3-secret-access-key=${secretAccessKey}`, `--s3-secret-access-key=${secretAccessKey}`,
`--s3-region=${region}`, `--s3-region=${region}`,
@@ -52,6 +51,9 @@ export const destinationRouter = createTRPCRouter({
"--s3-no-check-bucket", "--s3-no-check-bucket",
"--s3-force-path-style", "--s3-force-path-style",
]; ];
if (provider) {
rcloneFlags.unshift(`--s3-provider=${provider}`);
}
const rcloneDestination = `:s3:${bucket}`; const rcloneDestination = `:s3:${bucket}`;
const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} "${rcloneDestination}"`; const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} "${rcloneDestination}"`;

View File

@@ -881,8 +881,8 @@ export const templates: TemplateData[] = [
}, },
tags: ["forum", "community", "discussion"], tags: ["forum", "community", "discussion"],
load: () => import("./discourse/index").then((m) => m.generate), load: () => import("./discourse/index").then((m) => m.generate),
}, },
{ {
id: "immich", id: "immich",
name: "Immich", name: "Immich",
version: "v1.121.0", version: "v1.121.0",
@@ -896,8 +896,8 @@ export const templates: TemplateData[] = [
}, },
tags: ["photos", "videos", "backup", "media"], tags: ["photos", "videos", "backup", "media"],
load: () => import("./immich/index").then((m) => m.generate), load: () => import("./immich/index").then((m) => m.generate),
}, },
{ {
id: "twenty", id: "twenty",
name: "Twenty CRM", name: "Twenty CRM",
version: "latest", version: "latest",
@@ -911,8 +911,8 @@ export const templates: TemplateData[] = [
}, },
tags: ["crm", "sales", "business"], tags: ["crm", "sales", "business"],
load: () => import("./twenty/index").then((m) => m.generate), load: () => import("./twenty/index").then((m) => m.generate),
}, },
{ {
id: "yourls", id: "yourls",
name: "YOURLS", name: "YOURLS",
version: "1.9.2", version: "1.9.2",
@@ -926,8 +926,8 @@ export const templates: TemplateData[] = [
}, },
tags: ["url-shortener", "php"], tags: ["url-shortener", "php"],
load: () => import("./yourls/index").then((m) => m.generate), load: () => import("./yourls/index").then((m) => m.generate),
}, },
{ {
id: "ryot", id: "ryot",
name: "Ryot", name: "Ryot",
version: "v7.10", version: "v7.10",

View File

@@ -12,6 +12,7 @@ export const destinations = pgTable("destination", {
.primaryKey() .primaryKey()
.$defaultFn(() => nanoid()), .$defaultFn(() => nanoid()),
name: text("name").notNull(), name: text("name").notNull(),
provider: text("provider"),
accessKey: text("accessKey").notNull(), accessKey: text("accessKey").notNull(),
secretAccessKey: text("secretAccessKey").notNull(), secretAccessKey: text("secretAccessKey").notNull(),
bucket: text("bucket").notNull(), bucket: text("bucket").notNull(),
@@ -37,6 +38,7 @@ export const destinationsRelations = relations(
const createSchema = createInsertSchema(destinations, { const createSchema = createInsertSchema(destinations, {
destinationId: z.string(), destinationId: z.string(),
name: z.string().min(1), name: z.string().min(1),
provider: z.string(),
accessKey: z.string(), accessKey: z.string(),
bucket: z.string(), bucket: z.string(),
endpoint: z.string(), endpoint: z.string(),
@@ -47,6 +49,7 @@ const createSchema = createInsertSchema(destinations, {
export const apiCreateDestination = createSchema export const apiCreateDestination = createSchema
.pick({ .pick({
name: true, name: true,
provider: true,
accessKey: true, accessKey: true,
bucket: true, bucket: true,
region: true, region: true,

View File

@@ -28,9 +28,9 @@ export const removeScheduleBackup = (backupId: string) => {
}; };
export const getS3Credentials = (destination: Destination) => { export const getS3Credentials = (destination: Destination) => {
const { accessKey, secretAccessKey, bucket, region, endpoint } = destination; const { accessKey, secretAccessKey, bucket, region, endpoint, provider } =
destination;
const rcloneFlags = [ const rcloneFlags = [
// `--s3-provider=Cloudflare`,
`--s3-access-key-id=${accessKey}`, `--s3-access-key-id=${accessKey}`,
`--s3-secret-access-key=${secretAccessKey}`, `--s3-secret-access-key=${secretAccessKey}`,
`--s3-region=${region}`, `--s3-region=${region}`,
@@ -39,5 +39,9 @@ export const getS3Credentials = (destination: Destination) => {
"--s3-force-path-style", "--s3-force-path-style",
]; ];
if (provider) {
rcloneFlags.unshift(`--s3-provider=${provider}`);
}
return rcloneFlags; return rcloneFlags;
}; };