Merge pull request #719 from Dokploy/443-implement-coolifyio-like-support-of-envs-and-railwayapp-envs-shared-project-service-env

feat: add shared enviroment variables
This commit is contained in:
Mauricio Siu 2024-11-17 18:40:15 -06:00 committed by GitHub
commit b73889dd4f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 8485 additions and 38 deletions

View File

@ -32,6 +32,14 @@ const baseApp: ApplicationNested = {
serverId: "",
branch: null,
dockerBuildStage: "",
project: {
env: "",
adminId: "",
name: "",
description: "",
createdAt: "",
projectId: "",
},
buildArgs: null,
buildPath: "/",
gitlabPathNamespace: "",

179
apps/dokploy/__test__/env/shared.test.ts vendored Normal file
View File

@ -0,0 +1,179 @@
import { prepareEnvironmentVariables } from "@dokploy/server/index";
import { describe, expect, it } from "vitest";
const projectEnv = `
ENVIRONMENT=staging
DATABASE_URL=postgres://postgres:postgres@localhost:5432/project_db
PORT=3000
`;
const serviceEnv = `
ENVIRONMENT=\${{project.ENVIRONMENT}}
DATABASE_URL=\${{project.DATABASE_URL}}
SERVICE_PORT=4000
`;
describe("prepareEnvironmentVariables", () => {
it("resolves project variables correctly", () => {
const resolved = prepareEnvironmentVariables(serviceEnv, projectEnv);
expect(resolved).toEqual([
"ENVIRONMENT=staging",
"DATABASE_URL=postgres://postgres:postgres@localhost:5432/project_db",
"SERVICE_PORT=4000",
]);
});
it("handles undefined project variables", () => {
const incompleteProjectEnv = `
NODE_ENV=production
`;
const invalidServiceEnv = `
UNDEFINED_VAR=\${{project.UNDEFINED_VAR}}
`;
expect(
() =>
prepareEnvironmentVariables(invalidServiceEnv, incompleteProjectEnv), // Cambiado el orden
).toThrow("Invalid project environment variable: project.UNDEFINED_VAR");
});
it("allows service-specific variables to override project variables", () => {
const serviceSpecificEnv = `
ENVIRONMENT=production
DATABASE_URL=\${{project.DATABASE_URL}}
`;
const resolved = prepareEnvironmentVariables(
serviceSpecificEnv,
projectEnv,
);
expect(resolved).toEqual([
"ENVIRONMENT=production", // Overrides project variable
"DATABASE_URL=postgres://postgres:postgres@localhost:5432/project_db",
]);
});
it("resolves complex references for dynamic endpoints", () => {
const projectEnv = `
BASE_URL=https://api.example.com
API_VERSION=v1
PORT=8000
`;
const serviceEnv = `
API_ENDPOINT=\${{project.BASE_URL}}/\${{project.API_VERSION}}/endpoint
SERVICE_PORT=9000
`;
const resolved = prepareEnvironmentVariables(serviceEnv, projectEnv);
expect(resolved).toEqual([
"API_ENDPOINT=https://api.example.com/v1/endpoint",
"SERVICE_PORT=9000",
]);
});
it("handles missing project variables gracefully", () => {
const projectEnv = `
PORT=8080
`;
const serviceEnv = `
MISSING_VAR=\${{project.MISSING_KEY}}
SERVICE_PORT=3000
`;
expect(() => prepareEnvironmentVariables(serviceEnv, projectEnv)).toThrow(
"Invalid project environment variable: project.MISSING_KEY",
);
});
it("overrides project variables with service-specific values", () => {
const projectEnv = `
ENVIRONMENT=staging
DATABASE_URL=postgres://project:project@localhost:5432/project_db
`;
const serviceEnv = `
ENVIRONMENT=\${{project.ENVIRONMENT}}
DATABASE_URL=postgres://service:service@localhost:5432/service_db
SERVICE_NAME=my-service
`;
const resolved = prepareEnvironmentVariables(serviceEnv, projectEnv);
expect(resolved).toEqual([
"ENVIRONMENT=staging",
"DATABASE_URL=postgres://service:service@localhost:5432/service_db",
"SERVICE_NAME=my-service",
]);
});
it("handles project variables with normal and unusual characters", () => {
const projectEnv = `
ENVIRONMENT=PRODUCTION
`;
// Needs to be in quotes
const serviceEnv = `
NODE_ENV=\${{project.ENVIRONMENT}}
SPECIAL_VAR="$^@$^@#$^@!#$@#$-\${{project.ENVIRONMENT}}"
`;
const resolved = prepareEnvironmentVariables(serviceEnv, projectEnv);
expect(resolved).toEqual([
"NODE_ENV=PRODUCTION",
"SPECIAL_VAR=$^@$^@#$^@!#$@#$-PRODUCTION",
]);
});
it("handles complex cases with multiple references, special characters, and spaces", () => {
const projectEnv = `
ENVIRONMENT=STAGING
APP_NAME=MyApp
`;
const serviceEnv = `
NODE_ENV=\${{project.ENVIRONMENT}}
COMPLEX_VAR="Prefix-$#^!@-\${{project.ENVIRONMENT}}--\${{project.APP_NAME}} Suffix "
`;
const resolved = prepareEnvironmentVariables(serviceEnv, projectEnv);
expect(resolved).toEqual([
"NODE_ENV=STAGING",
"COMPLEX_VAR=Prefix-$#^!@-STAGING--MyApp Suffix ",
]);
});
it("handles references enclosed in single quotes", () => {
const projectEnv = `
ENVIRONMENT=STAGING
APP_NAME=MyApp
`;
const serviceEnv = `
NODE_ENV='\${{project.ENVIRONMENT}}'
COMPLEX_VAR='Prefix-$#^!@-\${{project.ENVIRONMENT}}--\${{project.APP_NAME}} Suffix'
`;
const resolved = prepareEnvironmentVariables(serviceEnv, projectEnv);
expect(resolved).toEqual([
"NODE_ENV=STAGING",
"COMPLEX_VAR=Prefix-$#^!@-STAGING--MyApp Suffix",
]);
});
it("handles double and single quotes combined", () => {
const projectEnv = `
ENVIRONMENT=PRODUCTION
APP_NAME=MyApp
`;
const serviceEnv = `
NODE_ENV="'\${{project.ENVIRONMENT}}'"
COMPLEX_VAR="'Prefix \"DoubleQuoted\" and \${{project.APP_NAME}}'"
`;
const resolved = prepareEnvironmentVariables(serviceEnv, projectEnv);
expect(resolved).toEqual([
"NODE_ENV='PRODUCTION'",
"COMPLEX_VAR='Prefix \"DoubleQuoted\" and MyApp'",
]);
});
});

View File

@ -13,6 +13,14 @@ const baseApp: ApplicationNested = {
branch: null,
dockerBuildStage: "",
buildArgs: null,
project: {
env: "",
adminId: "",
name: "",
description: "",
createdAt: "",
projectId: "",
},
buildPath: "/",
gitlabPathNamespace: "",
buildType: "nixpacks",

View File

@ -0,0 +1,162 @@
import { AlertBlock } from "@/components/shared/alert-block";
import { CodeEditor } from "@/components/shared/code-editor";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import { DropdownMenuItem } from "@/components/ui/dropdown-menu";
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
import { api } from "@/utils/api";
import { zodResolver } from "@hookform/resolvers/zod";
import { AlertTriangle, FileIcon, SquarePen } from "lucide-react";
import { useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import { z } from "zod";
const updateProjectSchema = z.object({
env: z.string().optional(),
});
type UpdateProject = z.infer<typeof updateProjectSchema>;
interface Props {
projectId: string;
}
export const AddEnv = ({ projectId }: Props) => {
const [isOpen, setIsOpen] = useState(false);
const utils = api.useUtils();
const { mutateAsync, error, isError, isLoading } =
api.project.update.useMutation();
const { data } = api.project.one.useQuery(
{
projectId,
},
{
enabled: !!projectId,
},
);
console.log(data);
const form = useForm<UpdateProject>({
defaultValues: {
env: data?.env ?? "",
},
resolver: zodResolver(updateProjectSchema),
});
useEffect(() => {
if (data) {
form.reset({
env: data.env ?? "",
});
}
}, [data, form, form.reset]);
const onSubmit = async (formData: UpdateProject) => {
await mutateAsync({
env: formData.env || "",
projectId: projectId,
})
.then(() => {
toast.success("Project env updated succesfully");
utils.project.all.invalidate();
})
.catch(() => {
toast.error("Error to update the env");
})
.finally(() => {});
};
return (
<Dialog open={isOpen} onOpenChange={setIsOpen}>
<DialogTrigger asChild>
<DropdownMenuItem
className="w-full cursor-pointer space-x-3"
onSelect={(e) => e.preventDefault()}
>
<FileIcon className="size-4" />
<span>Add Env</span>
</DropdownMenuItem>
</DialogTrigger>
<DialogContent className="max-h-screen overflow-y-auto sm:max-w-6xl">
<DialogHeader>
<DialogTitle>Modify Shared Env</DialogTitle>
<DialogDescription>Update the env variables</DialogDescription>
</DialogHeader>
{isError && <AlertBlock type="error">{error?.message}</AlertBlock>}
<AlertBlock type="info">
To use a shared env, in one of your services, you need to use like
this: Let's say you have a shared env ENVIROMENT="development" and you
want to use it in your service, you need to use like this:
<ul>
<li>
<code>ENVIRONMENT=${"${{shared.ENVIRONMENT}}"}</code>
</li>
<li>
<code>DATABASE_URL=${"${{shared.DATABASE_URL}}"}</code>
</li>
</ul>{" "}
This allows the service to inherit and use the shared variables from
the project level, ensuring consistency across services.
</AlertBlock>
<div className="grid gap-4">
<div className="grid items-center gap-4">
<Form {...form}>
<form
onSubmit={form.handleSubmit(onSubmit)}
className="grid w-full gap-4 "
>
<FormField
control={form.control}
name="env"
render={({ field }) => (
<FormItem>
<FormLabel>Enviroment variables</FormLabel>
<FormControl>
<CodeEditor
lineWrapping
language="properties"
wrapperClassName="h-[35rem] font-mono"
placeholder={`NODE_ENV=production
PORT=3000
`}
{...field}
/>
</FormControl>
<pre>
<FormMessage />
</pre>
</FormItem>
)}
/>
<DialogFooter>
<Button isLoading={isLoading} type="submit">
Update
</Button>
</DialogFooter>
</form>
</Form>
</div>
</div>
</DialogContent>
</Dialog>
);
};

View File

@ -35,6 +35,7 @@ import {
import Link from "next/link";
import { Fragment } from "react";
import { toast } from "sonner";
import { AddEnv } from "./add-env";
import { UpdateProject } from "./update";
export const ShowProjects = () => {
@ -190,7 +191,9 @@ export const ShowProjects = () => {
<DropdownMenuLabel className="font-normal">
Actions
</DropdownMenuLabel>
<div onClick={(e) => e.stopPropagation()}>
<AddEnv projectId={project.projectId} />
</div>
<div onClick={(e) => e.stopPropagation()}>
<UpdateProject projectId={project.projectId} />
</div>

View File

@ -0,0 +1,2 @@
ALTER TABLE "admin" ADD COLUMN "env" text DEFAULT '' NOT NULL;--> statement-breakpoint
ALTER TABLE "project" ADD COLUMN "env" text DEFAULT '' NOT NULL;

View File

@ -0,0 +1 @@
ALTER TABLE "admin" DROP COLUMN IF EXISTS "env";

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -302,6 +302,20 @@
"when": 1729984439862,
"tag": "0042_fancy_havok",
"breakpoints": true
},
{
"idx": 43,
"version": "6",
"when": 1731873965888,
"tag": "0043_closed_naoko",
"breakpoints": true
},
{
"idx": 44,
"version": "6",
"when": 1731875539532,
"tag": "0044_sour_true_believers",
"breakpoints": true
}
]
}

View File

@ -26,6 +26,7 @@ export const projects = pgTable("project", {
adminId: text("adminId")
.notNull()
.references(() => admins.adminId, { onDelete: "cascade" }),
env: text("env").notNull().default(""),
});
export const projectRelations = relations(projects, ({ many, one }) => ({
@ -65,10 +66,16 @@ export const apiRemoveProject = createSchema
})
.required();
export const apiUpdateProject = createSchema
.pick({
name: true,
description: true,
projectId: true,
})
.required();
// export const apiUpdateProject = createSchema
// .pick({
// name: true,
// description: true,
// projectId: true,
// env: true,
// })
// .required();
export const apiUpdateProject = createSchema.partial().extend({
projectId: z.string().min(1),
});
// .omit({ serverId: true });

View File

@ -180,7 +180,10 @@ const createEnvFile = (compose: ComposeNested) => {
envContent += `\nCOMPOSE_PREFIX=${compose.suffix}`;
}
const envFileContent = prepareEnvironmentVariables(envContent).join("\n");
const envFileContent = prepareEnvironmentVariables(
envContent,
compose.project.env,
).join("\n");
if (!existsSync(dirname(envFilePath))) {
mkdirSync(dirname(envFilePath), { recursive: true });
@ -206,7 +209,10 @@ export const getCreateEnvFileCommand = (compose: ComposeNested) => {
envContent += `\nCOMPOSE_PREFIX=${compose.suffix}`;
}
const envFileContent = prepareEnvironmentVariables(envContent).join("\n");
const envFileContent = prepareEnvironmentVariables(
envContent,
compose.project.env,
).join("\n");
const encodedContent = encodeBase64(envFileContent);
return `

View File

@ -20,7 +20,10 @@ export const buildCustomDocker = async (
const defaultContextPath =
dockerFilePath.substring(0, dockerFilePath.lastIndexOf("/") + 1) || ".";
const args = prepareEnvironmentVariables(buildArgs);
const args = prepareEnvironmentVariables(
buildArgs,
application.project.env,
);
const dockerContextPath = getDockerContextPath(application);
@ -38,7 +41,7 @@ export const buildCustomDocker = async (
as it could be publicly exposed.
*/
if (!publishDirectory) {
createEnvFile(dockerFilePath, env);
createEnvFile(dockerFilePath, env, application.project.env);
}
await spawnAsync(
@ -71,7 +74,10 @@ export const getDockerCommand = (
const defaultContextPath =
dockerFilePath.substring(0, dockerFilePath.lastIndexOf("/") + 1) || ".";
const args = prepareEnvironmentVariables(buildArgs);
const args = prepareEnvironmentVariables(
buildArgs,
application.project.env,
);
const dockerContextPath =
getDockerContextPath(application) || defaultContextPath;
@ -92,7 +98,11 @@ export const getDockerCommand = (
*/
let command = "";
if (!publishDirectory) {
command += createEnvFileCommand(dockerFilePath, env);
command += createEnvFileCommand(
dockerFilePath,
env,
application.project.env,
);
}
command += `

View File

@ -11,7 +11,10 @@ export const buildHeroku = async (
) => {
const { env, appName } = application;
const buildAppDirectory = getBuildAppDirectory(application);
const envVariables = prepareEnvironmentVariables(env);
const envVariables = prepareEnvironmentVariables(
env,
application.project.env,
);
try {
const args = [
"build",
@ -44,7 +47,10 @@ export const getHerokuCommand = (
const { env, appName } = application;
const buildAppDirectory = getBuildAppDirectory(application);
const envVariables = prepareEnvironmentVariables(env);
const envVariables = prepareEnvironmentVariables(
env,
application.project.env,
);
const args = [
"build",

View File

@ -24,7 +24,14 @@ import { buildStatic, getStaticCommand } from "./static";
// DOCKERFILE codeDirectory = where is the exact path of the (Dockerfile)
export type ApplicationNested = InferResultType<
"applications",
{ mounts: true; security: true; redirects: true; ports: true; registry: true }
{
mounts: true;
security: true;
redirects: true;
ports: true;
registry: true;
project: true;
}
>;
export const buildApplication = async (
application: ApplicationNested,
@ -133,7 +140,10 @@ export const mechanizeDockerContainer = async (
const bindsMount = generateBindMounts(mounts);
const filesMount = generateFileMounts(appName, application);
const envVariables = prepareEnvironmentVariables(env);
const envVariables = prepareEnvironmentVariables(
env,
application.project.env,
);
const image = getImageName(application);
const authConfig = getAuthConfig(application);

View File

@ -18,7 +18,10 @@ export const buildNixpacks = async (
const buildAppDirectory = getBuildAppDirectory(application);
const buildContainerId = `${appName}-${nanoid(10)}`;
const envVariables = prepareEnvironmentVariables(env);
const envVariables = prepareEnvironmentVariables(
env,
application.project.env,
);
const writeToStream = (data: string) => {
if (writeStream.writable) {
@ -92,7 +95,10 @@ export const getNixpacksCommand = (
const buildAppDirectory = getBuildAppDirectory(application);
const buildContainerId = `${appName}-${nanoid(10)}`;
const envVariables = prepareEnvironmentVariables(env);
const envVariables = prepareEnvironmentVariables(
env,
application.project.env,
);
const args = ["build", buildAppDirectory, "--name", appName];

View File

@ -10,7 +10,10 @@ export const buildPaketo = async (
) => {
const { env, appName } = application;
const buildAppDirectory = getBuildAppDirectory(application);
const envVariables = prepareEnvironmentVariables(env);
const envVariables = prepareEnvironmentVariables(
env,
application.project.env,
);
try {
const args = [
"build",
@ -43,7 +46,10 @@ export const getPaketoCommand = (
const { env, appName } = application;
const buildAppDirectory = getBuildAppDirectory(application);
const envVariables = prepareEnvironmentVariables(env);
const envVariables = prepareEnvironmentVariables(
env,
application.project.env,
);
const args = [
"build",

View File

@ -2,17 +2,29 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { encodeBase64, prepareEnvironmentVariables } from "../docker/utils";
export const createEnvFile = (directory: string, env: string | null) => {
export const createEnvFile = (
directory: string,
env: string | null,
projectEnv?: string | null,
) => {
const envFilePath = join(dirname(directory), ".env");
if (!existsSync(dirname(envFilePath))) {
mkdirSync(dirname(envFilePath), { recursive: true });
}
const envFileContent = prepareEnvironmentVariables(env).join("\n");
const envFileContent = prepareEnvironmentVariables(env, projectEnv).join(
"\n",
);
writeFileSync(envFilePath, envFileContent);
};
export const createEnvFileCommand = (directory: string, env: string | null) => {
const envFileContent = prepareEnvironmentVariables(env).join("\n");
export const createEnvFileCommand = (
directory: string,
env: string | null,
projectEnv?: string | null,
) => {
const envFileContent = prepareEnvironmentVariables(env, projectEnv).join(
"\n",
);
const encodedContent = encodeBase64(envFileContent || "");
const envFilePath = join(dirname(directory), ".env");

View File

@ -9,7 +9,10 @@ import {
} from "../docker/utils";
import { getRemoteDocker } from "../servers/remote-docker";
export type MariadbNested = InferResultType<"mariadb", { mounts: true }>;
export type MariadbNested = InferResultType<
"mariadb",
{ mounts: true; project: true }
>;
export const buildMariadb = async (mariadb: MariadbNested) => {
const {
appName,
@ -37,7 +40,10 @@ export const buildMariadb = async (mariadb: MariadbNested) => {
cpuLimit,
cpuReservation,
});
const envVariables = prepareEnvironmentVariables(defaultMariadbEnv);
const envVariables = prepareEnvironmentVariables(
defaultMariadbEnv,
mariadb.project.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);
const filesMount = generateFileMounts(appName, mariadb);

View File

@ -9,7 +9,10 @@ import {
} from "../docker/utils";
import { getRemoteDocker } from "../servers/remote-docker";
export type MongoNested = InferResultType<"mongo", { mounts: true }>;
export type MongoNested = InferResultType<
"mongo",
{ mounts: true; project: true }
>;
export const buildMongo = async (mongo: MongoNested) => {
const {
@ -36,7 +39,10 @@ export const buildMongo = async (mongo: MongoNested) => {
cpuLimit,
cpuReservation,
});
const envVariables = prepareEnvironmentVariables(defaultMongoEnv);
const envVariables = prepareEnvironmentVariables(
defaultMongoEnv,
mongo.project.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);
const filesMount = generateFileMounts(appName, mongo);

View File

@ -9,7 +9,10 @@ import {
} from "../docker/utils";
import { getRemoteDocker } from "../servers/remote-docker";
export type MysqlNested = InferResultType<"mysql", { mounts: true }>;
export type MysqlNested = InferResultType<
"mysql",
{ mounts: true; project: true }
>;
export const buildMysql = async (mysql: MysqlNested) => {
const {
@ -43,7 +46,10 @@ export const buildMysql = async (mysql: MysqlNested) => {
cpuLimit,
cpuReservation,
});
const envVariables = prepareEnvironmentVariables(defaultMysqlEnv);
const envVariables = prepareEnvironmentVariables(
defaultMysqlEnv,
mysql.project.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);
const filesMount = generateFileMounts(appName, mysql);

View File

@ -9,7 +9,10 @@ import {
} from "../docker/utils";
import { getRemoteDocker } from "../servers/remote-docker";
export type PostgresNested = InferResultType<"postgres", { mounts: true }>;
export type PostgresNested = InferResultType<
"postgres",
{ mounts: true; project: true }
>;
export const buildPostgres = async (postgres: PostgresNested) => {
const {
appName,
@ -36,7 +39,10 @@ export const buildPostgres = async (postgres: PostgresNested) => {
cpuLimit,
cpuReservation,
});
const envVariables = prepareEnvironmentVariables(defaultPostgresEnv);
const envVariables = prepareEnvironmentVariables(
defaultPostgresEnv,
postgres.project.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);
const filesMount = generateFileMounts(appName, postgres);

View File

@ -9,7 +9,10 @@ import {
} from "../docker/utils";
import { getRemoteDocker } from "../servers/remote-docker";
export type RedisNested = InferResultType<"redis", { mounts: true }>;
export type RedisNested = InferResultType<
"redis",
{ mounts: true; project: true }
>;
export const buildRedis = async (redis: RedisNested) => {
const {
appName,
@ -34,7 +37,10 @@ export const buildRedis = async (redis: RedisNested) => {
cpuLimit,
cpuReservation,
});
const envVariables = prepareEnvironmentVariables(defaultRedisEnv);
const envVariables = prepareEnvironmentVariables(
defaultRedisEnv,
redis.project.env,
);
const volumesMount = generateVolumeMounts(mounts);
const bindsMount = generateBindMounts(mounts);
const filesMount = generateFileMounts(appName, redis);

View File

@ -258,8 +258,28 @@ export const removeService = async (
}
};
export const prepareEnvironmentVariables = (env: string | null) =>
Object.entries(parse(env ?? "")).map(([key, value]) => `${key}=${value}`);
export const prepareEnvironmentVariables = (
serviceEnv: string | null,
projectEnv?: string | null,
) => {
const projectVars = parse(projectEnv ?? "");
const serviceVars = parse(serviceEnv ?? "");
const resolvedVars = Object.entries(serviceVars).map(([key, value]) => {
let resolvedValue = value;
if (projectVars) {
resolvedValue = value.replace(/\$\{\{project\.(.*?)\}\}/g, (_, ref) => {
if (projectVars[ref] !== undefined) {
return projectVars[ref];
}
throw new Error(`Invalid project environment variable: project.${ref}`);
});
}
return `${key}=${resolvedValue}`;
});
return resolvedVars;
};
export const prepareBuildArgs = (input: string | null) => {
const pairs = (input ?? "").split("\n");