From 88f7cf25465e8fbbe8b841f965ae7d4a11143a96 Mon Sep 17 00:00:00 2001 From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com> Date: Thu, 28 Nov 2024 21:54:20 -0600 Subject: [PATCH] fix: add registry url and use spawnAsync --- .../general/generic/save-docker-provider.tsx | 19 +- apps/dokploy/drizzle/0046_purple_sleeper.sql | 1 + apps/dokploy/drizzle/meta/0046_snapshot.json | 3987 +++++++++++++++++ apps/dokploy/drizzle/meta/_journal.json | 7 + .../dokploy/server/api/routers/application.ts | 1 + packages/server/src/db/schema/application.ts | 3 + packages/server/src/services/application.ts | 8 - packages/server/src/utils/docker/utils.ts | 41 +- packages/server/src/utils/providers/docker.ts | 7 +- packages/server/src/utils/providers/github.ts | 480 +- 10 files changed, 4278 insertions(+), 276 deletions(-) create mode 100644 apps/dokploy/drizzle/0046_purple_sleeper.sql create mode 100644 apps/dokploy/drizzle/meta/0046_snapshot.json diff --git a/apps/dokploy/components/dashboard/application/general/generic/save-docker-provider.tsx b/apps/dokploy/components/dashboard/application/general/generic/save-docker-provider.tsx index 81a35049..07bf5842 100644 --- a/apps/dokploy/components/dashboard/application/general/generic/save-docker-provider.tsx +++ b/apps/dokploy/components/dashboard/application/general/generic/save-docker-provider.tsx @@ -21,6 +21,7 @@ const DockerProviderSchema = z.object({ }), username: z.string().optional(), password: z.string().optional(), + registryURL: z.string().optional(), }); type DockerProvider = z.infer; @@ -39,6 +40,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => { dockerImage: "", password: "", username: "", + registryURL: "", }, resolver: zodResolver(DockerProviderSchema), }); @@ -49,6 +51,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => { dockerImage: data.dockerImage || "", password: data.password || "", username: data.username || "", + registryURL: data.registryUrl || "", }); } }, [form.reset, data, form]); @@ -59,6 +62,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => { password: values.password || null, applicationId, username: values.username || null, + registryUrl: values.registryURL || null, }) .then(async () => { toast.success("Docker Provider Saved"); @@ -76,7 +80,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => { className="flex flex-col gap-4" >
-
+
{ )} />
+ ( + + Registry URL + + + + + + )} + />
{ - docker.pull(dockerImage, { authconfig: authConfig }, (err, stream) => { - if (err) { - reject(err); - return; - } - - docker.modem.followProgress( - stream as Readable, - (err: Error | null, res) => { - if (!err) { - resolve(res); - } - if (err) { - reject(err); - } - }, - (event) => { - onData?.(event); - }, - ); - }); - }); + if (authConfig?.username && authConfig?.password) { + await spawnAsync( + "docker", + [ + "login", + authConfig.registryUrl || "", + "-u", + authConfig.username, + "-p", + authConfig.password, + ], + onData, + ); + } + await spawnAsync("docker", ["pull", dockerImage], onData); } catch (error) { throw error; } diff --git a/packages/server/src/utils/providers/docker.ts b/packages/server/src/utils/providers/docker.ts index 77c4db57..7245dc51 100644 --- a/packages/server/src/utils/providers/docker.ts +++ b/packages/server/src/utils/providers/docker.ts @@ -5,7 +5,7 @@ import { pullImage } from "../docker/utils"; interface RegistryAuth { username: string; password: string; - serveraddress: string; + registryUrl: string; } export const buildDocker = async ( @@ -16,6 +16,7 @@ export const buildDocker = async ( const authConfig: Partial = { username: username || "", password: password || "", + registryUrl: application.registryUrl || "", }; const writeStream = createWriteStream(logPath, { flags: "a" }); @@ -33,7 +34,7 @@ export const buildDocker = async ( dockerImage, (data) => { if (writeStream.writable) { - writeStream.write(`${data.status}\n`); + writeStream.write(`${data}\n`); } }, authConfig, @@ -41,7 +42,7 @@ export const buildDocker = async ( await mechanizeDockerContainer(application); writeStream.write("\nDocker Deployed: ✅\n"); } catch (error) { - writeStream.write(`ERROR: ${error}: ❌`); + writeStream.write("❌ Error"); throw error; } finally { writeStream.end(); diff --git a/packages/server/src/utils/providers/github.ts b/packages/server/src/utils/providers/github.ts index 61d5d30c..0a814623 100644 --- a/packages/server/src/utils/providers/github.ts +++ b/packages/server/src/utils/providers/github.ts @@ -14,325 +14,325 @@ import { type Github, findGithubById } from "@dokploy/server/services/github"; import { execAsyncRemote } from "../process/execAsync"; export const authGithub = (githubProvider: Github): Octokit => { - if (!haveGithubRequirements(githubProvider)) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Github Account not configured correctly", - }); - } + if (!haveGithubRequirements(githubProvider)) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Github Account not configured correctly", + }); + } - const octokit: Octokit = new Octokit({ - authStrategy: createAppAuth, - auth: { - appId: githubProvider?.githubAppId || 0, - privateKey: githubProvider?.githubPrivateKey || "", - installationId: githubProvider?.githubInstallationId, - }, - }); + const octokit: Octokit = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: githubProvider?.githubAppId || 0, + privateKey: githubProvider?.githubPrivateKey || "", + installationId: githubProvider?.githubInstallationId, + }, + }); - return octokit; + return octokit; }; export const getGithubToken = async ( - octokit: ReturnType, + octokit: ReturnType ) => { - const installation = (await octokit.auth({ - type: "installation", - })) as { - token: string; - }; + const installation = (await octokit.auth({ + type: "installation", + })) as { + token: string; + }; - return installation.token; + return installation.token; }; export const haveGithubRequirements = (githubProvider: Github) => { - return !!( - githubProvider?.githubAppId && - githubProvider?.githubPrivateKey && - githubProvider?.githubInstallationId - ); + return !!( + githubProvider?.githubAppId && + githubProvider?.githubPrivateKey && + githubProvider?.githubInstallationId + ); }; const getErrorCloneRequirements = (entity: { - repository?: string | null; - owner?: string | null; - branch?: string | null; + repository?: string | null; + owner?: string | null; + branch?: string | null; }) => { - const reasons: string[] = []; - const { repository, owner, branch } = entity; + const reasons: string[] = []; + const { repository, owner, branch } = entity; - if (!repository) reasons.push("1. Repository not assigned."); - if (!owner) reasons.push("2. Owner not specified."); - if (!branch) reasons.push("3. Branch not defined."); + if (!repository) reasons.push("1. Repository not assigned."); + if (!owner) reasons.push("2. Owner not specified."); + if (!branch) reasons.push("3. Branch not defined."); - return reasons; + return reasons; }; export type ApplicationWithGithub = InferResultType< - "applications", - { github: true } + "applications", + { github: true } >; export type ComposeWithGithub = InferResultType<"compose", { github: true }>; export const cloneGithubRepository = async ( - entity: ApplicationWithGithub | ComposeWithGithub, - logPath: string, - isCompose = false, + entity: ApplicationWithGithub | ComposeWithGithub, + logPath: string, + isCompose = false ) => { - const { APPLICATIONS_PATH, COMPOSE_PATH } = paths(); - const writeStream = createWriteStream(logPath, { flags: "a" }); - const { appName, repository, owner, branch, githubId } = entity; + const { APPLICATIONS_PATH, COMPOSE_PATH } = paths(); + const writeStream = createWriteStream(logPath, { flags: "a" }); + const { appName, repository, owner, branch, githubId } = entity; - if (!githubId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "GitHub Provider not found", - }); - } + if (!githubId) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "GitHub Provider not found", + }); + } - const requirements = getErrorCloneRequirements(entity); + const requirements = getErrorCloneRequirements(entity); - // Check if requirements are met - if (requirements.length > 0) { - writeStream.write( - `\nGitHub Repository configuration failed for application: ${appName}\n`, - ); - writeStream.write("Reasons:\n"); - writeStream.write(requirements.join("\n")); - writeStream.end(); - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Error: GitHub repository information is incomplete.", - }); - } + // Check if requirements are met + if (requirements.length > 0) { + writeStream.write( + `\nGitHub Repository configuration failed for application: ${appName}\n` + ); + writeStream.write("Reasons:\n"); + writeStream.write(requirements.join("\n")); + writeStream.end(); + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Error: GitHub repository information is incomplete.", + }); + } - const githubProvider = await findGithubById(githubId); - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; - const outputPath = join(basePath, appName, "code"); - const octokit = authGithub(githubProvider); - const token = await getGithubToken(octokit); - const repoclone = `github.com/${owner}/${repository}.git`; - await recreateDirectory(outputPath); - const cloneUrl = `https://oauth2:${token}@${repoclone}`; + const githubProvider = await findGithubById(githubId); + const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; + const outputPath = join(basePath, appName, "code"); + const octokit = authGithub(githubProvider); + const token = await getGithubToken(octokit); + const repoclone = `github.com/${owner}/${repository}.git`; + await recreateDirectory(outputPath); + const cloneUrl = `https://oauth2:${token}@${repoclone}`; - try { - writeStream.write(`\nClonning Repo ${repoclone} to ${outputPath}: ✅\n`); - await spawnAsync( - "git", - [ - "clone", - "--branch", - branch!, - "--depth", - "1", - "--recurse-submodules", - cloneUrl, - outputPath, - "--progress", - ], - (data) => { - if (writeStream.writable) { - writeStream.write(data); - } - }, - ); - writeStream.write(`\nCloned ${repoclone}: ✅\n`); - } catch (error) { - writeStream.write(`ERROR Clonning: ${error}: ❌`); - throw error; - } finally { - writeStream.end(); - } + try { + writeStream.write(`\nClonning Repo ${repoclone} to ${outputPath}: ✅\n`); + await spawnAsync( + "git", + [ + "clone", + "--branch", + branch!, + "--depth", + "1", + "--recurse-submodules", + cloneUrl, + outputPath, + "--progress", + ], + (data) => { + if (writeStream.writable) { + writeStream.write(data); + } + } + ); + writeStream.write(`\nCloned ${repoclone}: ✅\n`); + } catch (error) { + writeStream.write(`ERROR Clonning: ${error}: ❌`); + throw error; + } finally { + writeStream.end(); + } }; export const getGithubCloneCommand = async ( - entity: ApplicationWithGithub | ComposeWithGithub, - logPath: string, - isCompose = false, + entity: ApplicationWithGithub | ComposeWithGithub, + logPath: string, + isCompose = false ) => { - const { appName, repository, owner, branch, githubId, serverId } = entity; + const { appName, repository, owner, branch, githubId, serverId } = entity; - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } + if (!serverId) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Server not found", + }); + } - if (!githubId) { - const command = ` + if (!githubId) { + const command = ` echo "Error: ❌ Github Provider not found" >> ${logPath}; exit 1; `; - await execAsyncRemote(serverId, command); - throw new TRPCError({ - code: "NOT_FOUND", - message: "GitHub Provider not found", - }); - } + await execAsyncRemote(serverId, command); + throw new TRPCError({ + code: "NOT_FOUND", + message: "GitHub Provider not found", + }); + } - const requirements = getErrorCloneRequirements(entity); + const requirements = getErrorCloneRequirements(entity); - // Build log messages - let logMessages = ""; - if (requirements.length > 0) { - logMessages += `\nGitHub Repository configuration failed for application: ${appName}\n`; - logMessages += "Reasons:\n"; - logMessages += requirements.join("\n"); - const escapedLogMessages = logMessages - .replace(/\\/g, "\\\\") - .replace(/"/g, '\\"') - .replace(/\n/g, "\\n"); + // Build log messages + let logMessages = ""; + if (requirements.length > 0) { + logMessages += `\nGitHub Repository configuration failed for application: ${appName}\n`; + logMessages += "Reasons:\n"; + logMessages += requirements.join("\n"); + const escapedLogMessages = logMessages + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\n/g, "\\n"); - const bashCommand = ` + const bashCommand = ` echo "${escapedLogMessages}" >> ${logPath}; exit 1; # Exit with error code `; - await execAsyncRemote(serverId, bashCommand); - return; - } - const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true); - const githubProvider = await findGithubById(githubId); - const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; - const outputPath = join(basePath, appName, "code"); - const octokit = authGithub(githubProvider); - const token = await getGithubToken(octokit); - const repoclone = `github.com/${owner}/${repository}.git`; - const cloneUrl = `https://oauth2:${token}@${repoclone}`; + await execAsyncRemote(serverId, bashCommand); + return; + } + const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true); + const githubProvider = await findGithubById(githubId); + const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; + const outputPath = join(basePath, appName, "code"); + const octokit = authGithub(githubProvider); + const token = await getGithubToken(octokit); + const repoclone = `github.com/${owner}/${repository}.git`; + const cloneUrl = `https://oauth2:${token}@${repoclone}`; - const cloneCommand = ` + const cloneCommand = ` rm -rf ${outputPath}; mkdir -p ${outputPath}; if ! git clone --branch ${branch} --depth 1 --recurse-submodules --progress ${cloneUrl} ${outputPath} >> ${logPath} 2>&1; then - echo "❌ [ERROR] Fallo al clonar el repositorio ${repoclone}" >> ${logPath}; + echo "❌ [ERROR] Fail to clone repository ${repoclone}" >> ${logPath}; exit 1; fi echo "Cloned ${repoclone} to ${outputPath}: ✅" >> ${logPath}; `; - return cloneCommand; + return cloneCommand; }; export const cloneRawGithubRepository = async (entity: Compose) => { - const { appName, repository, owner, branch, githubId } = entity; + const { appName, repository, owner, branch, githubId } = entity; - if (!githubId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "GitHub Provider not found", - }); - } - const { COMPOSE_PATH } = paths(); - const githubProvider = await findGithubById(githubId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const octokit = authGithub(githubProvider); - const token = await getGithubToken(octokit); - const repoclone = `github.com/${owner}/${repository}.git`; - await recreateDirectory(outputPath); - const cloneUrl = `https://oauth2:${token}@${repoclone}`; - try { - await spawnAsync("git", [ - "clone", - "--branch", - branch!, - "--depth", - "1", - "--recurse-submodules", - cloneUrl, - outputPath, - "--progress", - ]); - } catch (error) { - throw error; - } + if (!githubId) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "GitHub Provider not found", + }); + } + const { COMPOSE_PATH } = paths(); + const githubProvider = await findGithubById(githubId); + const basePath = COMPOSE_PATH; + const outputPath = join(basePath, appName, "code"); + const octokit = authGithub(githubProvider); + const token = await getGithubToken(octokit); + const repoclone = `github.com/${owner}/${repository}.git`; + await recreateDirectory(outputPath); + const cloneUrl = `https://oauth2:${token}@${repoclone}`; + try { + await spawnAsync("git", [ + "clone", + "--branch", + branch!, + "--depth", + "1", + "--recurse-submodules", + cloneUrl, + outputPath, + "--progress", + ]); + } catch (error) { + throw error; + } }; export const cloneRawGithubRepositoryRemote = async (compose: Compose) => { - const { appName, repository, owner, branch, githubId, serverId } = compose; + const { appName, repository, owner, branch, githubId, serverId } = compose; - if (!serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - if (!githubId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "GitHub Provider not found", - }); - } + if (!serverId) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Server not found", + }); + } + if (!githubId) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "GitHub Provider not found", + }); + } - const { COMPOSE_PATH } = paths(true); - const githubProvider = await findGithubById(githubId); - const basePath = COMPOSE_PATH; - const outputPath = join(basePath, appName, "code"); - const octokit = authGithub(githubProvider); - const token = await getGithubToken(octokit); - const repoclone = `github.com/${owner}/${repository}.git`; - const cloneUrl = `https://oauth2:${token}@${repoclone}`; - try { - const command = ` + const { COMPOSE_PATH } = paths(true); + const githubProvider = await findGithubById(githubId); + const basePath = COMPOSE_PATH; + const outputPath = join(basePath, appName, "code"); + const octokit = authGithub(githubProvider); + const token = await getGithubToken(octokit); + const repoclone = `github.com/${owner}/${repository}.git`; + const cloneUrl = `https://oauth2:${token}@${repoclone}`; + try { + const command = ` rm -rf ${outputPath}; git clone --branch ${branch} --depth 1 ${cloneUrl} ${outputPath} `; - await execAsyncRemote(serverId, command); - } catch (error) { - throw error; - } + await execAsyncRemote(serverId, command); + } catch (error) { + throw error; + } }; export const getGithubRepositories = async (githubId?: string) => { - if (!githubId) { - return []; - } + if (!githubId) { + return []; + } - const githubProvider = await findGithubById(githubId); + const githubProvider = await findGithubById(githubId); - const octokit = new Octokit({ - authStrategy: createAppAuth, - auth: { - appId: githubProvider.githubAppId, - privateKey: githubProvider.githubPrivateKey, - installationId: githubProvider.githubInstallationId, - }, - }); + const octokit = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: githubProvider.githubAppId, + privateKey: githubProvider.githubPrivateKey, + installationId: githubProvider.githubInstallationId, + }, + }); - const repositories = (await octokit.paginate( - octokit.rest.apps.listReposAccessibleToInstallation, - )) as unknown as Awaited< - ReturnType - >["data"]["repositories"]; + const repositories = (await octokit.paginate( + octokit.rest.apps.listReposAccessibleToInstallation + )) as unknown as Awaited< + ReturnType + >["data"]["repositories"]; - return repositories; + return repositories; }; export const getGithubBranches = async ( - input: typeof apiFindGithubBranches._type, + input: typeof apiFindGithubBranches._type ) => { - if (!input.githubId) { - return []; - } - const githubProvider = await findGithubById(input.githubId); + if (!input.githubId) { + return []; + } + const githubProvider = await findGithubById(input.githubId); - const octokit = new Octokit({ - authStrategy: createAppAuth, - auth: { - appId: githubProvider.githubAppId, - privateKey: githubProvider.githubPrivateKey, - installationId: githubProvider.githubInstallationId, - }, - }); + const octokit = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: githubProvider.githubAppId, + privateKey: githubProvider.githubPrivateKey, + installationId: githubProvider.githubInstallationId, + }, + }); - const branches = (await octokit.paginate(octokit.rest.repos.listBranches, { - owner: input.owner, - repo: input.repo, - })) as unknown as Awaited< - ReturnType - >["data"]; + const branches = (await octokit.paginate(octokit.rest.repos.listBranches, { + owner: input.owner, + repo: input.repo, + })) as unknown as Awaited< + ReturnType + >["data"]; - return branches; + return branches; };