fix: add registry url and use spawnAsync

This commit is contained in:
Mauricio Siu 2024-11-28 21:54:20 -06:00
parent 34ea7ad8c9
commit 88f7cf2546
10 changed files with 4278 additions and 276 deletions

View File

@ -21,6 +21,7 @@ const DockerProviderSchema = z.object({
}), }),
username: z.string().optional(), username: z.string().optional(),
password: z.string().optional(), password: z.string().optional(),
registryURL: z.string().optional(),
}); });
type DockerProvider = z.infer<typeof DockerProviderSchema>; type DockerProvider = z.infer<typeof DockerProviderSchema>;
@ -39,6 +40,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
dockerImage: "", dockerImage: "",
password: "", password: "",
username: "", username: "",
registryURL: "",
}, },
resolver: zodResolver(DockerProviderSchema), resolver: zodResolver(DockerProviderSchema),
}); });
@ -49,6 +51,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
dockerImage: data.dockerImage || "", dockerImage: data.dockerImage || "",
password: data.password || "", password: data.password || "",
username: data.username || "", username: data.username || "",
registryURL: data.registryUrl || "",
}); });
} }
}, [form.reset, data, form]); }, [form.reset, data, form]);
@ -59,6 +62,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
password: values.password || null, password: values.password || null,
applicationId, applicationId,
username: values.username || null, username: values.username || null,
registryUrl: values.registryURL || null,
}) })
.then(async () => { .then(async () => {
toast.success("Docker Provider Saved"); toast.success("Docker Provider Saved");
@ -76,7 +80,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
className="flex flex-col gap-4" className="flex flex-col gap-4"
> >
<div className="grid md:grid-cols-2 gap-4 "> <div className="grid md:grid-cols-2 gap-4 ">
<div className="md:col-span-2 space-y-4"> <div className="space-y-4">
<FormField <FormField
control={form.control} control={form.control}
name="dockerImage" name="dockerImage"
@ -91,6 +95,19 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
)} )}
/> />
</div> </div>
<FormField
control={form.control}
name="registryURL"
render={({ field }) => (
<FormItem>
<FormLabel>Registry URL</FormLabel>
<FormControl>
<Input placeholder="Registry URL" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<div className="space-y-4"> <div className="space-y-4">
<FormField <FormField
control={form.control} control={form.control}

View File

@ -0,0 +1 @@
ALTER TABLE "application" ADD COLUMN "registryUrl" text;

File diff suppressed because it is too large Load Diff

View File

@ -323,6 +323,13 @@
"when": 1732644181718, "when": 1732644181718,
"tag": "0045_smiling_blur", "tag": "0045_smiling_blur",
"breakpoints": true "breakpoints": true
},
{
"idx": 46,
"version": "6",
"when": 1732851191048,
"tag": "0046_purple_sleeper",
"breakpoints": true
} }
] ]
} }

View File

@ -384,6 +384,7 @@ export const applicationRouter = createTRPCRouter({
password: input.password, password: input.password,
sourceType: "docker", sourceType: "docker",
applicationStatus: "idle", applicationStatus: "idle",
registryUrl: input.registryUrl,
}); });
return true; return true;

View File

@ -147,6 +147,7 @@ export const applications = pgTable("application", {
username: text("username"), username: text("username"),
password: text("password"), password: text("password"),
dockerImage: text("dockerImage"), dockerImage: text("dockerImage"),
registryUrl: text("registryUrl"),
// Git // Git
customGitUrl: text("customGitUrl"), customGitUrl: text("customGitUrl"),
customGitBranch: text("customGitBranch"), customGitBranch: text("customGitBranch"),
@ -348,6 +349,7 @@ const createSchema = createInsertSchema(applications, {
dockerImage: z.string().optional(), dockerImage: z.string().optional(),
username: z.string().optional(), username: z.string().optional(),
password: z.string().optional(), password: z.string().optional(),
registryUrl: z.string().optional(),
customGitSSHKeyId: z.string().optional(), customGitSSHKeyId: z.string().optional(),
repository: z.string().optional(), repository: z.string().optional(),
dockerfile: z.string().optional(), dockerfile: z.string().optional(),
@ -451,6 +453,7 @@ export const apiSaveDockerProvider = createSchema
applicationId: true, applicationId: true,
username: true, username: true,
password: true, password: true,
registryUrl: true,
}) })
.required(); .required();

View File

@ -208,14 +208,6 @@ export const deployApplication = async ({
adminId: application.project.adminId, adminId: application.project.adminId,
}); });
console.log(
"Error on ",
application.buildType,
"/",
application.sourceType,
error,
);
throw error; throw error;
} }

View File

@ -11,12 +11,13 @@ import type { MysqlNested } from "../databases/mysql";
import type { PostgresNested } from "../databases/postgres"; import type { PostgresNested } from "../databases/postgres";
import type { RedisNested } from "../databases/redis"; import type { RedisNested } from "../databases/redis";
import { execAsync, execAsyncRemote } from "../process/execAsync"; import { execAsync, execAsyncRemote } from "../process/execAsync";
import { spawnAsync } from "../process/spawnAsync";
import { getRemoteDocker } from "../servers/remote-docker"; import { getRemoteDocker } from "../servers/remote-docker";
interface RegistryAuth { interface RegistryAuth {
username: string; username: string;
password: string; password: string;
serveraddress: string; registryUrl: string;
} }
export const pullImage = async ( export const pullImage = async (
@ -29,29 +30,21 @@ export const pullImage = async (
throw new Error("Docker image not found"); throw new Error("Docker image not found");
} }
await new Promise((resolve, reject) => { if (authConfig?.username && authConfig?.password) {
docker.pull(dockerImage, { authconfig: authConfig }, (err, stream) => { await spawnAsync(
if (err) { "docker",
reject(err); [
return; "login",
} authConfig.registryUrl || "",
"-u",
docker.modem.followProgress( authConfig.username,
stream as Readable, "-p",
(err: Error | null, res) => { authConfig.password,
if (!err) { ],
resolve(res); onData,
} );
if (err) { }
reject(err); await spawnAsync("docker", ["pull", dockerImage], onData);
}
},
(event) => {
onData?.(event);
},
);
});
});
} catch (error) { } catch (error) {
throw error; throw error;
} }

View File

@ -5,7 +5,7 @@ import { pullImage } from "../docker/utils";
interface RegistryAuth { interface RegistryAuth {
username: string; username: string;
password: string; password: string;
serveraddress: string; registryUrl: string;
} }
export const buildDocker = async ( export const buildDocker = async (
@ -16,6 +16,7 @@ export const buildDocker = async (
const authConfig: Partial<RegistryAuth> = { const authConfig: Partial<RegistryAuth> = {
username: username || "", username: username || "",
password: password || "", password: password || "",
registryUrl: application.registryUrl || "",
}; };
const writeStream = createWriteStream(logPath, { flags: "a" }); const writeStream = createWriteStream(logPath, { flags: "a" });
@ -33,7 +34,7 @@ export const buildDocker = async (
dockerImage, dockerImage,
(data) => { (data) => {
if (writeStream.writable) { if (writeStream.writable) {
writeStream.write(`${data.status}\n`); writeStream.write(`${data}\n`);
} }
}, },
authConfig, authConfig,
@ -41,7 +42,7 @@ export const buildDocker = async (
await mechanizeDockerContainer(application); await mechanizeDockerContainer(application);
writeStream.write("\nDocker Deployed: ✅\n"); writeStream.write("\nDocker Deployed: ✅\n");
} catch (error) { } catch (error) {
writeStream.write(`ERROR: ${error}: ❌`); writeStream.write("❌ Error");
throw error; throw error;
} finally { } finally {
writeStream.end(); writeStream.end();

View File

@ -14,325 +14,325 @@ import { type Github, findGithubById } from "@dokploy/server/services/github";
import { execAsyncRemote } from "../process/execAsync"; import { execAsyncRemote } from "../process/execAsync";
export const authGithub = (githubProvider: Github): Octokit => { export const authGithub = (githubProvider: Github): Octokit => {
if (!haveGithubRequirements(githubProvider)) { if (!haveGithubRequirements(githubProvider)) {
throw new TRPCError({ throw new TRPCError({
code: "NOT_FOUND", code: "NOT_FOUND",
message: "Github Account not configured correctly", message: "Github Account not configured correctly",
}); });
} }
const octokit: Octokit = new Octokit({ const octokit: Octokit = new Octokit({
authStrategy: createAppAuth, authStrategy: createAppAuth,
auth: { auth: {
appId: githubProvider?.githubAppId || 0, appId: githubProvider?.githubAppId || 0,
privateKey: githubProvider?.githubPrivateKey || "", privateKey: githubProvider?.githubPrivateKey || "",
installationId: githubProvider?.githubInstallationId, installationId: githubProvider?.githubInstallationId,
}, },
}); });
return octokit; return octokit;
}; };
export const getGithubToken = async ( export const getGithubToken = async (
octokit: ReturnType<typeof authGithub>, octokit: ReturnType<typeof authGithub>
) => { ) => {
const installation = (await octokit.auth({ const installation = (await octokit.auth({
type: "installation", type: "installation",
})) as { })) as {
token: string; token: string;
}; };
return installation.token; return installation.token;
}; };
export const haveGithubRequirements = (githubProvider: Github) => { export const haveGithubRequirements = (githubProvider: Github) => {
return !!( return !!(
githubProvider?.githubAppId && githubProvider?.githubAppId &&
githubProvider?.githubPrivateKey && githubProvider?.githubPrivateKey &&
githubProvider?.githubInstallationId githubProvider?.githubInstallationId
); );
}; };
const getErrorCloneRequirements = (entity: { const getErrorCloneRequirements = (entity: {
repository?: string | null; repository?: string | null;
owner?: string | null; owner?: string | null;
branch?: string | null; branch?: string | null;
}) => { }) => {
const reasons: string[] = []; const reasons: string[] = [];
const { repository, owner, branch } = entity; const { repository, owner, branch } = entity;
if (!repository) reasons.push("1. Repository not assigned."); if (!repository) reasons.push("1. Repository not assigned.");
if (!owner) reasons.push("2. Owner not specified."); if (!owner) reasons.push("2. Owner not specified.");
if (!branch) reasons.push("3. Branch not defined."); if (!branch) reasons.push("3. Branch not defined.");
return reasons; return reasons;
}; };
export type ApplicationWithGithub = InferResultType< export type ApplicationWithGithub = InferResultType<
"applications", "applications",
{ github: true } { github: true }
>; >;
export type ComposeWithGithub = InferResultType<"compose", { github: true }>; export type ComposeWithGithub = InferResultType<"compose", { github: true }>;
export const cloneGithubRepository = async ( export const cloneGithubRepository = async (
entity: ApplicationWithGithub | ComposeWithGithub, entity: ApplicationWithGithub | ComposeWithGithub,
logPath: string, logPath: string,
isCompose = false, isCompose = false
) => { ) => {
const { APPLICATIONS_PATH, COMPOSE_PATH } = paths(); const { APPLICATIONS_PATH, COMPOSE_PATH } = paths();
const writeStream = createWriteStream(logPath, { flags: "a" }); const writeStream = createWriteStream(logPath, { flags: "a" });
const { appName, repository, owner, branch, githubId } = entity; const { appName, repository, owner, branch, githubId } = entity;
if (!githubId) { if (!githubId) {
throw new TRPCError({ throw new TRPCError({
code: "NOT_FOUND", code: "NOT_FOUND",
message: "GitHub Provider not found", message: "GitHub Provider not found",
}); });
} }
const requirements = getErrorCloneRequirements(entity); const requirements = getErrorCloneRequirements(entity);
// Check if requirements are met // Check if requirements are met
if (requirements.length > 0) { if (requirements.length > 0) {
writeStream.write( writeStream.write(
`\nGitHub Repository configuration failed for application: ${appName}\n`, `\nGitHub Repository configuration failed for application: ${appName}\n`
); );
writeStream.write("Reasons:\n"); writeStream.write("Reasons:\n");
writeStream.write(requirements.join("\n")); writeStream.write(requirements.join("\n"));
writeStream.end(); writeStream.end();
throw new TRPCError({ throw new TRPCError({
code: "BAD_REQUEST", code: "BAD_REQUEST",
message: "Error: GitHub repository information is incomplete.", message: "Error: GitHub repository information is incomplete.",
}); });
} }
const githubProvider = await findGithubById(githubId); const githubProvider = await findGithubById(githubId);
const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
const outputPath = join(basePath, appName, "code"); const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider); const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit); const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`; const repoclone = `github.com/${owner}/${repository}.git`;
await recreateDirectory(outputPath); await recreateDirectory(outputPath);
const cloneUrl = `https://oauth2:${token}@${repoclone}`; const cloneUrl = `https://oauth2:${token}@${repoclone}`;
try { try {
writeStream.write(`\nClonning Repo ${repoclone} to ${outputPath}: ✅\n`); writeStream.write(`\nClonning Repo ${repoclone} to ${outputPath}: ✅\n`);
await spawnAsync( await spawnAsync(
"git", "git",
[ [
"clone", "clone",
"--branch", "--branch",
branch!, branch!,
"--depth", "--depth",
"1", "1",
"--recurse-submodules", "--recurse-submodules",
cloneUrl, cloneUrl,
outputPath, outputPath,
"--progress", "--progress",
], ],
(data) => { (data) => {
if (writeStream.writable) { if (writeStream.writable) {
writeStream.write(data); writeStream.write(data);
} }
}, }
); );
writeStream.write(`\nCloned ${repoclone}: ✅\n`); writeStream.write(`\nCloned ${repoclone}: ✅\n`);
} catch (error) { } catch (error) {
writeStream.write(`ERROR Clonning: ${error}: ❌`); writeStream.write(`ERROR Clonning: ${error}: ❌`);
throw error; throw error;
} finally { } finally {
writeStream.end(); writeStream.end();
} }
}; };
export const getGithubCloneCommand = async ( export const getGithubCloneCommand = async (
entity: ApplicationWithGithub | ComposeWithGithub, entity: ApplicationWithGithub | ComposeWithGithub,
logPath: string, logPath: string,
isCompose = false, isCompose = false
) => { ) => {
const { appName, repository, owner, branch, githubId, serverId } = entity; const { appName, repository, owner, branch, githubId, serverId } = entity;
if (!serverId) { if (!serverId) {
throw new TRPCError({ throw new TRPCError({
code: "NOT_FOUND", code: "NOT_FOUND",
message: "Server not found", message: "Server not found",
}); });
} }
if (!githubId) { if (!githubId) {
const command = ` const command = `
echo "Error: ❌ Github Provider not found" >> ${logPath}; echo "Error: ❌ Github Provider not found" >> ${logPath};
exit 1; exit 1;
`; `;
await execAsyncRemote(serverId, command); await execAsyncRemote(serverId, command);
throw new TRPCError({ throw new TRPCError({
code: "NOT_FOUND", code: "NOT_FOUND",
message: "GitHub Provider not found", message: "GitHub Provider not found",
}); });
} }
const requirements = getErrorCloneRequirements(entity); const requirements = getErrorCloneRequirements(entity);
// Build log messages // Build log messages
let logMessages = ""; let logMessages = "";
if (requirements.length > 0) { if (requirements.length > 0) {
logMessages += `\nGitHub Repository configuration failed for application: ${appName}\n`; logMessages += `\nGitHub Repository configuration failed for application: ${appName}\n`;
logMessages += "Reasons:\n"; logMessages += "Reasons:\n";
logMessages += requirements.join("\n"); logMessages += requirements.join("\n");
const escapedLogMessages = logMessages const escapedLogMessages = logMessages
.replace(/\\/g, "\\\\") .replace(/\\/g, "\\\\")
.replace(/"/g, '\\"') .replace(/"/g, '\\"')
.replace(/\n/g, "\\n"); .replace(/\n/g, "\\n");
const bashCommand = ` const bashCommand = `
echo "${escapedLogMessages}" >> ${logPath}; echo "${escapedLogMessages}" >> ${logPath};
exit 1; # Exit with error code exit 1; # Exit with error code
`; `;
await execAsyncRemote(serverId, bashCommand); await execAsyncRemote(serverId, bashCommand);
return; return;
} }
const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true); const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true);
const githubProvider = await findGithubById(githubId); const githubProvider = await findGithubById(githubId);
const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH; const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
const outputPath = join(basePath, appName, "code"); const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider); const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit); const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`; const repoclone = `github.com/${owner}/${repository}.git`;
const cloneUrl = `https://oauth2:${token}@${repoclone}`; const cloneUrl = `https://oauth2:${token}@${repoclone}`;
const cloneCommand = ` const cloneCommand = `
rm -rf ${outputPath}; rm -rf ${outputPath};
mkdir -p ${outputPath}; mkdir -p ${outputPath};
if ! git clone --branch ${branch} --depth 1 --recurse-submodules --progress ${cloneUrl} ${outputPath} >> ${logPath} 2>&1; then if ! git clone --branch ${branch} --depth 1 --recurse-submodules --progress ${cloneUrl} ${outputPath} >> ${logPath} 2>&1; then
echo "❌ [ERROR] Fallo al clonar el repositorio ${repoclone}" >> ${logPath}; echo "❌ [ERROR] Fail to clone repository ${repoclone}" >> ${logPath};
exit 1; exit 1;
fi fi
echo "Cloned ${repoclone} to ${outputPath}: ✅" >> ${logPath}; echo "Cloned ${repoclone} to ${outputPath}: ✅" >> ${logPath};
`; `;
return cloneCommand; return cloneCommand;
}; };
export const cloneRawGithubRepository = async (entity: Compose) => { export const cloneRawGithubRepository = async (entity: Compose) => {
const { appName, repository, owner, branch, githubId } = entity; const { appName, repository, owner, branch, githubId } = entity;
if (!githubId) { if (!githubId) {
throw new TRPCError({ throw new TRPCError({
code: "NOT_FOUND", code: "NOT_FOUND",
message: "GitHub Provider not found", message: "GitHub Provider not found",
}); });
} }
const { COMPOSE_PATH } = paths(); const { COMPOSE_PATH } = paths();
const githubProvider = await findGithubById(githubId); const githubProvider = await findGithubById(githubId);
const basePath = COMPOSE_PATH; const basePath = COMPOSE_PATH;
const outputPath = join(basePath, appName, "code"); const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider); const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit); const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`; const repoclone = `github.com/${owner}/${repository}.git`;
await recreateDirectory(outputPath); await recreateDirectory(outputPath);
const cloneUrl = `https://oauth2:${token}@${repoclone}`; const cloneUrl = `https://oauth2:${token}@${repoclone}`;
try { try {
await spawnAsync("git", [ await spawnAsync("git", [
"clone", "clone",
"--branch", "--branch",
branch!, branch!,
"--depth", "--depth",
"1", "1",
"--recurse-submodules", "--recurse-submodules",
cloneUrl, cloneUrl,
outputPath, outputPath,
"--progress", "--progress",
]); ]);
} catch (error) { } catch (error) {
throw error; throw error;
} }
}; };
export const cloneRawGithubRepositoryRemote = async (compose: Compose) => { export const cloneRawGithubRepositoryRemote = async (compose: Compose) => {
const { appName, repository, owner, branch, githubId, serverId } = compose; const { appName, repository, owner, branch, githubId, serverId } = compose;
if (!serverId) { if (!serverId) {
throw new TRPCError({ throw new TRPCError({
code: "NOT_FOUND", code: "NOT_FOUND",
message: "Server not found", message: "Server not found",
}); });
} }
if (!githubId) { if (!githubId) {
throw new TRPCError({ throw new TRPCError({
code: "NOT_FOUND", code: "NOT_FOUND",
message: "GitHub Provider not found", message: "GitHub Provider not found",
}); });
} }
const { COMPOSE_PATH } = paths(true); const { COMPOSE_PATH } = paths(true);
const githubProvider = await findGithubById(githubId); const githubProvider = await findGithubById(githubId);
const basePath = COMPOSE_PATH; const basePath = COMPOSE_PATH;
const outputPath = join(basePath, appName, "code"); const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider); const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit); const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`; const repoclone = `github.com/${owner}/${repository}.git`;
const cloneUrl = `https://oauth2:${token}@${repoclone}`; const cloneUrl = `https://oauth2:${token}@${repoclone}`;
try { try {
const command = ` const command = `
rm -rf ${outputPath}; rm -rf ${outputPath};
git clone --branch ${branch} --depth 1 ${cloneUrl} ${outputPath} git clone --branch ${branch} --depth 1 ${cloneUrl} ${outputPath}
`; `;
await execAsyncRemote(serverId, command); await execAsyncRemote(serverId, command);
} catch (error) { } catch (error) {
throw error; throw error;
} }
}; };
export const getGithubRepositories = async (githubId?: string) => { export const getGithubRepositories = async (githubId?: string) => {
if (!githubId) { if (!githubId) {
return []; return [];
} }
const githubProvider = await findGithubById(githubId); const githubProvider = await findGithubById(githubId);
const octokit = new Octokit({ const octokit = new Octokit({
authStrategy: createAppAuth, authStrategy: createAppAuth,
auth: { auth: {
appId: githubProvider.githubAppId, appId: githubProvider.githubAppId,
privateKey: githubProvider.githubPrivateKey, privateKey: githubProvider.githubPrivateKey,
installationId: githubProvider.githubInstallationId, installationId: githubProvider.githubInstallationId,
}, },
}); });
const repositories = (await octokit.paginate( const repositories = (await octokit.paginate(
octokit.rest.apps.listReposAccessibleToInstallation, octokit.rest.apps.listReposAccessibleToInstallation
)) as unknown as Awaited< )) as unknown as Awaited<
ReturnType<typeof octokit.rest.apps.listReposAccessibleToInstallation> ReturnType<typeof octokit.rest.apps.listReposAccessibleToInstallation>
>["data"]["repositories"]; >["data"]["repositories"];
return repositories; return repositories;
}; };
export const getGithubBranches = async ( export const getGithubBranches = async (
input: typeof apiFindGithubBranches._type, input: typeof apiFindGithubBranches._type
) => { ) => {
if (!input.githubId) { if (!input.githubId) {
return []; return [];
} }
const githubProvider = await findGithubById(input.githubId); const githubProvider = await findGithubById(input.githubId);
const octokit = new Octokit({ const octokit = new Octokit({
authStrategy: createAppAuth, authStrategy: createAppAuth,
auth: { auth: {
appId: githubProvider.githubAppId, appId: githubProvider.githubAppId,
privateKey: githubProvider.githubPrivateKey, privateKey: githubProvider.githubPrivateKey,
installationId: githubProvider.githubInstallationId, installationId: githubProvider.githubInstallationId,
}, },
}); });
const branches = (await octokit.paginate(octokit.rest.repos.listBranches, { const branches = (await octokit.paginate(octokit.rest.repos.listBranches, {
owner: input.owner, owner: input.owner,
repo: input.repo, repo: input.repo,
})) as unknown as Awaited< })) as unknown as Awaited<
ReturnType<typeof octokit.rest.repos.listBranches> ReturnType<typeof octokit.rest.repos.listBranches>
>["data"]; >["data"];
return branches; return branches;
}; };