fix: add registry url and use spawnAsync

This commit is contained in:
Mauricio Siu 2024-11-28 21:54:20 -06:00
parent 34ea7ad8c9
commit 88f7cf2546
10 changed files with 4278 additions and 276 deletions

View File

@ -21,6 +21,7 @@ const DockerProviderSchema = z.object({
}),
username: z.string().optional(),
password: z.string().optional(),
registryURL: z.string().optional(),
});
type DockerProvider = z.infer<typeof DockerProviderSchema>;
@ -39,6 +40,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
dockerImage: "",
password: "",
username: "",
registryURL: "",
},
resolver: zodResolver(DockerProviderSchema),
});
@ -49,6 +51,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
dockerImage: data.dockerImage || "",
password: data.password || "",
username: data.username || "",
registryURL: data.registryUrl || "",
});
}
}, [form.reset, data, form]);
@ -59,6 +62,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
password: values.password || null,
applicationId,
username: values.username || null,
registryUrl: values.registryURL || null,
})
.then(async () => {
toast.success("Docker Provider Saved");
@ -76,7 +80,7 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
className="flex flex-col gap-4"
>
<div className="grid md:grid-cols-2 gap-4 ">
<div className="md:col-span-2 space-y-4">
<div className="space-y-4">
<FormField
control={form.control}
name="dockerImage"
@ -91,6 +95,19 @@ export const SaveDockerProvider = ({ applicationId }: Props) => {
)}
/>
</div>
<FormField
control={form.control}
name="registryURL"
render={({ field }) => (
<FormItem>
<FormLabel>Registry URL</FormLabel>
<FormControl>
<Input placeholder="Registry URL" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<div className="space-y-4">
<FormField
control={form.control}

View File

@ -0,0 +1 @@
ALTER TABLE "application" ADD COLUMN "registryUrl" text;

File diff suppressed because it is too large Load Diff

View File

@ -323,6 +323,13 @@
"when": 1732644181718,
"tag": "0045_smiling_blur",
"breakpoints": true
},
{
"idx": 46,
"version": "6",
"when": 1732851191048,
"tag": "0046_purple_sleeper",
"breakpoints": true
}
]
}

View File

@ -384,6 +384,7 @@ export const applicationRouter = createTRPCRouter({
password: input.password,
sourceType: "docker",
applicationStatus: "idle",
registryUrl: input.registryUrl,
});
return true;

View File

@ -147,6 +147,7 @@ export const applications = pgTable("application", {
username: text("username"),
password: text("password"),
dockerImage: text("dockerImage"),
registryUrl: text("registryUrl"),
// Git
customGitUrl: text("customGitUrl"),
customGitBranch: text("customGitBranch"),
@ -348,6 +349,7 @@ const createSchema = createInsertSchema(applications, {
dockerImage: z.string().optional(),
username: z.string().optional(),
password: z.string().optional(),
registryUrl: z.string().optional(),
customGitSSHKeyId: z.string().optional(),
repository: z.string().optional(),
dockerfile: z.string().optional(),
@ -451,6 +453,7 @@ export const apiSaveDockerProvider = createSchema
applicationId: true,
username: true,
password: true,
registryUrl: true,
})
.required();

View File

@ -208,14 +208,6 @@ export const deployApplication = async ({
adminId: application.project.adminId,
});
console.log(
"Error on ",
application.buildType,
"/",
application.sourceType,
error,
);
throw error;
}

View File

@ -11,12 +11,13 @@ import type { MysqlNested } from "../databases/mysql";
import type { PostgresNested } from "../databases/postgres";
import type { RedisNested } from "../databases/redis";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { spawnAsync } from "../process/spawnAsync";
import { getRemoteDocker } from "../servers/remote-docker";
interface RegistryAuth {
username: string;
password: string;
serveraddress: string;
registryUrl: string;
}
export const pullImage = async (
@ -29,29 +30,21 @@ export const pullImage = async (
throw new Error("Docker image not found");
}
await new Promise((resolve, reject) => {
docker.pull(dockerImage, { authconfig: authConfig }, (err, stream) => {
if (err) {
reject(err);
return;
}
docker.modem.followProgress(
stream as Readable,
(err: Error | null, res) => {
if (!err) {
resolve(res);
}
if (err) {
reject(err);
}
},
(event) => {
onData?.(event);
},
);
});
});
if (authConfig?.username && authConfig?.password) {
await spawnAsync(
"docker",
[
"login",
authConfig.registryUrl || "",
"-u",
authConfig.username,
"-p",
authConfig.password,
],
onData,
);
}
await spawnAsync("docker", ["pull", dockerImage], onData);
} catch (error) {
throw error;
}

View File

@ -5,7 +5,7 @@ import { pullImage } from "../docker/utils";
interface RegistryAuth {
username: string;
password: string;
serveraddress: string;
registryUrl: string;
}
export const buildDocker = async (
@ -16,6 +16,7 @@ export const buildDocker = async (
const authConfig: Partial<RegistryAuth> = {
username: username || "",
password: password || "",
registryUrl: application.registryUrl || "",
};
const writeStream = createWriteStream(logPath, { flags: "a" });
@ -33,7 +34,7 @@ export const buildDocker = async (
dockerImage,
(data) => {
if (writeStream.writable) {
writeStream.write(`${data.status}\n`);
writeStream.write(`${data}\n`);
}
},
authConfig,
@ -41,7 +42,7 @@ export const buildDocker = async (
await mechanizeDockerContainer(application);
writeStream.write("\nDocker Deployed: ✅\n");
} catch (error) {
writeStream.write(`ERROR: ${error}: ❌`);
writeStream.write("❌ Error");
throw error;
} finally {
writeStream.end();

View File

@ -14,325 +14,325 @@ import { type Github, findGithubById } from "@dokploy/server/services/github";
import { execAsyncRemote } from "../process/execAsync";
export const authGithub = (githubProvider: Github): Octokit => {
if (!haveGithubRequirements(githubProvider)) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Github Account not configured correctly",
});
}
if (!haveGithubRequirements(githubProvider)) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Github Account not configured correctly",
});
}
const octokit: Octokit = new Octokit({
authStrategy: createAppAuth,
auth: {
appId: githubProvider?.githubAppId || 0,
privateKey: githubProvider?.githubPrivateKey || "",
installationId: githubProvider?.githubInstallationId,
},
});
const octokit: Octokit = new Octokit({
authStrategy: createAppAuth,
auth: {
appId: githubProvider?.githubAppId || 0,
privateKey: githubProvider?.githubPrivateKey || "",
installationId: githubProvider?.githubInstallationId,
},
});
return octokit;
return octokit;
};
export const getGithubToken = async (
octokit: ReturnType<typeof authGithub>,
octokit: ReturnType<typeof authGithub>
) => {
const installation = (await octokit.auth({
type: "installation",
})) as {
token: string;
};
const installation = (await octokit.auth({
type: "installation",
})) as {
token: string;
};
return installation.token;
return installation.token;
};
export const haveGithubRequirements = (githubProvider: Github) => {
return !!(
githubProvider?.githubAppId &&
githubProvider?.githubPrivateKey &&
githubProvider?.githubInstallationId
);
return !!(
githubProvider?.githubAppId &&
githubProvider?.githubPrivateKey &&
githubProvider?.githubInstallationId
);
};
const getErrorCloneRequirements = (entity: {
repository?: string | null;
owner?: string | null;
branch?: string | null;
repository?: string | null;
owner?: string | null;
branch?: string | null;
}) => {
const reasons: string[] = [];
const { repository, owner, branch } = entity;
const reasons: string[] = [];
const { repository, owner, branch } = entity;
if (!repository) reasons.push("1. Repository not assigned.");
if (!owner) reasons.push("2. Owner not specified.");
if (!branch) reasons.push("3. Branch not defined.");
if (!repository) reasons.push("1. Repository not assigned.");
if (!owner) reasons.push("2. Owner not specified.");
if (!branch) reasons.push("3. Branch not defined.");
return reasons;
return reasons;
};
export type ApplicationWithGithub = InferResultType<
"applications",
{ github: true }
"applications",
{ github: true }
>;
export type ComposeWithGithub = InferResultType<"compose", { github: true }>;
export const cloneGithubRepository = async (
entity: ApplicationWithGithub | ComposeWithGithub,
logPath: string,
isCompose = false,
entity: ApplicationWithGithub | ComposeWithGithub,
logPath: string,
isCompose = false
) => {
const { APPLICATIONS_PATH, COMPOSE_PATH } = paths();
const writeStream = createWriteStream(logPath, { flags: "a" });
const { appName, repository, owner, branch, githubId } = entity;
const { APPLICATIONS_PATH, COMPOSE_PATH } = paths();
const writeStream = createWriteStream(logPath, { flags: "a" });
const { appName, repository, owner, branch, githubId } = entity;
if (!githubId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "GitHub Provider not found",
});
}
if (!githubId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "GitHub Provider not found",
});
}
const requirements = getErrorCloneRequirements(entity);
const requirements = getErrorCloneRequirements(entity);
// Check if requirements are met
if (requirements.length > 0) {
writeStream.write(
`\nGitHub Repository configuration failed for application: ${appName}\n`,
);
writeStream.write("Reasons:\n");
writeStream.write(requirements.join("\n"));
writeStream.end();
throw new TRPCError({
code: "BAD_REQUEST",
message: "Error: GitHub repository information is incomplete.",
});
}
// Check if requirements are met
if (requirements.length > 0) {
writeStream.write(
`\nGitHub Repository configuration failed for application: ${appName}\n`
);
writeStream.write("Reasons:\n");
writeStream.write(requirements.join("\n"));
writeStream.end();
throw new TRPCError({
code: "BAD_REQUEST",
message: "Error: GitHub repository information is incomplete.",
});
}
const githubProvider = await findGithubById(githubId);
const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`;
await recreateDirectory(outputPath);
const cloneUrl = `https://oauth2:${token}@${repoclone}`;
const githubProvider = await findGithubById(githubId);
const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`;
await recreateDirectory(outputPath);
const cloneUrl = `https://oauth2:${token}@${repoclone}`;
try {
writeStream.write(`\nClonning Repo ${repoclone} to ${outputPath}: ✅\n`);
await spawnAsync(
"git",
[
"clone",
"--branch",
branch!,
"--depth",
"1",
"--recurse-submodules",
cloneUrl,
outputPath,
"--progress",
],
(data) => {
if (writeStream.writable) {
writeStream.write(data);
}
},
);
writeStream.write(`\nCloned ${repoclone}: ✅\n`);
} catch (error) {
writeStream.write(`ERROR Clonning: ${error}: ❌`);
throw error;
} finally {
writeStream.end();
}
try {
writeStream.write(`\nClonning Repo ${repoclone} to ${outputPath}: ✅\n`);
await spawnAsync(
"git",
[
"clone",
"--branch",
branch!,
"--depth",
"1",
"--recurse-submodules",
cloneUrl,
outputPath,
"--progress",
],
(data) => {
if (writeStream.writable) {
writeStream.write(data);
}
}
);
writeStream.write(`\nCloned ${repoclone}: ✅\n`);
} catch (error) {
writeStream.write(`ERROR Clonning: ${error}: ❌`);
throw error;
} finally {
writeStream.end();
}
};
export const getGithubCloneCommand = async (
entity: ApplicationWithGithub | ComposeWithGithub,
logPath: string,
isCompose = false,
entity: ApplicationWithGithub | ComposeWithGithub,
logPath: string,
isCompose = false
) => {
const { appName, repository, owner, branch, githubId, serverId } = entity;
const { appName, repository, owner, branch, githubId, serverId } = entity;
if (!serverId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Server not found",
});
}
if (!serverId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Server not found",
});
}
if (!githubId) {
const command = `
if (!githubId) {
const command = `
echo "Error: ❌ Github Provider not found" >> ${logPath};
exit 1;
`;
await execAsyncRemote(serverId, command);
throw new TRPCError({
code: "NOT_FOUND",
message: "GitHub Provider not found",
});
}
await execAsyncRemote(serverId, command);
throw new TRPCError({
code: "NOT_FOUND",
message: "GitHub Provider not found",
});
}
const requirements = getErrorCloneRequirements(entity);
const requirements = getErrorCloneRequirements(entity);
// Build log messages
let logMessages = "";
if (requirements.length > 0) {
logMessages += `\nGitHub Repository configuration failed for application: ${appName}\n`;
logMessages += "Reasons:\n";
logMessages += requirements.join("\n");
const escapedLogMessages = logMessages
.replace(/\\/g, "\\\\")
.replace(/"/g, '\\"')
.replace(/\n/g, "\\n");
// Build log messages
let logMessages = "";
if (requirements.length > 0) {
logMessages += `\nGitHub Repository configuration failed for application: ${appName}\n`;
logMessages += "Reasons:\n";
logMessages += requirements.join("\n");
const escapedLogMessages = logMessages
.replace(/\\/g, "\\\\")
.replace(/"/g, '\\"')
.replace(/\n/g, "\\n");
const bashCommand = `
const bashCommand = `
echo "${escapedLogMessages}" >> ${logPath};
exit 1; # Exit with error code
`;
await execAsyncRemote(serverId, bashCommand);
return;
}
const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true);
const githubProvider = await findGithubById(githubId);
const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`;
const cloneUrl = `https://oauth2:${token}@${repoclone}`;
await execAsyncRemote(serverId, bashCommand);
return;
}
const { COMPOSE_PATH, APPLICATIONS_PATH } = paths(true);
const githubProvider = await findGithubById(githubId);
const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`;
const cloneUrl = `https://oauth2:${token}@${repoclone}`;
const cloneCommand = `
const cloneCommand = `
rm -rf ${outputPath};
mkdir -p ${outputPath};
if ! git clone --branch ${branch} --depth 1 --recurse-submodules --progress ${cloneUrl} ${outputPath} >> ${logPath} 2>&1; then
echo "❌ [ERROR] Fallo al clonar el repositorio ${repoclone}" >> ${logPath};
echo "❌ [ERROR] Fail to clone repository ${repoclone}" >> ${logPath};
exit 1;
fi
echo "Cloned ${repoclone} to ${outputPath}: ✅" >> ${logPath};
`;
return cloneCommand;
return cloneCommand;
};
export const cloneRawGithubRepository = async (entity: Compose) => {
const { appName, repository, owner, branch, githubId } = entity;
const { appName, repository, owner, branch, githubId } = entity;
if (!githubId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "GitHub Provider not found",
});
}
const { COMPOSE_PATH } = paths();
const githubProvider = await findGithubById(githubId);
const basePath = COMPOSE_PATH;
const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`;
await recreateDirectory(outputPath);
const cloneUrl = `https://oauth2:${token}@${repoclone}`;
try {
await spawnAsync("git", [
"clone",
"--branch",
branch!,
"--depth",
"1",
"--recurse-submodules",
cloneUrl,
outputPath,
"--progress",
]);
} catch (error) {
throw error;
}
if (!githubId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "GitHub Provider not found",
});
}
const { COMPOSE_PATH } = paths();
const githubProvider = await findGithubById(githubId);
const basePath = COMPOSE_PATH;
const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`;
await recreateDirectory(outputPath);
const cloneUrl = `https://oauth2:${token}@${repoclone}`;
try {
await spawnAsync("git", [
"clone",
"--branch",
branch!,
"--depth",
"1",
"--recurse-submodules",
cloneUrl,
outputPath,
"--progress",
]);
} catch (error) {
throw error;
}
};
export const cloneRawGithubRepositoryRemote = async (compose: Compose) => {
const { appName, repository, owner, branch, githubId, serverId } = compose;
const { appName, repository, owner, branch, githubId, serverId } = compose;
if (!serverId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Server not found",
});
}
if (!githubId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "GitHub Provider not found",
});
}
if (!serverId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Server not found",
});
}
if (!githubId) {
throw new TRPCError({
code: "NOT_FOUND",
message: "GitHub Provider not found",
});
}
const { COMPOSE_PATH } = paths(true);
const githubProvider = await findGithubById(githubId);
const basePath = COMPOSE_PATH;
const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`;
const cloneUrl = `https://oauth2:${token}@${repoclone}`;
try {
const command = `
const { COMPOSE_PATH } = paths(true);
const githubProvider = await findGithubById(githubId);
const basePath = COMPOSE_PATH;
const outputPath = join(basePath, appName, "code");
const octokit = authGithub(githubProvider);
const token = await getGithubToken(octokit);
const repoclone = `github.com/${owner}/${repository}.git`;
const cloneUrl = `https://oauth2:${token}@${repoclone}`;
try {
const command = `
rm -rf ${outputPath};
git clone --branch ${branch} --depth 1 ${cloneUrl} ${outputPath}
`;
await execAsyncRemote(serverId, command);
} catch (error) {
throw error;
}
await execAsyncRemote(serverId, command);
} catch (error) {
throw error;
}
};
export const getGithubRepositories = async (githubId?: string) => {
if (!githubId) {
return [];
}
if (!githubId) {
return [];
}
const githubProvider = await findGithubById(githubId);
const githubProvider = await findGithubById(githubId);
const octokit = new Octokit({
authStrategy: createAppAuth,
auth: {
appId: githubProvider.githubAppId,
privateKey: githubProvider.githubPrivateKey,
installationId: githubProvider.githubInstallationId,
},
});
const octokit = new Octokit({
authStrategy: createAppAuth,
auth: {
appId: githubProvider.githubAppId,
privateKey: githubProvider.githubPrivateKey,
installationId: githubProvider.githubInstallationId,
},
});
const repositories = (await octokit.paginate(
octokit.rest.apps.listReposAccessibleToInstallation,
)) as unknown as Awaited<
ReturnType<typeof octokit.rest.apps.listReposAccessibleToInstallation>
>["data"]["repositories"];
const repositories = (await octokit.paginate(
octokit.rest.apps.listReposAccessibleToInstallation
)) as unknown as Awaited<
ReturnType<typeof octokit.rest.apps.listReposAccessibleToInstallation>
>["data"]["repositories"];
return repositories;
return repositories;
};
export const getGithubBranches = async (
input: typeof apiFindGithubBranches._type,
input: typeof apiFindGithubBranches._type
) => {
if (!input.githubId) {
return [];
}
const githubProvider = await findGithubById(input.githubId);
if (!input.githubId) {
return [];
}
const githubProvider = await findGithubById(input.githubId);
const octokit = new Octokit({
authStrategy: createAppAuth,
auth: {
appId: githubProvider.githubAppId,
privateKey: githubProvider.githubPrivateKey,
installationId: githubProvider.githubInstallationId,
},
});
const octokit = new Octokit({
authStrategy: createAppAuth,
auth: {
appId: githubProvider.githubAppId,
privateKey: githubProvider.githubPrivateKey,
installationId: githubProvider.githubInstallationId,
},
});
const branches = (await octokit.paginate(octokit.rest.repos.listBranches, {
owner: input.owner,
repo: input.repo,
})) as unknown as Awaited<
ReturnType<typeof octokit.rest.repos.listBranches>
>["data"];
const branches = (await octokit.paginate(octokit.rest.repos.listBranches, {
owner: input.owner,
repo: input.repo,
})) as unknown as Awaited<
ReturnType<typeof octokit.rest.repos.listBranches>
>["data"];
return branches;
return branches;
};