From 249fe8c7fea3f00531af3d52f40b3f2b2c2d1711 Mon Sep 17 00:00:00 2001
From: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com>
Date: Sun, 1 Sep 2024 00:18:45 -0600
Subject: [PATCH] feat: add clone by gitlab, github and bitbucket

---
 .../server/api/routers/git-provider.ts        |  50 +----
 .../server/api/services/application.ts        |  17 +-
 .../server/api/services/git-provider.ts       |   4 -
 .../server/utils/providers/bitbucket.ts       | 110 ++++++++++
 apps/dokploy/server/utils/providers/github.ts |  71 ++++---
 apps/dokploy/server/utils/providers/gitlab.ts | 194 ++++++++++++++++++
 6 files changed, 368 insertions(+), 78 deletions(-)
 create mode 100644 apps/dokploy/server/utils/providers/bitbucket.ts
 create mode 100644 apps/dokploy/server/utils/providers/gitlab.ts

diff --git a/apps/dokploy/server/api/routers/git-provider.ts b/apps/dokploy/server/api/routers/git-provider.ts
index 21008ba2..dbd96c4f 100644
--- a/apps/dokploy/server/api/routers/git-provider.ts
+++ b/apps/dokploy/server/api/routers/git-provider.ts
@@ -11,11 +11,13 @@ import {
 	getBitbucketProvider,
 	getGitlabProvider,
 	haveGithubRequirements,
-	haveGitlabRequirements,
 	removeGithubProvider,
-	updateGitlabProvider,
 } from "../services/git-provider";
 import { z } from "zod";
+import {
+	haveGitlabRequirements,
+	refreshGitlabToken,
+} from "@/server/utils/providers/gitlab";
 
 export const gitProvider = createTRPCRouter({
 	getAll: protectedProcedure.query(async () => {
@@ -164,8 +166,7 @@ export const gitProvider = createTRPCRouter({
 			}),
 		)
 		.query(async ({ input }) => {
-			console.log(input);
-			if (!input.gitlabProviderId || !input.repo || !input.owner) {
+			if (!input.gitlabProviderId) {
 				return [];
 			}
 
@@ -329,47 +330,6 @@ export const gitProvider = createTRPCRouter({
 			}
 		}),
 });
-async function refreshGitlabToken(gitlabProviderId: string) {
-	const gitlabProvider = await getGitlabProvider(gitlabProviderId);
-	const currentTime = Math.floor(Date.now() / 1000);
-
-	const safetyMargin = 60;
-	if (
-		gitlabProvider.expiresAt &&
-		currentTime + safetyMargin < gitlabProvider.expiresAt
-	) {
-		console.log("Token still valid, no need to refresh");
-		return;
-	}
-
-	const response = await fetch("https://gitlab.com/oauth/token", {
-		method: "POST",
-		headers: {
-			"Content-Type": "application/x-www-form-urlencoded",
-		},
-		body: new URLSearchParams({
-			grant_type: "refresh_token",
-			refresh_token: gitlabProvider.refreshToken as string,
-			client_id: gitlabProvider.applicationId as string,
-			client_secret: gitlabProvider.secret as string,
-		}),
-	});
-
-	if (!response.ok) {
-		throw new Error(`Failed to refresh token: ${response.statusText}`);
-	}
-
-	const data = await response.json();
-
-	const expiresAt = Math.floor(Date.now() / 1000) + data.expires_in;
-
-	await updateGitlabProvider(gitlabProviderId, {
-		accessToken: data.access_token,
-		refreshToken: data.refresh_token,
-		expiresAt,
-	});
-	return data;
-}
 // 1725175543
 // {
 // 	access_token: '11d422887d8fac712191ee9b09dfdb043a705938cd67a4a39f36b4bc65b3106d',
diff --git a/apps/dokploy/server/api/services/application.ts b/apps/dokploy/server/api/services/application.ts
index 24733ba2..ea56e09a 100644
--- a/apps/dokploy/server/api/services/application.ts
+++ b/apps/dokploy/server/api/services/application.ts
@@ -21,6 +21,8 @@ import { createDeployment, updateDeploymentStatus } from "./deployment";
 import { sendBuildErrorNotifications } from "@/server/utils/notifications/build-error";
 import { sendBuildSuccessNotifications } from "@/server/utils/notifications/build-success";
 import { validUniqueServerAppName } from "./project";
+import { cloneGitlabRepository } from "@/server/utils/providers/gitlab";
+import { cloneBitbucketRepository } from "@/server/utils/providers/bitbucket";
 export type Application = typeof applications.$inferSelect;
 
 export const createApplication = async (
@@ -81,6 +83,9 @@ export const findApplicationById = async (applicationId: string) => {
 			security: true,
 			ports: true,
 			registry: true,
+			gitlabProvider: true,
+			githubProvider: true,
+			bitbucketProvider: true,
 		},
 	});
 	if (!application) {
@@ -150,7 +155,13 @@ export const deployApplication = async ({
 
 	try {
 		if (application.sourceType === "github") {
-			await cloneGithubRepository(admin, application, deployment.logPath);
+			await cloneGithubRepository(application, deployment.logPath);
+			await buildApplication(application, deployment.logPath);
+		} else if (application.sourceType === "gitlab") {
+			await cloneGitlabRepository(application, deployment.logPath);
+			await buildApplication(application, deployment.logPath);
+		} else if (application.sourceType === "bitbucket") {
+			await cloneBitbucketRepository(application, deployment.logPath);
 			await buildApplication(application, deployment.logPath);
 		} else if (application.sourceType === "docker") {
 			await buildDocker(application, deployment.logPath);
@@ -214,6 +225,10 @@ export const rebuildApplication = async ({
 	try {
 		if (application.sourceType === "github") {
 			await buildApplication(application, deployment.logPath);
+		} else if (application.sourceType === "gitlab") {
+			await buildApplication(application, deployment.logPath);
+		} else if (application.sourceType === "bitbucket") {
+			await buildApplication(application, deployment.logPath);
 		} else if (application.sourceType === "docker") {
 			await buildDocker(application, deployment.logPath);
 		} else if (application.sourceType === "git") {
diff --git a/apps/dokploy/server/api/services/git-provider.ts b/apps/dokploy/server/api/services/git-provider.ts
index 79f00437..5aad1153 100644
--- a/apps/dokploy/server/api/services/git-provider.ts
+++ b/apps/dokploy/server/api/services/git-provider.ts
@@ -142,10 +142,6 @@ export const haveGithubRequirements = (githubProvider: GithubProvider) => {
 	);
 };
 
-export const haveGitlabRequirements = (gitlabProvider: GitlabProvider) => {
-	return !!(gitlabProvider?.accessToken && gitlabProvider?.refreshToken);
-};
-
 export const getGitlabProvider = async (gitlabProviderId: string) => {
 	const gitlabProviderResult = await db.query.gitlabProvider.findFirst({
 		where: eq(gitlabProvider.gitlabProviderId, gitlabProviderId),
diff --git a/apps/dokploy/server/utils/providers/bitbucket.ts b/apps/dokploy/server/utils/providers/bitbucket.ts
new file mode 100644
index 00000000..04529120
--- /dev/null
+++ b/apps/dokploy/server/utils/providers/bitbucket.ts
@@ -0,0 +1,110 @@
+import { createWriteStream } from "node:fs";
+import { join } from "node:path";
+import { APPLICATIONS_PATH, COMPOSE_PATH } from "@/server/constants";
+import { TRPCError } from "@trpc/server";
+import { recreateDirectory } from "../filesystem/directory";
+import { spawnAsync } from "../process/spawnAsync";
+import type { InferResultType } from "@/server/types/with";
+
+export type ApplicationWithBitbucket = InferResultType<
+	"applications",
+	{ bitbucketProvider: true }
+>;
+
+export const cloneBitbucketRepository = async (
+	entity: ApplicationWithBitbucket,
+	logPath: string,
+	isCompose = false,
+) => {
+	const writeStream = createWriteStream(logPath, { flags: "a" });
+	const {
+		appName,
+		bitbucketRepository,
+		bitbucketOwner,
+		bitbucketBranch,
+		bitbucketProviderId,
+		bitbucketProvider,
+	} = entity;
+
+	if (!bitbucketProviderId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Bitbucket Provider not found",
+		});
+	}
+
+	const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
+	const outputPath = join(basePath, appName, "code");
+	await recreateDirectory(outputPath);
+	const repoclone = `bitbucket.org/${bitbucketOwner}/${bitbucketRepository}.git`;
+	const cloneUrl = `https://${bitbucketProvider?.bitbucketUsername}:${bitbucketProvider?.appPassword}@${repoclone}`;
+
+	try {
+		writeStream.write(`\nCloning Repo ${repoclone} to ${outputPath}: ✅\n`);
+		await spawnAsync(
+			"git",
+			[
+				"clone",
+				"--branch",
+				bitbucketBranch!,
+				"--depth",
+				"1",
+				cloneUrl,
+				outputPath,
+				"--progress",
+			],
+			(data) => {
+				if (writeStream.writable) {
+					writeStream.write(data);
+				}
+			},
+		);
+		writeStream.write(`\nCloned ${repoclone} to ${outputPath}: ✅\n`);
+	} catch (error) {
+		writeStream.write(`ERROR Clonning: ${error}: ❌`);
+		throw error;
+	} finally {
+		writeStream.end();
+	}
+};
+
+export const cloneRawBitbucketRepository = async (
+	entity: ApplicationWithBitbucket,
+) => {
+	const {
+		appName,
+		bitbucketRepository,
+		bitbucketOwner,
+		bitbucketBranch,
+		bitbucketProviderId,
+		bitbucketProvider,
+	} = entity;
+
+	if (!bitbucketProviderId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Bitbucket Provider not found",
+		});
+	}
+
+	const basePath = COMPOSE_PATH;
+	const outputPath = join(basePath, appName, "code");
+	await recreateDirectory(outputPath);
+	const repoclone = `bitbucket.org/${bitbucketOwner}/${bitbucketRepository}.git`;
+	const cloneUrl = `https://${bitbucketProvider?.bitbucketUsername}:${bitbucketProvider?.appPassword}@${repoclone}`;
+
+	try {
+		await spawnAsync("git", [
+			"clone",
+			"--branch",
+			bitbucketBranch!,
+			"--depth",
+			"1",
+			cloneUrl,
+			outputPath,
+			"--progress",
+		]);
+	} catch (error) {
+		throw error;
+	}
+};
diff --git a/apps/dokploy/server/utils/providers/github.ts b/apps/dokploy/server/utils/providers/github.ts
index a49f2b56..6a693a80 100644
--- a/apps/dokploy/server/utils/providers/github.ts
+++ b/apps/dokploy/server/utils/providers/github.ts
@@ -1,15 +1,19 @@
 import { createWriteStream } from "node:fs";
 import { join } from "node:path";
-import { type Admin, findAdmin } from "@/server/api/services/admin";
 import { APPLICATIONS_PATH, COMPOSE_PATH } from "@/server/constants";
 import { createAppAuth } from "@octokit/auth-app";
 import { TRPCError } from "@trpc/server";
 import { Octokit } from "octokit";
 import { recreateDirectory } from "../filesystem/directory";
 import { spawnAsync } from "../process/spawnAsync";
+import type { InferResultType } from "@/server/types/with";
+import {
+	getGithubProvider,
+	type GithubProvider,
+} from "@/server/api/services/git-provider";
 
-export const authGithub = (admin: Admin) => {
-	if (!haveGithubRequirements(admin)) {
+export const authGithub = (githubProvider: GithubProvider) => {
+	if (!haveGithubRequirements(githubProvider)) {
 		throw new TRPCError({
 			code: "NOT_FOUND",
 			message: "Github Account not configured correctly",
@@ -19,9 +23,9 @@ export const authGithub = (admin: Admin) => {
 	const octokit = new Octokit({
 		authStrategy: createAppAuth,
 		auth: {
-			appId: admin?.githubAppId || 0,
-			privateKey: admin?.githubPrivateKey || "",
-			installationId: admin?.githubInstallationId,
+			appId: githubProvider?.githubAppId || 0,
+			privateKey: githubProvider?.githubPrivateKey || "",
+			installationId: githubProvider?.githubInstallationId,
 		},
 	});
 
@@ -40,11 +44,11 @@ export const getGithubToken = async (
 	return installation.token;
 };
 
-export const haveGithubRequirements = (admin: Admin) => {
+export const haveGithubRequirements = (githubProvider: GithubProvider) => {
 	return !!(
-		admin?.githubAppId &&
-		admin?.githubPrivateKey &&
-		admin?.githubInstallationId
+		githubProvider?.githubAppId &&
+		githubProvider?.githubPrivateKey &&
+		githubProvider?.githubInstallationId
 	);
 };
 
@@ -63,19 +67,24 @@ const getErrorCloneRequirements = (entity: {
 	return reasons;
 };
 
+export type ApplicationWithGithub = InferResultType<
+	"applications",
+	{ githubProvider: true }
+>;
 export const cloneGithubRepository = async (
-	admin: Admin,
-	entity: {
-		appName: string;
-		repository?: string | null;
-		owner?: string | null;
-		branch?: string | null;
-	},
+	entity: ApplicationWithGithub,
 	logPath: string,
 	isCompose = false,
 ) => {
 	const writeStream = createWriteStream(logPath, { flags: "a" });
-	const { appName, repository, owner, branch } = entity;
+	const { appName, repository, owner, branch, githubProviderId } = entity;
+
+	if (!githubProviderId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "GitHub Provider not found",
+		});
+	}
 
 	const requirements = getErrorCloneRequirements(entity);
 
@@ -92,9 +101,11 @@ export const cloneGithubRepository = async (
 			message: "Error: GitHub repository information is incomplete.",
 		});
 	}
+
+	const githubProvider = await getGithubProvider(githubProviderId);
 	const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
 	const outputPath = join(basePath, appName, "code");
-	const octokit = authGithub(admin);
+	const octokit = authGithub(githubProvider);
 	const token = await getGithubToken(octokit);
 	const repoclone = `github.com/${owner}/${repository}.git`;
 	await recreateDirectory(outputPath);
@@ -129,17 +140,21 @@ export const cloneGithubRepository = async (
 	}
 };
 
-export const cloneRawGithubRepository = async (entity: {
-	appName: string;
-	repository?: string | null;
-	owner?: string | null;
-	branch?: string | null;
-}) => {
-	const { appName, repository, owner, branch } = entity;
-	const admin = await findAdmin();
+export const cloneRawGithubRepository = async (
+	entity: ApplicationWithGithub,
+) => {
+	const { appName, repository, owner, branch, githubProviderId } = entity;
+
+	if (!githubProviderId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "GitHub Provider not found",
+		});
+	}
+	const githubProvider = await getGithubProvider(githubProviderId);
 	const basePath = COMPOSE_PATH;
 	const outputPath = join(basePath, appName, "code");
-	const octokit = authGithub(admin);
+	const octokit = authGithub(githubProvider);
 	const token = await getGithubToken(octokit);
 	const repoclone = `github.com/${owner}/${repository}.git`;
 	await recreateDirectory(outputPath);
diff --git a/apps/dokploy/server/utils/providers/gitlab.ts b/apps/dokploy/server/utils/providers/gitlab.ts
new file mode 100644
index 00000000..927d60fa
--- /dev/null
+++ b/apps/dokploy/server/utils/providers/gitlab.ts
@@ -0,0 +1,194 @@
+import { createWriteStream } from "node:fs";
+import { join } from "node:path";
+import { findAdmin } from "@/server/api/services/admin";
+import { APPLICATIONS_PATH, COMPOSE_PATH } from "@/server/constants";
+import { TRPCError } from "@trpc/server";
+import { recreateDirectory } from "../filesystem/directory";
+import { spawnAsync } from "../process/spawnAsync";
+import {
+	getGitlabProvider,
+	type GitlabProvider,
+	updateGitlabProvider,
+} from "@/server/api/services/git-provider";
+import type { InferResultType } from "@/server/types/with";
+
+export const refreshGitlabToken = async (gitlabProviderId: string) => {
+	const gitlabProvider = await getGitlabProvider(gitlabProviderId);
+	const currentTime = Math.floor(Date.now() / 1000);
+
+	const safetyMargin = 60;
+	if (
+		gitlabProvider.expiresAt &&
+		currentTime + safetyMargin < gitlabProvider.expiresAt
+	) {
+		console.log("Token still valid, no need to refresh");
+		return;
+	}
+
+	const response = await fetch("https://gitlab.com/oauth/token", {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/x-www-form-urlencoded",
+		},
+		body: new URLSearchParams({
+			grant_type: "refresh_token",
+			refresh_token: gitlabProvider.refreshToken as string,
+			client_id: gitlabProvider.applicationId as string,
+			client_secret: gitlabProvider.secret as string,
+		}),
+	});
+
+	if (!response.ok) {
+		throw new Error(`Failed to refresh token: ${response.statusText}`);
+	}
+
+	const data = await response.json();
+
+	const expiresAt = Math.floor(Date.now() / 1000) + data.expires_in;
+
+	await updateGitlabProvider(gitlabProviderId, {
+		accessToken: data.access_token,
+		refreshToken: data.refresh_token,
+		expiresAt,
+	});
+	return data;
+};
+
+export const haveGitlabRequirements = (gitlabProvider: GitlabProvider) => {
+	return !!(gitlabProvider?.accessToken && gitlabProvider?.refreshToken);
+};
+
+const getErrorCloneRequirements = (entity: {
+	repository?: string | null;
+	owner?: string | null;
+	branch?: string | null;
+}) => {
+	const reasons: string[] = [];
+	const { repository, owner, branch } = entity;
+
+	if (!repository) reasons.push("1. Repository not assigned.");
+	if (!owner) reasons.push("2. Owner not specified.");
+	if (!branch) reasons.push("3. Branch not defined.");
+
+	return reasons;
+};
+
+export type ApplicationWithGitlab = InferResultType<
+	"applications",
+	{ gitlabProvider: true }
+>;
+
+export const cloneGitlabRepository = async (
+	entity: ApplicationWithGitlab,
+	logPath: string,
+	isCompose = false,
+) => {
+	const writeStream = createWriteStream(logPath, { flags: "a" });
+	const {
+		appName,
+		gitlabRepository,
+		gitlabOwner,
+		gitlabBranch,
+		gitlabProviderId,
+		gitlabProvider,
+	} = entity;
+
+	if (!gitlabProviderId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Gitlab Provider not found",
+		});
+	}
+
+	await refreshGitlabToken(gitlabProviderId);
+
+	const requirements = getErrorCloneRequirements(entity);
+
+	// Check if requirements are met
+	if (requirements.length > 0) {
+		writeStream.write(
+			`\nGitLab Repository configuration failed for application: ${appName}\n`,
+		);
+		writeStream.write("Reasons:\n");
+		writeStream.write(requirements.join("\n"));
+		writeStream.end();
+		throw new TRPCError({
+			code: "BAD_REQUEST",
+			message: "Error: GitLab repository information is incomplete.",
+		});
+	}
+	const basePath = isCompose ? COMPOSE_PATH : APPLICATIONS_PATH;
+	const outputPath = join(basePath, appName, "code");
+	await recreateDirectory(outputPath);
+	const repoclone = `gitlab.com/${gitlabOwner}/${gitlabRepository}.git`;
+	const cloneUrl = `https://oauth2:${gitlabProvider?.accessToken}@${repoclone}`;
+
+	try {
+		writeStream.write(`\nClonning Repo ${repoclone} to ${outputPath}: ✅\n`);
+		await spawnAsync(
+			"git",
+			[
+				"clone",
+				"--branch",
+				gitlabBranch!,
+				"--depth",
+				"1",
+				cloneUrl,
+				outputPath,
+				"--progress",
+			],
+			(data) => {
+				if (writeStream.writable) {
+					writeStream.write(data);
+				}
+			},
+		);
+		writeStream.write(`\nCloned ${repoclone}: ✅\n`);
+	} catch (error) {
+		writeStream.write(`ERROR Clonning: ${error}: ❌`);
+		throw error;
+	} finally {
+		writeStream.end();
+	}
+};
+
+export const cloneRawGitlabRepository = async (
+	entity: ApplicationWithGitlab,
+) => {
+	const {
+		appName,
+		gitlabRepository,
+		gitlabOwner,
+		gitlabBranch,
+		gitlabProviderId,
+		gitlabProvider,
+	} = entity;
+
+	if (!gitlabProviderId) {
+		throw new TRPCError({
+			code: "NOT_FOUND",
+			message: "Gitlab Provider not found",
+		});
+	}
+
+	await refreshGitlabToken(gitlabProviderId);
+	const basePath = COMPOSE_PATH;
+	const outputPath = join(basePath, appName, "code");
+	await recreateDirectory(outputPath);
+	const repoclone = `gitlab.com/${gitlabOwner}/${gitlabRepository}.git`;
+	const cloneUrl = `https://oauth2:${gitlabProvider?.accessToken}@${repoclone}`;
+	try {
+		await spawnAsync("git", [
+			"clone",
+			"--branch",
+			gitlabBranch!,
+			"--depth",
+			"1",
+			cloneUrl,
+			outputPath,
+			"--progress",
+		]);
+	} catch (error) {
+		throw error;
+	}
+};