feat: add docker registry upload

server/api/root.ts

@@ -20,6 +20,7 @@ import { securityRouter } from "./routers/security";
 import { portRouter } from "./routers/port";
 import { adminRouter } from "./routers/admin";
 import { dockerRouter } from "./routers/docker";
+import { registryRouter } from "./routers/registry";
 /**
  * This is the primary router for your server.
  *
@@ -47,6 +48,7 @@ export const appRouter = createTRPCRouter({
 	security: securityRouter,
 	redirects: redirectsRouter,
 	port: portRouter,
+	registry: registryRouter,
 });
 
 // export type definition of API

server/api/routers/registry.ts

@@ -7,12 +7,16 @@ import {
 } from "@/server/db/schema";
 import {
 	createRegistry,
+	findAllRegistry,
 	findRegistryById,
 	removeRegistry,
 	updaterRegistry,
 } from "../services/registry";
 import { adminProcedure, createTRPCRouter, protectedProcedure } from "../trpc";
 import { TRPCError } from "@trpc/server";
+import { manageRegistry } from "@/server/utils/traefik/registry";
+import { initializeRegistry } from "@/server/setup/registry-setup";
+import { docker } from "@/server/constants";
 
 export const registryRouter = createTRPCRouter({
 	create: adminProcedure
@@ -42,25 +46,41 @@ export const registryRouter = createTRPCRouter({
 
 			return true;
 		}),
+	all: protectedProcedure.query(async () => {
+		return await findAllRegistry();
+	}),
 	findOne: adminProcedure.input(apiFindOneRegistry).query(async ({ input }) => {
 		return await findRegistryById(input.registryId);
 	}),
+	testRegistry: protectedProcedure
+		.input(apiCreateRegistry)
+		.mutation(async ({ input }) => {
+			try {
+				const result = await docker.checkAuth({
+					username: input.username,
+					password: input.password,
+					serveraddress: input.registryUrl,
+				});
 
-	enableSelfHostedRegistry: protectedProcedure
+				return true;
+			} catch (error) {
+				return false;
+			}
+		}),
+
+	enableSelfHostedRegistry: adminProcedure
 		.input(apiEnableSelfHostedRegistry)
 		.mutation(async ({ input }) => {
-			// return await createRegistry({
-			//     username:"CUSTOM"
-			//     adminId: input.adminId,
-			// });
-			// const application = await findRegistryById(input.registryId);
-			// const result = await db
-			//     .update(registry)
-			//     .set({
-			//         selfHosted: true,
-			//     })
-			//     .where(eq(registry.registryId, input.registryId))
-			//     .returning();
-			// return result[0];
+			const selfHostedRegistry = await createRegistry({
+				...input,
+				registryName: "Self Hosted Registry",
+				registryType: "selfHosted",
+				imagePrefix: null,
+			});
+
+			await manageRegistry(selfHostedRegistry);
+			await initializeRegistry(input.username, input.password);
+
+			return selfHostedRegistry;
 		}),
 });

server/api/services/application.ts

@@ -61,6 +61,7 @@ export const findApplicationById = async (applicationId: string) => {
 			redirects: true,
 			security: true,
 			ports: true,
+			registry: true,
 		},
 	});
 	if (!application) {

server/api/services/registry.ts

@@ -2,14 +2,20 @@ import { type apiCreateRegistry, registry } from "@/server/db/schema";
 import { TRPCError } from "@trpc/server";
 import { db } from "@/server/db";
 import { eq } from "drizzle-orm";
+import { findAdmin } from "./admin";
+import { removeSelfHostedRegistry } from "@/server/utils/traefik/registry";
+import { removeService } from "@/server/utils/docker/utils";
 
 export type Registry = typeof registry.$inferSelect;
 
 export const createRegistry = async (input: typeof apiCreateRegistry._type) => {
+	const admin = await findAdmin();
+
 	const newRegistry = await db
 		.insert(registry)
 		.values({
 			...input,
+			adminId: admin.adminId,
 		})
 		.returning()
 		.then((value) => value[0]);
@@ -38,6 +44,11 @@ export const removeRegistry = async (registryId: string) => {
 			});
 		}
 
+		if (response.registryType === "selfHosted") {
+			await removeSelfHostedRegistry();
+			await removeService("dokploy-registry");
+		}
+
 		return response;
 	} catch (error) {
 		throw new TRPCError({
@@ -82,3 +93,8 @@ export const findRegistryById = async (registryId: string) => {
 	}
 	return registryResponse;
 };
+
+export const findAllRegistry = async () => {
+	const registryResponse = await db.query.registry.findMany();
+	return registryResponse;
+};

server/constants/index.ts

@@ -11,5 +11,6 @@ export const LOGS_PATH = `${BASE_PATH}/logs`;
 export const APPLICATIONS_PATH = `${BASE_PATH}/applications`;
 export const SSH_PATH = `${BASE_PATH}/ssh`;
 export const CERTIFICATES_PATH = `${DYNAMIC_TRAEFIK_PATH}/certificates`;
+export const REGISTRY_PATH = `${DYNAMIC_TRAEFIK_PATH}/registry`;
 export const MONITORING_PATH = `${BASE_PATH}/monitoring`;
 export const docker = new Docker();

server/db/drizzle.config.ts

@@ -1,13 +1,14 @@
-import type { Config } from "drizzle-kit";
+import { defineConfig } from "drizzle-kit";
 
-console.log("> Generating PG Schema:", process.env.DATABASE_URL);
-export default {
+export default defineConfig({
 	schema: "./server/db/schema/index.ts",
-	driver: "pg",
+	dialect: "postgresql",
 	dbCredentials: {
-		connectionString: process.env.DATABASE_URL || "",
+		url: process.env.DATABASE_URL || "",
 	},
-	verbose: true,
-	strict: true,
 	out: "drizzle",
-} satisfies Config;
+	migrations: {
+		table: "migrations",
+		schema: "public",
+	},
+});

server/db/schema/application.ts

@@ -12,6 +12,7 @@ import { applicationStatus } from "./shared";
 import { ports } from "./port";
 import { boolean, integer, pgEnum, pgTable, text } from "drizzle-orm/pg-core";
 import { generateAppName } from "./utils";
+import { registry } from "./registry";
 
 export const sourceType = pgEnum("sourceType", ["docker", "git", "github"]);
 
@@ -60,6 +61,7 @@ export const applications = pgTable("application", {
 	customGitBuildPath: text("customGitBuildPath"),
 	customGitSSHKey: text("customGitSSHKey"),
 	dockerfile: text("dockerfile"),
+	replicas: integer("replicas").default(1).notNull(),
 	applicationStatus: applicationStatus("applicationStatus")
 		.notNull()
 		.default("idle"),
@@ -67,6 +69,9 @@ export const applications = pgTable("application", {
 	createdAt: text("createdAt")
 		.notNull()
 		.$defaultFn(() => new Date().toISOString()),
+	registryId: text("registryId").references(() => registry.registryId, {
+		onDelete: "set null",
+	}),
 	projectId: text("projectId")
 		.notNull()
 		.references(() => projects.projectId, { onDelete: "cascade" }),
@@ -85,6 +90,10 @@ export const applicationsRelations = relations(
 		redirects: many(redirects),
 		security: many(security),
 		ports: many(ports),
+		registry: one(registry, {
+			fields: [applications.registryId],
+			references: [registry.registryId],
+		}),
 	}),
 );
 

server/db/schema/registry.ts

@@ -5,6 +5,7 @@ import { boolean, pgEnum, pgTable, text, timestamp } from "drizzle-orm/pg-core";
 import { auth } from "./auth";
 import { admins } from "./admin";
 import { z } from "zod";
+import { applications } from "./application";
 /**
  * This is an example of how to use the multi-project schema feature of Drizzle ORM. Use the same
  * database instance for multiple projects.
@@ -19,6 +20,7 @@ export const registry = pgTable("registry", {
 		.primaryKey()
 		.$defaultFn(() => nanoid()),
 	registryName: text("registryName").notNull(),
+	imagePrefix: text("imagePrefix"),
 	username: text("username").notNull(),
 	password: text("password").notNull(),
 	registryUrl: text("registryUrl").notNull(),
@@ -31,11 +33,12 @@ export const registry = pgTable("registry", {
 		.references(() => admins.adminId, { onDelete: "cascade" }),
 });
 
-export const registryRelations = relations(registry, ({ one }) => ({
+export const registryRelations = relations(registry, ({ one, many }) => ({
 	admin: one(admins, {
 		fields: [registry.adminId],
 		references: [admins.adminId],
 	}),
+	applications: many(applications),
 }));
 
 const createSchema = createInsertSchema(registry, {
@@ -45,6 +48,8 @@ const createSchema = createInsertSchema(registry, {
 	registryUrl: z.string().min(1),
 	adminId: z.string().min(1),
 	registryId: z.string().min(1),
+	registryType: z.enum(["selfHosted", "cloud"]),
+	imagePrefix: z.string().nullable().optional(),
 });
 
 export const apiCreateRegistry = createSchema
@@ -53,8 +58,9 @@ export const apiCreateRegistry = createSchema
 		registryName: z.string().min(1),
 		username: z.string().min(1),
 		password: z.string().min(1),
-		registryUrl: z.string().min(1),
-		adminId: z.string().min(1),
+		registryUrl: z.string(),
+		registryType: z.enum(["selfHosted", "cloud"]),
+		imagePrefix: z.string().nullable().optional(),
 	})
 	.required();
 
@@ -82,6 +88,8 @@ export const apiUpdateRegistry = createSchema
 
 export const apiEnableSelfHostedRegistry = createSchema
 	.pick({
-		adminId: true,
+		registryUrl: true,
+		username: true,
+		password: true,
 	})
 	.required();

server/setup/registry-setup.ts

@@ -0,0 +1,89 @@
+import type { CreateServiceOptions } from "dockerode";
+import { docker, REGISTRY_PATH } from "../constants";
+import { pullImage } from "../utils/docker/utils";
+import { execAsync } from "../utils/process/execAsync";
+import { generateRandomPassword } from "../auth/random-password";
+
+export const initializeRegistry = async (
+	username: string,
+	password: string,
+) => {
+	const imageName = "registry:2.8.3";
+	const containerName = "dokploy-registry";
+	await generatePassword(username, password);
+	const randomPass = await generateRandomPassword();
+	const settings: CreateServiceOptions = {
+		Name: containerName,
+		TaskTemplate: {
+			ContainerSpec: {
+				Image: imageName,
+				Env: [
+					"REGISTRY_STORAGE_DELETE_ENABLED=true",
+					"REGISTRY_AUTH=htpasswd",
+					"REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm",
+					"REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd",
+					`REGISTRY_HTTP_SECRET=${randomPass.hashedPassword}`,
+				],
+				Mounts: [
+					{
+						Type: "bind",
+						Source: `${REGISTRY_PATH}/htpasswd`,
+						Target: "/auth/htpasswd",
+						ReadOnly: true,
+					},
+					{
+						Type: "volume",
+						Source: "registry-data",
+						Target: "/var/lib/registry",
+						ReadOnly: false,
+					},
+				],
+			},
+			Networks: [{ Target: "dokploy-network" }],
+			RestartPolicy: {
+				Condition: "on-failure",
+			},
+		},
+		Mode: {
+			Replicated: {
+				Replicas: 1,
+			},
+		},
+		EndpointSpec: {
+			Ports: [
+				{
+					TargetPort: 5000,
+					PublishedPort: 5000,
+					Protocol: "tcp",
+					PublishMode: "host",
+				},
+			],
+		},
+	};
+	try {
+		await pullImage(imageName);
+
+		const service = docker.getService(containerName);
+		const inspect = await service.inspect();
+		await service.update({
+			version: Number.parseInt(inspect.Version.Index),
+			...settings,
+		});
+		console.log("Registry Started ✅");
+	} catch (error) {
+		await docker.createService(settings);
+		console.log("Registry Not Found: Starting ✅");
+	}
+};
+
+const generatePassword = async (username: string, password: string) => {
+	try {
+		const command = `htpasswd -nbB ${username} "${password}" > ${REGISTRY_PATH}/htpasswd`;
+		const result = await execAsync(command);
+		console.log("Password generated ✅");
+		return result.stdout.trim();
+	} catch (error) {
+		console.error("Error generating password:", error);
+		return null;
+	}
+};

server/utils/builders/index.ts

@@ -13,6 +13,7 @@ import { buildCustomDocker } from "./docker-file";
 import { buildHeroku } from "./heroku";
 import { buildNixpacks } from "./nixpacks";
 import { buildPaketo } from "./paketo";
+import { uploadImage } from "../cluster/upload";
 
 // NIXPACKS codeDirectory = where is the path of the code directory
 // HEROKU codeDirectory = where is the path of the code directory
@@ -20,7 +21,7 @@ import { buildPaketo } from "./paketo";
 // DOCKERFILE codeDirectory = where is the exact path of the (Dockerfile)
 export type ApplicationNested = InferResultType<
 	"applications",
-	{ mounts: true; security: true; redirects: true; ports: true }
+	{ mounts: true; security: true; redirects: true; ports: true; registry: true }
 >;
 export const buildApplication = async (
 	application: ApplicationNested,
@@ -42,6 +43,10 @@ export const buildApplication = async (
 		} else if (buildType === "dockerfile") {
 			await buildCustomDocker(application, writeStream);
 		}
+
+		if (application.registryId) {
+			await uploadImage(application, writeStream);
+		}
 		await mechanizeDockerContainer(application);
 		writeStream.write("Docker Deployed: ✅");
 	} catch (error) {
@@ -67,6 +72,7 @@ export const mechanizeDockerContainer = async (
 		cpuReservation,
 		command,
 		ports,
+		replicas,
 	} = application;
 
 	const resources = calculateResources({
@@ -104,7 +110,7 @@ export const mechanizeDockerContainer = async (
 		},
 		Mode: {
 			Replicated: {
-				Replicas: 1,
+				Replicas: replicas,
 			},
 		},
 		EndpointSpec: {

server/utils/cluster/upload.ts

@@ -0,0 +1,67 @@
+import type { ApplicationNested } from "../builders";
+import { execAsync } from "../process/execAsync";
+import { spawnAsync } from "../process/spawnAsync";
+import type { WriteStream } from "node:fs";
+
+export const uploadImage = async (
+	application: ApplicationNested,
+	writeStream: WriteStream,
+) => {
+	const registry = application.registry;
+
+	if (!registry) {
+		throw new Error("Registry not found");
+	}
+
+	const { registryUrl, imagePrefix } = registry;
+	const { appName } = application;
+	const imageName = `${appName}:latest`;
+
+	let finalURL = registryUrl;
+
+	let registryTag = `${registryUrl}/${imageName}`;
+
+	if (imagePrefix) {
+		registryTag = `${registryUrl}/${imagePrefix}/${imageName}`;
+	}
+
+	//  registry.digitalocean.com/<my-registry>/<my-image>
+	//  index.docker.io/siumauricio/app-parse-multi-byte-port-e32uh7:latest
+	if (registry.registryType === "selfHosted") {
+		finalURL =
+			process.env.NODE_ENV === "development" ? "localhost:5000" : registryUrl;
+		registryTag = `${finalURL}/${imageName}`;
+	}
+
+	try {
+		console.log(finalURL, registryTag);
+		writeStream.write(
+			`📦 [Enabled Registry] Uploading image to ${registry.registryType} | ${registryTag} | ${finalURL}\n`,
+		);
+
+		await spawnAsync(
+			"docker",
+			["login", finalURL, "-u", registry.username, "-p", registry.password],
+			(data) => {
+				if (writeStream.writable) {
+					writeStream.write(data);
+				}
+			},
+		);
+
+		await spawnAsync("docker", ["tag", imageName, registryTag], (data) => {
+			if (writeStream.writable) {
+				writeStream.write(data);
+			}
+		});
+
+		await spawnAsync("docker", ["push", registryTag], (data) => {
+			if (writeStream.writable) {
+				writeStream.write(data);
+			}
+		});
+	} catch (error) {
+		console.log(error);
+		throw error;
+	}
+};

server/utils/traefik/domain.ts

@@ -47,10 +47,7 @@ export const removeDomain = async (appName: string, uniqueKey: number) => {
 	}
 };
 
-export const createRouterConfig = async (
-	app: ApplicationNested,
-	domain: Domain,
-) => {
+const createRouterConfig = async (app: ApplicationNested, domain: Domain) => {
 	const { appName, redirects, security } = app;
 	const { certificateType } = domain;
 

server/utils/traefik/registry.ts

@@ -0,0 +1,67 @@
+import { loadOrCreateConfig } from "./application";
+import type { FileConfig, HttpRouter } from "./file-types";
+import type { Registry } from "@/server/api/services/registry";
+import { removeDirectoryIfExistsContent } from "../filesystem/directory";
+import { REGISTRY_PATH } from "@/server/constants";
+import { dump } from "js-yaml";
+import { join } from "node:path";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+
+export const manageRegistry = async (registry: Registry) => {
+	if (!existsSync(REGISTRY_PATH)) {
+		mkdirSync(REGISTRY_PATH, { recursive: true });
+	}
+	const appName = "dokploy-registry";
+	const config: FileConfig = loadOrCreateConfig(appName);
+	const serviceName = `${appName}-service`;
+	const routerName = `${appName}-router`;
+
+	config.http = config.http || { routers: {}, services: {} };
+	config.http.routers = config.http.routers || {};
+	config.http.services = config.http.services || {};
+
+	config.http.routers[routerName] = await createRegistryRouterConfig(registry);
+
+	config.http.services[serviceName] = {
+		loadBalancer: {
+			servers: [{ url: `http://${appName}:5000` }],
+			passHostHeader: true,
+		},
+	};
+
+	const yamlConfig = dump(config);
+	const configFile = join(REGISTRY_PATH, "registry.yml");
+	writeFileSync(configFile, yamlConfig);
+};
+
+export const removeSelfHostedRegistry = async () => {
+	await removeDirectoryIfExistsContent(REGISTRY_PATH);
+};
+
+const createRegistryRouterConfig = async (registry: Registry) => {
+	const { registryUrl } = registry;
+	const url =
+		process.env.NODE_ENV === "production"
+			? registryUrl
+			: "dokploy-registry.docker.localhost";
+	const routerConfig: HttpRouter = {
+		rule: `Host(\`${url}\`)`,
+		service: "dokploy-registry-service",
+		...(process.env.NODE_ENV === "production"
+			? {
+					middlewares: ["redirect-to-https"],
+				}
+			: {}),
+		entryPoints: [
+			"web",
+			...(process.env.NODE_ENV === "production" ? ["websecure"] : []),
+		],
+		...(process.env.NODE_ENV === "production"
+			? {
+					tls: { certResolver: "letsencrypt" },
+				}
+			: {}),
+	};
+
+	return routerConfig;
+};