Merge branch 'canary' into feat/mongo-replica-sets

This commit is contained in:
Mauricio Siu
2024-12-25 03:27:00 -06:00
126 changed files with 9622 additions and 1542 deletions

View File

@@ -7,6 +7,8 @@ import {
cleanUpSystemPrune,
cleanUpUnusedImages,
} from "../docker/utils";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup";
import { runMariadbBackup } from "./mariadb";
import { runMongoBackup } from "./mongo";
import { runMySqlBackup } from "./mysql";
@@ -25,14 +27,15 @@ export const initCronJobs = async () => {
await cleanUpUnusedImages();
await cleanUpDockerBuilder();
await cleanUpSystemPrune();
await sendDockerCleanupNotifications(admin.adminId);
});
}
const servers = await getAllServers();
for (const server of servers) {
const { appName, serverId } = server;
if (serverId) {
const { appName, serverId, enableDockerCleanup } = server;
if (enableDockerCleanup) {
scheduleJob(serverId, "0 0 * * *", async () => {
console.log(
`SERVER-BACKUP[${new Date().toLocaleString()}] Running Cleanup ${appName}`,
@@ -40,12 +43,17 @@ export const initCronJobs = async () => {
await cleanUpUnusedImages(serverId);
await cleanUpDockerBuilder(serverId);
await cleanUpSystemPrune(serverId);
await sendDockerCleanupNotifications(
admin.adminId,
`Docker cleanup for Server ${appName}`,
);
});
}
}
const pgs = await db.query.postgres.findMany({
with: {
project: true,
backups: {
with: {
destination: true,
@@ -61,18 +69,39 @@ export const initCronJobs = async () => {
for (const backup of pg.backups) {
const { schedule, backupId, enabled } = backup;
if (enabled) {
scheduleJob(backupId, schedule, async () => {
console.log(
`PG-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
runPostgresBackup(pg, backup);
});
try {
scheduleJob(backupId, schedule, async () => {
console.log(
`PG-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
runPostgresBackup(pg, backup);
});
await sendDatabaseBackupNotifications({
applicationName: pg.name,
projectName: pg.project.name,
databaseType: "postgres",
type: "success",
adminId: pg.project.adminId,
});
} catch (error) {
await sendDatabaseBackupNotifications({
applicationName: pg.name,
projectName: pg.project.name,
databaseType: "postgres",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
adminId: pg.project.adminId,
});
}
}
}
}
const mariadbs = await db.query.mariadb.findMany({
with: {
project: true,
backups: {
with: {
destination: true,
@@ -89,18 +118,38 @@ export const initCronJobs = async () => {
for (const backup of maria.backups) {
const { schedule, backupId, enabled } = backup;
if (enabled) {
scheduleJob(backupId, schedule, async () => {
console.log(
`MARIADB-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMariadbBackup(maria, backup);
});
try {
scheduleJob(backupId, schedule, async () => {
console.log(
`MARIADB-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMariadbBackup(maria, backup);
});
await sendDatabaseBackupNotifications({
applicationName: maria.name,
projectName: maria.project.name,
databaseType: "mariadb",
type: "success",
adminId: maria.project.adminId,
});
} catch (error) {
await sendDatabaseBackupNotifications({
applicationName: maria.name,
projectName: maria.project.name,
databaseType: "mariadb",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
adminId: maria.project.adminId,
});
}
}
}
}
const mongodbs = await db.query.mongo.findMany({
with: {
project: true,
backups: {
with: {
destination: true,
@@ -117,18 +166,38 @@ export const initCronJobs = async () => {
for (const backup of mongo.backups) {
const { schedule, backupId, enabled } = backup;
if (enabled) {
scheduleJob(backupId, schedule, async () => {
console.log(
`MONGO-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMongoBackup(mongo, backup);
});
try {
scheduleJob(backupId, schedule, async () => {
console.log(
`MONGO-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMongoBackup(mongo, backup);
});
await sendDatabaseBackupNotifications({
applicationName: mongo.name,
projectName: mongo.project.name,
databaseType: "mongodb",
type: "success",
adminId: mongo.project.adminId,
});
} catch (error) {
await sendDatabaseBackupNotifications({
applicationName: mongo.name,
projectName: mongo.project.name,
databaseType: "mongodb",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
adminId: mongo.project.adminId,
});
}
}
}
}
const mysqls = await db.query.mysql.findMany({
with: {
project: true,
backups: {
with: {
destination: true,
@@ -145,12 +214,31 @@ export const initCronJobs = async () => {
for (const backup of mysql.backups) {
const { schedule, backupId, enabled } = backup;
if (enabled) {
scheduleJob(backupId, schedule, async () => {
console.log(
`MYSQL-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMySqlBackup(mysql, backup);
});
try {
scheduleJob(backupId, schedule, async () => {
console.log(
`MYSQL-SERVER[${new Date().toLocaleString()}] Running Backup ${backupId}`,
);
await runMySqlBackup(mysql, backup);
});
await sendDatabaseBackupNotifications({
applicationName: mysql.name,
projectName: mysql.project.name,
databaseType: "mysql",
type: "success",
adminId: mysql.project.adminId,
});
} catch (error) {
await sendDatabaseBackupNotifications({
applicationName: mysql.name,
projectName: mysql.project.name,
databaseType: "mysql",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
adminId: mysql.project.adminId,
});
}
}
}
}

View File

@@ -48,6 +48,7 @@ Compose Type: ${composeType} ✅`;
writeStream.write(`\n${logBox}\n`);
const projectPath = join(COMPOSE_PATH, compose.appName, "code");
await spawnAsync(
"docker",
[...command.split(" ")],
@@ -67,7 +68,7 @@ Compose Type: ${composeType} ✅`;
writeStream.write("Docker Compose Deployed: ✅");
} catch (error) {
writeStream.write("Error ❌");
writeStream.write(`Error ❌ ${(error as Error).message}`);
throw error;
} finally {
writeStream.end();
@@ -144,6 +145,10 @@ const sanitizeCommand = (command: string) => {
export const createCommand = (compose: ComposeNested) => {
const { composeType, appName, sourceType } = compose;
if (compose.command) {
return `${sanitizeCommand(compose.command)}`;
}
const path =
sourceType === "raw" ? "docker-compose.yml" : compose.composePath;
let command = "";
@@ -154,12 +159,6 @@ export const createCommand = (compose: ComposeNested) => {
command = `stack deploy -c ${path} ${appName} --prune`;
}
const customCommand = sanitizeCommand(compose.command);
if (customCommand) {
command = `${command} ${customCommand}`;
}
return command;
};

View File

@@ -2,6 +2,7 @@ import { createWriteStream } from "node:fs";
import { join } from "node:path";
import type { InferResultType } from "@dokploy/server/types/with";
import type { CreateServiceOptions } from "dockerode";
import { nanoid } from "nanoid";
import { uploadImage, uploadImageRemoteCommand } from "../cluster/upload";
import {
calculateResources,
@@ -17,7 +18,6 @@ import { buildHeroku, getHerokuCommand } from "./heroku";
import { buildNixpacks, getNixpacksCommand } from "./nixpacks";
import { buildPaketo, getPaketoCommand } from "./paketo";
import { buildStatic, getStaticCommand } from "./static";
import { nanoid } from "nanoid";
// NIXPACKS codeDirectory = where is the path of the code directory
// HEROKU codeDirectory = where is the path of the code directory
@@ -211,21 +211,21 @@ const getImageName = (application: ApplicationNested) => {
}
if (registry) {
return join(registry.imagePrefix || "", appName);
return join(registry.registryUrl, registry.imagePrefix || "", appName);
}
return `${appName}:latest`;
};
const getAuthConfig = (application: ApplicationNested) => {
const { registry, username, password, sourceType } = application;
const { registry, username, password, sourceType, registryUrl } = application;
if (sourceType === "docker") {
if (username && password) {
return {
password,
username,
serveraddress: "https://index.docker.io/v1/",
serveraddress: registryUrl || "",
};
}
} else if (registry) {

View File

@@ -1,5 +1,5 @@
import type { WriteStream } from "node:fs";
import { join } from "node:path";
import path, { join } from "node:path";
import type { ApplicationNested } from "../builders";
import { spawnAsync } from "../process/spawnAsync";
@@ -13,27 +13,32 @@ export const uploadImage = async (
throw new Error("Registry not found");
}
const { registryUrl, imagePrefix, registryType } = registry;
const { registryUrl, imagePrefix } = registry;
const { appName } = application;
const imageName = `${appName}:latest`;
const finalURL = registryUrl;
const registryTag = join(imagePrefix || "", imageName);
const registryTag = path
.join(registryUrl, join(imagePrefix || "", imageName))
.replace(/\/+/g, "/");
try {
writeStream.write(
`📦 [Enabled Registry] Uploading image to ${registry.registryType} | ${registryTag} | ${finalURL}\n`,
`📦 [Enabled Registry] Uploading image to ${registry.registryType} | ${imageName} | ${finalURL}\n`,
);
await spawnAsync(
const loginCommand = spawnAsync(
"docker",
["login", finalURL, "-u", registry.username, "-p", registry.password],
["login", finalURL, "-u", registry.username, "--password-stdin"],
(data) => {
if (writeStream.writable) {
writeStream.write(data);
}
},
);
loginCommand.child?.stdin?.write(registry.password);
loginCommand.child?.stdin?.end();
await loginCommand;
await spawnAsync("docker", ["tag", imageName, registryTag], (data) => {
if (writeStream.writable) {
@@ -68,22 +73,23 @@ export const uploadImageRemoteCommand = (
const finalURL = registryUrl;
const registryTag = join(imagePrefix || "", imageName);
const registryTag = path
.join(registryUrl, join(imagePrefix || "", imageName))
.replace(/\/+/g, "/");
try {
const command = `
echo "📦 [Enabled Registry] Uploading image to '${registry.registryType}' | '${registryTag}'" >> ${logPath};
docker login ${finalURL} -u ${registry.username} -p ${registry.password} >> ${logPath} 2>> ${logPath} || {
echo "${registry.password}" | docker login ${finalURL} -u ${registry.username} --password-stdin >> ${logPath} 2>> ${logPath} || {
echo "❌ DockerHub Failed" >> ${logPath};
exit 1;
}
echo "✅ DockerHub Login Success" >> ${logPath};
echo "✅ Registry Login Success" >> ${logPath};
docker tag ${imageName} ${registryTag} >> ${logPath} 2>> ${logPath} || {
echo "❌ Error tagging image" >> ${logPath};
exit 1;
}
echo "✅ Image Tagged" >> ${logPath};
echo "✅ Image Tagged" >> ${logPath};
docker push ${registryTag} 2>> ${logPath} || {
echo "❌ Error pushing image" >> ${logPath};
exit 1;
@@ -92,7 +98,6 @@ export const uploadImageRemoteCommand = (
`;
return command;
} catch (error) {
console.log(error);
throw error;
}
};

View File

@@ -238,9 +238,11 @@ export const startServiceRemote = async (serverId: string, appName: string) => {
export const removeService = async (
appName: string,
serverId?: string | null,
deleteVolumes = false,
) => {
try {
const command = `docker service rm ${appName}`;
if (serverId) {
await execAsyncRemote(serverId, command);
} else {

View File

@@ -28,7 +28,7 @@ export const sendBuildErrorNotifications = async ({
adminId,
}: Props) => {
const date = new Date();
const unixDate = ~~((Number(date)) / 1000);
const unixDate = ~~(Number(date) / 1000);
const notificationList = await db.query.notifications.findMany({
where: and(
eq(notifications.appBuildError, true),
@@ -59,46 +59,49 @@ export const sendBuildErrorNotifications = async ({
}
if (discord) {
const decorate = (decoration: string, text: string) =>
`${discord.decoration ? decoration : ""} ${text}`.trim();
await sendDiscordNotification(discord, {
title: "> `⚠️` - Build Failed",
title: decorate(">", "`⚠️` Build Failed"),
color: 0xed4245,
fields: [
{
name: "`🛠️`・Project",
name: decorate("`🛠️`", "Project"),
value: projectName,
inline: true,
},
{
name: "`⚙️`・Application",
name: decorate("`⚙️`", "Application"),
value: applicationName,
inline: true,
},
{
name: "`❔`・Type",
name: decorate("`❔`", "Type"),
value: applicationType,
inline: true,
},
{
name: "`📅`・Date",
name: decorate("`📅`", "Date"),
value: `<t:${unixDate}:D>`,
inline: true,
},
{
name: "`⌚`・Time",
name: decorate("`⌚`", "Time"),
value: `<t:${unixDate}:t>`,
inline: true,
},
{
name: "`❓`・Type",
name: decorate("`❓`", "Type"),
value: "Failed",
inline: true,
},
{
name: "`⚠️`・Error Message",
name: decorate("`⚠️`", "Error Message"),
value: `\`\`\`${errorMessage}\`\`\``,
},
{
name: "`🧷`・Build Link",
name: decorate("`🧷`", "Build Link"),
value: `[Click here to access build link](${buildLink})`,
},
],
@@ -114,15 +117,15 @@ export const sendBuildErrorNotifications = async ({
telegram,
`
<b>⚠️ Build Failed</b>
<b>Project:</b> ${projectName}
<b>Application:</b> ${applicationName}
<b>Type:</b> ${applicationType}
<b>Time:</b> ${date.toLocaleString()}
<b>Error:</b>
<pre>${errorMessage}</pre>
<b>Build Details:</b> ${buildLink}
`,
);

View File

@@ -26,7 +26,7 @@ export const sendBuildSuccessNotifications = async ({
adminId,
}: Props) => {
const date = new Date();
const unixDate = ~~((Number(date)) / 1000);
const unixDate = ~~(Number(date) / 1000);
const notificationList = await db.query.notifications.findMany({
where: and(
eq(notifications.appDeploy, true),
@@ -57,42 +57,45 @@ export const sendBuildSuccessNotifications = async ({
}
if (discord) {
const decorate = (decoration: string, text: string) =>
`${discord.decoration ? decoration : ""} ${text}`.trim();
await sendDiscordNotification(discord, {
title: "> `✅` - Build Success",
title: "> `✅` Build Success",
color: 0x57f287,
fields: [
{
name: "`🛠️`・Project",
name: decorate("`🛠️`", "Project"),
value: projectName,
inline: true,
},
{
name: "`⚙️`・Application",
name: decorate("`⚙️`", "Application"),
value: applicationName,
inline: true,
},
{
name: "`❔`・Application Type",
name: decorate("`❔`", "Type"),
value: applicationType,
inline: true,
},
{
name: "`📅`・Date",
name: decorate("`📅`", "Date"),
value: `<t:${unixDate}:D>`,
inline: true,
},
{
name: "`⌚`・Time",
name: decorate("`⌚`", "Time"),
value: `<t:${unixDate}:t>`,
inline: true,
},
{
name: "`❓`・Type",
name: decorate("`❓`", "Type"),
value: "Successful",
inline: true,
},
{
name: "`🧷`・Build Link",
name: decorate("`🧷`", "Build Link"),
value: `[Click here to access build link](${buildLink})`,
},
],
@@ -108,12 +111,12 @@ export const sendBuildSuccessNotifications = async ({
telegram,
`
<b>✅ Build Success</b>
<b>Project:</b> ${projectName}
<b>Application:</b> ${applicationName}
<b>Type:</b> ${applicationType}
<b>Time:</b> ${date.toLocaleString()}
<b>Build Details:</b> ${buildLink}
`,
);

View File

@@ -26,7 +26,7 @@ export const sendDatabaseBackupNotifications = async ({
errorMessage?: string;
}) => {
const date = new Date();
const unixDate = ~~((Number(date)) / 1000);
const unixDate = ~~(Number(date) / 1000);
const notificationList = await db.query.notifications.findMany({
where: and(
eq(notifications.databaseBackup, true),
@@ -62,40 +62,43 @@ export const sendDatabaseBackupNotifications = async ({
}
if (discord) {
const decorate = (decoration: string, text: string) =>
`${discord.decoration ? decoration : ""} ${text}`.trim();
await sendDiscordNotification(discord, {
title:
type === "success"
? "> `✅` - Database Backup Successful"
: "> `❌` - Database Backup Failed",
? decorate(">", "`✅` Database Backup Successful")
: decorate(">", "`❌` Database Backup Failed"),
color: type === "success" ? 0x57f287 : 0xed4245,
fields: [
{
name: "`🛠️`・Project",
name: decorate("`🛠️`", "Project"),
value: projectName,
inline: true,
},
{
name: "`⚙️`・Application",
name: decorate("`⚙️`", "Application"),
value: applicationName,
inline: true,
},
{
name: "`❔`・Database",
name: decorate("`❔`", "Database"),
value: databaseType,
inline: true,
},
{
name: "`📅`・Date",
name: decorate("`📅`", "Date"),
value: `<t:${unixDate}:D>`,
inline: true,
},
{
name: "`⌚`・Time",
name: decorate("`⌚`", "Time"),
value: `<t:${unixDate}:t>`,
inline: true,
},
{
name: "`❓`・Type",
name: decorate("`❓`", "Type"),
value: type
.replace("error", "Failed")
.replace("success", "Successful"),
@@ -104,7 +107,7 @@ export const sendDatabaseBackupNotifications = async ({
...(type === "error" && errorMessage
? [
{
name: "`⚠️`・Error Message",
name: decorate("`⚠️`", "Error Message"),
value: `\`\`\`${errorMessage}\`\`\``,
},
]
@@ -121,12 +124,12 @@ export const sendDatabaseBackupNotifications = async ({
const statusEmoji = type === "success" ? "✅" : "❌";
const messageText = `
<b>${statusEmoji} Database Backup ${type === "success" ? "Successful" : "Failed"}</b>
<b>Project:</b> ${projectName}
<b>Application:</b> ${applicationName}
<b>Type:</b> ${databaseType}
<b>Time:</b> ${date.toLocaleString()}
<b>Status:</b> ${type === "success" ? "Successful" : "Failed"}
${type === "error" && errorMessage ? `<b>Error:</b> ${errorMessage}` : ""}
`;

View File

@@ -15,7 +15,7 @@ export const sendDockerCleanupNotifications = async (
message = "Docker cleanup for dokploy",
) => {
const date = new Date();
const unixDate = ~~((Number(date)) / 1000);
const unixDate = ~~(Number(date) / 1000);
const notificationList = await db.query.notifications.findMany({
where: and(
eq(notifications.dockerCleanup, true),
@@ -45,27 +45,30 @@ export const sendDockerCleanupNotifications = async (
}
if (discord) {
const decorate = (decoration: string, text: string) =>
`${discord.decoration ? decoration : ""} ${text}`.trim();
await sendDiscordNotification(discord, {
title: "> `✅` - Docker Cleanup",
title: decorate(">", "`✅` Docker Cleanup"),
color: 0x57f287,
fields: [
{
name: "`📅`・Date",
name: decorate("`📅`", "Date"),
value: `<t:${unixDate}:D>`,
inline: true,
},
{
name: "`⌚`・Time",
name: decorate("`⌚`", "Time"),
value: `<t:${unixDate}:t>`,
inline: true,
},
{
name: "`❓`・Type",
name: decorate("`❓`", "Type"),
value: "Successful",
inline: true,
},
{
name: "`📜`・Message",
name: decorate("`📜`", "Message"),
value: `\`\`\`${message}\`\`\``,
},
],

View File

@@ -12,7 +12,7 @@ import {
export const sendDokployRestartNotifications = async () => {
const date = new Date();
const unixDate = ~~((Number(date)) / 1000);
const unixDate = ~~(Number(date) / 1000);
const notificationList = await db.query.notifications.findMany({
where: eq(notifications.dokployRestart, true),
with: {
@@ -34,22 +34,25 @@ export const sendDokployRestartNotifications = async () => {
}
if (discord) {
const decorate = (decoration: string, text: string) =>
`${discord.decoration ? decoration : ""} ${text}`.trim();
await sendDiscordNotification(discord, {
title: "> `✅` - Dokploy Server Restarted",
title: decorate(">", "`✅` Dokploy Server Restarted"),
color: 0x57f287,
fields: [
{
name: "`📅`・Date",
name: decorate("`📅`", "Date"),
value: `<t:${unixDate}:D>`,
inline: true,
},
{
name: "`⌚`・Time",
name: decorate("`⌚`", "Time"),
value: `<t:${unixDate}:t>`,
inline: true,
},
{
name: "`❓`・Type",
name: decorate("`❓`", "Type"),
value: "Successful",
inline: true,
},

View File

@@ -53,7 +53,7 @@ export const buildRemoteDocker = async (
application: ApplicationNested,
logPath: string,
) => {
const { sourceType, dockerImage, username, password } = application;
const { registryUrl, dockerImage, username, password } = application;
try {
if (!dockerImage) {
@@ -65,7 +65,7 @@ echo "Pulling ${dockerImage}" >> ${logPath};
if (username && password) {
command += `
if ! docker login --username ${username} --password ${password} https://index.docker.io/v1/ >> ${logPath} 2>&1; then
if ! echo "${password}" | docker login --username "${username}" --password-stdin "${registryUrl || ""}" >> ${logPath} 2>&1; then
echo "❌ Login failed" >> ${logPath};
exit 1;
fi