From de2177d1bdbe8feede4a56267669f8e3bf5ee975 Mon Sep 17 00:00:00 2001 From: Mohamed Marrouchi Date: Thu, 2 Jan 2025 08:34:33 +0100 Subject: [PATCH] feat: add migration module and cli --- api/src/app.module.ts | 5 +- api/src/cli.ts | 22 ++ api/src/config/index.ts | 8 +- api/src/config/types.ts | 3 +- api/src/migration/README.md | 131 ++++++++++++ api/src/migration/migration.command.ts | 58 +++++ api/src/migration/migration.module.ts | 37 ++++ api/src/migration/migration.schema.ts | 27 +++ api/src/migration/migration.service.ts | 285 +++++++++++++++++++++++++ api/src/migration/migrations/.gitkeep | 0 api/src/migration/types.ts | 25 +++ api/src/utils/helpers/misc.ts | 18 ++ 12 files changed, 614 insertions(+), 5 deletions(-) create mode 100644 api/src/cli.ts create mode 100644 api/src/migration/README.md create mode 100644 api/src/migration/migration.command.ts create mode 100644 api/src/migration/migration.module.ts create mode 100644 api/src/migration/migration.schema.ts create mode 100644 api/src/migration/migration.service.ts create mode 100644 api/src/migration/migrations/.gitkeep create mode 100644 api/src/migration/types.ts diff --git a/api/src/app.module.ts b/api/src/app.module.ts index 52b779d6..4712d01d 100644 --- a/api/src/app.module.ts +++ b/api/src/app.module.ts @@ -1,5 +1,5 @@ /* - * Copyright © 2024 Hexastack. All rights reserved. + * Copyright © 2025 Hexastack. All rights reserved. * * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. @@ -39,6 +39,7 @@ import extraModules from './extra'; import { HelperModule } from './helper/helper.module'; import { I18nModule } from './i18n/i18n.module'; import { LoggerModule } from './logger/logger.module'; +import { MigrationModule } from './migration/migration.module'; import { NlpModule } from './nlp/nlp.module'; import { PluginsModule } from './plugins/plugins.module'; import { SettingModule } from './setting/setting.module'; @@ -142,7 +143,7 @@ const i18nOptions: I18nOptions = { ttl: config.cache.ttl, max: config.cache.max, }), - + MigrationModule, ...extraModules, ], controllers: [AppController], diff --git a/api/src/cli.ts b/api/src/cli.ts new file mode 100644 index 00000000..68e58a8d --- /dev/null +++ b/api/src/cli.ts @@ -0,0 +1,22 @@ +/* + * Copyright © 2025 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import moduleAlias from 'module-alias'; +import { CommandFactory } from 'nest-commander'; + +moduleAlias.addAliases({ + '@': __dirname, +}); + +import { HexabotModule } from './app.module'; + +async function bootstrap() { + await CommandFactory.run(HexabotModule); +} + +bootstrap(); diff --git a/api/src/config/index.ts b/api/src/config/index.ts index abba4b5a..858ef6b6 100644 --- a/api/src/config/index.ts +++ b/api/src/config/index.ts @@ -1,5 +1,5 @@ /* - * Copyright © 2024 Hexastack. All rights reserved. + * Copyright © 2025 Hexastack. All rights reserved. * * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. @@ -145,8 +145,12 @@ export const config: Config = { mongo: { user: process.env.MONGO_USER || 'dev_only', password: process.env.MONGO_PASSWORD || 'dev_only', - uri: process.env.MONGO_URI || 'mongodb://dev_only:dev_only@mongo:27017/', + uri: + process.env.MONGO_URI || 'mongodb://dev_only:dev_only@localhost:27017/', dbName: process.env.MONGO_DB || 'hexabot', + autoMigrate: process.env.MONGO_AUTO_MIGRATE + ? Boolean(process.env.MONGO_AUTO_MIGRATE) + : false, }, env: process.env.NODE_ENV || 'development', authentication: { diff --git a/api/src/config/types.ts b/api/src/config/types.ts index 57ac8cf9..97850130 100644 --- a/api/src/config/types.ts +++ b/api/src/config/types.ts @@ -1,5 +1,5 @@ /* - * Copyright © 2024 Hexastack. All rights reserved. + * Copyright © 2025 Hexastack. All rights reserved. * * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. @@ -102,6 +102,7 @@ export type Config = { password: string; uri: string; dbName: string; + autoMigrate: boolean; }; env: string; authentication: { diff --git a/api/src/migration/README.md b/api/src/migration/README.md new file mode 100644 index 00000000..dc3a511f --- /dev/null +++ b/api/src/migration/README.md @@ -0,0 +1,131 @@ +# Migration Module + +The `migration` module for **Hexabot** provides a simple and effective way to manage database migrations. It allows you to create, execute, and roll back migrations, ensuring the database schema stays in sync with the version DB updates. + +Whenever a new version is released which requires some DB updates, the onApplicationBootstrap() +will apply migrations automatically but only if it's a dev environement and `config.mongo.autoMigrate` is enabled. + +## Features + +- Generate timestamped migration files automatically in kebab-case. +- Track migration execution status in a MongoDB collection (`migrations`). +- Run individual or all migrations with ease. +- Built-in support for rollback logic. + +## Usage + +### Creating a Migration + +To create a new migration: + +```bash +npm run cli migration create +``` + +Replace `` with the next version for your migration, such as `v2.1.1`. + +Example: + +```bash +npm run cli migration create v2.1.1 +``` + +This will generate a new file under `src/migration/migrations/` with a timestamped filename in kebab-case. + +### Running Migrations + +#### Running a Specific Migration + +To execute a specific migration, use: + +```bash +npm run cli migration migrate up +``` + +Example: + +```bash +npm run cli migration migrate up v2.1.1 +``` + +#### Rolling Back a Specific Migration + +To roll back a specific migration, use: + +```bash +npm run cli migration migrate down +``` + +Example: + +```bash +npm run cli migration migrate down v2.1.1 +``` + +#### Running All Migrations + +To execute all pending migrations: + +```bash +npm run cli migration migrate up +``` + +#### Rolling Back All Migrations + +To roll back all migrations: + +```bash +npm run cli migration migrate down +``` + +### Tracking Migration Status + +The migration status is stored in a MongoDB collection called `migrations`. This collection helps ensure that each migration is executed or rolled back only once, avoiding duplicate operations. + +## Example Migration File + +Below is an example migration file: + +```typescript +import mongoose from 'mongoose'; +import attachmentSchema, { + Attachment, +} from '@/attachment/schemas/attachment.schema'; + +module.exports = { + async up() { + const AttachmentModel = mongoose.model( + Attachment.name, + attachmentSchema, + ); + await AttachmentModel.updateMany({ + type: 'csv' + }, { + $set: { type: 'text/csv' } + }); + }, + async down() { + // Rollback logic + const AttachmentModel = mongoose.model( + Attachment.name, + attachmentSchema, + ); + await AttachmentModel.updateMany({ + type: 'text/csv' + }, { + $set: { type: 'csv' } + }); + }, +}; +``` + +### Explanation + +- **`up` Method**: Defines the operations to apply the migration (e.g., modifying schemas or inserting data). +- **`down` Method**: Defines the rollback logic to revert the migration. + +## Best Practices + +- Use semantic versioning (e.g., `v2.1.1`) for your migration names to keep track of changes systematically. +- Always test migrations in a development or staging environment before running them in production. +- Keep the `up` and `down` methods idempotent to avoid side effects from repeated execution. diff --git a/api/src/migration/migration.command.ts b/api/src/migration/migration.command.ts new file mode 100644 index 00000000..ef56da2a --- /dev/null +++ b/api/src/migration/migration.command.ts @@ -0,0 +1,58 @@ +/* + * Copyright © 2025 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { Command, CommandRunner } from 'nest-commander'; + +import { LoggerService } from '@/logger/logger.service'; + +import { MigrationService } from './migration.service'; +import { MigrationAction } from './types'; + +@Command({ + name: 'migration', + description: 'Manage Mongodb Migrations', +}) +export class MigrationCommand extends CommandRunner { + constructor( + private readonly logger: LoggerService, + private readonly migrationService: MigrationService, + ) { + super(); + } + + async run(passedParam: string[]): Promise { + const [subcommand] = passedParam; + switch (subcommand) { + case 'create': + const [, filename] = passedParam; + return await this.migrationService.create(filename); + case 'migrate': + const [, action, name] = passedParam; + if ( + !Object.values(MigrationAction).includes(action as MigrationAction) + ) { + this.logger.error('Invalid Operation'); + this.exit(); + } + return await this.migrationService.run({ + action: action as MigrationAction, + name, + }); + + default: + this.logger.error('No valid command provided'); + this.exit(); + break; + } + } + + exit(): void { + this.logger.log('Exiting migration process.'); + process.exit(0); + } +} diff --git a/api/src/migration/migration.module.ts b/api/src/migration/migration.module.ts new file mode 100644 index 00000000..154c3eca --- /dev/null +++ b/api/src/migration/migration.module.ts @@ -0,0 +1,37 @@ +/* + * Copyright © 2025 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { join } from 'path'; + +import { Module } from '@nestjs/common'; +import { MongooseModule } from '@nestjs/mongoose'; + +import { LoggerModule } from '@/logger/logger.module'; + +import { MigrationCommand } from './migration.command'; +import { Migration, MigrationSchema } from './migration.schema'; +import { MigrationService } from './migration.service'; + +@Module({ + imports: [ + MongooseModule.forFeature([ + { name: Migration.name, schema: MigrationSchema }, + ]), + LoggerModule, + ], + providers: [ + MigrationService, + MigrationCommand, + { + provide: 'MONGO_MIGRATION_DIR', + useValue: join(process.cwd(), 'src', 'migration', 'migrations'), + }, + ], + exports: [MigrationService], +}) +export class MigrationModule {} diff --git a/api/src/migration/migration.schema.ts b/api/src/migration/migration.schema.ts new file mode 100644 index 00000000..a344d143 --- /dev/null +++ b/api/src/migration/migration.schema.ts @@ -0,0 +1,27 @@ +/* + * Copyright © 2025 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose'; +import { Document, Model } from 'mongoose'; + +import { MigrationAction } from './types'; + +@Schema({ timestamps: true }) +export class Migration { + @Prop({ required: true, unique: true }) + name: string; + + @Prop({ required: true, enum: MigrationAction }) + status: string; +} + +export const MigrationSchema = SchemaFactory.createForClass(Migration); + +export type MigrationDocument = Migration & Document; + +export type MigrationModel = Model; diff --git a/api/src/migration/migration.service.ts b/api/src/migration/migration.service.ts new file mode 100644 index 00000000..b0be1536 --- /dev/null +++ b/api/src/migration/migration.service.ts @@ -0,0 +1,285 @@ +/* + * Copyright © 2025 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { existsSync, readdirSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import { Injectable, OnApplicationBootstrap } from '@nestjs/common'; +import { ModuleRef } from '@nestjs/core'; +import { InjectModel } from '@nestjs/mongoose'; +import { kebabCase } from 'lodash'; +import mongoose from 'mongoose'; +import leanDefaults from 'mongoose-lean-defaults'; +import leanGetters from 'mongoose-lean-getters'; +import leanVirtuals from 'mongoose-lean-virtuals'; + +import { config } from '@/config'; +import { LoggerService } from '@/logger/logger.service'; +import idPlugin from '@/utils/schema-plugin/id.plugin'; + +import { + Migration, + MigrationDocument, + MigrationModel, +} from './migration.schema'; +import { + MigrationAction, + MigrationRunParams, + MigrationSuccessCallback, +} from './types'; + +@Injectable() +export class MigrationService implements OnApplicationBootstrap { + constructor( + private moduleRef: ModuleRef, + private readonly logger: LoggerService, + @InjectModel(Migration.name) + private readonly migrationModel: MigrationModel, + ) { + this.validateMigrationPath(); + } + + async onApplicationBootstrap() { + if (mongoose.connection.readyState !== 1) { + await this.connect(); + } + this.logger.log('Mongoose connection established'); + + const isProduction = config.env.toLowerCase().includes('prod'); + if (!isProduction && config.mongo.autoMigrate) { + this.logger.log('Executing migrations ...'); + await this.run({ action: MigrationAction.UP }); + } + } + + public exit() { + process.exit(0); + } + + // CREATE + public get migrationFilePath() { + return this.moduleRef.get('MONGO_MIGRATION_DIR'); + } + + public validateMigrationPath() { + if (!existsSync(this.migrationFilePath)) { + this.logger.error( + `Migration directory "${this.migrationFilePath}" not exists.`, + ); + this.exit(); + } + } + + public async create(name: string) { + const fileName: string = kebabCase(name) + '.migration.ts'; + + // check if file already exists + const files = await this.getDirFiles(); + const exist = files.some((file) => { + const [, ...actualFileName] = file.split('-'); + const migrationName = actualFileName.join('-'); + return migrationName === fileName; + }); + + if (exist) { + this.logger.error(`Migration file for "${name}" already exists`); + this.exit(); + } + + const migrationFileName = `${Date.now()}-${fileName}`; + const filePath = join(this.migrationFilePath, migrationFileName); + const template = this.getMigrationTemplate(); + try { + writeFileSync(filePath, template); + this.logger.log( + `Migration file for "${name}" created: ${migrationFileName}`, + ); + } catch (e) { + this.logger.error(e.stack); + } finally { + this.exit(); + } + } + + private getMigrationTemplate() { + return `import mongoose from 'mongoose'; + +module.exports = { + async up() { + // Migration logic + }, + async down() { + // Rollback logic + }, +};`; + } + + private async connect() { + try { + const connection = await mongoose.connect(config.mongo.uri, { + dbName: config.mongo.dbName, + }); + + connection.plugin(idPlugin); + connection.plugin(leanVirtuals); + connection.plugin(leanGetters); + connection.plugin(leanDefaults); + } catch (err) { + this.logger.error('Failed to connect to MongoDB'); + throw err; + } + } + + public async run({ action, name }: MigrationRunParams) { + if (!name) { + await this.runAll(action); + } else { + await this.runOne({ action, name }); + } + this.exit(); + } + + private async runOne({ name, action }: MigrationRunParams) { + // verify DB status + const { exist, migrationDocument } = await this.verifyStatus({ + name, + action, + }); + if (exist) { + return true; // stop exec; + } + + try { + const migration = await this.loadMigrationFile(name); + await migration[action](); + await this.successCallback({ + name, + action, + migrationDocument, + }); + } catch (e) { + this.failureCallback({ + name, + action, + }); + this.logger.log(e.stack); + } + } + + private async runAll(action: MigrationAction) { + const files = await this.getDirFiles(); + const migrationFiles = files + .filter((fileName) => fileName.includes('migration')) + .map((fileName) => { + const [migrationFileName] = fileName.split('.'); + const [, ...migrationName] = migrationFileName.split('-'); + return migrationName.join('-'); + }); + + for (const name of migrationFiles) { + await this.runOne({ name, action }); + } + } + + private async getDirFiles() { + return readdirSync(this.migrationFilePath); + } + + private async verifyStatus({ name, action }: MigrationRunParams): Promise<{ + exist: boolean; + migrationDocument: MigrationDocument | null; + }> { + let exist = false; + const migrationDocument = await this.migrationModel.findOne({ name }); + + if (migrationDocument) { + exist = Boolean(migrationDocument.status === action); + if (exist) { + this.logger.warn( + `Cannot proceed migration "${name}" is already in "${action}" state`, + ); + } + } + + return { exist, migrationDocument }; + } + + private async getMigrationFiles() { + const files = await this.getDirFiles(); + return files.filter((file) => /\.migration\.(js|ts)/.test(file)); + } + + private async findMigrationFileByName(name: string): Promise { + const files = await this.getMigrationFiles(); + return ( + files.find((file) => { + const [, ...migrationNameParts] = file.split('-'); + const migrationName = migrationNameParts + .join('-') + .replace(/\.migration\.(js|ts)/, ''); + + return migrationName === kebabCase(name); + }) || null + ); + } + + private async loadMigrationFile(name: string) { + try { + // Map the provided name to the actual file with timestamp + const fileName = await this.findMigrationFileByName(name); + if (!fileName) { + this.logger.error(`Migration file for "${name}" not found.`); + process.exit(1); + } + + const filePath = join(this.migrationFilePath, fileName); + const migration = await import(filePath); + if ( + !migration || + typeof migration.up !== 'function' || + typeof migration.down !== 'function' + ) { + throw new Error( + `Migration file "${name}" must export an object with "up" and "down" methods.`, + ); + } + return migration; + } catch (e) { + throw new Error(`Failed to load migration "${name}".\n${e.message}`); + } + } + + private async updateStatus({ + name, + action, + migrationDocument, + }: Omit) { + const document = + migrationDocument || + new this.migrationModel({ + name, + }); + document.status = action; + await document.save(); + } + + private async successCallback({ + name, + action, + migrationDocument, + }: MigrationSuccessCallback) { + await this.updateStatus({ name, action, migrationDocument }); + const migrationDisplayName = `${name} [${action}]`; + this.logger.log(`"${migrationDisplayName}" migration done`); + } + + private failureCallback({ name, action }: MigrationRunParams) { + const migrationDisplayName = `${name} [${action}]`; + this.logger.error(`"${migrationDisplayName}" migration failed`); + } +} diff --git a/api/src/migration/migrations/.gitkeep b/api/src/migration/migrations/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/api/src/migration/types.ts b/api/src/migration/types.ts new file mode 100644 index 00000000..43161c56 --- /dev/null +++ b/api/src/migration/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright © 2025 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { MigrationDocument } from './migration.schema'; + +enum MigrationAction { + UP = 'up', + DOWN = 'down', +} + +interface MigrationRunParams { + name?: string; + action: MigrationAction; +} + +interface MigrationSuccessCallback extends MigrationRunParams { + migrationDocument: MigrationDocument; +} + +export { MigrationAction, MigrationRunParams, MigrationSuccessCallback }; diff --git a/api/src/utils/helpers/misc.ts b/api/src/utils/helpers/misc.ts index be1aafb4..ff716923 100644 --- a/api/src/utils/helpers/misc.ts +++ b/api/src/utils/helpers/misc.ts @@ -13,3 +13,21 @@ export const isEmpty = (value: string): boolean => { export const hyphenToUnderscore = (str: string) => { return str.replaceAll('-', '_'); }; + +export const kebabCase = (input: string): string => { + return input + .replace(/([a-z])([A-Z])/g, '$1-$2') // Add a dash between lowercase and uppercase letters + .replace(/[\s_]+/g, '-') // Replace spaces and underscores with a dash + .toLowerCase(); // Convert the entire string to lowercase +}; + +export const camelCase = (input: string): string => { + return input + .replace(/[-_\s]+(.)?/g, (_, char) => (char ? char.toUpperCase() : '')) // Replace dashes, underscores, and spaces, capitalizing the following letter + .replace(/^./, (char) => char.toLowerCase()); // Ensure the first character is lowercase +}; + +export const upperFirst = (input: string): string => { + if (!input) return input; // Return as is if the input is empty + return input.charAt(0).toUpperCase() + input.slice(1); +};