Merge pull request #274 from Hexastack/feat/add-extra-deps

feat: add extra deps + docker base image

api/.dockerignore

@@ -1,6 +1,8 @@
 dist
 node_modules
 .dockerignore
+.Dockerfile
+.Dockerfile.base
 .env
 .eslintrc.js
 .git

api/.eslintrc.js

@@ -15,7 +15,7 @@ module.exports = {
     node: true,
     jest: true,
   },
-  ignorePatterns: ['.eslintrc.js'],
+  ignorePatterns: ['.eslintrc.js', 'add-extra-deps.js'],
   rules: {
     '@typescript-eslint/interface-name-prefix': 'off',
     '@typescript-eslint/explicit-function-return-type': 'off',

api/Dockerfile.base

@@ -0,0 +1,9 @@
+FROM scratch AS base
+
+WORKDIR /app
+
+COPY package*.json ./
+
+COPY patches ./patches
+
+COPY . .

api/add-extra-deps.js

@@ -0,0 +1,51 @@
+const fs = require('fs');
+const path = require('path');
+
+const excludingPackages = ['hexabot'];
+
+// File paths for package.json and package.extra.json
+const packageJsonPath = path.join(__dirname, 'package.json');
+const packageExtraJsonPath = path.join(__dirname, 'package.extra.json');
+
+// Helper function to read and parse JSON files
+function readJsonFile(filePath) {
+  if (fs.existsSync(filePath)) {
+    return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+  }
+  throw new Error(`File not found: ${filePath}`);
+}
+
+// Helper function to write JSON data to a file
+function writeJsonFile(filePath, data) {
+  fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
+}
+
+try {
+  // Load package.json and package.extra.json
+  const packageJson = readJsonFile(packageJsonPath);
+  const packageExtraJson = readJsonFile(packageExtraJsonPath);
+
+  // Get dependencies from package.extra.json (excluding hexabot and typescript)
+  const extraDeps = packageExtraJson.dependencies || {};
+  const filteredDeps = Object.keys(extraDeps).reduce((acc, dep) => {
+    if (!excludingPackages.includes(dep)) {
+      acc[dep] = extraDeps[dep];
+    }
+    return acc;
+  }, {});
+
+  // Merge the filtered dependencies into the package.json dependencies
+  packageJson.dependencies = {
+    ...filteredDeps,
+    ...packageJson.dependencies,
+  };
+
+  // Write the updated package.json back to the file
+  writeJsonFile(packageJsonPath, packageJson);
+
+  console.log(
+    'Dependencies from package.extra.json have been added to package.json.',
+  );
+} catch (error) {
+  console.error(`Error: ${error.message}`);
+}

api/package-lock.json

@@ -33,8 +33,6 @@
         "dotenv": "^16.3.1",
         "ejs": "^3.1.9",
         "express-session": "^1.17.3",
-        "hexabot-helper-ollama": "^2.0.0",
-        "hexabot-plugin-ollama": "^2.0.0",
         "joi": "^17.11.0",
         "module-alias": "^2.2.3",
         "mongoose": "^8.0.0",
@@ -11357,22 +11355,6 @@
         "he": "bin/he"
       }
     },
-    "node_modules/hexabot-helper-ollama": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/hexabot-helper-ollama/-/hexabot-helper-ollama-2.0.0.tgz",
-      "integrity": "sha512-VYY+zlOBmCrkhBML/ZUBA96VXXpsHcrGiAVk2s1GsyaOUVOjQtovQE9rHlU6qf38biGd72LUQ827323pZim6GA==",
-      "dependencies": {
-        "ollama": "^0.5.9"
-      }
-    },
-    "node_modules/hexabot-plugin-ollama": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/hexabot-plugin-ollama/-/hexabot-plugin-ollama-2.0.0.tgz",
-      "integrity": "sha512-/YE2o2UYNIG1vymViTcOrVlpFFmMswZVEI9EkDC9Xs9RzzHtIRD473RQE1PhTDFLHBAG3Qx3NgbiaJR/el4GXQ==",
-      "dependencies": {
-        "hexabot-helper-ollama": "2.0.0"
-      }
-    },
     "node_modules/hexoid": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz",

api/package.json

@@ -1,30 +1,30 @@
 {
   "name": "hexabot",
-  "private": true,
+  "version": "2.0.2",
-  "version": "2.0.0",
   "description": "Hexabot is a solution for creating and managing chatbots across multiple channels, leveraging AI for advanced conversational capabilities. It provides a user-friendly interface for building, training, and deploying chatbots with integrated support for various messaging platforms.",
   "author": "Hexastack",
   "license": "AGPL-3.0-only",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
   "files": [
-    "dist"
+    "src",
+    "types",
+    "!src/**/*.spec.ts",
+    "!src/.hexabot"
   ],
   "scripts": {
     "postinstall": "patch-package",
-    "build:clean": "rm -rf src/.hexabot",
+    "cache:channels": "mkdir -p src/.hexabot/contrib/extensions/channels && find node_modules/ -name 'hexabot-channel-*' -exec cp -R {} src/.hexabot/contrib/extensions/channels/ \\;",
-    "build:channels": "mkdir -p src/.hexabot/extensions/channels && find node_modules/ -name 'hexabot-channel-*' -exec cp -R {} src/.hexabot/extensions/channels/ \\;",
+    "cache:helpers": "mkdir -p src/.hexabot/contrib/extensions/helpers && find node_modules/ -name 'hexabot-helper-*' -exec cp -R {} src/.hexabot/contrib/extensions/helpers/ \\;",
-    "build:helpers": "mkdir -p src/.hexabot/extensions/helpers && find node_modules/ -name 'hexabot-helper-*' -exec cp -R {} src/.hexabot/extensions/helpers/ \\;",
+    "cache:plugins": "mkdir -p src/.hexabot/contrib/extensions/plugins && find node_modules/ -name 'hexabot-plugin-*' -exec cp -R {} src/.hexabot/contrib/extensions/plugins/ \\;",
-    "build:plugins": "mkdir -p src/.hexabot/extensions/plugins && find node_modules/ -name 'hexabot-plugin-*' -exec cp -R {} src/.hexabot/extensions/plugins/ \\;",
+    "cache:contrib": "rm -rf src/.hexabot/contrib && npm run cache:channels && npm run cache:helpers && npm run cache:plugins",
-    "build:extensions": "npm run build:channels && npm run build:helpers && npm run build:plugins",
+    "cache:custom": "mkdir -p src/.hexabot/custom/extensions",
-    "build:prepare": "npm run build:clean && npm run build:extensions",
+    "cache:prepare": "npm run cache:contrib && npm run cache:custom",
-    "build": "npm run build:prepare && nest build && npm run copy-types",
+    "build": "npm run cache:prepare && nest build && npm run copy-types",
     "copy-types": "cp -R types dist/types",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"libs/**/*.ts\"",
     "start": "nest start",
     "doc": "npx @compodoc/compodoc --hideGenerator -p tsconfig.doc.json -s -r 9003 -w",
-    "start:dev": "npm run build:prepare && nest start --watch",
+    "start:dev": "npm run cache:prepare && nest start --watch",
-    "start:debug": "npm run build:prepare && nest start --debug 0.0.0.0:9229 --watch",
+    "start:debug": "npm run cache:prepare && nest start --debug 0.0.0.0:9229 --watch",
     "start:prod": "node dist/main",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"",
     "lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
@@ -39,7 +39,11 @@
     "typecheck": "tsc --noEmit",
     "reset": "npm install && npm run containers:restart",
     "reset:hard": "npm clean-install && npm run containers:rebuild",
-    "migrate": "npx ts-migrate-mongoose --config-path ./migrations/config/migrate.ts"
+    "migrate": "npx ts-migrate-mongoose --config-path ./migrations/config/migrate.ts",
+    "docker:build-base": "docker build --pull --rm -f Dockerfile.base -t hexabot-base:latest .",
+    "docker:tag-base": "docker tag hexabot-base:latest hexastack/hexabot-base:latest",
+    "docker:push-base": "docker push hexastack/hexabot-base:latest",
+    "docker:release-base": "npm run docker:build-base && npm run docker:tag-base && npm run docker:push-base"
   },
   "dependencies": {
     "@nestjs-modules/mailer": "^1.11.2",
@@ -65,8 +69,6 @@
     "dotenv": "^16.3.1",
     "ejs": "^3.1.9",
     "express-session": "^1.17.3",
-    "hexabot-helper-ollama": "^2.0.0",
-    "hexabot-plugin-ollama": "^2.0.0",
     "joi": "^17.11.0",
     "module-alias": "^2.2.3",
     "mongoose": "^8.0.0",
@@ -173,4 +175,4 @@
       "@/(.*)": "<rootDir>/$1"
     }
   }
-}
+}

api/src/app.module.ts

@@ -8,11 +8,11 @@
 
 import path from 'path';
 
+import { CacheModule } from '@nestjs/cache-manager';
 // eslint-disable-next-line import/order
 import { MailerModule } from '@nestjs-modules/mailer';
 // eslint-disable-next-line import/order
 import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter';
-import { CacheModule } from '@nestjs/cache-manager';
 import { Module } from '@nestjs/common';
 import { APP_GUARD } from '@nestjs/core';
 import { EventEmitterModule } from '@nestjs/event-emitter';
@@ -33,6 +33,7 @@ import { ChannelModule } from './channel/channel.module';
 import { ChatModule } from './chat/chat.module';
 import { CmsModule } from './cms/cms.module';
 import { config } from './config';
+import extraModules from './extra';
 import { HelperModule } from './helper/helper.module';
 import { I18nModule } from './i18n/i18n.module';
 import { LoggerModule } from './logger/logger.module';
@@ -128,6 +129,7 @@ const i18nOptions: I18nOptions = {
       ttl: config.cache.ttl,
       max: config.cache.max,
     }),
+    ...extraModules,
   ],
   controllers: [AppController],
   providers: [

api/src/channel/channel.module.ts

@@ -32,8 +32,10 @@ export interface ChannelModuleOptions {
 @InjectDynamicProviders(
   // Core & under dev channels
   'dist/extensions/**/*.channel.js',
-  // Installed channels via npm
+  // Community extensions installed via npm
-  'dist/.hexabot/extensions/channels/**/*.channel.js',
+  'dist/.hexabot/contrib/extensions/channels/**/*.channel.js',
+  // Custom extensions under dev
+  'dist/.hexabot/custom/extensions/channels/**/*.channel.js',
 )
 @Module({
   controllers: [WebhookController, ChannelController],

api/src/extra/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+export default [];

api/src/helper/helper.module.ts

@@ -17,8 +17,10 @@ import { HelperService } from './helper.service';
 @InjectDynamicProviders(
   // Core & under dev helpers
   'dist/extensions/**/*.helper.js',
-  // Installed helpers via npm
+  // Community extensions installed via npm
-  'dist/.hexabot/extensions/helpers/**/*.helper.js',
+  'dist/.hexabot/contrib/extensions/helpers/**/*.helper.js',
+  // Custom extensions under dev
+  'dist/.hexabot/custom/extensions/helpers/**/*.helper.js',
 )
 @Module({
   imports: [HttpModule],

api/src/plugins/plugins.module.ts

@@ -23,8 +23,10 @@ import { PluginService } from './plugins.service';
 @InjectDynamicProviders(
   // Core & under dev plugins
   'dist/extensions/**/*.plugin.js',
-  // Installed plugins via npm
+  // Community extensions installed via npm
-  'dist/.hexabot/extensions/plugins/**/*.plugin.js',
+  'dist/.hexabot/contrib/extensions/plugins/**/*.plugin.js',
+  // Custom extensions under dev
+  'dist/.hexabot/custom/extensions/plugins/**/*.plugin.js',
 )
 @Global()
 @Module({

api/tsconfig.json

@@ -21,7 +21,8 @@
     "resolveJsonModule": true,
     "esModuleInterop": true,
     "paths": {
-      "@/*": ["src/*", "src/.hexabot/*"]
+      "@/*": ["src/*", "src/.hexabot/*"],
+      "hexabot/src/*": ["src/*", "src/.hexabot/*"]
     }
   },
   "include": [