Merge pull request #254 from Hexastack/revert-253-revert-250-feat/refactor-helpers

feat/refactor helpers

api/.dockerignore

@@ -10,3 +10,7 @@ node_modules
 coverage*
 README.md
 test
+*.spec.ts
+*.mock.ts
+__mock__
+__test__

api/src/app.module.ts

@@ -8,13 +8,15 @@
 
 import path from 'path';
 
+// eslint-disable-next-line import/order
+import { MailerModule } from '@nestjs-modules/mailer';
+// eslint-disable-next-line import/order
+import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter';
 import { CacheModule } from '@nestjs/cache-manager';
 import { Module } from '@nestjs/common';
 import { APP_GUARD } from '@nestjs/core';
 import { EventEmitterModule } from '@nestjs/event-emitter';
 import { MongooseModule } from '@nestjs/mongoose';
-import { MailerModule } from '@nestjs-modules/mailer';
-import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter';
 import { CsrfGuard, CsrfModule } from '@tekuconcept/nestjs-csrf';
 import {
   AcceptLanguageResolver,
@@ -31,6 +33,7 @@ import { ChannelModule } from './channel/channel.module';
 import { ChatModule } from './chat/chat.module';
 import { CmsModule } from './cms/cms.module';
 import { config } from './config';
+import { HelperModule } from './helper/helper.module';
 import { I18nModule } from './i18n/i18n.module';
 import { LoggerModule } from './logger/logger.module';
 import { NlpModule } from './nlp/nlp.module';
@@ -99,6 +102,7 @@ const i18nOptions: I18nOptions = {
     ChatModule,
     ChannelModule,
     PluginsModule,
+    HelperModule,
     LoggerModule,
     WebsocketModule,
     EventEmitterModule.forRoot({

api/src/channel/channel.module.ts

@@ -13,7 +13,6 @@ import { InjectDynamicProviders } from 'nestjs-dynamic-providers';
 import { AttachmentModule } from '@/attachment/attachment.module';
 import { ChatModule } from '@/chat/chat.module';
 import { CmsModule } from '@/cms/cms.module';
-import { NlpModule } from '@/nlp/nlp.module';
 
 import { ChannelController } from './channel.controller';
 import { ChannelMiddleware } from './channel.middleware';
@@ -29,7 +28,7 @@ export interface ChannelModuleOptions {
   controllers: [WebhookController, ChannelController],
   providers: [ChannelService],
   exports: [ChannelService],
-  imports: [NlpModule, ChatModule, AttachmentModule, CmsModule, HttpModule],
+  imports: [ChatModule, AttachmentModule, CmsModule, HttpModule],
 })
 export class ChannelModule {
   configure(consumer: MiddlewareConsumer) {

api/src/channel/lib/EventWrapper.ts

@@ -17,7 +17,7 @@ import {
   StdIncomingMessage,
 } from '@/chat/schemas/types/message';
 import { Payload } from '@/chat/schemas/types/quick-reply';
-import { Nlp } from '@/nlp/lib/types';
+import { Nlp } from '@/helper/types';
 
 import ChannelHandler from './Handler';
 

api/src/channel/lib/Handler.ts

@@ -16,8 +16,6 @@ import {
   StdOutgoingMessage,
 } from '@/chat/schemas/types/message';
 import { LoggerService } from '@/logger/logger.service';
-import BaseNlpHelper from '@/nlp/lib/BaseNlpHelper';
-import { NlpService } from '@/nlp/services/nlp.service';
 import { SettingService } from '@/setting/services/setting.service';
 import { hyphenToUnderscore } from '@/utils/helpers/misc';
 import { SocketRequest } from '@/websocket/utils/socket-request';
@@ -34,14 +32,11 @@ export default abstract class ChannelHandler<N extends string = string> {
 
   private readonly settings: ChannelSetting<N>[];
 
-  protected NLP: BaseNlpHelper;
-
   constructor(
     name: N,
     settings: ChannelSetting<N>[],
     protected readonly settingService: SettingService,
     private readonly channelService: ChannelService,
-    protected readonly nlpService: NlpService,
     protected readonly logger: LoggerService,
   ) {
     this.name = name;
@@ -56,10 +51,6 @@ export default abstract class ChannelHandler<N extends string = string> {
     this.setup();
   }
 
-  protected getGroup() {
-    return hyphenToUnderscore(this.getChannel()) as ChannelSetting<N>['group'];
-  }
-
   async setup() {
     await this.settingService.seedIfNotExist(
       this.getChannel(),
@@ -68,19 +59,9 @@ export default abstract class ChannelHandler<N extends string = string> {
         weight: i + 1,
       })),
     );
-    const nlp = this.nlpService.getNLP();
-    this.setNLP(nlp);
     this.init();
   }
 
-  setNLP(nlp: BaseNlpHelper) {
-    this.NLP = nlp;
-  }
-
-  getNLP() {
-    return this.NLP;
-  }
-
   /**
    * Returns the channel's name
    * @returns Channel's name
@@ -89,6 +70,14 @@ export default abstract class ChannelHandler<N extends string = string> {
     return this.name;
   }
 
+  /**
+   * Returns the channel's group
+   * @returns Channel's group
+   */
+  protected getGroup() {
+    return hyphenToUnderscore(this.getChannel()) as ChannelSetting<N>['group'];
+  }
+
   /**
    * Returns the channel's settings
    * @returns Channel's settings

api/src/chat/chat.module.ts

@@ -13,7 +13,6 @@ import { MongooseModule } from '@nestjs/mongoose';
 import { AttachmentModule } from '@/attachment/attachment.module';
 import { ChannelModule } from '@/channel/channel.module';
 import { CmsModule } from '@/cms/cms.module';
-import { NlpModule } from '@/nlp/nlp.module';
 import { UserModule } from '@/user/user.module';
 
 import { BlockController } from './controllers/block.controller';
@@ -63,7 +62,6 @@ import { SubscriberService } from './services/subscriber.service';
     forwardRef(() => ChannelModule),
     CmsModule,
     AttachmentModule,
-    NlpModule,
     EventEmitter2,
     UserModule,
   ],

api/src/chat/repositories/message.repository.ts

@@ -6,16 +6,11 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import { Injectable, Optional } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import { InjectModel } from '@nestjs/mongoose';
 import { Model } from 'mongoose';
 
-import { LanguageService } from '@/i18n/services/language.service';
-import { LoggerService } from '@/logger/logger.service';
-import { NlpSampleCreateDto } from '@/nlp/dto/nlp-sample.dto';
-import { NlpSampleState } from '@/nlp/schemas/types';
-import { NlpSampleService } from '@/nlp/services/nlp-sample.service';
 import { BaseRepository } from '@/utils/generics/base-repository';
 
 import {
@@ -33,18 +28,9 @@ export class MessageRepository extends BaseRepository<
   MessagePopulate,
   MessageFull
 > {
-  private readonly nlpSampleService: NlpSampleService;
-
-  private readonly logger: LoggerService;
-
-  private readonly languageService: LanguageService;
-
   constructor(
     readonly eventEmitter: EventEmitter2,
     @InjectModel(Message.name) readonly model: Model<AnyMessage>,
-    @Optional() nlpSampleService?: NlpSampleService,
-    @Optional() logger?: LoggerService,
-    @Optional() languageService?: LanguageService,
   ) {
     super(
       eventEmitter,
@@ -53,9 +39,6 @@ export class MessageRepository extends BaseRepository<
       MESSAGE_POPULATE,
       MessageFull,
     );
-    this.logger = logger;
-    this.nlpSampleService = nlpSampleService;
-    this.languageService = languageService;
   }
 
   /**
@@ -69,35 +52,8 @@ export class MessageRepository extends BaseRepository<
   async preCreate(_doc: AnyMessage): Promise<void> {
     if (_doc) {
       if (!('sender' in _doc) && !('recipient' in _doc)) {
-        this.logger.error('Either sender or recipient must be provided!', _doc);
         throw new Error('Either sender or recipient must be provided!');
       }
-      // If message is sent by the user then add it as an inbox sample
-      if (
-        'sender' in _doc &&
-        _doc.sender &&
-        'message' in _doc &&
-        'text' in _doc.message
-      ) {
-        const defaultLang = await this.languageService?.getDefaultLanguage();
-        const record: NlpSampleCreateDto = {
-          text: _doc.message.text,
-          type: NlpSampleState.inbox,
-          trained: false,
-          // @TODO : We need to define the language in the message entity
-          language: defaultLang.id,
-        };
-        try {
-          await this.nlpSampleService.findOneOrCreate(record, record);
-          this.logger.debug('User message saved as a inbox sample !');
-        } catch (err) {
-          this.logger.error(
-            'Unable to add message as a new inbox sample!',
-            err,
-          );
-          throw err;
-        }
-      }
     }
   }
 

api/src/chat/schemas/types/context.ts

@@ -6,7 +6,7 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import { Nlp } from '@/nlp/lib/types';
+import { Nlp } from '@/helper/types';
 
 import { Subscriber } from '../subscriber.schema';
 

api/src/chat/services/block.service.ts

@@ -12,10 +12,10 @@ import { Attachment } from '@/attachment/schemas/attachment.schema';
 import { AttachmentService } from '@/attachment/services/attachment.service';
 import EventWrapper from '@/channel/lib/EventWrapper';
 import { ContentService } from '@/cms/services/content.service';
+import { Nlp } from '@/helper/types';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LanguageService } from '@/i18n/services/language.service';
 import { LoggerService } from '@/logger/logger.service';
-import { Nlp } from '@/nlp/lib/types';
 import { PluginService } from '@/plugins/plugins.service';
 import { PluginType } from '@/plugins/types';
 import { SettingService } from '@/setting/services/setting.service';

api/src/chat/services/bot.service.spec.ts

@@ -27,24 +27,12 @@ import { MenuService } from '@/cms/services/menu.service';
 import { offlineEventText } from '@/extensions/channels/offline/__test__/events.mock';
 import OfflineHandler from '@/extensions/channels/offline/index.channel';
 import OfflineEventWrapper from '@/extensions/channels/offline/wrapper';
+import { HelperService } from '@/helper/helper.service';
 import { LanguageRepository } from '@/i18n/repositories/language.repository';
 import { LanguageModel } from '@/i18n/schemas/language.schema';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LanguageService } from '@/i18n/services/language.service';
 import { LoggerService } from '@/logger/logger.service';
-import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository';
-import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository';
-import { NlpSampleRepository } from '@/nlp/repositories/nlp-sample.repository';
-import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository';
-import { NlpEntityModel } from '@/nlp/schemas/nlp-entity.schema';
-import { NlpSampleEntityModel } from '@/nlp/schemas/nlp-sample-entity.schema';
-import { NlpSampleModel } from '@/nlp/schemas/nlp-sample.schema';
-import { NlpValueModel } from '@/nlp/schemas/nlp-value.schema';
-import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
-import { NlpSampleEntityService } from '@/nlp/services/nlp-sample-entity.service';
-import { NlpSampleService } from '@/nlp/services/nlp-sample.service';
-import { NlpValueService } from '@/nlp/services/nlp-value.service';
-import { NlpService } from '@/nlp/services/nlp.service';
 import { PluginService } from '@/plugins/plugins.service';
 import { SettingService } from '@/setting/services/setting.service';
 import { installBlockFixtures } from '@/utils/test/fixtures/block';
@@ -109,10 +97,6 @@ describe('BlockService', () => {
           SubscriberModel,
           MessageModel,
           MenuModel,
-          NlpValueModel,
-          NlpEntityModel,
-          NlpSampleEntityModel,
-          NlpSampleModel,
           ContextVarModel,
           LanguageModel,
         ]),
@@ -130,10 +114,6 @@ describe('BlockService', () => {
         SubscriberRepository,
         MessageRepository,
         MenuRepository,
-        NlpValueRepository,
-        NlpEntityRepository,
-        NlpSampleEntityRepository,
-        NlpSampleRepository,
         LanguageRepository,
         BlockService,
         CategoryService,
@@ -147,14 +127,13 @@ describe('BlockService', () => {
         MessageService,
         MenuService,
         OfflineHandler,
-        NlpValueService,
-        NlpEntityService,
-        NlpSampleEntityService,
-        NlpSampleService,
-        NlpService,
         ContextVarService,
         ContextVarRepository,
         LanguageService,
+        {
+          provide: HelperService,
+          useValue: {},
+        },
         {
           provide: PluginService,
           useValue: {},

api/src/chat/services/chat.service.ts

@@ -11,8 +11,8 @@ import { EventEmitter2, OnEvent } from '@nestjs/event-emitter';
 
 import EventWrapper from '@/channel/lib/EventWrapper';
 import { config } from '@/config';
+import { HelperService } from '@/helper/helper.service';
 import { LoggerService } from '@/logger/logger.service';
-import { NlpService } from '@/nlp/services/nlp.service';
 import { WebsocketGateway } from '@/websocket/websocket.gateway';
 
 import { MessageCreateDto } from '../dto/message.dto';
@@ -35,7 +35,7 @@ export class ChatService {
     private readonly subscriberService: SubscriberService,
     private readonly botService: BotService,
     private readonly websocketGateway: WebsocketGateway,
-    private readonly nlpService: NlpService,
+    private readonly helperService: HelperService,
   ) {}
 
   /**
@@ -268,9 +268,9 @@ export class ChatService {
       }
 
       if (event.getText() && !event.getNLP()) {
-        const nlpAdapter = this.nlpService.getNLP();
         try {
-          const nlp = await nlpAdapter.parse(event.getText());
+          const helper = await this.helperService.getDefaultNluHelper();
+          const nlp = await helper.predict(event.getText());
           event.setNLP(nlp);
         } catch (err) {
           this.logger.error('Unable to perform NLP parse', err);

api/src/extensions/channels/live-chat-tester/index.channel.ts

@@ -16,7 +16,6 @@ import { SubscriberService } from '@/chat/services/subscriber.service';
 import { MenuService } from '@/cms/services/menu.service';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LoggerService } from '@/logger/logger.service';
-import { NlpService } from '@/nlp/services/nlp.service';
 import { SettingService } from '@/setting/services/setting.service';
 import { WebsocketGateway } from '@/websocket/websocket.gateway';
 
@@ -34,7 +33,6 @@ export default class LiveChatTesterHandler extends BaseWebChannelHandler<
   constructor(
     settingService: SettingService,
     channelService: ChannelService,
-    nlpService: NlpService,
     logger: LoggerService,
     eventEmitter: EventEmitter2,
     i18n: I18nService,
@@ -49,7 +47,6 @@ export default class LiveChatTesterHandler extends BaseWebChannelHandler<
       DEFAULT_LIVE_CHAT_TEST_SETTINGS,
       settingService,
       channelService,
-      nlpService,
       logger,
       eventEmitter,
       i18n,

api/src/extensions/channels/offline/__test__/index.spec.ts

@@ -36,7 +36,6 @@ import { MenuModel } from '@/cms/schemas/menu.schema';
 import { MenuService } from '@/cms/services/menu.service';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LoggerService } from '@/logger/logger.service';
-import { NlpService } from '@/nlp/services/nlp.service';
 import { SettingService } from '@/setting/services/setting.service';
 import { UserModel } from '@/user/schemas/user.schema';
 import { installMessageFixtures } from '@/utils/test/fixtures/message';
@@ -92,12 +91,6 @@ describe('Offline Handler', () => {
             })),
           },
         },
-        {
-          provide: NlpService,
-          useValue: {
-            getNLP: jest.fn(() => undefined),
-          },
-        },
         ChannelService,
         WebsocketGateway,
         SocketEventDispatcherService,

api/src/extensions/channels/offline/base-web-channel.ts

@@ -53,7 +53,6 @@ import { MenuService } from '@/cms/services/menu.service';
 import { config } from '@/config';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LoggerService } from '@/logger/logger.service';
-import { NlpService } from '@/nlp/services/nlp.service';
 import { SettingService } from '@/setting/services/setting.service';
 import { SocketRequest } from '@/websocket/utils/socket-request';
 import { SocketResponse } from '@/websocket/utils/socket-response';
@@ -72,7 +71,6 @@ export default class BaseWebChannelHandler<
     settings: ChannelSetting<N>[],
     settingService: SettingService,
     channelService: ChannelService,
-    nlpService: NlpService,
     logger: LoggerService,
     protected readonly eventEmitter: EventEmitter2,
     protected readonly i18n: I18nService,
@@ -82,7 +80,7 @@ export default class BaseWebChannelHandler<
     protected readonly menuService: MenuService,
     private readonly websocketGateway: WebsocketGateway,
   ) {
-    super(name, settings, settingService, channelService, nlpService, logger);
+    super(name, settings, settingService, channelService, logger);
   }
 
   /**

api/src/extensions/channels/offline/index.channel.ts

@@ -16,7 +16,6 @@ import { SubscriberService } from '@/chat/services/subscriber.service';
 import { MenuService } from '@/cms/services/menu.service';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LoggerService } from '@/logger/logger.service';
-import { NlpService } from '@/nlp/services/nlp.service';
 import { SettingService } from '@/setting/services/setting.service';
 import { WebsocketGateway } from '@/websocket/websocket.gateway';
 
@@ -30,7 +29,6 @@ export default class OfflineHandler extends BaseWebChannelHandler<
   constructor(
     settingService: SettingService,
     channelService: ChannelService,
-    nlpService: NlpService,
     logger: LoggerService,
     eventEmitter: EventEmitter2,
     i18n: I18nService,
@@ -45,7 +43,6 @@ export default class OfflineHandler extends BaseWebChannelHandler<
       DEFAULT_OFFLINE_SETTINGS,
       settingService,
       channelService,
-      nlpService,
       logger,
       eventEmitter,
       i18n,

api/src/extensions/helpers/core-nlu/__test__/index.mock.ts

@@ -6,11 +6,11 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import { Nlp } from '@/nlp/lib/types';
+import { Nlp } from '@/helper/types';
 
-import { DatasetType, NlpParseResultType } from '../types';
+import { NlpParseResultType, RasaNlu } from '../types';
 
-export const nlpEmptyFormated: DatasetType = {
+export const nlpEmptyFormated: RasaNlu.Dataset = {
   common_examples: [],
   regex_features: [],
   lookup_tables: [
@@ -35,7 +35,7 @@ export const nlpEmptyFormated: DatasetType = {
   ],
 };
 
-export const nlpFormatted: DatasetType = {
+export const nlpFormatted: RasaNlu.Dataset = {
   common_examples: [
     {
       text: 'Hello',

api/src/extensions/helpers/core-nlu/__test__/index.spec.ts

@@ -12,31 +12,19 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 import { MongooseModule } from '@nestjs/mongoose';
 import { Test, TestingModule } from '@nestjs/testing';
 
+import { HelperService } from '@/helper/helper.service';
 import { LanguageRepository } from '@/i18n/repositories/language.repository';
 import { LanguageModel } from '@/i18n/schemas/language.schema';
 import { LanguageService } from '@/i18n/services/language.service';
 import { LoggerService } from '@/logger/logger.service';
-import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository';
-import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository';
-import { NlpSampleRepository } from '@/nlp/repositories/nlp-sample.repository';
-import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository';
-import { NlpEntityModel } from '@/nlp/schemas/nlp-entity.schema';
-import { NlpSampleEntityModel } from '@/nlp/schemas/nlp-sample-entity.schema';
-import { NlpSampleModel } from '@/nlp/schemas/nlp-sample.schema';
-import { NlpValueModel } from '@/nlp/schemas/nlp-value.schema';
-import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
-import { NlpSampleEntityService } from '@/nlp/services/nlp-sample-entity.service';
-import { NlpSampleService } from '@/nlp/services/nlp-sample.service';
-import { NlpValueService } from '@/nlp/services/nlp-value.service';
-import { NlpService } from '@/nlp/services/nlp.service';
 import { SettingService } from '@/setting/services/setting.service';
-import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity';
+import { installLanguageFixtures } from '@/utils/test/fixtures/language';
 import {
   closeInMongodConnection,
   rootMongooseTestModule,
 } from '@/utils/test/test';
 
-import DefaultNlpHelper from '../index.nlp.helper';
+import CoreNluHelper from '../index.helper';
 
 import { entitiesMock, samplesMock } from './__mock__/base.mock';
 import {
@@ -46,45 +34,31 @@ import {
   nlpParseResult,
 } from './index.mock';
 
-describe('NLP Default Helper', () => {
+describe('Core NLU Helper', () => {
   let settingService: SettingService;
-  let nlpService: NlpService;
+  let defaultNlpHelper: CoreNluHelper;
-  let defaultNlpHelper: DefaultNlpHelper;
 
   beforeAll(async () => {
     const module: TestingModule = await Test.createTestingModule({
       imports: [
-        rootMongooseTestModule(installNlpSampleEntityFixtures),
+        rootMongooseTestModule(async () => {
-        MongooseModule.forFeature([
+          await installLanguageFixtures();
-          NlpEntityModel,
+        }),
-          NlpValueModel,
+        MongooseModule.forFeature([LanguageModel]),
-          NlpSampleModel,
-          NlpSampleEntityModel,
-          LanguageModel,
-        ]),
         HttpModule,
       ],
       providers: [
-        NlpService,
-        NlpSampleService,
-        NlpSampleRepository,
-        NlpEntityService,
-        NlpEntityRepository,
-        NlpValueService,
-        NlpValueRepository,
-        NlpSampleEntityService,
-        NlpSampleEntityRepository,
         LanguageService,
         LanguageRepository,
         EventEmitter2,
-        DefaultNlpHelper,
+        HelperService,
+        CoreNluHelper,
         LoggerService,
         {
           provide: SettingService,
           useValue: {
             getSettings: jest.fn(() => ({
-              nlp_settings: {
+              core_nlu: {
-                provider: 'default',
                 endpoint: 'path',
                 token: 'token',
                 threshold: '0.5',
@@ -103,56 +77,51 @@ describe('NLP Default Helper', () => {
       ],
     }).compile();
     settingService = module.get<SettingService>(SettingService);
-    nlpService = module.get<NlpService>(NlpService);
+    defaultNlpHelper = module.get<CoreNluHelper>(CoreNluHelper);
-    defaultNlpHelper = module.get<DefaultNlpHelper>(DefaultNlpHelper);
-    nlpService.setHelper('default', defaultNlpHelper);
-    nlpService.initNLP();
   });
 
   afterAll(closeInMongodConnection);
 
-  it('should init() properly', () => {
-    const nlp = nlpService.getNLP();
-    expect(nlp).toBeDefined();
-  });
-
   it('should format empty training set properly', async () => {
-    const nlp = nlpService.getNLP();
+    const results = await defaultNlpHelper.format([], entitiesMock);
-    const results = await nlp.format([], entitiesMock);
     expect(results).toEqual(nlpEmptyFormated);
   });
 
   it('should format training set properly', async () => {
-    const nlp = nlpService.getNLP();
+    const results = await defaultNlpHelper.format(samplesMock, entitiesMock);
-    const results = await nlp.format(samplesMock, entitiesMock);
     expect(results).toEqual(nlpFormatted);
   });
 
-  it('should return best guess from empty parse results', () => {
+  it('should return best guess from empty parse results', async () => {
-    const nlp = nlpService.getNLP();
+    const results = await defaultNlpHelper.filterEntitiesByConfidence(
-    const results = nlp.bestGuess(
       {
         entities: [],
-        intent: {},
+        intent: { name: 'greeting', confidence: 0 },
         intent_ranking: [],
         text: 'test',
       },
       false,
     );
-    expect(results).toEqual({ entities: [] });
+    expect(results).toEqual({
+      entities: [{ entity: 'intent', value: 'greeting', confidence: 0 }],
+    });
   });
 
-  it('should return best guess from parse results', () => {
+  it('should return best guess from parse results', async () => {
-    const nlp = nlpService.getNLP();
+    const results = await defaultNlpHelper.filterEntitiesByConfidence(
-    const results = nlp.bestGuess(nlpParseResult, false);
+      nlpParseResult,
+      false,
+    );
     expect(results).toEqual(nlpBestGuess);
   });
 
   it('should return best guess from parse results with threshold', async () => {
-    const nlp = nlpService.getNLP();
+    const results = await defaultNlpHelper.filterEntitiesByConfidence(
-    const results = nlp.bestGuess(nlpParseResult, true);
+      nlpParseResult,
+      true,
+    );
     const settings = await settingService.getSettings();
-    const threshold = settings.nlp_settings.threshold;
+    const threshold = settings.core_nlu.threshold;
     const thresholdGuess = {
       entities: nlpBestGuess.entities.filter(
         (g) =>

api/src/extensions/helpers/core-nlu/i18n/en/help.json

@@ -0,0 +1,5 @@
+{
+  "endpoint": "Enter the endpoint URL for the Core NLU API where requests will be sent.",
+  "token": "Provide the API token for authenticating requests to the Core NLU API.",
+  "threshold": "Set the minimum confidence score for predictions to be considered valid."
+}

api/src/extensions/helpers/core-nlu/i18n/en/label.json

@@ -0,0 +1,5 @@
+{
+  "endpoint": "Core NLU API",
+  "token": "API Token",
+  "threshold": "Confidence Threshold"
+}

api/src/extensions/helpers/core-nlu/i18n/en/title.json

@@ -0,0 +1,3 @@
+{
+  "core_nlu": "Core NLU Engine"
+}

api/src/extensions/helpers/core-nlu/i18n/fr/help.json

@@ -0,0 +1,5 @@
+{
+  "endpoint": "Entrez l'URL de point de terminaison pour l'API NLU Core où les requêtes seront envoyées.",
+  "token": "Fournissez le jeton d'API pour authentifier les requêtes à l'API NLU Core.",
+  "threshold": "Définissez le score de confiance minimum pour que les prédictions soient considérées comme valides."
+}

api/src/extensions/helpers/core-nlu/i18n/fr/label.json

@@ -0,0 +1,5 @@
+{
+  "endpoint": "API NLU Core",
+  "token": "Jeton d'API",
+  "threshold": "Seuil de Confiance"
+}

api/src/extensions/helpers/core-nlu/i18n/fr/title.json

@@ -0,0 +1,3 @@
+{
+  "core_nlu": "Core NLU Engine"
+}

api/src/extensions/helpers/core-nlu/index.d.ts

@@ -0,0 +1,14 @@
+import { CORE_NLU_HELPER_GROUP, CORE_NLU_HELPER_SETTINGS } from './settings';
+
+declare global {
+  interface Settings extends SettingTree<typeof CORE_NLU_HELPER_SETTINGS> {}
+}
+
+declare module '@nestjs/event-emitter' {
+  interface IHookExtensionsOperationMap {
+    [CORE_NLU_HELPER_GROUP]: TDefinition<
+      object,
+      SettingMapByType<typeof CORE_NLU_HELPER_SETTINGS>
+    >;
+  }
+}

api/src/extensions/helpers/core-nlu/index.helper.ts

@@ -0,0 +1,283 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { HttpService } from '@nestjs/axios';
+import { Injectable } from '@nestjs/common';
+
+import { HelperService } from '@/helper/helper.service';
+import BaseNlpHelper from '@/helper/lib/base-nlp-helper';
+import { Nlp } from '@/helper/types';
+import { LanguageService } from '@/i18n/services/language.service';
+import { LoggerService } from '@/logger/logger.service';
+import { NlpEntity, NlpEntityFull } from '@/nlp/schemas/nlp-entity.schema';
+import { NlpSampleFull } from '@/nlp/schemas/nlp-sample.schema';
+import { NlpValue } from '@/nlp/schemas/nlp-value.schema';
+import { SettingService } from '@/setting/services/setting.service';
+import { buildURL } from '@/utils/helpers/URL';
+
+import { CORE_NLU_HELPER_NAME, CORE_NLU_HELPER_SETTINGS } from './settings';
+import { NlpParseResultType, RasaNlu } from './types';
+
+@Injectable()
+export default class CoreNluHelper extends BaseNlpHelper<
+  typeof CORE_NLU_HELPER_NAME
+> {
+  constructor(
+    settingService: SettingService,
+    helperService: HelperService,
+    logger: LoggerService,
+    private readonly httpService: HttpService,
+    private readonly languageService: LanguageService,
+  ) {
+    super(
+      CORE_NLU_HELPER_NAME,
+      CORE_NLU_HELPER_SETTINGS,
+      settingService,
+      helperService,
+      logger,
+    );
+  }
+
+  /**
+   * Formats a set of NLP samples into the Rasa NLU-compatible training dataset format.
+   *
+   * @param samples - The NLP samples to format.
+   * @param entities - The NLP entities available in the dataset.
+   *
+   * @returns The formatted Rasa NLU training dataset.
+   */
+  async format(
+    samples: NlpSampleFull[],
+    entities: NlpEntityFull[],
+  ): Promise<RasaNlu.Dataset> {
+    const entityMap = NlpEntity.getEntityMap(entities);
+    const valueMap = NlpValue.getValueMap(
+      NlpValue.getValuesFromEntities(entities),
+    );
+
+    const common_examples: RasaNlu.CommonExample[] = samples
+      .filter((s) => s.entities.length > 0)
+      .map((s) => {
+        const intent = s.entities.find(
+          (e) => entityMap[e.entity].name === 'intent',
+        );
+        if (!intent) {
+          throw new Error('Unable to find the `intent` nlp entity.');
+        }
+        const sampleEntities: RasaNlu.ExampleEntity[] = s.entities
+          .filter((e) => entityMap[<string>e.entity].name !== 'intent')
+          .map((e) => {
+            const res: RasaNlu.ExampleEntity = {
+              entity: entityMap[<string>e.entity].name,
+              value: valueMap[<string>e.value].value,
+            };
+            if ('start' in e && 'end' in e) {
+              Object.assign(res, {
+                start: e.start,
+                end: e.end,
+              });
+            }
+            return res;
+          })
+          // TODO : place language at the same level as the intent
+          .concat({
+            entity: 'language',
+            value: s.language.code,
+          });
+
+        return {
+          text: s.text,
+          intent: valueMap[intent.value].value,
+          entities: sampleEntities,
+        };
+      });
+
+    const languages = await this.languageService.getLanguages();
+    const lookup_tables: RasaNlu.LookupTable[] = entities
+      .map((e) => {
+        return {
+          name: e.name,
+          elements: e.values.map((v) => {
+            return v.value;
+          }),
+        };
+      })
+      .concat({
+        name: 'language',
+        elements: Object.keys(languages),
+      });
+    const entity_synonyms = entities
+      .reduce((acc, e) => {
+        const synonyms = e.values.map((v) => {
+          return {
+            value: v.value,
+            synonyms: v.expressions,
+          };
+        });
+        return acc.concat(synonyms);
+      }, [] as RasaNlu.EntitySynonym[])
+      .filter((s) => {
+        return s.synonyms.length > 0;
+      });
+    return {
+      common_examples,
+      regex_features: [],
+      lookup_tables,
+      entity_synonyms,
+    };
+  }
+
+  /**
+   * Perform a training request
+   *
+   * @param samples - Samples to train
+   * @param entities - All available entities
+   * @returns The training result
+   */
+  async train(
+    samples: NlpSampleFull[],
+    entities: NlpEntityFull[],
+  ): Promise<any> {
+    const nluData: RasaNlu.Dataset = await this.format(samples, entities);
+    const settings = await this.getSettings();
+    // Train samples
+    return await this.httpService.axiosRef.post(
+      buildURL(settings.endpoint, `/train`),
+      nluData,
+      {
+        params: {
+          token: settings.token,
+        },
+      },
+    );
+  }
+
+  /**
+   * Perform evaluation request
+   *
+   * @param samples - Samples to evaluate
+   * @param entities - All available entities
+   * @returns Evaluation results
+   */
+  async evaluate(
+    samples: NlpSampleFull[],
+    entities: NlpEntityFull[],
+  ): Promise<any> {
+    const settings = await this.getSettings();
+    const nluTestData: RasaNlu.Dataset = await this.format(samples, entities);
+    // Evaluate model with test samples
+    return await this.httpService.axiosRef.post(
+      buildURL(settings.endpoint, `/evaluate`),
+      nluTestData,
+      {
+        params: {
+          token: settings.token,
+        },
+      },
+    );
+  }
+
+  /**
+   * Returns only the entities that have strong confidence (> than the threshold), can return an empty result
+   *
+   * @param nlp - The nlp returned result
+   * @param threshold - Whenever to apply threshold filter or not
+   *
+   * @returns The parsed entities
+   */
+  async filterEntitiesByConfidence(
+    nlp: NlpParseResultType,
+    threshold: boolean,
+  ): Promise<Nlp.ParseEntities> {
+    try {
+      let minConfidence = 0;
+      const guess: Nlp.ParseEntities = {
+        entities: nlp.entities.slice(),
+      };
+      if (threshold) {
+        const settings = await this.getSettings();
+        const threshold = settings.threshold;
+        minConfidence =
+          typeof threshold === 'string'
+            ? Number.parseFloat(threshold)
+            : threshold;
+        guess.entities = guess.entities
+          .map((e) => {
+            e.confidence =
+              typeof e.confidence === 'string'
+                ? Number.parseFloat(e.confidence)
+                : e.confidence;
+            return e;
+          })
+          .filter((e) => e.confidence >= minConfidence);
+        // Get past threshold and the highest confidence for the same entity
+        // .filter((e, idx, self) => {
+        //   const sameEntities = self.filter((s) => s.entity === e.entity);
+        //   const max = Math.max.apply(Math, sameEntities.map((e) => { return e.confidence; }));
+        //   return e.confidence === max;
+        // });
+      }
+
+      ['intent', 'language'].forEach((trait) => {
+        if (trait in nlp && (nlp as any)[trait].confidence >= minConfidence) {
+          guess.entities.push({
+            entity: trait,
+            value: (nlp as any)[trait].name,
+            confidence: (nlp as any)[trait].confidence,
+          });
+        }
+      });
+      return guess;
+    } catch (e) {
+      this.logger.error(
+        'Core NLU Helper : Unable to parse nlp result to extract best guess!',
+        e,
+      );
+      return {
+        entities: [],
+      };
+    }
+  }
+
+  /**
+   * Returns only the entities that have strong confidence (> than the threshold), can return an empty result
+   *
+   * @param text - The text to parse
+   * @param threshold - Whenever to apply threshold filter or not
+   * @param project - Whenever to request a specific model
+   *
+   * @returns The prediction
+   */
+  async predict(
+    text: string,
+    threshold: boolean,
+    project: string = 'current',
+  ): Promise<Nlp.ParseEntities> {
+    try {
+      const settings = await this.getSettings();
+      const { data: nlp } =
+        await this.httpService.axiosRef.post<NlpParseResultType>(
+          buildURL(settings.endpoint, '/parse'),
+          {
+            q: text,
+            project,
+          },
+          {
+            params: {
+              token: settings.token,
+            },
+          },
+        );
+
+      return this.filterEntitiesByConfidence(nlp, threshold);
+    } catch (err) {
+      this.logger.error('Core NLU Helper : Unable to parse nlp', err);
+      throw err;
+    }
+  }
+}

api/src/extensions/helpers/core-nlu/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "hexabot-core-nlu",
+  "version": "2.0.0",
+  "description": "The Core NLU Helper Extension for Hexabot Chatbot / Agent Builder to enable the Intent Classification and Language Detection",
+  "dependencies": {},
+  "author": "Hexastack",
+  "license": "AGPL-3.0-only"
+}

api/src/extensions/helpers/core-nlu/settings.ts

@@ -0,0 +1,32 @@
+import { HelperSetting } from '@/helper/types';
+import { SettingType } from '@/setting/schemas/types';
+
+export const CORE_NLU_HELPER_NAME = 'core-nlu';
+
+export const CORE_NLU_HELPER_GROUP = 'core_nlu';
+
+export const CORE_NLU_HELPER_SETTINGS = [
+  {
+    group: CORE_NLU_HELPER_GROUP,
+    label: 'endpoint',
+    value: 'http://nlu-api:5000/',
+    type: SettingType.text,
+  },
+  {
+    group: CORE_NLU_HELPER_GROUP,
+    label: 'token',
+    value: 'token123',
+    type: SettingType.text,
+  },
+  {
+    group: CORE_NLU_HELPER_GROUP,
+    label: 'threshold',
+    value: 0.1,
+    type: SettingType.number,
+    config: {
+      min: 0,
+      max: 1,
+      step: 0.01,
+    },
+  },
+] as const satisfies HelperSetting<typeof CORE_NLU_HELPER_NAME>[];

api/src/extensions/helpers/core-nlu/types.ts

@@ -6,34 +6,36 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-export interface ExampleEntity {
+export namespace RasaNlu {
-  entity: string;
+  export interface ExampleEntity {
-  value: string;
+    entity: string;
-  start?: number;
+    value: string;
-  end?: number;
+    start?: number;
-}
+    end?: number;
+  }
 
-export interface CommonExample {
+  export interface CommonExample {
-  text: string;
+    text: string;
-  intent: string;
+    intent: string;
-  entities: ExampleEntity[];
+    entities: ExampleEntity[];
-}
+  }
 
-export interface LookupTable {
+  export interface LookupTable {
-  name: string;
+    name: string;
-  elements: string[];
+    elements: string[];
-}
+  }
 
-export interface EntitySynonym {
+  export interface EntitySynonym {
-  value: string;
+    value: string;
-  synonyms: string[];
+    synonyms: string[];
-}
+  }
 
-export interface DatasetType {
+  export interface Dataset {
-  common_examples: CommonExample[];
+    common_examples: CommonExample[];
-  regex_features: any[];
+    regex_features: any[];
-  lookup_tables: LookupTable[];
+    lookup_tables: LookupTable[];
-  entity_synonyms: EntitySynonym[];
+    entity_synonyms: EntitySynonym[];
+  }
 }
 
 export interface ParseEntity {

api/src/extensions/helpers/nlp/default/index.nlp.helper.ts

@@ -1,215 +0,0 @@
-/*
- * Copyright © 2024 Hexastack. All rights reserved.
- *
- * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
- * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
- * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
- */
-
-import { HttpService } from '@nestjs/axios';
-import { Injectable } from '@nestjs/common';
-
-import { LoggerService } from '@/logger/logger.service';
-import BaseNlpHelper from '@/nlp/lib/BaseNlpHelper';
-import { Nlp } from '@/nlp/lib/types';
-import { NlpEntityFull } from '@/nlp/schemas/nlp-entity.schema';
-import { NlpSampleFull } from '@/nlp/schemas/nlp-sample.schema';
-import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
-import { NlpSampleService } from '@/nlp/services/nlp-sample.service';
-import { NlpService } from '@/nlp/services/nlp.service';
-import { buildURL } from '@/utils/helpers/URL';
-
-import { DatasetType, NlpParseResultType } from './types';
-
-@Injectable()
-export default class DefaultNlpHelper extends BaseNlpHelper {
-  /**
-   * Instantiate a nlp helper
-   *
-   * @param settings - NLP settings
-   */
-  constructor(
-    logger: LoggerService,
-    nlpService: NlpService,
-    nlpSampleService: NlpSampleService,
-    nlpEntityService: NlpEntityService,
-    protected readonly httpService: HttpService,
-  ) {
-    super(logger, nlpService, nlpSampleService, nlpEntityService);
-  }
-
-  onModuleInit() {
-    this.nlpService.setHelper(this.getName(), this);
-  }
-
-  getName() {
-    return 'default';
-  }
-
-  /**
-   * Return training dataset in compatible format
-   *
-   * @param samples - Sample to train
-   * @param entities - All available entities
-   * @returns {DatasetType} - The formatted RASA training set
-   */
-  async format(
-    samples: NlpSampleFull[],
-    entities: NlpEntityFull[],
-  ): Promise<DatasetType> {
-    const nluData = await this.nlpSampleService.formatRasaNlu(
-      samples,
-      entities,
-    );
-
-    return nluData;
-  }
-
-  /**
-   * Perform Rasa training request
-   *
-   * @param samples - Samples to train
-   * @param entities - All available entities
-   * @returns {Promise<any>} - Rasa training result
-   */
-  async train(
-    samples: NlpSampleFull[],
-    entities: NlpEntityFull[],
-  ): Promise<any> {
-    const self = this;
-    const nluData: DatasetType = await self.format(samples, entities);
-    // Train samples
-    const result = await this.httpService.axiosRef.post(
-      buildURL(this.settings.endpoint, `/train`),
-      nluData,
-      {
-        params: {
-          token: this.settings.token,
-        },
-      },
-    );
-    // Mark samples as trained
-    await this.nlpSampleService.updateMany(
-      { type: 'train' },
-      { trained: true },
-    );
-    return result;
-  }
-
-  /**
-   * Perform evaluation request
-   *
-   * @param samples - Samples to evaluate
-   * @param entities - All available entities
-   * @returns {Promise<any>} - Evaluation results
-   */
-  async evaluate(
-    samples: NlpSampleFull[],
-    entities: NlpEntityFull[],
-  ): Promise<any> {
-    const self = this;
-    const nluTestData: DatasetType = await self.format(samples, entities);
-    // Evaluate model with test samples
-    return await this.httpService.axiosRef.post(
-      buildURL(this.settings.endpoint, `/evaluate`),
-      nluTestData,
-      {
-        params: {
-          token: this.settings.token,
-        },
-      },
-    );
-  }
-
-  /**
-   * Returns only the entities that have strong confidence (> than the threshold), can return an empty result
-   *
-   * @param nlp - The nlp returned result
-   * @param threshold - Whenever to apply threshold filter or not
-   * @returns {Nlp.ParseEntities}
-   */
-  bestGuess(nlp: NlpParseResultType, threshold: boolean): Nlp.ParseEntities {
-    try {
-      let minConfidence = 0;
-      const guess: Nlp.ParseEntities = {
-        entities: nlp.entities.slice(),
-      };
-      if (threshold) {
-        const threshold = this.settings.threshold;
-        minConfidence =
-          typeof threshold === 'string'
-            ? Number.parseFloat(threshold)
-            : threshold;
-        guess.entities = guess.entities
-          .map((e) => {
-            e.confidence =
-              typeof e.confidence === 'string'
-                ? Number.parseFloat(e.confidence)
-                : e.confidence;
-            return e;
-          })
-          .filter((e) => e.confidence >= minConfidence);
-        // Get past threshold and the highest confidence for the same entity
-        // .filter((e, idx, self) => {
-        //   const sameEntities = self.filter((s) => s.entity === e.entity);
-        //   const max = Math.max.apply(Math, sameEntities.map((e) => { return e.confidence; }));
-        //   return e.confidence === max;
-        // });
-      }
-
-      ['intent', 'language'].forEach((trait) => {
-        if (trait in nlp && (nlp as any)[trait].confidence >= minConfidence) {
-          guess.entities.push({
-            entity: trait,
-            value: (nlp as any)[trait].name,
-            confidence: (nlp as any)[trait].confidence,
-          });
-        }
-      });
-      return guess;
-    } catch (e) {
-      this.logger.error(
-        'NLP RasaAdapter : Unable to parse nlp result to extract best guess!',
-        e,
-      );
-      return {
-        entities: [],
-      };
-    }
-  }
-
-  /**
-   * Returns only the entities that have strong confidence (> than the threshold), can return an empty result
-   *
-   * @param text - The text to parse
-   * @param threshold - Whenever to apply threshold filter or not
-   * @param project - Whenever to request a specific model
-   * @returns {Promise<Nlp.ParseEntities>}
-   */
-  async parse(
-    text: string,
-    threshold: boolean,
-    project: string = 'current',
-  ): Promise<Nlp.ParseEntities> {
-    try {
-      const { data: nlp } =
-        await this.httpService.axiosRef.post<NlpParseResultType>(
-          buildURL(this.settings.endpoint, '/parse'),
-          {
-            q: text,
-            project,
-          },
-          {
-            params: {
-              token: this.settings.token,
-            },
-          },
-        );
-
-      return this.bestGuess(nlp, threshold);
-    } catch (err) {
-      this.logger.error('NLP RasaAdapter : Unable to parse nlp', err);
-      throw err;
-    }
-  }
-}

api/src/helper/helper.controller.ts

@@ -6,22 +6,26 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import { Controller, Get } from '@nestjs/common';
+import { Controller, Get, Param } from '@nestjs/common';
 
-import { NlpService } from '../services/nlp.service';
+import { Roles } from '@/utils/decorators/roles.decorator';
 
-@Controller('nlp')
+import { HelperService } from './helper.service';
-export class NlpController {
+import { HelperType } from './types';
-  constructor(private readonly nlpService: NlpService) {}
+
+@Controller('helper')
+export class HelperController {
+  constructor(private readonly helperService: HelperService) {}
 
   /**
-   * Retrieves a list of NLP helpers.
+   * Retrieves a list of helpers.
    *
    * @returns An array of objects containing the name of each NLP helper.
    */
-  @Get()
+  @Roles('public')
-  getNlpHelpers(): { name: string }[] {
+  @Get(':type')
-    return this.nlpService.getAll().map((helper) => {
+  getHelpers(@Param('type') type: HelperType) {
+    return this.helperService.getAllByType(type).map((helper) => {
       return {
         name: helper.getName(),
       };

api/src/helper/helper.module.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { HttpModule } from '@nestjs/axios';
+import { Global, Module } from '@nestjs/common';
+import { InjectDynamicProviders } from 'nestjs-dynamic-providers';
+
+import { HelperController } from './helper.controller';
+import { HelperService } from './helper.service';
+
+@Global()
+@InjectDynamicProviders('dist/extensions/**/*.helper.js')
+@Module({
+  imports: [HttpModule],
+  controllers: [HelperController],
+  providers: [HelperService],
+  exports: [HelperService],
+})
+export class HelperModule {}

api/src/helper/helper.service.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { Injectable } from '@nestjs/common';
+
+import { LoggerService } from '@/logger/logger.service';
+import { SettingService } from '@/setting/services/setting.service';
+
+import BaseHelper from './lib/base-helper';
+import { HelperRegistry, HelperType, TypeOfHelper } from './types';
+
+@Injectable()
+export class HelperService {
+  private registry: HelperRegistry = new Map();
+
+  constructor(
+    private readonly settingService: SettingService,
+    private readonly logger: LoggerService,
+  ) {
+    // Init empty registry
+    Object.values(HelperType).forEach((type: HelperType) => {
+      this.registry.set(type, new Map());
+    });
+  }
+
+  /**
+   * Registers a helper.
+   *
+   * @param name - The helper to be registered.
+   */
+  public register<H extends BaseHelper<string>>(helper: H) {
+    const helpers = this.registry.get(helper.getType());
+    helpers.set(helper.getName(), helper);
+    this.logger.log(`Helper "${helper.getName()}" has been registered!`);
+  }
+
+  /**
+   * Get a helper by name and type.
+   *
+   * @param type - The type of helper.
+   * @param name - The helper's name.
+   *
+   * @returns - The helper
+   */
+  public get<T extends HelperType>(type: T, name: string) {
+    const helpers = this.registry.get(type);
+
+    if (!helpers.has(name)) {
+      throw new Error('Uknown type of helpers');
+    }
+    return helpers.get(name) as TypeOfHelper<T>;
+  }
+
+  /**
+   * Get all helpers by type.
+   *
+   * @returns - The helpers
+   */
+  public getAllByType<T extends HelperType>(type: T) {
+    const helpers = this.registry.get(type) as Map<string, TypeOfHelper<T>>;
+
+    return Array.from(helpers.values());
+  }
+
+  /**
+   * Get default NLU helper.
+   *
+   * @returns - The helper
+   */
+  async getDefaultNluHelper() {
+    const settings = await this.settingService.getSettings();
+
+    const defaultHelper = this.get(
+      HelperType.NLU,
+      settings.chatbot_settings.default_nlu_helper,
+    );
+
+    if (!defaultHelper) {
+      throw new Error(`Unable to find default NLU helper`);
+    }
+
+    return defaultHelper;
+  }
+}

api/src/helper/lib/base-helper.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { LoggerService } from '@nestjs/common';
+
+import { SettingService } from '@/setting/services/setting.service';
+import { hyphenToUnderscore } from '@/utils/helpers/misc';
+
+import { HelperService } from '../helper.service';
+import { HelperSetting, HelperType } from '../types';
+
+export default abstract class BaseHelper<N extends string = string> {
+  protected readonly name: N;
+
+  protected readonly settings: HelperSetting<N>[] = [];
+
+  protected abstract type: HelperType;
+
+  constructor(
+    name: N,
+    settings: HelperSetting<N>[],
+    protected readonly settingService: SettingService,
+    protected readonly helperService: HelperService,
+    protected readonly logger: LoggerService,
+  ) {
+    this.name = name;
+    this.settings = settings;
+  }
+
+  onModuleInit() {
+    this.helperService.register(this);
+    this.setup();
+  }
+
+  async setup() {
+    await this.settingService.seedIfNotExist(
+      this.getName(),
+      this.settings.map((s, i) => ({
+        ...s,
+        weight: i + 1,
+      })),
+    );
+  }
+
+  /**
+   * Returns the helper's name
+   *
+   * @returns Helper's name
+   */
+  public getName() {
+    return this.name;
+  }
+
+  /**
+   * Returns the helper's group
+   * @returns Helper's group
+   */
+  protected getGroup() {
+    return hyphenToUnderscore(this.getName()) as HelperSetting<N>['group'];
+  }
+
+  /**
+   * Get the helper's type
+   *
+   * @returns Helper's type
+   */
+  public getType() {
+    return this.type;
+  }
+
+  /**
+   * Get the helper's settings
+   *
+   * @returns Helper's settings
+   */
+  async getSettings<S extends string = HyphenToUnderscore<N>>() {
+    const settings = await this.settingService.getSettings();
+    // @ts-expect-error workaround typing
+    return settings[this.getGroup() as keyof Settings] as Settings[S];
+  }
+}

api/src/helper/lib/base-nlp-helper.ts

@@ -6,17 +6,6 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-/**
- * @file NlpAdapter is an abstract class for define an NLP provider adapter
- * @author Hexastack <contact@hexastack.com>
- */
-
-/**
- * @module Services/NLP
- *
- * NlpAdapter is an abstract class from which each NLP provider adapter should extend from.
- */
-
 import { v4 as uuidv4 } from 'uuid';
 
 import { LoggerService } from '@/logger/logger.service';
@@ -31,34 +20,29 @@ import {
   NlpValueDocument,
   NlpValueFull,
 } from '@/nlp/schemas/nlp-value.schema';
+import { SettingService } from '@/setting/services/setting.service';
 
-import { NlpEntityService } from '../services/nlp-entity.service';
+import { HelperService } from '../helper.service';
-import { NlpSampleService } from '../services/nlp-sample.service';
+import { HelperSetting, HelperType, Nlp } from '../types';
-import { NlpService } from '../services/nlp.service';
 
-import { Nlp } from './types';
+import BaseHelper from './base-helper';
 
-export default abstract class BaseNlpHelper {
+// eslint-disable-next-line prettier/prettier
-  protected settings: Settings['nlp_settings'];
+export default abstract class BaseNlpHelper<
+  N extends string,
+> extends BaseHelper<N> {
+  protected readonly type: HelperType = HelperType.NLU;
 
   constructor(
-    protected readonly logger: LoggerService,
+    name: N,
-    protected readonly nlpService: NlpService,
+    settings: HelperSetting<N>[],
-    protected readonly nlpSampleService: NlpSampleService,
+    settingService: SettingService,
-    protected readonly nlpEntityService: NlpEntityService,
+    helperService: HelperService,
-  ) {}
+    logger: LoggerService,
-
+  ) {
-  setSettings(settings: Settings['nlp_settings']) {
+    super(name, settings, settingService, helperService, logger);
-    this.settings = settings;
   }
 
-  /**
-   * Returns the helper's name
-   *
-   * @returns Helper's name
-   */
-  abstract getName(): string;
-
   /**
    * Updates an entity
    *
@@ -183,7 +167,10 @@ export default abstract class BaseNlpHelper {
    *
    * @returns NLP Parsed entities
    */
-  abstract bestGuess(nlp: any, threshold: boolean): Nlp.ParseEntities;
+  abstract filterEntitiesByConfidence(
+    nlp: any,
+    threshold: boolean,
+  ): Promise<Nlp.ParseEntities>;
 
   /**
    * Returns only the entities that have strong confidence (> than the threshold), can return an empty result
@@ -194,7 +181,7 @@ export default abstract class BaseNlpHelper {
    *
    * @returns NLP Parsed entities
    */
-  abstract parse(
+  abstract predict(
     text: string,
     threshold?: boolean,
     project?: string,

api/src/helper/types.ts

@@ -0,0 +1,44 @@
+import { SettingCreateDto } from '@/setting/dto/setting.dto';
+
+import BaseHelper from './lib/base-helper';
+import BaseNlpHelper from './lib/base-nlp-helper';
+
+export namespace Nlp {
+  export interface Config {
+    endpoint?: string;
+    token: string;
+  }
+
+  export interface ParseEntity {
+    entity: string; // Entity name
+    value: string; // Value name
+    confidence: number;
+    start?: number;
+    end?: number;
+  }
+
+  export interface ParseEntities {
+    entities: ParseEntity[];
+  }
+}
+
+export enum HelperType {
+  NLU = 'nlu',
+  UTIL = 'util',
+}
+
+export type TypeOfHelper<T extends HelperType> = T extends HelperType.NLU
+  ? BaseNlpHelper<string>
+  : BaseHelper;
+
+export type HelperRegistry<H extends BaseHelper = BaseHelper> = Map<
+  HelperType,
+  Map<string, H>
+>;
+
+export type HelperSetting<N extends string = string> = Omit<
+  SettingCreateDto,
+  'group' | 'weight'
+> & {
+  group: HyphenToUnderscore<N>;
+};

api/src/nlp/controllers/nlp-sample.controller.spec.ts

@@ -17,6 +17,7 @@ import { Test, TestingModule } from '@nestjs/testing';
 import { AttachmentRepository } from '@/attachment/repositories/attachment.repository';
 import { AttachmentModel } from '@/attachment/schemas/attachment.schema';
 import { AttachmentService } from '@/attachment/services/attachment.service';
+import { HelperService } from '@/helper/helper.service';
 import { LanguageRepository } from '@/i18n/repositories/language.repository';
 import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
 import { I18nService } from '@/i18n/services/i18n.service';
@@ -98,6 +99,7 @@ describe('NlpSampleController', () => {
         LanguageService,
         EventEmitter2,
         NlpService,
+        HelperService,
         SettingRepository,
         SettingService,
         SettingSeeder,

api/src/nlp/controllers/nlp-sample.controller.ts

@@ -17,6 +17,7 @@ import {
   Delete,
   Get,
   HttpCode,
+  InternalServerErrorException,
   NotFoundException,
   Param,
   Patch,
@@ -33,6 +34,7 @@ import Papa from 'papaparse';
 
 import { AttachmentService } from '@/attachment/services/attachment.service';
 import { config } from '@/config';
+import { HelperService } from '@/helper/helper.service';
 import { LanguageService } from '@/i18n/services/language.service';
 import { CsrfInterceptor } from '@/interceptors/csrf.interceptor';
 import { LoggerService } from '@/logger/logger.service';
@@ -72,6 +74,7 @@ export class NlpSampleController extends BaseController<
     private readonly logger: LoggerService,
     private readonly nlpService: NlpService,
     private readonly languageService: LanguageService,
+    private readonly helperService: HelperService,
   ) {
     super(nlpSampleService);
   }
@@ -93,7 +96,8 @@ export class NlpSampleController extends BaseController<
       type ? { type } : {},
     );
     const entities = await this.nlpEntityService.findAllAndPopulate();
-    const result = await this.nlpSampleService.formatRasaNlu(samples, entities);
+    const helper = await this.helperService.getDefaultNluHelper();
+    const result = helper.format(samples, entities);
 
     // Sending the JSON data as a file
     const buffer = Buffer.from(JSON.stringify(result));
@@ -171,7 +175,8 @@ export class NlpSampleController extends BaseController<
    */
   @Get('message')
   async message(@Query('text') text: string) {
-    return this.nlpService.getNLP().parse(text);
+    const helper = await this.helperService.getDefaultNluHelper();
+    return helper.predict(text);
   }
 
   /**
@@ -201,7 +206,21 @@ export class NlpSampleController extends BaseController<
     const { samples, entities } =
       await this.getSamplesAndEntitiesByType('train');
 
-    return await this.nlpService.getNLP().train(samples, entities);
+    try {
+      const helper = await this.helperService.getDefaultNluHelper();
+      const response = await helper.train(samples, entities);
+      // Mark samples as trained
+      await this.nlpSampleService.updateMany(
+        { type: 'train' },
+        { trained: true },
+      );
+      return response;
+    } catch (err) {
+      this.logger.error(err);
+      throw new InternalServerErrorException(
+        'Unable to perform the train operation',
+      );
+    }
   }
 
   /**
@@ -214,7 +233,8 @@ export class NlpSampleController extends BaseController<
     const { samples, entities } =
       await this.getSamplesAndEntitiesByType('test');
 
-    return await this.nlpService.getNLP().evaluate(samples, entities);
+    const helper = await this.helperService.getDefaultNluHelper();
+    return await helper.evaluate(samples, entities);
   }
 
   /**

api/src/nlp/nlp.module.ts

@@ -16,7 +16,6 @@ import { AttachmentModule } from '@/attachment/attachment.module';
 import { NlpEntityController } from './controllers/nlp-entity.controller';
 import { NlpSampleController } from './controllers/nlp-sample.controller';
 import { NlpValueController } from './controllers/nlp-value.controller';
-import { NlpController } from './controllers/nlp.controller';
 import { NlpEntityRepository } from './repositories/nlp-entity.repository';
 import { NlpSampleEntityRepository } from './repositories/nlp-sample-entity.repository';
 import { NlpSampleRepository } from './repositories/nlp-sample.repository';
@@ -45,12 +44,7 @@ import { NlpService } from './services/nlp.service';
     AttachmentModule,
     HttpModule,
   ],
-  controllers: [
+  controllers: [NlpEntityController, NlpValueController, NlpSampleController],
-    NlpEntityController,
-    NlpValueController,
-    NlpSampleController,
-    NlpController,
-  ],
   providers: [
     NlpEntityRepository,
     NlpValueRepository,

api/src/nlp/services/nlp-sample.service.spec.ts

@@ -14,6 +14,7 @@ import { Test, TestingModule } from '@nestjs/testing';
 import { LanguageRepository } from '@/i18n/repositories/language.repository';
 import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
 import { LanguageService } from '@/i18n/services/language.service';
+import { LoggerService } from '@/logger/logger.service';
 import { nlpSampleFixtures } from '@/utils/test/fixtures/nlpsample';
 import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity';
 import { getPageQuery } from '@/utils/test/pagination';
@@ -28,10 +29,10 @@ import { NlpSampleRepository } from '../repositories/nlp-sample.repository';
 import { NlpValueRepository } from '../repositories/nlp-value.repository';
 import { NlpEntityModel } from '../schemas/nlp-entity.schema';
 import {
-  NlpSampleEntityModel,
   NlpSampleEntity,
+  NlpSampleEntityModel,
 } from '../schemas/nlp-sample-entity.schema';
-import { NlpSampleModel, NlpSample } from '../schemas/nlp-sample.schema';
+import { NlpSample, NlpSampleModel } from '../schemas/nlp-sample.schema';
 import { NlpValueModel } from '../schemas/nlp-value.schema';
 
 import { NlpEntityService } from './nlp-entity.service';
@@ -72,6 +73,7 @@ describe('NlpSampleService', () => {
         NlpValueService,
         LanguageService,
         EventEmitter2,
+        LoggerService,
         {
           provide: CACHE_MANAGER,
           useValue: {

api/src/nlp/services/nlp-sample.service.ts

@@ -9,25 +9,20 @@
 import { Injectable } from '@nestjs/common';
 import { OnEvent } from '@nestjs/event-emitter';
 
-import {
+import { AnyMessage } from '@/chat/schemas/types/message';
-  CommonExample,
-  DatasetType,
-  EntitySynonym,
-  ExampleEntity,
-  LookupTable,
-} from '@/extensions/helpers/nlp/default/types';
 import { Language } from '@/i18n/schemas/language.schema';
 import { LanguageService } from '@/i18n/services/language.service';
+import { LoggerService } from '@/logger/logger.service';
 import { BaseService } from '@/utils/generics/base-service';
 
+import { NlpSampleCreateDto } from '../dto/nlp-sample.dto';
 import { NlpSampleRepository } from '../repositories/nlp-sample.repository';
-import { NlpEntity, NlpEntityFull } from '../schemas/nlp-entity.schema';
 import {
   NlpSample,
   NlpSampleFull,
   NlpSamplePopulate,
 } from '../schemas/nlp-sample.schema';
-import { NlpValue } from '../schemas/nlp-value.schema';
+import { NlpSampleState } from '../schemas/types';
 
 @Injectable()
 export class NlpSampleService extends BaseService<
@@ -38,6 +33,7 @@ export class NlpSampleService extends BaseService<
   constructor(
     readonly repository: NlpSampleRepository,
     private readonly languageService: LanguageService,
+    private readonly logger: LoggerService,
   ) {
     super(repository);
   }
@@ -53,95 +49,6 @@ export class NlpSampleService extends BaseService<
     return await this.repository.deleteOne(id);
   }
 
-  /**
-   * Formats a set of NLP samples into the Rasa NLU-compatible training dataset format.
-   *
-   * @param samples - The NLP samples to format.
-   * @param entities - The NLP entities available in the dataset.
-   *
-   * @returns The formatted Rasa NLU training dataset.
-   */
-  async formatRasaNlu(
-    samples: NlpSampleFull[],
-    entities: NlpEntityFull[],
-  ): Promise<DatasetType> {
-    const entityMap = NlpEntity.getEntityMap(entities);
-    const valueMap = NlpValue.getValueMap(
-      NlpValue.getValuesFromEntities(entities),
-    );
-
-    const common_examples: CommonExample[] = samples
-      .filter((s) => s.entities.length > 0)
-      .map((s) => {
-        const intent = s.entities.find(
-          (e) => entityMap[e.entity].name === 'intent',
-        );
-        if (!intent) {
-          throw new Error('Unable to find the `intent` nlp entity.');
-        }
-        const sampleEntities: ExampleEntity[] = s.entities
-          .filter((e) => entityMap[<string>e.entity].name !== 'intent')
-          .map((e) => {
-            const res: ExampleEntity = {
-              entity: entityMap[<string>e.entity].name,
-              value: valueMap[<string>e.value].value,
-            };
-            if ('start' in e && 'end' in e) {
-              Object.assign(res, {
-                start: e.start,
-                end: e.end,
-              });
-            }
-            return res;
-          })
-          // TODO : place language at the same level as the intent
-          .concat({
-            entity: 'language',
-            value: s.language.code,
-          });
-
-        return {
-          text: s.text,
-          intent: valueMap[intent.value].value,
-          entities: sampleEntities,
-        };
-      });
-
-    const languages = await this.languageService.getLanguages();
-    const lookup_tables: LookupTable[] = entities
-      .map((e) => {
-        return {
-          name: e.name,
-          elements: e.values.map((v) => {
-            return v.value;
-          }),
-        };
-      })
-      .concat({
-        name: 'language',
-        elements: Object.keys(languages),
-      });
-    const entity_synonyms = entities
-      .reduce((acc, e) => {
-        const synonyms = e.values.map((v) => {
-          return {
-            value: v.value,
-            synonyms: v.expressions,
-          };
-        });
-        return acc.concat(synonyms);
-      }, [] as EntitySynonym[])
-      .filter((s) => {
-        return s.synonyms.length > 0;
-      });
-    return {
-      common_examples,
-      regex_features: [],
-      lookup_tables,
-      entity_synonyms,
-    };
-  }
-
   /**
    * When a language gets deleted, we need to set related samples to null
    *
@@ -158,4 +65,31 @@ export class NlpSampleService extends BaseService<
       },
     );
   }
+
+  @OnEvent('hook:message:preCreate')
+  async handleNewMessage(doc: AnyMessage) {
+    // If message is sent by the user then add it as an inbox sample
+    if (
+      'sender' in doc &&
+      doc.sender &&
+      'message' in doc &&
+      'text' in doc.message
+    ) {
+      const defaultLang = await this.languageService.getDefaultLanguage();
+      const record: NlpSampleCreateDto = {
+        text: doc.message.text,
+        type: NlpSampleState.inbox,
+        trained: false,
+        // @TODO : We need to define the language in the message entity
+        language: defaultLang.id,
+      };
+      try {
+        await this.findOneOrCreate(record, record);
+        this.logger.debug('User message saved as a inbox sample !');
+      } catch (err) {
+        this.logger.error('Unable to add message as a new inbox sample!', err);
+        throw err;
+      }
+    }
+  }
 }

api/src/nlp/services/nlp.service.ts

@@ -6,13 +6,12 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { OnEvent } from '@nestjs/event-emitter';
 
+import { HelperService } from '@/helper/helper.service';
 import { LoggerService } from '@/logger/logger.service';
-import { SettingService } from '@/setting/services/setting.service';
 
-import BaseNlpHelper from '../lib/BaseNlpHelper';
 import { NlpEntity, NlpEntityDocument } from '../schemas/nlp-entity.schema';
 import { NlpValue, NlpValueDocument } from '../schemas/nlp-value.schema';
 
@@ -21,93 +20,15 @@ import { NlpSampleService } from './nlp-sample.service';
 import { NlpValueService } from './nlp-value.service';
 
 @Injectable()
-export class NlpService implements OnApplicationBootstrap {
+export class NlpService {
-  private registry: Map<string, BaseNlpHelper> = new Map();
-
-  private nlp: BaseNlpHelper;
-
   constructor(
-    private readonly settingService: SettingService,
     private readonly logger: LoggerService,
     protected readonly nlpSampleService: NlpSampleService,
     protected readonly nlpEntityService: NlpEntityService,
     protected readonly nlpValueService: NlpValueService,
+    protected readonly helperService: HelperService,
   ) {}
 
-  onApplicationBootstrap() {
-    this.initNLP();
-  }
-
-  /**
-   * Registers a helper with a specific name in the registry.
-   *
-   * @param name - The name of the helper to register.
-   * @param helper - The NLP helper to be associated with the given name.
-   * @typeParam C - The type of the helper, which must extend `BaseNlpHelper`.
-   */
-  public setHelper<C extends BaseNlpHelper>(name: string, helper: C) {
-    this.registry.set(name, helper);
-  }
-
-  /**
-   * Retrieves all registered helpers.
-   *
-   * @returns An array of all helpers currently registered.
-   */
-  public getAll() {
-    return Array.from(this.registry.values());
-  }
-
-  /**
-   * Retrieves the appropriate helper based on the helper name.
-   *
-   * @param helperName - The name of the helper (messenger, offline, ...).
-   *
-   * @returns The specified helper.
-   */
-  public getHelper<C extends BaseNlpHelper>(name: string): C {
-    const handler = this.registry.get(name);
-    if (!handler) {
-      throw new Error(`NLP Helper ${name} not found`);
-    }
-    return handler as C;
-  }
-
-  async initNLP() {
-    try {
-      const settings = await this.settingService.getSettings();
-      const nlpSettings = settings.nlp_settings;
-      const helper = this.getHelper(nlpSettings.provider);
-
-      if (helper) {
-        this.nlp = helper;
-        this.nlp.setSettings(nlpSettings);
-      } else {
-        throw new Error(`Undefined NLP Helper ${nlpSettings.provider}`);
-      }
-    } catch (e) {
-      this.logger.error('NLP Service : Unable to instantiate NLP Helper !', e);
-      // throw e;
-    }
-  }
-
-  /**
-   * Retrieves the currently active NLP helper.
-   *
-   * @returns The current NLP helper.
-   */
-  getNLP() {
-    return this.nlp;
-  }
-
-  /**
-   * Handles the event triggered when NLP settings are updated. Re-initializes the NLP service.
-   */
-  @OnEvent('hook:nlp_settings:*')
-  async handleSettingsUpdate() {
-    this.initNLP();
-  }
-
   /**
    * Handles the event triggered when a new NLP entity is created. Synchronizes the entity with the external NLP provider.
    *
@@ -118,7 +39,8 @@ export class NlpService implements OnApplicationBootstrap {
   async handleEntityCreate(entity: NlpEntityDocument) {
     // Synchonize new entity with NLP
     try {
-      const foreignId = await this.getNLP().addEntity(entity);
+      const helper = await this.helperService.getDefaultNluHelper();
+      const foreignId = await helper.addEntity(entity);
       this.logger.debug('New entity successfully synced!', foreignId);
       return await this.nlpEntityService.updateOne(entity._id, {
         foreign_id: foreignId,
@@ -138,7 +60,8 @@ export class NlpService implements OnApplicationBootstrap {
   async handleEntityUpdate(entity: NlpEntity) {
     // Synchonize new entity with NLP provider
     try {
-      await this.getNLP().updateEntity(entity);
+      const helper = await this.helperService.getDefaultNluHelper();
+      await helper.updateEntity(entity);
       this.logger.debug('Updated entity successfully synced!', entity);
     } catch (err) {
       this.logger.error('Unable to sync updated entity', err);
@@ -154,7 +77,8 @@ export class NlpService implements OnApplicationBootstrap {
   async handleEntityDelete(entity: NlpEntity) {
     // Synchonize new entity with NLP provider
     try {
-      await this.getNLP().deleteEntity(entity.foreign_id);
+      const helper = await this.helperService.getDefaultNluHelper();
+      await helper.deleteEntity(entity.foreign_id);
       this.logger.debug('Deleted entity successfully synced!', entity);
     } catch (err) {
       this.logger.error('Unable to sync deleted entity', err);
@@ -172,7 +96,8 @@ export class NlpService implements OnApplicationBootstrap {
   async handleValueCreate(value: NlpValueDocument) {
     // Synchonize new value with NLP provider
     try {
-      const foreignId = await this.getNLP().addValue(value);
+      const helper = await this.helperService.getDefaultNluHelper();
+      const foreignId = await helper.addValue(value);
       this.logger.debug('New value successfully synced!', foreignId);
       return await this.nlpValueService.updateOne(value._id, {
         foreign_id: foreignId,
@@ -192,7 +117,8 @@ export class NlpService implements OnApplicationBootstrap {
   async handleValueUpdate(value: NlpValue) {
     // Synchonize new value with NLP provider
     try {
-      await this.getNLP().updateValue(value);
+      const helper = await this.helperService.getDefaultNluHelper();
+      await helper.updateValue(value);
       this.logger.debug('Updated value successfully synced!', value);
     } catch (err) {
       this.logger.error('Unable to sync updated value', err);
@@ -208,10 +134,11 @@ export class NlpService implements OnApplicationBootstrap {
   async handleValueDelete(value: NlpValue) {
     // Synchonize new value with NLP provider
     try {
+      const helper = await this.helperService.getDefaultNluHelper();
       const populatedValue = await this.nlpValueService.findOneAndPopulate(
         value.id,
       );
-      await this.getNLP().deleteValue(populatedValue);
+      await helper.deleteValue(populatedValue);
       this.logger.debug('Deleted value successfully synced!', value);
     } catch (err) {
       this.logger.error('Unable to sync deleted value', err);

api/src/setting/seeds/setting.seed-model.ts

@@ -10,12 +10,33 @@ import { SettingCreateDto } from '../dto/setting.dto';
 import { SettingType } from '../schemas/types';
 
 export const DEFAULT_SETTINGS = [
+  {
+    group: 'chatbot_settings',
+    label: 'default_nlu_helper',
+    value: 'core-nlu',
+    type: SettingType.select,
+    config: {
+      multiple: false,
+      allowCreate: false,
+      entity: 'Helper',
+      idKey: 'name',
+      labelKey: 'name',
+    },
+    weight: 1,
+  },
   {
     group: 'chatbot_settings',
     label: 'global_fallback',
     value: true,
     type: SettingType.checkbox,
-    weight: 1,
+    weight: 3,
+  },
+  {
+    group: 'chatbot_settings',
+    label: 'global_fallback',
+    value: true,
+    type: SettingType.checkbox,
+    weight: 4,
   },
   {
     group: 'chatbot_settings',
@@ -26,11 +47,11 @@ export const DEFAULT_SETTINGS = [
     config: {
       multiple: false,
       allowCreate: false,
-      source: '/Block/',
+      entity: 'Block',
-      valueKey: 'id',
+      idKey: 'id',
       labelKey: 'name',
     },
-    weight: 2,
+    weight: 5,
   },
   {
     group: 'chatbot_settings',
@@ -40,41 +61,7 @@ export const DEFAULT_SETTINGS = [
       "I'm really sorry but i don't quite understand what you are saying :(",
     ] as string[],
     type: SettingType.multiple_text,
-    weight: 3,
+    weight: 6,
-  },
-  {
-    group: 'nlp_settings',
-    label: 'provider',
-    value: 'default',
-    options: ['default'],
-    type: SettingType.select,
-    weight: 1,
-  },
-  {
-    group: 'nlp_settings',
-    label: 'endpoint',
-    value: 'http://nlu-api:5000/',
-    type: SettingType.text,
-    weight: 2,
-  },
-  {
-    group: 'nlp_settings',
-    label: 'token',
-    value: 'token123',
-    type: SettingType.text,
-    weight: 3,
-  },
-  {
-    group: 'nlp_settings',
-    label: 'threshold',
-    value: 0.1,
-    type: SettingType.number,
-    config: {
-      min: 0,
-      max: 1,
-      step: 0.01,
-    },
-    weight: 4,
   },
   {
     group: 'contact',

api/src/utils/test/mocks/nlp.ts

@@ -6,7 +6,7 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import { Nlp } from '@/nlp/lib/types';
+import { Nlp } from '@/helper/types';
 
 export const nlpEntitiesGreeting: Nlp.ParseEntities = {
   entities: [

frontend/public/locales/en/chatbot_settings.json

@@ -4,10 +4,12 @@
   },
   "label": {
     "global_fallback": "Enable Global Fallback?",
-    "fallback_message": "Fallback Message"
+    "fallback_message": "Fallback Message",
+    "default_nlu_helper": "Default NLU Helper"
   },
   "help": {
     "global_fallback": "Global fallback allows you to send custom messages when user entry does not match any of the block messages.",
-    "fallback_message": "If no fallback block is selected, then one of these messages will be sent."
+    "fallback_message": "If no fallback block is selected, then one of these messages will be sent.",
+    "default_nlu_helper": "The NLU helper is responsible for processing and understanding user inputs, including tasks like intent prediction, language detection, and entity recognition."
   }
 }

frontend/public/locales/fr/chatbot_settings.json

@@ -1,13 +1,15 @@
 {
   "title": {
-    "chatbot_settings": "Chatbot"
+    "chatbot_settings": "Paramètres du Chatbot"
   },
   "label": {
-    "global_fallback": "Activer le message de secours global?",
+    "global_fallback": "Activer la réponse de secours globale ?",
-    "fallback_message": "Message de secours"
+    "fallback_message": "Message de secours",
+    "default_nlu_helper": "Utilitaire NLU par défaut"
   },
   "help": {
-    "global_fallback": "Le message de secours global vous permet d'envoyer des messages personnalisés lorsque le message de l'utilisateur ne déclenche aucun bloc de message.",
+    "global_fallback": "La réponse de secours globale vous permet d'envoyer des messages personnalisés lorsque l'entrée de l'utilisateur ne correspond à aucun des messages des blocs.",
-    "fallback_message": "Si aucun bloc de secours n'est spécifié, alors de ces messages sera envoyé."
+    "fallback_message": "Si aucun bloc de secours n'est sélectionné, l'un de ces messages sera envoyé.",
+    "default_nlu_helper": "Utilitaire du traitement et de la compréhension des entrées des utilisateurs, incluant des tâches telles que la prédiction d'intention, la détection de langue et la reconnaissance d'entités."
   }
 }

frontend/src/app-components/inputs/AutoCompleteSelect.tsx

@@ -11,7 +11,7 @@ import Autocomplete, {
   AutocompleteProps,
   AutocompleteValue,
 } from "@mui/material/Autocomplete";
-import { useState, useCallback, useMemo, useEffect, forwardRef } from "react";
+import { forwardRef, useCallback, useEffect, useMemo, useState } from "react";
 
 import { Input } from "@/app-components/inputs/Input";
 

frontend/src/components/settings/SettingInput.tsx

@@ -19,6 +19,7 @@ import { PasswordInput } from "@/app-components/inputs/PasswordInput";
 import { useTranslate } from "@/hooks/useTranslate";
 import { EntityType, Format } from "@/services/types";
 import { IBlock } from "@/types/block.types";
+import { IHelper } from "@/types/helper.types";
 import { ISetting } from "@/types/setting.types";
 import { MIME_TYPES } from "@/utils/attachment";
 
@@ -115,11 +116,29 @@ const SettingInput: React.FC<RenderSettingInputProps> = ({
             format={Format.BASIC}
             labelKey="name"
             label={t("label.fallback_block")}
+            helperText={t("help.fallback_block")}
             multiple={false}
             onChange={(_e, selected, ..._) => onChange(selected?.id)}
             {...rest}
           />
         );
+      } else if (setting.label === "default_nlu_helper") {
+        const { onChange, ...rest } = field;
+
+        return (
+          <AutoCompleteEntitySelect<IHelper, "name", false>
+            searchFields={["name"]}
+            entity={EntityType.NLU_HELPER}
+            format={Format.BASIC}
+            labelKey="name"
+            idKey="name"
+            label={t("label.default_nlu_helper")}
+            helperText={t("help.default_nlu_helper")}
+            multiple={false}
+            onChange={(_e, selected, ..._) => onChange(selected?.name)}
+            {...rest}
+          />
+        );
       }
 
       return (

frontend/src/services/api.class.ts

@@ -64,6 +64,8 @@ export const ROUTES = {
   [EntityType.TRANSLATION]: "/translation",
   [EntityType.ATTACHMENT]: "/attachment",
   [EntityType.CHANNEL]: "/channel",
+  [EntityType.HELPER]: "/helper",
+  [EntityType.NLU_HELPER]: "/helper/nlu",
 } as const;
 
 export class ApiClient {

frontend/src/services/entities.ts

@@ -284,6 +284,19 @@ export const ChannelEntity = new schema.Entity(EntityType.CHANNEL, undefined, {
   idAttribute: ({ name }) => name,
 });
 
+export const HelperEntity = new schema.Entity(EntityType.HELPER, undefined, {
+  idAttribute: ({ name }) => name,
+});
+
+export const NluHelperEntity = new schema.Entity(
+  EntityType.NLU_HELPER,
+  undefined,
+  {
+    idAttribute: ({ name }) => name,
+  },
+);
+
+
 export const ENTITY_MAP = {
   [EntityType.SUBSCRIBER]: SubscriberEntity,
   [EntityType.LABEL]: LabelEntity,
@@ -310,4 +323,6 @@ export const ENTITY_MAP = {
   [EntityType.CUSTOM_BLOCK]: CustomBlockEntity,
   [EntityType.CUSTOM_BLOCK_SETTINGS]: CustomBlockSettingEntity,
   [EntityType.CHANNEL]: ChannelEntity,
+  [EntityType.HELPER]: HelperEntity,
+  [EntityType.NLU_HELPER]: NluHelperEntity,
 } as const;

frontend/src/services/types.ts

@@ -35,6 +35,8 @@ export enum EntityType {
   TRANSLATION = "Translation",
   ATTACHMENT = "Attachment",
   CHANNEL = "Channel",
+  HELPER = "Helper",
+  NLU_HELPER = "NluHelper",
 }
 
 export type NormalizedEntities = Record<string, Record<string, any>>;

frontend/src/types/base.types.ts

@@ -22,6 +22,7 @@ import { IChannel, IChannelAttributes } from "./channel.types";
 import { IContentType, IContentTypeAttributes } from "./content-type.types";
 import { IContent, IContentAttributes, IContentFull } from "./content.types";
 import { IContextVar, IContextVarAttributes } from "./context-var.types";
+import { IHelper, IHelperAttributes } from "./helper.types";
 import { ILabel, ILabelAttributes, ILabelFull } from "./label.types";
 import { ILanguage, ILanguageAttributes } from "./language.types";
 import {
@@ -112,6 +113,8 @@ export const POPULATE_BY_TYPE = {
   [EntityType.CUSTOM_BLOCK]: [],
   [EntityType.CUSTOM_BLOCK_SETTINGS]: [],
   [EntityType.CHANNEL]: [],
+  [EntityType.HELPER]: [],
+  [EntityType.NLU_HELPER]: [],
 } as const;
 
 export type Populate<C extends EntityType> =
@@ -200,6 +203,8 @@ export interface IEntityMapTypes {
     IMessageFull
   >;
   [EntityType.CHANNEL]: IEntityTypes<IChannelAttributes, IChannel>;
+  [EntityType.HELPER]: IEntityTypes<IHelperAttributes, IHelper>;
+  [EntityType.NLU_HELPER]: IEntityTypes<IHelperAttributes, IHelper>;
 }
 
 export type TType<TParam extends keyof IEntityMapTypes> =

frontend/src/types/helper.types.ts

@@ -6,21 +6,11 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-export namespace Nlp {
+import { IBaseSchema } from "./base.types";
-  export interface Config {
-    endpoint?: string;
-    token: string;
-  }
 
-  export interface ParseEntity {
+export interface IHelperAttributes {
-    entity: string; // Entity name
+  name: string;
-    value: string; // Value name
-    confidence: number;
-    start?: number;
-    end?: number;
-  }
-
-  export interface ParseEntities {
-    entities: ParseEntity[];
-  }
 }
+
+// @TODO: not all entities extend from IBaseSchema
+export interface IHelper extends IHelperAttributes, IBaseSchema {}