feat: refactor helpers (nlu)

api/src/helper/helper.controller.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { Controller, Get, Param } from '@nestjs/common';
+
+import { Roles } from '@/utils/decorators/roles.decorator';
+
+import { HelperService } from './helper.service';
+import { HelperType } from './types';
+
+@Controller('helper')
+export class HelperController {
+  constructor(private readonly helperService: HelperService) {}
+
+  /**
+   * Retrieves a list of helpers.
+   *
+   * @returns An array of objects containing the name of each NLP helper.
+   */
+  @Roles('public')
+  @Get(':type')
+  getHelpers(@Param('type') type: HelperType) {
+    return this.helperService.getAllByType(type).map((helper) => {
+      return {
+        name: helper.getName(),
+      };
+    });
+  }
+}

api/src/helper/helper.module.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { HttpModule } from '@nestjs/axios';
+import { Global, Module } from '@nestjs/common';
+import { InjectDynamicProviders } from 'nestjs-dynamic-providers';
+
+import { HelperController } from './helper.controller';
+import { HelperService } from './helper.service';
+
+@Global()
+@InjectDynamicProviders('dist/extensions/**/*.helper.js')
+@Module({
+  imports: [HttpModule],
+  controllers: [HelperController],
+  providers: [HelperService],
+  exports: [HelperService],
+})
+export class HelperModule {}

api/src/helper/helper.service.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { Injectable } from '@nestjs/common';
+
+import { LoggerService } from '@/logger/logger.service';
+import { SettingService } from '@/setting/services/setting.service';
+
+import BaseHelper from './lib/base-helper';
+import { HelperRegistry, HelperType, TypeOfHelper } from './types';
+
+@Injectable()
+export class HelperService {
+  private registry: HelperRegistry = new Map();
+
+  constructor(
+    private readonly settingService: SettingService,
+    private readonly logger: LoggerService,
+  ) {
+    // Init empty registry
+    Object.values(HelperType).forEach((type: HelperType) => {
+      this.registry.set(type, new Map());
+    });
+  }
+
+  /**
+   * Registers a helper.
+   *
+   * @param name - The helper to be registered.
+   */
+  public register<H extends BaseHelper<string>>(helper: H) {
+    const helpers = this.registry.get(helper.getType());
+    helpers.set(helper.getName(), helper);
+    this.logger.log(`Helper "${helper.getName()}" has been registered!`);
+  }
+
+  /**
+   * Get a helper by name and type.
+   *
+   * @param type - The type of helper.
+   * @param name - The helper's name.
+   *
+   * @returns - The helper
+   */
+  public get<T extends HelperType>(type: T, name: string) {
+    const helpers = this.registry.get(type);
+
+    if (!helpers.has(name)) {
+      throw new Error('Uknown type of helpers');
+    }
+    return helpers.get(name) as TypeOfHelper<T>;
+  }
+
+  /**
+   * Get all helpers by type.
+   *
+   * @returns - The helpers
+   */
+  public getAllByType<T extends HelperType>(type: T) {
+    const helpers = this.registry.get(type) as Map<string, TypeOfHelper<T>>;
+
+    return Array.from(helpers.values());
+  }
+
+  /**
+   * Get default NLU helper.
+   *
+   * @returns - The helper
+   */
+  async getDefaultNluHelper() {
+    const settings = await this.settingService.getSettings();
+
+    const defaultHelper = this.get(
+      HelperType.NLU,
+      settings.chatbot_settings.default_nlu_helper,
+    );
+
+    if (!defaultHelper) {
+      throw new Error(`Unable to find default NLU helper`);
+    }
+
+    return defaultHelper;
+  }
+}

api/src/helper/lib/base-helper.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { LoggerService } from '@nestjs/common';
+
+import { SettingService } from '@/setting/services/setting.service';
+import { hyphenToUnderscore } from '@/utils/helpers/misc';
+
+import { HelperService } from '../helper.service';
+import { HelperSetting, HelperType } from '../types';
+
+export default abstract class BaseHelper<N extends string = string> {
+  protected readonly name: N;
+
+  protected readonly settings: HelperSetting<N>[] = [];
+
+  protected abstract type: HelperType;
+
+  constructor(
+    name: N,
+    settings: HelperSetting<N>[],
+    protected readonly settingService: SettingService,
+    protected readonly helperService: HelperService,
+    protected readonly logger: LoggerService,
+  ) {
+    this.name = name;
+    this.settings = settings;
+  }
+
+  onModuleInit() {
+    this.helperService.register(this);
+    this.setup();
+  }
+
+  async setup() {
+    await this.settingService.seedIfNotExist(
+      this.getName(),
+      this.settings.map((s, i) => ({
+        ...s,
+        weight: i + 1,
+      })),
+    );
+  }
+
+  /**
+   * Returns the helper's name
+   *
+   * @returns Helper's name
+   */
+  public getName() {
+    return this.name;
+  }
+
+  /**
+   * Returns the helper's group
+   * @returns Helper's group
+   */
+  protected getGroup() {
+    return hyphenToUnderscore(this.getName()) as HelperSetting<N>['group'];
+  }
+
+  /**
+   * Get the helper's type
+   *
+   * @returns Helper's type
+   */
+  public getType() {
+    return this.type;
+  }
+
+  /**
+   * Get the helper's settings
+   *
+   * @returns Helper's settings
+   */
+  async getSettings<S extends string = HyphenToUnderscore<N>>() {
+    const settings = await this.settingService.getSettings();
+    // @ts-expect-error workaround typing
+    return settings[this.getGroup() as keyof Settings] as Settings[S];
+  }
+}

api/src/helper/lib/base-nlp-helper.ts

@@ -0,0 +1,189 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { v4 as uuidv4 } from 'uuid';
+
+import { LoggerService } from '@/logger/logger.service';
+import {
+  NlpEntity,
+  NlpEntityDocument,
+  NlpEntityFull,
+} from '@/nlp/schemas/nlp-entity.schema';
+import { NlpSample, NlpSampleFull } from '@/nlp/schemas/nlp-sample.schema';
+import {
+  NlpValue,
+  NlpValueDocument,
+  NlpValueFull,
+} from '@/nlp/schemas/nlp-value.schema';
+import { SettingService } from '@/setting/services/setting.service';
+
+import { HelperService } from '../helper.service';
+import { HelperSetting, HelperType, Nlp } from '../types';
+
+import BaseHelper from './base-helper';
+
+// eslint-disable-next-line prettier/prettier
+export default abstract class BaseNlpHelper<
+  N extends string,
+> extends BaseHelper<N> {
+  protected readonly type: HelperType = HelperType.NLU;
+
+  constructor(
+    name: N,
+    settings: HelperSetting<N>[],
+    settingService: SettingService,
+    helperService: HelperService,
+    logger: LoggerService,
+  ) {
+    super(name, settings, settingService, helperService, logger);
+  }
+
+  /**
+   * Updates an entity
+   *
+   * @param entity - The updated entity
+   *
+   * @returns The updated entity otherwise an error
+   */
+  async updateEntity(entity: NlpEntity): Promise<NlpEntity> {
+    return entity;
+  }
+
+  /**
+   * Adds an entity
+   *
+   * @param entity - The entity to add
+   * @returns The added entity otherwise an error
+   */
+  addEntity(_entity: NlpEntityDocument): Promise<string> {
+    return new Promise((resolve, _reject) => {
+      return resolve(uuidv4());
+    });
+  }
+
+  /**
+   * Deletes an entity
+   *
+   * @param entityId - The entity ID to delete
+   *
+   * @return The deleted entity otherwise an error
+   */
+  async deleteEntity(entityId: string): Promise<any> {
+    return entityId;
+  }
+
+  /**
+   * Update an entity value
+   *
+   * @param value - The updated update
+   *
+   * @returns The updated value otherwise it should throw an error
+   */
+  async updateValue(value: NlpValue): Promise<NlpValue> {
+    return value;
+  }
+
+  /**
+   * Adds an entity value
+   *
+   * @param value - The value to add
+   *
+   * @returns The added value otherwise it should throw an error
+   */
+  addValue(_value: NlpValueDocument): Promise<string> {
+    return new Promise((resolve, _reject) => {
+      return resolve(uuidv4());
+    });
+  }
+
+  /**
+   * Delete an entity value
+   *
+   * @param value - The value to delete
+   *
+   * @returns The deleted value otherwise an error
+   */
+  async deleteValue(value: NlpValueFull): Promise<NlpValueFull> {
+    return value;
+  }
+
+  /**
+   * Returns training dataset in NLP provider compatible format
+   *
+   * @param samples - Sample to train
+   * @param entities - All available entities
+   *
+   * @returns The formatted NLP training set
+   */
+  abstract format(samples: NlpSampleFull[], entities: NlpEntityFull[]): unknown;
+
+  /**
+   * Perform training request
+   *
+   * @param samples - Samples to train
+   * @param entities - All available entities
+   *
+   * @returns Training result
+   */
+  abstract train(
+    samples: NlpSampleFull[],
+    entities: NlpEntityFull[],
+  ): Promise<any>;
+
+  /**
+   * Perform evaluation request
+   *
+   * @param samples - Samples to evaluate
+   * @param entities - All available entities
+   *
+   * @returns NLP evaluation result
+   */
+  abstract evaluate(
+    samples: NlpSampleFull[],
+    entities: NlpEntityFull[],
+  ): Promise<any>;
+
+  /**
+   * Delete/Forget a sample
+   *
+   * @param sample - The sample to delete/forget
+   *
+   * @returns The deleted sample otherwise an error
+   */
+  async forget(sample: NlpSample): Promise<NlpSample> {
+    return sample;
+  }
+
+  /**
+   * Returns only the entities that have strong confidence (> than the threshold), can return an empty result
+   *
+   * @param nlp - The nlp provider parse returned result
+   * @param threshold - Whenever to apply threshold filter or not
+   *
+   * @returns NLP Parsed entities
+   */
+  abstract filterEntitiesByConfidence(
+    nlp: any,
+    threshold: boolean,
+  ): Promise<Nlp.ParseEntities>;
+
+  /**
+   * Returns only the entities that have strong confidence (> than the threshold), can return an empty result
+   *
+   * @param text - The text to parse
+   * @param threshold - Whenever to apply threshold filter or not
+   * @param project - Whenever to request a specific model
+   *
+   * @returns NLP Parsed entities
+   */
+  abstract predict(
+    text: string,
+    threshold?: boolean,
+    project?: string,
+  ): Promise<Nlp.ParseEntities>;
+}

api/src/helper/types.ts

@@ -0,0 +1,44 @@
+import { SettingCreateDto } from '@/setting/dto/setting.dto';
+
+import BaseHelper from './lib/base-helper';
+import BaseNlpHelper from './lib/base-nlp-helper';
+
+export namespace Nlp {
+  export interface Config {
+    endpoint?: string;
+    token: string;
+  }
+
+  export interface ParseEntity {
+    entity: string; // Entity name
+    value: string; // Value name
+    confidence: number;
+    start?: number;
+    end?: number;
+  }
+
+  export interface ParseEntities {
+    entities: ParseEntity[];
+  }
+}
+
+export enum HelperType {
+  NLU = 'nlu',
+  UTIL = 'util',
+}
+
+export type TypeOfHelper<T extends HelperType> = T extends HelperType.NLU
+  ? BaseNlpHelper<string>
+  : BaseHelper;
+
+export type HelperRegistry<H extends BaseHelper = BaseHelper> = Map<
+  HelperType,
+  Map<string, H>
+>;
+
+export type HelperSetting<N extends string = string> = Omit<
+  SettingCreateDto,
+  'group' | 'weight'
+> & {
+  group: HyphenToUnderscore<N>;
+};