feat: implement nlp based blocks prioritization strategy

api/src/chat/chat.module.ts

@@ -16,6 +16,9 @@ import { AttachmentModel } from '@/attachment/schemas/attachment.schema';
 import { AttachmentService } from '@/attachment/services/attachment.service';
 import { ChannelModule } from '@/channel/channel.module';
 import { CmsModule } from '@/cms/cms.module';
+import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository';
+import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository';
+import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository';
 import { UserModule } from '@/user/user.module';
 
 import { BlockController } from './controllers/block.controller';
@@ -85,6 +88,9 @@ import { SubscriberService } from './services/subscriber.service';
     MessageRepository,
     SubscriberRepository,
     ConversationRepository,
+    NlpEntityRepository,
+    NlpSampleEntityRepository,
+    NlpValueRepository,
     CategoryService,
     ContextVarService,
     LabelService,

api/src/chat/controllers/block.controller.spec.ts

@@ -20,6 +20,15 @@ import { LanguageRepository } from '@/i18n/repositories/language.repository';
 import { LanguageModel } from '@/i18n/schemas/language.schema';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LanguageService } from '@/i18n/services/language.service';
+import { LoggerService } from '@/logger/logger.service';
+import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository';
+import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository';
+import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository';
+import { NlpEntityModel } from '@/nlp/schemas/nlp-entity.schema';
+import { NlpSampleEntityModel } from '@/nlp/schemas/nlp-sample-entity.schema';
+import { NlpValueModel } from '@/nlp/schemas/nlp-value.schema';
+import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
+import { NlpValueService } from '@/nlp/services/nlp-value.service';
 import { PluginService } from '@/plugins/plugins.service';
 import { SettingService } from '@/setting/services/setting.service';
 import { InvitationRepository } from '@/user/repositories/invitation.repository';
@@ -93,6 +102,9 @@ describe('BlockController', () => {
           RoleModel,
           PermissionModel,
           LanguageModel,
+          NlpEntityModel,
+          NlpSampleEntityModel,
+          NlpValueModel,
         ]),
       ],
       providers: [
@@ -116,6 +128,12 @@ describe('BlockController', () => {
         PermissionService,
         LanguageService,
         PluginService,
+        LoggerService,
+        NlpEntityService,
+        NlpEntityRepository,
+        NlpSampleEntityRepository,
+        NlpValueRepository,
+        NlpValueService,
         {
           provide: I18nService,
           useValue: {

api/src/chat/services/block.service.spec.ts

@@ -31,6 +31,14 @@ import { LanguageRepository } from '@/i18n/repositories/language.repository';
 import { LanguageModel } from '@/i18n/schemas/language.schema';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LanguageService } from '@/i18n/services/language.service';
+import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository';
+import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository';
+import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository';
+import { NlpEntityModel } from '@/nlp/schemas/nlp-entity.schema';
+import { NlpSampleEntityModel } from '@/nlp/schemas/nlp-sample-entity.schema';
+import { NlpValueModel } from '@/nlp/schemas/nlp-value.schema';
+import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
+import { NlpValueService } from '@/nlp/services/nlp-value.service';
 import { PluginService } from '@/plugins/plugins.service';
 import { SettingService } from '@/setting/services/setting.service';
 import {
@@ -43,6 +51,8 @@ import {
   blockGetStarted,
   blockProductListMock,
   blocks,
+  mockNlpBlock,
+  nlpBlocks,
 } from '@/utils/test/mocks/block';
 import {
   contextBlankInstance,
@@ -66,6 +76,25 @@ import { CategoryRepository } from './../repositories/category.repository';
 import { BlockService } from './block.service';
 import { CategoryService } from './category.service';
 
+// Create a mock for the NlpEntityService
+const mockNlpEntityService = {
+  findOne: jest.fn().mockImplementation((query) => {
+    if (query.name === 'intent') {
+      return Promise.resolve({
+        lookups: ['trait'],
+        id: '67e3e41eff551ca5be70559c',
+      });
+    }
+    if (query.name === 'firstname') {
+      return Promise.resolve({
+        lookups: ['trait'],
+        id: '67e3e41eff551ca5be70559d',
+      });
+    }
+    return Promise.resolve(null); // Default response if the entity isn't found
+  }),
+};
+
 describe('BlockService', () => {
   let blockRepository: BlockRepository;
   let categoryRepository: CategoryRepository;
@@ -75,6 +104,9 @@ describe('BlockService', () => {
   let hasPreviousBlocks: Block;
   let contentService: ContentService;
   let contentTypeService: ContentTypeService;
+  let nlpEntityService: NlpEntityService;
+  let settingService: SettingService;
+  let settings: Settings;
 
   beforeAll(async () => {
     const { getMocks } = await buildTestingMocks({
@@ -91,6 +123,9 @@ describe('BlockService', () => {
           AttachmentModel,
           LabelModel,
           LanguageModel,
+          NlpEntityModel,
+          NlpValueModel,
+          NlpSampleEntityModel,
         ]),
       ],
       providers: [
@@ -100,12 +135,20 @@ describe('BlockService', () => {
         ContentRepository,
         AttachmentRepository,
         LanguageRepository,
+        NlpEntityRepository,
+        NlpSampleEntityRepository,
+        NlpValueRepository,
         BlockService,
         CategoryService,
         ContentTypeService,
         ContentService,
         AttachmentService,
         LanguageService,
+        NlpValueService,
+        {
+          provide: NlpEntityService, // Mocking NlpEntityService
+          useValue: mockNlpEntityService,
+        },
         {
           provide: PluginService,
           useValue: {},
@@ -138,20 +181,14 @@ describe('BlockService', () => {
           },
         },
       ],
-    });
-    [
-      blockService,
-      contentService,
-      contentTypeService,
-      categoryRepository,
-      blockRepository,
-    ] = await getMocks([
-      BlockService,
-      ContentService,
-      ContentTypeService,
-      CategoryRepository,
-      BlockRepository,
-    ]);
+    }).compile();
+    blockService = module.get<BlockService>(BlockService);
+    contentService = module.get<ContentService>(ContentService);
+    settingService = module.get<SettingService>(SettingService);
+    contentTypeService = module.get<ContentTypeService>(ContentTypeService);
+    categoryRepository = module.get<CategoryRepository>(CategoryRepository);
+    blockRepository = module.get<BlockRepository>(BlockRepository);
+    nlpEntityService = module.get<NlpEntityService>(NlpEntityService);
     category = (await categoryRepository.findOne({ label: 'default' }))!;
     hasPreviousBlocks = (await blockRepository.findOne({
       name: 'hasPreviousBlocks',
@@ -381,6 +418,59 @@ describe('BlockService', () => {
     });
   });
 
+  describe('matchBestNLP', () => {
+    it('should return undefined if blocks is empty', async () => {
+      const result = await blockService.matchBestNLP([]);
+      expect(result).toBeUndefined();
+    });
+
+    it('should return the only block if there is one', async () => {
+      const result = await blockService.matchBestNLP([blockEmpty]);
+      expect(result).toBe(blockEmpty);
+    });
+
+    it('should correctly select the best block based on NLP scores', async () => {
+      const result = await blockService.matchBestNLP(nlpBlocks);
+      expect(result).toBe(mockNlpBlock);
+
+      // Iterate over each block
+      for (const block of nlpBlocks) {
+        // Flatten the patterns array and filter valid NLP patterns
+        block.patterns
+          .flatMap((pattern) => (Array.isArray(pattern) ? pattern : []))
+          .filter((p) => typeof p === 'object' && 'entity' in p && 'match' in p) // Filter only valid patterns with entity and match
+          .forEach((p) => {
+            // Check if findOne was called with the correct entity
+            expect(nlpEntityService.findOne).toHaveBeenCalledWith(
+              { name: p.entity },
+              undefined,
+              { _id: 0, lookups: 1 },
+            );
+          });
+      }
+    });
+
+    it('should return the block with the highest combined score', async () => {
+      const result = await blockService.matchBestNLP(nlpBlocks);
+      expect(result).toBe(mockNlpBlock);
+      // Iterate over each block
+      for (const block of nlpBlocks) {
+        // Flatten the patterns array and filter valid NLP patterns
+        block.patterns
+          .flatMap((pattern) => (Array.isArray(pattern) ? pattern : []))
+          .filter((p) => typeof p === 'object' && 'entity' in p && 'match' in p) // Filter only valid patterns with entity and match
+          .forEach((p) => {
+            // Check if findOne was called with the correct entity
+            expect(nlpEntityService.findOne).toHaveBeenCalledWith(
+              { name: p.entity },
+              undefined,
+              { _id: 0, lookups: 1 },
+            );
+          });
+      }
+    });
+  });
+
   describe('matchText', () => {
     it('should return false for matching an empty text', () => {
       const result = blockService.matchText('', blockGetStarted);

api/src/chat/services/block.service.ts

@@ -16,6 +16,7 @@ import { CONSOLE_CHANNEL_NAME } from '@/extensions/channels/console/settings';
 import { NLU } from '@/helper/types';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LanguageService } from '@/i18n/services/language.service';
+import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
 import { PluginService } from '@/plugins/plugins.service';
 import { PluginType } from '@/plugins/types';
 import { SettingService } from '@/setting/services/setting.service';
@@ -53,6 +54,7 @@ export class BlockService extends BaseService<
     private readonly pluginService: PluginService,
     protected readonly i18n: I18nService,
     protected readonly languageService: LanguageService,
+    protected readonly entityService: NlpEntityService,
   ) {
     super(repository);
   }
@@ -183,18 +185,13 @@ export class BlockService extends BaseService<
       // Perform an NLP Match
       if (!block && nlp) {
         // Find block pattern having the best match of nlp entities
-        let nlpBest = 0;
-        filteredBlocks.forEach((b, index, self) => {
-          const nlpPattern = this.matchNLP(nlp, b);
-          if (nlpPattern && nlpPattern.length > nlpBest) {
-            nlpBest = nlpPattern.length;
-            block = self[index];
-          }
+        const newBlocks = filteredBlocks.filter((b) => {
+          return this.matchNLP(nlp, b);
         });
+        block = (await this.matchBestNLP(newBlocks)) as BlockFull | undefined;
       }
     }
-    // Uknown event type => return false;
-    // this.logger.error('Unable to recognize event type while matching', event);
+
     return block;
   }
 
@@ -338,6 +335,81 @@ export class BlockService extends BaseService<
     });
   }
 
+  /**
+   *  Identifies and returns the best-matching block based on NLP entity scores.
+   *
+   * This function evaluates a list of blocks by analyzing their associated NLP entities
+   * and scoring them based on predefined lookup entities. The block with the highest
+   * score is selected as the best match.
+   * @param blocks - Blocks on which to perform the filtering
+   *
+   * @returns The best block
+   */
+  async matchBestNLP(
+    blocks: Block[] | BlockFull[] | undefined,
+  ): Promise<Block | BlockFull | undefined> {
+    // @TODO make lookup scores configurable in hexabot settings
+    const lookupScores: { [key: string]: number } = {
+      trait: 2,
+      keywords: 1,
+    };
+
+    // No blocks to check against
+    if (blocks?.length === 0 || !blocks) {
+      return undefined;
+    }
+
+    // If there's only one block, return it immediately.
+    if (blocks.length === 1) {
+      return blocks[0];
+    }
+    let bestBlock: Block | BlockFull | undefined;
+    let highestScore = 0;
+
+    // Iterate over each block in blocks
+    for (const block of blocks) {
+      let nlpScore = 0;
+
+      // Gather all entity lookups for patterns that include an entity
+      const entityLookups = await Promise.all(
+        block.patterns
+          .flatMap((pattern) => (Array.isArray(pattern) ? pattern : []))
+          .filter((p) => typeof p === 'object' && 'entity' in p && 'match' in p)
+          .map(async (pattern) => {
+            const entityName = pattern.entity;
+            return await this.entityService.findOne(
+              { name: entityName },
+              undefined,
+              { lookups: 1, _id: 0 },
+            );
+          }),
+      );
+
+      nlpScore += entityLookups.reduce((score, entityLookup) => {
+        if (
+          entityLookup &&
+          entityLookup.lookups[0] &&
+          lookupScores[entityLookup.lookups[0]]
+        ) {
+          return score + lookupScores[entityLookup.lookups[0]]; // Add points based on the lookup type
+        }
+        return score; // Return the current score if no match
+      }, 0);
+
+      // Update the best block if the current block has a higher NLP score
+      if (nlpScore > highestScore) {
+        highestScore = nlpScore;
+        bestBlock = block;
+      }
+    }
+
+    this.logger.debug(`Best Nlp Score obtained ${highestScore}`);
+    this.logger.debug(
+      `Best retrieved block based on NLP entities ${JSON.stringify(bestBlock)}`,
+    );
+    return bestBlock;
+  }
+
   /**
    * Matches an outcome-based block from a list of available blocks
    * based on the outcome of a system message.

api/src/chat/services/bot.service.spec.ts

@@ -33,6 +33,14 @@ import { LanguageRepository } from '@/i18n/repositories/language.repository';
 import { LanguageModel } from '@/i18n/schemas/language.schema';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LanguageService } from '@/i18n/services/language.service';
+import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository';
+import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository';
+import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository';
+import { NlpEntityModel } from '@/nlp/schemas/nlp-entity.schema';
+import { NlpSampleEntityModel } from '@/nlp/schemas/nlp-sample-entity.schema';
+import { NlpValueModel } from '@/nlp/schemas/nlp-value.schema';
+import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
+import { NlpValueService } from '@/nlp/services/nlp-value.service';
 import { PluginService } from '@/plugins/plugins.service';
 import { SettingService } from '@/setting/services/setting.service';
 import { installBlockFixtures } from '@/utils/test/fixtures/block';
@@ -100,6 +108,9 @@ describe('BlockService', () => {
           MenuModel,
           ContextVarModel,
           LanguageModel,
+          NlpEntityModel,
+          NlpSampleEntityModel,
+          NlpValueModel,
         ]),
         JwtModule,
       ],
@@ -131,6 +142,11 @@ describe('BlockService', () => {
         ContextVarService,
         ContextVarRepository,
         LanguageService,
+        NlpEntityService,
+        NlpEntityRepository,
+        NlpSampleEntityRepository,
+        NlpValueRepository,
+        NlpValueService,
         {
           provide: HelperService,
           useValue: {},

api/src/i18n/controllers/translation.controller.spec.ts

@@ -1,4 +1,5 @@
 /*
+ * Copyright © 2025 Hexastack. All rights reserved.
  * Copyright © 2025 Hexastack. All rights reserved.
  *
  * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
@@ -30,6 +31,14 @@ import { MenuModel } from '@/cms/schemas/menu.schema';
 import { ContentService } from '@/cms/services/content.service';
 import { MenuService } from '@/cms/services/menu.service';
 import { I18nService } from '@/i18n/services/i18n.service';
+import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository';
+import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository';
+import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository';
+import { NlpEntityModel } from '@/nlp/schemas/nlp-entity.schema';
+import { NlpSampleEntityModel } from '@/nlp/schemas/nlp-sample-entity.schema';
+import { NlpValueModel } from '@/nlp/schemas/nlp-value.schema';
+import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
+import { NlpValueService } from '@/nlp/services/nlp-value.service';
 import { NlpService } from '@/nlp/services/nlp.service';
 import { PluginService } from '@/plugins/plugins.service';
 import { SettingService } from '@/setting/services/setting.service';
@@ -75,6 +84,9 @@ describe('TranslationController', () => {
           BlockModel,
           ContentModel,
           LanguageModel,
+          NlpEntityModel,
+          NlpSampleEntityModel,
+          NlpValueModel,
         ]),
       ],
       providers: [
@@ -130,6 +142,11 @@ describe('TranslationController', () => {
         },
         LanguageService,
         LanguageRepository,
+        NlpEntityRepository,
+        NlpEntityService,
+        NlpValueRepository,
+        NlpValueService,
+        NlpSampleEntityRepository,
       ],
     });
     [translationService, translationController] = await getMocks([

api/src/nlp/nlp.module.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Hexastack. All rights reserved.
+ * Copyright © 2025 Hexastack. All rights reserved.
  *
  * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
  * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
@@ -31,6 +31,7 @@ import { NlpSampleService } from './services/nlp-sample.service';
 import { NlpValueService } from './services/nlp-value.service';
 import { NlpService } from './services/nlp.service';
 
+// @Global()
 @Module({
   imports: [
     MongooseModule.forFeature([

api/src/utils/test/mocks/block.ts

@@ -246,6 +246,35 @@ export const blockGetStarted = {
   message: ['Welcome! How are you ? '],
 } as unknown as BlockFull;
 
+export const mockNlpBlock = {
+  ...baseBlockInstance,
+  name: 'Mock Nlp',
+  patterns: [
+    'Hello',
+    '/we*lcome/',
+    { label: 'Mock Nlp', value: 'MOCK_NLP' },
+    [
+      {
+        entity: 'intent',
+        match: 'value',
+        value: 'greeting',
+      },
+      {
+        entity: 'intent',
+        match: 'value',
+        value: 'want',
+      },
+      {
+        entity: 'intent',
+        match: 'value',
+        value: 'affirmative',
+      },
+    ],
+  ],
+  trigger_labels: customerLabelsMock,
+  message: ['Good to see you again '],
+} as unknown as BlockFull;
+
 const patternsProduct: Pattern[] = [
   'produit',
   [
@@ -285,3 +314,5 @@ export const blockCarouselMock = {
 } as unknown as BlockFull;
 
 export const blocks: BlockFull[] = [blockGetStarted, blockEmpty];
+
+export const nlpBlocks: BlockFull[] = [blockGetStarted, mockNlpBlock];