feat: refactor to use findAndPopulate in block score calculation

api/src/chat/services/block.service.spec.ts

@@ -85,39 +85,24 @@ import { BlockService } from './block.service';
 import { CategoryService } from './category.service';
 
 // Create a mock for the NlpEntityService
-const mockNlpEntityService = {
-  entities: {
-    intent: {
-      lookups: ['trait'],
-      id: '67e3e41eff551ca5be70559c',
-      weight: 1,
-    },
-    firstname: {
-      lookups: ['trait'],
-      id: '67e3e41eff551ca5be70559d',
-      weight: 1,
-    },
-  },
-  findOne: jest.fn().mockImplementation((query) => {
-    const entity = mockNlpEntityService.entities[query.name];
-    if (entity) {
-      return Promise.resolve(entity);
-    }
-    return Promise.resolve(null); // Default response if the entity isn't found
-  }),
-};
+const mockNlpEntityService: Partial<Record<keyof NlpEntityService, jest.Mock>> =
+  {
+    findAndPopulate: jest.fn().mockResolvedValue([
+      {
+        _id: '67e3e41eff551ca5be70559c',
+        name: 'intent',
+        weight: 1,
+        values: [{ value: 'greeting' }, { value: 'affirmation' }],
+      },
+      {
+        _id: '67e3e41eff551ca5be70559d',
+        name: 'firstname',
+        weight: 1,
+        values: [{ value: 'jhon' }, { value: 'doe' }],
+      },
+    ]),
+  };
 
-const mockNlpValueService = {
-  find: jest.fn().mockImplementation((query) => {
-    if (query.entity === '67e3e41eff551ca5be70559c') {
-      return Promise.resolve([{ value: 'greeting' }, { value: 'affirmation' }]); // Simulating multiple values for 'intent'
-    }
-    if (query.entity === '67e3e41eff551ca5be70559d') {
-      return Promise.resolve([{ value: 'jhon' }, { value: 'doe' }]); // Simulating multiple values for 'firstname'
-    }
-    return Promise.resolve([]); // Default response for no matching entity
-  }),
-};
 describe('BlockService', () => {
   let blockRepository: BlockRepository;
   let categoryRepository: CategoryRepository;
@@ -169,8 +154,8 @@ describe('BlockService', () => {
           useValue: mockNlpEntityService,
         },
         {
-          provide: NlpValueService, // Mocking NlpValueService
-          useValue: mockNlpValueService,
+          provide: NlpValueService,
+          useValue: {},
         },
         {
           provide: PluginService,
@@ -513,41 +498,40 @@ describe('BlockService', () => {
     it('should correctly use entity cache to avoid redundant database calls', async () => {
       const nlpCacheMap: NlpCacheMap = new Map();
 
-      // Create spies on the services
-      const entityServiceSpy = jest.spyOn(mockNlpEntityService, 'findOne');
-      const valueServiceSpy = jest.spyOn(mockNlpValueService, 'find');
+      // Spy on findAndPopulate
+      const findAndPopulateSpy = jest.spyOn(
+        mockNlpEntityService,
+        'findAndPopulate',
+      );
 
-      // First call should calculate and cache entity data
+      // First call: should trigger findAndPopulate and cache results
       await blockService.calculateBlockScore(
         mockNlpPatternsSetOne,
         mockNlpEntitiesSetOne,
         nlpCacheMap,
         nlpPenaltyFactor,
       );
-      const cacheSizeBefore = nlpCacheMap.size;
-      const entityCallsBefore = entityServiceSpy.mock.calls.length;
-      const valueCallsBefore = valueServiceSpy.mock.calls.length;
 
-      // Second call should use cached entity data, without redundant DB calls
+      const cacheSizeAfterFirstCall = nlpCacheMap.size;
+      const callsAfterFirstCall = findAndPopulateSpy.mock.calls.length;
+
+      // should not call findAndPopulate again since data is cached
       await blockService.calculateBlockScore(
         mockNlpPatternsSetOne,
         mockNlpEntitiesSetOne,
         nlpCacheMap,
         nlpPenaltyFactor,
       );
-      const cacheSizeAfter = nlpCacheMap.size;
-      const entityCallsAfter = entityServiceSpy.mock.calls.length;
-      const valueCallsAfter = valueServiceSpy.mock.calls.length;
 
-      // Assert that the cache size hasn't increased after the second call
-      expect(cacheSizeBefore).toBe(cacheSizeAfter);
-      // Assert that the services weren't called again
-      expect(entityCallsAfter).toBe(entityCallsBefore);
-      expect(valueCallsAfter).toBe(valueCallsBefore);
+      const cacheSizeAfterSecondCall = nlpCacheMap.size;
+      const callsAfterSecondCall = findAndPopulateSpy.mock.calls.length;
+
+      expect(cacheSizeAfterSecondCall).toBe(cacheSizeAfterFirstCall);
+      expect(callsAfterSecondCall).toBe(callsAfterFirstCall); // No new call
+      expect(findAndPopulateSpy).toHaveBeenCalledTimes(1); // Should only be called once
 
       // Cleanup
-      entityServiceSpy.mockRestore();
-      valueServiceSpy.mockRestore();
+      findAndPopulateSpy.mockRestore();
     });
   });
 

api/src/chat/services/block.service.ts

@@ -16,9 +16,9 @@ import { CONSOLE_CHANNEL_NAME } from '@/extensions/channels/console/settings';
 import { NLU } from '@/helper/types';
 import { I18nService } from '@/i18n/services/i18n.service';
 import { LanguageService } from '@/i18n/services/language.service';
+import { NlpEntityFull } from '@/nlp/schemas/nlp-entity.schema';
 import { NlpCacheMap } from '@/nlp/schemas/types';
 import { NlpEntityService } from '@/nlp/services/nlp-entity.service';
-import { NlpValueService } from '@/nlp/services/nlp-value.service';
 import { PluginService } from '@/plugins/plugins.service';
 import { PluginType } from '@/plugins/types';
 import { SettingService } from '@/setting/services/setting.service';
@@ -61,7 +61,6 @@ export class BlockService extends BaseService<
     protected readonly i18n: I18nService,
     protected readonly languageService: LanguageService,
     protected readonly entityService: NlpEntityService,
-    protected readonly valueService: NlpValueService,
   ) {
     super(repository);
   }
@@ -410,7 +409,6 @@ export class BlockService extends BaseService<
         nlpCacheMap,
         nlpPenaltyFactor,
       );
-
       if (nlpScore > highestScore) {
         highestScore = nlpScore;
         bestBlock = block;
@@ -446,52 +444,49 @@ export class BlockService extends BaseService<
   ): Promise<number> {
     let nlpScore = 0;
 
-    const patternScores = await Promise.all(
-      patterns.map(async (pattern) => {
-        const entityName = pattern.entity;
+    // Collect all unique entity names from patterns
+    const entityNames = [...new Set(patterns.map((pattern) => pattern.entity))];
 
-        // Retrieve entity data from cache or database if not cached
-        let entityData = nlpCacheMap.get(entityName);
-        if (!entityData) {
-          const entityLookup = await this.entityService.findOne(
-            { name: entityName },
-            undefined,
-            { lookups: 1, weight: 1, _id: 1 },
-          );
-          if (!entityLookup?.id || !entityLookup.weight) return 0;
-
-          const valueLookups = await this.valueService.find(
-            { entity: entityLookup.id },
-            undefined,
-            { value: 1, _id: 0 },
-          );
-          const values = valueLookups.map((v) => v.value);
-
-          // Cache the entity data
-          entityData = {
-            id: entityLookup.id,
-            weight: entityLookup.weight,
-            values,
-          };
-          nlpCacheMap.set(entityName, entityData);
-        }
-
-        // Check if the NLP entity matches with the cached data
-        const matchedEntity = nlp.entities.find(
-          (e) =>
-            e.entity === entityName &&
-            entityData?.values.some((v) => v === e.value) &&
-            (pattern.match !== 'value' || e.value === pattern.value),
-        );
-        return matchedEntity?.confidence
-          ? matchedEntity.confidence *
-              entityData.weight *
-              (pattern.match === 'entity' ? nlpPenaltyFactor : 1)
-          : 0;
-      }),
+    // Check the cache for existing lookups first
+    const uncachedEntityNames = entityNames.filter(
+      (name) => !nlpCacheMap.has(name),
     );
+    // Fetch only uncached entities in one query, with values already populated
+    const entityLookups: NlpEntityFull[] = uncachedEntityNames.length
+      ? await this.entityService.findAndPopulate({
+          name: { $in: uncachedEntityNames },
+        })
+      : [];
 
-    // Sum up the scores for all patterns
+    // Populate the cache with the fetched lookups
+    entityLookups.forEach((lookup) => {
+      nlpCacheMap.set(lookup.name, {
+        id: lookup.id,
+        weight: lookup.weight,
+        values: lookup.values?.map((v) => v.value) ?? [],
+      });
+    });
+
+    // Calculate the score for each pattern
+    const patternScores = patterns.map((pattern) => {
+      const entityData = nlpCacheMap.get(pattern.entity);
+      if (!entityData) return 0;
+
+      const matchedEntity = nlp.entities.find(
+        (e) =>
+          e.entity === pattern.entity &&
+          entityData.values.includes(e.value) &&
+          (pattern.match !== 'value' || e.value === pattern.value),
+      );
+
+      return matchedEntity?.confidence
+        ? matchedEntity.confidence *
+            entityData.weight *
+            (pattern.match === 'entity' ? nlpPenaltyFactor : 1)
+        : 0;
+    });
+
+    // Sum up all scores
     nlpScore = patternScores.reduce((sum, score) => sum + score, 0);
 
     return nlpScore;