Merge pull request #1097 from Hexastack/feat/nlp-sample-filter-by-entities

Feat/nlp sample filter by entities
This commit is contained in:
Med Marrouchi
2025-06-11 11:10:18 +01:00
committed by GitHub
23 changed files with 1528 additions and 173 deletions

View File

@@ -10,6 +10,7 @@ import { CACHE_MANAGER } from '@nestjs/cache-manager';
import { BadRequestException, NotFoundException } from '@nestjs/common';
import { MongooseModule } from '@nestjs/mongoose';
import { NlpValueMatchPattern } from '@/chat/schemas/types/pattern';
import { HelperService } from '@/helper/helper.service';
import { LanguageRepository } from '@/i18n/repositories/language.repository';
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
@@ -181,11 +182,51 @@ describe('NlpSampleController', () => {
})),
);
});
it('should find nlp samples with patterns', async () => {
const pageQuery = getPageQuery<NlpSample>({ sort: ['text', 'desc'] });
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'greeting' },
];
const result = await nlpSampleController.findPage(
pageQuery,
['language', 'entities'],
{},
patterns,
);
// Should only return samples matching the pattern
const nlpSamples = await nlpSampleService.findByPatternsAndPopulate(
{ filters: {}, patterns },
pageQuery,
);
expect(result).toEqualPayload(nlpSamples);
});
it('should return empty array if no samples match the patterns', async () => {
const pageQuery = getPageQuery<NlpSample>({ sort: ['text', 'desc'] });
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'nonexistent' },
];
jest.spyOn(nlpSampleService, 'findByPatternsAndPopulate');
const result = await nlpSampleController.findPage(
pageQuery,
['language', 'entities'],
{},
patterns,
);
expect(nlpSampleService.findByPatternsAndPopulate).toHaveBeenCalledTimes(
1,
);
expect(Array.isArray(result)).toBe(true);
expect(result).toHaveLength(0);
});
});
describe('count', () => {
it('should count the nlp samples', async () => {
jest.spyOn(nlpSampleService, 'count');
const result = await nlpSampleController.count({});
expect(nlpSampleService.count).toHaveBeenCalledTimes(1);
const count = nlpSampleFixtures.length;
expect(result).toEqual({ count });
});
@@ -439,4 +480,34 @@ describe('NlpSampleController', () => {
).rejects.toThrow(NotFoundException);
});
});
describe('filterCount', () => {
it('should count the nlp samples without patterns', async () => {
const filters = { text: 'Hello' };
jest.spyOn(nlpSampleService, 'countByPatterns');
const result = await nlpSampleController.filterCount(filters, []);
expect(nlpSampleService.countByPatterns).toHaveBeenCalledTimes(1);
expect(result).toEqual({ count: 1 });
});
it('should count the nlp samples with patterns', async () => {
const filters = { text: 'Hello' };
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'greeting' },
];
jest.spyOn(nlpSampleService, 'countByPatterns');
const result = await nlpSampleController.filterCount(filters, patterns);
expect(nlpSampleService.countByPatterns).toHaveBeenCalledTimes(1);
expect(result).toEqual({ count: 1 });
});
it('should return zero count when no samples match the filters and patterns', async () => {
const filters = { text: 'Nonexistent' };
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'nonexistent' },
];
const result = await nlpSampleController.filterCount(filters, patterns);
expect(result).toEqual({ count: 0 });
});
});
});

View File

@@ -29,7 +29,12 @@ import {
import { FileInterceptor } from '@nestjs/platform-express';
import { CsrfCheck } from '@tekuconcept/nestjs-csrf';
import { Response } from 'express';
import { z } from 'zod';
import {
NlpValueMatchPattern,
nlpValueMatchPatternSchema,
} from '@/chat/schemas/types/pattern';
import { HelperService } from '@/helper/helper.service';
import { HelperType } from '@/helper/types';
import { LanguageService } from '@/i18n/services/language.service';
@@ -40,6 +45,7 @@ import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
import { PageQueryPipe } from '@/utils/pagination/pagination-query.pipe';
import { PopulatePipe } from '@/utils/pipes/populate.pipe';
import { SearchFilterPipe } from '@/utils/pipes/search-filter.pipe';
import { ZodQueryParamPipe } from '@/utils/pipes/zod.pipe';
import { TFilterQuery } from '@/utils/types/filter.types';
import { NlpSampleDto, TNlpSampleDto } from '../dto/nlp-sample.dto';
@@ -184,9 +190,22 @@ export class NlpSampleController extends BaseController<
allowedFields: ['text', 'type', 'language'],
}),
)
filters?: TFilterQuery<NlpSample>,
filters: TFilterQuery<NlpSample> = {},
@Query(
new ZodQueryParamPipe(
z.array(nlpValueMatchPatternSchema),
(q) => q?.where?.patterns,
),
)
patterns: NlpValueMatchPattern[] = [],
) {
return await this.count(filters);
const count = await this.nlpSampleService.countByPatterns({
filters,
patterns,
});
return {
count,
};
}
/**
@@ -285,10 +304,23 @@ export class NlpSampleController extends BaseController<
}),
)
filters: TFilterQuery<NlpSample>,
@Query(
new ZodQueryParamPipe(
z.array(nlpValueMatchPatternSchema),
(q) => q?.where?.patterns,
),
)
patterns: NlpValueMatchPattern[] = [],
) {
return this.canPopulate(populate)
? await this.nlpSampleService.findAndPopulate(filters, pageQuery)
: await this.nlpSampleService.find(filters, pageQuery);
? await this.nlpSampleService.findByPatternsAndPopulate(
{ filters, patterns },
pageQuery,
)
: await this.nlpSampleService.findByPatterns(
{ filters, patterns },
pageQuery,
);
}
/**

View File

@@ -7,9 +7,11 @@
*/
import { MongooseModule } from '@nestjs/mongoose';
import { Types } from 'mongoose';
import { LanguageRepository } from '@/i18n/repositories/language.repository';
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
import { nlpSampleFixtures } from '@/utils/test/fixtures/nlpsample';
import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity';
import { getPageQuery } from '@/utils/test/pagination';
@@ -29,13 +31,16 @@ import {
NlpSampleFull,
NlpSampleModel,
} from '../schemas/nlp-sample.schema';
import { NlpValue, NlpValueModel } from '../schemas/nlp-value.schema';
import { NlpSampleEntityRepository } from './nlp-sample-entity.repository';
import { NlpSampleRepository } from './nlp-sample.repository';
import { NlpValueRepository } from './nlp-value.repository';
describe('NlpSampleRepository', () => {
let nlpSampleRepository: NlpSampleRepository;
let nlpSampleEntityRepository: NlpSampleEntityRepository;
let nlpValueRepository: NlpValueRepository;
let languageRepository: LanguageRepository;
let nlpSampleEntity: NlpSampleEntity | null;
let noNlpSample: NlpSample | null;
@@ -48,21 +53,28 @@ describe('NlpSampleRepository', () => {
MongooseModule.forFeature([
NlpSampleModel,
NlpSampleEntityModel,
NlpValueModel,
LanguageModel,
]),
],
providers: [
NlpSampleRepository,
NlpSampleEntityRepository,
NlpValueRepository,
LanguageRepository,
],
});
[nlpSampleRepository, nlpSampleEntityRepository, languageRepository] =
await getMocks([
NlpSampleRepository,
NlpSampleEntityRepository,
LanguageRepository,
]);
[
nlpSampleRepository,
nlpSampleEntityRepository,
nlpValueRepository,
languageRepository,
] = await getMocks([
NlpSampleRepository,
NlpSampleEntityRepository,
NlpValueRepository,
LanguageRepository,
]);
noNlpSample = await nlpSampleRepository.findOne({ text: 'No' });
nlpSampleEntity = await nlpSampleEntityRepository.findOne({
sample: noNlpSample!.id,
@@ -141,4 +153,149 @@ describe('NlpSampleRepository', () => {
expect(sampleEntities.length).toEqual(0);
});
});
describe('findByEntities', () => {
it('should return mapped NlpSample instances for matching entities', async () => {
const filters = {};
const values = await nlpValueRepository.find({ value: 'greeting' });
const result = await nlpSampleRepository.findByEntities({
filters,
values,
});
expect(result).toHaveLength(2);
expect(result[0]).toBeInstanceOf(NlpSample);
expect(result[0].text).toBe('Hello');
});
it('should return an empty array if no samples match', async () => {
const filters = {};
const values = [
{
id: new Types.ObjectId().toHexString(),
entity: new Types.ObjectId().toHexString(),
value: 'nonexistent',
},
] as NlpValue[];
const result = await nlpSampleRepository.findByEntities({
filters,
values,
});
expect(Array.isArray(result)).toBe(true);
expect(result).toHaveLength(0);
});
});
describe('findByEntitiesAndPopulate', () => {
it('should return populated NlpSampleFull instances for matching entities', async () => {
const filters = {};
const values = await nlpValueRepository.find({ value: 'greeting' });
const result = await nlpSampleRepository.findByEntitiesAndPopulate({
filters,
values,
});
expect(result.length).toBe(2);
result.forEach((sample) => {
expect(sample).toBeInstanceOf(NlpSampleFull);
expect(sample.entities).toBeDefined();
expect(Array.isArray(sample.entities)).toBe(true);
expect(sample.language).toBeDefined();
});
});
it('should return an empty array if no samples match', async () => {
const filters = {};
const values = [
{
id: new Types.ObjectId().toHexString(),
entity: new Types.ObjectId().toHexString(),
value: 'nonexistent',
},
] as NlpValue[];
const result = await nlpSampleRepository.findByEntitiesAndPopulate({
filters,
values,
});
expect(Array.isArray(result)).toBe(true);
expect(result).toHaveLength(0);
});
it('should support pagination and projection', async () => {
const filters = {};
const values = await nlpValueRepository.find({ value: 'greeting' });
const page = {
limit: 1,
skip: 0,
sort: ['text', 'asc'],
} as PageQueryDto<NlpSample>;
const projection = { text: 1 };
const result = await nlpSampleRepository.findByEntitiesAndPopulate(
{ filters, values },
page,
projection,
);
expect(Array.isArray(result)).toBe(true);
expect(result.length).toBe(1);
if (result.length > 0) {
expect(result[0]).toHaveProperty('text');
}
});
});
describe('countByEntities', () => {
it('should return the correct count for matching entities', async () => {
const filters = {};
const values = await nlpValueRepository.find({ value: 'greeting' });
const count = await nlpSampleRepository.countByEntities({
filters,
values,
});
expect(typeof count).toBe('number');
expect(count).toBe(2);
});
it('should return 0 if no samples match', async () => {
const filters = {};
const values = [
{
id: new Types.ObjectId().toHexString(),
entity: new Types.ObjectId().toHexString(),
value: 'nonexistent',
},
] as NlpValue[];
const count = await nlpSampleRepository.countByEntities({
filters,
values,
});
expect(count).toBe(0);
});
it('should respect filters (e.g. language)', async () => {
const values = await nlpValueRepository.find({ value: 'greeting' });
const language = languages[0];
const filters = { language: language.id };
const count = await nlpSampleRepository.countByEntities({
filters,
values,
});
// Should be <= total greeting samples, and >= 0
expect(typeof count).toBe('number');
expect(count).toBeGreaterThanOrEqual(0);
expect(count).toBeLessThanOrEqual(2);
});
});
});

View File

@@ -8,18 +8,31 @@
import { Injectable } from '@nestjs/common';
import { InjectModel } from '@nestjs/mongoose';
import { Document, Model, Query } from 'mongoose';
import { plainToClass } from 'class-transformer';
import {
Aggregate,
Document,
Model,
PipelineStage,
ProjectionType,
Query,
Types,
} from 'mongoose';
import { BaseRepository, DeleteResult } from '@/utils/generics/base-repository';
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
import { TFilterQuery } from '@/utils/types/filter.types';
import { TNlpSampleDto } from '../dto/nlp-sample.dto';
import { NlpSampleEntity } from '../schemas/nlp-sample-entity.schema';
import {
NLP_SAMPLE_POPULATE,
NlpSample,
NlpSampleDocument,
NlpSampleFull,
NlpSamplePopulate,
} from '../schemas/nlp-sample.schema';
import { NlpValue } from '../schemas/nlp-value.schema';
import { NlpSampleEntityRepository } from './nlp-sample-entity.repository';
@@ -32,11 +45,257 @@ export class NlpSampleRepository extends BaseRepository<
> {
constructor(
@InjectModel(NlpSample.name) readonly model: Model<NlpSample>,
@InjectModel(NlpSampleEntity.name)
readonly sampleEntityModel: Model<NlpSampleEntity>,
private readonly nlpSampleEntityRepository: NlpSampleEntityRepository,
) {
super(model, NlpSample, NLP_SAMPLE_POPULATE, NlpSampleFull);
}
/**
* Normalize the filter query.
*
* @param filters - The filters to normalize.
* @returns The normalized filters.
*/
private normalizeFilters(
filters: TFilterQuery<NlpSample>,
): TFilterQuery<NlpSample> {
if (filters?.$and) {
return {
...filters,
$and: filters.$and.map((condition) => {
// @todo: think of a better way to handle language to objectId conversion
// This is a workaround for the fact that language is stored as an ObjectId
// in the database, but we want to filter by its string representation.
if ('language' in condition && condition.language) {
return {
...condition,
language: new Types.ObjectId(condition.language as string),
};
}
return condition;
}),
};
}
return filters;
}
/**
* Build the aggregation stages that restrict a *nlpSampleEntities* collection
* to links which:
* 1. Reference all of the supplied `values`, and
* 2. Whose document satisfies the optional `filters`.
*
* @param criterias Object with:
* @param criterias.filters Extra filters to be applied on *nlpsamples*.
* @param criterias.entities Entity documents whose IDs should match `entity`.
* @param criterias.values Value documents whose IDs should match `value`.
* @returns Array of aggregation `PipelineStage`s ready to be concatenated
* into a larger pipeline.
*/
buildFindByEntitiesStages({
filters,
values,
}: {
filters: TFilterQuery<NlpSample>;
values: NlpValue[];
}): PipelineStage[] {
const requiredPairs = values.map(({ id, entity }) => ({
entity: new Types.ObjectId(entity),
value: new Types.ObjectId(id),
}));
const normalizedFilters = this.normalizeFilters(filters);
return [
{
$match: {
...normalizedFilters,
},
},
// Fetch the entities for each sample
{
$lookup: {
from: 'nlpsampleentities',
localField: '_id', // nlpsamples._id
foreignField: 'sample', // nlpsampleentities.sample
as: 'sampleentities',
pipeline: [
{
$match: {
$or: requiredPairs,
},
},
],
},
},
// Filter out empty or less matching
{
$match: {
$expr: {
$gte: [{ $size: '$sampleentities' }, requiredPairs.length],
},
},
},
// Collapse each link into an { entity, value } object
{
$addFields: {
entities: {
$ifNull: [
{
$map: {
input: '$sampleentities',
as: 's',
in: { entity: '$$s.entity', value: '$$s.value' },
},
},
[],
],
},
},
},
// Keep only the samples whose `entities` array ⊇ `requiredPairs`
{
$match: {
$expr: {
$eq: [
requiredPairs.length, // target size
{
$size: {
$setIntersection: ['$entities', requiredPairs],
},
},
],
},
},
},
//drop helper array if you dont need it downstream
{ $project: { entities: 0, sampleentities: 0 } },
];
}
findByEntitiesAggregation(
criterias: {
filters: TFilterQuery<NlpSample>;
values: NlpValue[];
},
page?: PageQueryDto<NlpSample>,
projection?: ProjectionType<NlpSample>,
): Aggregate<NlpSampleDocument[]> {
return this.model.aggregate<NlpSampleDocument>([
...this.buildFindByEntitiesStages(criterias),
// sort / skip / limit
...this.buildPaginationPipelineStages(page),
// projection
...(projection
? [
{
$project:
typeof projection === 'string'
? { [projection]: 1 }
: projection,
},
]
: []),
]);
}
async findByEntities(
criterias: {
filters: TFilterQuery<NlpSample>;
values: NlpValue[];
},
page?: PageQueryDto<NlpSample>,
projection?: ProjectionType<NlpSample>,
): Promise<NlpSample[]> {
const aggregation = this.findByEntitiesAggregation(
criterias,
page,
projection,
);
const resultSet = await aggregation.exec();
return resultSet.map((doc) =>
plainToClass(NlpSample, doc, this.transformOpts),
);
}
/**
* Find NLP samples by entities and populate them with their related data.
*
* @param criterias - Criteria containing filters and values to match.
* @param page - Optional pagination parameters.
* @param projection - Optional projection to limit fields returned.
* @returns Promise resolving to an array of populated NlpSampleFull objects.
*/
async findByEntitiesAndPopulate(
criterias: {
filters: TFilterQuery<NlpSample>;
values: NlpValue[];
},
page?: PageQueryDto<NlpSample>,
projection?: ProjectionType<NlpSample>,
): Promise<NlpSampleFull[]> {
const aggregation = this.findByEntitiesAggregation(
criterias,
page,
projection,
);
const docs = await aggregation.exec();
const populatedResultSet = await this.populate(docs);
return populatedResultSet.map((doc) =>
plainToClass(NlpSampleFull, doc, this.transformOpts),
);
}
/**
* Build an aggregation pipeline that counts NLP samples satisfying:
* the extra `filters` (passed to `$match` later on), and
* All of the supplied `entities` / `values`.
*
* @param criterias `{ filters, entities, values }`
* @returns Un-executed aggregation cursor.
*/
countByEntitiesAggregation(criterias: {
filters: TFilterQuery<NlpSample>;
values: NlpValue[];
}): Aggregate<{ count: number }[]> {
return this.model.aggregate<{ count: number }>([
...this.buildFindByEntitiesStages(criterias),
// Final count
{ $count: 'count' },
]);
}
/**
* Returns the count of samples by filters, entities and/or values
*
* @param criterias `{ filters, entities, values }`
* @returns Promise resolving to the count.
*/
async countByEntities(criterias: {
filters: TFilterQuery<NlpSample>;
values: NlpValue[];
}): Promise<number> {
const aggregation = this.countByEntitiesAggregation(criterias);
const [result] = await aggregation.exec();
return result?.count || 0;
}
/**
* Deletes NLP sample entities associated with the provided criteria before deleting the sample itself.
*

View File

@@ -10,9 +10,11 @@ import { CACHE_MANAGER } from '@nestjs/cache-manager';
import { BadRequestException, NotFoundException } from '@nestjs/common';
import { MongooseModule } from '@nestjs/mongoose';
import { NlpValueMatchPattern } from '@/chat/schemas/types/pattern';
import { LanguageRepository } from '@/i18n/repositories/language.repository';
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
import { LanguageService } from '@/i18n/services/language.service';
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
import { nlpSampleFixtures } from '@/utils/test/fixtures/nlpsample';
import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity';
import { getPageQuery } from '@/utils/test/pagination';
@@ -52,6 +54,7 @@ describe('NlpSampleService', () => {
let nlpEntityService: NlpEntityService;
let nlpSampleService: NlpSampleService;
let nlpSampleEntityService: NlpSampleEntityService;
let nlpValueService: NlpValueService;
let languageService: LanguageService;
let nlpSampleEntityRepository: NlpSampleEntityRepository;
let nlpSampleRepository: NlpSampleRepository;
@@ -98,6 +101,7 @@ describe('NlpSampleService', () => {
nlpEntityService,
nlpSampleService,
nlpSampleEntityService,
nlpValueService,
nlpSampleRepository,
nlpSampleEntityRepository,
nlpSampleEntityRepository,
@@ -107,6 +111,7 @@ describe('NlpSampleService', () => {
NlpEntityService,
NlpSampleService,
NlpSampleEntityService,
NlpValueService,
NlpSampleRepository,
NlpSampleEntityRepository,
NlpSampleEntityRepository,
@@ -360,4 +365,200 @@ describe('NlpSampleService', () => {
expect(extractSpy).not.toHaveBeenCalled();
});
});
describe('findByPatterns', () => {
it('should return samples without providing patterns', async () => {
const result = await nlpSampleService.findByPatterns(
{ filters: {}, patterns: [] },
undefined,
);
expect(Array.isArray(result)).toBe(true);
expect(result.length).toBeGreaterThan(0);
});
it('should return samples matching the given patterns', async () => {
// Assume pattern: entity 'intent', value 'greeting'
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'greeting' },
];
jest.spyOn(nlpSampleRepository, 'findByEntities');
jest.spyOn(nlpValueService, 'findByPatterns');
const result = await nlpSampleService.findByPatterns(
{ filters: {}, patterns },
undefined,
);
expect(nlpSampleRepository.findByEntities).toHaveBeenCalled();
expect(nlpValueService.findByPatterns).toHaveBeenCalled();
expect(Array.isArray(result)).toBe(true);
expect(result[0].text).toBe('Hello');
});
it('should return an empty array if no samples match the patterns', async () => {
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'nonexistent' },
];
jest.spyOn(nlpSampleRepository, 'findByEntities');
jest.spyOn(nlpValueService, 'findByPatterns');
const result = await nlpSampleService.findByPatterns(
{ filters: {}, patterns },
undefined,
);
expect(nlpSampleRepository.findByEntities).not.toHaveBeenCalled();
expect(nlpValueService.findByPatterns).toHaveBeenCalled();
expect(Array.isArray(result)).toBe(true);
expect(result).toHaveLength(0);
});
it('should support pagination', async () => {
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'greeting' },
];
const page: PageQueryDto<NlpSample> = {
limit: 1,
skip: 0,
sort: ['text', 'asc'],
};
const result = await nlpSampleService.findByPatterns(
{ filters: {}, patterns },
page,
);
expect(Array.isArray(result)).toBe(true);
expect(result.length).toBe(1);
});
});
describe('findByPatternsAndPopulate', () => {
it('should return populated NlpSampleFull instances for matching patterns', async () => {
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'greeting' },
];
const result = await nlpSampleService.findByPatternsAndPopulate(
{ filters: {}, patterns },
undefined,
);
expect(Array.isArray(result)).toBe(true);
expect(result.length).toBeGreaterThan(0);
result.forEach((sample) => {
expect(sample).toBeInstanceOf(NlpSampleFull);
expect(sample.entities).toBeDefined();
expect(Array.isArray(sample.entities)).toBe(true);
expect(sample.language).toBeDefined();
});
});
it('should return populated NlpSampleFull without providing patterns', async () => {
const result = await nlpSampleService.findByPatternsAndPopulate(
{ filters: { text: /Hello/gi }, patterns: [] },
undefined,
);
expect(Array.isArray(result)).toBe(true);
expect(result.length).toBe(1);
expect(result[0]).toBeInstanceOf(NlpSampleFull);
expect(result[0].entities).toBeDefined();
expect(Array.isArray(result[0].entities)).toBe(true);
});
it('should return an empty array if no samples match the patterns', async () => {
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'nonexistent' },
];
const result = await nlpSampleService.findByPatternsAndPopulate(
{ filters: {}, patterns },
undefined,
);
expect(Array.isArray(result)).toBe(true);
expect(result).toHaveLength(0);
});
it('should support pagination and projection', async () => {
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'greeting' },
];
const page: PageQueryDto<NlpSample> = {
limit: 1,
skip: 0,
sort: ['text', 'asc'],
};
const result = await nlpSampleService.findByPatternsAndPopulate(
{ filters: {}, patterns },
page,
);
expect(Array.isArray(result)).toBe(true);
expect(result.length).toBe(1);
});
});
describe('countByPatterns', () => {
it('should return the correct count for matching patterns', async () => {
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'greeting' },
];
jest.spyOn(nlpSampleRepository, 'countByEntities');
jest.spyOn(nlpValueService, 'findByPatterns');
const count = await nlpSampleService.countByPatterns({
filters: {},
patterns,
});
expect(nlpSampleRepository.countByEntities).toHaveBeenCalled();
expect(nlpValueService.findByPatterns).toHaveBeenCalled();
expect(typeof count).toBe('number');
expect(count).toBe(2);
});
it('should return the correct count without providing patterns', async () => {
jest.spyOn(nlpSampleRepository, 'findByEntities');
jest.spyOn(nlpValueService, 'findByPatterns');
const count = await nlpSampleService.countByPatterns({
filters: {},
patterns: [],
});
expect(nlpSampleRepository.findByEntities).not.toHaveBeenCalled();
expect(nlpValueService.findByPatterns).not.toHaveBeenCalled();
expect(typeof count).toBe('number');
expect(count).toBeGreaterThan(2);
});
it('should return 0 if no samples match the patterns', async () => {
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'nonexistent' },
];
const count = await nlpSampleService.countByPatterns({
filters: {},
patterns,
});
expect(count).toBe(0);
});
it('should respect filters (e.g. language)', async () => {
const patterns: NlpValueMatchPattern[] = [
{ entity: 'intent', match: 'value', value: 'greeting' },
];
const filters = { text: 'Hello' };
const count = await nlpSampleService.countByPatterns({
filters,
patterns,
});
expect(typeof count).toBe('number');
expect(count).toBe(1);
});
});
});

View File

@@ -12,14 +12,16 @@ import {
NotFoundException,
} from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { Document, Query } from 'mongoose';
import { Document, ProjectionType, Query } from 'mongoose';
import Papa from 'papaparse';
import { Message } from '@/chat/schemas/message.schema';
import { NlpValueMatchPattern } from '@/chat/schemas/types/pattern';
import { Language } from '@/i18n/schemas/language.schema';
import { LanguageService } from '@/i18n/services/language.service';
import { DeleteResult } from '@/utils/generics/base-repository';
import { BaseService } from '@/utils/generics/base-service';
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
import { TFilterQuery, THydratedDocument } from '@/utils/types/filter.types';
import { NlpSampleEntityCreateDto } from '../dto/nlp-sample-entity.dto';
@@ -35,6 +37,7 @@ import { NlpSampleEntityValue, NlpSampleState } from '../schemas/types';
import { NlpEntityService } from './nlp-entity.service';
import { NlpSampleEntityService } from './nlp-sample-entity.service';
import { NlpValueService } from './nlp-value.service';
@Injectable()
export class NlpSampleService extends BaseService<
@@ -47,11 +50,126 @@ export class NlpSampleService extends BaseService<
readonly repository: NlpSampleRepository,
private readonly nlpSampleEntityService: NlpSampleEntityService,
private readonly nlpEntityService: NlpEntityService,
private readonly nlpValueService: NlpValueService,
private readonly languageService: LanguageService,
) {
super(repository);
}
/**
* Retrieve samples that satisfy `filters` **and** reference any entity / value
* contained in `patterns`.
*
* The pattern list is first resolved via `NlpEntityService.findByPatterns`
* and `NlpValueService.findByPatterns`, then delegated to
* `repository.findByEntities`.
*
* @param criterias `{ filters, patterns }`
* @param page Optional paging / sorting descriptor.
* @param projection Optional Mongo projection.
* @returns Promise resolving to the matching samples.
*/
async findByPatterns(
{
filters,
patterns,
}: {
filters: TFilterQuery<NlpSample>;
patterns: NlpValueMatchPattern[];
},
page?: PageQueryDto<NlpSample>,
projection?: ProjectionType<NlpSample>,
): Promise<NlpSample[]> {
if (!patterns.length) {
return await this.repository.find(filters, page, projection);
}
const values = await this.nlpValueService.findByPatterns(patterns);
if (!values.length) {
return [];
}
return await this.repository.findByEntities(
{
filters,
values,
},
page,
projection,
);
}
/**
* Same as `findByPatterns`, but also populates all relations declared
* in the repository (`populatePaths`).
*
* @param criteria `{ filters, patterns }`
* @param page Optional paging / sorting descriptor.
* @param projection Optional Mongo projection.
* @returns Promise resolving to the populated samples.
*/
async findByPatternsAndPopulate(
{
filters,
patterns,
}: {
filters: TFilterQuery<NlpSample>;
patterns: NlpValueMatchPattern[];
},
page?: PageQueryDto<NlpSample>,
projection?: ProjectionType<NlpSample>,
): Promise<NlpSampleFull[]> {
if (!patterns.length) {
return await this.repository.findAndPopulate(filters, page, projection);
}
const values = await this.nlpValueService.findByPatterns(patterns);
if (!values.length) {
return [];
}
return await this.repository.findByEntitiesAndPopulate(
{
filters,
values,
},
page,
projection,
);
}
/**
* Count how many samples satisfy `filters` and reference any entity / value
* present in `patterns`.
*
* @param param0 `{ filters, patterns }`
* @returns Promise resolving to the count.
*/
async countByPatterns({
filters,
patterns,
}: {
filters: TFilterQuery<NlpSample>;
patterns: NlpValueMatchPattern[];
}): Promise<number> {
if (!patterns.length) {
return await this.repository.count(filters);
}
const values = await this.nlpValueService.findByPatterns(patterns);
if (!values.length) {
return 0;
}
return await this.repository.countByEntities({
filters,
values,
});
}
/**
* Fetches the samples and entities for a given sample type.
*

View File

@@ -8,6 +8,7 @@
import { forwardRef, Inject, Injectable } from '@nestjs/common';
import { NlpValueMatchPattern } from '@/chat/schemas/types/pattern';
import { DeleteResult } from '@/utils/generics/base-repository';
import { BaseService } from '@/utils/generics/base-service';
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
@@ -42,6 +43,20 @@ export class NlpValueService extends BaseService<
super(repository);
}
/**
* Fetch values whose `value` field matches the patterns provided.
*
* @param patterns Pattern list
* @returns Promise resolving to the matching values.
*/
async findByPatterns(patterns: NlpValueMatchPattern[]) {
return await this.find({
value: {
$in: patterns.map((p) => p.value),
},
});
}
/**
* Deletes an NLP value by its ID, cascading any dependent data.
*