mirror of
https://github.com/hexastack/hexabot
synced 2025-06-26 18:27:28 +00:00
Merge pull request #1097 from Hexastack/feat/nlp-sample-filter-by-entities
Feat/nlp sample filter by entities
This commit is contained in:
@@ -10,6 +10,7 @@ import { CACHE_MANAGER } from '@nestjs/cache-manager';
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { MongooseModule } from '@nestjs/mongoose';
|
||||
|
||||
import { NlpValueMatchPattern } from '@/chat/schemas/types/pattern';
|
||||
import { HelperService } from '@/helper/helper.service';
|
||||
import { LanguageRepository } from '@/i18n/repositories/language.repository';
|
||||
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
|
||||
@@ -181,11 +182,51 @@ describe('NlpSampleController', () => {
|
||||
})),
|
||||
);
|
||||
});
|
||||
|
||||
it('should find nlp samples with patterns', async () => {
|
||||
const pageQuery = getPageQuery<NlpSample>({ sort: ['text', 'desc'] });
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'greeting' },
|
||||
];
|
||||
const result = await nlpSampleController.findPage(
|
||||
pageQuery,
|
||||
['language', 'entities'],
|
||||
{},
|
||||
patterns,
|
||||
);
|
||||
// Should only return samples matching the pattern
|
||||
const nlpSamples = await nlpSampleService.findByPatternsAndPopulate(
|
||||
{ filters: {}, patterns },
|
||||
pageQuery,
|
||||
);
|
||||
expect(result).toEqualPayload(nlpSamples);
|
||||
});
|
||||
|
||||
it('should return empty array if no samples match the patterns', async () => {
|
||||
const pageQuery = getPageQuery<NlpSample>({ sort: ['text', 'desc'] });
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'nonexistent' },
|
||||
];
|
||||
jest.spyOn(nlpSampleService, 'findByPatternsAndPopulate');
|
||||
const result = await nlpSampleController.findPage(
|
||||
pageQuery,
|
||||
['language', 'entities'],
|
||||
{},
|
||||
patterns,
|
||||
);
|
||||
expect(nlpSampleService.findByPatternsAndPopulate).toHaveBeenCalledTimes(
|
||||
1,
|
||||
);
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('count', () => {
|
||||
it('should count the nlp samples', async () => {
|
||||
jest.spyOn(nlpSampleService, 'count');
|
||||
const result = await nlpSampleController.count({});
|
||||
expect(nlpSampleService.count).toHaveBeenCalledTimes(1);
|
||||
const count = nlpSampleFixtures.length;
|
||||
expect(result).toEqual({ count });
|
||||
});
|
||||
@@ -439,4 +480,34 @@ describe('NlpSampleController', () => {
|
||||
).rejects.toThrow(NotFoundException);
|
||||
});
|
||||
});
|
||||
|
||||
describe('filterCount', () => {
|
||||
it('should count the nlp samples without patterns', async () => {
|
||||
const filters = { text: 'Hello' };
|
||||
jest.spyOn(nlpSampleService, 'countByPatterns');
|
||||
const result = await nlpSampleController.filterCount(filters, []);
|
||||
expect(nlpSampleService.countByPatterns).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual({ count: 1 });
|
||||
});
|
||||
|
||||
it('should count the nlp samples with patterns', async () => {
|
||||
const filters = { text: 'Hello' };
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'greeting' },
|
||||
];
|
||||
jest.spyOn(nlpSampleService, 'countByPatterns');
|
||||
const result = await nlpSampleController.filterCount(filters, patterns);
|
||||
expect(nlpSampleService.countByPatterns).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual({ count: 1 });
|
||||
});
|
||||
|
||||
it('should return zero count when no samples match the filters and patterns', async () => {
|
||||
const filters = { text: 'Nonexistent' };
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'nonexistent' },
|
||||
];
|
||||
const result = await nlpSampleController.filterCount(filters, patterns);
|
||||
expect(result).toEqual({ count: 0 });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -29,7 +29,12 @@ import {
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { CsrfCheck } from '@tekuconcept/nestjs-csrf';
|
||||
import { Response } from 'express';
|
||||
import { z } from 'zod';
|
||||
|
||||
import {
|
||||
NlpValueMatchPattern,
|
||||
nlpValueMatchPatternSchema,
|
||||
} from '@/chat/schemas/types/pattern';
|
||||
import { HelperService } from '@/helper/helper.service';
|
||||
import { HelperType } from '@/helper/types';
|
||||
import { LanguageService } from '@/i18n/services/language.service';
|
||||
@@ -40,6 +45,7 @@ import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
|
||||
import { PageQueryPipe } from '@/utils/pagination/pagination-query.pipe';
|
||||
import { PopulatePipe } from '@/utils/pipes/populate.pipe';
|
||||
import { SearchFilterPipe } from '@/utils/pipes/search-filter.pipe';
|
||||
import { ZodQueryParamPipe } from '@/utils/pipes/zod.pipe';
|
||||
import { TFilterQuery } from '@/utils/types/filter.types';
|
||||
|
||||
import { NlpSampleDto, TNlpSampleDto } from '../dto/nlp-sample.dto';
|
||||
@@ -184,9 +190,22 @@ export class NlpSampleController extends BaseController<
|
||||
allowedFields: ['text', 'type', 'language'],
|
||||
}),
|
||||
)
|
||||
filters?: TFilterQuery<NlpSample>,
|
||||
filters: TFilterQuery<NlpSample> = {},
|
||||
@Query(
|
||||
new ZodQueryParamPipe(
|
||||
z.array(nlpValueMatchPatternSchema),
|
||||
(q) => q?.where?.patterns,
|
||||
),
|
||||
)
|
||||
patterns: NlpValueMatchPattern[] = [],
|
||||
) {
|
||||
return await this.count(filters);
|
||||
const count = await this.nlpSampleService.countByPatterns({
|
||||
filters,
|
||||
patterns,
|
||||
});
|
||||
return {
|
||||
count,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -285,10 +304,23 @@ export class NlpSampleController extends BaseController<
|
||||
}),
|
||||
)
|
||||
filters: TFilterQuery<NlpSample>,
|
||||
@Query(
|
||||
new ZodQueryParamPipe(
|
||||
z.array(nlpValueMatchPatternSchema),
|
||||
(q) => q?.where?.patterns,
|
||||
),
|
||||
)
|
||||
patterns: NlpValueMatchPattern[] = [],
|
||||
) {
|
||||
return this.canPopulate(populate)
|
||||
? await this.nlpSampleService.findAndPopulate(filters, pageQuery)
|
||||
: await this.nlpSampleService.find(filters, pageQuery);
|
||||
? await this.nlpSampleService.findByPatternsAndPopulate(
|
||||
{ filters, patterns },
|
||||
pageQuery,
|
||||
)
|
||||
: await this.nlpSampleService.findByPatterns(
|
||||
{ filters, patterns },
|
||||
pageQuery,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -7,9 +7,11 @@
|
||||
*/
|
||||
|
||||
import { MongooseModule } from '@nestjs/mongoose';
|
||||
import { Types } from 'mongoose';
|
||||
|
||||
import { LanguageRepository } from '@/i18n/repositories/language.repository';
|
||||
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
|
||||
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
|
||||
import { nlpSampleFixtures } from '@/utils/test/fixtures/nlpsample';
|
||||
import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity';
|
||||
import { getPageQuery } from '@/utils/test/pagination';
|
||||
@@ -29,13 +31,16 @@ import {
|
||||
NlpSampleFull,
|
||||
NlpSampleModel,
|
||||
} from '../schemas/nlp-sample.schema';
|
||||
import { NlpValue, NlpValueModel } from '../schemas/nlp-value.schema';
|
||||
|
||||
import { NlpSampleEntityRepository } from './nlp-sample-entity.repository';
|
||||
import { NlpSampleRepository } from './nlp-sample.repository';
|
||||
import { NlpValueRepository } from './nlp-value.repository';
|
||||
|
||||
describe('NlpSampleRepository', () => {
|
||||
let nlpSampleRepository: NlpSampleRepository;
|
||||
let nlpSampleEntityRepository: NlpSampleEntityRepository;
|
||||
let nlpValueRepository: NlpValueRepository;
|
||||
let languageRepository: LanguageRepository;
|
||||
let nlpSampleEntity: NlpSampleEntity | null;
|
||||
let noNlpSample: NlpSample | null;
|
||||
@@ -48,21 +53,28 @@ describe('NlpSampleRepository', () => {
|
||||
MongooseModule.forFeature([
|
||||
NlpSampleModel,
|
||||
NlpSampleEntityModel,
|
||||
NlpValueModel,
|
||||
LanguageModel,
|
||||
]),
|
||||
],
|
||||
providers: [
|
||||
NlpSampleRepository,
|
||||
NlpSampleEntityRepository,
|
||||
NlpValueRepository,
|
||||
LanguageRepository,
|
||||
],
|
||||
});
|
||||
[nlpSampleRepository, nlpSampleEntityRepository, languageRepository] =
|
||||
await getMocks([
|
||||
NlpSampleRepository,
|
||||
NlpSampleEntityRepository,
|
||||
LanguageRepository,
|
||||
]);
|
||||
[
|
||||
nlpSampleRepository,
|
||||
nlpSampleEntityRepository,
|
||||
nlpValueRepository,
|
||||
languageRepository,
|
||||
] = await getMocks([
|
||||
NlpSampleRepository,
|
||||
NlpSampleEntityRepository,
|
||||
NlpValueRepository,
|
||||
LanguageRepository,
|
||||
]);
|
||||
noNlpSample = await nlpSampleRepository.findOne({ text: 'No' });
|
||||
nlpSampleEntity = await nlpSampleEntityRepository.findOne({
|
||||
sample: noNlpSample!.id,
|
||||
@@ -141,4 +153,149 @@ describe('NlpSampleRepository', () => {
|
||||
expect(sampleEntities.length).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findByEntities', () => {
|
||||
it('should return mapped NlpSample instances for matching entities', async () => {
|
||||
const filters = {};
|
||||
const values = await nlpValueRepository.find({ value: 'greeting' });
|
||||
|
||||
const result = await nlpSampleRepository.findByEntities({
|
||||
filters,
|
||||
values,
|
||||
});
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0]).toBeInstanceOf(NlpSample);
|
||||
expect(result[0].text).toBe('Hello');
|
||||
});
|
||||
|
||||
it('should return an empty array if no samples match', async () => {
|
||||
const filters = {};
|
||||
const values = [
|
||||
{
|
||||
id: new Types.ObjectId().toHexString(),
|
||||
entity: new Types.ObjectId().toHexString(),
|
||||
value: 'nonexistent',
|
||||
},
|
||||
] as NlpValue[];
|
||||
|
||||
const result = await nlpSampleRepository.findByEntities({
|
||||
filters,
|
||||
values,
|
||||
});
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findByEntitiesAndPopulate', () => {
|
||||
it('should return populated NlpSampleFull instances for matching entities', async () => {
|
||||
const filters = {};
|
||||
const values = await nlpValueRepository.find({ value: 'greeting' });
|
||||
|
||||
const result = await nlpSampleRepository.findByEntitiesAndPopulate({
|
||||
filters,
|
||||
values,
|
||||
});
|
||||
|
||||
expect(result.length).toBe(2);
|
||||
result.forEach((sample) => {
|
||||
expect(sample).toBeInstanceOf(NlpSampleFull);
|
||||
expect(sample.entities).toBeDefined();
|
||||
expect(Array.isArray(sample.entities)).toBe(true);
|
||||
expect(sample.language).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
it('should return an empty array if no samples match', async () => {
|
||||
const filters = {};
|
||||
const values = [
|
||||
{
|
||||
id: new Types.ObjectId().toHexString(),
|
||||
entity: new Types.ObjectId().toHexString(),
|
||||
value: 'nonexistent',
|
||||
},
|
||||
] as NlpValue[];
|
||||
|
||||
const result = await nlpSampleRepository.findByEntitiesAndPopulate({
|
||||
filters,
|
||||
values,
|
||||
});
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should support pagination and projection', async () => {
|
||||
const filters = {};
|
||||
const values = await nlpValueRepository.find({ value: 'greeting' });
|
||||
const page = {
|
||||
limit: 1,
|
||||
skip: 0,
|
||||
sort: ['text', 'asc'],
|
||||
} as PageQueryDto<NlpSample>;
|
||||
const projection = { text: 1 };
|
||||
|
||||
const result = await nlpSampleRepository.findByEntitiesAndPopulate(
|
||||
{ filters, values },
|
||||
page,
|
||||
projection,
|
||||
);
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBe(1);
|
||||
if (result.length > 0) {
|
||||
expect(result[0]).toHaveProperty('text');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('countByEntities', () => {
|
||||
it('should return the correct count for matching entities', async () => {
|
||||
const filters = {};
|
||||
const values = await nlpValueRepository.find({ value: 'greeting' });
|
||||
|
||||
const count = await nlpSampleRepository.countByEntities({
|
||||
filters,
|
||||
values,
|
||||
});
|
||||
|
||||
expect(typeof count).toBe('number');
|
||||
expect(count).toBe(2);
|
||||
});
|
||||
|
||||
it('should return 0 if no samples match', async () => {
|
||||
const filters = {};
|
||||
const values = [
|
||||
{
|
||||
id: new Types.ObjectId().toHexString(),
|
||||
entity: new Types.ObjectId().toHexString(),
|
||||
value: 'nonexistent',
|
||||
},
|
||||
] as NlpValue[];
|
||||
|
||||
const count = await nlpSampleRepository.countByEntities({
|
||||
filters,
|
||||
values,
|
||||
});
|
||||
|
||||
expect(count).toBe(0);
|
||||
});
|
||||
|
||||
it('should respect filters (e.g. language)', async () => {
|
||||
const values = await nlpValueRepository.find({ value: 'greeting' });
|
||||
const language = languages[0];
|
||||
const filters = { language: language.id };
|
||||
|
||||
const count = await nlpSampleRepository.countByEntities({
|
||||
filters,
|
||||
values,
|
||||
});
|
||||
|
||||
// Should be <= total greeting samples, and >= 0
|
||||
expect(typeof count).toBe('number');
|
||||
expect(count).toBeGreaterThanOrEqual(0);
|
||||
expect(count).toBeLessThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,18 +8,31 @@
|
||||
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectModel } from '@nestjs/mongoose';
|
||||
import { Document, Model, Query } from 'mongoose';
|
||||
import { plainToClass } from 'class-transformer';
|
||||
import {
|
||||
Aggregate,
|
||||
Document,
|
||||
Model,
|
||||
PipelineStage,
|
||||
ProjectionType,
|
||||
Query,
|
||||
Types,
|
||||
} from 'mongoose';
|
||||
|
||||
import { BaseRepository, DeleteResult } from '@/utils/generics/base-repository';
|
||||
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
|
||||
import { TFilterQuery } from '@/utils/types/filter.types';
|
||||
|
||||
import { TNlpSampleDto } from '../dto/nlp-sample.dto';
|
||||
import { NlpSampleEntity } from '../schemas/nlp-sample-entity.schema';
|
||||
import {
|
||||
NLP_SAMPLE_POPULATE,
|
||||
NlpSample,
|
||||
NlpSampleDocument,
|
||||
NlpSampleFull,
|
||||
NlpSamplePopulate,
|
||||
} from '../schemas/nlp-sample.schema';
|
||||
import { NlpValue } from '../schemas/nlp-value.schema';
|
||||
|
||||
import { NlpSampleEntityRepository } from './nlp-sample-entity.repository';
|
||||
|
||||
@@ -32,11 +45,257 @@ export class NlpSampleRepository extends BaseRepository<
|
||||
> {
|
||||
constructor(
|
||||
@InjectModel(NlpSample.name) readonly model: Model<NlpSample>,
|
||||
@InjectModel(NlpSampleEntity.name)
|
||||
readonly sampleEntityModel: Model<NlpSampleEntity>,
|
||||
private readonly nlpSampleEntityRepository: NlpSampleEntityRepository,
|
||||
) {
|
||||
super(model, NlpSample, NLP_SAMPLE_POPULATE, NlpSampleFull);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize the filter query.
|
||||
*
|
||||
* @param filters - The filters to normalize.
|
||||
* @returns The normalized filters.
|
||||
*/
|
||||
private normalizeFilters(
|
||||
filters: TFilterQuery<NlpSample>,
|
||||
): TFilterQuery<NlpSample> {
|
||||
if (filters?.$and) {
|
||||
return {
|
||||
...filters,
|
||||
$and: filters.$and.map((condition) => {
|
||||
// @todo: think of a better way to handle language to objectId conversion
|
||||
// This is a workaround for the fact that language is stored as an ObjectId
|
||||
// in the database, but we want to filter by its string representation.
|
||||
if ('language' in condition && condition.language) {
|
||||
return {
|
||||
...condition,
|
||||
language: new Types.ObjectId(condition.language as string),
|
||||
};
|
||||
}
|
||||
return condition;
|
||||
}),
|
||||
};
|
||||
}
|
||||
return filters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the aggregation stages that restrict a *nlpSampleEntities* collection
|
||||
* to links which:
|
||||
* 1. Reference all of the supplied `values`, and
|
||||
* 2. Whose document satisfies the optional `filters`.
|
||||
*
|
||||
* @param criterias Object with:
|
||||
* @param criterias.filters Extra filters to be applied on *nlpsamples*.
|
||||
* @param criterias.entities Entity documents whose IDs should match `entity`.
|
||||
* @param criterias.values Value documents whose IDs should match `value`.
|
||||
* @returns Array of aggregation `PipelineStage`s ready to be concatenated
|
||||
* into a larger pipeline.
|
||||
*/
|
||||
buildFindByEntitiesStages({
|
||||
filters,
|
||||
values,
|
||||
}: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
values: NlpValue[];
|
||||
}): PipelineStage[] {
|
||||
const requiredPairs = values.map(({ id, entity }) => ({
|
||||
entity: new Types.ObjectId(entity),
|
||||
value: new Types.ObjectId(id),
|
||||
}));
|
||||
|
||||
const normalizedFilters = this.normalizeFilters(filters);
|
||||
|
||||
return [
|
||||
{
|
||||
$match: {
|
||||
...normalizedFilters,
|
||||
},
|
||||
},
|
||||
|
||||
// Fetch the entities for each sample
|
||||
{
|
||||
$lookup: {
|
||||
from: 'nlpsampleentities',
|
||||
localField: '_id', // nlpsamples._id
|
||||
foreignField: 'sample', // nlpsampleentities.sample
|
||||
as: 'sampleentities',
|
||||
pipeline: [
|
||||
{
|
||||
$match: {
|
||||
$or: requiredPairs,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
// Filter out empty or less matching
|
||||
{
|
||||
$match: {
|
||||
$expr: {
|
||||
$gte: [{ $size: '$sampleentities' }, requiredPairs.length],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
// Collapse each link into an { entity, value } object
|
||||
{
|
||||
$addFields: {
|
||||
entities: {
|
||||
$ifNull: [
|
||||
{
|
||||
$map: {
|
||||
input: '$sampleentities',
|
||||
as: 's',
|
||||
in: { entity: '$$s.entity', value: '$$s.value' },
|
||||
},
|
||||
},
|
||||
[],
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
// Keep only the samples whose `entities` array ⊇ `requiredPairs`
|
||||
{
|
||||
$match: {
|
||||
$expr: {
|
||||
$eq: [
|
||||
requiredPairs.length, // target size
|
||||
{
|
||||
$size: {
|
||||
$setIntersection: ['$entities', requiredPairs],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
//drop helper array if you don’t need it downstream
|
||||
{ $project: { entities: 0, sampleentities: 0 } },
|
||||
];
|
||||
}
|
||||
|
||||
findByEntitiesAggregation(
|
||||
criterias: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
values: NlpValue[];
|
||||
},
|
||||
page?: PageQueryDto<NlpSample>,
|
||||
projection?: ProjectionType<NlpSample>,
|
||||
): Aggregate<NlpSampleDocument[]> {
|
||||
return this.model.aggregate<NlpSampleDocument>([
|
||||
...this.buildFindByEntitiesStages(criterias),
|
||||
|
||||
// sort / skip / limit
|
||||
...this.buildPaginationPipelineStages(page),
|
||||
|
||||
// projection
|
||||
...(projection
|
||||
? [
|
||||
{
|
||||
$project:
|
||||
typeof projection === 'string'
|
||||
? { [projection]: 1 }
|
||||
: projection,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]);
|
||||
}
|
||||
|
||||
async findByEntities(
|
||||
criterias: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
values: NlpValue[];
|
||||
},
|
||||
page?: PageQueryDto<NlpSample>,
|
||||
projection?: ProjectionType<NlpSample>,
|
||||
): Promise<NlpSample[]> {
|
||||
const aggregation = this.findByEntitiesAggregation(
|
||||
criterias,
|
||||
page,
|
||||
projection,
|
||||
);
|
||||
|
||||
const resultSet = await aggregation.exec();
|
||||
return resultSet.map((doc) =>
|
||||
plainToClass(NlpSample, doc, this.transformOpts),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find NLP samples by entities and populate them with their related data.
|
||||
*
|
||||
* @param criterias - Criteria containing filters and values to match.
|
||||
* @param page - Optional pagination parameters.
|
||||
* @param projection - Optional projection to limit fields returned.
|
||||
* @returns Promise resolving to an array of populated NlpSampleFull objects.
|
||||
*/
|
||||
async findByEntitiesAndPopulate(
|
||||
criterias: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
values: NlpValue[];
|
||||
},
|
||||
page?: PageQueryDto<NlpSample>,
|
||||
projection?: ProjectionType<NlpSample>,
|
||||
): Promise<NlpSampleFull[]> {
|
||||
const aggregation = this.findByEntitiesAggregation(
|
||||
criterias,
|
||||
page,
|
||||
projection,
|
||||
);
|
||||
|
||||
const docs = await aggregation.exec();
|
||||
|
||||
const populatedResultSet = await this.populate(docs);
|
||||
|
||||
return populatedResultSet.map((doc) =>
|
||||
plainToClass(NlpSampleFull, doc, this.transformOpts),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an aggregation pipeline that counts NLP samples satisfying:
|
||||
* – the extra `filters` (passed to `$match` later on), and
|
||||
* – All of the supplied `entities` / `values`.
|
||||
*
|
||||
* @param criterias `{ filters, entities, values }`
|
||||
* @returns Un-executed aggregation cursor.
|
||||
*/
|
||||
countByEntitiesAggregation(criterias: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
values: NlpValue[];
|
||||
}): Aggregate<{ count: number }[]> {
|
||||
return this.model.aggregate<{ count: number }>([
|
||||
...this.buildFindByEntitiesStages(criterias),
|
||||
|
||||
// Final count
|
||||
{ $count: 'count' },
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the count of samples by filters, entities and/or values
|
||||
*
|
||||
* @param criterias `{ filters, entities, values }`
|
||||
* @returns Promise resolving to the count.
|
||||
*/
|
||||
async countByEntities(criterias: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
values: NlpValue[];
|
||||
}): Promise<number> {
|
||||
const aggregation = this.countByEntitiesAggregation(criterias);
|
||||
|
||||
const [result] = await aggregation.exec();
|
||||
|
||||
return result?.count || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes NLP sample entities associated with the provided criteria before deleting the sample itself.
|
||||
*
|
||||
|
||||
@@ -10,9 +10,11 @@ import { CACHE_MANAGER } from '@nestjs/cache-manager';
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { MongooseModule } from '@nestjs/mongoose';
|
||||
|
||||
import { NlpValueMatchPattern } from '@/chat/schemas/types/pattern';
|
||||
import { LanguageRepository } from '@/i18n/repositories/language.repository';
|
||||
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
|
||||
import { LanguageService } from '@/i18n/services/language.service';
|
||||
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
|
||||
import { nlpSampleFixtures } from '@/utils/test/fixtures/nlpsample';
|
||||
import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity';
|
||||
import { getPageQuery } from '@/utils/test/pagination';
|
||||
@@ -52,6 +54,7 @@ describe('NlpSampleService', () => {
|
||||
let nlpEntityService: NlpEntityService;
|
||||
let nlpSampleService: NlpSampleService;
|
||||
let nlpSampleEntityService: NlpSampleEntityService;
|
||||
let nlpValueService: NlpValueService;
|
||||
let languageService: LanguageService;
|
||||
let nlpSampleEntityRepository: NlpSampleEntityRepository;
|
||||
let nlpSampleRepository: NlpSampleRepository;
|
||||
@@ -98,6 +101,7 @@ describe('NlpSampleService', () => {
|
||||
nlpEntityService,
|
||||
nlpSampleService,
|
||||
nlpSampleEntityService,
|
||||
nlpValueService,
|
||||
nlpSampleRepository,
|
||||
nlpSampleEntityRepository,
|
||||
nlpSampleEntityRepository,
|
||||
@@ -107,6 +111,7 @@ describe('NlpSampleService', () => {
|
||||
NlpEntityService,
|
||||
NlpSampleService,
|
||||
NlpSampleEntityService,
|
||||
NlpValueService,
|
||||
NlpSampleRepository,
|
||||
NlpSampleEntityRepository,
|
||||
NlpSampleEntityRepository,
|
||||
@@ -360,4 +365,200 @@ describe('NlpSampleService', () => {
|
||||
expect(extractSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('findByPatterns', () => {
|
||||
it('should return samples without providing patterns', async () => {
|
||||
const result = await nlpSampleService.findByPatterns(
|
||||
{ filters: {}, patterns: [] },
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should return samples matching the given patterns', async () => {
|
||||
// Assume pattern: entity 'intent', value 'greeting'
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'greeting' },
|
||||
];
|
||||
jest.spyOn(nlpSampleRepository, 'findByEntities');
|
||||
jest.spyOn(nlpValueService, 'findByPatterns');
|
||||
const result = await nlpSampleService.findByPatterns(
|
||||
{ filters: {}, patterns },
|
||||
undefined,
|
||||
);
|
||||
expect(nlpSampleRepository.findByEntities).toHaveBeenCalled();
|
||||
expect(nlpValueService.findByPatterns).toHaveBeenCalled();
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result[0].text).toBe('Hello');
|
||||
});
|
||||
|
||||
it('should return an empty array if no samples match the patterns', async () => {
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'nonexistent' },
|
||||
];
|
||||
|
||||
jest.spyOn(nlpSampleRepository, 'findByEntities');
|
||||
jest.spyOn(nlpValueService, 'findByPatterns');
|
||||
const result = await nlpSampleService.findByPatterns(
|
||||
{ filters: {}, patterns },
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(nlpSampleRepository.findByEntities).not.toHaveBeenCalled();
|
||||
expect(nlpValueService.findByPatterns).toHaveBeenCalled();
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should support pagination', async () => {
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'greeting' },
|
||||
];
|
||||
const page: PageQueryDto<NlpSample> = {
|
||||
limit: 1,
|
||||
skip: 0,
|
||||
sort: ['text', 'asc'],
|
||||
};
|
||||
|
||||
const result = await nlpSampleService.findByPatterns(
|
||||
{ filters: {}, patterns },
|
||||
page,
|
||||
);
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findByPatternsAndPopulate', () => {
|
||||
it('should return populated NlpSampleFull instances for matching patterns', async () => {
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'greeting' },
|
||||
];
|
||||
|
||||
const result = await nlpSampleService.findByPatternsAndPopulate(
|
||||
{ filters: {}, patterns },
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
result.forEach((sample) => {
|
||||
expect(sample).toBeInstanceOf(NlpSampleFull);
|
||||
expect(sample.entities).toBeDefined();
|
||||
expect(Array.isArray(sample.entities)).toBe(true);
|
||||
expect(sample.language).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
it('should return populated NlpSampleFull without providing patterns', async () => {
|
||||
const result = await nlpSampleService.findByPatternsAndPopulate(
|
||||
{ filters: { text: /Hello/gi }, patterns: [] },
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBe(1);
|
||||
expect(result[0]).toBeInstanceOf(NlpSampleFull);
|
||||
expect(result[0].entities).toBeDefined();
|
||||
expect(Array.isArray(result[0].entities)).toBe(true);
|
||||
});
|
||||
|
||||
it('should return an empty array if no samples match the patterns', async () => {
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'nonexistent' },
|
||||
];
|
||||
|
||||
const result = await nlpSampleService.findByPatternsAndPopulate(
|
||||
{ filters: {}, patterns },
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should support pagination and projection', async () => {
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'greeting' },
|
||||
];
|
||||
const page: PageQueryDto<NlpSample> = {
|
||||
limit: 1,
|
||||
skip: 0,
|
||||
sort: ['text', 'asc'],
|
||||
};
|
||||
|
||||
const result = await nlpSampleService.findByPatternsAndPopulate(
|
||||
{ filters: {}, patterns },
|
||||
page,
|
||||
);
|
||||
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('countByPatterns', () => {
|
||||
it('should return the correct count for matching patterns', async () => {
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'greeting' },
|
||||
];
|
||||
|
||||
jest.spyOn(nlpSampleRepository, 'countByEntities');
|
||||
jest.spyOn(nlpValueService, 'findByPatterns');
|
||||
const count = await nlpSampleService.countByPatterns({
|
||||
filters: {},
|
||||
patterns,
|
||||
});
|
||||
|
||||
expect(nlpSampleRepository.countByEntities).toHaveBeenCalled();
|
||||
expect(nlpValueService.findByPatterns).toHaveBeenCalled();
|
||||
expect(typeof count).toBe('number');
|
||||
expect(count).toBe(2);
|
||||
});
|
||||
|
||||
it('should return the correct count without providing patterns', async () => {
|
||||
jest.spyOn(nlpSampleRepository, 'findByEntities');
|
||||
jest.spyOn(nlpValueService, 'findByPatterns');
|
||||
const count = await nlpSampleService.countByPatterns({
|
||||
filters: {},
|
||||
patterns: [],
|
||||
});
|
||||
|
||||
expect(nlpSampleRepository.findByEntities).not.toHaveBeenCalled();
|
||||
expect(nlpValueService.findByPatterns).not.toHaveBeenCalled();
|
||||
expect(typeof count).toBe('number');
|
||||
expect(count).toBeGreaterThan(2);
|
||||
});
|
||||
|
||||
it('should return 0 if no samples match the patterns', async () => {
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'nonexistent' },
|
||||
];
|
||||
|
||||
const count = await nlpSampleService.countByPatterns({
|
||||
filters: {},
|
||||
patterns,
|
||||
});
|
||||
|
||||
expect(count).toBe(0);
|
||||
});
|
||||
|
||||
it('should respect filters (e.g. language)', async () => {
|
||||
const patterns: NlpValueMatchPattern[] = [
|
||||
{ entity: 'intent', match: 'value', value: 'greeting' },
|
||||
];
|
||||
const filters = { text: 'Hello' };
|
||||
|
||||
const count = await nlpSampleService.countByPatterns({
|
||||
filters,
|
||||
patterns,
|
||||
});
|
||||
|
||||
expect(typeof count).toBe('number');
|
||||
expect(count).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,14 +12,16 @@ import {
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { OnEvent } from '@nestjs/event-emitter';
|
||||
import { Document, Query } from 'mongoose';
|
||||
import { Document, ProjectionType, Query } from 'mongoose';
|
||||
import Papa from 'papaparse';
|
||||
|
||||
import { Message } from '@/chat/schemas/message.schema';
|
||||
import { NlpValueMatchPattern } from '@/chat/schemas/types/pattern';
|
||||
import { Language } from '@/i18n/schemas/language.schema';
|
||||
import { LanguageService } from '@/i18n/services/language.service';
|
||||
import { DeleteResult } from '@/utils/generics/base-repository';
|
||||
import { BaseService } from '@/utils/generics/base-service';
|
||||
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
|
||||
import { TFilterQuery, THydratedDocument } from '@/utils/types/filter.types';
|
||||
|
||||
import { NlpSampleEntityCreateDto } from '../dto/nlp-sample-entity.dto';
|
||||
@@ -35,6 +37,7 @@ import { NlpSampleEntityValue, NlpSampleState } from '../schemas/types';
|
||||
|
||||
import { NlpEntityService } from './nlp-entity.service';
|
||||
import { NlpSampleEntityService } from './nlp-sample-entity.service';
|
||||
import { NlpValueService } from './nlp-value.service';
|
||||
|
||||
@Injectable()
|
||||
export class NlpSampleService extends BaseService<
|
||||
@@ -47,11 +50,126 @@ export class NlpSampleService extends BaseService<
|
||||
readonly repository: NlpSampleRepository,
|
||||
private readonly nlpSampleEntityService: NlpSampleEntityService,
|
||||
private readonly nlpEntityService: NlpEntityService,
|
||||
private readonly nlpValueService: NlpValueService,
|
||||
private readonly languageService: LanguageService,
|
||||
) {
|
||||
super(repository);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve samples that satisfy `filters` **and** reference any entity / value
|
||||
* contained in `patterns`.
|
||||
*
|
||||
* The pattern list is first resolved via `NlpEntityService.findByPatterns`
|
||||
* and `NlpValueService.findByPatterns`, then delegated to
|
||||
* `repository.findByEntities`.
|
||||
*
|
||||
* @param criterias `{ filters, patterns }`
|
||||
* @param page Optional paging / sorting descriptor.
|
||||
* @param projection Optional Mongo projection.
|
||||
* @returns Promise resolving to the matching samples.
|
||||
*/
|
||||
async findByPatterns(
|
||||
{
|
||||
filters,
|
||||
patterns,
|
||||
}: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
patterns: NlpValueMatchPattern[];
|
||||
},
|
||||
page?: PageQueryDto<NlpSample>,
|
||||
projection?: ProjectionType<NlpSample>,
|
||||
): Promise<NlpSample[]> {
|
||||
if (!patterns.length) {
|
||||
return await this.repository.find(filters, page, projection);
|
||||
}
|
||||
|
||||
const values = await this.nlpValueService.findByPatterns(patterns);
|
||||
|
||||
if (!values.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return await this.repository.findByEntities(
|
||||
{
|
||||
filters,
|
||||
values,
|
||||
},
|
||||
page,
|
||||
projection,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as `findByPatterns`, but also populates all relations declared
|
||||
* in the repository (`populatePaths`).
|
||||
*
|
||||
* @param criteria `{ filters, patterns }`
|
||||
* @param page Optional paging / sorting descriptor.
|
||||
* @param projection Optional Mongo projection.
|
||||
* @returns Promise resolving to the populated samples.
|
||||
*/
|
||||
async findByPatternsAndPopulate(
|
||||
{
|
||||
filters,
|
||||
patterns,
|
||||
}: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
patterns: NlpValueMatchPattern[];
|
||||
},
|
||||
page?: PageQueryDto<NlpSample>,
|
||||
projection?: ProjectionType<NlpSample>,
|
||||
): Promise<NlpSampleFull[]> {
|
||||
if (!patterns.length) {
|
||||
return await this.repository.findAndPopulate(filters, page, projection);
|
||||
}
|
||||
|
||||
const values = await this.nlpValueService.findByPatterns(patterns);
|
||||
|
||||
if (!values.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return await this.repository.findByEntitiesAndPopulate(
|
||||
{
|
||||
filters,
|
||||
values,
|
||||
},
|
||||
page,
|
||||
projection,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Count how many samples satisfy `filters` and reference any entity / value
|
||||
* present in `patterns`.
|
||||
*
|
||||
* @param param0 `{ filters, patterns }`
|
||||
* @returns Promise resolving to the count.
|
||||
*/
|
||||
async countByPatterns({
|
||||
filters,
|
||||
patterns,
|
||||
}: {
|
||||
filters: TFilterQuery<NlpSample>;
|
||||
patterns: NlpValueMatchPattern[];
|
||||
}): Promise<number> {
|
||||
if (!patterns.length) {
|
||||
return await this.repository.count(filters);
|
||||
}
|
||||
|
||||
const values = await this.nlpValueService.findByPatterns(patterns);
|
||||
|
||||
if (!values.length) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return await this.repository.countByEntities({
|
||||
filters,
|
||||
values,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the samples and entities for a given sample type.
|
||||
*
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
import { forwardRef, Inject, Injectable } from '@nestjs/common';
|
||||
|
||||
import { NlpValueMatchPattern } from '@/chat/schemas/types/pattern';
|
||||
import { DeleteResult } from '@/utils/generics/base-repository';
|
||||
import { BaseService } from '@/utils/generics/base-service';
|
||||
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
|
||||
@@ -42,6 +43,20 @@ export class NlpValueService extends BaseService<
|
||||
super(repository);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch values whose `value` field matches the patterns provided.
|
||||
*
|
||||
* @param patterns Pattern list
|
||||
* @returns Promise resolving to the matching values.
|
||||
*/
|
||||
async findByPatterns(patterns: NlpValueMatchPattern[]) {
|
||||
return await this.find({
|
||||
value: {
|
||||
$in: patterns.map((p) => p.value),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an NLP value by its ID, cascading any dependent data.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user