From b79fcac080e3723ff42aae7f9a72aff0d0c412e2 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Fri, 21 Mar 2025 14:52:45 +0100 Subject: [PATCH 01/20] feat: support the display samples count per value --- .../nlp/controllers/nlp-value.controller.ts | 18 ++- api/src/nlp/services/nlp-value.service.ts | 124 +++++++++++++++++- frontend/public/locales/en/translation.json | 1 + frontend/public/locales/fr/translation.json | 1 + .../components/nlp/components/NlpValue.tsx | 19 ++- frontend/src/types/nlp-value.types.ts | 1 + 6 files changed, 160 insertions(+), 4 deletions(-) diff --git a/api/src/nlp/controllers/nlp-value.controller.ts b/api/src/nlp/controllers/nlp-value.controller.ts index a8c3c6be..42cb334f 100644 --- a/api/src/nlp/controllers/nlp-value.controller.ts +++ b/api/src/nlp/controllers/nlp-value.controller.ts @@ -125,6 +125,22 @@ export class NlpValueController extends BaseController< return doc; } + @Get('') + async findAndPopulateNlpValuesWithCount( + @Query(PageQueryPipe) pageQuery: PageQueryDto, + @Query(PopulatePipe) populate: string[], + @Query( + new SearchFilterPipe({ allowedFields: ['entity', 'value'] }), + ) + filters?: TFilterQuery, + ) { + return await this.nlpValueService.findAndPopulateNlpValuesWithCount( + populate, + filters, + pageQuery, + ); + } + /** * Retrieves a paginated list of NLP values. * @@ -136,7 +152,7 @@ export class NlpValueController extends BaseController< * * @returns A promise resolving to a paginated list of NLP values. */ - @Get() + // @Get('') disabled async findPage( @Query(PageQueryPipe) pageQuery: PageQueryDto, @Query(PopulatePipe) populate: string[], diff --git a/api/src/nlp/services/nlp-value.service.ts b/api/src/nlp/services/nlp-value.service.ts index f246ad61..c9ec6309 100644 --- a/api/src/nlp/services/nlp-value.service.ts +++ b/api/src/nlp/services/nlp-value.service.ts @@ -1,5 +1,5 @@ /* - * Copyright © 2024 Hexastack. All rights reserved. + * Copyright © 2025 Hexastack. All rights reserved. * * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. @@ -7,11 +7,15 @@ */ import { forwardRef, Inject, Injectable } from '@nestjs/common'; +import { Types } from 'mongoose'; import { DeleteResult } from '@/utils/generics/base-repository'; import { BaseService } from '@/utils/generics/base-service'; +import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; +import { TFilterQuery } from '@/utils/types/filter.types'; import { NlpValueCreateDto, NlpValueDto } from '../dto/nlp-value.dto'; +import { NlpSampleEntityRepository } from '../repositories/nlp-sample-entity.repository'; import { NlpValueRepository } from '../repositories/nlp-value.repository'; import { NlpEntity } from '../schemas/nlp-entity.schema'; import { @@ -34,6 +38,7 @@ export class NlpValueService extends BaseService< readonly repository: NlpValueRepository, @Inject(forwardRef(() => NlpEntityService)) private readonly nlpEntityService: NlpEntityService, + private readonly nlpSampleEntityRepository: NlpSampleEntityRepository, ) { super(repository); } @@ -218,4 +223,121 @@ export class NlpValueService extends BaseService< }); return Promise.all(promises); } + + async findAndPopulateNlpValuesWithCount( + populate: string[], + filters?: TFilterQuery, + pageQuery?: PageQueryDto, + ) { + const { $and = [], ...rest } = filters || ({} as TFilterQuery); + + const entityValueModel = this.getRepository().model; + + return entityValueModel + .aggregate([ + { + // support filters + $match: { + ...rest, + ...($and?.length && { + $and: + $and?.map((v) => { + if (v.entity) { + return { + ...v, + entity: new Types.ObjectId(String(v.entity)), + }; + } + + return v; + }) || [], + }), + }, + }, + // support pageQuery + { + $skip: pageQuery?.skip || 0, + }, + { + $limit: pageQuery?.limit || 10, + }, + { + $sort: { + [pageQuery?.sort?.[0] || 'createdAt']: + pageQuery?.sort?.[1] === 'desc' ? -1 : 1, + }, + }, + { + $lookup: { + from: 'nlpsampleentities', + localField: '_id', + foreignField: 'value', + as: 'sampleEntities', + }, + }, + { + $unwind: { + path: '$sampleEntities', + preserveNullAndEmptyArrays: true, + }, + }, + { + $lookup: { + from: 'nlpsamples', + localField: 'sampleEntities.sample', + foreignField: '_id', + as: 'samples', + }, + }, + { + $lookup: { + from: 'nlpentities', + localField: 'entity', + foreignField: '_id', + as: 'entities', + }, + }, + { + $group: { + _id: '$_id', + value: { $first: '$value' }, + expressions: { $first: '$expressions' }, + builtin: { $first: '$builtin' }, + metadata: { $first: '$metadata' }, + createdAt: { $first: '$createdAt' }, + updatedAt: { $first: '$updatedAt' }, + entity: { + // support populate + $first: populate.some((p) => + this.getRepository() + .getPopulate() + .map((p) => p.toString()) + .includes(p), + ) + ? '$entities' + : '$entity', + }, + //TODO when samples is empty array we need to return 0 not 1 + nlpSamplesCount: { + $sum: { $cond: [{ $ifNull: ['samples', false] }, 1, 0] }, + }, + }, + }, + { + $project: { + id: '$_id', + _id: 0, + value: 1, + expressions: 1, + builtin: 1, + entity: 1, + metadata: 1, + createdAt: 1, + updatedAt: 1, + nlpSamplesCount: 1, + }, + }, + ]) + .exec(); + } } diff --git a/frontend/public/locales/en/translation.json b/frontend/public/locales/en/translation.json index 93781e15..b243c3ae 100644 --- a/frontend/public/locales/en/translation.json +++ b/frontend/public/locales/en/translation.json @@ -334,6 +334,7 @@ "nlp": "NLU", "nlp_entity": "Entity", "nlp_entity_value": "Value", + "nlp_samples_count": "Nlp Samples count", "value": "Value", "synonyms": "Synonyms", "lookups": "Lookups", diff --git a/frontend/public/locales/fr/translation.json b/frontend/public/locales/fr/translation.json index 208dfc25..cd32248c 100644 --- a/frontend/public/locales/fr/translation.json +++ b/frontend/public/locales/fr/translation.json @@ -334,6 +334,7 @@ "nlp": "NLU", "nlp_entity": "Entité NLU", "nlp_entity_value": "Valeur NLU", + "nlp_samples_count": "Nombre des échantillons", "value": "Valeur", "lookups": "Stratégies", "lookup_strategies": "Stratégie de recherche", diff --git a/frontend/src/components/nlp/components/NlpValue.tsx b/frontend/src/components/nlp/components/NlpValue.tsx index 12c462e8..92cc7d04 100644 --- a/frontend/src/components/nlp/components/NlpValue.tsx +++ b/frontend/src/components/nlp/components/NlpValue.tsx @@ -55,7 +55,7 @@ export const NlpValues = ({ entityId }: { entityId: string }) => { const canHaveSynonyms = nlpEntity?.lookups?.[0] === NlpLookups.keywords; const { onSearch, searchPayload } = useSearch({ $eq: [{ entity: entityId }], - $or: ["doc", "value"] + $or: ["doc", "value"], }); const { dataGridProps } = useFind( { entity: EntityType.NLP_VALUE }, @@ -103,7 +103,7 @@ export const NlpValues = ({ entityId }: { entityId: string }) => { ], t("label.operations"), ); - const synonymsColumn = { + const synonymsColumn = { flex: 3, field: "synonyms", headerName: t("label.synonyms"), @@ -125,6 +125,21 @@ export const NlpValues = ({ entityId }: { entityId: string }) => { disableColumnMenu: true, renderHeader, }, + { + flex: 3, + field: "nlpSamplesCount", + headerName: t("label.nlp_samples_count"), + sortable: true, + disableColumnMenu: true, + renderHeader, + renderCell: ({ row }) => ( + + ), + }, { flex: 3, field: "doc", diff --git a/frontend/src/types/nlp-value.types.ts b/frontend/src/types/nlp-value.types.ts index 7b7a1e5e..4986ee8d 100644 --- a/frontend/src/types/nlp-value.types.ts +++ b/frontend/src/types/nlp-value.types.ts @@ -19,6 +19,7 @@ export interface INlpValueAttributes { expressions?: string[]; metadata?: Record; builtin?: boolean; + nlpSamplesCount?: number; } export interface INlpValueStub extends IBaseSchema, INlpValueAttributes {} From 7ecf821c48b2acb1b085b5b1f8dbae770ac25fc6 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Fri, 21 Mar 2025 15:49:27 +0100 Subject: [PATCH 02/20] fix: move findAndPopulateNlpValuesWithCount to the repository --- .../nlp/repositories/nlp-value.repository.ts | 117 +++++++++++++++++- api/src/nlp/services/nlp-value.service.ts | 116 +---------------- 2 files changed, 121 insertions(+), 112 deletions(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index a476c79e..99cfdd11 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -8,9 +8,10 @@ import { Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; -import { Document, Model, Query } from 'mongoose'; +import { Document, Model, Query, Types } from 'mongoose'; import { BaseRepository, DeleteResult } from '@/utils/generics/base-repository'; +import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; import { TFilterQuery } from '@/utils/types/filter.types'; import { NlpValueDto } from '../dto/nlp-value.dto'; @@ -106,4 +107,118 @@ export class NlpValueRepository extends BaseRepository< throw new Error('Attempted to delete a NLP value using unknown criteria'); } } + + async findAndPopulateNlpValuesWithCount( + populate: string[], + filters?: TFilterQuery, + pageQuery?: PageQueryDto, + ) { + const { $and = [], ...rest } = filters || ({} as TFilterQuery); + + return this.model + .aggregate([ + { + // support filters + $match: { + ...rest, + ...($and?.length && { + $and: + $and?.map((v) => { + if (v.entity) { + return { + ...v, + entity: new Types.ObjectId(String(v.entity)), + }; + } + + return v; + }) || [], + }), + }, + }, + // support pageQuery + { + $skip: pageQuery?.skip || 0, + }, + { + $limit: pageQuery?.limit || 10, + }, + { + $sort: { + [pageQuery?.sort?.[0] || 'createdAt']: + pageQuery?.sort?.[1] === 'desc' ? -1 : 1, + }, + }, + { + $lookup: { + from: 'nlpsampleentities', + localField: '_id', + foreignField: 'value', + as: 'sampleEntities', + }, + }, + { + $unwind: { + path: '$sampleEntities', + preserveNullAndEmptyArrays: true, + }, + }, + { + $lookup: { + from: 'nlpsamples', + localField: 'sampleEntities.sample', + foreignField: '_id', + as: 'samples', + }, + }, + { + $lookup: { + from: 'nlpentities', + localField: 'entity', + foreignField: '_id', + as: 'entities', + }, + }, + { + $group: { + _id: '$_id', + value: { $first: '$value' }, + expressions: { $first: '$expressions' }, + builtin: { $first: '$builtin' }, + metadata: { $first: '$metadata' }, + createdAt: { $first: '$createdAt' }, + updatedAt: { $first: '$updatedAt' }, + entity: { + // support populate + $first: populate.some((p) => + this.getPopulate() + .map((p) => p.toString()) + .includes(p), + ) + ? '$entities' + : '$entity', + }, + //TODO when samples is empty array we need to return 0 not 1 + nlpSamplesCount: { + $sum: { $cond: [{ $ifNull: ['samples', false] }, 1, 0] }, + }, + }, + }, + { + $project: { + id: '$_id', + _id: 0, + value: 1, + expressions: 1, + builtin: 1, + entity: 1, + metadata: 1, + createdAt: 1, + updatedAt: 1, + nlpSamplesCount: 1, + }, + }, + ]) + .exec(); + } } diff --git a/api/src/nlp/services/nlp-value.service.ts b/api/src/nlp/services/nlp-value.service.ts index c9ec6309..f4147333 100644 --- a/api/src/nlp/services/nlp-value.service.ts +++ b/api/src/nlp/services/nlp-value.service.ts @@ -7,7 +7,6 @@ */ import { forwardRef, Inject, Injectable } from '@nestjs/common'; -import { Types } from 'mongoose'; import { DeleteResult } from '@/utils/generics/base-repository'; import { BaseService } from '@/utils/generics/base-service'; @@ -229,115 +228,10 @@ export class NlpValueService extends BaseService< filters?: TFilterQuery, pageQuery?: PageQueryDto, ) { - const { $and = [], ...rest } = filters || ({} as TFilterQuery); - - const entityValueModel = this.getRepository().model; - - return entityValueModel - .aggregate([ - { - // support filters - $match: { - ...rest, - ...($and?.length && { - $and: - $and?.map((v) => { - if (v.entity) { - return { - ...v, - entity: new Types.ObjectId(String(v.entity)), - }; - } - - return v; - }) || [], - }), - }, - }, - // support pageQuery - { - $skip: pageQuery?.skip || 0, - }, - { - $limit: pageQuery?.limit || 10, - }, - { - $sort: { - [pageQuery?.sort?.[0] || 'createdAt']: - pageQuery?.sort?.[1] === 'desc' ? -1 : 1, - }, - }, - { - $lookup: { - from: 'nlpsampleentities', - localField: '_id', - foreignField: 'value', - as: 'sampleEntities', - }, - }, - { - $unwind: { - path: '$sampleEntities', - preserveNullAndEmptyArrays: true, - }, - }, - { - $lookup: { - from: 'nlpsamples', - localField: 'sampleEntities.sample', - foreignField: '_id', - as: 'samples', - }, - }, - { - $lookup: { - from: 'nlpentities', - localField: 'entity', - foreignField: '_id', - as: 'entities', - }, - }, - { - $group: { - _id: '$_id', - value: { $first: '$value' }, - expressions: { $first: '$expressions' }, - builtin: { $first: '$builtin' }, - metadata: { $first: '$metadata' }, - createdAt: { $first: '$createdAt' }, - updatedAt: { $first: '$updatedAt' }, - entity: { - // support populate - $first: populate.some((p) => - this.getRepository() - .getPopulate() - .map((p) => p.toString()) - .includes(p), - ) - ? '$entities' - : '$entity', - }, - //TODO when samples is empty array we need to return 0 not 1 - nlpSamplesCount: { - $sum: { $cond: [{ $ifNull: ['samples', false] }, 1, 0] }, - }, - }, - }, - { - $project: { - id: '$_id', - _id: 0, - value: 1, - expressions: 1, - builtin: 1, - entity: 1, - metadata: 1, - createdAt: 1, - updatedAt: 1, - nlpSamplesCount: 1, - }, - }, - ]) - .exec(); + return await this.repository.findAndPopulateNlpValuesWithCount( + populate, + filters, + pageQuery, + ); } } From 6a25d47349c251caa447e85c0d1e1ad6ab6574ae Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sun, 23 Mar 2025 13:03:56 +0100 Subject: [PATCH 03/20] fix: enhance logic --- .../nlp/controllers/nlp-value.controller.ts | 4 +- .../nlp/repositories/nlp-value.repository.ts | 60 +++++-------------- api/src/nlp/services/nlp-value.service.ts | 8 +-- 3 files changed, 21 insertions(+), 51 deletions(-) diff --git a/api/src/nlp/controllers/nlp-value.controller.ts b/api/src/nlp/controllers/nlp-value.controller.ts index 42cb334f..f1e09dad 100644 --- a/api/src/nlp/controllers/nlp-value.controller.ts +++ b/api/src/nlp/controllers/nlp-value.controller.ts @@ -132,12 +132,12 @@ export class NlpValueController extends BaseController< @Query( new SearchFilterPipe({ allowedFields: ['entity', 'value'] }), ) - filters?: TFilterQuery, + filters: TFilterQuery, ) { return await this.nlpValueService.findAndPopulateNlpValuesWithCount( + pageQuery, populate, filters, - pageQuery, ); } diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 99cfdd11..5c5af5bd 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -109,44 +109,39 @@ export class NlpValueRepository extends BaseRepository< } async findAndPopulateNlpValuesWithCount( + { limit = 10, skip = 0, sort = ['createdAt', -1] }: PageQueryDto, populate: string[], - filters?: TFilterQuery, - pageQuery?: PageQueryDto, + { $and = [], ...rest }: TFilterQuery, ) { - const { $and = [], ...rest } = filters || ({} as TFilterQuery); - return this.model .aggregate([ { // support filters $match: { ...rest, - ...($and?.length && { + ...($and.length && { $and: - $and?.map((v) => { - if (v.entity) { - return { - ...v, - entity: new Types.ObjectId(String(v.entity)), - }; - } - - return v; - }) || [], + $and.map(({ entity, ...rest }) => + entity + ? { + ...rest, + entity: new Types.ObjectId(String(entity)), + } + : rest, + ) || [], }), }, }, // support pageQuery { - $skip: pageQuery?.skip || 0, + $limit: limit, }, { - $limit: pageQuery?.limit || 10, + $skip: skip, }, { $sort: { - [pageQuery?.sort?.[0] || 'createdAt']: - pageQuery?.sort?.[1] === 'desc' ? -1 : 1, + [sort[0]]: sort[1] === 'desc' ? -1 : 1, }, }, { @@ -163,22 +158,6 @@ export class NlpValueRepository extends BaseRepository< preserveNullAndEmptyArrays: true, }, }, - { - $lookup: { - from: 'nlpsamples', - localField: 'sampleEntities.sample', - foreignField: '_id', - as: 'samples', - }, - }, - { - $lookup: { - from: 'nlpentities', - localField: 'entity', - foreignField: '_id', - as: 'entities', - }, - }, { $group: { _id: '$_id', @@ -190,17 +169,10 @@ export class NlpValueRepository extends BaseRepository< updatedAt: { $first: '$updatedAt' }, entity: { // support populate - $first: populate.some((p) => - this.getPopulate() - .map((p) => p.toString()) - .includes(p), - ) - ? '$entities' - : '$entity', + $first: this.canPopulate(populate) ? '$entities' : '$entity', }, - //TODO when samples is empty array we need to return 0 not 1 nlpSamplesCount: { - $sum: { $cond: [{ $ifNull: ['samples', false] }, 1, 0] }, + $sum: { $cond: [{ $ifNull: ['$sampleEntities', false] }, 1, 0] }, }, }, }, diff --git a/api/src/nlp/services/nlp-value.service.ts b/api/src/nlp/services/nlp-value.service.ts index f4147333..71ccd031 100644 --- a/api/src/nlp/services/nlp-value.service.ts +++ b/api/src/nlp/services/nlp-value.service.ts @@ -14,7 +14,6 @@ import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; import { TFilterQuery } from '@/utils/types/filter.types'; import { NlpValueCreateDto, NlpValueDto } from '../dto/nlp-value.dto'; -import { NlpSampleEntityRepository } from '../repositories/nlp-sample-entity.repository'; import { NlpValueRepository } from '../repositories/nlp-value.repository'; import { NlpEntity } from '../schemas/nlp-entity.schema'; import { @@ -37,7 +36,6 @@ export class NlpValueService extends BaseService< readonly repository: NlpValueRepository, @Inject(forwardRef(() => NlpEntityService)) private readonly nlpEntityService: NlpEntityService, - private readonly nlpSampleEntityRepository: NlpSampleEntityRepository, ) { super(repository); } @@ -224,14 +222,14 @@ export class NlpValueService extends BaseService< } async findAndPopulateNlpValuesWithCount( + pageQuery: PageQueryDto, populate: string[], - filters?: TFilterQuery, - pageQuery?: PageQueryDto, + filters: TFilterQuery, ) { return await this.repository.findAndPopulateNlpValuesWithCount( + pageQuery, populate, filters, - pageQuery, ); } } From 61d8938212cd8e4dae2f3578ae1d67bd066b925d Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sun, 23 Mar 2025 15:05:51 +0100 Subject: [PATCH 04/20] fix: add nlpentities populate --- api/src/nlp/repositories/nlp-value.repository.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 5c5af5bd..8d68a654 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -158,6 +158,14 @@ export class NlpValueRepository extends BaseRepository< preserveNullAndEmptyArrays: true, }, }, + { + $lookup: { + from: 'nlpentities', + localField: 'entity', + foreignField: '_id', + as: 'entities', + }, + }, { $group: { _id: '$_id', From 486def7e7566eb83124b820d320f5575beca892d Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sun, 23 Mar 2025 15:14:13 +0100 Subject: [PATCH 05/20] fix: rename findAndPopulate method --- api/src/nlp/controllers/nlp-value.controller.ts | 4 ++-- api/src/nlp/repositories/nlp-value.repository.ts | 4 ++-- api/src/nlp/services/nlp-value.service.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/src/nlp/controllers/nlp-value.controller.ts b/api/src/nlp/controllers/nlp-value.controller.ts index f1e09dad..6c8c0e42 100644 --- a/api/src/nlp/controllers/nlp-value.controller.ts +++ b/api/src/nlp/controllers/nlp-value.controller.ts @@ -126,7 +126,7 @@ export class NlpValueController extends BaseController< } @Get('') - async findAndPopulateNlpValuesWithCount( + async findAndPopulateWithCount( @Query(PageQueryPipe) pageQuery: PageQueryDto, @Query(PopulatePipe) populate: string[], @Query( @@ -134,7 +134,7 @@ export class NlpValueController extends BaseController< ) filters: TFilterQuery, ) { - return await this.nlpValueService.findAndPopulateNlpValuesWithCount( + return await this.nlpValueService.findAndPopulateWithCount( pageQuery, populate, filters, diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 8d68a654..54869623 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -108,13 +108,13 @@ export class NlpValueRepository extends BaseRepository< } } - async findAndPopulateNlpValuesWithCount( + async findAndPopulateWithCount( { limit = 10, skip = 0, sort = ['createdAt', -1] }: PageQueryDto, populate: string[], { $and = [], ...rest }: TFilterQuery, ) { return this.model - .aggregate([ + .aggregate([ { // support filters $match: { diff --git a/api/src/nlp/services/nlp-value.service.ts b/api/src/nlp/services/nlp-value.service.ts index 71ccd031..6e5ccc35 100644 --- a/api/src/nlp/services/nlp-value.service.ts +++ b/api/src/nlp/services/nlp-value.service.ts @@ -221,12 +221,12 @@ export class NlpValueService extends BaseService< return Promise.all(promises); } - async findAndPopulateNlpValuesWithCount( + async findAndPopulateWithCount( pageQuery: PageQueryDto, populate: string[], filters: TFilterQuery, ) { - return await this.repository.findAndPopulateNlpValuesWithCount( + return await this.repository.findAndPopulateWithCount( pageQuery, populate, filters, From 1eb09ab84ee413bb24442ee1e451c8cbad1094e2 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Wed, 26 Mar 2025 08:55:48 +0100 Subject: [PATCH 06/20] fix: update nlp value aggregate --- .../controllers/nlp-value.controller.spec.ts | 67 +---- .../nlp/controllers/nlp-value.controller.ts | 28 +-- .../nlp/repositories/nlp-value.repository.ts | 229 ++++++++++++------ api/src/nlp/schemas/nlp-value.schema.ts | 16 ++ api/src/nlp/services/nlp-value.service.ts | 18 +- 5 files changed, 185 insertions(+), 173 deletions(-) diff --git a/api/src/nlp/controllers/nlp-value.controller.spec.ts b/api/src/nlp/controllers/nlp-value.controller.spec.ts index 14277994..deedf872 100644 --- a/api/src/nlp/controllers/nlp-value.controller.spec.ts +++ b/api/src/nlp/controllers/nlp-value.controller.spec.ts @@ -10,18 +10,14 @@ import { BadRequestException, NotFoundException } from '@nestjs/common'; import { MongooseModule } from '@nestjs/mongoose'; import { getUpdateOneError } from '@/utils/test/errors/messages'; -import { nlpEntityFixtures } from '@/utils/test/fixtures/nlpentity'; import { installNlpValueFixtures, nlpValueFixtures, } from '@/utils/test/fixtures/nlpvalue'; -import { getPageQuery } from '@/utils/test/pagination'; import { closeInMongodConnection, rootMongooseTestModule, } from '@/utils/test/test'; -import { TFixtures } from '@/utils/test/types'; -import { buildTestingMocks } from '@/utils/test/utils'; import { NlpValueCreateDto } from '../dto/nlp-value.dto'; import { NlpEntityRepository } from '../repositories/nlp-entity.repository'; @@ -29,11 +25,7 @@ import { NlpSampleEntityRepository } from '../repositories/nlp-sample-entity.rep import { NlpValueRepository } from '../repositories/nlp-value.repository'; import { NlpEntityModel } from '../schemas/nlp-entity.schema'; import { NlpSampleEntityModel } from '../schemas/nlp-sample-entity.schema'; -import { - NlpValue, - NlpValueFull, - NlpValueModel, -} from '../schemas/nlp-value.schema'; +import { NlpValue, NlpValueModel } from '../schemas/nlp-value.schema'; import { NlpEntityService } from '../services/nlp-entity.service'; import { NlpValueService } from '../services/nlp-value.service'; @@ -80,63 +72,6 @@ describe('NlpValueController', () => { afterEach(jest.clearAllMocks); - describe('findPage', () => { - it('should find nlp Values, and foreach nlp value populate the corresponding entity', async () => { - const pageQuery = getPageQuery({ - sort: ['value', 'desc'], - }); - const result = await nlpValueController.findPage( - pageQuery, - ['entity'], - {}, - ); - - const nlpValueFixturesWithEntities = nlpValueFixtures.reduce( - (acc, curr) => { - acc.push({ - ...curr, - entity: nlpEntityFixtures[ - parseInt(curr.entity!) - ] as NlpValueFull['entity'], - builtin: curr.builtin!, - expressions: curr.expressions!, - metadata: curr.metadata!, - }); - return acc; - }, - [] as TFixtures[], - ); - expect(result).toEqualPayload(nlpValueFixturesWithEntities); - }); - - it('should find nlp Values', async () => { - const pageQuery = getPageQuery({ - sort: ['value', 'desc'], - }); - const result = await nlpValueController.findPage( - pageQuery, - ['invalidCriteria'], - {}, - ); - const nlpEntities = await nlpEntityService.findAll(); - const nlpValueFixturesWithEntities = nlpValueFixtures.reduce( - (acc, curr) => { - const ValueWithEntities = { - ...curr, - entity: curr.entity ? nlpEntities[parseInt(curr.entity!)].id : null, - expressions: curr.expressions!, - metadata: curr.metadata!, - builtin: curr.builtin!, - }; - acc.push(ValueWithEntities); - return acc; - }, - [] as TFixtures[], - ); - expect(result).toEqualPayload(nlpValueFixturesWithEntities); - }); - }); - describe('count', () => { it('should count the nlp Values', async () => { const result = await nlpValueController.filterCount(); diff --git a/api/src/nlp/controllers/nlp-value.controller.ts b/api/src/nlp/controllers/nlp-value.controller.ts index 6c8c0e42..c7845f43 100644 --- a/api/src/nlp/controllers/nlp-value.controller.ts +++ b/api/src/nlp/controllers/nlp-value.controller.ts @@ -125,24 +125,8 @@ export class NlpValueController extends BaseController< return doc; } - @Get('') - async findAndPopulateWithCount( - @Query(PageQueryPipe) pageQuery: PageQueryDto, - @Query(PopulatePipe) populate: string[], - @Query( - new SearchFilterPipe({ allowedFields: ['entity', 'value'] }), - ) - filters: TFilterQuery, - ) { - return await this.nlpValueService.findAndPopulateWithCount( - pageQuery, - populate, - filters, - ); - } - /** - * Retrieves a paginated list of NLP values. + * Retrieves a paginated list of NLP values with NLP Samples count. * * Supports filtering, pagination, and optional population of related entities. * @@ -150,10 +134,10 @@ export class NlpValueController extends BaseController< * @param populate - An array of related entities to populate. * @param filters - Filters to apply when retrieving the NLP values. * - * @returns A promise resolving to a paginated list of NLP values. + * @returns A promise resolving to a paginated list of NLP values with NLP Samples count. */ - // @Get('') disabled - async findPage( + @Get() + async findWithCount( @Query(PageQueryPipe) pageQuery: PageQueryDto, @Query(PopulatePipe) populate: string[], @Query( @@ -164,8 +148,8 @@ export class NlpValueController extends BaseController< filters: TFilterQuery, ) { return this.canPopulate(populate) - ? await this.nlpValueService.findAndPopulate(filters, pageQuery) - : await this.nlpValueService.find(filters, pageQuery); + ? await this.nlpValueService.findAndPopulateWithCount(pageQuery, filters) + : await this.nlpValueService.findWithCount(pageQuery, filters); } /** diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 54869623..b6a5eec8 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -8,21 +8,27 @@ import { Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; -import { Document, Model, Query, Types } from 'mongoose'; +import { plainToClass } from 'class-transformer'; +import { Document, Model, PipelineStage, Query, Types } from 'mongoose'; import { BaseRepository, DeleteResult } from '@/utils/generics/base-repository'; import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; import { TFilterQuery } from '@/utils/types/filter.types'; import { NlpValueDto } from '../dto/nlp-value.dto'; +import { NlpEntity } from '../schemas/nlp-entity.schema'; import { NLP_VALUE_POPULATE, NlpValue, NlpValueDocument, NlpValueFull, + NlpValueFullWithCount, NlpValuePopulate, + NlpValueWithCount, + TNlpValueCountFormat, } from '../schemas/nlp-value.schema'; +import { NlpEntityRepository } from './nlp-entity.repository'; import { NlpSampleEntityRepository } from './nlp-sample-entity.repository'; @Injectable() @@ -35,6 +41,8 @@ export class NlpValueRepository extends BaseRepository< constructor( @InjectModel(NlpValue.name) readonly model: Model, private readonly nlpSampleEntityRepository: NlpSampleEntityRepository, + @Inject(forwardRef(() => NlpEntityRepository)) + private readonly nlpEntityRepository: NlpEntityRepository, ) { super(model, NlpValue, NLP_VALUE_POPULATE, NlpValueFull); } @@ -108,97 +116,162 @@ export class NlpValueRepository extends BaseRepository< } } - async findAndPopulateWithCount( + private async aggregateWithCount( { limit = 10, skip = 0, sort = ['createdAt', -1] }: PageQueryDto, - populate: string[], { $and = [], ...rest }: TFilterQuery, + populatePipelineStages: PipelineStage[] = [], ) { - return this.model - .aggregate([ - { - // support filters - $match: { - ...rest, - ...($and.length && { - $and: - $and.map(({ entity, ...rest }) => - entity - ? { - ...rest, - entity: new Types.ObjectId(String(entity)), - } - : rest, - ) || [], - }), + const pipeline: PipelineStage[] = [ + // support pageQuery + { + $limit: limit, + }, + { + $skip: skip, + }, + { + $sort: { + [sort[0]]: sort[1] === 'desc' ? -1 : 1, + _id: sort[1] === 'desc' ? -1 : 1, + }, + }, + { + // support filters + $match: { + ...rest, + ...($and.length && { + $and: + $and.map(({ entity, ...rest }) => + entity + ? { + ...rest, + entity: new Types.ObjectId(String(entity)), + } + : rest, + ) || [], + }), + }, + }, + { + $lookup: { + from: 'nlpsampleentities', + localField: '_id', + foreignField: 'value', + as: 'sampleEntities', + }, + }, + { + $unwind: { + path: '$sampleEntities', + preserveNullAndEmptyArrays: true, + }, + }, + { + $group: { + _id: '$_id', + value: { $first: '$value' }, + expressions: { $first: '$expressions' }, + builtin: { $first: '$builtin' }, + metadata: { $first: '$metadata' }, + createdAt: { $first: '$createdAt' }, + updatedAt: { $first: '$updatedAt' }, + entity: { $first: '$entity' }, + nlpSamplesCount: { + $sum: { $cond: [{ $ifNull: ['$sampleEntities', false] }, 1, 0] }, }, }, - // support pageQuery - { - $limit: limit, - }, - { - $skip: skip, - }, - { - $sort: { - [sort[0]]: sort[1] === 'desc' ? -1 : 1, - }, - }, - { - $lookup: { - from: 'nlpsampleentities', - localField: '_id', - foreignField: 'value', - as: 'sampleEntities', - }, - }, - { - $unwind: { - path: '$sampleEntities', - preserveNullAndEmptyArrays: true, - }, + }, + { + $project: { + id: '$_id', + _id: 0, + value: 1, + expressions: 1, + builtin: 1, + entity: 1, + metadata: 1, + createdAt: 1, + updatedAt: 1, + nlpSamplesCount: 1, }, + }, + ...populatePipelineStages, + ]; + + return await this.model.aggregate>(pipeline).exec(); + } + + private async plainToClass( + format: 'full' | 'stub', + aggregatedResults: (NlpValueWithCount | NlpValueFullWithCount)[], + ): Promise[]> { + if (format === 'full') { + const nestedNlpEntities: NlpValueFullWithCount[] = []; + for (const { entity, ...rest } of aggregatedResults) { + const plainNlpValue = { + ...rest, + entity: plainToClass( + NlpEntity, + await this.nlpEntityRepository.findOne(entity), + { + excludePrefixes: ['_'], + }, + ), + }; + nestedNlpEntities.push( + plainToClass(NlpValueFullWithCount, plainNlpValue, { + excludePrefixes: ['_'], + }), + ); + } + return nestedNlpEntities as TNlpValueCountFormat[]; + } else { + const nestedNlpEntities: NlpValueWithCount[] = []; + for (const aggregatedResult of aggregatedResults) { + nestedNlpEntities.push( + plainToClass(NlpValueWithCount, aggregatedResult, { + excludePrefixes: ['_'], + }), + ); + } + return nestedNlpEntities as TNlpValueCountFormat[]; + } + } + + async findWithCount( + pageQuery: PageQueryDto, + filterQuery: TFilterQuery, + ): Promise { + const aggregatedResults = await this.aggregateWithCount<'stub'>( + pageQuery, + filterQuery, + ); + + return await this.plainToClass<'stub'>('stub', aggregatedResults); + } + + async findAndPopulateWithCount( + pageQuery: PageQueryDto, + filterQuery: TFilterQuery, + ): Promise { + const aggregatedResults = await this.aggregateWithCount<'full'>( + pageQuery, + filterQuery, + [ { $lookup: { from: 'nlpentities', localField: 'entity', foreignField: '_id', - as: 'entities', + as: 'entity', }, }, { - $group: { - _id: '$_id', - value: { $first: '$value' }, - expressions: { $first: '$expressions' }, - builtin: { $first: '$builtin' }, - metadata: { $first: '$metadata' }, - createdAt: { $first: '$createdAt' }, - updatedAt: { $first: '$updatedAt' }, - entity: { - // support populate - $first: this.canPopulate(populate) ? '$entities' : '$entity', - }, - nlpSamplesCount: { - $sum: { $cond: [{ $ifNull: ['$sampleEntities', false] }, 1, 0] }, - }, - }, + $unwind: '$entity', }, - { - $project: { - id: '$_id', - _id: 0, - value: 1, - expressions: 1, - builtin: 1, - entity: 1, - metadata: 1, - createdAt: 1, - updatedAt: 1, - nlpSamplesCount: 1, - }, - }, - ]) - .exec(); + ], + ); + + return await this.plainToClass<'full'>('full', aggregatedResults); } } diff --git a/api/src/nlp/schemas/nlp-value.schema.ts b/api/src/nlp/schemas/nlp-value.schema.ts index 523eaa3e..77edcae1 100644 --- a/api/src/nlp/schemas/nlp-value.schema.ts +++ b/api/src/nlp/schemas/nlp-value.schema.ts @@ -106,6 +106,18 @@ export class NlpValueFull extends NlpValueStub { entity: NlpEntity; } +export class NlpValueWithCount extends NlpValue { + nlpSamplesCount: number; +} + +export class NlpValueFullWithCount extends NlpValueFull { + nlpSamplesCount: number; +} + +export class NlpValueFullWithCountDto { + nlpSamplesCount: number; +} + export type NlpValueDocument = THydratedDocument; export const NlpValueModel: ModelDefinition = LifecycleHookManager.attach({ @@ -121,3 +133,7 @@ export type NlpValuePopulate = keyof TFilterPopulateFields< >; export const NLP_VALUE_POPULATE: NlpValuePopulate[] = ['entity']; + +export type TNlpValueCountFormat = T extends 'stub' + ? NlpValueWithCount + : NlpValueFullWithCount; diff --git a/api/src/nlp/services/nlp-value.service.ts b/api/src/nlp/services/nlp-value.service.ts index 6e5ccc35..23448440 100644 --- a/api/src/nlp/services/nlp-value.service.ts +++ b/api/src/nlp/services/nlp-value.service.ts @@ -19,7 +19,9 @@ import { NlpEntity } from '../schemas/nlp-entity.schema'; import { NlpValue, NlpValueFull, + NlpValueFullWithCount, NlpValuePopulate, + NlpValueWithCount, } from '../schemas/nlp-value.schema'; import { NlpSampleEntityValue } from '../schemas/types'; @@ -221,15 +223,17 @@ export class NlpValueService extends BaseService< return Promise.all(promises); } + async findWithCount( + pageQuery: PageQueryDto, + filters: TFilterQuery, + ): Promise { + return await this.repository.findWithCount(pageQuery, filters); + } + async findAndPopulateWithCount( pageQuery: PageQueryDto, - populate: string[], filters: TFilterQuery, - ) { - return await this.repository.findAndPopulateWithCount( - pageQuery, - populate, - filters, - ); + ): Promise { + return await this.repository.findAndPopulateWithCount(pageQuery, filters); } } From 1cc99d00206dd08c12303fe28fe181cb98d2f26e Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Thu, 3 Apr 2025 08:17:43 +0100 Subject: [PATCH 07/20] fix: apply feedback updates --- .../nlp/repositories/nlp-value.repository.ts | 32 +++++++++++-------- .../components/nlp/components/NlpValue.tsx | 5 ++- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index b6a5eec8..73236796 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -117,24 +117,15 @@ export class NlpValueRepository extends BaseRepository< } private async aggregateWithCount( - { limit = 10, skip = 0, sort = ['createdAt', -1] }: PageQueryDto, + { + limit = 10, + skip = 0, + sort = ['createdAt', 'desc'], + }: PageQueryDto, { $and = [], ...rest }: TFilterQuery, populatePipelineStages: PipelineStage[] = [], ) { const pipeline: PipelineStage[] = [ - // support pageQuery - { - $limit: limit, - }, - { - $skip: skip, - }, - { - $sort: { - [sort[0]]: sort[1] === 'desc' ? -1 : 1, - _id: sort[1] === 'desc' ? -1 : 1, - }, - }, { // support filters $match: { @@ -152,6 +143,13 @@ export class NlpValueRepository extends BaseRepository< }), }, }, + // support pageQuery + { + $skip: skip, + }, + { + $limit: limit, + }, { $lookup: { from: 'nlpsampleentities', @@ -196,6 +194,12 @@ export class NlpValueRepository extends BaseRepository< }, }, ...populatePipelineStages, + { + $sort: { + [sort[0]]: sort[1].toString().startsWith('desc') ? -1 : 1, + _id: sort[1].toString().startsWith('desc') ? -1 : 1, + }, + }, ]; return await this.model.aggregate>(pipeline).exec(); diff --git a/frontend/src/components/nlp/components/NlpValue.tsx b/frontend/src/components/nlp/components/NlpValue.tsx index 92cc7d04..c0a6b18d 100644 --- a/frontend/src/components/nlp/components/NlpValue.tsx +++ b/frontend/src/components/nlp/components/NlpValue.tsx @@ -126,14 +126,17 @@ export const NlpValues = ({ entityId }: { entityId: string }) => { renderHeader, }, { - flex: 3, + flex: 2, field: "nlpSamplesCount", + align: "center", headerName: t("label.nlp_samples_count"), sortable: true, disableColumnMenu: true, + headerAlign: "center", renderHeader, renderCell: ({ row }) => ( Date: Thu, 3 Apr 2025 08:45:09 +0100 Subject: [PATCH 08/20] fix: update NlpValue to have full format --- frontend/src/components/nlp/components/NlpValue.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/nlp/components/NlpValue.tsx b/frontend/src/components/nlp/components/NlpValue.tsx index c0a6b18d..3c49543b 100644 --- a/frontend/src/components/nlp/components/NlpValue.tsx +++ b/frontend/src/components/nlp/components/NlpValue.tsx @@ -58,7 +58,7 @@ export const NlpValues = ({ entityId }: { entityId: string }) => { $or: ["doc", "value"], }); const { dataGridProps } = useFind( - { entity: EntityType.NLP_VALUE }, + { entity: EntityType.NLP_VALUE, format: Format.FULL }, { params: searchPayload, }, From ebd2a66bbd4b4883f4e68d6b1b44de5c2e2f4852 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Thu, 3 Apr 2025 08:55:54 +0100 Subject: [PATCH 09/20] fix: apply feedbacks --- frontend/src/components/nlp/components/NlpValue.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/nlp/components/NlpValue.tsx b/frontend/src/components/nlp/components/NlpValue.tsx index 3c49543b..eb6b6be9 100644 --- a/frontend/src/components/nlp/components/NlpValue.tsx +++ b/frontend/src/components/nlp/components/NlpValue.tsx @@ -137,8 +137,8 @@ export const NlpValues = ({ entityId }: { entityId: string }) => { renderCell: ({ row }) => ( ), From a38824a23a10bac9ede5530436e322e5eaff339d Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Fri, 4 Apr 2025 11:58:54 +0100 Subject: [PATCH 10/20] fix: resolve Circular dependency in unit tests --- .../controllers/nlp-value.controller.spec.ts | 1 + .../nlp/repositories/nlp-value.repository.ts | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/api/src/nlp/controllers/nlp-value.controller.spec.ts b/api/src/nlp/controllers/nlp-value.controller.spec.ts index deedf872..a72b8571 100644 --- a/api/src/nlp/controllers/nlp-value.controller.spec.ts +++ b/api/src/nlp/controllers/nlp-value.controller.spec.ts @@ -18,6 +18,7 @@ import { closeInMongodConnection, rootMongooseTestModule, } from '@/utils/test/test'; +import { buildTestingMocks } from '@/utils/test/utils'; import { NlpValueCreateDto } from '../dto/nlp-value.dto'; import { NlpEntityRepository } from '../repositories/nlp-entity.repository'; diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 73236796..03f22af4 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -9,14 +9,20 @@ import { Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; import { plainToClass } from 'class-transformer'; -import { Document, Model, PipelineStage, Query, Types } from 'mongoose'; +import mongoose, { + Document, + Model, + PipelineStage, + Query, + Types, +} from 'mongoose'; import { BaseRepository, DeleteResult } from '@/utils/generics/base-repository'; import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; import { TFilterQuery } from '@/utils/types/filter.types'; import { NlpValueDto } from '../dto/nlp-value.dto'; -import { NlpEntity } from '../schemas/nlp-entity.schema'; +import { NlpEntity, NlpEntityModel } from '../schemas/nlp-entity.schema'; import { NLP_VALUE_POPULATE, NlpValue, @@ -28,7 +34,6 @@ import { TNlpValueCountFormat, } from '../schemas/nlp-value.schema'; -import { NlpEntityRepository } from './nlp-entity.repository'; import { NlpSampleEntityRepository } from './nlp-sample-entity.repository'; @Injectable() @@ -41,8 +46,6 @@ export class NlpValueRepository extends BaseRepository< constructor( @InjectModel(NlpValue.name) readonly model: Model, private readonly nlpSampleEntityRepository: NlpSampleEntityRepository, - @Inject(forwardRef(() => NlpEntityRepository)) - private readonly nlpEntityRepository: NlpEntityRepository, ) { super(model, NlpValue, NLP_VALUE_POPULATE, NlpValueFull); } @@ -216,7 +219,9 @@ export class NlpValueRepository extends BaseRepository< ...rest, entity: plainToClass( NlpEntity, - await this.nlpEntityRepository.findOne(entity), + await mongoose + .model(NlpEntityModel.name, NlpEntityModel.schema) + .findById(entity), { excludePrefixes: ['_'], }, From 18f26df7bb019c2fcced4ccfd411270fac5b43aa Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Fri, 4 Apr 2025 12:04:40 +0100 Subject: [PATCH 11/20] fix: remove N/A defaultvalue --- frontend/src/components/nlp/components/NlpValue.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/nlp/components/NlpValue.tsx b/frontend/src/components/nlp/components/NlpValue.tsx index eb6b6be9..c6094204 100644 --- a/frontend/src/components/nlp/components/NlpValue.tsx +++ b/frontend/src/components/nlp/components/NlpValue.tsx @@ -138,7 +138,7 @@ export const NlpValues = ({ entityId }: { entityId: string }) => { ), From 524f30576b16aefcac13b7815e65556245e340e0 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Fri, 4 Apr 2025 12:07:02 +0100 Subject: [PATCH 12/20] fix: update Nlp Sample en translation --- frontend/public/locales/en/translation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/public/locales/en/translation.json b/frontend/public/locales/en/translation.json index b243c3ae..94124e91 100644 --- a/frontend/public/locales/en/translation.json +++ b/frontend/public/locales/en/translation.json @@ -334,7 +334,7 @@ "nlp": "NLU", "nlp_entity": "Entity", "nlp_entity_value": "Value", - "nlp_samples_count": "Nlp Samples count", + "nlp_samples_count": "Samples count", "value": "Value", "synonyms": "Synonyms", "lookups": "Lookups", From a3b92da470bce5a69bd0ee43aa7eeebc7ae64980 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sat, 5 Apr 2025 20:34:12 +0100 Subject: [PATCH 13/20] fix: update NlpEntity reponse to be plain --- api/src/nlp/repositories/nlp-value.repository.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 03f22af4..836990ce 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -221,7 +221,8 @@ export class NlpValueRepository extends BaseRepository< NlpEntity, await mongoose .model(NlpEntityModel.name, NlpEntityModel.schema) - .findById(entity), + .findById(entity) + .lean(), { excludePrefixes: ['_'], }, From e592a781b38ae66259ae236c02c4c9211aae5775 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sat, 5 Apr 2025 21:59:19 +0100 Subject: [PATCH 14/20] fix: refactor logic --- .../nlp/controllers/nlp-value.controller.ts | 9 +- .../nlp/repositories/nlp-value.repository.ts | 182 ++++++++++-------- api/src/nlp/schemas/nlp-value.schema.ts | 13 +- api/src/nlp/services/nlp-value.service.ts | 18 +- api/src/utils/types/format.types.ts | 18 ++ 5 files changed, 133 insertions(+), 107 deletions(-) create mode 100644 api/src/utils/types/format.types.ts diff --git a/api/src/nlp/controllers/nlp-value.controller.ts b/api/src/nlp/controllers/nlp-value.controller.ts index c7845f43..c9d98078 100644 --- a/api/src/nlp/controllers/nlp-value.controller.ts +++ b/api/src/nlp/controllers/nlp-value.controller.ts @@ -30,6 +30,7 @@ import { PageQueryPipe } from '@/utils/pagination/pagination-query.pipe'; import { PopulatePipe } from '@/utils/pipes/populate.pipe'; import { SearchFilterPipe } from '@/utils/pipes/search-filter.pipe'; import { TFilterQuery } from '@/utils/types/filter.types'; +import { Format } from '@/utils/types/format.types'; import { NlpValueCreateDto, NlpValueUpdateDto } from '../dto/nlp-value.dto'; import { @@ -147,9 +148,11 @@ export class NlpValueController extends BaseController< ) filters: TFilterQuery, ) { - return this.canPopulate(populate) - ? await this.nlpValueService.findAndPopulateWithCount(pageQuery, filters) - : await this.nlpValueService.findWithCount(pageQuery, filters); + return await this.nlpValueService.findWithCount( + this.canPopulate(populate) ? Format.FULL : Format.STUB, + pageQuery, + filters, + ); } /** diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 836990ce..5ae353b3 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -20,6 +20,7 @@ import mongoose, { import { BaseRepository, DeleteResult } from '@/utils/generics/base-repository'; import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; import { TFilterQuery } from '@/utils/types/filter.types'; +import { Format } from '@/utils/types/format.types'; import { NlpValueDto } from '../dto/nlp-value.dto'; import { NlpEntity, NlpEntityModel } from '../schemas/nlp-entity.schema'; @@ -31,7 +32,7 @@ import { NlpValueFullWithCount, NlpValuePopulate, NlpValueWithCount, - TNlpValueCountFormat, + TNlpValueCount, } from '../schemas/nlp-value.schema'; import { NlpSampleEntityRepository } from './nlp-sample-entity.repository'; @@ -119,31 +120,38 @@ export class NlpValueRepository extends BaseRepository< } } - private async aggregateWithCount( + /** + * Performs an aggregation to retrieve NLP values with their sample counts. + * + * @param pageQuery - The pagination parameters + * @param filterQuery - The filter criteria + * @param populatePipelineStages - Optional additional pipeline stages for populating related data + * @returns Aggregated results with sample counts + */ + private async aggregateWithCount( + format: F, { limit = 10, skip = 0, sort = ['createdAt', 'desc'], }: PageQueryDto, { $and = [], ...rest }: TFilterQuery, - populatePipelineStages: PipelineStage[] = [], - ) { + ): Promise[]> { const pipeline: PipelineStage[] = [ { // support filters $match: { ...rest, - ...($and.length && { - $and: - $and.map(({ entity, ...rest }) => - entity - ? { - ...rest, - entity: new Types.ObjectId(String(entity)), - } - : rest, - ) || [], - }), + ...($and.length + ? { + $and: $and.map(({ entity, ...rest }) => ({ + ...rest, + ...(entity + ? { entity: new Types.ObjectId(String(entity)) } + : {}), + })), + } + : {}), }, }, // support pageQuery @@ -196,92 +204,96 @@ export class NlpValueRepository extends BaseRepository< nlpSamplesCount: 1, }, }, - ...populatePipelineStages, + ...(format === Format.FULL + ? [ + { + $lookup: { + from: 'nlpentities', + localField: 'entity', + foreignField: '_id', + as: 'entity', + }, + }, + { + $unwind: '$entity', + }, + ] + : []), { $sort: { - [sort[0]]: sort[1].toString().startsWith('desc') ? -1 : 1, - _id: sort[1].toString().startsWith('desc') ? -1 : 1, + [sort[0]]: + typeof sort[1] === 'number' + ? sort[1] + : sort[1].toString().toLowerCase() === 'desc' + ? -1 + : 1, + _id: + typeof sort[1] === 'number' + ? sort[1] + : sort[1].toString().toLowerCase() === 'desc' + ? -1 + : 1, }, }, ]; - return await this.model.aggregate>(pipeline).exec(); + return await this.model.aggregate>(pipeline).exec(); } - private async plainToClass( - format: 'full' | 'stub', - aggregatedResults: (NlpValueWithCount | NlpValueFullWithCount)[], - ): Promise[]> { - if (format === 'full') { - const nestedNlpEntities: NlpValueFullWithCount[] = []; - for (const { entity, ...rest } of aggregatedResults) { - const plainNlpValue = { + private async plainToClass( + format: F, + aggregatedResults: TNlpValueCount[], + ): Promise[]> { + const result: typeof aggregatedResults = []; + + for (const item of aggregatedResults as TNlpValueCount[]) { + if (format === Format.FULL) { + const { entity, ...rest } = item; + const entityData = await mongoose + .model(NlpEntityModel.name, NlpEntityModel.schema) + .findById(entity) + .lean(); + + const plainNlpValue: NlpValueFull = { ...rest, - entity: plainToClass( - NlpEntity, - await mongoose - .model(NlpEntityModel.name, NlpEntityModel.schema) - .findById(entity) - .lean(), - { - excludePrefixes: ['_'], - }, - ), + entity: plainToClass(NlpEntity, entityData, { + excludePrefixes: ['_'], + }), }; - nestedNlpEntities.push( + + result.push( plainToClass(NlpValueFullWithCount, plainNlpValue, { excludePrefixes: ['_'], - }), + }) as TNlpValueCount, ); - } - return nestedNlpEntities as TNlpValueCountFormat[]; - } else { - const nestedNlpEntities: NlpValueWithCount[] = []; - for (const aggregatedResult of aggregatedResults) { - nestedNlpEntities.push( - plainToClass(NlpValueWithCount, aggregatedResult, { + } else { + result.push( + plainToClass(NlpValueWithCount, item, { excludePrefixes: ['_'], - }), + }) as TNlpValueCount, ); } - return nestedNlpEntities as TNlpValueCountFormat[]; + } + + return result; + } + + async findWithCount( + format: F, + pageQuery: PageQueryDto, + filterQuery: TFilterQuery, + ): Promise[]> { + try { + const aggregatedResults = await this.aggregateWithCount( + format, + pageQuery, + filterQuery, + ); + + return await this.plainToClass(format, aggregatedResults); + } catch (error) { + this.logger.error(`Error in findWithCount: ${error.message}`, error); + throw error; } } - - async findWithCount( - pageQuery: PageQueryDto, - filterQuery: TFilterQuery, - ): Promise { - const aggregatedResults = await this.aggregateWithCount<'stub'>( - pageQuery, - filterQuery, - ); - - return await this.plainToClass<'stub'>('stub', aggregatedResults); - } - - async findAndPopulateWithCount( - pageQuery: PageQueryDto, - filterQuery: TFilterQuery, - ): Promise { - const aggregatedResults = await this.aggregateWithCount<'full'>( - pageQuery, - filterQuery, - [ - { - $lookup: { - from: 'nlpentities', - localField: 'entity', - foreignField: '_id', - as: 'entity', - }, - }, - { - $unwind: '$entity', - }, - ], - ); - - return await this.plainToClass<'full'>('full', aggregatedResults); - } } diff --git a/api/src/nlp/schemas/nlp-value.schema.ts b/api/src/nlp/schemas/nlp-value.schema.ts index 77edcae1..a49eea5c 100644 --- a/api/src/nlp/schemas/nlp-value.schema.ts +++ b/api/src/nlp/schemas/nlp-value.schema.ts @@ -16,6 +16,7 @@ import { TFilterPopulateFields, THydratedDocument, } from '@/utils/types/filter.types'; +import { TStubOrFull } from '@/utils/types/format.types'; import { NlpEntity, NlpEntityFull } from './nlp-entity.schema'; import { NlpValueMap } from './types'; @@ -114,10 +115,6 @@ export class NlpValueFullWithCount extends NlpValueFull { nlpSamplesCount: number; } -export class NlpValueFullWithCountDto { - nlpSamplesCount: number; -} - export type NlpValueDocument = THydratedDocument; export const NlpValueModel: ModelDefinition = LifecycleHookManager.attach({ @@ -134,6 +131,8 @@ export type NlpValuePopulate = keyof TFilterPopulateFields< export const NLP_VALUE_POPULATE: NlpValuePopulate[] = ['entity']; -export type TNlpValueCountFormat = T extends 'stub' - ? NlpValueWithCount - : NlpValueFullWithCount; +export type TNlpValueCount = TStubOrFull< + T, + NlpValueWithCount, + NlpValueFullWithCount +>; diff --git a/api/src/nlp/services/nlp-value.service.ts b/api/src/nlp/services/nlp-value.service.ts index 23448440..87988140 100644 --- a/api/src/nlp/services/nlp-value.service.ts +++ b/api/src/nlp/services/nlp-value.service.ts @@ -12,6 +12,7 @@ import { DeleteResult } from '@/utils/generics/base-repository'; import { BaseService } from '@/utils/generics/base-service'; import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; import { TFilterQuery } from '@/utils/types/filter.types'; +import { Format } from '@/utils/types/format.types'; import { NlpValueCreateDto, NlpValueDto } from '../dto/nlp-value.dto'; import { NlpValueRepository } from '../repositories/nlp-value.repository'; @@ -19,9 +20,8 @@ import { NlpEntity } from '../schemas/nlp-entity.schema'; import { NlpValue, NlpValueFull, - NlpValueFullWithCount, NlpValuePopulate, - NlpValueWithCount, + TNlpValueCount, } from '../schemas/nlp-value.schema'; import { NlpSampleEntityValue } from '../schemas/types'; @@ -223,17 +223,11 @@ export class NlpValueService extends BaseService< return Promise.all(promises); } - async findWithCount( + async findWithCount( + format: F, pageQuery: PageQueryDto, filters: TFilterQuery, - ): Promise { - return await this.repository.findWithCount(pageQuery, filters); - } - - async findAndPopulateWithCount( - pageQuery: PageQueryDto, - filters: TFilterQuery, - ): Promise { - return await this.repository.findAndPopulateWithCount(pageQuery, filters); + ): Promise[]> { + return await this.repository.findWithCount(format, pageQuery, filters); } } diff --git a/api/src/utils/types/format.types.ts b/api/src/utils/types/format.types.ts new file mode 100644 index 00000000..2336a6c1 --- /dev/null +++ b/api/src/utils/types/format.types.ts @@ -0,0 +1,18 @@ +/* + * Copyright © 2025 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +export enum Format { + NONE = 0, + STUB = 1, + BASIC = 2, + FULL = 3, +} + +export type TStubOrFull = TF extends Format.STUB + ? TStub + : TFull; From f18c0618885a54d590dbae3dca9e67bd3b41801e Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sat, 5 Apr 2025 22:25:59 +0100 Subject: [PATCH 15/20] fix: update doc comment of aggregateWithCount method --- api/src/nlp/repositories/nlp-value.repository.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 5ae353b3..05f716b0 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -123,10 +123,10 @@ export class NlpValueRepository extends BaseRepository< /** * Performs an aggregation to retrieve NLP values with their sample counts. * + * @param format - The format can be full or stub * @param pageQuery - The pagination parameters * @param filterQuery - The filter criteria - * @param populatePipelineStages - Optional additional pipeline stages for populating related data - * @returns Aggregated results with sample counts + * @returns Aggregated Nlp Value results with sample counts */ private async aggregateWithCount( format: F, From a729fe2f15fb0c40af1c209c6657280323458b5c Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sat, 5 Apr 2025 22:46:37 +0100 Subject: [PATCH 16/20] fix: update nlpEntity model injection --- .../nlp/repositories/nlp-value.repository.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 05f716b0..18632a3f 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -9,13 +9,7 @@ import { Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; import { plainToClass } from 'class-transformer'; -import mongoose, { - Document, - Model, - PipelineStage, - Query, - Types, -} from 'mongoose'; +import { Document, Model, PipelineStage, Query, Types } from 'mongoose'; import { BaseRepository, DeleteResult } from '@/utils/generics/base-repository'; import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; @@ -23,7 +17,7 @@ import { TFilterQuery } from '@/utils/types/filter.types'; import { Format } from '@/utils/types/format.types'; import { NlpValueDto } from '../dto/nlp-value.dto'; -import { NlpEntity, NlpEntityModel } from '../schemas/nlp-entity.schema'; +import { NlpEntity } from '../schemas/nlp-entity.schema'; import { NLP_VALUE_POPULATE, NlpValue, @@ -47,6 +41,8 @@ export class NlpValueRepository extends BaseRepository< constructor( @InjectModel(NlpValue.name) readonly model: Model, private readonly nlpSampleEntityRepository: NlpSampleEntityRepository, + @InjectModel(NlpEntity.name) + private readonly nlpEntityModel: Model, ) { super(model, NlpValue, NLP_VALUE_POPULATE, NlpValueFull); } @@ -249,14 +245,11 @@ export class NlpValueRepository extends BaseRepository< for (const item of aggregatedResults as TNlpValueCount[]) { if (format === Format.FULL) { const { entity, ...rest } = item; - const entityData = await mongoose - .model(NlpEntityModel.name, NlpEntityModel.schema) - .findById(entity) - .lean(); + const nlpEntityData = await this.nlpEntityModel.findById(entity).lean(); const plainNlpValue: NlpValueFull = { ...rest, - entity: plainToClass(NlpEntity, entityData, { + entity: plainToClass(NlpEntity, nlpEntityData, { excludePrefixes: ['_'], }), }; From bdf92af8883595160b1c0f15b811af325ee4571f Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sat, 5 Apr 2025 23:04:46 +0100 Subject: [PATCH 17/20] fix: add getSortDirection method --- .../nlp/repositories/nlp-value.repository.ts | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 18632a3f..60f6afea 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -9,7 +9,14 @@ import { Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; import { plainToClass } from 'class-transformer'; -import { Document, Model, PipelineStage, Query, Types } from 'mongoose'; +import { + Document, + Model, + PipelineStage, + Query, + SortOrder, + Types, +} from 'mongoose'; import { BaseRepository, DeleteResult } from '@/utils/generics/base-repository'; import { PageQueryDto } from '@/utils/pagination/pagination-query.dto'; @@ -217,18 +224,8 @@ export class NlpValueRepository extends BaseRepository< : []), { $sort: { - [sort[0]]: - typeof sort[1] === 'number' - ? sort[1] - : sort[1].toString().toLowerCase() === 'desc' - ? -1 - : 1, - _id: - typeof sort[1] === 'number' - ? sort[1] - : sort[1].toString().toLowerCase() === 'desc' - ? -1 - : 1, + [sort[0]]: this.getSortDirection(sort[1]), + _id: this.getSortDirection(sort[1]), }, }, ]; @@ -289,4 +286,12 @@ export class NlpValueRepository extends BaseRepository< throw error; } } + + private getSortDirection(sortOrder: SortOrder) { + return typeof sortOrder === 'number' + ? sortOrder + : sortOrder.toString().toLowerCase() === 'desc' + ? -1 + : 1; + } } From a6d9c9dae5b4bda5cd7b820b4ba7071928db0838 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sun, 6 Apr 2025 00:07:56 +0100 Subject: [PATCH 18/20] fix: refactor plainToClass method --- .../nlp/repositories/nlp-value.repository.ts | 46 ++++++++----------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 60f6afea..4ac2efe1 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -8,7 +8,7 @@ import { Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; -import { plainToClass } from 'class-transformer'; +import { ClassTransformOptions, plainToClass } from 'class-transformer'; import { Document, Model, @@ -48,8 +48,6 @@ export class NlpValueRepository extends BaseRepository< constructor( @InjectModel(NlpValue.name) readonly model: Model, private readonly nlpSampleEntityRepository: NlpSampleEntityRepository, - @InjectModel(NlpEntity.name) - private readonly nlpEntityModel: Model, ) { super(model, NlpValue, NLP_VALUE_POPULATE, NlpValueFull); } @@ -236,36 +234,28 @@ export class NlpValueRepository extends BaseRepository< private async plainToClass( format: F, aggregatedResults: TNlpValueCount[], + options: ClassTransformOptions = { excludePrefixes: ['_'] }, ): Promise[]> { - const result: typeof aggregatedResults = []; - - for (const item of aggregatedResults as TNlpValueCount[]) { + return aggregatedResults.map(({ entity, ...rest }) => { if (format === Format.FULL) { - const { entity, ...rest } = item; - const nlpEntityData = await this.nlpEntityModel.findById(entity).lean(); - - const plainNlpValue: NlpValueFull = { - ...rest, - entity: plainToClass(NlpEntity, nlpEntityData, { + return plainToClass( + NlpValueFullWithCount, + { + ...rest, + entity: plainToClass(NlpEntity, entity, options), + }, + { excludePrefixes: ['_'], - }), - }; - - result.push( - plainToClass(NlpValueFullWithCount, plainNlpValue, { - excludePrefixes: ['_'], - }) as TNlpValueCount, - ); + }, + ) as TNlpValueCount; } else { - result.push( - plainToClass(NlpValueWithCount, item, { - excludePrefixes: ['_'], - }) as TNlpValueCount, - ); + return plainToClass( + NlpValueWithCount, + { entity, ...rest }, + options, + ) as TNlpValueCount; } - } - - return result; + }); } async findWithCount( From 903ff0bbf91422892526dec7dedae83ab49fe3bb Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sun, 6 Apr 2025 10:56:43 +0100 Subject: [PATCH 19/20] fix: update findWithCount method logic --- .../nlp/repositories/nlp-value.repository.ts | 56 ++++++------------- 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index 4ac2efe1..ab276904 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -8,7 +8,7 @@ import { Injectable } from '@nestjs/common'; import { InjectModel } from '@nestjs/mongoose'; -import { ClassTransformOptions, plainToClass } from 'class-transformer'; +import { plainToInstance } from 'class-transformer'; import { Document, Model, @@ -24,7 +24,6 @@ import { TFilterQuery } from '@/utils/types/filter.types'; import { Format } from '@/utils/types/format.types'; import { NlpValueDto } from '../dto/nlp-value.dto'; -import { NlpEntity } from '../schemas/nlp-entity.schema'; import { NLP_VALUE_POPULATE, NlpValue, @@ -121,6 +120,14 @@ export class NlpValueRepository extends BaseRepository< } } + private getSortDirection(sortOrder: SortOrder) { + return typeof sortOrder === 'number' + ? sortOrder + : sortOrder.toString().toLowerCase() === 'desc' + ? -1 + : 1; + } + /** * Performs an aggregation to retrieve NLP values with their sample counts. * @@ -231,33 +238,6 @@ export class NlpValueRepository extends BaseRepository< return await this.model.aggregate>(pipeline).exec(); } - private async plainToClass( - format: F, - aggregatedResults: TNlpValueCount[], - options: ClassTransformOptions = { excludePrefixes: ['_'] }, - ): Promise[]> { - return aggregatedResults.map(({ entity, ...rest }) => { - if (format === Format.FULL) { - return plainToClass( - NlpValueFullWithCount, - { - ...rest, - entity: plainToClass(NlpEntity, entity, options), - }, - { - excludePrefixes: ['_'], - }, - ) as TNlpValueCount; - } else { - return plainToClass( - NlpValueWithCount, - { entity, ...rest }, - options, - ) as TNlpValueCount; - } - }); - } - async findWithCount( format: F, pageQuery: PageQueryDto, @@ -270,18 +250,18 @@ export class NlpValueRepository extends BaseRepository< filterQuery, ); - return await this.plainToClass(format, aggregatedResults); + if (format === Format.FULL) { + return plainToInstance(NlpValueFullWithCount, aggregatedResults, { + excludePrefixes: ['_'], + }) as TNlpValueCount[]; + } + + return plainToInstance(NlpValueWithCount, aggregatedResults, { + excludePrefixes: ['_'], + }) as TNlpValueCount[]; } catch (error) { this.logger.error(`Error in findWithCount: ${error.message}`, error); throw error; } } - - private getSortDirection(sortOrder: SortOrder) { - return typeof sortOrder === 'number' - ? sortOrder - : sortOrder.toString().toLowerCase() === 'desc' - ? -1 - : 1; - } } From 83849c2df2d68649e58e67ee6bd732e4f43eb97f Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sun, 6 Apr 2025 11:46:37 +0100 Subject: [PATCH 20/20] fix: enhance aggregation pipeline --- .../nlp/repositories/nlp-value.repository.ts | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/api/src/nlp/repositories/nlp-value.repository.ts b/api/src/nlp/repositories/nlp-value.repository.ts index ab276904..bab94d7c 100644 --- a/api/src/nlp/repositories/nlp-value.repository.ts +++ b/api/src/nlp/repositories/nlp-value.repository.ts @@ -147,7 +147,6 @@ export class NlpValueRepository extends BaseRepository< ): Promise[]> { const pipeline: PipelineStage[] = [ { - // support filters $match: { ...rest, ...($and.length @@ -162,7 +161,6 @@ export class NlpValueRepository extends BaseRepository< : {}), }, }, - // support pageQuery { $skip: skip, }, @@ -174,42 +172,34 @@ export class NlpValueRepository extends BaseRepository< from: 'nlpsampleentities', localField: '_id', foreignField: 'value', - as: 'sampleEntities', + as: '_sampleEntities', }, }, { $unwind: { - path: '$sampleEntities', + path: '$_sampleEntities', preserveNullAndEmptyArrays: true, }, }, { $group: { _id: '$_id', - value: { $first: '$value' }, - expressions: { $first: '$expressions' }, - builtin: { $first: '$builtin' }, - metadata: { $first: '$metadata' }, - createdAt: { $first: '$createdAt' }, - updatedAt: { $first: '$updatedAt' }, - entity: { $first: '$entity' }, + _originalDoc: { + $first: { + $unsetField: { input: '$$ROOT', field: 'nlpSamplesCount' }, + }, + }, nlpSamplesCount: { - $sum: { $cond: [{ $ifNull: ['$sampleEntities', false] }, 1, 0] }, + $sum: { $cond: [{ $ifNull: ['$_sampleEntities', false] }, 1, 0] }, }, }, }, { - $project: { - id: '$_id', - _id: 0, - value: 1, - expressions: 1, - builtin: 1, - entity: 1, - metadata: 1, - createdAt: 1, - updatedAt: 1, - nlpSamplesCount: 1, + $replaceWith: { + $mergeObjects: [ + '$_originalDoc', + { nlpSamplesCount: '$nlpSamplesCount' }, + ], }, }, ...(format === Format.FULL