UniqAI/hexabot
1
0
Fork 0
mirror of https://github.com/hexastack/hexabot synced 2025-06-26 18:27:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: refactor helpers (nlu)

Browse Source
This commit is contained in:
Mohamed Marrouchi
2024-10-21 15:09:59 +01:00
parent b2c32fe27d
commit b7eef89981
53 changed files with 901 additions and 731 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/nlp/controllers/nlp-sample.controller.spec.ts
View File

@@ -17,6 +17,7 @@ import { Test, TestingModule } from '@nestjs/testing';
import { AttachmentRepository } from '@/attachment/repositories/attachment.repository';
import { AttachmentModel } from '@/attachment/schemas/attachment.schema';
import { AttachmentService } from '@/attachment/services/attachment.service';
import { HelperService } from '@/helper/helper.service';
import { LanguageRepository } from '@/i18n/repositories/language.repository';
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
import { I18nService } from '@/i18n/services/i18n.service';
@@ -98,6 +99,7 @@ describe('NlpSampleController', () => {
LanguageService,
EventEmitter2,
NlpService,
HelperService,
SettingRepository,
SettingService,
SettingSeeder,

28
api/src/nlp/controllers/nlp-sample.controller.ts
View File

@@ -17,6 +17,7 @@ import {
Delete,
Get,
HttpCode,
InternalServerErrorException,
NotFoundException,
Param,
Patch,
@@ -33,6 +34,7 @@ import Papa from 'papaparse';
import { AttachmentService } from '@/attachment/services/attachment.service';
import { config } from '@/config';
import { HelperService } from '@/helper/helper.service';
import { LanguageService } from '@/i18n/services/language.service';
import { CsrfInterceptor } from '@/interceptors/csrf.interceptor';
import { LoggerService } from '@/logger/logger.service';
@@ -72,6 +74,7 @@ export class NlpSampleController extends BaseController<
private readonly logger: LoggerService,
private readonly nlpService: NlpService,
private readonly languageService: LanguageService,
private readonly helperService: HelperService,
) {
super(nlpSampleService);
}
@@ -93,7 +96,8 @@ export class NlpSampleController extends BaseController<
type ? { type } : {},
);
const entities = await this.nlpEntityService.findAllAndPopulate();
const result = await this.nlpSampleService.formatRasaNlu(samples, entities);
const helper = await this.helperService.getDefaultNluHelper();
const result = helper.format(samples, entities);
// Sending the JSON data as a file
const buffer = Buffer.from(JSON.stringify(result));
@@ -171,7 +175,8 @@ export class NlpSampleController extends BaseController<
*/
@Get('message')
async message(@Query('text') text: string) {
return this.nlpService.getNLP().parse(text);
const helper = await this.helperService.getDefaultNluHelper();
return helper.predict(text);
}
/**
@@ -201,7 +206,21 @@ export class NlpSampleController extends BaseController<
const { samples, entities } =
await this.getSamplesAndEntitiesByType('train');
return await this.nlpService.getNLP().train(samples, entities);
try {
const helper = await this.helperService.getDefaultNluHelper();
const response = await helper.train(samples, entities);
// Mark samples as trained
await this.nlpSampleService.updateMany(
{ type: 'train' },
{ trained: true },
);
return response;
} catch (err) {
this.logger.error(err);
throw new InternalServerErrorException(
'Unable to perform the train operation',
);
}
}
/**
@@ -214,7 +233,8 @@ export class NlpSampleController extends BaseController<
const { samples, entities } =
await this.getSamplesAndEntitiesByType('test');
return await this.nlpService.getNLP().evaluate(samples, entities);
const helper = await this.helperService.getDefaultNluHelper();
return await helper.evaluate(samples, entities);
}
/**

30
api/src/nlp/controllers/nlp.controller.ts
View File

@@ -1,30 +0,0 @@
/*
* Copyright © 2024 Hexastack. All rights reserved.
*
* Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
* 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
* 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
*/
import { Controller, Get } from '@nestjs/common';
import { NlpService } from '../services/nlp.service';
@Controller('nlp')
export class NlpController {
constructor(private readonly nlpService: NlpService) {}
/**
* Retrieves a list of NLP helpers.
*
* @returns An array of objects containing the name of each NLP helper.
*/
@Get()
getNlpHelpers(): { name: string }[] {
return this.nlpService.getAll().map((helper) => {
return {
name: helper.getName(),
};
});
}
}

202
api/src/nlp/lib/BaseNlpHelper.ts
View File

@@ -1,202 +0,0 @@
/*
* Copyright © 2024 Hexastack. All rights reserved.
*
* Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
* 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
* 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
*/
/**
* @file NlpAdapter is an abstract class for define an NLP provider adapter
* @author Hexastack <contact@hexastack.com>
*/
/**
* @module Services/NLP
*
* NlpAdapter is an abstract class from which each NLP provider adapter should extend from.
*/
import { v4 as uuidv4 } from 'uuid';
import { LoggerService } from '@/logger/logger.service';
import {
NlpEntity,
NlpEntityDocument,
NlpEntityFull,
} from '@/nlp/schemas/nlp-entity.schema';
import { NlpSample, NlpSampleFull } from '@/nlp/schemas/nlp-sample.schema';
import {
NlpValue,
NlpValueDocument,
NlpValueFull,
} from '@/nlp/schemas/nlp-value.schema';
import { NlpEntityService } from '../services/nlp-entity.service';
import { NlpSampleService } from '../services/nlp-sample.service';
import { NlpService } from '../services/nlp.service';
import { Nlp } from './types';
export default abstract class BaseNlpHelper {
protected settings: Settings['nlp_settings'];
constructor(
protected readonly logger: LoggerService,
protected readonly nlpService: NlpService,
protected readonly nlpSampleService: NlpSampleService,
protected readonly nlpEntityService: NlpEntityService,
) {}
setSettings(settings: Settings['nlp_settings']) {
this.settings = settings;
}
/**
* Returns the helper's name
*
* @returns Helper's name
*/
abstract getName(): string;
/**
* Updates an entity
*
* @param entity - The updated entity
*
* @returns The updated entity otherwise an error
*/
async updateEntity(entity: NlpEntity): Promise<NlpEntity> {
return entity;
}
/**
* Adds an entity
*
* @param entity - The entity to add
* @returns The added entity otherwise an error
*/
addEntity(_entity: NlpEntityDocument): Promise<string> {
return new Promise((resolve, _reject) => {
return resolve(uuidv4());
});
}
/**
* Deletes an entity
*
* @param entityId - The entity ID to delete
*
* @return The deleted entity otherwise an error
*/
async deleteEntity(entityId: string): Promise<any> {
return entityId;
}
/**
* Update an entity value
*
* @param value - The updated update
*
* @returns The updated value otherwise it should throw an error
*/
async updateValue(value: NlpValue): Promise<NlpValue> {
return value;
}
/**
* Adds an entity value
*
* @param value - The value to add
*
* @returns The added value otherwise it should throw an error
*/
addValue(_value: NlpValueDocument): Promise<string> {
return new Promise((resolve, _reject) => {
return resolve(uuidv4());
});
}
/**
* Delete an entity value
*
* @param value - The value to delete
*
* @returns The deleted value otherwise an error
*/
async deleteValue(value: NlpValueFull): Promise<NlpValueFull> {
return value;
}
/**
* Returns training dataset in NLP provider compatible format
*
* @param samples - Sample to train
* @param entities - All available entities
*
* @returns The formatted NLP training set
*/
abstract format(samples: NlpSampleFull[], entities: NlpEntityFull[]): unknown;
/**
* Perform training request
*
* @param samples - Samples to train
* @param entities - All available entities
*
* @returns Training result
*/
abstract train(
samples: NlpSampleFull[],
entities: NlpEntityFull[],
): Promise<any>;
/**
* Perform evaluation request
*
* @param samples - Samples to evaluate
* @param entities - All available entities
*
* @returns NLP evaluation result
*/
abstract evaluate(
samples: NlpSampleFull[],
entities: NlpEntityFull[],
): Promise<any>;
/**
* Delete/Forget a sample
*
* @param sample - The sample to delete/forget
*
* @returns The deleted sample otherwise an error
*/
async forget(sample: NlpSample): Promise<NlpSample> {
return sample;
}
/**
* Returns only the entities that have strong confidence (> than the threshold), can return an empty result
*
* @param nlp - The nlp provider parse returned result
* @param threshold - Whenever to apply threshold filter or not
*
* @returns NLP Parsed entities
*/
abstract bestGuess(nlp: any, threshold: boolean): Nlp.ParseEntities;
/**
* Returns only the entities that have strong confidence (> than the threshold), can return an empty result
*
* @param text - The text to parse
* @param threshold - Whenever to apply threshold filter or not
* @param project - Whenever to request a specific model
*
* @returns NLP Parsed entities
*/
abstract parse(
text: string,
threshold?: boolean,
project?: string,
): Promise<Nlp.ParseEntities>;
}

26
api/src/nlp/lib/types.ts
View File

@@ -1,26 +0,0 @@
/*
* Copyright © 2024 Hexastack. All rights reserved.
*
* Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
* 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
* 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
*/
export namespace Nlp {
export interface Config {
endpoint?: string;
token: string;
}
export interface ParseEntity {
entity: string; // Entity name
value: string; // Value name
confidence: number;
start?: number;
end?: number;
}
export interface ParseEntities {
entities: ParseEntity[];
}
}

8
api/src/nlp/nlp.module.ts
View File

@@ -16,7 +16,6 @@ import { AttachmentModule } from '@/attachment/attachment.module';
import { NlpEntityController } from './controllers/nlp-entity.controller';
import { NlpSampleController } from './controllers/nlp-sample.controller';
import { NlpValueController } from './controllers/nlp-value.controller';
import { NlpController } from './controllers/nlp.controller';
import { NlpEntityRepository } from './repositories/nlp-entity.repository';
import { NlpSampleEntityRepository } from './repositories/nlp-sample-entity.repository';
import { NlpSampleRepository } from './repositories/nlp-sample.repository';
@@ -45,12 +44,7 @@ import { NlpService } from './services/nlp.service';
AttachmentModule,
HttpModule,
],
controllers: [
NlpEntityController,
NlpValueController,
NlpSampleController,
NlpController,
],
controllers: [NlpEntityController, NlpValueController, NlpSampleController],
providers: [
NlpEntityRepository,
NlpValueRepository,

6
api/src/nlp/services/nlp-sample.service.spec.ts
View File

@@ -14,6 +14,7 @@ import { Test, TestingModule } from '@nestjs/testing';
import { LanguageRepository } from '@/i18n/repositories/language.repository';
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
import { LanguageService } from '@/i18n/services/language.service';
import { LoggerService } from '@/logger/logger.service';
import { nlpSampleFixtures } from '@/utils/test/fixtures/nlpsample';
import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity';
import { getPageQuery } from '@/utils/test/pagination';
@@ -28,10 +29,10 @@ import { NlpSampleRepository } from '../repositories/nlp-sample.repository';
import { NlpValueRepository } from '../repositories/nlp-value.repository';
import { NlpEntityModel } from '../schemas/nlp-entity.schema';
import {
NlpSampleEntityModel,
NlpSampleEntity,
NlpSampleEntityModel,
} from '../schemas/nlp-sample-entity.schema';
import { NlpSampleModel, NlpSample } from '../schemas/nlp-sample.schema';
import { NlpSample, NlpSampleModel } from '../schemas/nlp-sample.schema';
import { NlpValueModel } from '../schemas/nlp-value.schema';
import { NlpEntityService } from './nlp-entity.service';
@@ -72,6 +73,7 @@ describe('NlpSampleService', () => {
NlpValueService,
LanguageService,
EventEmitter2,
LoggerService,
{
provide: CACHE_MANAGER,
useValue: {

130
api/src/nlp/services/nlp-sample.service.ts
View File

@@ -9,25 +9,20 @@
import { Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import {
CommonExample,
DatasetType,
EntitySynonym,
ExampleEntity,
LookupTable,
} from '@/extensions/helpers/nlp/default/types';
import { AnyMessage } from '@/chat/schemas/types/message';
import { Language } from '@/i18n/schemas/language.schema';
import { LanguageService } from '@/i18n/services/language.service';
import { LoggerService } from '@/logger/logger.service';
import { BaseService } from '@/utils/generics/base-service';
import { NlpSampleCreateDto } from '../dto/nlp-sample.dto';
import { NlpSampleRepository } from '../repositories/nlp-sample.repository';
import { NlpEntity, NlpEntityFull } from '../schemas/nlp-entity.schema';
import {
NlpSample,
NlpSampleFull,
NlpSamplePopulate,
} from '../schemas/nlp-sample.schema';
import { NlpValue } from '../schemas/nlp-value.schema';
import { NlpSampleState } from '../schemas/types';
@Injectable()
export class NlpSampleService extends BaseService<
@@ -38,6 +33,7 @@ export class NlpSampleService extends BaseService<
constructor(
readonly repository: NlpSampleRepository,
private readonly languageService: LanguageService,
private readonly logger: LoggerService,
) {
super(repository);
}
@@ -53,95 +49,6 @@ export class NlpSampleService extends BaseService<
return await this.repository.deleteOne(id);
}
/**
* Formats a set of NLP samples into the Rasa NLU-compatible training dataset format.
*
* @param samples - The NLP samples to format.
* @param entities - The NLP entities available in the dataset.
*
* @returns The formatted Rasa NLU training dataset.
*/
async formatRasaNlu(
samples: NlpSampleFull[],
entities: NlpEntityFull[],
): Promise<DatasetType> {
const entityMap = NlpEntity.getEntityMap(entities);
const valueMap = NlpValue.getValueMap(
NlpValue.getValuesFromEntities(entities),
);
const common_examples: CommonExample[] = samples
.filter((s) => s.entities.length > 0)
.map((s) => {
const intent = s.entities.find(
(e) => entityMap[e.entity].name === 'intent',
);
if (!intent) {
throw new Error('Unable to find the `intent` nlp entity.');
}
const sampleEntities: ExampleEntity[] = s.entities
.filter((e) => entityMap[<string>e.entity].name !== 'intent')
.map((e) => {
const res: ExampleEntity = {
entity: entityMap[<string>e.entity].name,
value: valueMap[<string>e.value].value,
};
if ('start' in e && 'end' in e) {
Object.assign(res, {
start: e.start,
end: e.end,
});
}
return res;
})
// TODO : place language at the same level as the intent
.concat({
entity: 'language',
value: s.language.code,
});
return {
text: s.text,
intent: valueMap[intent.value].value,
entities: sampleEntities,
};
});
const languages = await this.languageService.getLanguages();
const lookup_tables: LookupTable[] = entities
.map((e) => {
return {
name: e.name,
elements: e.values.map((v) => {
return v.value;
}),
};
})
.concat({
name: 'language',
elements: Object.keys(languages),
});
const entity_synonyms = entities
.reduce((acc, e) => {
const synonyms = e.values.map((v) => {
return {
value: v.value,
synonyms: v.expressions,
};
});
return acc.concat(synonyms);
}, [] as EntitySynonym[])
.filter((s) => {
return s.synonyms.length > 0;
});
return {
common_examples,
regex_features: [],
lookup_tables,
entity_synonyms,
};
}
/**
* When a language gets deleted, we need to set related samples to null
*
@@ -158,4 +65,31 @@ export class NlpSampleService extends BaseService<
},
);
}
@OnEvent('hook:message:preCreate')
async handleNewMessage(doc: AnyMessage) {
// If message is sent by the user then add it as an inbox sample
if (
'sender' in doc &&
doc.sender &&
'message' in doc &&
'text' in doc.message
) {
const defaultLang = await this.languageService.getDefaultLanguage();
const record: NlpSampleCreateDto = {
text: doc.message.text,
type: NlpSampleState.inbox,
trained: false,
// @TODO : We need to define the language in the message entity
language: defaultLang.id,
};
try {
await this.findOneOrCreate(record, record);
this.logger.debug('User message saved as a inbox sample !');
} catch (err) {
this.logger.error('Unable to add message as a new inbox sample!', err);
throw err;
}
}
}
}

105
api/src/nlp/services/nlp.service.ts
View File

@@ -6,13 +6,12 @@
* 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
*/
import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
import { Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { HelperService } from '@/helper/helper.service';
import { LoggerService } from '@/logger/logger.service';
import { SettingService } from '@/setting/services/setting.service';
import BaseNlpHelper from '../lib/BaseNlpHelper';
import { NlpEntity, NlpEntityDocument } from '../schemas/nlp-entity.schema';
import { NlpValue, NlpValueDocument } from '../schemas/nlp-value.schema';
@@ -21,93 +20,15 @@ import { NlpSampleService } from './nlp-sample.service';
import { NlpValueService } from './nlp-value.service';
@Injectable()
export class NlpService implements OnApplicationBootstrap {
private registry: Map<string, BaseNlpHelper> = new Map();
private nlp: BaseNlpHelper;
export class NlpService {
constructor(
private readonly settingService: SettingService,
private readonly logger: LoggerService,
protected readonly nlpSampleService: NlpSampleService,
protected readonly nlpEntityService: NlpEntityService,
protected readonly nlpValueService: NlpValueService,
protected readonly helperService: HelperService,
) {}
onApplicationBootstrap() {
this.initNLP();
}
/**
* Registers a helper with a specific name in the registry.
*
* @param name - The name of the helper to register.
* @param helper - The NLP helper to be associated with the given name.
* @typeParam C - The type of the helper, which must extend `BaseNlpHelper`.
*/
public setHelper<C extends BaseNlpHelper>(name: string, helper: C) {
this.registry.set(name, helper);
}
/**
* Retrieves all registered helpers.
*
* @returns An array of all helpers currently registered.
*/
public getAll() {
return Array.from(this.registry.values());
}
/**
* Retrieves the appropriate helper based on the helper name.
*
* @param helperName - The name of the helper (messenger, offline, ...).
*
* @returns The specified helper.
*/
public getHelper<C extends BaseNlpHelper>(name: string): C {
const handler = this.registry.get(name);
if (!handler) {
throw new Error(`NLP Helper ${name} not found`);
}
return handler as C;
}
async initNLP() {
try {
const settings = await this.settingService.getSettings();
const nlpSettings = settings.nlp_settings;
const helper = this.getHelper(nlpSettings.provider);
if (helper) {
this.nlp = helper;
this.nlp.setSettings(nlpSettings);
} else {
throw new Error(`Undefined NLP Helper ${nlpSettings.provider}`);
}
} catch (e) {
this.logger.error('NLP Service : Unable to instantiate NLP Helper !', e);
// throw e;
}
}
/**
* Retrieves the currently active NLP helper.
*
* @returns The current NLP helper.
*/
getNLP() {
return this.nlp;
}
/**
* Handles the event triggered when NLP settings are updated. Re-initializes the NLP service.
*/
@OnEvent('hook:nlp_settings:*')
async handleSettingsUpdate() {
this.initNLP();
}
/**
* Handles the event triggered when a new NLP entity is created. Synchronizes the entity with the external NLP provider.
*
@@ -118,7 +39,8 @@ export class NlpService implements OnApplicationBootstrap {
async handleEntityCreate(entity: NlpEntityDocument) {
// Synchonize new entity with NLP
try {
const foreignId = await this.getNLP().addEntity(entity);
const helper = await this.helperService.getDefaultNluHelper();
const foreignId = await helper.addEntity(entity);
this.logger.debug('New entity successfully synced!', foreignId);
return await this.nlpEntityService.updateOne(entity._id, {
foreign_id: foreignId,
@@ -138,7 +60,8 @@ export class NlpService implements OnApplicationBootstrap {
async handleEntityUpdate(entity: NlpEntity) {
// Synchonize new entity with NLP provider
try {
await this.getNLP().updateEntity(entity);
const helper = await this.helperService.getDefaultNluHelper();
await helper.updateEntity(entity);
this.logger.debug('Updated entity successfully synced!', entity);
} catch (err) {
this.logger.error('Unable to sync updated entity', err);
@@ -154,7 +77,8 @@ export class NlpService implements OnApplicationBootstrap {
async handleEntityDelete(entity: NlpEntity) {
// Synchonize new entity with NLP provider
try {
await this.getNLP().deleteEntity(entity.foreign_id);
const helper = await this.helperService.getDefaultNluHelper();
await helper.deleteEntity(entity.foreign_id);
this.logger.debug('Deleted entity successfully synced!', entity);
} catch (err) {
this.logger.error('Unable to sync deleted entity', err);
@@ -172,7 +96,8 @@ export class NlpService implements OnApplicationBootstrap {
async handleValueCreate(value: NlpValueDocument) {
// Synchonize new value with NLP provider
try {
const foreignId = await this.getNLP().addValue(value);
const helper = await this.helperService.getDefaultNluHelper();
const foreignId = await helper.addValue(value);
this.logger.debug('New value successfully synced!', foreignId);
return await this.nlpValueService.updateOne(value._id, {
foreign_id: foreignId,
@@ -192,7 +117,8 @@ export class NlpService implements OnApplicationBootstrap {
async handleValueUpdate(value: NlpValue) {
// Synchonize new value with NLP provider
try {
await this.getNLP().updateValue(value);
const helper = await this.helperService.getDefaultNluHelper();
await helper.updateValue(value);
this.logger.debug('Updated value successfully synced!', value);
} catch (err) {
this.logger.error('Unable to sync updated value', err);
@@ -208,10 +134,11 @@ export class NlpService implements OnApplicationBootstrap {
async handleValueDelete(value: NlpValue) {
// Synchonize new value with NLP provider
try {
const helper = await this.helperService.getDefaultNluHelper();
const populatedValue = await this.nlpValueService.findOneAndPopulate(
value.id,
);
await this.getNLP().deleteValue(populatedValue);
await helper.deleteValue(populatedValue);
this.logger.debug('Deleted value successfully synced!', value);
} catch (err) {
this.logger.error('Unable to sync deleted value', err);