Merge pull request #462 from Hexastack/461-issue-saving-nlpsample-as-an-attachment

feat: import nlpsamples files without adding them as attachments

api/src/config/index.ts

@@ -112,7 +112,7 @@ export const config: Config = {
     storageMode: 'disk',
     maxUploadSize: process.env.UPLOAD_MAX_SIZE_IN_BYTES
       ? Number(process.env.UPLOAD_MAX_SIZE_IN_BYTES)
-      : 2000000,
+      : 50 * 1024 * 1024, // 50 MB in bytes
     appName: 'Hexabot.ai',
   },
   pagination: {

api/src/nlp/controllers/nlp-sample.controller.spec.ts

@@ -6,17 +6,12 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import fs from 'fs';
-
 import { CACHE_MANAGER } from '@nestjs/cache-manager';
 import { BadRequestException, NotFoundException } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import { MongooseModule } from '@nestjs/mongoose';
 import { Test, TestingModule } from '@nestjs/testing';
 
-import { AttachmentRepository } from '@/attachment/repositories/attachment.repository';
-import { AttachmentModel } from '@/attachment/schemas/attachment.schema';
-import { AttachmentService } from '@/attachment/services/attachment.service';
 import { HelperService } from '@/helper/helper.service';
 import { LanguageRepository } from '@/i18n/repositories/language.repository';
 import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
@@ -50,7 +45,6 @@ import { NlpEntityService } from '../services/nlp-entity.service';
 import { NlpSampleEntityService } from '../services/nlp-sample-entity.service';
 import { NlpSampleService } from '../services/nlp-sample.service';
 import { NlpValueService } from '../services/nlp-value.service';
-import { NlpService } from '../services/nlp.service';
 
 import { NlpSampleController } from './nlp-sample.controller';
 
@@ -60,7 +54,6 @@ describe('NlpSampleController', () => {
   let nlpSampleService: NlpSampleService;
   let nlpEntityService: NlpEntityService;
   let nlpValueService: NlpValueService;
-  let attachmentService: AttachmentService;
   let languageService: LanguageService;
   let byeJhonSampleId: string;
   let languages: Language[];
@@ -76,7 +69,6 @@ describe('NlpSampleController', () => {
         MongooseModule.forFeature([
           NlpSampleModel,
           NlpSampleEntityModel,
-          AttachmentModel,
           NlpEntityModel,
           NlpValueModel,
           SettingModel,
@@ -87,9 +79,7 @@ describe('NlpSampleController', () => {
         LoggerService,
         NlpSampleRepository,
         NlpSampleEntityRepository,
-        AttachmentService,
         NlpEntityService,
-        AttachmentRepository,
         NlpEntityRepository,
         NlpValueService,
         NlpValueRepository,
@@ -98,7 +88,6 @@ describe('NlpSampleController', () => {
         LanguageRepository,
         LanguageService,
         EventEmitter2,
-        NlpService,
         HelperService,
         SettingRepository,
         SettingService,
@@ -131,7 +120,6 @@ describe('NlpSampleController', () => {
         text: 'Bye Jhon',
       })
     ).id;
-    attachmentService = module.get<AttachmentService>(AttachmentService);
     languageService = module.get<LanguageService>(LanguageService);
     languages = await languageService.findAll();
   });
@@ -315,83 +303,44 @@ describe('NlpSampleController', () => {
     });
   });
 
-  describe('import', () => {
-    it('should throw exception when attachment is not found', async () => {
-      const invalidattachmentId = (
-        await attachmentService.findOne({
-          name: 'store2.jpg',
-        })
-      ).id;
-      await attachmentService.deleteOne({ name: 'store2.jpg' });
-      await expect(
-        nlpSampleController.import(invalidattachmentId),
-      ).rejects.toThrow(NotFoundException);
-    });
-
-    it('should throw exception when file location is not present', async () => {
-      const attachmentId = (
-        await attachmentService.findOne({
-          name: 'store1.jpg',
-        })
-      ).id;
-      jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false);
-      await expect(nlpSampleController.import(attachmentId)).rejects.toThrow(
-        NotFoundException,
+  describe('importFile', () => {
+    it('should throw exception when something is wrong with the upload', async () => {
+      const file = {
+        buffer: Buffer.from('', 'utf-8'),
+        size: 0,
+        mimetype: 'text/csv',
+      } as Express.Multer.File;
+      await expect(nlpSampleController.importFile(file)).rejects.toThrow(
+        'Bad Request Exception',
       );
     });
 
     it('should return a failure if an error occurs when parsing csv file ', async () => {
       const mockCsvDataWithErrors: string = `intent,entities,lang,question
           greeting,person,en`;
-      jest.spyOn(fs, 'existsSync').mockReturnValueOnce(true);
-      jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(mockCsvDataWithErrors);
-      const attachmentId = (
-        await attachmentService.findOne({
-          name: 'store1.jpg',
-        })
-      ).id;
 
-      const mockParsedCsvDataWithErrors = {
-        data: [{ intent: 'greeting', entities: 'person', lang: 'en' }],
-        errors: [
-          {
-            type: 'FieldMismatch',
-            code: 'TooFewFields',
-            message: 'Too few fields: expected 4 fields but parsed 3',
-            row: 0,
-          },
-        ],
-        meta: {
-          delimiter: ',',
-          linebreak: '\n',
-          aborted: false,
-          truncated: false,
-          cursor: 49,
-          fields: ['intent', 'entities', 'lang', 'question'],
-        },
-      };
-      await expect(nlpSampleController.import(attachmentId)).rejects.toThrow(
-        new BadRequestException({
-          cause: mockParsedCsvDataWithErrors.errors,
-          description: 'Error while parsing CSV',
-        }),
-      );
+      const buffer = Buffer.from(mockCsvDataWithErrors, 'utf-8');
+      const file = {
+        buffer,
+        size: buffer.length,
+        mimetype: 'text/csv',
+      } as Express.Multer.File;
+      await expect(nlpSampleController.importFile(file)).rejects.toThrow();
     });
 
     it('should import data from a CSV file', async () => {
-      const attachmentId = (
-        await attachmentService.findOne({
-          name: 'store1.jpg',
-        })
-      ).id;
       const mockCsvData: string = [
         `text,intent,language`,
         `How much does a BMW cost?,price,en`,
       ].join('\n');
-      jest.spyOn(fs, 'existsSync').mockReturnValueOnce(true);
-      jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(mockCsvData);
 
-      const result = await nlpSampleController.import(attachmentId);
+      const buffer = Buffer.from(mockCsvData, 'utf-8');
+      const file = {
+        buffer,
+        size: buffer.length,
+        mimetype: 'text/csv',
+      } as Express.Multer.File;
+      const result = await nlpSampleController.importFile(file);
       const intentEntityResult = await nlpEntityService.findOne({
         name: 'intent',
       });
@@ -429,9 +378,10 @@ describe('NlpSampleController', () => {
       expect(intentEntityResult).toEqualPayload(intentEntity);
       expect(priceValueResult).toEqualPayload(priceValue);
       expect(textSampleResult).toEqualPayload(textSample);
-      expect(result).toEqual({ success: true });
+      expect(result).toEqualPayload([textSample]);
     });
   });
+
   describe('deleteMany', () => {
     it('should delete multiple nlp samples', async () => {
       const samplesToDelete = [

api/src/nlp/controllers/nlp-sample.controller.ts

@@ -6,8 +6,6 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import fs from 'fs';
-import { join } from 'path';
 import { Readable } from 'stream';
 
 import {
@@ -25,14 +23,13 @@ import {
   Query,
   Res,
   StreamableFile,
+  UploadedFile,
   UseInterceptors,
 } from '@nestjs/common';
+import { FileInterceptor } from '@nestjs/platform-express';
 import { CsrfCheck } from '@tekuconcept/nestjs-csrf';
 import { Response } from 'express';
-import Papa from 'papaparse';
 
-import { AttachmentService } from '@/attachment/services/attachment.service';
-import { config } from '@/config';
 import { HelperService } from '@/helper/helper.service';
 import { LanguageService } from '@/i18n/services/language.service';
 import { CsrfInterceptor } from '@/interceptors/csrf.interceptor';
@@ -45,18 +42,17 @@ import { PopulatePipe } from '@/utils/pipes/populate.pipe';
 import { SearchFilterPipe } from '@/utils/pipes/search-filter.pipe';
 import { TFilterQuery } from '@/utils/types/filter.types';
 
-import { NlpSampleCreateDto, NlpSampleDto } from '../dto/nlp-sample.dto';
+import { NlpSampleDto } from '../dto/nlp-sample.dto';
 import {
   NlpSample,
   NlpSampleFull,
   NlpSamplePopulate,
   NlpSampleStub,
 } from '../schemas/nlp-sample.schema';
-import { NlpSampleEntityValue, NlpSampleState } from '../schemas/types';
+import { NlpSampleState } from '../schemas/types';
 import { NlpEntityService } from '../services/nlp-entity.service';
 import { NlpSampleEntityService } from '../services/nlp-sample-entity.service';
 import { NlpSampleService } from '../services/nlp-sample.service';
-import { NlpService } from '../services/nlp.service';
 
 @UseInterceptors(CsrfInterceptor)
 @Controller('nlpsample')
@@ -68,11 +64,9 @@ export class NlpSampleController extends BaseController<
 > {
   constructor(
     private readonly nlpSampleService: NlpSampleService,
-    private readonly attachmentService: AttachmentService,
     private readonly nlpSampleEntityService: NlpSampleEntityService,
     private readonly nlpEntityService: NlpEntityService,
     private readonly logger: LoggerService,
-    private readonly nlpService: NlpService,
     private readonly languageService: LanguageService,
     private readonly helperService: HelperService,
   ) {
@@ -369,129 +363,11 @@ export class NlpSampleController extends BaseController<
     return deleteResult;
   }
 
-  /**
-   * Imports NLP samples from a CSV file.
-   *
-   * @param file - The file path or ID of the CSV file to import.
-   *
-   * @returns A success message after the import process is completed.
-   */
   @CsrfCheck(true)
-  @Post('import/:file')
-  async import(
-    @Param('file')
-    file: string,
-  ) {
-    // Check if file is present
-    const importedFile = await this.attachmentService.findOne(file);
-    if (!importedFile) {
-      throw new NotFoundException('Missing file!');
-    }
-    const filePath = importedFile
-      ? join(config.parameters.uploadDir, importedFile.location)
-      : undefined;
-
-    // Check if file location is present
-    if (!fs.existsSync(filePath)) {
-      throw new NotFoundException('File does not exist');
-    }
-
-    const allEntities = await this.nlpEntityService.findAll();
-
-    // Check if file location is present
-    if (allEntities.length === 0) {
-      throw new NotFoundException(
-        'No entities found, please create them first.',
-      );
-    }
-
-    // Read file content
-    const data = fs.readFileSync(filePath, 'utf8');
-
-    // Parse local CSV file
-    const result: {
-      errors: any[];
-      data: Array<Record<string, string>>;
-    } = Papa.parse(data, {
-      header: true,
-      skipEmptyLines: true,
-    });
-
-    if (result.errors && result.errors.length > 0) {
-      this.logger.warn(
-        `Errors parsing the file: ${JSON.stringify(result.errors)}`,
-      );
-      throw new BadRequestException(result.errors, {
-        cause: result.errors,
-        description: 'Error while parsing CSV',
-      });
-    }
-    // Remove data with no intent
-    const filteredData = result.data.filter((d) => d.intent !== 'none');
-    const languages = await this.languageService.getLanguages();
-    const defaultLanguage = await this.languageService.getDefaultLanguage();
-    // Reduce function to ensure executing promises one by one
-    for (const d of filteredData) {
-      try {
-        // Check if a sample with the same text already exists
-        const existingSamples = await this.nlpSampleService.find({
-          text: d.text,
-        });
-
-        // Skip if sample already exists
-        if (Array.isArray(existingSamples) && existingSamples.length > 0) {
-          continue;
-        }
-
-        // Fallback to default language if 'language' is missing or invalid
-        if (!d.language || !(d.language in languages)) {
-          if (d.language) {
-            this.logger.warn(
-              `Language "${d.language}" does not exist, falling back to default.`,
-            );
-          }
-          d.language = defaultLanguage.code;
-        }
-
-        // Create a new sample dto
-        const sample: NlpSampleCreateDto = {
-          text: d.text,
-          trained: false,
-          language: languages[d.language].id,
-        };
-
-        // Create a new sample entity dto
-        const entities: NlpSampleEntityValue[] = allEntities
-          .filter(({ name }) => name in d)
-          .map(({ name }) => {
-            return {
-              entity: name,
-              value: d[name],
-            };
-          });
-
-        // Store any new entity/value
-        const storedEntities = await this.nlpEntityService.storeNewEntities(
-          sample.text,
-          entities,
-          ['trait'],
-        );
-        // Store sample
-        const createdSample = await this.nlpSampleService.create(sample);
-        // Map and assign the sample ID to each stored entity
-        const sampleEntities = storedEntities.map((se) => ({
-          ...se,
-          sample: createdSample?.id,
-        }));
-
-        // Store sample entities
-        await this.nlpSampleEntityService.createMany(sampleEntities);
-      } catch (err) {
-        this.logger.error('Error occurred when extracting data. ', err);
-      }
-    }
-
-    this.logger.log('Import process completed successfully.');
-    return { success: true };
+  @Post('import')
+  @UseInterceptors(FileInterceptor('file'))
+  async importFile(@UploadedFile() file: Express.Multer.File) {
+    const datasetContent = file.buffer.toString('utf-8');
+    return await this.nlpSampleService.parseAndSaveDataset(datasetContent);
   }
 }

api/src/nlp/services/nlp-sample.service.spec.ts

@@ -7,6 +7,7 @@
  */
 
 import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { BadRequestException, NotFoundException } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import { MongooseModule } from '@nestjs/mongoose';
 import { Test, TestingModule } from '@nestjs/testing';
@@ -27,7 +28,7 @@ import { NlpEntityRepository } from '../repositories/nlp-entity.repository';
 import { NlpSampleEntityRepository } from '../repositories/nlp-sample-entity.repository';
 import { NlpSampleRepository } from '../repositories/nlp-sample.repository';
 import { NlpValueRepository } from '../repositories/nlp-value.repository';
-import { NlpEntityModel } from '../schemas/nlp-entity.schema';
+import { NlpEntity, NlpEntityModel } from '../schemas/nlp-entity.schema';
 import {
   NlpSampleEntity,
   NlpSampleEntityModel,
@@ -41,7 +42,10 @@ import { NlpSampleService } from './nlp-sample.service';
 import { NlpValueService } from './nlp-value.service';
 
 describe('NlpSampleService', () => {
+  let nlpEntityService: NlpEntityService;
   let nlpSampleService: NlpSampleService;
+  let nlpSampleEntityService: NlpSampleEntityService;
+  let languageService: LanguageService;
   let nlpSampleEntityRepository: NlpSampleEntityRepository;
   let nlpSampleRepository: NlpSampleRepository;
   let languageRepository: LanguageRepository;
@@ -84,7 +88,11 @@ describe('NlpSampleService', () => {
         },
       ],
     }).compile();
+    nlpEntityService = module.get<NlpEntityService>(NlpEntityService);
     nlpSampleService = module.get<NlpSampleService>(NlpSampleService);
+    nlpSampleEntityService = module.get<NlpSampleEntityService>(
+      NlpSampleEntityService,
+    );
     nlpSampleRepository = module.get<NlpSampleRepository>(NlpSampleRepository);
     nlpSampleEntityRepository = module.get<NlpSampleEntityRepository>(
       NlpSampleEntityRepository,
@@ -92,6 +100,7 @@ describe('NlpSampleService', () => {
     nlpSampleEntityRepository = module.get<NlpSampleEntityRepository>(
       NlpSampleEntityRepository,
     );
+    languageService = module.get<LanguageService>(LanguageService);
     languageRepository = module.get<LanguageRepository>(LanguageRepository);
     noNlpSample = await nlpSampleService.findOne({ text: 'No' });
     nlpSampleEntity = await nlpSampleEntityRepository.findOne({
@@ -162,4 +171,104 @@ describe('NlpSampleService', () => {
       expect(result.deletedCount).toEqual(1);
     });
   });
+
+  describe('parseAndSaveDataset', () => {
+    it('should throw NotFoundException if no entities are found', async () => {
+      jest.spyOn(nlpEntityService, 'findAll').mockResolvedValue([]);
+
+      await expect(
+        nlpSampleService.parseAndSaveDataset(
+          'text,intent,language\nHello,none,en',
+        ),
+      ).rejects.toThrow(NotFoundException);
+
+      expect(nlpEntityService.findAll).toHaveBeenCalled();
+    });
+
+    it('should throw BadRequestException if CSV parsing fails', async () => {
+      const invalidCSV = 'text,intent,language\n"Hello,none'; // Malformed CSV
+      jest
+        .spyOn(nlpEntityService, 'findAll')
+        .mockResolvedValue([{ name: 'intent' } as NlpEntity]);
+      jest.spyOn(languageService, 'getLanguages').mockResolvedValue({});
+      jest
+        .spyOn(languageService, 'getDefaultLanguage')
+        .mockResolvedValue({ code: 'en' } as Language);
+
+      await expect(
+        nlpSampleService.parseAndSaveDataset(invalidCSV),
+      ).rejects.toThrow(BadRequestException);
+    });
+
+    it('should filter out rows with "none" as intent', async () => {
+      const mockData = 'text,intent,language\nHello,none,en\nHi,greet,en';
+      jest
+        .spyOn(nlpEntityService, 'findAll')
+        .mockResolvedValue([{ name: 'intent' } as NlpEntity]);
+      jest
+        .spyOn(languageService, 'getLanguages')
+        .mockResolvedValue({ en: { id: '1' } });
+      jest
+        .spyOn(languageService, 'getDefaultLanguage')
+        .mockResolvedValue({ code: 'en' } as Language);
+      jest.spyOn(nlpSampleService, 'find').mockResolvedValue([]);
+      jest
+        .spyOn(nlpSampleService, 'create')
+        .mockResolvedValue({ id: '1', text: 'Hi' } as NlpSample);
+      jest.spyOn(nlpSampleEntityService, 'createMany').mockResolvedValue([]);
+
+      const result = await nlpSampleService.parseAndSaveDataset(mockData);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].text).toEqual('Hi');
+    });
+
+    it('should fallback to the default language if the language is invalid', async () => {
+      const mockData = 'text,intent,language\nHi,greet,invalidLang';
+      jest
+        .spyOn(nlpEntityService, 'findAll')
+        .mockResolvedValue([{ name: 'intent' } as NlpEntity]);
+      jest
+        .spyOn(languageService, 'getLanguages')
+        .mockResolvedValue({ en: { id: '1' } });
+      jest
+        .spyOn(languageService, 'getDefaultLanguage')
+        .mockResolvedValue({ code: 'en' } as Language);
+      jest.spyOn(nlpSampleService, 'find').mockResolvedValue([]);
+      jest
+        .spyOn(nlpSampleService, 'create')
+        .mockResolvedValue({ id: '1', text: 'Hi' } as NlpSample);
+      jest.spyOn(nlpSampleEntityService, 'createMany').mockResolvedValue([]);
+
+      const result = await nlpSampleService.parseAndSaveDataset(mockData);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].text).toEqual('Hi');
+    });
+
+    it('should successfully process and save valid dataset rows', async () => {
+      const mockData = 'text,intent,language\nHi,greet,en\nBye,bye,en';
+      const mockLanguages = { en: { id: '1' } };
+
+      jest
+        .spyOn(languageService, 'getLanguages')
+        .mockResolvedValue(mockLanguages);
+      jest
+        .spyOn(languageService, 'getDefaultLanguage')
+        .mockResolvedValue({ code: 'en' } as Language);
+      jest.spyOn(nlpSampleService, 'find').mockResolvedValue([]);
+      let id = 0;
+      jest.spyOn(nlpSampleService, 'create').mockImplementation((s) => {
+        return Promise.resolve({ id: (++id).toString(), ...s } as NlpSample);
+      });
+      jest.spyOn(nlpSampleEntityService, 'createMany').mockResolvedValue([]);
+
+      const result = await nlpSampleService.parseAndSaveDataset(mockData);
+
+      expect(nlpSampleEntityService.createMany).toHaveBeenCalledTimes(2);
+      expect(result).toHaveLength(2);
+      expect(result[0].text).toEqual('Hi');
+      expect(result[1].text).toEqual('Bye');
+    });
+  });
 });

api/src/nlp/services/nlp-sample.service.ts

@@ -6,8 +6,13 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import { Injectable } from '@nestjs/common';
+import {
+  BadRequestException,
+  Injectable,
+  NotFoundException,
+} from '@nestjs/common';
 import { OnEvent } from '@nestjs/event-emitter';
+import Papa from 'papaparse';
 
 import { Message } from '@/chat/schemas/message.schema';
 import { Language } from '@/i18n/schemas/language.schema';
@@ -23,7 +28,10 @@ import {
   NlpSampleFull,
   NlpSamplePopulate,
 } from '../schemas/nlp-sample.schema';
-import { NlpSampleState } from '../schemas/types';
+import { NlpSampleEntityValue, NlpSampleState } from '../schemas/types';
+
+import { NlpEntityService } from './nlp-entity.service';
+import { NlpSampleEntityService } from './nlp-sample-entity.service';
 
 @Injectable()
 export class NlpSampleService extends BaseService<
@@ -33,6 +41,8 @@ export class NlpSampleService extends BaseService<
 > {
   constructor(
     readonly repository: NlpSampleRepository,
+    private readonly nlpSampleEntityService: NlpSampleEntityService,
+    private readonly nlpEntityService: NlpEntityService,
     private readonly languageService: LanguageService,
     private readonly logger: LoggerService,
   ) {
@@ -50,6 +60,110 @@ export class NlpSampleService extends BaseService<
     return await this.repository.deleteOne(id);
   }
 
+  /**
+   * This function is responsible for parsing a CSV dataset string and saving the parsed data into the database.
+   * It ensures that all necessary entities and languages exist, validates the dataset, and processes it row by row
+   * to create NLP samples and associated entities in the system.
+   *
+   * @param data - The raw CSV dataset as a string.
+   * @returns A promise that resolves to an array of created NLP samples.
+   */
+  async parseAndSaveDataset(data: string) {
+    const allEntities = await this.nlpEntityService.findAll();
+    // Check if file location is present
+    if (allEntities.length === 0) {
+      throw new NotFoundException(
+        'No entities found, please create them first.',
+      );
+    }
+
+    // Parse local CSV file
+    const result: {
+      errors: any[];
+      data: Array<Record<string, string>>;
+    } = Papa.parse(data, {
+      header: true,
+      skipEmptyLines: true,
+    });
+
+    if (result.errors && result.errors.length > 0) {
+      this.logger.warn(
+        `Errors parsing the file: ${JSON.stringify(result.errors)}`,
+      );
+      throw new BadRequestException(result.errors, {
+        cause: result.errors,
+        description: 'Error while parsing CSV',
+      });
+    }
+    // Remove data with no intent
+    const filteredData = result.data.filter((d) => d.intent !== 'none');
+    const languages = await this.languageService.getLanguages();
+    const defaultLanguage = await this.languageService.getDefaultLanguage();
+    const nlpSamples: NlpSample[] = [];
+    // Reduce function to ensure executing promises one by one
+    for (const d of filteredData) {
+      try {
+        // Check if a sample with the same text already exists
+        const existingSamples = await this.find({
+          text: d.text,
+        });
+
+        // Skip if sample already exists
+        if (Array.isArray(existingSamples) && existingSamples.length > 0) {
+          continue;
+        }
+
+        // Fallback to default language if 'language' is missing or invalid
+        if (!d.language || !(d.language in languages)) {
+          if (d.language) {
+            this.logger.warn(
+              `Language "${d.language}" does not exist, falling back to default.`,
+            );
+          }
+          d.language = defaultLanguage.code;
+        }
+
+        // Create a new sample dto
+        const sample: NlpSampleCreateDto = {
+          text: d.text,
+          trained: false,
+          language: languages[d.language].id,
+        };
+
+        // Create a new sample entity dto
+        const entities: NlpSampleEntityValue[] = allEntities
+          .filter(({ name }) => name in d)
+          .map(({ name }) => ({
+            entity: name,
+            value: d[name],
+          }));
+
+        // Store any new entity/value
+        const storedEntities = await this.nlpEntityService.storeNewEntities(
+          sample.text,
+          entities,
+          ['trait'],
+        );
+
+        // Store sample
+        const createdSample = await this.create(sample);
+        nlpSamples.push(createdSample);
+        // Map and assign the sample ID to each stored entity
+        const sampleEntities = storedEntities.map((storedEntity) => ({
+          ...storedEntity,
+          sample: createdSample?.id,
+        }));
+
+        // Store sample entities
+        await this.nlpSampleEntityService.createMany(sampleEntities);
+      } catch (err) {
+        this.logger.error('Error occurred when extracting data. ', err);
+      }
+    }
+
+    return nlpSamples;
+  }
+
   /**
    * When a language gets deleted, we need to set related samples to null
    *

frontend/public/locales/en/translation.json

@@ -58,8 +58,8 @@
     "custom_code_is_invalid": "Custom code seems to contain some errors.",
     "attachment_failure_format": "Attachment has invalid format",
     "drop_file_here": "Drop file here or click to upload",
-    "file_max_size": "File must have a size less than 25MB",
-    "attachment_failure_size": "Invalid size! File must have a size less than 25MB",
+    "file_max_size": "The file exceeds the maximum allowed size. Please ensure your file is within the size limit and try again.",
+    "attachment_failure_size": "The file exceeds the maximum allowed size. Please ensure your file is within the size limit and try again.",
     "upload_failed": "Unable to upload the file!",
     "value_is_required": "NLU Value is required",
     "nlp_entity_name_is_invalid": "NLU Entity name format is invalid! Only `A-z`, `0-9` and `_` are allowed.",
@@ -81,7 +81,9 @@
     "subtitle_is_required": "Subtitle is required",
     "category_is_required": "Flow is required",
     "attachment_is_required": "Attachment is required",
-    "success_import": "Content has been successfuly imported!",
+    "success_import": "Content has been successfully imported!",
+    "import_failed": "Import failed",
+    "import_duplicated_data": "Data already exists",
     "attachment_not_synced": "- Pending Sync. -",
     "success_translation_refresh": "Translations has been successfully refreshed!",
     "message_tag_is_required": "You need to specify a message tag.",
@@ -106,7 +108,7 @@
     "no_label_found": "No label found",
     "code_is_required": "Language code is required",
     "text_is_required": "Text is required",
-    "invalid_file_type": "Invalid file type",
+    "invalid_file_type": "Invalid file type. Please select a file in the supported format.",
     "select_category": "Select a flow"
   },
   "menu": {
@@ -341,6 +343,7 @@
     "precision": "Precision",
     "recall": "Recall",
     "f1score": "F1 Score",
+    "all": "All",
     "train": "Train",
     "test": "Test",
     "inbox": "Inbox",

frontend/public/locales/fr/translation.json

@@ -58,8 +58,8 @@
     "custom_code_is_invalid": "Le code personnalisé semble contenir quelques erreurs.",
     "attachment_failure_format": "La pièce jointe a un format invalide",
     "drop_file_here": "Déposez le fichier ici ou cliquez pour télécharger",
-    "file_max_size": "Le fichier doit avoir une taille inférieure à 25 Mo",
-    "attachment_failure_size": "Taille invalide! Le fichier doit avoir une taille inférieure à 25 Mo",
+    "file_max_size": "Le fichier dépasse la taille maximale autorisée. Veuillez vérifier que votre fichier respecte la limite de taille et réessayez.",
+    "attachment_failure_size": "Le fichier dépasse la taille maximale autorisée. Veuillez vérifier que votre fichier respecte la limite de taille et réessayez.",
     "upload_failed": "Impossible d'envoyer le fichier au serveur!",
     "value_is_required": "La valeur NLU est requise",
     "nlp_entity_name_is_invalid": "Le nom d'entité NLU n'est pas valide! Seuls `A-z`,` 0-9` et `_` sont autorisés.",
@@ -83,6 +83,8 @@
     "category_is_required": "La catégorie est requise",
     "attachment_is_required": "L'attachement est obligatoire",
     "success_import": "Le contenu a été importé avec succès!",
+    "import_failed": "Échec de l'importation",
+    "import_duplicated_data": "Les données existent déjà",
     "attachment_not_synced": "- En attente de Sync. -",
     "success_translation_refresh": "Les traductions ont été actualisées avec succès!",
     "message_tag_is_required": "Vous devez spécifier le tag de message.",
@@ -106,7 +108,7 @@
     "no_label_found": "Aucune étiquette trouvée",
     "code_is_required": "Le code est requis",
     "text_is_required": "Texte requis",
-    "invalid_file_type": "Type de fichier invalide",
+    "invalid_file_type": "Type de fichier invalide. Veuillez choisir un fichier dans un format pris en charge.",
     "select_category": "Sélectionner une catégorie"
   },
   "menu": {
@@ -341,6 +343,7 @@
     "precision": "Précision",
     "recall": "Rappel",
     "f1score": "F1-Score",
+    "all": "Tout",
     "train": "Apprentissage",
     "test": "Evaluation",
     "inbox": "Boîte de réception",

frontend/src/app-components/inputs/FileInput.tsx

@@ -0,0 +1,82 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import UploadIcon from "@mui/icons-material/Upload";
+import { Button, CircularProgress } from "@mui/material";
+import { ChangeEvent, forwardRef } from "react";
+
+import { useConfig } from "@/hooks/useConfig";
+import { useToast } from "@/hooks/useToast";
+import { useTranslate } from "@/hooks/useTranslate";
+
+import { Input } from "./Input";
+
+export type FileUploadButtonProps = {
+  label: string;
+  accept?: string;
+  onChange: (file: File) => void;
+  isLoading?: boolean;
+  error?: boolean;
+  helperText?: string;
+};
+
+const FileUploadButton = forwardRef<HTMLLabelElement, FileUploadButtonProps>(
+  ({ label, accept, isLoading = true, onChange }, ref) => {
+    const config = useConfig();
+    const { toast } = useToast();
+    const { t } = useTranslate();
+    const handleImportChange = async (event: ChangeEvent<HTMLInputElement>) => {
+      if (event.target.files?.length) {
+        const file = event.target.files.item(0);
+
+        if (!file) return false;
+
+        if (accept && !accept.split(",").includes(file.type)) {
+          toast.error(t("message.invalid_file_type"));
+
+          return false;
+        }
+
+        if (config.maxUploadSize && file.size > config.maxUploadSize) {
+          toast.error(t("message.file_max_size"));
+
+          return false;
+        }
+
+        onChange(file);
+      }
+    };
+
+    return (
+      <>
+        <Button
+          ref={ref}
+          htmlFor="importFile"
+          variant="contained"
+          component="label"
+          startIcon={<UploadIcon />}
+          endIcon={isLoading ? <CircularProgress size="1rem" /> : null}
+          disabled={isLoading}
+        >
+          {label}
+        </Button>
+        <Input
+          id="importFile"
+          type="file"
+          value="" // to trigger an automatic reset to allow the same file to be selected multiple times
+          sx={{ display: "none" }}
+          onChange={handleImportChange}
+        />
+      </>
+    );
+  },
+);
+
+FileUploadButton.displayName = "FileUploadButton";
+
+export default FileUploadButton;

frontend/src/components/nlp/components/NlpSample.tsx

@@ -7,9 +7,9 @@
  */
 
 import CircleIcon from "@mui/icons-material/Circle";
+import ClearIcon from "@mui/icons-material/Clear";
 import DeleteIcon from "@mui/icons-material/Delete";
 import DownloadIcon from "@mui/icons-material/Download";
-import UploadIcon from "@mui/icons-material/Upload";
 import {
   Box,
   Button,
@@ -17,15 +17,19 @@ import {
   Chip,
   Grid,
   IconButton,
+  InputAdornment,
   MenuItem,
   Stack,
+  Typography,
 } from "@mui/material";
 import { GridColDef, GridRowSelectionModel } from "@mui/x-data-grid";
 import { useState } from "react";
+import { useQueryClient } from "react-query";
 
 import { DeleteDialog } from "@/app-components/dialogs";
 import { ChipEntity } from "@/app-components/displays/ChipEntity";
 import AutoCompleteEntitySelect from "@/app-components/inputs/AutoCompleteEntitySelect";
+import FileUploadButton from "@/app-components/inputs/FileInput";
 import { FilterTextfield } from "@/app-components/inputs/FilterTextfield";
 import { Input } from "@/app-components/inputs/Input";
 import {
@@ -34,10 +38,12 @@ import {
 } from "@/app-components/tables/columns/getColumns";
 import { renderHeader } from "@/app-components/tables/columns/renderHeader";
 import { DataGrid } from "@/app-components/tables/DataGrid";
+import { isSameEntity } from "@/hooks/crud/helpers";
 import { useDelete } from "@/hooks/crud/useDelete";
 import { useDeleteMany } from "@/hooks/crud/useDeleteMany";
 import { useFind } from "@/hooks/crud/useFind";
 import { useGetFromCache } from "@/hooks/crud/useGet";
+import { useImport } from "@/hooks/crud/useImport";
 import { useConfig } from "@/hooks/useConfig";
 import { getDisplayDialogs, useDialog } from "@/hooks/useDialog";
 import { useHasPermission } from "@/hooks/useHasPermission";
@@ -60,6 +66,7 @@ import { NlpImportDialog } from "../NlpImportDialog";
 import { NlpSampleDialog } from "../NlpSampleDialog";
 
 const NLP_SAMPLE_TYPE_COLORS = {
+  all: "#fff",
   test: "#e6a23c",
   train: "#67c23a",
   inbox: "#909399",
@@ -69,7 +76,8 @@ export default function NlpSample() {
   const { apiUrl } = useConfig();
   const { toast } = useToast();
   const { t } = useTranslate();
-  const [type, setType] = useState<NlpSampleType | undefined>(undefined);
+  const queryClient = useQueryClient();
+  const [type, setType] = useState<NlpSampleType | "all">("all");
   const [language, setLanguage] = useState<string | undefined>(undefined);
   const hasPermission = useHasPermission();
   const getNlpEntityFromCache = useGetFromCache(EntityType.NLP_ENTITY);
@@ -79,7 +87,10 @@ export default function NlpSample() {
   );
   const getLanguageFromCache = useGetFromCache(EntityType.LANGUAGE);
   const { onSearch, searchPayload } = useSearch<INlpSample>({
-    $eq: [...(type ? [{ type }] : []), ...(language ? [{ language }] : [])],
+    $eq: [
+      ...(type !== "all" ? [{ type }] : []),
+      ...(language ? [{ language }] : []),
+    ],
     $iLike: ["text"],
   });
   const { mutateAsync: deleteNlpSample } = useDelete(EntityType.NLP_SAMPLE, {
@@ -104,6 +115,32 @@ export default function NlpSample() {
       },
     },
   );
+  const { mutateAsync: importDataset, isLoading } = useImport(
+    EntityType.NLP_SAMPLE,
+    {
+      onError: () => {
+        toast.error(t("message.import_failed"));
+      },
+      onSuccess: (data) => {
+        queryClient.removeQueries({
+          predicate: ({ queryKey }) => {
+            const [_qType, qEntity] = queryKey;
+
+            return (
+              isSameEntity(qEntity, EntityType.NLP_SAMPLE_ENTITY) ||
+              isSameEntity(qEntity, EntityType.NLP_ENTITY) ||
+              isSameEntity(qEntity, EntityType.NLP_VALUE)
+            );
+          },
+        });
+        if (data.length) {
+          toast.success(t("message.success_import"));
+        } else {
+          toast.error(t("message.import_duplicated_data"));
+        }
+      },
+    },
+  );
   const [selectedNlpSamples, setSelectedNlpSamples] = useState<string[]>([]);
   const { dataGridProps } = useFind(
     { entity: EntityType.NLP_SAMPLE, format: Format.FULL },
@@ -259,6 +296,9 @@ export default function NlpSample() {
   const handleSelectionChange = (selection: GridRowSelectionModel) => {
     setSelectedNlpSamples(selection as string[]);
   };
+  const handleImportChange = async (file: File) => {
+    await importDataset(file);
+  };
 
   return (
     <Grid item xs={12}>
@@ -292,7 +332,7 @@ export default function NlpSample() {
           <AutoCompleteEntitySelect<ILanguage, "title", false>
             fullWidth={false}
             sx={{
-              minWidth: "150px",
+              minWidth: "256px",
             }}
             autoFocus
             searchFields={["title", "code"]}
@@ -307,35 +347,38 @@ export default function NlpSample() {
             select
             fullWidth={false}
             sx={{
-              minWidth: "150px",
+              minWidth: "256px",
             }}
             label={t("label.dataset")}
             value={type}
             onChange={(e) => setType(e.target.value as NlpSampleType)}
             SelectProps={{
               ...(type && {
-                IconComponent: () => (
-                  <IconButton size="small" onClick={() => setType(undefined)}>
-                    <DeleteIcon />
-                  </IconButton>
+                endAdornment: (
+                  <InputAdornment sx={{ marginRight: "1rem" }} position="end">
+                    <IconButton size="small" onClick={() => setType("all")}>
+                      <ClearIcon sx={{ fontSize: "1.25rem" }} />
+                    </IconButton>
+                  </InputAdornment>
                 ),
               }),
               renderValue: (value) => <Box>{t(`label.${value}`)}</Box>,
             }}
           >
-            {Object.values(NlpSampleType).map((nlpSampleType, index) => (
-              <MenuItem key={index} value={nlpSampleType}>
-                <Grid container>
-                  <Grid item xs={4}>
+            {["all", ...Object.values(NlpSampleType)].map(
+              (nlpSampleType, index) => (
+                <MenuItem key={index} value={nlpSampleType}>
+                  <Box display="flex" gap={1}>
                     <CircleIcon
-                      fontSize="small"
-                      sx={{ color: NLP_SAMPLE_TYPE_COLORS[nlpSampleType] }}
+                      sx={{
+                        color: NLP_SAMPLE_TYPE_COLORS[nlpSampleType],
+                      }}
                     />
-                  </Grid>
-                  <Grid item>{nlpSampleType}</Grid>
-                </Grid>
-              </MenuItem>
-            ))}
+                    <Typography>{t(`label.${nlpSampleType}`)}</Typography>
+                  </Box>
+                </MenuItem>
+              ),
+            )}
           </Input>
           <ButtonGroup sx={{ marginLeft: "auto" }}>
             {hasPermission(EntityType.NLP_SAMPLE, PermissionAction.CREATE) &&
@@ -343,13 +386,12 @@ export default function NlpSample() {
               EntityType.NLP_SAMPLE_ENTITY,
               PermissionAction.CREATE,
             ) ? (
-              <Button
-                variant="contained"
-                onClick={() => importDialogCtl.openDialog()}
-                startIcon={<UploadIcon />}
-              >
-                {t("button.import")}
-              </Button>
+              <FileUploadButton
+                accept="text/csv"
+                label={t("button.import")}
+                onChange={handleImportChange}
+                isLoading={isLoading}
+              />
             ) : null}
             {hasPermission(EntityType.NLP_SAMPLE, PermissionAction.READ) &&
             hasPermission(

frontend/src/contexts/config.context.tsx

@@ -13,6 +13,7 @@ export const ConfigContext = createContext<IConfig | null>(null);
 export interface IConfig {
   apiUrl: string;
   ssoEnabled: boolean;
+  maxUploadSize: number;
 }
 
 export const ConfigProvider = ({ children }) => {

frontend/src/hooks/crud/useImport.tsx

@@ -0,0 +1,59 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { useMutation, useQueryClient } from "react-query";
+
+import { QueryType, TMutationOptions } from "@/services/types";
+import { IBaseSchema, IDynamicProps, TType } from "@/types/base.types";
+
+import { useEntityApiClient } from "../useApiClient";
+
+import { isSameEntity, useNormalizeAndCache } from "./helpers";
+
+export const useImport = <
+  TEntity extends IDynamicProps["entity"],
+  TAttr extends File = File,
+  TBasic extends IBaseSchema = TType<TEntity>["basic"],
+>(
+  entity: TEntity,
+  options: Omit<
+    TMutationOptions<TBasic[], Error, TAttr, TBasic[]>,
+    "mutationFn" | "mutationKey"
+  > = {},
+) => {
+  const api = useEntityApiClient<TAttr, TBasic>(entity);
+  const queryClient = useQueryClient();
+  const normalizeAndCache = useNormalizeAndCache<TBasic, string[], TBasic>(
+    entity,
+  );
+  const { invalidate = true, ...rest } = options;
+
+  return useMutation({
+    mutationFn: async (variables) => {
+      const data = await api.import(variables);
+      const { result, entities } = normalizeAndCache(data);
+
+      // Invalidate current entity count and collection
+      if (invalidate) {
+        queryClient.invalidateQueries({
+          predicate: ({ queryKey }) => {
+            const [qType, qEntity] = queryKey;
+
+            return (
+              (qType === QueryType.count || qType === QueryType.collection) &&
+              isSameEntity(qEntity, entity)
+            );
+          },
+        });
+      }
+
+      return result.map((id) => entities[entity][id]);
+    },
+    ...rest,
+  });
+};

frontend/src/pages/api/config.ts

@@ -11,6 +11,7 @@ import type { NextApiRequest, NextApiResponse } from "next";
 type ResponseData = {
   apiUrl: string;
   ssoEnabled: boolean;
+  maxUploadSize: number;
 };
 
 export default function handler(
@@ -20,5 +21,8 @@ export default function handler(
   res.status(200).json({
     apiUrl: process.env.NEXT_PUBLIC_API_ORIGIN || "http://localhost:4000",
     ssoEnabled: process.env.NEXT_PUBLIC_SSO_ENABLED === "true" || false,
+    maxUploadSize: process.env.UPLOAD_MAX_SIZE_IN_BYTES
+      ? Number(process.env.UPLOAD_MAX_SIZE_IN_BYTES)
+      : 50 * 1024 * 1024, // 50 MB in bytes
   });
 }

frontend/src/services/api.class.ts

@@ -269,6 +269,20 @@ export class EntityApiClient<TAttr, TBasic, TFull> extends ApiClient {
     return data;
   }
 
+  async import<T = TBasic>(file: File) {
+    const { _csrf } = await this.getCsrf();
+    const formData = new FormData();
+
+    formData.append("file", file);
+
+    const { data } = await this.request.post<T[], AxiosResponse<T[]>, FormData>(
+      `${ROUTES[this.type]}/import?_csrf=${_csrf}`,
+      formData,
+    );
+
+    return data;
+  }
+
   async upload(file: File) {
     const { _csrf } = await this.getCsrf();
     const formData = new FormData();