UniqAI/hexabot
1
0
Fork 0
mirror of https://github.com/hexastack/hexabot synced 2025-06-26 18:27:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: enhacement of content import

Browse Source
This commit is contained in:
Emnaghz
2025-02-19 18:03:58 +01:00
parent 06c999b7f3
commit 82f46ad65e
9 changed files with 204 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
api/src/cms/controllers/content.controller.spec.ts
View File

@@ -1,24 +1,16 @@
/*
* Copyright © 2024 Hexastack. All rights reserved.
* Copyright © 2025 Hexastack. All rights reserved.
*
* Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
* 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
* 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
*/
import fs from 'fs';
import { NotFoundException } from '@nestjs/common/exceptions';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { MongooseModule } from '@nestjs/mongoose';
import { Test, TestingModule } from '@nestjs/testing';
import { AttachmentRepository } from '@/attachment/repositories/attachment.repository';
import {
Attachment,
AttachmentModel,
} from '@/attachment/schemas/attachment.schema';
import { AttachmentService } from '@/attachment/services/attachment.service';
import { LoggerService } from '@/logger/logger.service';
import { NOT_FOUND_ID } from '@/utils/constants/mock';
import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
@@ -48,10 +40,8 @@ describe('ContentController', () => {
let contentController: ContentController;
let contentService: ContentService;
let contentTypeService: ContentTypeService;
let attachmentService: AttachmentService;
let contentType: ContentType | null;
let content: Content | null;
let attachment: Attachment | null;
let updatedContent;
let pageQuery: PageQueryDto<Content>;
@@ -60,34 +50,24 @@ describe('ContentController', () => {
controllers: [ContentController],
imports: [
rootMongooseTestModule(installContentFixtures),
MongooseModule.forFeature([
ContentTypeModel,
ContentModel,
AttachmentModel,
]),
MongooseModule.forFeature([ContentTypeModel, ContentModel]),
],
providers: [
LoggerService,
ContentTypeService,
ContentService,
ContentRepository,
AttachmentService,
ContentTypeRepository,
AttachmentRepository,
EventEmitter2,
],
}).compile();
contentController = module.get<ContentController>(ContentController);
contentService = module.get<ContentService>(ContentService);
attachmentService = module.get<AttachmentService>(AttachmentService);
contentTypeService = module.get<ContentTypeService>(ContentTypeService);
contentType = await contentTypeService.findOne({ name: 'Product' });
content = await contentService.findOne({
title: 'Jean',
});
attachment = await attachmentService.findOne({
name: 'store1.jpg',
});
pageQuery = getPageQuery<Content>({
limit: 1,
@@ -243,91 +223,76 @@ describe('ContentController', () => {
});
describe('import', () => {
const mockCsvData: string = `other,title,status,image
should not appear,store 3,true,image.jpg`;
const file: Express.Multer.File = {
buffer: Buffer.from(mockCsvData, 'utf-8'),
originalname: 'test.csv',
mimetype: 'text/csv',
size: mockCsvData.length,
fieldname: 'file',
encoding: '7bit',
stream: null,
destination: '',
filename: '',
path: '',
} as unknown as Express.Multer.File;
it('should import content from a CSV file', async () => {
const mockCsvData: string = `other,title,status,image
should not appear,store 3,true,image.jpg`;
const mockCsvContentDto: ContentCreateDto = {
entity: '0',
title: 'store 3',
status: true,
dynamicFields: {
image: 'image.jpg',
},
};
jest.spyOn(contentService, 'createMany');
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(true);
jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(mockCsvData);
const contentType = await contentTypeService.findOne({
const mockContentType = {
id: '0',
name: 'Store',
});
const result = await contentController.import({
idFileToImport: attachment!.id,
idTargetContentType: contentType!.id,
});
expect(contentService.createMany).toHaveBeenCalledWith([
{ ...mockCsvContentDto, entity: contentType!.id },
} as unknown as ContentType;
jest
.spyOn(contentTypeService, 'findOne')
.mockResolvedValueOnce(mockContentType);
jest.spyOn(contentService, 'parseAndSaveDataset').mockResolvedValueOnce([
{
entity: mockContentType.id,
title: 'store 3',
status: true,
dynamicFields: {
image: 'image.jpg',
},
id: '',
createdAt: null as unknown as Date,
updatedAt: null as unknown as Date,
},
]);
expect(result).toEqualPayload(
[
{
...mockCsvContentDto,
entity: contentType!.id,
},
],
[...IGNORED_TEST_FIELDS, 'rag'],
const result = await contentController.import(file, mockContentType.id);
expect(contentService.parseAndSaveDataset).toHaveBeenCalledWith(
mockCsvData,
mockContentType.id,
mockContentType,
);
expect(result).toEqual([
{
entity: mockContentType.id,
title: 'store 3',
status: true,
dynamicFields: {
image: 'image.jpg',
},
id: '',
createdAt: null as unknown as Date,
updatedAt: null as unknown as Date,
},
]);
});
it('should throw NotFoundException if content type is not found', async () => {
jest.spyOn(contentTypeService, 'findOne').mockResolvedValueOnce(null);
await expect(
contentController.import({
idFileToImport: attachment!.id,
idTargetContentType: NOT_FOUND_ID,
}),
contentController.import(file, 'INVALID_ID'),
).rejects.toThrow(new NotFoundException('Content type is not found'));
});
it('should throw NotFoundException if file is not found in attachment database', async () => {
const contentType = await contentTypeService.findOne({
name: 'Product',
});
jest.spyOn(contentTypeService, 'findOne');
await expect(
contentController.import({
idFileToImport: NOT_FOUND_ID,
idTargetContentType: contentType!.id.toString(),
}),
).rejects.toThrow(new NotFoundException('File does not exist'));
it('should throw NotFoundException if idTargetContentType is missing', async () => {
await expect(contentController.import(file, '')).rejects.toThrow(
new NotFoundException('Missing parameter'),
);
});
it('should throw NotFoundException if file does not exist in the given path ', async () => {
jest.spyOn(fs, 'existsSync').mockReturnValue(false);
await expect(
contentController.import({
idFileToImport: attachment!.id,
idTargetContentType: contentType!.id,
}),
).rejects.toThrow(new NotFoundException('File does not exist'));
});
it.each([
['file param and content type params are missing', '', ''],
['content type param is missing', '', NOT_FOUND_ID],
['file param is missing', NOT_FOUND_ID, ''],
])(
'should throw NotFoundException if %s',
async (_message, fileToImport, targetContentType) => {
await expect(
contentController.import({
idFileToImport: fileToImport,
idTargetContentType: targetContentType,
}),
).rejects.toThrow(new NotFoundException('Missing params'));
},
);
});
});

94
api/src/cms/controllers/content.controller.ts
View File

@@ -1,14 +1,11 @@
/*
* Copyright © 2024 Hexastack. All rights reserved.
* Copyright © 2025 Hexastack. All rights reserved.
*
* Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
* 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
* 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
*/
import fs from 'fs';
import path from 'path';
import {
Body,
Controller,
@@ -20,14 +17,12 @@ import {
Patch,
Post,
Query,
UploadedFile,
UseInterceptors,
} from '@nestjs/common';
import { BadRequestException } from '@nestjs/common/exceptions';
import { FileInterceptor } from '@nestjs/platform-express';
import { CsrfCheck } from '@tekuconcept/nestjs-csrf';
import Papa from 'papaparse';
import { AttachmentService } from '@/attachment/services/attachment.service';
import { config } from '@/config';
import { CsrfInterceptor } from '@/interceptors/csrf.interceptor';
import { LoggerService } from '@/logger/logger.service';
import { BaseController } from '@/utils/generics/base-controller';
@@ -59,7 +54,6 @@ export class ContentController extends BaseController<
constructor(
private readonly contentService: ContentService,
private readonly contentTypeService: ContentTypeService,
private readonly attachmentService: AttachmentService,
private readonly logger: LoggerService,
) {
super(contentService);
@@ -92,29 +86,22 @@ export class ContentController extends BaseController<
/**
* Imports content from a CSV file based on the provided content type and file ID.
*
* @param idTargetContentType - The content type to match the CSV data against.
* @param idFileToImport - The ID of the file to be imported.
*
* @param idTargetContentType - The content type to match the CSV data against. *
* @returns A promise that resolves to the newly created content documents.
*/
@Get('import/:idTargetContentType/:idFileToImport')
@CsrfCheck(true)
@Post('import')
@UseInterceptors(FileInterceptor('file'))
async import(
@Param()
{
idTargetContentType: targetContentType,
idFileToImport: fileToImport,
}: {
idTargetContentType: string;
idFileToImport: string;
},
@UploadedFile() file: Express.Multer.File,
@Query('idTargetContentType')
targetContentType: string,
) {
// Check params
if (!fileToImport || !targetContentType) {
this.logger.warn(`Parameters are missing`);
throw new NotFoundException(`Missing params`);
const datasetContent = file.buffer.toString('utf-8');
if (!targetContentType) {
this.logger.warn(`Parameter is missing`);
throw new NotFoundException(`Missing parameter`);
}
// Find the content type that corresponds to the given content
const contentType =
await this.contentTypeService.findOne(targetContentType);
if (!contentType) {
@@ -124,56 +111,11 @@ export class ContentController extends BaseController<
throw new NotFoundException(`Content type is not found`);
}
// Get file location
const file = await this.attachmentService.findOne(fileToImport);
// Check if file is present
const filePath = file
? path.join(config.parameters.uploadDir, file.location)
: undefined;
if (!file || !filePath || !fs.existsSync(filePath)) {
this.logger.warn(`Failed to find file type with id ${fileToImport}.`);
throw new NotFoundException(`File does not exist`);
}
//read file sync
const data = fs.readFileSync(filePath, 'utf8');
const result = Papa.parse<Record<string, string | boolean | number>>(data, {
header: true,
skipEmptyLines: true,
dynamicTyping: true,
});
if (result.errors.length > 0) {
this.logger.warn(
`Errors parsing the file: ${JSON.stringify(result.errors)}`,
);
throw new BadRequestException(result.errors, {
cause: result.errors,
description: 'Error while parsing CSV',
});
}
const contentsDto = result.data.reduce(
(acc, { title, status, ...rest }) => [
...acc,
{
title: String(title),
status: Boolean(status),
entity: targetContentType,
dynamicFields: Object.keys(rest)
.filter((key) =>
contentType.fields?.map((field) => field.name).includes(key),
)
.reduce((filtered, key) => ({ ...filtered, [key]: rest[key] }), {}),
},
],
[],
return await this.contentService.parseAndSaveDataset(
datasetContent,
targetContentType,
contentType,
);
// Create content
return await this.contentService.createMany(contentsDto);
}
/**

11
api/src/cms/services/content.service.spec.ts
View File

@@ -10,9 +10,6 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
import { MongooseModule } from '@nestjs/mongoose';
import { Test, TestingModule } from '@nestjs/testing';
import { AttachmentRepository } from '@/attachment/repositories/attachment.repository';
import { AttachmentModel } from '@/attachment/schemas/attachment.schema';
import { AttachmentService } from '@/attachment/services/attachment.service';
import { OutgoingMessageFormat } from '@/chat/schemas/types/message';
import { ContentOptions } from '@/chat/schemas/types/options';
import { LoggerService } from '@/logger/logger.service';
@@ -44,19 +41,13 @@ describe('ContentService', () => {
const module: TestingModule = await Test.createTestingModule({
imports: [
rootMongooseTestModule(installContentFixtures),
MongooseModule.forFeature([
ContentTypeModel,
ContentModel,
AttachmentModel,
]),
MongooseModule.forFeature([ContentTypeModel, ContentModel]),
],
providers: [
ContentTypeRepository,
ContentRepository,
AttachmentRepository,
ContentTypeService,
ContentService,
AttachmentService,
LoggerService,
EventEmitter2,
],

70
api/src/cms/services/content.service.ts
View File

@@ -6,9 +6,9 @@
* 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
*/
import { Injectable } from '@nestjs/common';
import { BadRequestException, Injectable } from '@nestjs/common';
import Papa from 'papaparse';
import { AttachmentService } from '@/attachment/services/attachment.service';
import { StdOutgoingListMessage } from '@/chat/schemas/types/message';
import { ContentOptions } from '@/chat/schemas/types/options';
import { LoggerService } from '@/logger/logger.service';
@@ -17,6 +17,7 @@ import { TFilterQuery } from '@/utils/types/filter.types';
import { ContentDto } from '../dto/content.dto';
import { ContentRepository } from '../repositories/content.repository';
import { ContentType } from '../schemas/content-type.schema';
import {
Content,
ContentFull,
@@ -32,7 +33,6 @@ export class ContentService extends BaseService<
> {
constructor(
readonly repository: ContentRepository,
private readonly attachmentService: AttachmentService,
private readonly logger: LoggerService,
) {
super(repository);
@@ -103,4 +103,68 @@ export class ContentService extends BaseService<
throw err;
}
}
/**
* Parses a CSV dataset and saves the content in the repository.
*
* @param data - The CSV data as a string to be parsed.
* @param targetContentType - The content type to associate with the parsed data.
* @param contentType - The content type metadata, including fields to validate the parsed data.
* @return A promise resolving to the created content objects.
*/
async parseAndSaveDataset(
data: string,
targetContentType: string,
contentType: ContentType,
) {
// Parse local CSV file
const result: {
errors: any[];
data: Array<Record<string, string>>;
} = Papa.parse(data, {
header: true,
skipEmptyLines: true,
});
if (result.errors && result.errors.length > 0) {
this.logger.warn(
`Errors parsing the file: ${JSON.stringify(result.errors)}`,
);
throw new BadRequestException(result.errors, {
cause: result.errors,
description: 'Error while parsing CSV',
});
}
if (!result.data.every((row) => row.title && row.status)) {
throw new BadRequestException(
'Missing required fields: "title" or "status"',
{
cause: 'Invalid CSV data',
description: 'CSV must include "title" and "status" columns',
},
);
}
const contentsDto = result.data.reduce(
(acc, { title, status, ...rest }) => [
...acc,
{
title: String(title),
status: Boolean(status),
entity: targetContentType,
dynamicFields: Object.keys(rest)
.filter((key) =>
contentType.fields?.map((field) => field.name).includes(key),
)
.reduce((filtered, key) => ({ ...filtered, [key]: rest[key] }), {}),
},
],
[],
);
this.logger.log(`Parsed ${result.data.length} rows from CSV.`);
try {
return await this.createMany(contentsDto);
} catch (err) {
this.logger.error('Error occurred when extracting data. ', err);
}
}
}