diff --git a/api/src/attachment/dto/attachment.dto.ts b/api/src/attachment/dto/attachment.dto.ts index a461a6f9..ce180dc0 100644 --- a/api/src/attachment/dto/attachment.dto.ts +++ b/api/src/attachment/dto/attachment.dto.ts @@ -9,54 +9,57 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { Type } from 'class-transformer'; import { + IsNotEmpty, IsObject, IsOptional, - MaxLength, - IsNotEmpty, IsString, + MaxLength, } from 'class-validator'; +import { ChannelName } from '@/channel/types'; import { ObjectIdDto } from '@/utils/dto/object-id.dto'; -export class AttachmentCreateDto { +export class AttachmentMetadataDto { /** - * Attachment channel + * Attachment original file name */ - @ApiPropertyOptional({ description: 'Attachment channel', type: Object }) - @IsNotEmpty() - @IsObject() - channel?: Partial>; - - /** - * Attachment location - */ - @ApiProperty({ description: 'Attachment location', type: String }) - @IsNotEmpty() - @IsString() - location: string; - - /** - * Attachment name - */ - @ApiProperty({ description: 'Attachment name', type: String }) + @ApiProperty({ description: 'Attachment original file name', type: String }) @IsNotEmpty() @IsString() name: string; /** - * Attachment size + * Attachment size in bytes */ - @ApiProperty({ description: 'Attachment size', type: Number }) + @ApiProperty({ description: 'Attachment size in bytes', type: Number }) @IsNotEmpty() size: number; /** - * Attachment type + * Attachment MIME type */ - @ApiProperty({ description: 'Attachment type', type: String }) + @ApiProperty({ description: 'Attachment MIME type', type: String }) @IsNotEmpty() @IsString() type: string; + + /** + * Attachment specia channel(s) metadata + */ + @ApiPropertyOptional({ description: 'Attachment channel', type: Object }) + @IsNotEmpty() + @IsObject() + channel?: Partial>; +} + +export class AttachmentCreateDto extends AttachmentMetadataDto { + /** + * Attachment location (file would/should be stored under a unique name) + */ + @ApiProperty({ description: 'Attachment location', type: String }) + @IsNotEmpty() + @IsString() + location: string; } export class AttachmentDownloadDto extends ObjectIdDto { diff --git a/api/src/attachment/mocks/attachment.mock.ts b/api/src/attachment/mocks/attachment.mock.ts index 3af4ef32..34966a39 100644 --- a/api/src/attachment/mocks/attachment.mock.ts +++ b/api/src/attachment/mocks/attachment.mock.ts @@ -42,7 +42,7 @@ export const attachments: Attachment[] = [ size: 343370, location: '/app/src/attachment/uploads/Screenshot from 2022-03-11 08-41-27-2a9799a8b6109c88fd9a7a690c1101934c.png', - channel: { dimelo: {} }, + channel: { 'web-channel': {} }, id: '65940d115178607da65c82b7', createdAt: new Date(), updatedAt: new Date(), @@ -53,7 +53,7 @@ export const attachments: Attachment[] = [ size: 33829, location: '/app/src/attachment/uploads/Screenshot from 2022-03-18 08-58-15-af61e7f71281f9fd3f1ad7ad10107741c.png', - channel: { dimelo: {} }, + channel: { 'web-channel': {} }, id: '65940d115178607da65c82b8', createdAt: new Date(), updatedAt: new Date(), diff --git a/api/src/attachment/services/attachment.service.ts b/api/src/attachment/services/attachment.service.ts index 52ad7d4b..ccbd0712 100644 --- a/api/src/attachment/services/attachment.service.ts +++ b/api/src/attachment/services/attachment.service.ts @@ -6,8 +6,9 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import fs, { createReadStream } from 'fs'; +import fs, { createReadStream, promises as fsPromises } from 'fs'; import path, { join } from 'path'; +import { Readable } from 'stream'; import { Injectable, @@ -24,9 +25,14 @@ import { PluginService } from '@/plugins/plugins.service'; import { PluginType } from '@/plugins/types'; import { BaseService } from '@/utils/generics/base-service'; +import { AttachmentMetadataDto } from '../dto/attachment.dto'; import { AttachmentRepository } from '../repositories/attachment.repository'; import { Attachment } from '../schemas/attachment.schema'; -import { fileExists, getStreamableFile } from '../utilities'; +import { + fileExists, + generateUniqueFilename, + getStreamableFile, +} from '../utilities'; @Injectable() export class AttachmentService extends BaseService { @@ -180,6 +186,50 @@ export class AttachmentService extends BaseService { return uploadedFiles; } + /** + * Uploads files to the server. If a storage plugin is configured it uploads files accordingly. + * Otherwise, uploads files to the local directory. + * + * @param file - The file path or the Buffer / Readable + * @returns A promise that resolves to an array of uploaded attachments. + */ + async store( + file: Buffer | Readable | string, + metadata: AttachmentMetadataDto, + ): Promise { + if (this.getStoragePlugin()) { + const storedDto = await this.getStoragePlugin().store(file, metadata); + return await this.create(storedDto); + } else { + const dirPath = path.join(config.parameters.uploadDir); + const uniqueFilename = generateUniqueFilename(metadata.name); + const filePath = path.resolve(dirPath, uniqueFilename); + + if (typeof file === 'string') { + // For example, if the file is an instance of `Express.Multer.File` (diskStorage case) + await fsPromises.copyFile(file, filePath); + await fsPromises.unlink(file); + } else if (Buffer.isBuffer(file)) { + await fsPromises.writeFile(filePath, file); + } else if (file instanceof Readable) { + await new Promise((resolve, reject) => { + const writeStream = fs.createWriteStream(filePath); + file.pipe(writeStream); + writeStream.on('finish', resolve); + writeStream.on('error', reject); + }); + } else { + throw new TypeError('Unrecognized file object'); + } + + const location = filePath.replace(dirPath, ''); + return await this.create({ + ...metadata, + location, + }); + } + } + /** * Downloads an attachment identified by the provided parameters. * diff --git a/api/src/attachment/utilities/index.ts b/api/src/attachment/utilities/index.ts index 3395030a..874ff0a0 100644 --- a/api/src/attachment/utilities/index.ts +++ b/api/src/attachment/utilities/index.ts @@ -7,19 +7,31 @@ */ import { createReadStream, existsSync } from 'fs'; -import { join } from 'path'; +import { extname, join } from 'path'; import { Logger, StreamableFile } from '@nestjs/common'; import { StreamableFileOptions } from '@nestjs/common/file-stream/interfaces/streamable-options.interface'; +import { v4 as uuidv4 } from 'uuid'; import { config } from '@/config'; export const MIME_REGEX = /^[a-z-]+\/[0-9a-z\-.]+$/gm; +/** + * Validates if a given string matches the MIME type format. + * + * @param type The string to validate. + * @returns Whether the string is a valid MIME type. + */ export const isMime = (type: string): boolean => { return MIME_REGEX.test(type); }; +/** + * Checks if a file exists in the specified upload directory. + * @param location The relative location of the file. + * @returns Whether the file exists. + */ export const fileExists = (location: string): boolean => { // bypass test env if (config.env === 'test') { @@ -35,6 +47,12 @@ export const fileExists = (location: string): boolean => { } }; +/** + * Creates a streamable file from a given file path and options. + * + * @param options The object containing the file path and optional settings. + * @returns A streamable file object. + */ export const getStreamableFile = ({ path, options, @@ -50,3 +68,15 @@ export const getStreamableFile = ({ return new StreamableFile(fileReadStream, options); }; + +/** + * Generates a unique filename by appending a UUID to the original name. + * + * @param originalname The original filename. + * @returns A unique filename. + */ +export const generateUniqueFilename = (originalname: string) => { + const extension = extname(originalname); + const name = originalname.slice(0, -extension.length); + return `${name}-${uuidv4()}${extension}`; +}; diff --git a/api/src/extensions/channels/web/base-web-channel.ts b/api/src/extensions/channels/web/base-web-channel.ts index 0f42f9f1..daecbd8f 100644 --- a/api/src/extensions/channels/web/base-web-channel.ts +++ b/api/src/extensions/channels/web/base-web-channel.ts @@ -6,14 +6,10 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import { promises as fsPromises } from 'fs'; -import path from 'path'; - import { Injectable } from '@nestjs/common'; import { EventEmitter2, OnEvent } from '@nestjs/event-emitter'; -import { Request, Response } from 'express'; -import multer, { diskStorage } from 'multer'; -import sanitize from 'sanitize-filename'; +import { NextFunction, Request, Response } from 'express'; +import multer, { diskStorage, memoryStorage } from 'multer'; import { Socket } from 'socket.io'; import { v4 as uuidv4 } from 'uuid'; @@ -31,7 +27,6 @@ import { Button, ButtonType } from '@/chat/schemas/types/button'; import { AnyMessage, ContentElement, - FileType, IncomingMessage, OutgoingMessage, OutgoingMessageFormat, @@ -597,133 +592,104 @@ export default abstract class BaseWebChannelHandler< /** * Upload file as attachment if provided * - * @param upload - * @param filename - */ - private async storeAttachment( - upload: Omit, - filename: string, - next: ( - err: Error | null, - payload: { type: string; url: string } | false, - ) => void, - ): Promise { - try { - this.logger.debug('Web Channel Handler : Successfully uploaded file'); - - const attachment = await this.attachmentService.create({ - name: upload.name || '', - type: upload.type || 'text/txt', - size: upload.size || 0, - location: filename, - channel: { web: {} }, - }); - - this.logger.debug( - 'Web Channel Handler : Successfully stored file as attachment', - ); - - next(null, { - type: Attachment.getTypeByMime(attachment.type), - url: Attachment.getAttachmentUrl(attachment.id, attachment.name), - }); - } catch (err) { - this.logger.error( - 'Web Channel Handler : Unable to store uploaded file as attachment', - err, - ); - next(err, false); - } - } - - /** - * Upload file as attachment if provided - * - * @param req - * @param res + * @param req Either a HTTP Express request or a WS request (Synthetic Object) + * @param res Either a HTTP Express response or a WS response (Synthetic Object) + * @param next Callback Function */ async handleFilesUpload( req: Request | SocketRequest, res: Response | SocketResponse, next: ( err: null | Error, - result: { type: string; url: string } | false, + result?: Web.IncomingAttachmentMessageData, ) => void, ): Promise { - const data: Web.IncomingMessage = req.body; // Check if any file is provided if (!req.session.web) { this.logger.debug('Web Channel Handler : No session provided'); - return next(null, false); - } - // Check if any file is provided - if (!data || !data.type || data.type !== 'file') { - this.logger.debug('Web Channel Handler : No files provided'); - return next(null, false); - } - - // Parse json form data (in case of content-type multipart/form-data) - data.data = - typeof data.data === 'string' ? JSON.parse(data.data) : data.data; - - // Check max size upload - const upload = data.data; - if (typeof upload.size === 'undefined') { - return next(new Error('File upload probably failed.'), false); - } - - // Store file as attachment - const dirPath = path.join(config.parameters.uploadDir); - const sanitizedFilename = sanitize( - `${req.session.web.profile.id}_${+new Date()}_${upload.name}`, - ); - const filePath = path.resolve(dirPath, sanitizedFilename); - - if (!filePath.startsWith(dirPath)) { - return next(new Error('Invalid file path!'), false); + return next(null); } if (this.isSocketRequest(req)) { - // @TODO : test this try { - await fsPromises.writeFile(filePath, upload.file); - this.storeAttachment(upload, sanitizedFilename, next); + const { type, data } = req.body as Web.IncomingMessage; + + // Check if any file is provided + if (type !== 'file' || !('file' in data) || !data.file) { + this.logger.debug('Web Channel Handler : No files provided'); + return next(null); + } + + const size = Buffer.byteLength(data.file); + + if (size > config.parameters.maxUploadSize) { + return next(new Error('Max upload size has been exceeded')); + } + + const attachment = await this.attachmentService.store(data.file, { + name: data.name, + size: Buffer.byteLength(data.file), + type: data.type, + }); + next(null, { + type: Attachment.getTypeByMime(attachment.type), + url: Attachment.getAttachmentUrl(attachment.id, attachment.name), + }); } catch (err) { this.logger.error( 'Web Channel Handler : Unable to write uploaded file', err, ); - return next(new Error('Unable to upload file!'), false); + return next(new Error('Unable to upload file!')); } } else { const upload = multer({ - storage: diskStorage({ - destination: dirPath, // Set the destination directory for file storage - filename: (_req, _file, cb) => { - cb(null, sanitizedFilename); // Set the file name - }, - }), + limits: { + fileSize: config.parameters.maxUploadSize, + }, + storage: (() => { + if (config.parameters.storageMode === 'memory') { + return memoryStorage(); + } else { + return diskStorage({}); + } + })(), }).single('file'); // 'file' is the field name in the form - upload(req as Request, res as Response, (err) => { + upload(req as Request, res as Response, async (err?: any) => { if (err) { this.logger.error( 'Web Channel Handler : Unable to write uploaded file', err, ); - return next(new Error('Unable to upload file!'), false); + return next(new Error('Unable to upload file!')); + } + + // Check if any file is provided + if (!req.file) { + this.logger.debug('Web Channel Handler : No files provided'); + return next(null); + } + + try { + const file = req.file; + const attachment = await this.attachmentService.store( + config.parameters.storageMode === 'memory' + ? file.buffer + : file.path, + { + name: file.originalname, + size: file.size, + type: file.mimetype, + }, + ); + next(null, { + type: Attachment.getTypeByMime(attachment.type), + url: Attachment.getAttachmentUrl(attachment.id, attachment.name), + }); + } catch (err) { + next(err); } - // @TODO : test upload - const file = req.file; - this.storeAttachment( - { - name: file.filename, - type: file.mimetype as FileType, // @Todo : test this - size: file.size, - }, - file.path.replace(dirPath, ''), - next, - ); }); } } @@ -751,7 +717,7 @@ export default abstract class BaseWebChannelHandler< /** * Return subscriber channel specific attributes * - * @param req + * @param req Either a HTTP Express request or a WS request (Synthetic Object) * * @returns The subscriber channel's attributes */ @@ -766,22 +732,44 @@ export default abstract class BaseWebChannelHandler< } /** - * Handle channel event (probably a message) - * + * Custom channel middleware * @param req * @param res + * @param next + */ + async middleware(_req: Request, _res: Response, next: NextFunction) { + // Do nothing, override in channel + next(); + } + + /** + * Handle channel event (probably a message) + * + * @param req Either a HTTP Express request or a WS request (Synthetic Object) + * @param res Either a HTTP Express response or a WS response (Synthetic Object) */ _handleEvent( req: Request | SocketRequest, res: Response | SocketResponse, ): void { - const data: Web.IncomingMessage = req.body; + // @TODO: perform payload validation + if (!req.body) { + this.logger.debug('Web Channel Handler : Empty body'); + res.status(400).json({ err: 'Web Channel Handler : Bad Request!' }); + return; + } else { + // Parse json form data (in case of content-type multipart/form-data) + req.body.data = + typeof req.body.data === 'string' + ? JSON.parse(req.body.data) + : req.body.data; + } + this.validateSession(req, res, (profile) => { this.handleFilesUpload( req, res, - // @ts-expect-error @TODO : This needs to be fixed at a later point @TODO - (err: Error, upload: Web.IncomingMessageData) => { + (err: Error, data?: Web.IncomingAttachmentMessageData) => { if (err) { this.logger.warn( 'Web Channel Handler : Unable to upload file ', @@ -792,14 +780,18 @@ export default abstract class BaseWebChannelHandler< .json({ err: 'Web Channel Handler : File upload failed!' }); } // Set data in file upload case - if (upload) { - data.data = upload; - } + const body: Web.IncomingMessage = data + ? { + ...req.body, + data, + } + : req.body; + const channelAttrs = this.getChannelAttributes(req); - const event = new WebEventWrapper(this, data, channelAttrs); + const event = new WebEventWrapper(this, body, channelAttrs); if (event.getEventType() === 'message') { // Handler sync message sent by chabbot - if (data.sync && data.author === 'chatbot') { + if (body.sync && body.author === 'chatbot') { const sentMessage: MessageCreateDto = { mid: event.getId(), message: event.getMessage() as StdOutgoingMessage, diff --git a/api/src/extensions/channels/web/types.ts b/api/src/extensions/channels/web/types.ts index e8e0180e..a650c1c0 100644 --- a/api/src/extensions/channels/web/types.ts +++ b/api/src/extensions/channels/web/types.ts @@ -59,14 +59,19 @@ export namespace Web { }; }; - export type IncomingAttachmentMessageData = { - type: FileType; // mime type in a file case - url: string; // file url - // Only when uploaded - size?: number; // file size - name?: string; - file?: any; - }; + // Depending if it's has been processed or not + export type IncomingAttachmentMessageData = + // After upload and attachment is processed + | { + type: FileType; + url: string; // file download url + } // Before upload and attachment is processed + | { + type: string; // mime type + size: number; // file size + name: string; + file: Buffer; + }; export type IncomingMessageData = | IncomingTextMessageData diff --git a/api/src/extensions/channels/web/wrapper.ts b/api/src/extensions/channels/web/wrapper.ts index b8c7cc7a..6ae44fa7 100644 --- a/api/src/extensions/channels/web/wrapper.ts +++ b/api/src/extensions/channels/web/wrapper.ts @@ -216,6 +216,10 @@ export default class WebEventWrapper extends EventWrapper }; } case IncomingMessageType.attachments: + if (!('url' in this._adapter.raw.data)) { + throw new Error('Attachment has not been processed'); + } + return { type: PayloadType.attachments, attachments: { @@ -263,6 +267,11 @@ export default class WebEventWrapper extends EventWrapper case IncomingMessageType.attachments: { const attachment = this._adapter.raw.data; + + if (!('url' in attachment)) { + throw new Error('Attachment has not been processed'); + } + return { type: PayloadType.attachments, serialized_text: `attachment:${attachment.type}:${attachment.url}`, diff --git a/api/src/plugins/base-storage-plugin.ts b/api/src/plugins/base-storage-plugin.ts index 3bbf4225..a1e1de05 100644 --- a/api/src/plugins/base-storage-plugin.ts +++ b/api/src/plugins/base-storage-plugin.ts @@ -6,9 +6,14 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ +import { Readable } from 'stream'; + import { Injectable, StreamableFile } from '@nestjs/common'; -import { AttachmentCreateDto } from '@/attachment/dto/attachment.dto'; +import { + AttachmentCreateDto, + AttachmentMetadataDto, +} from '@/attachment/dto/attachment.dto'; import { Attachment } from '@/attachment/schemas/attachment.schema'; import { BasePlugin } from './base-plugin.service'; @@ -34,4 +39,9 @@ export abstract class BaseStoragePlugin extends BasePlugin { abstract downloadProfilePic(name: string): Promise; readAsBuffer?(attachment: Attachment): Promise; + + store?( + file: Buffer | Readable | string, + metadata: AttachmentMetadataDto, + ): Promise; } diff --git a/api/src/utils/test/fixtures/attachment.ts b/api/src/utils/test/fixtures/attachment.ts index 5346769b..3319401d 100644 --- a/api/src/utils/test/fixtures/attachment.ts +++ b/api/src/utils/test/fixtures/attachment.ts @@ -18,7 +18,7 @@ export const attachmentFixtures: AttachmentCreateDto[] = [ size: 3539, location: '39991e51-55c6-4a26-9176-b6ba04f180dc.jpg', channel: { - dimelo: { + 'web-channel': { id: '1', }, }, @@ -30,7 +30,7 @@ export const attachmentFixtures: AttachmentCreateDto[] = [ size: 3539, location: '39991e51-55c6-4a26-9176-b6ba04f180dd.jpg', channel: { - dimelo: { + 'web-channel': { id: '2', }, },