fix: refactor file upload

api/src/attachment/dto/attachment.dto.ts

@@ -9,54 +9,57 @@
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
 import {
+  IsNotEmpty,
   IsObject,
   IsOptional,
-  MaxLength,
-  IsNotEmpty,
   IsString,
+  MaxLength,
 } from 'class-validator';
 
+import { ChannelName } from '@/channel/types';
 import { ObjectIdDto } from '@/utils/dto/object-id.dto';
 
-export class AttachmentCreateDto {
+export class AttachmentMetadataDto {
   /**
-   * Attachment channel
+   * Attachment original file name
    */
-  @ApiPropertyOptional({ description: 'Attachment channel', type: Object })
-  @IsNotEmpty()
-  @IsObject()
-  channel?: Partial<Record<string, any>>;
-
-  /**
-   * Attachment location
-   */
-  @ApiProperty({ description: 'Attachment location', type: String })
-  @IsNotEmpty()
-  @IsString()
-  location: string;
-
-  /**
-   * Attachment name
-   */
-  @ApiProperty({ description: 'Attachment name', type: String })
+  @ApiProperty({ description: 'Attachment original file name', type: String })
   @IsNotEmpty()
   @IsString()
   name: string;
 
   /**
-   * Attachment size
+   * Attachment size in bytes
    */
-  @ApiProperty({ description: 'Attachment size', type: Number })
+  @ApiProperty({ description: 'Attachment size in bytes', type: Number })
   @IsNotEmpty()
   size: number;
 
   /**
-   * Attachment type
+   * Attachment MIME type
    */
-  @ApiProperty({ description: 'Attachment type', type: String })
+  @ApiProperty({ description: 'Attachment MIME type', type: String })
   @IsNotEmpty()
   @IsString()
   type: string;
+
+  /**
+   * Attachment specia channel(s) metadata
+   */
+  @ApiPropertyOptional({ description: 'Attachment channel', type: Object })
+  @IsNotEmpty()
+  @IsObject()
+  channel?: Partial<Record<ChannelName, any>>;
+}
+
+export class AttachmentCreateDto extends AttachmentMetadataDto {
+  /**
+   * Attachment location (file would/should be stored under a unique name)
+   */
+  @ApiProperty({ description: 'Attachment location', type: String })
+  @IsNotEmpty()
+  @IsString()
+  location: string;
 }
 
 export class AttachmentDownloadDto extends ObjectIdDto {

api/src/attachment/mocks/attachment.mock.ts

@@ -42,7 +42,7 @@ export const attachments: Attachment[] = [
     size: 343370,
     location:
       '/app/src/attachment/uploads/Screenshot from 2022-03-11 08-41-27-2a9799a8b6109c88fd9a7a690c1101934c.png',
-    channel: { dimelo: {} },
+    channel: { 'web-channel': {} },
     id: '65940d115178607da65c82b7',
     createdAt: new Date(),
     updatedAt: new Date(),
@@ -53,7 +53,7 @@ export const attachments: Attachment[] = [
     size: 33829,
     location:
       '/app/src/attachment/uploads/Screenshot from 2022-03-18 08-58-15-af61e7f71281f9fd3f1ad7ad10107741c.png',
-    channel: { dimelo: {} },
+    channel: { 'web-channel': {} },
     id: '65940d115178607da65c82b8',
     createdAt: new Date(),
     updatedAt: new Date(),

api/src/attachment/services/attachment.service.ts

@@ -6,8 +6,9 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import fs, { createReadStream } from 'fs';
+import fs, { createReadStream, promises as fsPromises } from 'fs';
 import path, { join } from 'path';
+import { Readable } from 'stream';
 
 import {
   Injectable,
@@ -24,9 +25,14 @@ import { PluginService } from '@/plugins/plugins.service';
 import { PluginType } from '@/plugins/types';
 import { BaseService } from '@/utils/generics/base-service';
 
+import { AttachmentMetadataDto } from '../dto/attachment.dto';
 import { AttachmentRepository } from '../repositories/attachment.repository';
 import { Attachment } from '../schemas/attachment.schema';
-import { fileExists, getStreamableFile } from '../utilities';
+import {
+  fileExists,
+  generateUniqueFilename,
+  getStreamableFile,
+} from '../utilities';
 
 @Injectable()
 export class AttachmentService extends BaseService<Attachment> {
@@ -180,6 +186,50 @@ export class AttachmentService extends BaseService<Attachment> {
     return uploadedFiles;
   }
 
+  /**
+   * Uploads files to the server. If a storage plugin is configured it uploads files accordingly.
+   * Otherwise, uploads files to the local directory.
+   *
+   * @param file - The file path or the Buffer / Readable
+   * @returns A promise that resolves to an array of uploaded attachments.
+   */
+  async store(
+    file: Buffer | Readable | string,
+    metadata: AttachmentMetadataDto,
+  ): Promise<Attachment> {
+    if (this.getStoragePlugin()) {
+      const storedDto = await this.getStoragePlugin().store(file, metadata);
+      return await this.create(storedDto);
+    } else {
+      const dirPath = path.join(config.parameters.uploadDir);
+      const uniqueFilename = generateUniqueFilename(metadata.name);
+      const filePath = path.resolve(dirPath, uniqueFilename);
+
+      if (typeof file === 'string') {
+        // For example, if the file is an instance of `Express.Multer.File` (diskStorage case)
+        await fsPromises.copyFile(file, filePath);
+        await fsPromises.unlink(file);
+      } else if (Buffer.isBuffer(file)) {
+        await fsPromises.writeFile(filePath, file);
+      } else if (file instanceof Readable) {
+        await new Promise((resolve, reject) => {
+          const writeStream = fs.createWriteStream(filePath);
+          file.pipe(writeStream);
+          writeStream.on('finish', resolve);
+          writeStream.on('error', reject);
+        });
+      } else {
+        throw new TypeError('Unrecognized file object');
+      }
+
+      const location = filePath.replace(dirPath, '');
+      return await this.create({
+        ...metadata,
+        location,
+      });
+    }
+  }
+
   /**
    * Downloads an attachment identified by the provided parameters.
    *

api/src/attachment/utilities/index.ts

@@ -7,19 +7,31 @@
  */
 
 import { createReadStream, existsSync } from 'fs';
-import { join } from 'path';
+import { extname, join } from 'path';
 
 import { Logger, StreamableFile } from '@nestjs/common';
 import { StreamableFileOptions } from '@nestjs/common/file-stream/interfaces/streamable-options.interface';
+import { v4 as uuidv4 } from 'uuid';
 
 import { config } from '@/config';
 
 export const MIME_REGEX = /^[a-z-]+\/[0-9a-z\-.]+$/gm;
 
+/**
+ * Validates if a given string matches the MIME type format.
+ *
+ * @param type The string to validate.
+ * @returns Whether the string is a valid MIME type.
+ */
 export const isMime = (type: string): boolean => {
   return MIME_REGEX.test(type);
 };
 
+/**
+ * Checks if a file exists in the specified upload directory.
+ * @param location The relative location of the file.
+ * @returns Whether the file exists.
+ */
 export const fileExists = (location: string): boolean => {
   // bypass test env
   if (config.env === 'test') {
@@ -35,6 +47,12 @@ export const fileExists = (location: string): boolean => {
   }
 };
 
+/**
+ * Creates a streamable file from a given file path and options.
+ *
+ * @param options The object containing the file path and optional settings.
+ * @returns A streamable file object.
+ */
 export const getStreamableFile = ({
   path,
   options,
@@ -50,3 +68,15 @@ export const getStreamableFile = ({
 
   return new StreamableFile(fileReadStream, options);
 };
+
+/**
+ * Generates a unique filename by appending a UUID to the original name.
+ *
+ * @param originalname The original filename.
+ * @returns A unique filename.
+ */
+export const generateUniqueFilename = (originalname: string) => {
+  const extension = extname(originalname);
+  const name = originalname.slice(0, -extension.length);
+  return `${name}-${uuidv4()}${extension}`;
+};

api/src/extensions/channels/web/base-web-channel.ts

@@ -6,14 +6,10 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import { promises as fsPromises } from 'fs';
-import path from 'path';
-
 import { Injectable } from '@nestjs/common';
 import { EventEmitter2, OnEvent } from '@nestjs/event-emitter';
-import { Request, Response } from 'express';
-import multer, { diskStorage } from 'multer';
-import sanitize from 'sanitize-filename';
+import { NextFunction, Request, Response } from 'express';
+import multer, { diskStorage, memoryStorage } from 'multer';
 import { Socket } from 'socket.io';
 import { v4 as uuidv4 } from 'uuid';
 
@@ -31,7 +27,6 @@ import { Button, ButtonType } from '@/chat/schemas/types/button';
 import {
   AnyMessage,
   ContentElement,
-  FileType,
   IncomingMessage,
   OutgoingMessage,
   OutgoingMessageFormat,
@@ -597,133 +592,104 @@ export default abstract class BaseWebChannelHandler<
   /**
    * Upload file as attachment if provided
    *
-   * @param upload
-   * @param filename
-   */
-  private async storeAttachment(
-    upload: Omit<Web.IncomingAttachmentMessageData, 'url' | 'file'>,
-    filename: string,
-    next: (
-      err: Error | null,
-      payload: { type: string; url: string } | false,
-    ) => void,
-  ): Promise<void> {
-    try {
-      this.logger.debug('Web Channel Handler : Successfully uploaded file');
-
-      const attachment = await this.attachmentService.create({
-        name: upload.name || '',
-        type: upload.type || 'text/txt',
-        size: upload.size || 0,
-        location: filename,
-        channel: { web: {} },
-      });
-
-      this.logger.debug(
-        'Web Channel Handler : Successfully stored file as attachment',
-      );
-
-      next(null, {
-        type: Attachment.getTypeByMime(attachment.type),
-        url: Attachment.getAttachmentUrl(attachment.id, attachment.name),
-      });
-    } catch (err) {
-      this.logger.error(
-        'Web Channel Handler : Unable to store uploaded file as attachment',
-        err,
-      );
-      next(err, false);
-    }
-  }
-
-  /**
-   * Upload file as attachment if provided
-   *
-   * @param req
-   * @param res
+   * @param req Either a HTTP Express request or a WS request (Synthetic Object)
+   * @param res Either a HTTP Express response or a WS response (Synthetic Object)
+   * @param next Callback Function
    */
   async handleFilesUpload(
     req: Request | SocketRequest,
     res: Response | SocketResponse,
     next: (
       err: null | Error,
-      result: { type: string; url: string } | false,
+      result?: Web.IncomingAttachmentMessageData,
     ) => void,
   ): Promise<void> {
-    const data: Web.IncomingMessage = req.body;
     // Check if any file is provided
     if (!req.session.web) {
       this.logger.debug('Web Channel Handler : No session provided');
-      return next(null, false);
-    }
-    // Check if any file is provided
-    if (!data || !data.type || data.type !== 'file') {
-      this.logger.debug('Web Channel Handler : No files provided');
-      return next(null, false);
-    }
-
-    // Parse json form data (in case of content-type multipart/form-data)
-    data.data =
-      typeof data.data === 'string' ? JSON.parse(data.data) : data.data;
-
-    // Check max size upload
-    const upload = data.data;
-    if (typeof upload.size === 'undefined') {
-      return next(new Error('File upload probably failed.'), false);
-    }
-
-    // Store file as attachment
-    const dirPath = path.join(config.parameters.uploadDir);
-    const sanitizedFilename = sanitize(
-      `${req.session.web.profile.id}_${+new Date()}_${upload.name}`,
-    );
-    const filePath = path.resolve(dirPath, sanitizedFilename);
-
-    if (!filePath.startsWith(dirPath)) {
-      return next(new Error('Invalid file path!'), false);
+      return next(null);
     }
 
     if (this.isSocketRequest(req)) {
-      // @TODO : test this
       try {
-        await fsPromises.writeFile(filePath, upload.file);
-        this.storeAttachment(upload, sanitizedFilename, next);
+        const { type, data } = req.body as Web.IncomingMessage;
+
+        // Check if any file is provided
+        if (type !== 'file' || !('file' in data) || !data.file) {
+          this.logger.debug('Web Channel Handler : No files provided');
+          return next(null);
+        }
+
+        const size = Buffer.byteLength(data.file);
+
+        if (size > config.parameters.maxUploadSize) {
+          return next(new Error('Max upload size has been exceeded'));
+        }
+
+        const attachment = await this.attachmentService.store(data.file, {
+          name: data.name,
+          size: Buffer.byteLength(data.file),
+          type: data.type,
+        });
+        next(null, {
+          type: Attachment.getTypeByMime(attachment.type),
+          url: Attachment.getAttachmentUrl(attachment.id, attachment.name),
+        });
       } catch (err) {
         this.logger.error(
           'Web Channel Handler : Unable to write uploaded file',
           err,
         );
-        return next(new Error('Unable to upload file!'), false);
+        return next(new Error('Unable to upload file!'));
       }
     } else {
       const upload = multer({
-        storage: diskStorage({
-          destination: dirPath, // Set the destination directory for file storage
-          filename: (_req, _file, cb) => {
-            cb(null, sanitizedFilename); // Set the file name
-          },
-        }),
+        limits: {
+          fileSize: config.parameters.maxUploadSize,
+        },
+        storage: (() => {
+          if (config.parameters.storageMode === 'memory') {
+            return memoryStorage();
+          } else {
+            return diskStorage({});
+          }
+        })(),
       }).single('file'); // 'file' is the field name in the form
 
-      upload(req as Request, res as Response, (err) => {
+      upload(req as Request, res as Response, async (err?: any) => {
         if (err) {
           this.logger.error(
             'Web Channel Handler : Unable to write uploaded file',
             err,
           );
-          return next(new Error('Unable to upload file!'), false);
+          return next(new Error('Unable to upload file!'));
+        }
+
+        // Check if any file is provided
+        if (!req.file) {
+          this.logger.debug('Web Channel Handler : No files provided');
+          return next(null);
+        }
+
+        try {
+          const file = req.file;
+          const attachment = await this.attachmentService.store(
+            config.parameters.storageMode === 'memory'
+              ? file.buffer
+              : file.path,
+            {
+              name: file.originalname,
+              size: file.size,
+              type: file.mimetype,
+            },
+          );
+          next(null, {
+            type: Attachment.getTypeByMime(attachment.type),
+            url: Attachment.getAttachmentUrl(attachment.id, attachment.name),
+          });
+        } catch (err) {
+          next(err);
         }
-        // @TODO : test upload
-        const file = req.file;
-        this.storeAttachment(
-          {
-            name: file.filename,
-            type: file.mimetype as FileType, // @Todo : test this
-            size: file.size,
-          },
-          file.path.replace(dirPath, ''),
-          next,
-        );
       });
     }
   }
@@ -751,7 +717,7 @@ export default abstract class BaseWebChannelHandler<
   /**
    * Return subscriber channel specific attributes
    *
-   * @param req
+   * @param req Either a HTTP Express request or a WS request (Synthetic Object)
    *
    * @returns The subscriber channel's attributes
    */
@@ -766,22 +732,44 @@ export default abstract class BaseWebChannelHandler<
   }
 
   /**
-   * Handle channel event (probably a message)
-   *
+   * Custom channel middleware
    * @param req
    * @param res
+   * @param next
+   */
+  async middleware(_req: Request, _res: Response, next: NextFunction) {
+    // Do nothing, override in channel
+    next();
+  }
+
+  /**
+   * Handle channel event (probably a message)
+   *
+   * @param req Either a HTTP Express request or a WS request (Synthetic Object)
+   * @param res Either a HTTP Express response or a WS response (Synthetic Object)
    */
   _handleEvent(
     req: Request | SocketRequest,
     res: Response | SocketResponse,
   ): void {
-    const data: Web.IncomingMessage = req.body;
+    // @TODO: perform payload validation
+    if (!req.body) {
+      this.logger.debug('Web Channel Handler : Empty body');
+      res.status(400).json({ err: 'Web Channel Handler : Bad Request!' });
+      return;
+    } else {
+      // Parse json form data (in case of content-type multipart/form-data)
+      req.body.data =
+        typeof req.body.data === 'string'
+          ? JSON.parse(req.body.data)
+          : req.body.data;
+    }
+
     this.validateSession(req, res, (profile) => {
       this.handleFilesUpload(
         req,
         res,
-        // @ts-expect-error @TODO : This needs to be fixed at a later point @TODO
-        (err: Error, upload: Web.IncomingMessageData) => {
+        (err: Error, data?: Web.IncomingAttachmentMessageData) => {
           if (err) {
             this.logger.warn(
               'Web Channel Handler : Unable to upload file ',
@@ -792,14 +780,18 @@ export default abstract class BaseWebChannelHandler<
               .json({ err: 'Web Channel Handler : File upload failed!' });
           }
           // Set data in file upload case
-          if (upload) {
-            data.data = upload;
-          }
+          const body: Web.IncomingMessage = data
+            ? {
+                ...req.body,
+                data,
+              }
+            : req.body;
+
           const channelAttrs = this.getChannelAttributes(req);
-          const event = new WebEventWrapper<N>(this, data, channelAttrs);
+          const event = new WebEventWrapper<N>(this, body, channelAttrs);
           if (event.getEventType() === 'message') {
             // Handler sync message sent by chabbot
-            if (data.sync && data.author === 'chatbot') {
+            if (body.sync && body.author === 'chatbot') {
               const sentMessage: MessageCreateDto = {
                 mid: event.getId(),
                 message: event.getMessage() as StdOutgoingMessage,

api/src/extensions/channels/web/types.ts

@@ -59,14 +59,19 @@ export namespace Web {
     };
   };
 
-  export type IncomingAttachmentMessageData = {
-    type: FileType; // mime type in a file case
-    url: string; // file url
-    // Only when uploaded
-    size?: number; // file size
-    name?: string;
-    file?: any;
-  };
+  // Depending if it's has been processed or not
+  export type IncomingAttachmentMessageData =
+    // After upload and attachment is processed
+    | {
+        type: FileType;
+        url: string; // file download url
+      } // Before upload and attachment is processed
+    | {
+        type: string; // mime type
+        size: number; // file size
+        name: string;
+        file: Buffer;
+      };
 
   export type IncomingMessageData =
     | IncomingTextMessageData

api/src/extensions/channels/web/wrapper.ts

@@ -216,6 +216,10 @@ export default class WebEventWrapper<N extends ChannelName> extends EventWrapper
         };
       }
       case IncomingMessageType.attachments:
+        if (!('url' in this._adapter.raw.data)) {
+          throw new Error('Attachment has not been processed');
+        }
+
         return {
           type: PayloadType.attachments,
           attachments: {
@@ -263,6 +267,11 @@ export default class WebEventWrapper<N extends ChannelName> extends EventWrapper
 
       case IncomingMessageType.attachments: {
         const attachment = this._adapter.raw.data;
+
+        if (!('url' in attachment)) {
+          throw new Error('Attachment has not been processed');
+        }
+
         return {
           type: PayloadType.attachments,
           serialized_text: `attachment:${attachment.type}:${attachment.url}`,

api/src/plugins/base-storage-plugin.ts

@@ -6,9 +6,14 @@
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
+import { Readable } from 'stream';
+
 import { Injectable, StreamableFile } from '@nestjs/common';
 
-import { AttachmentCreateDto } from '@/attachment/dto/attachment.dto';
+import {
+  AttachmentCreateDto,
+  AttachmentMetadataDto,
+} from '@/attachment/dto/attachment.dto';
 import { Attachment } from '@/attachment/schemas/attachment.schema';
 
 import { BasePlugin } from './base-plugin.service';
@@ -34,4 +39,9 @@ export abstract class BaseStoragePlugin extends BasePlugin {
   abstract downloadProfilePic(name: string): Promise<StreamableFile>;
 
   readAsBuffer?(attachment: Attachment): Promise<Buffer>;
+
+  store?(
+    file: Buffer | Readable | string,
+    metadata: AttachmentMetadataDto,
+  ): Promise<Attachment>;
 }

api/src/utils/test/fixtures/attachment.ts

@@ -18,7 +18,7 @@ export const attachmentFixtures: AttachmentCreateDto[] = [
     size: 3539,
     location: '39991e51-55c6-4a26-9176-b6ba04f180dc.jpg',
     channel: {
-      dimelo: {
+      'web-channel': {
         id: '1',
       },
     },
@@ -30,7 +30,7 @@ export const attachmentFixtures: AttachmentCreateDto[] = [
     size: 3539,
     location: '39991e51-55c6-4a26-9176-b6ba04f180dd.jpg',
     channel: {
-      dimelo: {
+      'web-channel': {
         id: '2',
       },
     },