Merge pull request #335 from Hexastack/334-issue-redis-integration

feat(api): Redis integration REST/WS

api/package-lock.json

@@ -25,9 +25,11 @@
         "@nestjs/swagger": "^7.2.0",
         "@nestjs/websockets": "^10.3.7",
         "@resvg/resvg-js": "^2.6.2",
+        "@socket.io/redis-adapter": "^8.3.0",
         "@tekuconcept/nestjs-csrf": "^1.1.0",
         "bcryptjs": "^2.4.3",
         "cache-manager": "^5.3.2",
+        "cache-manager-redis-yet": "^4.1.2",
         "connect-mongo": "^5.1.0",
         "cookie-parser": "^1.4.6",
         "dotenv": "^16.3.1",
@@ -5159,6 +5161,64 @@
         "url": "https://opencollective.com/unts"
       }
     },
+    "node_modules/@redis/bloom": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
+      "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/client": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz",
+      "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==",
+      "dependencies": {
+        "cluster-key-slot": "1.1.2",
+        "generic-pool": "3.9.0",
+        "yallist": "4.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@redis/client/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+    },
+    "node_modules/@redis/graph": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
+      "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/json": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz",
+      "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/search": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz",
+      "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/time-series": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz",
+      "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
     "node_modules/@resvg/resvg-js": {
       "version": "2.6.2",
       "resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.2.tgz",
@@ -5877,6 +5937,22 @@
       "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.1.tgz",
       "integrity": "sha512-dzJtaDAAoXx4GCOJpbB2eG/Qj8VDpdwkLsWGzGm+0L7E8/434RyMbAHmk9ubXWVAb9nXmc44jUf8GKqVDiKezg=="
     },
+    "node_modules/@socket.io/redis-adapter": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/@socket.io/redis-adapter/-/redis-adapter-8.3.0.tgz",
+      "integrity": "sha512-ly0cra+48hDmChxmIpnESKrc94LjRL80TEmZVscuQ/WWkRP81nNj8W8cCGMqbI4L6NCuAaPRSzZF1a9GlAxxnA==",
+      "dependencies": {
+        "debug": "~4.3.1",
+        "notepack.io": "~3.0.1",
+        "uid2": "1.0.0"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "socket.io-adapter": "^2.5.4"
+      }
+    },
     "node_modules/@swc-node/core": {
       "version": "1.13.3",
       "resolved": "https://registry.npmjs.org/@swc-node/core/-/core-1.13.3.tgz",
@@ -7890,22 +7966,42 @@
       }
     },
     "node_modules/cache-manager": {
-      "version": "5.3.2",
+      "version": "5.7.6",
-      "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.3.2.tgz",
+      "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz",
-      "integrity": "sha512-MBpwYqCqf8LFSso5CjMs5Kj2i3RYYUP0fLwjMSnxyspYx0y8uT1SZbWiOk33fw5r+Sbpe5ERfk1+pgXEJ/omEw==",
+      "integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==",
       "dependencies": {
+        "eventemitter3": "^5.0.1",
         "lodash.clonedeep": "^4.5.0",
-        "lru-cache": "^10.1.0",
+        "lru-cache": "^10.2.2",
         "promise-coalesce": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/cache-manager-redis-yet": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/cache-manager-redis-yet/-/cache-manager-redis-yet-4.2.0.tgz",
+      "integrity": "sha512-3Exp8Hv52G/K8oY5U8OwS2A7MJEJmNUvn9DPEnNSo/yu+Ken4v9VEnIAq9YZ6b6E0LN1NwPXNF3N+IaQM9RcTQ==",
+      "deprecated": "With cache-manager v6 we now are using Keyv",
+      "dependencies": {
+        "@redis/bloom": "^1.2.0",
+        "@redis/client": "^1.5.14",
+        "@redis/graph": "^1.1.1",
+        "@redis/json": "^1.0.6",
+        "@redis/search": "^1.1.6",
+        "@redis/time-series": "^1.0.5",
+        "cache-manager": "^5.4.0",
+        "redis": "^4.6.13"
+      },
+      "engines": {
+        "node": ">= 16.17.0"
       }
     },
     "node_modules/cache-manager/node_modules/lru-cache": {
-      "version": "10.1.0",
+      "version": "10.4.3",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
-      "integrity": "sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==",
+      "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="
-      "engines": {
-        "node": "14 || >=16.14"
-      }
     },
     "node_modules/call-bind": {
       "version": "1.0.7",
@@ -8211,6 +8307,14 @@
         "node": ">=0.8"
       }
     },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -10265,6 +10369,11 @@
       "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.9.tgz",
       "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg=="
     },
+    "node_modules/eventemitter3": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
+      "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="
+    },
     "node_modules/events": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
@@ -10998,6 +11107,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/generic-pool": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
+      "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/gensync": {
       "version": "1.0.0-beta.2",
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -14749,6 +14866,11 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/notepack.io": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/notepack.io/-/notepack.io-3.0.1.tgz",
+      "integrity": "sha512-TKC/8zH5pXIAMVQio2TvVDTtPRX+DJPHDqjRbxogtFiByHyzKmy96RA0JtCQJ+WouyyL4A10xomQzgbUT+1jCg=="
+    },
     "node_modules/npm-run-path": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
@@ -16662,6 +16784,19 @@
         "node": ">=8.10.0"
       }
     },
+    "node_modules/redis": {
+      "version": "4.7.0",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz",
+      "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==",
+      "dependencies": {
+        "@redis/bloom": "1.2.0",
+        "@redis/client": "1.6.0",
+        "@redis/graph": "1.1.1",
+        "@redis/json": "1.0.7",
+        "@redis/search": "1.2.0",
+        "@redis/time-series": "1.1.0"
+      }
+    },
     "node_modules/reflect-metadata": {
       "version": "0.1.13",
       "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
@@ -18536,6 +18671,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/uid2": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/uid2/-/uid2-1.0.0.tgz",
+      "integrity": "sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ==",
+      "engines": {
+        "node": ">= 4.0.0"
+      }
+    },
     "node_modules/unbox-primitive": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz",

api/package.json

@@ -63,9 +63,11 @@
     "@nestjs/swagger": "^7.2.0",
     "@nestjs/websockets": "^10.3.7",
     "@resvg/resvg-js": "^2.6.2",
+    "@socket.io/redis-adapter": "^8.3.0",
     "@tekuconcept/nestjs-csrf": "^1.1.0",
     "bcryptjs": "^2.4.3",
     "cache-manager": "^5.3.2",
+    "cache-manager-redis-yet": "^4.1.2",
     "connect-mongo": "^5.1.0",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.3.1",

api/src/app.module.ts

@@ -11,19 +11,21 @@ import path from 'path';
 import { CacheModule } from '@nestjs/cache-manager';
 // eslint-disable-next-line import/order
 import { MailerModule } from '@nestjs-modules/mailer';
-// eslint-disable-next-line import/order
-import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter';
 import { Module } from '@nestjs/common';
 import { APP_GUARD } from '@nestjs/core';
 import { EventEmitterModule } from '@nestjs/event-emitter';
 import { MongooseModule } from '@nestjs/mongoose';
+// eslint-disable-next-line import/order
+import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter';
 import { CsrfGuard, CsrfModule } from '@tekuconcept/nestjs-csrf';
+import { redisStore } from 'cache-manager-redis-yet';
 import {
   AcceptLanguageResolver,
   I18nOptions,
   QueryResolver,
 } from 'nestjs-i18n';
 import SMTPTransport from 'nodemailer/lib/smtp-transport';
+import { RedisClientOptions } from 'redis';
 
 import { AnalyticsModule } from './analytics/analytics.module';
 import { AppController } from './app.controller';
@@ -124,11 +126,23 @@ const i18nOptions: I18nOptions = {
     }),
     CsrfModule,
     I18nModule.forRoot(i18nOptions),
-    CacheModule.register({
+    config.cache.type === 'redis'
-      isGlobal: true,
+      ? CacheModule.register<RedisClientOptions>({
-      ttl: config.cache.ttl,
+          isGlobal: true,
-      max: config.cache.max,
+          store: redisStore,
-    }),
+          socket: {
+            host: config.cache.host,
+            port: config.cache.port,
+          },
+          ttl: config.cache.ttl,
+          max: config.cache.max,
+        })
+      : CacheModule.register({
+          isGlobal: true,
+          ttl: config.cache.ttl,
+          max: config.cache.max,
+        }),
+
     ...extraModules,
   ],
   controllers: [AppController],

api/src/config/index.ts

@@ -129,9 +129,11 @@ export const config: Config = {
     level: 'verbose',
   },
   cache: {
-    type: 'memory',
+    type: process.env.REDIS_ENABLED === 'true' ? 'redis' : 'memory',
     ttl: 60 * 1000, // Milliseconds
     max: 100, // Maximum number of items in cache (defaults to 100)
+    host: process.env.REDIS_HOST || 'redis',
+    port: parseInt(process.env.REDIS_PORT || '6379'),
   },
   mongo: {
     user: process.env.MONGO_USER || 'dev_only',

api/src/config/types.ts

@@ -19,7 +19,10 @@ type TLogLevel = 'log' | 'fatal' | 'error' | 'warn' | 'debug' | 'verbose';
 type TCacheConfig = {
   ttl: number;
   max: number;
-} & { type: 'memory' };
+  host: string;
+  port: number;
+  type: 'memory' | 'redis';
+};
 
 export type Config = {
   i18n: { translationFilename: string };

api/src/main.ts

@@ -25,6 +25,7 @@ import { seedDatabase } from './seeder';
 import { swagger } from './swagger';
 import { getSessionStore } from './utils/constants/session-store';
 import { ObjectIdPipe } from './utils/pipes/object-id.pipe';
+import { RedisIoAdapter } from './websocket/adapters/redis-io.adapter';
 
 async function bootstrap() {
   const isProduction = config.env.toLowerCase().includes('prod');
@@ -77,6 +78,12 @@ async function bootstrap() {
   app.use(passport.initialize());
   app.use(passport.session());
 
+  if (config.cache.type === 'redis') {
+    const redisIoAdapter = new RedisIoAdapter(app);
+    await redisIoAdapter.connectToRedis();
+    app.useWebSocketAdapter(redisIoAdapter);
+  }
+
   process.on('uncaughtException', (error) => {
     if (error.stack.toLowerCase().includes('smtp'))
       app.get(LoggerService).error('SMTP error', error.stack);

api/src/websocket/adapters/redis-io.adapter.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright © 2024 Hexastack. All rights reserved.
+ *
+ * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
+ * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
+ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
+ */
+
+import { IoAdapter } from '@nestjs/platform-socket.io';
+import { createAdapter } from '@socket.io/redis-adapter';
+import { createClient } from 'redis';
+import { ServerOptions } from 'socket.io';
+
+import { config } from '@/config';
+
+export class RedisIoAdapter extends IoAdapter {
+  private adapter: ReturnType<typeof createAdapter>;
+
+  async connectToRedis(): Promise<void> {
+    const pubClient = createClient(
+      config.cache.type === 'redis' && {
+        socket: {
+          host: config.cache.host,
+          port: config.cache.port,
+        },
+      },
+    );
+    const subClient = pubClient.duplicate();
+    pubClient.on('error', (error) => {
+      throw error;
+    });
+    subClient.on('error', (error) => {
+      throw error;
+    });
+    await Promise.all([pubClient.connect(), subClient.connect()]);
+    this.adapter = createAdapter(pubClient, subClient);
+  }
+
+  createIOServer(port: number, options?: ServerOptions): any {
+    const server = super.createIOServer(port, options);
+    server.adapter(this.adapter);
+    return server;
+  }
+}

docker/.env.example

@@ -58,3 +58,9 @@ NEXT_PUBLIC_SSO_ENABLED=false
 APP_WIDGET_PORT=5173
 REACT_APP_WIDGET_API_URL=http://${APP_DOMAIN}:${API_PORT}
 REACT_APP_WIDGET_CHANNEL=web-channel
+
+# Redis
+APP_REDIS_PORT=9001
+REDIS_ENABLED=false
+REDIS_HOST=redis
+REDIS_PORT=6379

docker/docker-compose.redis.dev.yml

@@ -0,0 +1,7 @@
+version: "3.8"
+
+services:
+  redis:
+    image: redis/redis-stack:7.2.0-v6
+    ports:
+      - ${APP_REDIS_PORT}:8001

docker/docker-compose.redis.yml

@@ -0,0 +1,29 @@
+version: "3.8"
+
+services:
+  api:
+    networks:
+      - cache-network
+    depends_on:
+      redis:
+        condition: service_healthy
+
+  redis:
+    container_name: redis
+    image: redis/redis-stack-server:7.2.0-v6
+    networks:
+      - cache-network
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 10s
+    volumes:
+      - redis_data:/data
+
+volumes:
+  redis_data:
+
+networks:
+  cache-network: