From 1436b808e202d33597eb9ccbd464dcc6513a99e3 Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Fri, 15 Nov 2024 17:39:12 +0100 Subject: [PATCH 1/2] feat(api): Redis integration REST/WS --- api/package-lock.json | 163 ++++++++++++++++-- api/package.json | 2 + api/src/app.module.ts | 28 ++- api/src/config/index.ts | 4 +- api/src/config/types.ts | 4 +- api/src/main.ts | 7 + .../websocket/adapters/redis-io.adapter.ts | 44 +++++ docker/.env.example | 6 + docker/docker-compose.redis.dev.yml | 7 + docker/docker-compose.redis.yml | 29 ++++ 10 files changed, 275 insertions(+), 19 deletions(-) create mode 100644 api/src/websocket/adapters/redis-io.adapter.ts create mode 100644 docker/docker-compose.redis.dev.yml create mode 100644 docker/docker-compose.redis.yml diff --git a/api/package-lock.json b/api/package-lock.json index b50360df..09d8e7ed 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -25,9 +25,11 @@ "@nestjs/swagger": "^7.2.0", "@nestjs/websockets": "^10.3.7", "@resvg/resvg-js": "^2.6.2", + "@socket.io/redis-adapter": "^8.3.0", "@tekuconcept/nestjs-csrf": "^1.1.0", "bcryptjs": "^2.4.3", "cache-manager": "^5.3.2", + "cache-manager-redis-yet": "^4.1.2", "connect-mongo": "^5.1.0", "cookie-parser": "^1.4.6", "dotenv": "^16.3.1", @@ -5159,6 +5161,64 @@ "url": "https://opencollective.com/unts" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz", + "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/client/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@resvg/resvg-js": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.2.tgz", @@ -5877,6 +5937,22 @@ "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.1.tgz", "integrity": "sha512-dzJtaDAAoXx4GCOJpbB2eG/Qj8VDpdwkLsWGzGm+0L7E8/434RyMbAHmk9ubXWVAb9nXmc44jUf8GKqVDiKezg==" }, + "node_modules/@socket.io/redis-adapter": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@socket.io/redis-adapter/-/redis-adapter-8.3.0.tgz", + "integrity": "sha512-ly0cra+48hDmChxmIpnESKrc94LjRL80TEmZVscuQ/WWkRP81nNj8W8cCGMqbI4L6NCuAaPRSzZF1a9GlAxxnA==", + "dependencies": { + "debug": "~4.3.1", + "notepack.io": "~3.0.1", + "uid2": "1.0.0" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "socket.io-adapter": "^2.5.4" + } + }, "node_modules/@swc-node/core": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/@swc-node/core/-/core-1.13.3.tgz", @@ -7890,22 +7966,42 @@ } }, "node_modules/cache-manager": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.3.2.tgz", - "integrity": "sha512-MBpwYqCqf8LFSso5CjMs5Kj2i3RYYUP0fLwjMSnxyspYx0y8uT1SZbWiOk33fw5r+Sbpe5ERfk1+pgXEJ/omEw==", + "version": "5.7.6", + "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz", + "integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==", "dependencies": { + "eventemitter3": "^5.0.1", "lodash.clonedeep": "^4.5.0", - "lru-cache": "^10.1.0", + "lru-cache": "^10.2.2", "promise-coalesce": "^1.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/cache-manager-redis-yet": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/cache-manager-redis-yet/-/cache-manager-redis-yet-4.2.0.tgz", + "integrity": "sha512-3Exp8Hv52G/K8oY5U8OwS2A7MJEJmNUvn9DPEnNSo/yu+Ken4v9VEnIAq9YZ6b6E0LN1NwPXNF3N+IaQM9RcTQ==", + "deprecated": "With cache-manager v6 we now are using Keyv", + "dependencies": { + "@redis/bloom": "^1.2.0", + "@redis/client": "^1.5.14", + "@redis/graph": "^1.1.1", + "@redis/json": "^1.0.6", + "@redis/search": "^1.1.6", + "@redis/time-series": "^1.0.5", + "cache-manager": "^5.4.0", + "redis": "^4.6.13" + }, + "engines": { + "node": ">= 16.17.0" } }, "node_modules/cache-manager/node_modules/lru-cache": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.1.0.tgz", - "integrity": "sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==", - "engines": { - "node": "14 || >=16.14" - } + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, "node_modules/call-bind": { "version": "1.0.7", @@ -8211,6 +8307,14 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -10265,6 +10369,11 @@ "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.9.tgz", "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==" }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -10998,6 +11107,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -14749,6 +14866,11 @@ "node": ">=0.10.0" } }, + "node_modules/notepack.io": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/notepack.io/-/notepack.io-3.0.1.tgz", + "integrity": "sha512-TKC/8zH5pXIAMVQio2TvVDTtPRX+DJPHDqjRbxogtFiByHyzKmy96RA0JtCQJ+WouyyL4A10xomQzgbUT+1jCg==" + }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -16662,6 +16784,19 @@ "node": ">=8.10.0" } }, + "node_modules/redis": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz", + "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.0", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "node_modules/reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -18536,6 +18671,14 @@ "node": ">= 0.8" } }, + "node_modules/uid2": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/uid2/-/uid2-1.0.0.tgz", + "integrity": "sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ==", + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", diff --git a/api/package.json b/api/package.json index f52398be..f4bcc69d 100644 --- a/api/package.json +++ b/api/package.json @@ -63,9 +63,11 @@ "@nestjs/swagger": "^7.2.0", "@nestjs/websockets": "^10.3.7", "@resvg/resvg-js": "^2.6.2", + "@socket.io/redis-adapter": "^8.3.0", "@tekuconcept/nestjs-csrf": "^1.1.0", "bcryptjs": "^2.4.3", "cache-manager": "^5.3.2", + "cache-manager-redis-yet": "^4.1.2", "connect-mongo": "^5.1.0", "cookie-parser": "^1.4.6", "dotenv": "^16.3.1", diff --git a/api/src/app.module.ts b/api/src/app.module.ts index ec9e9dc7..9c90f471 100644 --- a/api/src/app.module.ts +++ b/api/src/app.module.ts @@ -11,19 +11,21 @@ import path from 'path'; import { CacheModule } from '@nestjs/cache-manager'; // eslint-disable-next-line import/order import { MailerModule } from '@nestjs-modules/mailer'; -// eslint-disable-next-line import/order -import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter'; import { Module } from '@nestjs/common'; import { APP_GUARD } from '@nestjs/core'; import { EventEmitterModule } from '@nestjs/event-emitter'; import { MongooseModule } from '@nestjs/mongoose'; +// eslint-disable-next-line import/order +import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter'; import { CsrfGuard, CsrfModule } from '@tekuconcept/nestjs-csrf'; +import { redisStore } from 'cache-manager-redis-yet'; import { AcceptLanguageResolver, I18nOptions, QueryResolver, } from 'nestjs-i18n'; import SMTPTransport from 'nodemailer/lib/smtp-transport'; +import { RedisClientOptions } from 'redis'; import { AnalyticsModule } from './analytics/analytics.module'; import { AppController } from './app.controller'; @@ -124,11 +126,23 @@ const i18nOptions: I18nOptions = { }), CsrfModule, I18nModule.forRoot(i18nOptions), - CacheModule.register({ - isGlobal: true, - ttl: config.cache.ttl, - max: config.cache.max, - }), + config.cache.type === 'redis' + ? CacheModule.register({ + isGlobal: true, + store: redisStore, + socket: { + host: config.cache.host, + port: config.cache.port, + }, + ttl: config.cache.ttl, + max: config.cache.max, + }) + : CacheModule.register({ + isGlobal: true, + ttl: config.cache.ttl, + max: config.cache.max, + }), + ...extraModules, ], controllers: [AppController], diff --git a/api/src/config/index.ts b/api/src/config/index.ts index d81992d3..32f4ec6e 100644 --- a/api/src/config/index.ts +++ b/api/src/config/index.ts @@ -129,9 +129,11 @@ export const config: Config = { level: 'verbose', }, cache: { - type: 'memory', + type: process.env.REDIS_ENABLED === 'true' ? 'redis' : 'memory', ttl: 60 * 1000, // Milliseconds max: 100, // Maximum number of items in cache (defaults to 100) + host: process.env.REDIS_HOST, + port: parseInt(process.env.REDIS_PORT || '6379'), }, mongo: { user: process.env.MONGO_USER || 'dev_only', diff --git a/api/src/config/types.ts b/api/src/config/types.ts index 35d0d9bc..f2131757 100644 --- a/api/src/config/types.ts +++ b/api/src/config/types.ts @@ -19,7 +19,9 @@ type TLogLevel = 'log' | 'fatal' | 'error' | 'warn' | 'debug' | 'verbose'; type TCacheConfig = { ttl: number; max: number; -} & { type: 'memory' }; + host: string; + port: number; +} & { type: 'memory' | 'redis' }; export type Config = { i18n: { translationFilename: string }; diff --git a/api/src/main.ts b/api/src/main.ts index 2572d9b5..c6be3147 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -25,6 +25,7 @@ import { seedDatabase } from './seeder'; import { swagger } from './swagger'; import { getSessionStore } from './utils/constants/session-store'; import { ObjectIdPipe } from './utils/pipes/object-id.pipe'; +import { RedisIoAdapter } from './websocket/adapters/redis-io.adapter'; async function bootstrap() { const isProduction = config.env.toLowerCase().includes('prod'); @@ -77,6 +78,12 @@ async function bootstrap() { app.use(passport.initialize()); app.use(passport.session()); + if (config.cache.type === 'redis') { + const redisIoAdapter = new RedisIoAdapter(app); + await redisIoAdapter.connectToRedis(); + app.useWebSocketAdapter(redisIoAdapter); + } + process.on('uncaughtException', (error) => { if (error.stack.toLowerCase().includes('smtp')) app.get(LoggerService).error('SMTP error', error.stack); diff --git a/api/src/websocket/adapters/redis-io.adapter.ts b/api/src/websocket/adapters/redis-io.adapter.ts new file mode 100644 index 00000000..8288fe07 --- /dev/null +++ b/api/src/websocket/adapters/redis-io.adapter.ts @@ -0,0 +1,44 @@ +/* + * Copyright © 2024 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { IoAdapter } from '@nestjs/platform-socket.io'; +import { createAdapter } from '@socket.io/redis-adapter'; +import { createClient } from 'redis'; +import { ServerOptions } from 'socket.io'; + +import { config } from '@/config'; + +export class RedisIoAdapter extends IoAdapter { + private adapter: ReturnType; + + async connectToRedis(): Promise { + const pubClient = createClient( + config.cache.type === 'redis' && { + socket: { + host: config.cache.host, + port: config.cache.port, + }, + }, + ); + const subClient = pubClient.duplicate(); + pubClient.on('error', (error) => { + throw error; + }); + subClient.on('error', (error) => { + throw error; + }); + await Promise.all([pubClient.connect(), subClient.connect()]); + this.adapter = createAdapter(pubClient, subClient); + } + + createIOServer(port: number, options?: ServerOptions): any { + const server = super.createIOServer(port, options); + server.adapter(this.adapter); + return server; + } +} diff --git a/docker/.env.example b/docker/.env.example index 985a6fac..15d8284f 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -58,3 +58,9 @@ NEXT_PUBLIC_SSO_ENABLED=false APP_WIDGET_PORT=5173 REACT_APP_WIDGET_API_URL=http://${APP_DOMAIN}:${API_PORT} REACT_APP_WIDGET_CHANNEL=web-channel + +# Redis +APP_REDIS_PORT=9001 +REDIS_ENABLED=false +REDIS_HOST=redis +REDIS_PORT=6379 diff --git a/docker/docker-compose.redis.dev.yml b/docker/docker-compose.redis.dev.yml new file mode 100644 index 00000000..46e82824 --- /dev/null +++ b/docker/docker-compose.redis.dev.yml @@ -0,0 +1,7 @@ +version: "3.8" + +services: + redis: + image: redis/redis-stack:7.2.0-v6 + ports: + - ${APP_REDIS_PORT}:8001 diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml new file mode 100644 index 00000000..78600ff8 --- /dev/null +++ b/docker/docker-compose.redis.yml @@ -0,0 +1,29 @@ +version: "3.8" + +services: + api: + networks: + - cache-network + depends_on: + redis: + condition: service_healthy + + redis: + container_name: redis + image: redis/redis-stack-server:7.2.0-v6 + networks: + - cache-network + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 10s + retries: 5 + start_period: 10s + volumes: + - redis_data:/data + +volumes: + redis_data: + +networks: + cache-network: From f192dc2c031416360683ef3d4013edad6aa1dd0f Mon Sep 17 00:00:00 2001 From: yassinedorbozgithub Date: Sun, 17 Nov 2024 10:39:10 +0100 Subject: [PATCH 2/2] fix: redis feedback updates --- api/src/config/index.ts | 2 +- api/src/config/types.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/config/index.ts b/api/src/config/index.ts index 32f4ec6e..5cb430a6 100644 --- a/api/src/config/index.ts +++ b/api/src/config/index.ts @@ -132,7 +132,7 @@ export const config: Config = { type: process.env.REDIS_ENABLED === 'true' ? 'redis' : 'memory', ttl: 60 * 1000, // Milliseconds max: 100, // Maximum number of items in cache (defaults to 100) - host: process.env.REDIS_HOST, + host: process.env.REDIS_HOST || 'redis', port: parseInt(process.env.REDIS_PORT || '6379'), }, mongo: { diff --git a/api/src/config/types.ts b/api/src/config/types.ts index f2131757..7b472909 100644 --- a/api/src/config/types.ts +++ b/api/src/config/types.ts @@ -21,7 +21,8 @@ type TCacheConfig = { max: number; host: string; port: number; -} & { type: 'memory' | 'redis' }; + type: 'memory' | 'redis'; +}; export type Config = { i18n: { translationFilename: string };