feat: allow to use redis cache instead of memory cache (#832)

* feat(backend/cache): allow to use redis cache instead as memory * feat(frontend/admin): add cache section Add a new section for cache attributes. Also add US translation. --------- Co-authored-by: Jules Lefebvre <jules.lefebvre@diabolocom.com>
2025-05-25 22:16:19 +02:00
parent cfdb29ed4d
commit 85f514316b
8 changed files with 168 additions and 17 deletions
--- a/backend/package-lock.json
+++ b/backend/package-lock.json
@@ -9,6 +9,7 @@
      "version": "1.12.0",
      "dependencies": {
        "@aws-sdk/client-s3": "^3.787.0",
+        "@keyv/redis": "^4.4.0",
        "@nestjs/cache-manager": "^3.0.1",
        "@nestjs/common": "^11.0.17",
        "@nestjs/config": "^4.0.2",
@@ -25,6 +26,7 @@
        "argon2": "^0.41.1",
        "body-parser": "^2.2.0",
        "cache-manager": "^6.4.2",
+        "cacheable": "^1.9.0",
        "clamscan": "^2.4.0",
        "class-transformer": "^0.5.1",
        "class-validator": "^0.14.1",
@@ -2573,6 +2575,22 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
+    "node_modules/@keyv/redis": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/@keyv/redis/-/redis-4.4.0.tgz",
+      "integrity": "sha512-n/KEj3S7crVkoykggqsMUtcjNGvjagGPlJYgO/r6m9hhGZfhp1txJElHxcdJ1ANi/LJoBuOSILj15g6HD2ucqQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@redis/client": "^1.6.0",
+        "cluster-key-slot": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "keyv": "^5.3.3"
+      }
+    },
    "node_modules/@keyv/serialize": {
      "version": "1.0.3",
      "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.0.3.tgz",
@@ -3259,6 +3277,20 @@
        "@prisma/debug": "6.6.0"
      }
    },
+    "node_modules/@redis/client": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz",
+      "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==",
+      "license": "MIT",
+      "dependencies": {
+        "cluster-key-slot": "1.1.2",
+        "generic-pool": "3.9.0",
+        "yallist": "4.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
    "node_modules/@scarf/scarf": {
      "version": "1.4.0",
      "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz",
@@ -5233,6 +5265,16 @@
        "keyv": "^5.3.2"
      }
    },
+    "node_modules/cacheable": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-1.9.0.tgz",
+      "integrity": "sha512-8D5htMCxPDUULux9gFzv30f04Xo3wCnik0oOxKoRTPIBoqA7HtOcJ87uBhQTs3jCfZZTrUBGsYIZOgE0ZRgMAg==",
+      "license": "MIT",
+      "dependencies": {
+        "hookified": "^1.8.2",
+        "keyv": "^5.3.3"
+      }
+    },
    "node_modules/call-bind-apply-helpers": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.1.tgz",
@@ -5444,6 +5486,15 @@
        "node": ">=0.8"
      }
    },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/color": {
      "version": "4.2.3",
      "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
@@ -6897,6 +6948,15 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/generic-pool": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
+      "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
    "node_modules/get-intrinsic": {
      "version": "1.2.7",
      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.7.tgz",
@@ -7134,6 +7194,12 @@
        "node": ">= 0.4"
      }
    },
+    "node_modules/hookified": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.9.0.tgz",
+      "integrity": "sha512-2yEEGqphImtKIe1NXWEhu6yD3hlFR4Mxk4Mtp3XEyScpSt4pQ4ymmXA1zzxZpj99QkFK+nN0nzjeb2+RUi/6CQ==",
+      "license": "MIT"
+    },
    "node_modules/http-errors": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -7634,9 +7700,10 @@
      }
    },
    "node_modules/keyv": {
-      "version": "5.3.2",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.3.2.tgz",
-      "integrity": "sha512-Lji2XRxqqa5Wg+CHLVfFKBImfJZ4pCSccu9eVWK6w4c2SDFLd8JAn1zqTuSFnsxb7ope6rMsnIHfp+eBbRBRZQ==",
+      "version": "5.3.3",
+      "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.3.3.tgz",
+      "integrity": "sha512-Rwu4+nXI9fqcxiEHtbkvoes2X+QfkTRo1TMkPfwzipGsJlJO/z69vqB4FNl9xJ3xCpAcbkvmEabZfPzrwN3+gQ==",
+      "license": "MIT",
      "dependencies": {
        "@keyv/serialize": "^1.0.3"
      }
@@ -10855,8 +10922,7 @@
    "node_modules/yallist": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
-      "dev": true
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
    },
    "node_modules/yaml": {
      "version": "2.7.1",
--- a/backend/package.json
+++ b/backend/package.json
@@ -14,6 +14,7 @@
  },
  "dependencies": {
    "@aws-sdk/client-s3": "^3.787.0",
+    "@keyv/redis": "^4.4.0",
    "@nestjs/cache-manager": "^3.0.1",
    "@nestjs/common": "^11.0.17",
    "@nestjs/config": "^4.0.2",
@@ -30,6 +31,7 @@
    "argon2": "^0.41.1",
    "body-parser": "^2.2.0",
    "cache-manager": "^6.4.2",
+    "cacheable": "^1.9.0",
    "clamscan": "^2.4.0",
    "class-transformer": "^0.5.1",
    "class-validator": "^0.14.1",
--- a/backend/prisma/seed/config.seed.ts
+++ b/backend/prisma/seed/config.seed.ts
@@ -76,6 +76,25 @@ export const configVariables = {
      secret: false,
    },
  },
+  cache: {
+    "redis-enabled": {
+      type: "boolean",
+      defaultValue: "false",
+    },
+    "redis-url": {
+      type: "string",
+      defaultValue: "redis://pingvin-redis:6379",
+      secret: true,
+    },
+    ttl: {
+      type: "number",
+      defaultValue: "60",
+    },
+    maxItems: {
+      type: "number",
+      defaultValue: "1000",
+    },
+  },
  email: {
    enableShareEmailRecipients: {
      type: "boolean",
@@ -419,11 +438,11 @@ const prisma = new PrismaClient({

 async function seedConfigVariables() {
  for (const [category, configVariablesOfCategory] of Object.entries(
-    configVariables
+    configVariables,
  )) {
    let order = 0;
    for (const [name, properties] of Object.entries(
-      configVariablesOfCategory
+      configVariablesOfCategory,
    )) {
      const existingConfigVariable = await prisma.config.findUnique({
        where: { name_category: { name, category } },
@@ -469,7 +488,7 @@ async function migrateConfigVariables() {
      // Update the config variable if it exists in the seed
    } else {
      const variableOrder = Object.keys(
-        configVariables[existingConfigVariable.category]
+        configVariables[existingConfigVariable.category],
      ).indexOf(existingConfigVariable.name);
      await prisma.config.update({
        where: {
--- a/backend/src/app.module.ts
+++ b/backend/src/app.module.ts
@@ -3,9 +3,9 @@ import { Module } from "@nestjs/common";
 import { ScheduleModule } from "@nestjs/schedule";
 import { AuthModule } from "./auth/auth.module";

-import { CacheModule } from "@nestjs/cache-manager";
 import { APP_GUARD } from "@nestjs/core";
 import { ThrottlerGuard, ThrottlerModule } from "@nestjs/throttler";
+import { AppCacheModule } from "./cache/cache.module";
 import { AppController } from "./app.controller";
 import { ClamScanModule } from "./clamscan/clamscan.module";
 import { ConfigModule } from "./config/config.module";
@@ -38,9 +38,7 @@ import { UserModule } from "./user/user.module";
    ClamScanModule,
    ReverseShareModule,
    OAuthModule,
-    CacheModule.register({
-      isGlobal: true,
-    }),
+    AppCacheModule,
  ],
  controllers: [AppController],
  providers: [
--- a/backend/src/cache/cache.module.ts
+++ b/backend/src/cache/cache.module.ts
@@ -0,0 +1,41 @@
+import { Module } from "@nestjs/common";
+import { CacheModule } from "@nestjs/cache-manager";
+import { CacheableMemory } from "cacheable";
+import { createKeyv } from "@keyv/redis";
+import { Keyv } from "keyv";
+import { ConfigModule } from "src/config/config.module";
+import { ConfigService } from "src/config/config.service";
+
+@Module({
+  imports: [
+    ConfigModule,
+    CacheModule.registerAsync({
+      isGlobal: true,
+      imports: [ConfigModule],
+      inject: [ConfigService],
+      useFactory: async (configService: ConfigService) => {
+        const useRedis = configService.get("cache.redis-enabled");
+        const ttl = configService.get("cache.ttl");
+        const max = configService.get("cache.maxItems");
+
+        let config = {
+          ttl,
+          max,
+          stores: [],
+        };
+
+        if (useRedis) {
+          const redisUrl = configService.get("cache.redis-url");
+          config.stores = [
+            new Keyv({ store: new CacheableMemory({ ttl, lruSize: 5000 }) }),
+            createKeyv(redisUrl),
+          ];
+        }
+
+        return config;
+      },
+    }),
+  ],
+  exports: [CacheModule],
+})
+export class AppCacheModule {}
--- a/config.example.yaml
+++ b/config.example.yaml
@@ -29,6 +29,15 @@ share:
  chunkSize: "10000000"
  #The share creation modal automatically appears when a user selects files, eliminating the need to manually click the button.
  autoOpenShareModal: "false"
+cache:
+  #Normally Pingvin Share caches information in memory. If you run multiple instances of Pingvin Share, you need to enable Redis caching to share the cache between the instances.
+  redis-enabled: "false"
+  #Url to connect to the Redis instance used for caching.
+  redis-url: redis://pingvin-redis:6379
+  #Time in second to keep information inside the cache.
+  ttl: "60"
+  #Maximum number of items inside the cache.
+  maxItems: "1000"
 email:
  #Whether to allow email sharing with recipients. Only enable this if SMTP is activated.
  enableShareEmailRecipients: "false"
--- a/frontend/src/components/admin/configuration/ConfigurationNavBar.tsx
+++ b/frontend/src/components/admin/configuration/ConfigurationNavBar.tsx
@@ -13,13 +13,14 @@ import Link from "next/link";
 import { Dispatch, SetStateAction } from "react";
 import {
  TbAt,
+  TbBinaryTree,
+  TbBucket,
  TbMail,
+  TbScale,
+  TbServerBolt,
+  TbSettings,
  TbShare,
  TbSocial,
-  TbBucket,
-  TbBinaryTree,
-  TbSettings,
-  TbScale,
 } from "react-icons/tb";
 import { FormattedMessage } from "react-intl";

@@ -32,6 +33,7 @@ const categories = [
  { name: "LDAP", icon: <TbBinaryTree /> },
  { name: "S3", icon: <TbBucket /> },
  { name: "Legal", icon: <TbScale /> },
+  { name: "Cache", icon: <TbServerBolt /> },
 ];

 const useStyles = createStyles((theme) => ({
--- a/frontend/src/i18n/translations/en-US.ts
+++ b/frontend/src/i18n/translations/en-US.ts
@@ -423,6 +423,7 @@ export default {
  "admin.config.title": "Configuration",
  "admin.config.category.general": "General",
  "admin.config.category.share": "Share",
+  "admin.config.category.cache": "Cache",
  "admin.config.category.email": "Email",
  "admin.config.category.smtp": "SMTP",
  "admin.config.category.oauth": "Social Login",
@@ -446,6 +447,19 @@ export default {
    "Change your logo by uploading a new image. The image must be a PNG and should have the format 1:1.",
  "admin.config.general.logo.placeholder": "Pick image",

+  "admin.config.cache.ttl": "TTL",
+  "admin.config.cache.ttl.description":
+    "Time in second to keep information inside the cache.",
+  "admin.config.cache.max-items": "Maximum items",
+  "admin.config.cache.max-items.description":
+    "Maximum number of items inside the cache.",
+  "admin.config.cache.redis-enabled": "Redis enabled",
+  "admin.config.cache.redis-enabled.description":
+    "Normally Pingvin Share caches information in memory. If you run multiple instances of Pingvin Share, you need to enable Redis caching to share the cache between the instances.",
+  "admin.config.cache.redis-url": "Redis URL",
+  "admin.config.cache.redis-url.description":
+    "Url to connect to the Redis instance used for caching.",
+
  "admin.config.email.enable-share-email-recipients":
    "Enable email recipient sharing",
  "admin.config.email.enable-share-email-recipients.description":