feat: allow to use redis cache instead of memory cache (#832)

* feat(backend/cache): allow to use redis cache instead as memory

* feat(frontend/admin): add cache section

Add a new section for cache attributes. Also add US translation.

---------

Co-authored-by: Jules Lefebvre <jules.lefebvre@diabolocom.com>

backend/package-lock.json

@@ -9,6 +9,7 @@
       "version": "1.12.0",
       "dependencies": {
         "@aws-sdk/client-s3": "^3.787.0",
+        "@keyv/redis": "^4.4.0",
         "@nestjs/cache-manager": "^3.0.1",
         "@nestjs/common": "^11.0.17",
         "@nestjs/config": "^4.0.2",
@@ -25,6 +26,7 @@
         "argon2": "^0.41.1",
         "body-parser": "^2.2.0",
         "cache-manager": "^6.4.2",
+        "cacheable": "^1.9.0",
         "clamscan": "^2.4.0",
         "class-transformer": "^0.5.1",
         "class-validator": "^0.14.1",
@@ -2573,6 +2575,22 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@keyv/redis": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/@keyv/redis/-/redis-4.4.0.tgz",
+      "integrity": "sha512-n/KEj3S7crVkoykggqsMUtcjNGvjagGPlJYgO/r6m9hhGZfhp1txJElHxcdJ1ANi/LJoBuOSILj15g6HD2ucqQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@redis/client": "^1.6.0",
+        "cluster-key-slot": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "keyv": "^5.3.3"
+      }
+    },
     "node_modules/@keyv/serialize": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.0.3.tgz",
@@ -3259,6 +3277,20 @@
         "@prisma/debug": "6.6.0"
       }
     },
+    "node_modules/@redis/client": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz",
+      "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==",
+      "license": "MIT",
+      "dependencies": {
+        "cluster-key-slot": "1.1.2",
+        "generic-pool": "3.9.0",
+        "yallist": "4.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@scarf/scarf": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz",
@@ -5233,6 +5265,16 @@
         "keyv": "^5.3.2"
       }
     },
+    "node_modules/cacheable": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-1.9.0.tgz",
+      "integrity": "sha512-8D5htMCxPDUULux9gFzv30f04Xo3wCnik0oOxKoRTPIBoqA7HtOcJ87uBhQTs3jCfZZTrUBGsYIZOgE0ZRgMAg==",
+      "license": "MIT",
+      "dependencies": {
+        "hookified": "^1.8.2",
+        "keyv": "^5.3.3"
+      }
+    },
     "node_modules/call-bind-apply-helpers": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.1.tgz",
@@ -5444,6 +5486,15 @@
         "node": ">=0.8"
       }
     },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/color": {
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
@@ -6897,6 +6948,15 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/generic-pool": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
+      "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/get-intrinsic": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.7.tgz",
@@ -7134,6 +7194,12 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/hookified": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.9.0.tgz",
+      "integrity": "sha512-2yEEGqphImtKIe1NXWEhu6yD3hlFR4Mxk4Mtp3XEyScpSt4pQ4ymmXA1zzxZpj99QkFK+nN0nzjeb2+RUi/6CQ==",
+      "license": "MIT"
+    },
     "node_modules/http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -7634,9 +7700,10 @@
       }
     },
     "node_modules/keyv": {
-      "version": "5.3.2",
+      "version": "5.3.3",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.3.2.tgz",
+      "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.3.3.tgz",
-      "integrity": "sha512-Lji2XRxqqa5Wg+CHLVfFKBImfJZ4pCSccu9eVWK6w4c2SDFLd8JAn1zqTuSFnsxb7ope6rMsnIHfp+eBbRBRZQ==",
+      "integrity": "sha512-Rwu4+nXI9fqcxiEHtbkvoes2X+QfkTRo1TMkPfwzipGsJlJO/z69vqB4FNl9xJ3xCpAcbkvmEabZfPzrwN3+gQ==",
+      "license": "MIT",
       "dependencies": {
         "@keyv/serialize": "^1.0.3"
       }
@@ -10855,8 +10922,7 @@
     "node_modules/yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
-      "dev": true
     },
     "node_modules/yaml": {
       "version": "2.7.1",

backend/package.json

@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.787.0",
+    "@keyv/redis": "^4.4.0",
     "@nestjs/cache-manager": "^3.0.1",
     "@nestjs/common": "^11.0.17",
     "@nestjs/config": "^4.0.2",
@@ -30,6 +31,7 @@
     "argon2": "^0.41.1",
     "body-parser": "^2.2.0",
     "cache-manager": "^6.4.2",
+    "cacheable": "^1.9.0",
     "clamscan": "^2.4.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",

backend/prisma/seed/config.seed.ts

@@ -76,6 +76,25 @@ export const configVariables = {
       secret: false,
     },
   },
+  cache: {
+    "redis-enabled": {
+      type: "boolean",
+      defaultValue: "false",
+    },
+    "redis-url": {
+      type: "string",
+      defaultValue: "redis://pingvin-redis:6379",
+      secret: true,
+    },
+    ttl: {
+      type: "number",
+      defaultValue: "60",
+    },
+    maxItems: {
+      type: "number",
+      defaultValue: "1000",
+    },
+  },
   email: {
     enableShareEmailRecipients: {
       type: "boolean",
@@ -419,11 +438,11 @@ const prisma = new PrismaClient({
 
 async function seedConfigVariables() {
   for (const [category, configVariablesOfCategory] of Object.entries(
-    configVariables
+    configVariables,
   )) {
     let order = 0;
     for (const [name, properties] of Object.entries(
-      configVariablesOfCategory
+      configVariablesOfCategory,
     )) {
       const existingConfigVariable = await prisma.config.findUnique({
         where: { name_category: { name, category } },
@@ -469,7 +488,7 @@ async function migrateConfigVariables() {
       // Update the config variable if it exists in the seed
     } else {
       const variableOrder = Object.keys(
-        configVariables[existingConfigVariable.category]
+        configVariables[existingConfigVariable.category],
       ).indexOf(existingConfigVariable.name);
       await prisma.config.update({
         where: {

backend/src/app.module.ts

@@ -3,9 +3,9 @@ import { Module } from "@nestjs/common";
 import { ScheduleModule } from "@nestjs/schedule";
 import { AuthModule } from "./auth/auth.module";
 
-import { CacheModule } from "@nestjs/cache-manager";
 import { APP_GUARD } from "@nestjs/core";
 import { ThrottlerGuard, ThrottlerModule } from "@nestjs/throttler";
+import { AppCacheModule } from "./cache/cache.module";
 import { AppController } from "./app.controller";
 import { ClamScanModule } from "./clamscan/clamscan.module";
 import { ConfigModule } from "./config/config.module";
@@ -38,9 +38,7 @@ import { UserModule } from "./user/user.module";
     ClamScanModule,
     ReverseShareModule,
     OAuthModule,
-    CacheModule.register({
+    AppCacheModule,
-      isGlobal: true,
-    }),
   ],
   controllers: [AppController],
   providers: [

backend/src/cache/cache.module.ts

@@ -0,0 +1,41 @@
+import { Module } from "@nestjs/common";
+import { CacheModule } from "@nestjs/cache-manager";
+import { CacheableMemory } from "cacheable";
+import { createKeyv } from "@keyv/redis";
+import { Keyv } from "keyv";
+import { ConfigModule } from "src/config/config.module";
+import { ConfigService } from "src/config/config.service";
+
+@Module({
+  imports: [
+    ConfigModule,
+    CacheModule.registerAsync({
+      isGlobal: true,
+      imports: [ConfigModule],
+      inject: [ConfigService],
+      useFactory: async (configService: ConfigService) => {
+        const useRedis = configService.get("cache.redis-enabled");
+        const ttl = configService.get("cache.ttl");
+        const max = configService.get("cache.maxItems");
+
+        let config = {
+          ttl,
+          max,
+          stores: [],
+        };
+
+        if (useRedis) {
+          const redisUrl = configService.get("cache.redis-url");
+          config.stores = [
+            new Keyv({ store: new CacheableMemory({ ttl, lruSize: 5000 }) }),
+            createKeyv(redisUrl),
+          ];
+        }
+
+        return config;
+      },
+    }),
+  ],
+  exports: [CacheModule],
+})
+export class AppCacheModule {}

config.example.yaml

@@ -29,6 +29,15 @@ share:
   chunkSize: "10000000"
   #The share creation modal automatically appears when a user selects files, eliminating the need to manually click the button.
   autoOpenShareModal: "false"
+cache:
+  #Normally Pingvin Share caches information in memory. If you run multiple instances of Pingvin Share, you need to enable Redis caching to share the cache between the instances.
+  redis-enabled: "false"
+  #Url to connect to the Redis instance used for caching.
+  redis-url: redis://pingvin-redis:6379
+  #Time in second to keep information inside the cache.
+  ttl: "60"
+  #Maximum number of items inside the cache.
+  maxItems: "1000"
 email:
   #Whether to allow email sharing with recipients. Only enable this if SMTP is activated.
   enableShareEmailRecipients: "false"

frontend/src/components/admin/configuration/ConfigurationNavBar.tsx

@@ -13,13 +13,14 @@ import Link from "next/link";
 import { Dispatch, SetStateAction } from "react";
 import {
   TbAt,
+  TbBinaryTree,
+  TbBucket,
   TbMail,
+  TbScale,
+  TbServerBolt,
+  TbSettings,
   TbShare,
   TbSocial,
-  TbBucket,
-  TbBinaryTree,
-  TbSettings,
-  TbScale,
 } from "react-icons/tb";
 import { FormattedMessage } from "react-intl";
 
@@ -32,6 +33,7 @@ const categories = [
   { name: "LDAP", icon: <TbBinaryTree /> },
   { name: "S3", icon: <TbBucket /> },
   { name: "Legal", icon: <TbScale /> },
+  { name: "Cache", icon: <TbServerBolt /> },
 ];
 
 const useStyles = createStyles((theme) => ({

frontend/src/i18n/translations/en-US.ts

@@ -423,6 +423,7 @@ export default {
   "admin.config.title": "Configuration",
   "admin.config.category.general": "General",
   "admin.config.category.share": "Share",
+  "admin.config.category.cache": "Cache",
   "admin.config.category.email": "Email",
   "admin.config.category.smtp": "SMTP",
   "admin.config.category.oauth": "Social Login",
@@ -446,6 +447,19 @@ export default {
     "Change your logo by uploading a new image. The image must be a PNG and should have the format 1:1.",
   "admin.config.general.logo.placeholder": "Pick image",
 
+  "admin.config.cache.ttl": "TTL",
+  "admin.config.cache.ttl.description":
+    "Time in second to keep information inside the cache.",
+  "admin.config.cache.max-items": "Maximum items",
+  "admin.config.cache.max-items.description":
+    "Maximum number of items inside the cache.",
+  "admin.config.cache.redis-enabled": "Redis enabled",
+  "admin.config.cache.redis-enabled.description":
+    "Normally Pingvin Share caches information in memory. If you run multiple instances of Pingvin Share, you need to enable Redis caching to share the cache between the instances.",
+  "admin.config.cache.redis-url": "Redis URL",
+  "admin.config.cache.redis-url.description":
+    "Url to connect to the Redis instance used for caching.",
+
   "admin.config.email.enable-share-email-recipients":
     "Enable email recipient sharing",
   "admin.config.email.enable-share-email-recipients.description":