feat(s3): stream s3 content over a zip file (#822)

* chore: update package.json and enhance file handling in backend

* refactor: remove StreamResponseFilter and update getZip method for improved file handling

- Deleted StreamResponseFilter as it was no longer needed.
- Updated getZip method in LocalFileService to return a Promise and handle errors more effectively.
- Adjusted getZip method in FileService to await the storage service's getZip call.

* refactor: remove unused UseFilters import in file.controller.ts

backend/src/file/file.controller.ts

@@ -54,7 +54,7 @@ export class FileController {
     @Res({ passthrough: true }) res: Response,
     @Param("shareId") shareId: string,
   ) {
-    const zipStream = this.fileService.getZip(shareId);
+    const zipStream = await this.fileService.getZip(shareId);
 
     res.set({
       "Content-Type": "application/zip",

backend/src/file/file.service.ts

@@ -59,9 +59,9 @@ export class FileService {
     return storageService.deleteAllFiles(shareId);
   }
 
-  getZip(shareId: string) {
+  async getZip(shareId: string): Promise<Readable> {
     const storageService = this.getStorageService();
-    return storageService.getZip(shareId) as Readable;
+    return await storageService.getZip(shareId);
   }
 
   private async streamToUint8Array(stream: Readable): Promise<Uint8Array> {

backend/src/file/local.service.ts

@@ -14,6 +14,7 @@ import { ConfigService } from "src/config/config.service";
 import { PrismaService } from "src/prisma/prisma.service";
 import { validate as isValidUUID } from "uuid";
 import { SHARE_DIRECTORY } from "../constants";
+import { Readable } from "stream";
 
 @Injectable()
 export class LocalFileService {
@@ -155,7 +156,19 @@ export class LocalFileService {
     });
   }
 
-  getZip(shareId: string) {
-    return createReadStream(`${SHARE_DIRECTORY}/${shareId}/archive.zip`);
+  async getZip(shareId: string): Promise<Readable> {
+    return new Promise((resolve, reject) => {
+      const zipStream = createReadStream(
+        `${SHARE_DIRECTORY}/${shareId}/archive.zip`,
+      );
+
+      zipStream.on("error", (err) => {
+        reject(new InternalServerErrorException(err));
+      });
+
+      zipStream.on("open", () => {
+        resolve(zipStream);
+      });
+    });
   }
 }

backend/src/file/s3.service.ts

@@ -25,6 +25,7 @@ import * as mime from "mime-types";
 import { File } from "./file.service";
 import { Readable } from "stream";
 import { validate as isValidUUID } from "uuid";
+import * as archiver from "archiver";
 
 @Injectable()
 export class S3FileService {
@@ -275,7 +276,8 @@ export class S3FileService {
   }
 
   getS3Instance(): S3Client {
-    const checksumCalculation = this.config.get("s3.useChecksum") === true ? null : "WHEN_REQUIRED";
+    const checksumCalculation =
+      this.config.get("s3.useChecksum") === true ? null : "WHEN_REQUIRED";
 
     return new S3Client({
       endpoint: this.config.get("s3.endpoint"),
@@ -290,10 +292,95 @@ export class S3FileService {
     });
   }
 
-  getZip() {
-    throw new BadRequestException(
-      "ZIP download is not supported with S3 storage",
-    );
+  getZip(shareId: string) {
+    return new Promise<Readable>(async (resolve, reject) => {
+      const s3Instance = this.getS3Instance();
+      const bucketName = this.config.get("s3.bucketName");
+      const compressionLevel = this.config.get("share.zipCompressionLevel");
+
+      const prefix = `${this.getS3Path()}${shareId}/`;
+
+      try {
+        const listResponse = await s3Instance.send(
+          new ListObjectsV2Command({
+            Bucket: bucketName,
+            Prefix: prefix,
+          }),
+        );
+
+        if (!listResponse.Contents || listResponse.Contents.length === 0) {
+          throw new NotFoundException(`No files found for share ${shareId}`);
+        }
+
+        const archive = archiver("zip", {
+          zlib: { level: parseInt(compressionLevel) },
+        });
+
+        archive.on("error", (err) => {
+          this.logger.error("Archive error", err);
+          reject(new InternalServerErrorException("Error creating ZIP file"));
+        });
+
+        const fileKeys = listResponse.Contents.filter(
+          (object) => object.Key && object.Key !== prefix,
+        ).map((object) => object.Key as string);
+
+        if (fileKeys.length === 0) {
+          throw new NotFoundException(
+            `No valid files found for share ${shareId}`,
+          );
+        }
+
+        let filesAdded = 0;
+
+        const processNextFile = async (index: number) => {
+          if (index >= fileKeys.length) {
+            archive.finalize();
+            return;
+          }
+
+          const key = fileKeys[index];
+          const fileName = key.replace(prefix, "");
+
+          try {
+            const response = await s3Instance.send(
+              new GetObjectCommand({
+                Bucket: bucketName,
+                Key: key,
+              }),
+            );
+
+            if (response.Body instanceof Readable) {
+              const fileStream = response.Body;
+
+              fileStream.on("end", () => {
+                filesAdded++;
+                processNextFile(index + 1);
+              });
+
+              fileStream.on("error", (err) => {
+                this.logger.error(`Error streaming file ${fileName}`, err);
+                processNextFile(index + 1);
+              });
+
+              archive.append(fileStream, { name: fileName });
+            } else {
+              processNextFile(index + 1);
+            }
+          } catch (error) {
+            this.logger.error(`Error processing file ${fileName}`, error);
+            processNextFile(index + 1);
+          }
+        };
+
+        resolve(archive);
+        processNextFile(0);
+      } catch (error) {
+        this.logger.error("Error creating ZIP file", error);
+
+        reject(new InternalServerErrorException("Error creating ZIP file"));
+      }
+    });
   }
 
   getS3Path(): string {