Files
pingvin-share/backend/src/file/local.service.ts
Mattia Müggler 5a54fe4cb7 feat: add support for S3 as a storage provider (#659)
* add s3

* instance the s3 client dynamically

* refactor code

* fix format

* add docs

* add docs

* fix issue with s3 upload if you use the base path,
fix issue with archiving -> disable archiving for s3

* split file service in local and s3 file service and fix s3 upload chunking

* add working download/view

* add new features to local service (from main branch)

* revert s3 service and add working delete/remove functionality

* refactor s3 service

* Update backend/src/file/s3.service.ts

Co-authored-by: Elias Schneider <login@eliasschneider.com>

* Update frontend/src/components/admin/configuration/ConfigurationNavBar.tsx

Co-authored-by: Elias Schneider <login@eliasschneider.com>

* Update docs/docs/setup/s3.md

Co-authored-by: Elias Schneider <login@eliasschneider.com>

* Update backend/prisma/seed/config.seed.ts

Co-authored-by: Elias Schneider <login@eliasschneider.com>

* add note for ZIP archive in docs

* create logger instance

* make s3 instance dynamic

* add icon import

* remove console.logs

* add correct pdf viewing format

* add storage provider to share

* refactor: run formatter

* chore: add prisma migration

* fix: don't expose `storageProvider`

* chore: improve config variables description

---------

Co-authored-by: Elias Schneider <login@eliasschneider.com>
2024-12-19 12:06:49 +01:00

162 lines
4.4 KiB
TypeScript

import {
BadRequestException,
HttpException,
HttpStatus,
Injectable,
InternalServerErrorException,
NotFoundException,
} from "@nestjs/common";
import * as crypto from "crypto";
import { createReadStream } from "fs";
import * as fs from "fs/promises";
import * as mime from "mime-types";
import { ConfigService } from "src/config/config.service";
import { PrismaService } from "src/prisma/prisma.service";
import { validate as isValidUUID } from "uuid";
import { SHARE_DIRECTORY } from "../constants";
@Injectable()
export class LocalFileService {
constructor(
private prisma: PrismaService,
private config: ConfigService,
) {}
async create(
data: string,
chunk: { index: number; total: number },
file: { id?: string; name: string },
shareId: string,
) {
if (!file.id) {
file.id = crypto.randomUUID();
} else if (!isValidUUID(file.id)) {
throw new BadRequestException("Invalid file ID format");
}
const share = await this.prisma.share.findUnique({
where: { id: shareId },
include: { files: true, reverseShare: true },
});
if (share.uploadLocked)
throw new BadRequestException("Share is already completed");
let diskFileSize: number;
try {
diskFileSize = (
await fs.stat(`${SHARE_DIRECTORY}/${shareId}/${file.id}.tmp-chunk`)
).size;
} catch {
diskFileSize = 0;
}
// If the sent chunk index and the expected chunk index doesn't match throw an error
const chunkSize = this.config.get("share.chunkSize");
const expectedChunkIndex = Math.ceil(diskFileSize / chunkSize);
if (expectedChunkIndex != chunk.index)
throw new BadRequestException({
message: "Unexpected chunk received",
error: "unexpected_chunk_index",
expectedChunkIndex,
});
const buffer = Buffer.from(data, "base64");
// Check if there is enough space on the server
const space = await fs.statfs(SHARE_DIRECTORY);
const availableSpace = space.bavail * space.bsize;
if (availableSpace < buffer.byteLength) {
throw new InternalServerErrorException("Not enough space on the server");
}
// Check if share size limit is exceeded
const fileSizeSum = share.files.reduce(
(n, { size }) => n + parseInt(size),
0,
);
const shareSizeSum = fileSizeSum + diskFileSize + buffer.byteLength;
if (
shareSizeSum > this.config.get("share.maxSize") ||
(share.reverseShare?.maxShareSize &&
shareSizeSum > parseInt(share.reverseShare.maxShareSize))
) {
throw new HttpException(
"Max share size exceeded",
HttpStatus.PAYLOAD_TOO_LARGE,
);
}
await fs.appendFile(
`${SHARE_DIRECTORY}/${shareId}/${file.id}.tmp-chunk`,
buffer,
);
const isLastChunk = chunk.index == chunk.total - 1;
if (isLastChunk) {
await fs.rename(
`${SHARE_DIRECTORY}/${shareId}/${file.id}.tmp-chunk`,
`${SHARE_DIRECTORY}/${shareId}/${file.id}`,
);
const fileSize = (
await fs.stat(`${SHARE_DIRECTORY}/${shareId}/${file.id}`)
).size;
await this.prisma.file.create({
data: {
id: file.id,
name: file.name,
size: fileSize.toString(),
share: { connect: { id: shareId } },
},
});
}
return file;
}
async get(shareId: string, fileId: string) {
const fileMetaData = await this.prisma.file.findUnique({
where: { id: fileId },
});
if (!fileMetaData) throw new NotFoundException("File not found");
const file = createReadStream(`${SHARE_DIRECTORY}/${shareId}/${fileId}`);
return {
metaData: {
mimeType: mime.contentType(fileMetaData.name.split(".").pop()),
...fileMetaData,
size: fileMetaData.size,
},
file,
};
}
async remove(shareId: string, fileId: string) {
const fileMetaData = await this.prisma.file.findUnique({
where: { id: fileId },
});
if (!fileMetaData) throw new NotFoundException("File not found");
await fs.unlink(`${SHARE_DIRECTORY}/${shareId}/${fileId}`);
await this.prisma.file.delete({ where: { id: fileId } });
}
async deleteAllFiles(shareId: string) {
await fs.rm(`${SHARE_DIRECTORY}/${shareId}`, {
recursive: true,
force: true,
});
}
getZip(shareId: string) {
return createReadStream(`${SHARE_DIRECTORY}/${shareId}/archive.zip`);
}
}