iio/pingvin-share
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: chunk uploads (#76)

Browse Source 
* add first concept

* finished first concept

* allow 3 uploads at same time

* retry if chunk failed

* updated clean temporary files job

* fix throttling for chunk uploads

* update tests

* remove multer

* migrate from `MAX_FILE_SIZE` to `MAX_SHARE_SIZE`

* improve error handling if file failed to upload

* fix promise limit

* improve file progress
This commit is contained in:
Elias Schneider
2023-01-09 11:43:48 +01:00
committed by GitHub
parent a5bef5d4a4
commit 653d72bcb9
20 changed files with 364 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
frontend/src/components/upload/Dropzone.tsx
View File

@@ -33,9 +33,11 @@ const useStyles = createStyles((theme) => ({
const Dropzone = ({
isUploading,
files,
setFiles,
}: {
isUploading: boolean;
files: FileUpload[];
setFiles: Dispatch<SetStateAction<FileUpload[]>>;
}) => {
const config = useConfig();
@@ -45,18 +47,27 @@ const Dropzone = ({
return (
<div className={classes.wrapper}>
<MantineDropzone
maxSize={parseInt(config.get("MAX_FILE_SIZE"))}
onReject={(e) => {
toast.error(e[0].errors[0].message);
}}
disabled={isUploading}
openRef={openRef as ForwardedRef<() => void>}
onDrop={(files) => {
const newFiles = files.map((file) => {
(file as FileUpload).uploadingProgress = 0;
return file as FileUpload;
});
setFiles(newFiles);
onDrop={(newFiles: FileUpload[]) => {
const fileSizeSum = [...newFiles, ...files].reduce((n, { size }) => n + size, 0);
if (fileSizeSum > config.get("MAX_SHARE_SIZE")) {
toast.error(
`Your files exceed the maximum share size of ${byteStringToHumanSizeString(
config.get("MAX_SHARE_SIZE")
)}.`
);
} else {
newFiles = newFiles.map((newFile) => {
newFile.uploadingProgress = 0;
return newFile;
});
setFiles([...newFiles, ...files]);
}
}}
className={classes.dropzone}
radius="md"
@@ -71,7 +82,8 @@ const Dropzone = ({
<Text align="center" size="sm" mt="xs" color="dimmed">
Drag&apos;n&apos;drop files here to start your share. We can accept
only files that are less than{" "}
{byteStringToHumanSizeString(config.get("MAX_FILE_SIZE"))} in size.
{byteStringToHumanSizeString(config.get("MAX_SHARE_SIZE"))} in
total.
</Text>
</div>
</MantineDropzone>

6
frontend/src/components/upload/UploadProgressIndicator.tsx
View File

@@ -1,5 +1,5 @@
import { RingProgress } from "@mantine/core";
import { TbCircleCheck, TbCircleX } from "react-icons/tb";
import { Loader, RingProgress } from "@mantine/core";
import { TbCircleCheck } from "react-icons/tb";
const UploadProgressIndicator = ({ progress }: { progress: number }) => {
if (progress > 0 && progress < 100) {
return (
@@ -12,7 +12,7 @@ const UploadProgressIndicator = ({ progress }: { progress: number }) => {
} else if (progress >= 100) {
return <TbCircleCheck color="green" size={22} />;
} else {
return <TbCircleX color="red" size={22} />;
return <Loader color="red" size={19} />;
}
};

1
frontend/src/components/upload/modals/showCompletedUploadModal.tsx
View File

@@ -37,6 +37,7 @@ const Body = ({ share }: { share: Share }) => {
return (
<Stack align="stretch">
<TextInput
readOnly
variant="filled"
value={link}
rightSection={

158
frontend/src/pages/upload.tsx
View File

@@ -1,5 +1,7 @@
import { Button, Group } from "@mantine/core";
import { useModals } from "@mantine/modals";
import { cleanNotifications } from "@mantine/notifications";
import { AxiosError } from "axios";
import { useRouter } from "next/router";
import pLimit from "p-limit";
import { useEffect, useState } from "react";
@@ -15,8 +17,10 @@ import { FileUpload } from "../types/File.type";
import { CreateShare, Share } from "../types/share.type";
import toast from "../utils/toast.util";
let createdShare: Share;
const promiseLimit = pLimit(3);
const chunkSize = 10 * 1024 * 1024; // 10MB
let errorToastShown = false;
let createdShare: Share;
const Upload = () => {
const router = useRouter();
@@ -29,70 +33,122 @@ const Upload = () => {
const uploadFiles = async (share: CreateShare) => {
setisUploading(true);
try {
setFiles((files) =>
files.map((file) => {
file.uploadingProgress = 1;
return file;
})
);
createdShare = await shareService.create(share);
createdShare = await shareService.create(share);
const uploadPromises = files.map((file, i) => {
// Callback to indicate current upload progress
const progressCallBack = (progress: number) => {
setFiles((files) => {
return files.map((file, callbackIndex) => {
if (i == callbackIndex) {
const fileUploadPromises = files.map(async (file, fileIndex) =>
// Limit the number of concurrent uploads to 3
promiseLimit(async () => {
let fileId: string;
const setFileProgress = (progress: number) => {
setFiles((files) =>
files.map((file, callbackIndex) => {
if (fileIndex == callbackIndex) {
file.uploadingProgress = progress;
}
return file;
});
});
})
);
};
try {
return promiseLimit(() =>
shareService.uploadFile(share.id, file, progressCallBack)
);
} catch {
file.uploadingProgress = -1;
}
});
setFileProgress(1);
await Promise.all(uploadPromises);
} catch (e) {
toast.axiosError(e);
setisUploading(false);
}
const chunks = Math.ceil(file.size / chunkSize);
for (let chunkIndex = 0; chunkIndex < chunks; chunkIndex++) {
const from = chunkIndex * chunkSize;
const to = from + chunkSize;
const blob = file.slice(from, to);
try {
await new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = async (event) =>
await shareService
.uploadFile(
createdShare.id,
event,
{
id: fileId,
name: file.name,
},
chunkIndex,
Math.ceil(file.size / chunkSize)
)
.then((response) => {
fileId = response.id;
resolve(response);
})
.catch(reject);
reader.readAsDataURL(blob);
});
setFileProgress(((chunkIndex + 1) / chunks) * 100);
} catch (e) {
if (
e instanceof AxiosError &&
e.response?.data.error == "unexpected_chunk_index"
) {
// Retry with the expected chunk index
chunkIndex = e.response!.data!.expectedChunkIndex - 1;
continue;
} else {
setFileProgress(-1);
// Retry after 5 seconds
await new Promise((resolve) => setTimeout(resolve, 5000));
chunkIndex = -1;
continue;
}
}
}
})
);
Promise.all(fileUploadPromises);
};
useEffect(() => {
// Check if there are any files that failed to upload
const fileErrorCount = files.filter(
(file) => file.uploadingProgress == -1
).length;
if (fileErrorCount > 0) {
if (!errorToastShown) {
toast.error(
`${fileErrorCount} file(s) failed to upload. Trying again.`,
{
disallowClose: true,
autoClose: false,
}
);
}
errorToastShown = true;
} else {
cleanNotifications();
errorToastShown = false;
}
// Complete share
if (
files.length > 0 &&
files.every(
(file) => file.uploadingProgress >= 100 || file.uploadingProgress == -1
)
files.every((file) => file.uploadingProgress >= 100) &&
fileErrorCount == 0
) {
const fileErrorCount = files.filter(
(file) => file.uploadingProgress == -1
).length;
setisUploading(false);
if (fileErrorCount > 0) {
toast.error(`${fileErrorCount} file(s) failed to upload. Try again.`);
} else {
shareService
.completeShare(createdShare.id)
.then(() => {
showCompletedUploadModal(modals, createdShare);
setFiles([]);
})
.catch(() =>
toast.error("An error occurred while finishing your share.")
);
}
shareService
.completeShare(createdShare.id)
.then(() => {
setisUploading(false);
showCompletedUploadModal(modals, createdShare);
setFiles([]);
})
.catch(() =>
toast.error("An error occurred while finishing your share.")
);
}
}, [files]);
if (!user && !config.get("ALLOW_UNAUTHENTICATED_SHARES")) {
router.replace("/");
} else {
@@ -120,7 +176,7 @@ const Upload = () => {
Share
</Button>
</Group>
<Dropzone setFiles={setFiles} isUploading={isUploading} />
<Dropzone files={files} setFiles={setFiles} isUploading={isUploading} />
{files.length > 0 && <FileList files={files} setFiles={setFiles} />}
</>
);

36
frontend/src/services/share.service.ts
View File

@@ -1,3 +1,4 @@
import { FileUploadResponse } from "../types/File.type";
import {
CreateShare,
MyShare,
@@ -74,22 +75,27 @@ const downloadFile = async (shareId: string, fileId: string) => {
const uploadFile = async (
shareId: string,
file: File,
progressCallBack: (uploadingProgress: number) => void
) => {
let formData = new FormData();
formData.append("file", file);
readerEvent: ProgressEvent<FileReader>,
file: {
id?: string;
name: string;
},
chunkIndex: number,
totalChunks: number
): Promise<FileUploadResponse> => {
const data = readerEvent.target!.result;
const response = await api.post(`shares/${shareId}/files`, formData, {
onUploadProgress: (progressEvent) => {
const uploadingProgress = Math.round(
(100 * progressEvent.loaded) / (progressEvent.total ?? 1)
);
if (uploadingProgress < 100) progressCallBack(uploadingProgress);
},
});
progressCallBack(100);
return response;
return (
await api.post(`shares/${shareId}/files`, data, {
headers: { "Content-Type": "application/octet-stream" },
params: {
id: file.id,
name: file.name,
chunkIndex,
totalChunks,
},
})
).data;
};
export default {

2
frontend/src/types/File.type.ts
View File

@@ -1 +1,3 @@
export type FileUpload = File & { uploadingProgress: number };
export type FileUploadResponse = {id: string, name: string}

16
frontend/src/utils/toast.util.tsx
View File

@@ -1,25 +1,33 @@
import { showNotification } from "@mantine/notifications";
import { NotificationProps, showNotification } from "@mantine/notifications";
import { TbCheck, TbX } from "react-icons/tb";
const error = (message: string) =>
const error = (message: string, config?: Omit<NotificationProps, "message">) =>
showNotification({
icon: <TbX />,
color: "red",
radius: "md",
title: "Error",
message: message,
autoClose: true,
...config,
});
const axiosError = (axiosError: any) =>
error(axiosError?.response?.data?.message ?? "An unknown error occurred");
const success = (message: string) =>
const success = (
message: string,
config?: Omit<NotificationProps, "message">
) =>
showNotification({
icon: <TbCheck />,
color: "green",
radius: "md",
title: "Success",
message: message,
autoClose: true,
...config,
});
const toast = {