Add chunked uploads (Resumable curently broken)

2026-04-14 16:39:43 +02:00
parent 8ae5dfc483
commit 6065b4d95f
8 changed files with 428 additions and 84 deletions
--- a/static/js/upload.js
+++ b/static/js/upload.js
@@ -4,7 +4,6 @@ const uploadBtn = document.getElementById('uploadBtn');
 const cancelBtn = document.getElementById('cancelBtn');

 const progressText = document.getElementById("progress-text");
-const statsText = document.getElementById("stats-text");
 const speedText = document.getElementById("speed-text");
 const etaText = document.getElementById("eta-text");
 const progressBar = document.getElementById("progress-bar");
@@ -12,13 +11,17 @@ const progressContainer = document.getElementById("progress-container");

 let currentXhr = null;

-function formatBytes(bytes, decimals = 2) {
+const CHUNK_THRESHOLD = 1024 * 1024 * 1024;
+const CHUNK_SIZE = 10 * 1024 * 1024;
+const MAX_PARALLEL_UPLOADS = 4;
+const MAX_RETRIES = 3;
+
+function formatBytes(bytes) {
    if (bytes === 0) return '0 Bytes';
    const k = 1024;
-    const dm = decimals < 0 ? 0 : decimals;
    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
    const i = Math.floor(Math.log(bytes) / Math.log(k));
-    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+    return (bytes / Math.pow(k, i)).toFixed(2) + ' ' + sizes[i];
 }

 function formatTime(seconds) {
@@ -30,18 +33,52 @@ function formatTime(seconds) {
        h > 0 ? h : null,
        (h > 0 ? m.toString().padStart(2, '0') : m),
        s.toString().padStart(2, '0')
-    ].filter(x => x !== null).join(':');
+    ].filter(Boolean).join(':');
+}
+
+function startUI() {
+    uploadBtn.disabled = true;
+    uploadBtn.innerText = "UPLOADING...";
+    cancelBtn.classList.remove('hidden');
+    progressContainer.classList.remove("hidden");
+    progressText.classList.remove("hidden");
+    document.getElementById("stats-text").classList.remove("hidden");
+}
+
+function updateProgress(loaded, total, startTime) {
+    const percent = Math.round((loaded / total) * 100);
+    progressBar.style.width = percent + "%";
+    progressText.innerText = percent + "%";
+
+    const elapsed = (Date.now() - startTime) / 1000;
+    if (elapsed <= 0) return;
+
+    const speed = loaded / elapsed;
+    const remaining = total - loaded;
+
+    speedText.innerText = formatBytes(speed) + "/S";
+    etaText.innerText = formatTime(remaining / speed);
+}
+
+function redirect(data) {
+    const key = data.view_key;
+    if (!key) {
+        alert("Invalid server response");
+        return;
+    }
+    window.location.href = "/f/" + key;
 }

 zone.onclick = () => input.click();

-zone.ondragover = (e) => {
+zone.ondragover = e => {
    e.preventDefault();
    zone.classList.add('active');
 };
+
 zone.ondragleave = () => zone.classList.remove('active');

-zone.ondrop = (e) => {
+zone.ondrop = e => {
    e.preventDefault();
    zone.classList.remove('active');
    if (e.dataTransfer.files.length) {
@@ -51,94 +88,69 @@ zone.ondrop = (e) => {
 };

 input.onchange = () => {
-    if (input.files.length) {
-        showFiles(input.files);
-        uploadBtn.disabled = false;
-    } else {
+    const files = Array.from(input.files || []);
+    if (!files.length) {
        uploadBtn.disabled = true;
+        return;
    }
-};

-function showFiles(fileList) {
-    const files = Array.from(fileList || []);
-    if (files.length === 0) return;
-    const total = files.reduce((acc, f) => acc + f.size, 0);
-    if (files.length === 1) {
-        document.getElementById('dz-text').innerText =
-            `${files[0].name}  [${formatBytes(files[0].size)}]`;
-    } else {
-        document.getElementById('dz-text').innerText =
-            `${files.length} FILES  [${formatBytes(total)}]  — will be zipped`;
-    }
-}
+    const total = files.reduce((a, f) => a + f.size, 0);
+    document.getElementById('dz-text').innerText =
+        files.length === 1
+            ? `${files[0].name}  [${formatBytes(files[0].size)}]`
+            : `${files.length} FILES  [${formatBytes(total)}]`;
+
+    uploadBtn.disabled = false;
+};

 uploadBtn.onclick = () => {
-    if (!input.files.length) return;
-    if (input.files.length === 1) {
-        handleUploadSingle(input.files[0]);
+    const files = input.files;
+    if (!files.length) return;
+
+    if (files.length === 1 && files[0].size > CHUNK_THRESHOLD) {
+        uploadChunked(files[0]);
+    } else if (files.length === 1) {
+        uploadSingle(files[0]);
    } else {
-        handleUploadMulti(input.files);
+        uploadMulti(files);
    }
 };

-cancelBtn.onclick = (e) => {
+cancelBtn.onclick = e => {
    e.stopPropagation();
-    if (currentXhr) {
-        currentXhr.abort();
-        alert("Upload cancelled.");
-        location.reload();
-    }
+    if (currentXhr) currentXhr.abort();
+    localStorage.clear();
+    location.reload();
 };

 function commonFormData() {
    const fd = new FormData();
    fd.append("once", document.getElementById("once").checked ? "true" : "false");
-    const hours = parseInt(document.getElementById("duration").value, 10);
-    fd.append("duration", hours);
+    fd.append("duration", parseInt(document.getElementById("duration").value, 10));
    return fd;
 }

-function startUploadUI() {
-    uploadBtn.disabled = true;
-    uploadBtn.innerText = "UPLOADING...";
-    cancelBtn.classList.remove('hidden');
-    progressContainer.classList.remove("hidden");
-    progressText.classList.remove("hidden");
-    statsText.classList.remove("hidden");
-}
+function uploadSingle(file) {
+    startUI();

-function setupXHRHandlers(xhr) {
-    let startTime = Date.now();
+    const fd = commonFormData();
+    fd.append("file", file);

-    xhr.upload.onprogress = (e) => {
-        if (e.lengthComputable) {
-            const percent = Math.round((e.loaded / e.total) * 100);
-            progressBar.style.width = percent + "%";
-            progressText.innerText = percent + "%";
+    const xhr = new XMLHttpRequest();
+    currentXhr = xhr;

-            const elapsedSeconds = (Date.now() - startTime) / 1000;
-            if (elapsedSeconds > 0) {
-                const bytesPerSecond = e.loaded / elapsedSeconds;
-                const remainingBytes = e.total - e.loaded;
-                const secondsRemaining = remainingBytes / bytesPerSecond;
-                speedText.innerText = formatBytes(bytesPerSecond) + "/S";
-                etaText.innerText = formatTime(secondsRemaining);
-            }
-        }
+    const startTime = Date.now();
+
+    xhr.upload.onprogress = e => {
+        if (e.lengthComputable) updateProgress(e.loaded, file.size, startTime);
    };

    xhr.onload = () => {
-        if (xhr.status >= 200 && xhr.status < 300) {
-            try {
-                const data = JSON.parse(xhr.responseText);
-                if (data.error) throw new Error(data.error);
-                window.location.href = "/f/" + data.view_key;
-            } catch (err) {
-                console.error("Invalid response:", xhr.responseText);
-                alert("Server error");
-            }
-        } else {
-            alert("Upload failed");
+        try {
+            if (xhr.status < 200 || xhr.status >= 300) throw new Error();
+            redirect(JSON.parse(xhr.responseText));
+        } catch {
+            alert("Server error");
        }
    };

@@ -148,30 +160,170 @@ function setupXHRHandlers(xhr) {
            location.reload();
        }
    };
-}

-function handleUploadSingle(file) {
-    startUploadUI();
-    const fd = commonFormData();
-    fd.append("file", file);
-    const xhr = new XMLHttpRequest();
-    currentXhr = xhr;
-    setupXHRHandlers(xhr);
    xhr.open("POST", "/api/files/upload");
    xhr.send(fd);
 }

-function handleUploadMulti(fileList) {
-    startUploadUI();
+function uploadMulti(files) {
+    startUI();
+
    const fd = commonFormData();
-    Array.from(fileList).forEach(f => fd.append("files", f));
+    const list = Array.from(files);
+    list.forEach(f => fd.append("files", f));
+
+    const total = list.reduce((a, f) => a + f.size, 0);
+
    const xhr = new XMLHttpRequest();
    currentXhr = xhr;
-    setupXHRHandlers(xhr);
+
+    const startTime = Date.now();
+
+    xhr.upload.onprogress = e => {
+        if (e.lengthComputable) updateProgress(e.loaded, total, startTime);
+    };
+
+    xhr.onload = () => {
+        try {
+            if (xhr.status < 200 || xhr.status >= 300) throw new Error();
+            redirect(JSON.parse(xhr.responseText));
+        } catch {
+            alert("Server error");
+        }
+    };
+
+    xhr.onerror = () => {
+        if (xhr.statusText !== "abort") {
+            alert("Upload failed");
+            location.reload();
+        }
+    };
+
    xhr.open("POST", "/api/files/upload-multi");
    xhr.send(fd);
 }

+async function uploadChunked(file) {
+    startUI();
+
+    const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
+
+    const initRes = await fetch("/api/files/upload/init", {
+        method: "POST",
+        headers: {"Content-Type": "application/json"},
+        body: JSON.stringify({
+            filename: file.name,
+            totalChunks,
+            size: file.size
+        })
+    });
+
+    const { fileId } = await initRes.json();
+
+    let uploadedBytes = 0;
+    const startTime = Date.now();
+
+    const chunks = Array.from({ length: totalChunks }, (_, i) => ({
+        index: i,
+        start: i * CHUNK_SIZE,
+        end: Math.min((i + 1) * CHUNK_SIZE, file.size),
+        retries: 0,
+        uploading: false,
+        done: false
+    }));
+
+    let active = 0;
+    let completed = 0;
+
+    function uploadChunk(chunk) {
+        return new Promise((res, rej) => {
+            const blob = file.slice(chunk.start, chunk.end);
+            const fd = new FormData();
+            fd.append("chunk", blob);
+
+            const xhr = new XMLHttpRequest();
+            currentXhr = xhr;
+
+            let last = 0;
+
+            xhr.upload.onprogress = e => {
+                if (!e.lengthComputable) return;
+                const delta = e.loaded - last;
+                last = e.loaded;
+                uploadedBytes += delta;
+                updateProgress(uploadedBytes, file.size, startTime);
+            };
+
+            xhr.onload = () => {
+                if (xhr.status >= 200 && xhr.status < 300) {
+                    chunk.done = true;
+                    completed++;
+                    res();
+                } else {
+                    rej();
+                }
+            };
+
+            xhr.onerror = rej;
+
+            xhr.open("POST", "/api/files/upload/chunk");
+            xhr.setRequestHeader("fileId", fileId);
+            xhr.setRequestHeader("chunkIndex", chunk.index);
+            xhr.send(fd);
+        });
+    }
+
+    return new Promise((resolve, reject) => {
+        function next() {
+            if (completed === totalChunks) return finish();
+
+            while (active < MAX_PARALLEL_UPLOADS) {
+                const chunk = chunks.find(c => !c.done && !c.uploading);
+                if (!chunk) break;
+
+                chunk.uploading = true;
+                active++;
+
+                uploadChunk(chunk)
+                    .then(() => {
+                        active--;
+                        next();
+                    })
+                    .catch(() => {
+                        active--;
+                        if (chunk.retries++ < MAX_RETRIES) {
+                            chunk.uploading = false;
+                        } else {
+                            reject();
+                        }
+                        next();
+                    });
+            }
+        }
+
+        async function finish() {
+            try {
+                const res = await fetch("/api/files/upload/complete", {
+                    method: "POST",
+                    headers: {"Content-Type": "application/json"},
+                    body: JSON.stringify({
+                        fileId,
+                        filename: file.name,
+                        totalChunks
+                    })
+                });
+
+                redirect(await res.json());
+                resolve();
+            } catch {
+                reject();
+            }
+        }
+
+        next();
+    });
+}
+
 function copy(id) {
    const el = document.getElementById(id);
    el.select();