fix: asset operations

src/contexts/AdminAssetsContext.jsx

@@ -2,6 +2,7 @@ import { createContext, useCallback, useContext, useRef, useState } from "react"
 import { nanoid } from "nanoid";
 import api from "@/utils/api.util";
 import { useAuth } from "@/contexts/AuthContext";
+import { presignAssetUpload, uploadPresigned } from "@/utils/presignedUpload.util";
 import { toast } from "sonner";
 
 // ─── Generic authenticated SSE reader ──────────────────────────────────────
@@ -48,13 +49,9 @@ function streamSSE(url, token, onEvent) {
     return () => controller.abort();
 }
 
-// Upload progress (Express -> Garage, real bytes) — see uploadAsset() below.
-const streamUploadProgress = (uploadId, token, onProgress) =>
-    streamSSE(`${import.meta.env.VITE_API_URL}/admin/assets/upload-progress/${uploadId}`, token, onProgress);
-
 // Document-conversion stage progress (compiling/validating/generating) — see
-// convertAssetToMarkdown() below. Same broadcaster/channel shape on the
+// convertAssetToMarkdown() below. Backend broadcaster is
-// backend (services/uploadProgress.service.js), just a different job id.
+// services/uploadProgress.service.js, keyed by a client-generated job id.
 const streamConvertProgress = (jobId, token, onProgress) =>
     streamSSE(`${import.meta.env.VITE_API_URL}/admin/assets/convert-progress/${jobId}`, token, onProgress);
 
@@ -242,52 +239,51 @@ export function AssetsProvider({ children }) {
         [request]
     );
 
-    // ─── POST /api/admin/assets ───────────────────────────────────────────────
+    // ─── POST /api/admin/assets/presign + direct PUT + POST /api/admin/assets ──
     //
-    // onProgress?: ({ phase: 'uploading'|'storing'|'done'|'error', pct }) => void
+    // onProgress?: ({ phase: 'uploading'|'processing'|'done', pct }) => void
-    // Two honest, sequential phases — not one blended/estimated number:
+    // "uploading"  — browser -> storage, real bytes sent directly (this
-    //   "uploading" — browser -> this backend, real bytes sent (axios onUploadProgress).
+    //                backend is never in that data path at all anymore).
-    //   "storing"   — this backend -> Garage, the actual S3 PUT, streamed live over
+    // "processing" — brief server-side step once the upload lands: reads the
-    //                 SSE (see streamUploadProgress above / uploadProgress.service.js
+    //                object back (HeadObjectCommand), runs ffprobe for
-    //                 on the backend). Real numbers from the real transfer, both legs.
+    //                video/audio, inserts the DB row.
     const uploadAsset = useCallback(
         ({ file, thumbnail, onProgress, ...rest }) =>
             request(async () => {
-                const uploadId = nanoid();
+                const [mainPresign, thumbPresign] = await Promise.all([
-                const form = new FormData();
+                    presignAssetUpload(file),
-                form.append("file", file);
+                    thumbnail ? presignAssetUpload(thumbnail) : Promise.resolve(null),
-                if (thumbnail) form.append("thumbnail", thumbnail);
+                ]);
-                form.append("uploadId", uploadId);
+                const storage_key = mainPresign.key;
-                Object.entries(rest).forEach(([k, v]) => {
+
-                    if (v !== undefined && v !== null) form.append(k, v);
+                await Promise.all([
+                    uploadPresigned(file, mainPresign, (pct) => onProgress?.({ phase: "uploading", pct })),
+                    thumbPresign ? uploadPresigned(thumbnail, thumbPresign) : Promise.resolve(),
+                ]);
+
+                onProgress?.({ phase: "processing", pct: 100 });
+
+                const res = await api.post("/admin/assets", {
+                    storage_key,
+                    thumbnail_storage_key: thumbPresign?.key,
+                    original_name: file.name,
+                    // Fallback only — the backend prefers storage's own
+                    // Content-Type, this just covers the rare case a browser
+                    // sent the PUT with no Content-Type at all (empty File.type).
+                    mimetype: file.type || undefined,
+                    ...rest,
                 });
 
-                const stopStream = onProgress
+                const asset = res.data?.data?.data ?? null;
-                    ? streamUploadProgress(uploadId, accessTokenRef.current, onProgress)
+                if (asset) {
-                    : null;
+                    setAssets((prev) => [asset, ...prev]);
-
+                    invalidateListCache();
-                try {
+                    toast("Asset uploaded successfully.");
-                    const res = await api.post("/admin/assets", form, {
-                        onUploadProgress: onProgress
-                            ? (evt) => {
-                                  const pct = evt.total ? Math.round((evt.loaded / evt.total) * 100) : 0;
-                                  onProgress({ phase: "uploading", pct });
-                              }
-                            : undefined,
-                    });
-                    const asset = res.data?.data?.data ?? null;
-                    if (asset) {
-                        setAssets((prev) => [asset, ...prev]);
-                        invalidateListCache();
-                        toast("Asset uploaded successfully.");
-                    }
-                    onProgress?.({ phase: "done", pct: 100 });
-                    return res.data;
-                } finally {
-                    stopStream?.();
                 }
+                onProgress?.({ phase: "done", pct: 100 });
+                return res.data;
             }),
-        [request, accessTokenRef]
+        [request]
     );
 
     // ─── POST /api/admin/assets/:assetId/convert-to-markdown ─────────────────

src/contexts/UploadQueueContext.jsx

@@ -6,25 +6,23 @@
 // UploadProgressToast reads the same state from anywhere in /admin.
 //
 // Every job — whether it's part of an initial addBatch() or a single retry —
-// goes out as its OWN POST /admin/assets/batch request (that endpoint has
+// uploads directly to storage via its own presigned PUT (see
-// always accepted 1-20 files; sending 1 is a fully supported path). This
+// presignedUpload.util.js), then finalizes with its own POST /admin/assets
-// matters beyond just per-file progress: the backend processes a batch
+// call — the same per-file isolation as before (a dropped connection can
-// request's files sequentially inside one request/response cycle and never
+// only ever affect the one job riding it), now with the added benefit that
-// aborts on client disconnect, so a connection drop anywhere during a
+// the file's bytes never pass through this backend's memory at all, no
-// combined multi-file request used to fail EVERY job in that request, even
+// matter how large the file is. uploadOne() below is shared by both the
-// ones the server had already finished (S3 object + DB row both created) —
+// initial run and retries; runBatch() just fans it out over a small worker
-// clicking Retry on those then silently created a duplicate. One request per
+// pool so a big batch doesn't fire 20 uploads at once (MAX_CONCURRENT
-// file means a dropped connection can only affect the one job riding it.
+// mirrors the Garage-edge Caddy's own concurrent-upload cap, so the
-// uploadOne() below is shared by both the initial run and retries; runBatch()
+// frontend doesn't just relocate the pile-up into tripping that limit
-// just fans it out over a small worker pool so a big batch doesn't open 20
+// instead).
-// connections at once (MAX_CONCURRENT mirrors garage-anon-proxy's own
-// MAX_CONCURRENT_UPLOADS=3 ceiling on the S3 side, so the frontend doesn't
-// just relocate the pile-up into tripping that limit instead).
 
 import { createContext, useCallback, useContext, useRef, useState } from "react";
 import { nanoid } from "nanoid";
 import api from "@/utils/api.util";
 import { validateAssetFile } from "@/utils/assetUpload.util";
+import { presignAssetUpload, uploadPresigned } from "@/utils/presignedUpload.util";
 
 const MAX_CONCURRENT = 3;
 
@@ -47,31 +45,34 @@ export function UploadQueueProvider({ children }) {
         setJobs((prev) => prev.map((j) => (idSet.has(j.id) ? { ...j, ...(typeof patch === "function" ? patch(j) : patch) } : j)));
     }, []);
 
-    // ─── Upload exactly one job through its own request ────────────────────
+    // ─── Upload exactly one job: presign -> direct PUT -> finalize ─────────
     // Shared by the initial batch run and single-job retries — a dropped
-    // connection here can only ever fail the one job riding this request,
+    // connection here can only ever fail the one job riding it, never its
-    // never its batch-mates.
+    // batch-mates. Same isolation guarantee as before, plus the file's bytes
+    // now go straight to storage instead of buffering through the backend.
     const uploadOne = useCallback(async (job) => {
-        const form = new FormData();
-        form.append("files", job.file);
-        Object.entries(job.meta).forEach(([k, v]) => {
-            if (v !== undefined && v !== null) form.append(k, v);
-        });
-        // Bulk display_name derivation only applies when the caller didn't
-        // already provide one — each job keeps whatever name it resolves to.
-        if (!job.meta.display_name) form.append("display_name", baseNameOf(job.name));
-
         try {
-            const { data } = await api.post("/admin/assets/batch", form, {
+            const presigned = await presignAssetUpload(job.file);
-                onUploadProgress: (evt) => {
+            const storage_key = presigned.key;
-                    const pct = evt.total ? Math.round((evt.loaded / evt.total) * 100) : 0;
+
-                    patchJobs([job.id], { progress: pct });
+            await uploadPresigned(job.file, presigned, (pct) => patchJobs([job.id], { progress: pct }));
-                },
+
+            // Bulk display_name derivation only applies when the caller didn't
+            // already provide one — each job keeps whatever name it resolves to.
+            const display_name = job.meta.display_name || baseNameOf(job.name);
+
+            const { data } = await api.post("/admin/assets", {
+                ...job.meta,
+                storage_key,
+                original_name: job.name,
+                display_name,
+                // Fallback only — the backend prefers storage's own
+                // Content-Type, this just covers the rare case a browser
+                // sent the PUT with no Content-Type at all (empty File.type).
+                mimetype: job.mime || undefined,
             });
-            const result = data?.data?.results?.[0];
+            const asset = data?.data?.data;
-            patchJobs([job.id], result?.success
+            patchJobs([job.id], { status: "uploaded", progress: 100, asset, error: null });
-                ? { status: "uploaded", progress: 100, asset: result.data, error: null }
-                : { status: "failed", progress: 100, error: result?.message || "Upload failed." });
         } catch (err) {
             // No sonner toast() here — the floating widget (both corners
             // would collide, see UploadProgressToast) already surfaces this

src/modules/admin/pages/assets/AddAsset.jsx

@@ -9,6 +9,7 @@ import { ArrowLeft, ArrowRight, UploadCloud, X, FileVideo, FileText, Image, File
 
 import { useAssets } from "@/contexts/AdminAssetsContext";
 import { useAuth } from "@/contexts/AuthContext";
+import { MAX_ASSET_FILE_SIZE, MAX_ASSET_FILE_SIZE_LABEL } from "@/utils/assetUpload.util";
 import { useUnsavedChangesGuard } from "@/hooks/useUnsavedChangesGuard";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
@@ -34,11 +35,6 @@ function resolveFileType(mimeType = "") {
     return "document";
 }
 
-// Matches asset_upload.middleware.js on the backend — checked here too so an
-// oversized file is rejected instantly instead of only after a full upload
-// attempt round-trips to the server.
-const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
-
 // ─── Schema ───────────────────────────────────────────────────────────────────
 
 const schema = z.object({
@@ -128,7 +124,7 @@ export default function AddAsset() {
     const fileRef      = useRef(null);
     const thumbnailRef = useRef(null);
     const [thumbKey, setThumbKey] = useState(0);
-    const [progress, setProgress] = useState(null); // { phase: 'uploading'|'storing'|'done'|'error', pct } | null
+    const [progress, setProgress] = useState(null); // { phase: 'uploading'|'processing'|'done'|'error', pct } | null
 
     const {
         register,
@@ -161,8 +157,8 @@ export default function AddAsset() {
     const fileType = file ? resolveFileType(file.type) : null;
 
     const setFile = (f) => {
-        if (f.size > MAX_FILE_SIZE) {
+        if (f.size > MAX_ASSET_FILE_SIZE) {
-            setError("_file", { message: `File exceeds the ${MAX_FILE_SIZE / (1024 * 1024)} MB size limit.` });
+            setError("_file", { message: `File exceeds the ${MAX_ASSET_FILE_SIZE_LABEL} size limit.` });
             return;
         }
         fileRef.current = f;
@@ -205,10 +201,10 @@ export default function AddAsset() {
     };
 
     const progressLabel = {
-        uploading: "Uploading to server…",
+        uploading:  "Uploading…",
-        storing:   "Storing to server...",
+        processing: "Processing…",
-        done:      "Done.",
+        done:       "Done.",
-        error:     "Upload failed.",
+        error:      "Upload failed.",
     }[progress?.phase];
 
     return (

src/utils/assetUpload.util.js

@@ -28,10 +28,15 @@ const ALLOWED_DOCUMENT_EXTENSIONS = new Set([
     "zip", "json", "rtf", "txt", "csv", "md",
 ]);
 
-// Matches asset_upload.middleware.js on the backend — rejecting an oversized
+// Uploads go straight to storage via a presigned URL (see
-// file here means the queue shows a clear "invalid" reason instantly instead
+// presignedUpload.util.js) — this backend never buffers the file. Above 5GB
-// of a job that uploads for a while and then fails with a generic error.
+// (S3-compatible storage's own single-PUT ceiling) uploads switch to real
-export const MAX_ASSET_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
+// multipart automatically; 15GB is the app-level ceiling chosen for future
+// assets, well within multipart's own much larger real limit. Rejecting an
+// oversized file here still means the queue shows a clear "invalid" reason
+// instantly instead of a job that uploads for a while and then fails.
+export const MAX_ASSET_FILE_SIZE = 15 * 1024 * 1024 * 1024; // 15 GB
+export const MAX_ASSET_FILE_SIZE_LABEL = "15 GB";
 
 export function fileExtension(filename = "") {
     const dot = filename.lastIndexOf(".");
@@ -43,7 +48,7 @@ export function validateAssetFile(file) {
     if (file.size > MAX_ASSET_FILE_SIZE) {
         return {
             ok: false,
-            reason: `File exceeds the ${MAX_ASSET_FILE_SIZE / (1024 * 1024)} MB size limit.`,
+            reason: `File exceeds the ${MAX_ASSET_FILE_SIZE_LABEL} size limit.`,
         };
     }
 

src/utils/presignedUpload.util.js

@@ -0,0 +1,169 @@
+// utils/presignedUpload.util.js
+//
+// Shared by AdminAssetsContext.jsx (single-file Add Asset) and
+// UploadQueueContext.jsx (Add Assets Bulk) — the browser uploads file bytes
+// directly to storage via a short-lived presigned URL, so this backend
+// never buffers them regardless of file size. presignAssetUpload() mints
+// the URL(s); putDirect()/putMultipart() send the bytes with real progress
+// events, and deliberately bypass the shared `api` axios instance since
+// they target a different origin (the storage host) and must not pick up
+// the app's Authorization-header interceptor or any other header that
+// wasn't part of what the URL was signed for.
+//
+// Files at or below storage's single-PUT ceiling (5GB) get one presigned
+// PUT (putDirect). Anything larger uses real multipart upload (putMultipart)
+// — the backend splits it into 50MB parts, presigns each one, and the
+// browser uploads them through a small concurrent worker pool, retrying an
+// individual failed part a couple of times before giving up entirely.
+
+import api from "@/utils/api.util";
+
+const MAX_PART_RETRIES = 2;
+const MAX_CONCURRENT_PARTS = 3;
+
+export function resolveFileType(mimetype = "") {
+    if (mimetype.startsWith("image/")) return "image";
+    if (mimetype.startsWith("video/")) return "video";
+    if (mimetype.startsWith("audio/")) return "audio";
+    return "document";
+}
+
+// -> { key, uploadUrl }                                   (size <= 5GB)
+// -> { key, multipart: true, uploadId, partSize, parts }   (size > 5GB)
+export async function presignAssetUpload(file) {
+    const { data } = await api.post("/admin/assets/presign", {
+        filename: file.name,
+        mimetype: file.type,
+        file_type: resolveFileType(file.type),
+        size: file.size,
+    });
+    return data.data;
+}
+
+export function putDirect(uploadUrl, file, onProgress) {
+    return new Promise((resolve, reject) => {
+        const xhr = new XMLHttpRequest();
+        xhr.open("PUT", uploadUrl);
+        if (onProgress) {
+            xhr.upload.onprogress = (evt) => {
+                onProgress(evt.lengthComputable ? Math.round((evt.loaded / evt.total) * 100) : 0);
+            };
+        }
+        xhr.onload = () => {
+            if (xhr.status >= 200 && xhr.status < 300) resolve();
+            else reject(new Error(`Upload failed (${xhr.status}).`));
+        };
+        xhr.onerror = () => reject(new Error("Network error during upload."));
+        xhr.send(file);
+    });
+}
+
+// Single part PUT — resolves with the ETag S3 returns for that part (needed
+// to build the CompleteMultipartUpload part list later). Requires ETag to be
+// listed in the storage edge's Access-Control-Expose-Headers, since it's
+// otherwise invisible to JS on a cross-origin response.
+function putPart(uploadUrl, blob, onLoaded) {
+    return new Promise((resolve, reject) => {
+        const xhr = new XMLHttpRequest();
+        xhr.open("PUT", uploadUrl);
+        xhr.upload.onprogress = (evt) => {
+            if (evt.lengthComputable) onLoaded(evt.loaded);
+        };
+        xhr.onload = () => {
+            if (xhr.status >= 200 && xhr.status < 300) {
+                const etag = xhr.getResponseHeader("ETag");
+                if (!etag) { reject(new Error("Part upload succeeded but no ETag was returned.")); return; }
+                onLoaded(blob.size);
+                resolve(etag);
+            } else {
+                reject(new Error(`Part upload failed (${xhr.status}).`));
+            }
+        };
+        xhr.onerror = () => reject(new Error("Network error during part upload."));
+        xhr.send(blob);
+    });
+}
+
+async function putPartWithRetry(uploadUrl, blob, onLoaded) {
+    let lastErr;
+    for (let attempt = 0; attempt <= MAX_PART_RETRIES; attempt++) {
+        try {
+            return await putPart(uploadUrl, blob, onLoaded);
+        } catch (err) {
+            lastErr = err;
+            onLoaded(0); // reset this part's contribution before retrying
+        }
+    }
+    throw lastErr;
+}
+
+// multipartInfo: { uploadId, partSize, parts: [{ partNumber, uploadUrl }] }
+// -> [{ partNumber, etag }, ...] (unordered — completeMultipartUpload sorts)
+export async function putMultipart(file, multipartInfo, onProgress) {
+    const { partSize, parts } = multipartInfo;
+    const loadedByPart = new Array(parts.length).fill(0);
+
+    const reportProgress = () => {
+        if (!onProgress) return;
+        const loaded = loadedByPart.reduce((sum, n) => sum + n, 0);
+        onProgress(Math.min(100, Math.round((loaded / file.size) * 100)));
+    };
+
+    const results = new Array(parts.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < parts.length) {
+            const i = next++;
+            const { partNumber, uploadUrl } = parts[i];
+            const start = (partNumber - 1) * partSize;
+            const end = Math.min(start + partSize, file.size);
+            const blob = file.slice(start, end);
+
+            const etag = await putPartWithRetry(uploadUrl, blob, (loaded) => {
+                loadedByPart[i] = loaded;
+                reportProgress();
+            });
+            results[i] = { partNumber, etag };
+        }
+    };
+
+    await Promise.all(
+        Array.from({ length: Math.min(MAX_CONCURRENT_PARTS, parts.length) }, worker)
+    );
+
+    return results;
+}
+
+// Uploads one file to whatever presignAssetUpload() returned — single PUT or
+// full multipart, whichever the file's size required — and leaves storage
+// clean on failure (aborts a partially-uploaded multipart session rather
+// than leaving it to linger). Shared by AdminAssetsContext.jsx and
+// UploadQueueContext.jsx so neither has to duplicate this branch.
+export async function uploadPresigned(file, presigned, onProgress) {
+    if (!presigned.multipart) {
+        await putDirect(presigned.uploadUrl, file, onProgress);
+        return;
+    }
+
+    try {
+        const parts = await putMultipart(file, presigned, onProgress);
+        await completeMultipartUpload(presigned.key, presigned.uploadId, parts);
+    } catch (err) {
+        await abortMultipartUpload(presigned.key, presigned.uploadId);
+        throw err;
+    }
+}
+
+export async function completeMultipartUpload(storage_key, uploadId, parts) {
+    await api.post("/admin/assets/complete-multipart", { storage_key, uploadId, parts });
+}
+
+export async function abortMultipartUpload(storage_key, uploadId) {
+    // Best-effort — a failed cleanup call shouldn't mask the real upload
+    // error that triggered the abort in the first place.
+    try {
+        await api.post("/admin/assets/abort-multipart", { storage_key, uploadId });
+    } catch {
+        // ignore
+    }
+}