From c647b06cd1e5fdfab9c91732c5f845c57dbc2cb3 Mon Sep 17 00:00:00 2001 From: rgrgogu Date: Sat, 1 Aug 2026 12:07:48 +0800 Subject: [PATCH] fix: asset operations --- src/contexts/AdminAssetsContext.jsx | 84 +++++----- src/contexts/UploadQueueContext.jsx | 71 ++++---- src/modules/admin/pages/assets/AddAsset.jsx | 20 +-- src/utils/assetUpload.util.js | 15 +- src/utils/presignedUpload.util.js | 169 ++++++++++++++++++++ 5 files changed, 263 insertions(+), 96 deletions(-) create mode 100644 src/utils/presignedUpload.util.js diff --git a/src/contexts/AdminAssetsContext.jsx b/src/contexts/AdminAssetsContext.jsx index d083c95..b8b3c62 100644 --- a/src/contexts/AdminAssetsContext.jsx +++ b/src/contexts/AdminAssetsContext.jsx @@ -2,6 +2,7 @@ import { createContext, useCallback, useContext, useRef, useState } from "react" import { nanoid } from "nanoid"; import api from "@/utils/api.util"; import { useAuth } from "@/contexts/AuthContext"; +import { presignAssetUpload, uploadPresigned } from "@/utils/presignedUpload.util"; import { toast } from "sonner"; // ─── Generic authenticated SSE reader ────────────────────────────────────── @@ -48,13 +49,9 @@ function streamSSE(url, token, onEvent) { return () => controller.abort(); } -// Upload progress (Express -> Garage, real bytes) — see uploadAsset() below. -const streamUploadProgress = (uploadId, token, onProgress) => - streamSSE(`${import.meta.env.VITE_API_URL}/admin/assets/upload-progress/${uploadId}`, token, onProgress); - // Document-conversion stage progress (compiling/validating/generating) — see -// convertAssetToMarkdown() below. Same broadcaster/channel shape on the -// backend (services/uploadProgress.service.js), just a different job id. +// convertAssetToMarkdown() below. Backend broadcaster is +// services/uploadProgress.service.js, keyed by a client-generated job id. const streamConvertProgress = (jobId, token, onProgress) => streamSSE(`${import.meta.env.VITE_API_URL}/admin/assets/convert-progress/${jobId}`, token, onProgress); @@ -242,52 +239,51 @@ export function AssetsProvider({ children }) { [request] ); - // ─── POST /api/admin/assets ─────────────────────────────────────────────── + // ─── POST /api/admin/assets/presign + direct PUT + POST /api/admin/assets ── // - // onProgress?: ({ phase: 'uploading'|'storing'|'done'|'error', pct }) => void - // Two honest, sequential phases — not one blended/estimated number: - // "uploading" — browser -> this backend, real bytes sent (axios onUploadProgress). - // "storing" — this backend -> Garage, the actual S3 PUT, streamed live over - // SSE (see streamUploadProgress above / uploadProgress.service.js - // on the backend). Real numbers from the real transfer, both legs. + // onProgress?: ({ phase: 'uploading'|'processing'|'done', pct }) => void + // "uploading" — browser -> storage, real bytes sent directly (this + // backend is never in that data path at all anymore). + // "processing" — brief server-side step once the upload lands: reads the + // object back (HeadObjectCommand), runs ffprobe for + // video/audio, inserts the DB row. const uploadAsset = useCallback( ({ file, thumbnail, onProgress, ...rest }) => request(async () => { - const uploadId = nanoid(); - const form = new FormData(); - form.append("file", file); - if (thumbnail) form.append("thumbnail", thumbnail); - form.append("uploadId", uploadId); - Object.entries(rest).forEach(([k, v]) => { - if (v !== undefined && v !== null) form.append(k, v); + const [mainPresign, thumbPresign] = await Promise.all([ + presignAssetUpload(file), + thumbnail ? presignAssetUpload(thumbnail) : Promise.resolve(null), + ]); + const storage_key = mainPresign.key; + + await Promise.all([ + uploadPresigned(file, mainPresign, (pct) => onProgress?.({ phase: "uploading", pct })), + thumbPresign ? uploadPresigned(thumbnail, thumbPresign) : Promise.resolve(), + ]); + + onProgress?.({ phase: "processing", pct: 100 }); + + const res = await api.post("/admin/assets", { + storage_key, + thumbnail_storage_key: thumbPresign?.key, + original_name: file.name, + // Fallback only — the backend prefers storage's own + // Content-Type, this just covers the rare case a browser + // sent the PUT with no Content-Type at all (empty File.type). + mimetype: file.type || undefined, + ...rest, }); - const stopStream = onProgress - ? streamUploadProgress(uploadId, accessTokenRef.current, onProgress) - : null; - - try { - const res = await api.post("/admin/assets", form, { - onUploadProgress: onProgress - ? (evt) => { - const pct = evt.total ? Math.round((evt.loaded / evt.total) * 100) : 0; - onProgress({ phase: "uploading", pct }); - } - : undefined, - }); - const asset = res.data?.data?.data ?? null; - if (asset) { - setAssets((prev) => [asset, ...prev]); - invalidateListCache(); - toast("Asset uploaded successfully."); - } - onProgress?.({ phase: "done", pct: 100 }); - return res.data; - } finally { - stopStream?.(); + const asset = res.data?.data?.data ?? null; + if (asset) { + setAssets((prev) => [asset, ...prev]); + invalidateListCache(); + toast("Asset uploaded successfully."); } + onProgress?.({ phase: "done", pct: 100 }); + return res.data; }), - [request, accessTokenRef] + [request] ); // ─── POST /api/admin/assets/:assetId/convert-to-markdown ───────────────── diff --git a/src/contexts/UploadQueueContext.jsx b/src/contexts/UploadQueueContext.jsx index 3877bf3..8b622e6 100644 --- a/src/contexts/UploadQueueContext.jsx +++ b/src/contexts/UploadQueueContext.jsx @@ -6,25 +6,23 @@ // UploadProgressToast reads the same state from anywhere in /admin. // // Every job — whether it's part of an initial addBatch() or a single retry — -// goes out as its OWN POST /admin/assets/batch request (that endpoint has -// always accepted 1-20 files; sending 1 is a fully supported path). This -// matters beyond just per-file progress: the backend processes a batch -// request's files sequentially inside one request/response cycle and never -// aborts on client disconnect, so a connection drop anywhere during a -// combined multi-file request used to fail EVERY job in that request, even -// ones the server had already finished (S3 object + DB row both created) — -// clicking Retry on those then silently created a duplicate. One request per -// file means a dropped connection can only affect the one job riding it. -// uploadOne() below is shared by both the initial run and retries; runBatch() -// just fans it out over a small worker pool so a big batch doesn't open 20 -// connections at once (MAX_CONCURRENT mirrors garage-anon-proxy's own -// MAX_CONCURRENT_UPLOADS=3 ceiling on the S3 side, so the frontend doesn't -// just relocate the pile-up into tripping that limit instead). +// uploads directly to storage via its own presigned PUT (see +// presignedUpload.util.js), then finalizes with its own POST /admin/assets +// call — the same per-file isolation as before (a dropped connection can +// only ever affect the one job riding it), now with the added benefit that +// the file's bytes never pass through this backend's memory at all, no +// matter how large the file is. uploadOne() below is shared by both the +// initial run and retries; runBatch() just fans it out over a small worker +// pool so a big batch doesn't fire 20 uploads at once (MAX_CONCURRENT +// mirrors the Garage-edge Caddy's own concurrent-upload cap, so the +// frontend doesn't just relocate the pile-up into tripping that limit +// instead). import { createContext, useCallback, useContext, useRef, useState } from "react"; import { nanoid } from "nanoid"; import api from "@/utils/api.util"; import { validateAssetFile } from "@/utils/assetUpload.util"; +import { presignAssetUpload, uploadPresigned } from "@/utils/presignedUpload.util"; const MAX_CONCURRENT = 3; @@ -47,31 +45,34 @@ export function UploadQueueProvider({ children }) { setJobs((prev) => prev.map((j) => (idSet.has(j.id) ? { ...j, ...(typeof patch === "function" ? patch(j) : patch) } : j))); }, []); - // ─── Upload exactly one job through its own request ──────────────────── + // ─── Upload exactly one job: presign -> direct PUT -> finalize ───────── // Shared by the initial batch run and single-job retries — a dropped - // connection here can only ever fail the one job riding this request, - // never its batch-mates. + // connection here can only ever fail the one job riding it, never its + // batch-mates. Same isolation guarantee as before, plus the file's bytes + // now go straight to storage instead of buffering through the backend. const uploadOne = useCallback(async (job) => { - const form = new FormData(); - form.append("files", job.file); - Object.entries(job.meta).forEach(([k, v]) => { - if (v !== undefined && v !== null) form.append(k, v); - }); - // Bulk display_name derivation only applies when the caller didn't - // already provide one — each job keeps whatever name it resolves to. - if (!job.meta.display_name) form.append("display_name", baseNameOf(job.name)); - try { - const { data } = await api.post("/admin/assets/batch", form, { - onUploadProgress: (evt) => { - const pct = evt.total ? Math.round((evt.loaded / evt.total) * 100) : 0; - patchJobs([job.id], { progress: pct }); - }, + const presigned = await presignAssetUpload(job.file); + const storage_key = presigned.key; + + await uploadPresigned(job.file, presigned, (pct) => patchJobs([job.id], { progress: pct })); + + // Bulk display_name derivation only applies when the caller didn't + // already provide one — each job keeps whatever name it resolves to. + const display_name = job.meta.display_name || baseNameOf(job.name); + + const { data } = await api.post("/admin/assets", { + ...job.meta, + storage_key, + original_name: job.name, + display_name, + // Fallback only — the backend prefers storage's own + // Content-Type, this just covers the rare case a browser + // sent the PUT with no Content-Type at all (empty File.type). + mimetype: job.mime || undefined, }); - const result = data?.data?.results?.[0]; - patchJobs([job.id], result?.success - ? { status: "uploaded", progress: 100, asset: result.data, error: null } - : { status: "failed", progress: 100, error: result?.message || "Upload failed." }); + const asset = data?.data?.data; + patchJobs([job.id], { status: "uploaded", progress: 100, asset, error: null }); } catch (err) { // No sonner toast() here — the floating widget (both corners // would collide, see UploadProgressToast) already surfaces this diff --git a/src/modules/admin/pages/assets/AddAsset.jsx b/src/modules/admin/pages/assets/AddAsset.jsx index 2e40530..da5c69e 100644 --- a/src/modules/admin/pages/assets/AddAsset.jsx +++ b/src/modules/admin/pages/assets/AddAsset.jsx @@ -9,6 +9,7 @@ import { ArrowLeft, ArrowRight, UploadCloud, X, FileVideo, FileText, Image, File import { useAssets } from "@/contexts/AdminAssetsContext"; import { useAuth } from "@/contexts/AuthContext"; +import { MAX_ASSET_FILE_SIZE, MAX_ASSET_FILE_SIZE_LABEL } from "@/utils/assetUpload.util"; import { useUnsavedChangesGuard } from "@/hooks/useUnsavedChangesGuard"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -34,11 +35,6 @@ function resolveFileType(mimeType = "") { return "document"; } -// Matches asset_upload.middleware.js on the backend — checked here too so an -// oversized file is rejected instantly instead of only after a full upload -// attempt round-trips to the server. -const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB - // ─── Schema ─────────────────────────────────────────────────────────────────── const schema = z.object({ @@ -128,7 +124,7 @@ export default function AddAsset() { const fileRef = useRef(null); const thumbnailRef = useRef(null); const [thumbKey, setThumbKey] = useState(0); - const [progress, setProgress] = useState(null); // { phase: 'uploading'|'storing'|'done'|'error', pct } | null + const [progress, setProgress] = useState(null); // { phase: 'uploading'|'processing'|'done'|'error', pct } | null const { register, @@ -161,8 +157,8 @@ export default function AddAsset() { const fileType = file ? resolveFileType(file.type) : null; const setFile = (f) => { - if (f.size > MAX_FILE_SIZE) { - setError("_file", { message: `File exceeds the ${MAX_FILE_SIZE / (1024 * 1024)} MB size limit.` }); + if (f.size > MAX_ASSET_FILE_SIZE) { + setError("_file", { message: `File exceeds the ${MAX_ASSET_FILE_SIZE_LABEL} size limit.` }); return; } fileRef.current = f; @@ -205,10 +201,10 @@ export default function AddAsset() { }; const progressLabel = { - uploading: "Uploading to server…", - storing: "Storing to server...", - done: "Done.", - error: "Upload failed.", + uploading: "Uploading…", + processing: "Processing…", + done: "Done.", + error: "Upload failed.", }[progress?.phase]; return ( diff --git a/src/utils/assetUpload.util.js b/src/utils/assetUpload.util.js index 416af64..433de9f 100644 --- a/src/utils/assetUpload.util.js +++ b/src/utils/assetUpload.util.js @@ -28,10 +28,15 @@ const ALLOWED_DOCUMENT_EXTENSIONS = new Set([ "zip", "json", "rtf", "txt", "csv", "md", ]); -// Matches asset_upload.middleware.js on the backend — rejecting an oversized -// file here means the queue shows a clear "invalid" reason instantly instead -// of a job that uploads for a while and then fails with a generic error. -export const MAX_ASSET_FILE_SIZE = 500 * 1024 * 1024; // 500 MB +// Uploads go straight to storage via a presigned URL (see +// presignedUpload.util.js) — this backend never buffers the file. Above 5GB +// (S3-compatible storage's own single-PUT ceiling) uploads switch to real +// multipart automatically; 15GB is the app-level ceiling chosen for future +// assets, well within multipart's own much larger real limit. Rejecting an +// oversized file here still means the queue shows a clear "invalid" reason +// instantly instead of a job that uploads for a while and then fails. +export const MAX_ASSET_FILE_SIZE = 15 * 1024 * 1024 * 1024; // 15 GB +export const MAX_ASSET_FILE_SIZE_LABEL = "15 GB"; export function fileExtension(filename = "") { const dot = filename.lastIndexOf("."); @@ -43,7 +48,7 @@ export function validateAssetFile(file) { if (file.size > MAX_ASSET_FILE_SIZE) { return { ok: false, - reason: `File exceeds the ${MAX_ASSET_FILE_SIZE / (1024 * 1024)} MB size limit.`, + reason: `File exceeds the ${MAX_ASSET_FILE_SIZE_LABEL} size limit.`, }; } diff --git a/src/utils/presignedUpload.util.js b/src/utils/presignedUpload.util.js new file mode 100644 index 0000000..2dd6104 --- /dev/null +++ b/src/utils/presignedUpload.util.js @@ -0,0 +1,169 @@ +// utils/presignedUpload.util.js +// +// Shared by AdminAssetsContext.jsx (single-file Add Asset) and +// UploadQueueContext.jsx (Add Assets Bulk) — the browser uploads file bytes +// directly to storage via a short-lived presigned URL, so this backend +// never buffers them regardless of file size. presignAssetUpload() mints +// the URL(s); putDirect()/putMultipart() send the bytes with real progress +// events, and deliberately bypass the shared `api` axios instance since +// they target a different origin (the storage host) and must not pick up +// the app's Authorization-header interceptor or any other header that +// wasn't part of what the URL was signed for. +// +// Files at or below storage's single-PUT ceiling (5GB) get one presigned +// PUT (putDirect). Anything larger uses real multipart upload (putMultipart) +// — the backend splits it into 50MB parts, presigns each one, and the +// browser uploads them through a small concurrent worker pool, retrying an +// individual failed part a couple of times before giving up entirely. + +import api from "@/utils/api.util"; + +const MAX_PART_RETRIES = 2; +const MAX_CONCURRENT_PARTS = 3; + +export function resolveFileType(mimetype = "") { + if (mimetype.startsWith("image/")) return "image"; + if (mimetype.startsWith("video/")) return "video"; + if (mimetype.startsWith("audio/")) return "audio"; + return "document"; +} + +// -> { key, uploadUrl } (size <= 5GB) +// -> { key, multipart: true, uploadId, partSize, parts } (size > 5GB) +export async function presignAssetUpload(file) { + const { data } = await api.post("/admin/assets/presign", { + filename: file.name, + mimetype: file.type, + file_type: resolveFileType(file.type), + size: file.size, + }); + return data.data; +} + +export function putDirect(uploadUrl, file, onProgress) { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open("PUT", uploadUrl); + if (onProgress) { + xhr.upload.onprogress = (evt) => { + onProgress(evt.lengthComputable ? Math.round((evt.loaded / evt.total) * 100) : 0); + }; + } + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) resolve(); + else reject(new Error(`Upload failed (${xhr.status}).`)); + }; + xhr.onerror = () => reject(new Error("Network error during upload.")); + xhr.send(file); + }); +} + +// Single part PUT — resolves with the ETag S3 returns for that part (needed +// to build the CompleteMultipartUpload part list later). Requires ETag to be +// listed in the storage edge's Access-Control-Expose-Headers, since it's +// otherwise invisible to JS on a cross-origin response. +function putPart(uploadUrl, blob, onLoaded) { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open("PUT", uploadUrl); + xhr.upload.onprogress = (evt) => { + if (evt.lengthComputable) onLoaded(evt.loaded); + }; + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + const etag = xhr.getResponseHeader("ETag"); + if (!etag) { reject(new Error("Part upload succeeded but no ETag was returned.")); return; } + onLoaded(blob.size); + resolve(etag); + } else { + reject(new Error(`Part upload failed (${xhr.status}).`)); + } + }; + xhr.onerror = () => reject(new Error("Network error during part upload.")); + xhr.send(blob); + }); +} + +async function putPartWithRetry(uploadUrl, blob, onLoaded) { + let lastErr; + for (let attempt = 0; attempt <= MAX_PART_RETRIES; attempt++) { + try { + return await putPart(uploadUrl, blob, onLoaded); + } catch (err) { + lastErr = err; + onLoaded(0); // reset this part's contribution before retrying + } + } + throw lastErr; +} + +// multipartInfo: { uploadId, partSize, parts: [{ partNumber, uploadUrl }] } +// -> [{ partNumber, etag }, ...] (unordered — completeMultipartUpload sorts) +export async function putMultipart(file, multipartInfo, onProgress) { + const { partSize, parts } = multipartInfo; + const loadedByPart = new Array(parts.length).fill(0); + + const reportProgress = () => { + if (!onProgress) return; + const loaded = loadedByPart.reduce((sum, n) => sum + n, 0); + onProgress(Math.min(100, Math.round((loaded / file.size) * 100))); + }; + + const results = new Array(parts.length); + let next = 0; + const worker = async () => { + while (next < parts.length) { + const i = next++; + const { partNumber, uploadUrl } = parts[i]; + const start = (partNumber - 1) * partSize; + const end = Math.min(start + partSize, file.size); + const blob = file.slice(start, end); + + const etag = await putPartWithRetry(uploadUrl, blob, (loaded) => { + loadedByPart[i] = loaded; + reportProgress(); + }); + results[i] = { partNumber, etag }; + } + }; + + await Promise.all( + Array.from({ length: Math.min(MAX_CONCURRENT_PARTS, parts.length) }, worker) + ); + + return results; +} + +// Uploads one file to whatever presignAssetUpload() returned — single PUT or +// full multipart, whichever the file's size required — and leaves storage +// clean on failure (aborts a partially-uploaded multipart session rather +// than leaving it to linger). Shared by AdminAssetsContext.jsx and +// UploadQueueContext.jsx so neither has to duplicate this branch. +export async function uploadPresigned(file, presigned, onProgress) { + if (!presigned.multipart) { + await putDirect(presigned.uploadUrl, file, onProgress); + return; + } + + try { + const parts = await putMultipart(file, presigned, onProgress); + await completeMultipartUpload(presigned.key, presigned.uploadId, parts); + } catch (err) { + await abortMultipartUpload(presigned.key, presigned.uploadId); + throw err; + } +} + +export async function completeMultipartUpload(storage_key, uploadId, parts) { + await api.post("/admin/assets/complete-multipart", { storage_key, uploadId, parts }); +} + +export async function abortMultipartUpload(storage_key, uploadId) { + // Best-effort — a failed cleanup call shouldn't mask the real upload + // error that triggered the abort in the first place. + try { + await api.post("/admin/assets/abort-multipart", { storage_key, uploadId }); + } catch { + // ignore + } +}