mirror of
https://github.com/rgrgogu/new_starr.git
synced 2026-09-27 00:12:54 +08:00
fix: asset operations
This commit is contained in:
rgrgogu
@@ -2,6 +2,7 @@ import { createContext, useCallback, useContext, useRef, useState } from "react"
|
||||
import { nanoid } from "nanoid";
|
||||
import api from "@/utils/api.util";
|
||||
import { useAuth } from "@/contexts/AuthContext";
|
||||
import { presignAssetUpload, uploadPresigned } from "@/utils/presignedUpload.util";
|
||||
import { toast } from "sonner";
|
||||
|
||||
// ─── Generic authenticated SSE reader ──────────────────────────────────────
|
||||
@@ -48,13 +49,9 @@ function streamSSE(url, token, onEvent) {
|
||||
return () => controller.abort();
|
||||
}
|
||||
|
||||
// Upload progress (Express -> Garage, real bytes) — see uploadAsset() below.
|
||||
const streamUploadProgress = (uploadId, token, onProgress) =>
|
||||
streamSSE(`${import.meta.env.VITE_API_URL}/admin/assets/upload-progress/${uploadId}`, token, onProgress);
|
||||
|
||||
// Document-conversion stage progress (compiling/validating/generating) — see
|
||||
// convertAssetToMarkdown() below. Same broadcaster/channel shape on the
|
||||
// backend (services/uploadProgress.service.js), just a different job id.
|
||||
// convertAssetToMarkdown() below. Backend broadcaster is
|
||||
// services/uploadProgress.service.js, keyed by a client-generated job id.
|
||||
const streamConvertProgress = (jobId, token, onProgress) =>
|
||||
streamSSE(`${import.meta.env.VITE_API_URL}/admin/assets/convert-progress/${jobId}`, token, onProgress);
|
||||
|
||||
@@ -242,52 +239,51 @@ export function AssetsProvider({ children }) {
|
||||
[request]
|
||||
);
|
||||
|
||||
// ─── POST /api/admin/assets ───────────────────────────────────────────────
|
||||
// ─── POST /api/admin/assets/presign + direct PUT + POST /api/admin/assets ──
|
||||
//
|
||||
// onProgress?: ({ phase: 'uploading'|'storing'|'done'|'error', pct }) => void
|
||||
// Two honest, sequential phases — not one blended/estimated number:
|
||||
// "uploading" — browser -> this backend, real bytes sent (axios onUploadProgress).
|
||||
// "storing" — this backend -> Garage, the actual S3 PUT, streamed live over
|
||||
// SSE (see streamUploadProgress above / uploadProgress.service.js
|
||||
// on the backend). Real numbers from the real transfer, both legs.
|
||||
// onProgress?: ({ phase: 'uploading'|'processing'|'done', pct }) => void
|
||||
// "uploading" — browser -> storage, real bytes sent directly (this
|
||||
// backend is never in that data path at all anymore).
|
||||
// "processing" — brief server-side step once the upload lands: reads the
|
||||
// object back (HeadObjectCommand), runs ffprobe for
|
||||
// video/audio, inserts the DB row.
|
||||
const uploadAsset = useCallback(
|
||||
({ file, thumbnail, onProgress, ...rest }) =>
|
||||
request(async () => {
|
||||
const uploadId = nanoid();
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
if (thumbnail) form.append("thumbnail", thumbnail);
|
||||
form.append("uploadId", uploadId);
|
||||
Object.entries(rest).forEach(([k, v]) => {
|
||||
if (v !== undefined && v !== null) form.append(k, v);
|
||||
const [mainPresign, thumbPresign] = await Promise.all([
|
||||
presignAssetUpload(file),
|
||||
thumbnail ? presignAssetUpload(thumbnail) : Promise.resolve(null),
|
||||
]);
|
||||
const storage_key = mainPresign.key;
|
||||
|
||||
await Promise.all([
|
||||
uploadPresigned(file, mainPresign, (pct) => onProgress?.({ phase: "uploading", pct })),
|
||||
thumbPresign ? uploadPresigned(thumbnail, thumbPresign) : Promise.resolve(),
|
||||
]);
|
||||
|
||||
onProgress?.({ phase: "processing", pct: 100 });
|
||||
|
||||
const res = await api.post("/admin/assets", {
|
||||
storage_key,
|
||||
thumbnail_storage_key: thumbPresign?.key,
|
||||
original_name: file.name,
|
||||
// Fallback only — the backend prefers storage's own
|
||||
// Content-Type, this just covers the rare case a browser
|
||||
// sent the PUT with no Content-Type at all (empty File.type).
|
||||
mimetype: file.type || undefined,
|
||||
...rest,
|
||||
});
|
||||
|
||||
const stopStream = onProgress
|
||||
? streamUploadProgress(uploadId, accessTokenRef.current, onProgress)
|
||||
: null;
|
||||
|
||||
try {
|
||||
const res = await api.post("/admin/assets", form, {
|
||||
onUploadProgress: onProgress
|
||||
? (evt) => {
|
||||
const pct = evt.total ? Math.round((evt.loaded / evt.total) * 100) : 0;
|
||||
onProgress({ phase: "uploading", pct });
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
const asset = res.data?.data?.data ?? null;
|
||||
if (asset) {
|
||||
setAssets((prev) => [asset, ...prev]);
|
||||
invalidateListCache();
|
||||
toast("Asset uploaded successfully.");
|
||||
}
|
||||
onProgress?.({ phase: "done", pct: 100 });
|
||||
return res.data;
|
||||
} finally {
|
||||
stopStream?.();
|
||||
const asset = res.data?.data?.data ?? null;
|
||||
if (asset) {
|
||||
setAssets((prev) => [asset, ...prev]);
|
||||
invalidateListCache();
|
||||
toast("Asset uploaded successfully.");
|
||||
}
|
||||
onProgress?.({ phase: "done", pct: 100 });
|
||||
return res.data;
|
||||
}),
|
||||
[request, accessTokenRef]
|
||||
[request]
|
||||
);
|
||||
|
||||
// ─── POST /api/admin/assets/:assetId/convert-to-markdown ─────────────────
|
||||
|
||||
@@ -6,25 +6,23 @@
|
||||
// UploadProgressToast reads the same state from anywhere in /admin.
|
||||
//
|
||||
// Every job — whether it's part of an initial addBatch() or a single retry —
|
||||
// goes out as its OWN POST /admin/assets/batch request (that endpoint has
|
||||
// always accepted 1-20 files; sending 1 is a fully supported path). This
|
||||
// matters beyond just per-file progress: the backend processes a batch
|
||||
// request's files sequentially inside one request/response cycle and never
|
||||
// aborts on client disconnect, so a connection drop anywhere during a
|
||||
// combined multi-file request used to fail EVERY job in that request, even
|
||||
// ones the server had already finished (S3 object + DB row both created) —
|
||||
// clicking Retry on those then silently created a duplicate. One request per
|
||||
// file means a dropped connection can only affect the one job riding it.
|
||||
// uploadOne() below is shared by both the initial run and retries; runBatch()
|
||||
// just fans it out over a small worker pool so a big batch doesn't open 20
|
||||
// connections at once (MAX_CONCURRENT mirrors garage-anon-proxy's own
|
||||
// MAX_CONCURRENT_UPLOADS=3 ceiling on the S3 side, so the frontend doesn't
|
||||
// just relocate the pile-up into tripping that limit instead).
|
||||
// uploads directly to storage via its own presigned PUT (see
|
||||
// presignedUpload.util.js), then finalizes with its own POST /admin/assets
|
||||
// call — the same per-file isolation as before (a dropped connection can
|
||||
// only ever affect the one job riding it), now with the added benefit that
|
||||
// the file's bytes never pass through this backend's memory at all, no
|
||||
// matter how large the file is. uploadOne() below is shared by both the
|
||||
// initial run and retries; runBatch() just fans it out over a small worker
|
||||
// pool so a big batch doesn't fire 20 uploads at once (MAX_CONCURRENT
|
||||
// mirrors the Garage-edge Caddy's own concurrent-upload cap, so the
|
||||
// frontend doesn't just relocate the pile-up into tripping that limit
|
||||
// instead).
|
||||
|
||||
import { createContext, useCallback, useContext, useRef, useState } from "react";
|
||||
import { nanoid } from "nanoid";
|
||||
import api from "@/utils/api.util";
|
||||
import { validateAssetFile } from "@/utils/assetUpload.util";
|
||||
import { presignAssetUpload, uploadPresigned } from "@/utils/presignedUpload.util";
|
||||
|
||||
const MAX_CONCURRENT = 3;
|
||||
|
||||
@@ -47,31 +45,34 @@ export function UploadQueueProvider({ children }) {
|
||||
setJobs((prev) => prev.map((j) => (idSet.has(j.id) ? { ...j, ...(typeof patch === "function" ? patch(j) : patch) } : j)));
|
||||
}, []);
|
||||
|
||||
// ─── Upload exactly one job through its own request ────────────────────
|
||||
// ─── Upload exactly one job: presign -> direct PUT -> finalize ─────────
|
||||
// Shared by the initial batch run and single-job retries — a dropped
|
||||
// connection here can only ever fail the one job riding this request,
|
||||
// never its batch-mates.
|
||||
// connection here can only ever fail the one job riding it, never its
|
||||
// batch-mates. Same isolation guarantee as before, plus the file's bytes
|
||||
// now go straight to storage instead of buffering through the backend.
|
||||
const uploadOne = useCallback(async (job) => {
|
||||
const form = new FormData();
|
||||
form.append("files", job.file);
|
||||
Object.entries(job.meta).forEach(([k, v]) => {
|
||||
if (v !== undefined && v !== null) form.append(k, v);
|
||||
});
|
||||
// Bulk display_name derivation only applies when the caller didn't
|
||||
// already provide one — each job keeps whatever name it resolves to.
|
||||
if (!job.meta.display_name) form.append("display_name", baseNameOf(job.name));
|
||||
|
||||
try {
|
||||
const { data } = await api.post("/admin/assets/batch", form, {
|
||||
onUploadProgress: (evt) => {
|
||||
const pct = evt.total ? Math.round((evt.loaded / evt.total) * 100) : 0;
|
||||
patchJobs([job.id], { progress: pct });
|
||||
},
|
||||
const presigned = await presignAssetUpload(job.file);
|
||||
const storage_key = presigned.key;
|
||||
|
||||
await uploadPresigned(job.file, presigned, (pct) => patchJobs([job.id], { progress: pct }));
|
||||
|
||||
// Bulk display_name derivation only applies when the caller didn't
|
||||
// already provide one — each job keeps whatever name it resolves to.
|
||||
const display_name = job.meta.display_name || baseNameOf(job.name);
|
||||
|
||||
const { data } = await api.post("/admin/assets", {
|
||||
...job.meta,
|
||||
storage_key,
|
||||
original_name: job.name,
|
||||
display_name,
|
||||
// Fallback only — the backend prefers storage's own
|
||||
// Content-Type, this just covers the rare case a browser
|
||||
// sent the PUT with no Content-Type at all (empty File.type).
|
||||
mimetype: job.mime || undefined,
|
||||
});
|
||||
const result = data?.data?.results?.[0];
|
||||
patchJobs([job.id], result?.success
|
||||
? { status: "uploaded", progress: 100, asset: result.data, error: null }
|
||||
: { status: "failed", progress: 100, error: result?.message || "Upload failed." });
|
||||
const asset = data?.data?.data;
|
||||
patchJobs([job.id], { status: "uploaded", progress: 100, asset, error: null });
|
||||
} catch (err) {
|
||||
// No sonner toast() here — the floating widget (both corners
|
||||
// would collide, see UploadProgressToast) already surfaces this
|
||||
|
||||
@@ -9,6 +9,7 @@ import { ArrowLeft, ArrowRight, UploadCloud, X, FileVideo, FileText, Image, File
|
||||
|
||||
import { useAssets } from "@/contexts/AdminAssetsContext";
|
||||
import { useAuth } from "@/contexts/AuthContext";
|
||||
import { MAX_ASSET_FILE_SIZE, MAX_ASSET_FILE_SIZE_LABEL } from "@/utils/assetUpload.util";
|
||||
import { useUnsavedChangesGuard } from "@/hooks/useUnsavedChangesGuard";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
@@ -34,11 +35,6 @@ function resolveFileType(mimeType = "") {
|
||||
return "document";
|
||||
}
|
||||
|
||||
// Matches asset_upload.middleware.js on the backend — checked here too so an
|
||||
// oversized file is rejected instantly instead of only after a full upload
|
||||
// attempt round-trips to the server.
|
||||
const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
|
||||
|
||||
// ─── Schema ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const schema = z.object({
|
||||
@@ -128,7 +124,7 @@ export default function AddAsset() {
|
||||
const fileRef = useRef(null);
|
||||
const thumbnailRef = useRef(null);
|
||||
const [thumbKey, setThumbKey] = useState(0);
|
||||
const [progress, setProgress] = useState(null); // { phase: 'uploading'|'storing'|'done'|'error', pct } | null
|
||||
const [progress, setProgress] = useState(null); // { phase: 'uploading'|'processing'|'done'|'error', pct } | null
|
||||
|
||||
const {
|
||||
register,
|
||||
@@ -161,8 +157,8 @@ export default function AddAsset() {
|
||||
const fileType = file ? resolveFileType(file.type) : null;
|
||||
|
||||
const setFile = (f) => {
|
||||
if (f.size > MAX_FILE_SIZE) {
|
||||
setError("_file", { message: `File exceeds the ${MAX_FILE_SIZE / (1024 * 1024)} MB size limit.` });
|
||||
if (f.size > MAX_ASSET_FILE_SIZE) {
|
||||
setError("_file", { message: `File exceeds the ${MAX_ASSET_FILE_SIZE_LABEL} size limit.` });
|
||||
return;
|
||||
}
|
||||
fileRef.current = f;
|
||||
@@ -205,10 +201,10 @@ export default function AddAsset() {
|
||||
};
|
||||
|
||||
const progressLabel = {
|
||||
uploading: "Uploading to server…",
|
||||
storing: "Storing to server...",
|
||||
done: "Done.",
|
||||
error: "Upload failed.",
|
||||
uploading: "Uploading…",
|
||||
processing: "Processing…",
|
||||
done: "Done.",
|
||||
error: "Upload failed.",
|
||||
}[progress?.phase];
|
||||
|
||||
return (
|
||||
|
||||
@@ -28,10 +28,15 @@ const ALLOWED_DOCUMENT_EXTENSIONS = new Set([
|
||||
"zip", "json", "rtf", "txt", "csv", "md",
|
||||
]);
|
||||
|
||||
// Matches asset_upload.middleware.js on the backend — rejecting an oversized
|
||||
// file here means the queue shows a clear "invalid" reason instantly instead
|
||||
// of a job that uploads for a while and then fails with a generic error.
|
||||
export const MAX_ASSET_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
|
||||
// Uploads go straight to storage via a presigned URL (see
|
||||
// presignedUpload.util.js) — this backend never buffers the file. Above 5GB
|
||||
// (S3-compatible storage's own single-PUT ceiling) uploads switch to real
|
||||
// multipart automatically; 15GB is the app-level ceiling chosen for future
|
||||
// assets, well within multipart's own much larger real limit. Rejecting an
|
||||
// oversized file here still means the queue shows a clear "invalid" reason
|
||||
// instantly instead of a job that uploads for a while and then fails.
|
||||
export const MAX_ASSET_FILE_SIZE = 15 * 1024 * 1024 * 1024; // 15 GB
|
||||
export const MAX_ASSET_FILE_SIZE_LABEL = "15 GB";
|
||||
|
||||
export function fileExtension(filename = "") {
|
||||
const dot = filename.lastIndexOf(".");
|
||||
@@ -43,7 +48,7 @@ export function validateAssetFile(file) {
|
||||
if (file.size > MAX_ASSET_FILE_SIZE) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: `File exceeds the ${MAX_ASSET_FILE_SIZE / (1024 * 1024)} MB size limit.`,
|
||||
reason: `File exceeds the ${MAX_ASSET_FILE_SIZE_LABEL} size limit.`,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
// utils/presignedUpload.util.js
|
||||
//
|
||||
// Shared by AdminAssetsContext.jsx (single-file Add Asset) and
|
||||
// UploadQueueContext.jsx (Add Assets Bulk) — the browser uploads file bytes
|
||||
// directly to storage via a short-lived presigned URL, so this backend
|
||||
// never buffers them regardless of file size. presignAssetUpload() mints
|
||||
// the URL(s); putDirect()/putMultipart() send the bytes with real progress
|
||||
// events, and deliberately bypass the shared `api` axios instance since
|
||||
// they target a different origin (the storage host) and must not pick up
|
||||
// the app's Authorization-header interceptor or any other header that
|
||||
// wasn't part of what the URL was signed for.
|
||||
//
|
||||
// Files at or below storage's single-PUT ceiling (5GB) get one presigned
|
||||
// PUT (putDirect). Anything larger uses real multipart upload (putMultipart)
|
||||
// — the backend splits it into 50MB parts, presigns each one, and the
|
||||
// browser uploads them through a small concurrent worker pool, retrying an
|
||||
// individual failed part a couple of times before giving up entirely.
|
||||
|
||||
import api from "@/utils/api.util";
|
||||
|
||||
const MAX_PART_RETRIES = 2;
|
||||
const MAX_CONCURRENT_PARTS = 3;
|
||||
|
||||
export function resolveFileType(mimetype = "") {
|
||||
if (mimetype.startsWith("image/")) return "image";
|
||||
if (mimetype.startsWith("video/")) return "video";
|
||||
if (mimetype.startsWith("audio/")) return "audio";
|
||||
return "document";
|
||||
}
|
||||
|
||||
// -> { key, uploadUrl } (size <= 5GB)
|
||||
// -> { key, multipart: true, uploadId, partSize, parts } (size > 5GB)
|
||||
export async function presignAssetUpload(file) {
|
||||
const { data } = await api.post("/admin/assets/presign", {
|
||||
filename: file.name,
|
||||
mimetype: file.type,
|
||||
file_type: resolveFileType(file.type),
|
||||
size: file.size,
|
||||
});
|
||||
return data.data;
|
||||
}
|
||||
|
||||
export function putDirect(uploadUrl, file, onProgress) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("PUT", uploadUrl);
|
||||
if (onProgress) {
|
||||
xhr.upload.onprogress = (evt) => {
|
||||
onProgress(evt.lengthComputable ? Math.round((evt.loaded / evt.total) * 100) : 0);
|
||||
};
|
||||
}
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) resolve();
|
||||
else reject(new Error(`Upload failed (${xhr.status}).`));
|
||||
};
|
||||
xhr.onerror = () => reject(new Error("Network error during upload."));
|
||||
xhr.send(file);
|
||||
});
|
||||
}
|
||||
|
||||
// Single part PUT — resolves with the ETag S3 returns for that part (needed
|
||||
// to build the CompleteMultipartUpload part list later). Requires ETag to be
|
||||
// listed in the storage edge's Access-Control-Expose-Headers, since it's
|
||||
// otherwise invisible to JS on a cross-origin response.
|
||||
function putPart(uploadUrl, blob, onLoaded) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("PUT", uploadUrl);
|
||||
xhr.upload.onprogress = (evt) => {
|
||||
if (evt.lengthComputable) onLoaded(evt.loaded);
|
||||
};
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
const etag = xhr.getResponseHeader("ETag");
|
||||
if (!etag) { reject(new Error("Part upload succeeded but no ETag was returned.")); return; }
|
||||
onLoaded(blob.size);
|
||||
resolve(etag);
|
||||
} else {
|
||||
reject(new Error(`Part upload failed (${xhr.status}).`));
|
||||
}
|
||||
};
|
||||
xhr.onerror = () => reject(new Error("Network error during part upload."));
|
||||
xhr.send(blob);
|
||||
});
|
||||
}
|
||||
|
||||
async function putPartWithRetry(uploadUrl, blob, onLoaded) {
|
||||
let lastErr;
|
||||
for (let attempt = 0; attempt <= MAX_PART_RETRIES; attempt++) {
|
||||
try {
|
||||
return await putPart(uploadUrl, blob, onLoaded);
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
onLoaded(0); // reset this part's contribution before retrying
|
||||
}
|
||||
}
|
||||
throw lastErr;
|
||||
}
|
||||
|
||||
// multipartInfo: { uploadId, partSize, parts: [{ partNumber, uploadUrl }] }
|
||||
// -> [{ partNumber, etag }, ...] (unordered — completeMultipartUpload sorts)
|
||||
export async function putMultipart(file, multipartInfo, onProgress) {
|
||||
const { partSize, parts } = multipartInfo;
|
||||
const loadedByPart = new Array(parts.length).fill(0);
|
||||
|
||||
const reportProgress = () => {
|
||||
if (!onProgress) return;
|
||||
const loaded = loadedByPart.reduce((sum, n) => sum + n, 0);
|
||||
onProgress(Math.min(100, Math.round((loaded / file.size) * 100)));
|
||||
};
|
||||
|
||||
const results = new Array(parts.length);
|
||||
let next = 0;
|
||||
const worker = async () => {
|
||||
while (next < parts.length) {
|
||||
const i = next++;
|
||||
const { partNumber, uploadUrl } = parts[i];
|
||||
const start = (partNumber - 1) * partSize;
|
||||
const end = Math.min(start + partSize, file.size);
|
||||
const blob = file.slice(start, end);
|
||||
|
||||
const etag = await putPartWithRetry(uploadUrl, blob, (loaded) => {
|
||||
loadedByPart[i] = loaded;
|
||||
reportProgress();
|
||||
});
|
||||
results[i] = { partNumber, etag };
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(MAX_CONCURRENT_PARTS, parts.length) }, worker)
|
||||
);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// Uploads one file to whatever presignAssetUpload() returned — single PUT or
|
||||
// full multipart, whichever the file's size required — and leaves storage
|
||||
// clean on failure (aborts a partially-uploaded multipart session rather
|
||||
// than leaving it to linger). Shared by AdminAssetsContext.jsx and
|
||||
// UploadQueueContext.jsx so neither has to duplicate this branch.
|
||||
export async function uploadPresigned(file, presigned, onProgress) {
|
||||
if (!presigned.multipart) {
|
||||
await putDirect(presigned.uploadUrl, file, onProgress);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const parts = await putMultipart(file, presigned, onProgress);
|
||||
await completeMultipartUpload(presigned.key, presigned.uploadId, parts);
|
||||
} catch (err) {
|
||||
await abortMultipartUpload(presigned.key, presigned.uploadId);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export async function completeMultipartUpload(storage_key, uploadId, parts) {
|
||||
await api.post("/admin/assets/complete-multipart", { storage_key, uploadId, parts });
|
||||
}
|
||||
|
||||
export async function abortMultipartUpload(storage_key, uploadId) {
|
||||
// Best-effort — a failed cleanup call shouldn't mask the real upload
|
||||
// error that triggered the abort in the first place.
|
||||
try {
|
||||
await api.post("/admin/assets/abort-multipart", { storage_key, uploadId });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user