kennethscoffee/starr-philproperties
1
0
Fork 0
mirror of https://github.com/rgrgogu/new_starr.git synced 2026-09-27 00:12:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: asset operations

Browse Source
This commit is contained in:
rgrgogu
2026-08-01 12:07:48 +08:00
parent 297f02186c
commit c647b06cd1
5 changed files with 263 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/contexts/AdminAssetsContext.jsx
+33 -37
View File
@@ -2,6 +2,7 @@ import { createContext, useCallback, useContext, useRef, useState } from "react"
import { nanoid } from "nanoid";
import api from "@/utils/api.util";
import { useAuth } from "@/contexts/AuthContext";
import { presignAssetUpload, uploadPresigned } from "@/utils/presignedUpload.util";
import { toast } from "sonner";
// ─── Generic authenticated SSE reader ──────────────────────────────────────
@@ -48,13 +49,9 @@ function streamSSE(url, token, onEvent) {
return () => controller.abort();
}
// Upload progress (Express -> Garage, real bytes) — see uploadAsset() below.
const streamUploadProgress = (uploadId, token, onProgress) =>
streamSSE(`${import.meta.env.VITE_API_URL}/admin/assets/upload-progress/${uploadId}`, token, onProgress);
// Document-conversion stage progress (compiling/validating/generating) — see
// convertAssetToMarkdown() below. Same broadcaster/channel shape on the
// backend (services/uploadProgress.service.js), just a different job id.
// convertAssetToMarkdown() below. Backend broadcaster is
// services/uploadProgress.service.js, keyed by a client-generated job id.
const streamConvertProgress = (jobId, token, onProgress) =>
streamSSE(`${import.meta.env.VITE_API_URL}/admin/assets/convert-progress/${jobId}`, token, onProgress);
@@ -242,39 +239,41 @@ export function AssetsProvider({ children }) {
[request]
);
// ─── POST /api/admin/assets ───────────────────────────────────────────────
// ─── POST /api/admin/assets/presign + direct PUT + POST /api/admin/assets ──
//
// onProgress?: ({ phase: 'uploading'|'storing'|'done'|'error', pct }) => void
// Two honest, sequential phases — not one blended/estimated number:
// "uploading" — browser -> this backend, real bytes sent (axios onUploadProgress).
// "storing" — this backend -> Garage, the actual S3 PUT, streamed live over
// SSE (see streamUploadProgress above / uploadProgress.service.js
// on the backend). Real numbers from the real transfer, both legs.
// onProgress?: ({ phase: 'uploading'|'processing'|'done', pct }) => void
// "uploading" — browser -> storage, real bytes sent directly (this
// backend is never in that data path at all anymore).
// "processing" — brief server-side step once the upload lands: reads the
// object back (HeadObjectCommand), runs ffprobe for
// video/audio, inserts the DB row.
const uploadAsset = useCallback(
({ file, thumbnail, onProgress, ...rest }) =>
request(async () => {
const uploadId = nanoid();
const form = new FormData();
form.append("file", file);
if (thumbnail) form.append("thumbnail", thumbnail);
form.append("uploadId", uploadId);
Object.entries(rest).forEach(([k, v]) => {
if (v !== undefined && v !== null) form.append(k, v);
const [mainPresign, thumbPresign] = await Promise.all([
presignAssetUpload(file),
thumbnail ? presignAssetUpload(thumbnail) : Promise.resolve(null),
]);
const storage_key = mainPresign.key;
await Promise.all([
uploadPresigned(file, mainPresign, (pct) => onProgress?.({ phase: "uploading", pct })),
thumbPresign ? uploadPresigned(thumbnail, thumbPresign) : Promise.resolve(),
]);
onProgress?.({ phase: "processing", pct: 100 });
const res = await api.post("/admin/assets", {
storage_key,
thumbnail_storage_key: thumbPresign?.key,
original_name: file.name,
// Fallback only — the backend prefers storage's own
// Content-Type, this just covers the rare case a browser
// sent the PUT with no Content-Type at all (empty File.type).
mimetype: file.type || undefined,
...rest,
});
const stopStream = onProgress
? streamUploadProgress(uploadId, accessTokenRef.current, onProgress)
: null;
try {
const res = await api.post("/admin/assets", form, {
onUploadProgress: onProgress
? (evt) => {
const pct = evt.total ? Math.round((evt.loaded / evt.total) * 100) : 0;
onProgress({ phase: "uploading", pct });
}
: undefined,
});
const asset = res.data?.data?.data ?? null;
if (asset) {
setAssets((prev) => [asset, ...prev]);
@@ -283,11 +282,8 @@ export function AssetsProvider({ children }) {
}
onProgress?.({ phase: "done", pct: 100 });
return res.data;
} finally {
stopStream?.();
}
}),
[request, accessTokenRef]
[request]
);
// ─── POST /api/admin/assets/:assetId/convert-to-markdown ─────────────────
src/contexts/UploadQueueContext.jsx
+34 -33
View File
@@ -6,25 +6,23 @@
// UploadProgressToast reads the same state from anywhere in /admin.
//
// Every job — whether it's part of an initial addBatch() or a single retry —
// goes out as its OWN POST /admin/assets/batch request (that endpoint has
// always accepted 1-20 files; sending 1 is a fully supported path). This
// matters beyond just per-file progress: the backend processes a batch
// request's files sequentially inside one request/response cycle and never
// aborts on client disconnect, so a connection drop anywhere during a
// combined multi-file request used to fail EVERY job in that request, even
// ones the server had already finished (S3 object + DB row both created) —
// clicking Retry on those then silently created a duplicate. One request per
// file means a dropped connection can only affect the one job riding it.
// uploadOne() below is shared by both the initial run and retries; runBatch()
// just fans it out over a small worker pool so a big batch doesn't open 20
// connections at once (MAX_CONCURRENT mirrors garage-anon-proxy's own
// MAX_CONCURRENT_UPLOADS=3 ceiling on the S3 side, so the frontend doesn't
// just relocate the pile-up into tripping that limit instead).
// uploads directly to storage via its own presigned PUT (see
// presignedUpload.util.js), then finalizes with its own POST /admin/assets
// call — the same per-file isolation as before (a dropped connection can
// only ever affect the one job riding it), now with the added benefit that
// the file's bytes never pass through this backend's memory at all, no
// matter how large the file is. uploadOne() below is shared by both the
// initial run and retries; runBatch() just fans it out over a small worker
// pool so a big batch doesn't fire 20 uploads at once (MAX_CONCURRENT
// mirrors the Garage-edge Caddy's own concurrent-upload cap, so the
// frontend doesn't just relocate the pile-up into tripping that limit
// instead).
import { createContext, useCallback, useContext, useRef, useState } from "react";
import { nanoid } from "nanoid";
import api from "@/utils/api.util";
import { validateAssetFile } from "@/utils/assetUpload.util";
import { presignAssetUpload, uploadPresigned } from "@/utils/presignedUpload.util";
const MAX_CONCURRENT = 3;
@@ -47,31 +45,34 @@ export function UploadQueueProvider({ children }) {
setJobs((prev) => prev.map((j) => (idSet.has(j.id) ? { ...j, ...(typeof patch === "function" ? patch(j) : patch) } : j)));
}, []);
// ─── Upload exactly one job through its own request ────────────────────
// ─── Upload exactly one job: presign -> direct PUT -> finalize ─────────
// Shared by the initial batch run and single-job retries — a dropped
// connection here can only ever fail the one job riding this request,
// never its batch-mates.
// connection here can only ever fail the one job riding it, never its
// batch-mates. Same isolation guarantee as before, plus the file's bytes
// now go straight to storage instead of buffering through the backend.
const uploadOne = useCallback(async (job) => {
const form = new FormData();
form.append("files", job.file);
Object.entries(job.meta).forEach(([k, v]) => {
if (v !== undefined && v !== null) form.append(k, v);
});
try {
const presigned = await presignAssetUpload(job.file);
const storage_key = presigned.key;
await uploadPresigned(job.file, presigned, (pct) => patchJobs([job.id], { progress: pct }));
// Bulk display_name derivation only applies when the caller didn't
// already provide one — each job keeps whatever name it resolves to.
if (!job.meta.display_name) form.append("display_name", baseNameOf(job.name));
const display_name = job.meta.display_name || baseNameOf(job.name);
try {
const { data } = await api.post("/admin/assets/batch", form, {
onUploadProgress: (evt) => {
const pct = evt.total ? Math.round((evt.loaded / evt.total) * 100) : 0;
patchJobs([job.id], { progress: pct });
},
const { data } = await api.post("/admin/assets", {
...job.meta,
storage_key,
original_name: job.name,
display_name,
// Fallback only — the backend prefers storage's own
// Content-Type, this just covers the rare case a browser
// sent the PUT with no Content-Type at all (empty File.type).
mimetype: job.mime || undefined,
});
const result = data?.data?.results?.[0];
patchJobs([job.id], result?.success
? { status: "uploaded", progress: 100, asset: result.data, error: null }
: { status: "failed", progress: 100, error: result?.message || "Upload failed." });
const asset = data?.data?.data;
patchJobs([job.id], { status: "uploaded", progress: 100, asset, error: null });
} catch (err) {
// No sonner toast() here — the floating widget (both corners
// would collide, see UploadProgressToast) already surfaces this
src/modules/admin/pages/assets/AddAsset.jsx
+6 -10
View File
@@ -9,6 +9,7 @@ import { ArrowLeft, ArrowRight, UploadCloud, X, FileVideo, FileText, Image, File
import { useAssets } from "@/contexts/AdminAssetsContext";
import { useAuth } from "@/contexts/AuthContext";
import { MAX_ASSET_FILE_SIZE, MAX_ASSET_FILE_SIZE_LABEL } from "@/utils/assetUpload.util";
import { useUnsavedChangesGuard } from "@/hooks/useUnsavedChangesGuard";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@@ -34,11 +35,6 @@ function resolveFileType(mimeType = "") {
return "document";
}
// Matches asset_upload.middleware.js on the backend — checked here too so an
// oversized file is rejected instantly instead of only after a full upload
// attempt round-trips to the server.
const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
// ─── Schema ───────────────────────────────────────────────────────────────────
const schema = z.object({
@@ -128,7 +124,7 @@ export default function AddAsset() {
const fileRef = useRef(null);
const thumbnailRef = useRef(null);
const [thumbKey, setThumbKey] = useState(0);
const [progress, setProgress] = useState(null); // { phase: 'uploading'|'storing'|'done'|'error', pct } | null
const [progress, setProgress] = useState(null); // { phase: 'uploading'|'processing'|'done'|'error', pct } | null
const {
register,
@@ -161,8 +157,8 @@ export default function AddAsset() {
const fileType = file ? resolveFileType(file.type) : null;
const setFile = (f) => {
if (f.size > MAX_FILE_SIZE) {
setError("_file", { message: `File exceeds the ${MAX_FILE_SIZE / (1024 * 1024)} MB size limit.` });
if (f.size > MAX_ASSET_FILE_SIZE) {
setError("_file", { message: `File exceeds the ${MAX_ASSET_FILE_SIZE_LABEL} size limit.` });
return;
}
fileRef.current = f;
@@ -205,8 +201,8 @@ export default function AddAsset() {
};
const progressLabel = {
uploading: "Uploading to server…",
storing: "Storing to server...",
uploading: "Uploading…",
processing: "Processing…",
done: "Done.",
error: "Upload failed.",
}[progress?.phase];
src/utils/assetUpload.util.js
+10 -5
View File
@@ -28,10 +28,15 @@ const ALLOWED_DOCUMENT_EXTENSIONS = new Set([
"zip", "json", "rtf", "txt", "csv", "md",
]);
// Matches asset_upload.middleware.js on the backend — rejecting an oversized
// file here means the queue shows a clear "invalid" reason instantly instead
// of a job that uploads for a while and then fails with a generic error.
export const MAX_ASSET_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
// Uploads go straight to storage via a presigned URL (see
// presignedUpload.util.js) — this backend never buffers the file. Above 5GB
// (S3-compatible storage's own single-PUT ceiling) uploads switch to real
// multipart automatically; 15GB is the app-level ceiling chosen for future
// assets, well within multipart's own much larger real limit. Rejecting an
// oversized file here still means the queue shows a clear "invalid" reason
// instantly instead of a job that uploads for a while and then fails.
export const MAX_ASSET_FILE_SIZE = 15 * 1024 * 1024 * 1024; // 15 GB
export const MAX_ASSET_FILE_SIZE_LABEL = "15 GB";
export function fileExtension(filename = "") {
const dot = filename.lastIndexOf(".");
@@ -43,7 +48,7 @@ export function validateAssetFile(file) {
if (file.size > MAX_ASSET_FILE_SIZE) {
return {
ok: false,
reason: `File exceeds the ${MAX_ASSET_FILE_SIZE / (1024 * 1024)} MB size limit.`,
reason: `File exceeds the ${MAX_ASSET_FILE_SIZE_LABEL} size limit.`,
};
}
src/utils/presignedUpload.util.js
+169
View File
@@ -0,0 +1,169 @@
// utils/presignedUpload.util.js
//
// Shared by AdminAssetsContext.jsx (single-file Add Asset) and
// UploadQueueContext.jsx (Add Assets Bulk) — the browser uploads file bytes
// directly to storage via a short-lived presigned URL, so this backend
// never buffers them regardless of file size. presignAssetUpload() mints
// the URL(s); putDirect()/putMultipart() send the bytes with real progress
// events, and deliberately bypass the shared `api` axios instance since
// they target a different origin (the storage host) and must not pick up
// the app's Authorization-header interceptor or any other header that
// wasn't part of what the URL was signed for.
//
// Files at or below storage's single-PUT ceiling (5GB) get one presigned
// PUT (putDirect). Anything larger uses real multipart upload (putMultipart)
// — the backend splits it into 50MB parts, presigns each one, and the
// browser uploads them through a small concurrent worker pool, retrying an
// individual failed part a couple of times before giving up entirely.
import api from "@/utils/api.util";
const MAX_PART_RETRIES = 2;
const MAX_CONCURRENT_PARTS = 3;
export function resolveFileType(mimetype = "") {
if (mimetype.startsWith("image/")) return "image";
if (mimetype.startsWith("video/")) return "video";
if (mimetype.startsWith("audio/")) return "audio";
return "document";
}
// -> { key, uploadUrl } (size <= 5GB)
// -> { key, multipart: true, uploadId, partSize, parts } (size > 5GB)
export async function presignAssetUpload(file) {
const { data } = await api.post("/admin/assets/presign", {
filename: file.name,
mimetype: file.type,
file_type: resolveFileType(file.type),
size: file.size,
});
return data.data;
}
export function putDirect(uploadUrl, file, onProgress) {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("PUT", uploadUrl);
if (onProgress) {
xhr.upload.onprogress = (evt) => {
onProgress(evt.lengthComputable ? Math.round((evt.loaded / evt.total) * 100) : 0);
};
}
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) resolve();
else reject(new Error(`Upload failed (${xhr.status}).`));
};
xhr.onerror = () => reject(new Error("Network error during upload."));
xhr.send(file);
});
}
// Single part PUT — resolves with the ETag S3 returns for that part (needed
// to build the CompleteMultipartUpload part list later). Requires ETag to be
// listed in the storage edge's Access-Control-Expose-Headers, since it's
// otherwise invisible to JS on a cross-origin response.
function putPart(uploadUrl, blob, onLoaded) {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("PUT", uploadUrl);
xhr.upload.onprogress = (evt) => {
if (evt.lengthComputable) onLoaded(evt.loaded);
};
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) {
const etag = xhr.getResponseHeader("ETag");
if (!etag) { reject(new Error("Part upload succeeded but no ETag was returned.")); return; }
onLoaded(blob.size);
resolve(etag);
} else {
reject(new Error(`Part upload failed (${xhr.status}).`));
}
};
xhr.onerror = () => reject(new Error("Network error during part upload."));
xhr.send(blob);
});
}
async function putPartWithRetry(uploadUrl, blob, onLoaded) {
let lastErr;
for (let attempt = 0; attempt <= MAX_PART_RETRIES; attempt++) {
try {
return await putPart(uploadUrl, blob, onLoaded);
} catch (err) {
lastErr = err;
onLoaded(0); // reset this part's contribution before retrying
}
}
throw lastErr;
}
// multipartInfo: { uploadId, partSize, parts: [{ partNumber, uploadUrl }] }
// -> [{ partNumber, etag }, ...] (unordered — completeMultipartUpload sorts)
export async function putMultipart(file, multipartInfo, onProgress) {
const { partSize, parts } = multipartInfo;
const loadedByPart = new Array(parts.length).fill(0);
const reportProgress = () => {
if (!onProgress) return;
const loaded = loadedByPart.reduce((sum, n) => sum + n, 0);
onProgress(Math.min(100, Math.round((loaded / file.size) * 100)));
};
const results = new Array(parts.length);
let next = 0;
const worker = async () => {
while (next < parts.length) {
const i = next++;
const { partNumber, uploadUrl } = parts[i];
const start = (partNumber - 1) * partSize;
const end = Math.min(start + partSize, file.size);
const blob = file.slice(start, end);
const etag = await putPartWithRetry(uploadUrl, blob, (loaded) => {
loadedByPart[i] = loaded;
reportProgress();
});
results[i] = { partNumber, etag };
}
};
await Promise.all(
Array.from({ length: Math.min(MAX_CONCURRENT_PARTS, parts.length) }, worker)
);
return results;
}
// Uploads one file to whatever presignAssetUpload() returned — single PUT or
// full multipart, whichever the file's size required — and leaves storage
// clean on failure (aborts a partially-uploaded multipart session rather
// than leaving it to linger). Shared by AdminAssetsContext.jsx and
// UploadQueueContext.jsx so neither has to duplicate this branch.
export async function uploadPresigned(file, presigned, onProgress) {
if (!presigned.multipart) {
await putDirect(presigned.uploadUrl, file, onProgress);
return;
}
try {
const parts = await putMultipart(file, presigned, onProgress);
await completeMultipartUpload(presigned.key, presigned.uploadId, parts);
} catch (err) {
await abortMultipartUpload(presigned.key, presigned.uploadId);
throw err;
}
}
export async function completeMultipartUpload(storage_key, uploadId, parts) {
await api.post("/admin/assets/complete-multipart", { storage_key, uploadId, parts });
}
export async function abortMultipartUpload(storage_key, uploadId) {
// Best-effort — a failed cleanup call shouldn't mask the real upload
// error that triggered the abort in the first place.
try {
await api.post("/admin/assets/abort-multipart", { storage_key, uploadId });
} catch {
// ignore
}
}