mirror of
https://github.com/rgrgogu/new_starr.git
synced 2026-09-27 00:12:54 +08:00
assets: go 15GB limit
Signed-off-by: Kenneth Obsequio <k80308392@gmail.com>
This commit is contained in:
@@ -320,96 +320,68 @@ exports.getAsset = async (req, res) => {
|
|||||||
// ─── UPLOAD (shared core) ──────────────────────────────────────────────────────
|
// ─── UPLOAD (shared core) ──────────────────────────────────────────────────────
|
||||||
//
|
//
|
||||||
// ┌─────────────────────────────────────────────────────────────────────────┐
|
// ┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
// │ TRANSACTION STRATEGY │
|
// │ PRESIGNED-UPLOAD STRATEGY │
|
||||||
// │ │
|
// │ │
|
||||||
// │ Phase 1 — SLOW WORK (outside transaction): │
|
// │ The browser already PUT the file's bytes straight to storage via a │
|
||||||
// │ • Input validation │
|
// │ presigned URL (see presignAssetUpload below) — this backend never │
|
||||||
// │ • ffprobe metadata extraction │
|
// │ buffers or even touches them (this is what removes the old 500MB │
|
||||||
// │ • Provider upload (chibi or s3) → track for rollback │
|
// │ multer-memoryStorage RAM ceiling entirely, regardless of file size). │
|
||||||
// │ • Thumbnail upload → track for rollback │
|
// │ Finalizing an asset from an already-uploaded object is just: │
|
||||||
// │ │
|
// │ • HeadObjectCommand → real file_size/mime_type/checksum (=ETag) │
|
||||||
// │ Phase 2 — FAST WORK (transaction open milliseconds only): │
|
// │ • ffprobe by URL → video/audio metadata only, no download │
|
||||||
// │ • BEGIN → Asset.create() → COMMIT │
|
// │ • BEGIN → Asset.create() → COMMIT │
|
||||||
// │ │
|
// │ │
|
||||||
// │ On any error: │
|
// │ On any error: rollbackUploads([{ key, provider }]) deletes the │
|
||||||
// │ • ROLLBACK transaction (if opened) │
|
// │ already-uploaded object(s) — same cleanup as before, just always │
|
||||||
// │ • rollbackUploads([{ key, provider }]) to clean orphans │
|
// │ covering both file + thumbnail upfront, since both already exist in │
|
||||||
|
// │ storage by the time this runs (the browser uploaded them first). │
|
||||||
// └─────────────────────────────────────────────────────────────────────────┘
|
// └─────────────────────────────────────────────────────────────────────────┘
|
||||||
//
|
//
|
||||||
// Shared by the single-file POST / and the multi-file POST /batch routes.
|
|
||||||
// Thumbnails are optional for both video and audio — a video/audio asset can
|
// Thumbnails are optional for both video and audio — a video/audio asset can
|
||||||
// land with thumbnail_url null and pick one up later via the existing
|
// land with thumbnail_url null and pick one up later via the existing
|
||||||
// "thumbnail-only" path in updateAsset().
|
// "thumbnail-only" path in updateAsset().
|
||||||
//
|
//
|
||||||
async function createAssetFromUpload({ file, thumbFile, body, user, uploadId }) {
|
async function finalizeAssetFromStorage({ storage_key, thumbnail_storage_key, original_name, body, user }) {
|
||||||
const uploadedFiles = []; // [{ key, provider }]
|
const {
|
||||||
|
display_name,
|
||||||
|
description,
|
||||||
|
is_public = false,
|
||||||
|
storage_provider = "s3",
|
||||||
|
storage_bucket,
|
||||||
|
createdBy,
|
||||||
|
} = body;
|
||||||
|
|
||||||
|
const uploadedFiles = [{ key: storage_key, provider: storage_provider }];
|
||||||
|
if (thumbnail_storage_key) uploadedFiles.push({ key: thumbnail_storage_key, provider: storage_provider });
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (!file) throw Object.assign(new Error("No file uploaded."), { status: 400 });
|
if (!storage_key) throw Object.assign(new Error("storage_key is required."), { status: 400 });
|
||||||
|
|
||||||
const {
|
|
||||||
display_name,
|
|
||||||
description,
|
|
||||||
is_public = false,
|
|
||||||
// TEMPORARY: defaulting to "s3" instead of "chibisafe" — zrok tunnel is
|
|
||||||
// dropping Chibisafe uploads (502s). Revert this default once off zrok/VPS migration.
|
|
||||||
storage_provider = "s3",
|
|
||||||
storage_bucket,
|
|
||||||
storage_key,
|
|
||||||
createdBy,
|
|
||||||
} = body;
|
|
||||||
|
|
||||||
if (!createdBy) throw Object.assign(new Error("createdBy is required."), { status: 400 });
|
if (!createdBy) throw Object.assign(new Error("createdBy is required."), { status: 400 });
|
||||||
|
|
||||||
const mime_type = file.mimetype;
|
const svc = getProvider(storage_provider);
|
||||||
const file_type = resolveFileType(mime_type);
|
if (!svc || !svc.getFileMetadata) {
|
||||||
const extension = resolveExtension(file.originalname);
|
throw Object.assign(new Error("Presigned uploads are only supported for S3 storage."), { status: 400 });
|
||||||
const checksum = file.buffer ? resolveChecksum(file.buffer) : null;
|
|
||||||
const usesProvider = ["chibisafe", "s3"].includes(storage_provider);
|
|
||||||
|
|
||||||
if (usesProvider && !file.buffer) {
|
|
||||||
throw Object.assign(new Error("File buffer is required. Ensure multer uses memoryStorage."), { status: 400 });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Phase 1b: Upload main file ────────────────────────────────────────────
|
let meta;
|
||||||
|
try {
|
||||||
let file_url = null;
|
meta = await svc.getFileMetadata(storage_key);
|
||||||
let storage_key_resolved = null;
|
} catch {
|
||||||
|
throw Object.assign(new Error("Uploaded file not found in storage — the upload may have failed or expired."), { status: 400 });
|
||||||
if (usesProvider) {
|
|
||||||
const svc = getProvider(storage_provider);
|
|
||||||
// Real Express -> Garage progress, main file only (not the thumbnail —
|
|
||||||
// it's small enough that tracking it wouldn't add anything useful).
|
|
||||||
// Relayed live to the browser over SSE; see uploadProgress.service.js.
|
|
||||||
const onProgress = (storage_provider === "s3" && uploadId)
|
|
||||||
? ({ loaded, total }) => uploadProgress.publish(uploadId, {
|
|
||||||
phase: "storing",
|
|
||||||
loaded,
|
|
||||||
total,
|
|
||||||
pct: total ? Math.round((loaded / total) * 100) : 0,
|
|
||||||
})
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
const result = await svc.uploadFile({
|
|
||||||
buffer: file.buffer,
|
|
||||||
originalname: file.originalname,
|
|
||||||
mimetype: mime_type,
|
|
||||||
ownerType: file_type,
|
|
||||||
onProgress,
|
|
||||||
});
|
|
||||||
file_url = result.url;
|
|
||||||
storage_key_resolved = result.uuid;
|
|
||||||
uploadedFiles.push({ key: storage_key_resolved, provider: storage_provider });
|
|
||||||
|
|
||||||
} else {
|
|
||||||
file_url = storage_provider === "local"
|
|
||||||
? `/uploads/${file.filename}`
|
|
||||||
: body.file_url;
|
|
||||||
|
|
||||||
if (!file_url) throw Object.assign(new Error("file_url is required for non-local storage."), { status: 400 });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Phase 1c: ffprobe + thumbnail ─────────────────────────────────────────
|
// S3's own Content-Type is authoritative when present, but browsers only
|
||||||
|
// send one automatically when the File object's own .type is non-empty —
|
||||||
|
// fall back to whatever the client reported at presign time, and finally
|
||||||
|
// to a generic default, rather than ever letting a NOT NULL column see
|
||||||
|
// null here (mime_type also drives file_type below, so a null here would
|
||||||
|
// misclassify the asset entirely, not just leave a field blank).
|
||||||
|
const mime_type = meta.mimetype || body.mimetype || "application/octet-stream";
|
||||||
|
const file_type = resolveFileType(mime_type);
|
||||||
|
const extension = resolveExtension(original_name || storage_key);
|
||||||
|
const file_url = await svc.buildPublicUrl(storage_key);
|
||||||
|
|
||||||
|
// ── ffprobe (video/audio only) ────────────────────────────────────────────
|
||||||
|
|
||||||
let width = null, height = null, resolution = null;
|
let width = null, height = null, resolution = null;
|
||||||
let duration = null, frame_rate = null, bitrate = null;
|
let duration = null, frame_rate = null, bitrate = null;
|
||||||
@@ -417,34 +389,18 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId })
|
|||||||
let thumbnail_url = null;
|
let thumbnail_url = null;
|
||||||
|
|
||||||
if (file_type === "video" || file_type === "audio") {
|
if (file_type === "video" || file_type === "audio") {
|
||||||
const meta = await extractVideoMeta({ buffer: file.buffer, extension: extension || (file_type === "video" ? "mp4" : "mp3") });
|
const probeUrl = await svc.getSignedDownloadUrl(storage_key);
|
||||||
width = meta.width;
|
const videoMeta = await extractVideoMeta({ url: probeUrl });
|
||||||
height = meta.height;
|
width = videoMeta.width;
|
||||||
resolution = meta.resolution;
|
height = videoMeta.height;
|
||||||
duration = meta.duration;
|
resolution = videoMeta.resolution;
|
||||||
frame_rate = meta.frame_rate;
|
duration = videoMeta.duration;
|
||||||
bitrate = meta.bitrate;
|
frame_rate = videoMeta.frame_rate;
|
||||||
video_codec = meta.video_codec;
|
bitrate = videoMeta.bitrate;
|
||||||
audio_codec = meta.audio_codec;
|
video_codec = videoMeta.video_codec;
|
||||||
|
audio_codec = videoMeta.audio_codec;
|
||||||
|
|
||||||
if (usesProvider && thumbFile) {
|
if (thumbnail_storage_key) thumbnail_url = await svc.buildPublicUrl(thumbnail_storage_key);
|
||||||
if (!thumbFile.buffer) {
|
|
||||||
await rollbackUploads(uploadedFiles);
|
|
||||||
throw Object.assign(new Error("Thumbnail buffer is required."), { status: 400 });
|
|
||||||
}
|
|
||||||
const baseName = file.originalname.replace(/\.[^.]+$/, "");
|
|
||||||
const svc = getProvider(storage_provider);
|
|
||||||
const thumbResult = await svc.uploadFile({
|
|
||||||
buffer: thumbFile.buffer,
|
|
||||||
originalname: `thumb_${baseName}.${resolveExtension(thumbFile.originalname) || "jpg"}`,
|
|
||||||
mimetype: thumbFile.mimetype,
|
|
||||||
ownerType: "thumbnail",
|
|
||||||
});
|
|
||||||
thumbnail_url = thumbResult.url;
|
|
||||||
uploadedFiles.push({ key: thumbResult.uuid, provider: storage_provider });
|
|
||||||
} else if (!usesProvider) {
|
|
||||||
thumbnail_url = thumbFile?.filename ? `/uploads/${thumbFile.filename}` : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
const parsedWidth = body.width ? parseInt(body.width) : null;
|
const parsedWidth = body.width ? parseInt(body.width) : null;
|
||||||
@@ -454,18 +410,18 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId })
|
|||||||
resolution = resolveResolution(parsedWidth, parsedHeight);
|
resolution = resolveResolution(parsedWidth, parsedHeight);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Phase 2: DB insert ────────────────────────────────────────────────────
|
// ── DB insert ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const t = await sequelize.transaction();
|
const t = await sequelize.transaction();
|
||||||
try {
|
try {
|
||||||
const asset = await Asset.create({
|
const asset = await Asset.create({
|
||||||
original_name: file.originalname,
|
original_name: original_name || storage_key,
|
||||||
display_name: display_name || file.originalname,
|
display_name: display_name || original_name || storage_key,
|
||||||
file_url,
|
file_url,
|
||||||
file_size: file.size,
|
file_size: meta.size,
|
||||||
mime_type,
|
mime_type,
|
||||||
extension,
|
extension,
|
||||||
checksum,
|
checksum: meta.checksum,
|
||||||
file_type,
|
file_type,
|
||||||
width,
|
width,
|
||||||
height,
|
height,
|
||||||
@@ -479,7 +435,7 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId })
|
|||||||
description,
|
description,
|
||||||
storage_provider,
|
storage_provider,
|
||||||
storage_bucket: storage_bucket || (storage_provider === "s3" ? process.env.S3_BUCKET : null) || null,
|
storage_bucket: storage_bucket || (storage_provider === "s3" ? process.env.S3_BUCKET : null) || null,
|
||||||
storage_key: storage_key_resolved || storage_key || file.filename || null,
|
storage_key,
|
||||||
is_public,
|
is_public,
|
||||||
createdBy,
|
createdBy,
|
||||||
}, { transaction: t });
|
}, { transaction: t });
|
||||||
@@ -500,90 +456,117 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId })
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.uploadAsset = async (req, res) => {
|
// ─── PRESIGN UPLOAD ─────────────────────────────────────────────────────────
|
||||||
const { uploadId } = req.body;
|
//
|
||||||
|
// Mints a short-lived presigned PUT URL so the browser can upload the file's
|
||||||
|
// bytes directly to storage — this backend never buffers them. Called once
|
||||||
|
// for the main file, and once more for a thumbnail if the admin picked one
|
||||||
|
// (see s3.service.js#presignUpload for the key-naming convention).
|
||||||
|
//
|
||||||
|
exports.presignAssetUpload = async (req, res) => {
|
||||||
try {
|
try {
|
||||||
const file = req.files?.file?.[0];
|
const { filename, mimetype, file_type, size = 0, storage_provider = "s3" } = req.body;
|
||||||
const thumbFile = req.files?.thumbnail?.[0];
|
if (!filename) return R.error(res, "filename is required.", 400);
|
||||||
|
|
||||||
const asset = await createAssetFromUpload({ file, thumbFile, body: req.body, user: req.user, uploadId });
|
const svc = getProvider(storage_provider);
|
||||||
|
if (!svc || !svc.presignUpload) {
|
||||||
|
return R.error(res, "Presigned uploads are only supported for S3 storage.", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
const ownerType = file_type || resolveFileType(mimetype || "") || "document";
|
||||||
|
// Result is either { key, uploadUrl } or, above the single-PUT ceiling,
|
||||||
|
// { key, multipart: true, uploadId, partSize, parts } — see
|
||||||
|
// s3.service.js#presignUpload. The client branches on `multipart`.
|
||||||
|
const presigned = await svc.presignUpload(filename, ownerType, Number(size) || 0);
|
||||||
|
return R.success(res, "Presigned URL generated.", presigned);
|
||||||
|
|
||||||
|
} catch (err) {
|
||||||
|
console.error("[ASSET][PRESIGN]", err);
|
||||||
|
return R.error(res, "Could not generate upload URL.", 500);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// ─── COMPLETE / ABORT MULTIPART ─────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Only used above presignAssetUpload's single-PUT ceiling (see
|
||||||
|
// s3.service.js's MULTIPART_THRESHOLD) — the browser PUTs every part
|
||||||
|
// directly, then calls complete-multipart with the ETags each part's PUT
|
||||||
|
// response returned. abort-multipart is the failure-path cleanup (a part
|
||||||
|
// exhausted its retries, or the admin cancelled) so an abandoned multipart
|
||||||
|
// upload doesn't linger as orphaned storage forever.
|
||||||
|
//
|
||||||
|
exports.completeMultipartAssetUpload = async (req, res) => {
|
||||||
|
try {
|
||||||
|
const { storage_key, uploadId, parts, storage_provider = "s3" } = req.body;
|
||||||
|
if (!storage_key || !uploadId || !Array.isArray(parts) || !parts.length) {
|
||||||
|
return R.error(res, "storage_key, uploadId, and parts are required.", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
const svc = getProvider(storage_provider);
|
||||||
|
if (!svc || !svc.completeMultipartUpload) {
|
||||||
|
return R.error(res, "Multipart uploads are only supported for S3 storage.", 400);
|
||||||
|
}
|
||||||
|
|
||||||
|
await svc.completeMultipartUpload(storage_key, uploadId, parts);
|
||||||
|
return R.success(res, "Multipart upload completed.", {});
|
||||||
|
|
||||||
|
} catch (err) {
|
||||||
|
console.error("[ASSET][COMPLETE MULTIPART]", err);
|
||||||
|
return R.error(res, "Could not complete multipart upload.", 500);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.abortMultipartAssetUpload = async (req, res) => {
|
||||||
|
try {
|
||||||
|
const { storage_key, uploadId, storage_provider = "s3" } = req.body;
|
||||||
|
if (!storage_key || !uploadId) return R.error(res, "storage_key and uploadId are required.", 400);
|
||||||
|
|
||||||
|
const svc = getProvider(storage_provider);
|
||||||
|
if (svc?.abortMultipartUpload) {
|
||||||
|
try {
|
||||||
|
await svc.abortMultipartUpload(storage_key, uploadId);
|
||||||
|
} catch (err) {
|
||||||
|
// Best-effort, same tolerance as rollbackUploads() — an already-gone
|
||||||
|
// or already-completed upload isn't worth failing the request over.
|
||||||
|
console.error(`[ASSET][ABORT MULTIPART] Failed to abort "${storage_key}":`, err.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return R.success(res, "Multipart upload aborted.", {});
|
||||||
|
|
||||||
|
} catch (err) {
|
||||||
|
console.error("[ASSET][ABORT MULTIPART]", err);
|
||||||
|
return R.error(res, "Could not abort multipart upload.", 500);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// ─── UPLOAD (finalize) ──────────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Called once the browser's direct-to-storage PUT(s) have completed. Body is
|
||||||
|
// plain JSON — no file bytes here, just the storage key(s) presignAssetUpload
|
||||||
|
// handed back plus asset metadata. Also the endpoint the bulk queue
|
||||||
|
// (UploadQueueContext.jsx) calls once per file, reusing this single-asset
|
||||||
|
// path instead of a separate batch endpoint.
|
||||||
|
//
|
||||||
|
exports.uploadAsset = async (req, res) => {
|
||||||
|
try {
|
||||||
|
const { storage_key, thumbnail_storage_key, original_name } = req.body;
|
||||||
|
const asset = await finalizeAssetFromStorage({ storage_key, thumbnail_storage_key, original_name, body: req.body, user: req.user });
|
||||||
invalidateListCache();
|
invalidateListCache();
|
||||||
if (uploadId) uploadProgress.complete(uploadId, { phase: "done", pct: 100 });
|
|
||||||
return R.success(res, "Asset uploaded.", { data: asset }, 201);
|
return R.success(res, "Asset uploaded.", { data: asset }, 201);
|
||||||
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error("[ASSET][UPLOAD]", err);
|
console.error("[ASSET][UPLOAD]", err);
|
||||||
if (uploadId) uploadProgress.complete(uploadId, { phase: "error" });
|
|
||||||
if (err.status) return R.error(res, err.message, err.status, { detail: err.chibiBody });
|
if (err.status) return R.error(res, err.message, err.status, { detail: err.chibiBody });
|
||||||
return R.error(res, "Internal server error.", 500);
|
return R.error(res, "Internal server error.", 500);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// ─── UPLOAD PROGRESS (SSE) ──────────────────────────────────────────────────
|
|
||||||
//
|
|
||||||
// Client opens this before POSTing the file, correlated by a client-generated
|
|
||||||
// uploadId sent as a form field on the upload request itself. Streams the
|
|
||||||
// real Express -> Garage httpUploadProgress events from s3.service.js's
|
|
||||||
// Upload — not a simulated or estimated number.
|
|
||||||
//
|
|
||||||
exports.streamUploadProgress = (req, res) => {
|
|
||||||
uploadProgress.subscribe(req.params.uploadId, res);
|
|
||||||
};
|
|
||||||
|
|
||||||
// Same broadcaster, reused as-is for document-conversion job progress — it's
|
// Same broadcaster, reused as-is for document-conversion job progress — it's
|
||||||
// just a generic string-keyed SSE channel, nothing upload-specific about it.
|
// just a generic string-keyed SSE channel, nothing upload-specific about it.
|
||||||
exports.streamConvertProgress = (req, res) => {
|
exports.streamConvertProgress = (req, res) => {
|
||||||
uploadProgress.subscribe(req.params.jobId, res);
|
uploadProgress.subscribe(req.params.jobId, res);
|
||||||
};
|
};
|
||||||
|
|
||||||
// ─── UPLOAD (batch) ─────────────────────────────────────────────────────────
|
|
||||||
//
|
|
||||||
// Accepts multiple files under the "files" field in one multipart request,
|
|
||||||
// all sharing the same is_public / storage_provider / createdBy. Each file
|
|
||||||
// is uploaded independently — one failing (bad codec, DB constraint, etc.)
|
|
||||||
// does not roll back the others. display_name defaults to the filename
|
|
||||||
// (minus extension) since there's no per-file metadata step in bulk mode.
|
|
||||||
// Videos land without a thumbnail (see createAssetFromUpload) — add one
|
|
||||||
// later via the existing "replace thumbnail" path on PATCH /:assetId.
|
|
||||||
//
|
|
||||||
exports.uploadAssetsBatch = async (req, res) => {
|
|
||||||
const files = req.files ?? [];
|
|
||||||
if (!files.length) return R.error(res, "No files uploaded.", 400);
|
|
||||||
|
|
||||||
// TEMPORARY: defaulting to "s3" instead of "chibisafe" — zrok tunnel is
|
|
||||||
// dropping Chibisafe uploads (502s). Revert this default once off zrok/VPS migration.
|
|
||||||
const { display_name, description, is_public = false, storage_provider = "s3", createdBy } = req.body;
|
|
||||||
if (!createdBy) return R.error(res, "createdBy is required.", 400);
|
|
||||||
|
|
||||||
const results = [];
|
|
||||||
|
|
||||||
for (const file of files) {
|
|
||||||
try {
|
|
||||||
const baseName = file.originalname.replace(/\.[^.]+$/, "");
|
|
||||||
const asset = await createAssetFromUpload({
|
|
||||||
file,
|
|
||||||
thumbFile: null,
|
|
||||||
body: {
|
|
||||||
display_name: files.length === 1 ? (display_name || baseName) : baseName,
|
|
||||||
description,
|
|
||||||
is_public,
|
|
||||||
storage_provider,
|
|
||||||
createdBy,
|
|
||||||
},
|
|
||||||
user: req.user,
|
|
||||||
});
|
|
||||||
results.push({ originalname: file.originalname, success: true, data: asset });
|
|
||||||
} catch (err) {
|
|
||||||
console.error("[ASSET][UPLOAD BATCH]", file.originalname, err.stack || err);
|
|
||||||
results.push({ originalname: file.originalname, success: false, message: err.status ? err.message : "Internal server error." });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
invalidateListCache();
|
|
||||||
const createdCount = results.filter((r) => r.success).length;
|
|
||||||
return R.success(res, `${createdCount} of ${files.length} asset(s) uploaded.`, { results }, 201);
|
|
||||||
};
|
|
||||||
|
|
||||||
// ─── CONVERT TO MARKDOWN ────────────────────────────────────────────────────
|
// ─── CONVERT TO MARKDOWN ────────────────────────────────────────────────────
|
||||||
//
|
//
|
||||||
// PDF/PPTX -> Markdown, text only (see services/documentConversion.service.js
|
// PDF/PPTX -> Markdown, text only (see services/documentConversion.service.js
|
||||||
|
|||||||
@@ -11,13 +11,20 @@ router.get('/field-values', controller.getAssetFieldValues);
|
|||||||
router.delete('/bulk', sensitiveOpsLimiter, controller.archiveAssets);
|
router.delete('/bulk', sensitiveOpsLimiter, controller.archiveAssets);
|
||||||
router.delete('/bulk/permanent', sensitiveOpsLimiter, controller.permanentlyDeleteAssets);
|
router.delete('/bulk/permanent', sensitiveOpsLimiter, controller.permanentlyDeleteAssets);
|
||||||
router.patch('/bulk-restore', controller.restoreAssets);
|
router.patch('/bulk-restore', controller.restoreAssets);
|
||||||
router.get('/upload-progress/:uploadId', controller.streamUploadProgress);
|
|
||||||
router.get('/convert-progress/:jobId', controller.streamConvertProgress);
|
router.get('/convert-progress/:jobId', controller.streamConvertProgress);
|
||||||
|
|
||||||
// ─── Collection ───────────────────────────────────────────────────────────────
|
// ─── Collection ───────────────────────────────────────────────────────────────
|
||||||
router.get('/', controller.getAssets);
|
router.get('/', controller.getAssets);
|
||||||
router.post('/', handleUpload(upload.fields([{ name: 'file', maxCount: 1 }, { name: 'thumbnail', maxCount: 1 }])), controller.uploadAsset);
|
// Presigned-upload flow: the browser PUTs file bytes straight to storage, this
|
||||||
router.post('/batch', handleUpload(upload.array('files', 20)), controller.uploadAssetsBatch);
|
// backend never buffers them (see assets.controller.js). presign mints the
|
||||||
|
// upload URL(s); the POST / body that follows is plain JSON (no multer here).
|
||||||
|
router.post('/presign', controller.presignAssetUpload);
|
||||||
|
// Above presign's single-PUT ceiling only (see s3.service.js's
|
||||||
|
// MULTIPART_THRESHOLD) — the browser PUTs every part directly, then calls
|
||||||
|
// complete/abort here. Plain JSON, no file bytes.
|
||||||
|
router.post('/complete-multipart', controller.completeMultipartAssetUpload);
|
||||||
|
router.post('/abort-multipart', controller.abortMultipartAssetUpload);
|
||||||
|
router.post('/', controller.uploadAsset);
|
||||||
|
|
||||||
// ─── Dynamic routes last ──────────────────────────────────────────────────────
|
// ─── Dynamic routes last ──────────────────────────────────────────────────────
|
||||||
router.get('/:assetId', controller.getAsset);
|
router.get('/:assetId', controller.getAsset);
|
||||||
|
|||||||
@@ -73,15 +73,43 @@ function probeFile(filePath) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ffprobe/libavformat accepts http(s):// URLs directly — used for presigned
|
||||||
|
// -upload assets, where no buffer or local temp file exists at all (the
|
||||||
|
// browser PUT the bytes straight to storage, this backend never touched
|
||||||
|
// them). No default network timeout applies to a remote input the way it
|
||||||
|
// would to a local file, so this races the probe against an explicit one to
|
||||||
|
// avoid hanging the finalize request on a slow/stuck remote read.
|
||||||
|
function probeUrl(url, timeoutMs = 30_000) {
|
||||||
|
return Promise.race([
|
||||||
|
new Promise((resolve, reject) => {
|
||||||
|
ffmpeg.ffprobe(url, (err, metadata) => {
|
||||||
|
if (err) return reject(err);
|
||||||
|
resolve(metadata);
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
new Promise((_, reject) =>
|
||||||
|
setTimeout(() => reject(new Error(`ffprobe timed out after ${timeoutMs}ms`)), timeoutMs)
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
// ─── Public API ───────────────────────────────────────────────────────────────
|
// ─── Public API ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extract video metadata from a Buffer.
|
* Extract video metadata from either a Buffer or a remote URL.
|
||||||
* Thumbnail is NOT generated here — the client uploads it as a separate file.
|
* Thumbnail is NOT generated here — the client uploads it as a separate file.
|
||||||
*
|
*
|
||||||
* @param {object} opts
|
* @param {object} opts
|
||||||
* @param {Buffer} opts.buffer raw video bytes (multer memoryStorage)
|
* @param {Buffer} [opts.buffer] raw video bytes (multer memoryStorage) —
|
||||||
* @param {string} opts.extension file extension without dot, e.g. "mp4"
|
* written to a temp file and probed there.
|
||||||
|
* @param {string} [opts.extension] file extension without dot, e.g. "mp4" —
|
||||||
|
* only used with `buffer`.
|
||||||
|
* @param {string} [opts.url] a presigned GET URL (or any ffprobe/
|
||||||
|
* libavformat-reachable address) to probe
|
||||||
|
* directly, with no local buffer or temp
|
||||||
|
* file at all — used for presigned-upload
|
||||||
|
* assets. Takes precedence over `buffer`
|
||||||
|
* when both are given.
|
||||||
*
|
*
|
||||||
* @returns {Promise<VideoMeta>}
|
* @returns {Promise<VideoMeta>}
|
||||||
*
|
*
|
||||||
@@ -95,11 +123,11 @@ function probeFile(filePath) {
|
|||||||
* @property {string|null} video_codec "H.264", "H.265", …
|
* @property {string|null} video_codec "H.264", "H.265", …
|
||||||
* @property {string|null} audio_codec "AAC", "MP3", …
|
* @property {string|null} audio_codec "AAC", "MP3", …
|
||||||
*/
|
*/
|
||||||
async function extractVideoMeta({ buffer, extension }) {
|
async function extractVideoMeta({ buffer, extension, url }) {
|
||||||
const tmpPath = writeTempFile(buffer, extension || "mp4");
|
const tmpPath = url ? null : writeTempFile(buffer, extension || "mp4");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const raw = await probeFile(tmpPath);
|
const raw = await (tmpPath ? probeFile(tmpPath) : probeUrl(url));
|
||||||
|
|
||||||
const videoStream = raw.streams?.find((s) => s.codec_type === "video") || {};
|
const videoStream = raw.streams?.find((s) => s.codec_type === "video") || {};
|
||||||
const audioStream = raw.streams?.find((s) => s.codec_type === "audio") || {};
|
const audioStream = raw.streams?.find((s) => s.codec_type === "audio") || {};
|
||||||
@@ -135,7 +163,7 @@ async function extractVideoMeta({ buffer, extension }) {
|
|||||||
return { width, height, resolution, duration, frame_rate, bitrate, video_codec, audio_codec };
|
return { width, height, resolution, duration, frame_rate, bitrate, video_codec, audio_codec };
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
cleanupTempFile(tmpPath);
|
if (tmpPath) cleanupTempFile(tmpPath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+147
-3
@@ -22,7 +22,10 @@
|
|||||||
// whenever S3_ENDPOINT isn't reachable from this machine —
|
// whenever S3_ENDPOINT isn't reachable from this machine —
|
||||||
// see resolvePublicHost() below)
|
// see resolvePublicHost() below)
|
||||||
|
|
||||||
const { S3Client, DeleteObjectCommand, GetObjectCommand, HeadBucketCommand } = require("@aws-sdk/client-s3");
|
const {
|
||||||
|
S3Client, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, HeadBucketCommand, PutObjectCommand,
|
||||||
|
CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand,
|
||||||
|
} = require("@aws-sdk/client-s3");
|
||||||
const { Upload } = require("@aws-sdk/lib-storage");
|
const { Upload } = require("@aws-sdk/lib-storage");
|
||||||
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
|
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
|
||||||
const crypto = require("crypto");
|
const crypto = require("crypto");
|
||||||
@@ -44,7 +47,14 @@ const credentials = {
|
|||||||
// WHEN_REQUIRED skips validation unless the operation mandates it (GetObject
|
// WHEN_REQUIRED skips validation unless the operation mandates it (GetObject
|
||||||
// never does) — safe here since Garage isn't computing/verifying per AWS spec
|
// never does) — safe here since Garage isn't computing/verifying per AWS spec
|
||||||
// anyway.
|
// anyway.
|
||||||
const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED" };
|
//
|
||||||
|
// requestChecksumCalculation: "WHEN_REQUIRED" — same fix, request side. SDK
|
||||||
|
// v3's default "WHEN_SUPPORTED" makes presignUpload()'s PutObjectCommand carry
|
||||||
|
// a signed x-amz-checksum-crc32 computed over an empty/phantom body (there's
|
||||||
|
// no real body yet at presign time), so Garage rejects the browser's actual
|
||||||
|
// PUT once real bytes arrive with "InvalidDigest ... expected Crc32([0,0,0,0])".
|
||||||
|
// PutObject doesn't require a checksum, so WHEN_REQUIRED just omits it.
|
||||||
|
const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED", requestChecksumCalculation: "WHEN_REQUIRED" };
|
||||||
|
|
||||||
// Internal client — uploads, deletes, direct streams from the server itself.
|
// Internal client — uploads, deletes, direct streams from the server itself.
|
||||||
// Always targets S3_ENDPOINT. When this backend is co-located with Garage,
|
// Always targets S3_ENDPOINT. When this backend is co-located with Garage,
|
||||||
@@ -271,4 +281,138 @@ async function ping() {
|
|||||||
await client.send(new HeadBucketCommand({ Bucket: DEFAULT_BUCKET }));
|
await client.send(new HeadBucketCommand({ Bucket: DEFAULT_BUCKET }));
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { uploadFile, deleteFile, getSignedDownloadUrl, getPublicUrl, getObjectStream, ping };
|
// S3's own hard ceiling for a single (non-multipart) PUT — files above this
|
||||||
|
// need real multipart upload instead (see presignUpload() below).
|
||||||
|
const MULTIPART_THRESHOLD = 5 * 1024 ** 3; // 5GB
|
||||||
|
|
||||||
|
// Fixed part size for multipart uploads — comfortably above S3's 5MB-per-part
|
||||||
|
// minimum, and keeps the part count reasonable even at the largest files this
|
||||||
|
// app expects (15GB / 50MB = 300 parts, well under S3's 10,000-part ceiling).
|
||||||
|
// No adaptive sizing needed at this scale.
|
||||||
|
const PART_SIZE = 50 * 1024 ** 2; // 50MB
|
||||||
|
|
||||||
|
// ─── presignUpload ────────────────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Presigned upload URL(s) for browser-direct uploads — the browser PUTs the
|
||||||
|
// file's bytes straight to storage; this backend never buffers them. Signed
|
||||||
|
// against the same public-facing host as getPublicUrl() (browsers can't
|
||||||
|
// reach S3_ENDPOINT when that's an internal-only address), and deliberately
|
||||||
|
// does NOT pin Content-Type on any signed command — the browser's actual
|
||||||
|
// Content-Type header (from the File object) would have to match whatever
|
||||||
|
// was signed exactly, and leaving it unsigned avoids that fragility.
|
||||||
|
//
|
||||||
|
// input: originalname, ownerType — same as uploadFile(), used to build the
|
||||||
|
// same {prefix}/{uuid}.{ext} key convention.
|
||||||
|
// size — total file size in bytes, decides single-PUT vs multipart.
|
||||||
|
// output: { key, uploadUrl } (size <= 5GB)
|
||||||
|
// or { key, multipart: true, uploadId, partSize, parts } (size > 5GB)
|
||||||
|
// parts: [{ partNumber, uploadUrl }, ...], one presigned UploadPart
|
||||||
|
// URL per part, all generated upfront in this one call.
|
||||||
|
//
|
||||||
|
async function presignUpload(originalname, ownerType = "image", size = 0) {
|
||||||
|
const key = buildKey(originalname, ownerType);
|
||||||
|
const client = getPublicClient();
|
||||||
|
|
||||||
|
if (size <= MULTIPART_THRESHOLD) {
|
||||||
|
// Generous expiry — even a file near the 5GB ceiling can take a long
|
||||||
|
// time to PUT on a slow connection, and the signature must still be
|
||||||
|
// valid when the browser actually gets around to sending it.
|
||||||
|
const uploadUrl = await getSignedUrl(
|
||||||
|
client,
|
||||||
|
new PutObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key }),
|
||||||
|
{ expiresIn: 4 * 60 * 60 } // 4h
|
||||||
|
);
|
||||||
|
return { key, uploadUrl };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Above the single-PUT ceiling — the backend initiates the multipart
|
||||||
|
// session itself with real credentials (a lightweight control-plane call,
|
||||||
|
// no file bytes involved); only the individual part uploads, which do
|
||||||
|
// carry real bytes, get presigned for the browser.
|
||||||
|
const { UploadId: uploadId } = await s3.send(new CreateMultipartUploadCommand({
|
||||||
|
Bucket: DEFAULT_BUCKET,
|
||||||
|
Key: key,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const partCount = Math.ceil(size / PART_SIZE);
|
||||||
|
const parts = [];
|
||||||
|
for (let partNumber = 1; partNumber <= partCount; partNumber++) {
|
||||||
|
const uploadUrl = await getSignedUrl(
|
||||||
|
client,
|
||||||
|
new UploadPartCommand({ Bucket: DEFAULT_BUCKET, Key: key, UploadId: uploadId, PartNumber: partNumber }),
|
||||||
|
{ expiresIn: 24 * 60 * 60 } // 24h — a 15GB upload can genuinely take a while on a slow connection
|
||||||
|
);
|
||||||
|
parts.push({ partNumber, uploadUrl });
|
||||||
|
}
|
||||||
|
|
||||||
|
return { key, multipart: true, uploadId, partSize: PART_SIZE, parts };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── completeMultipartUpload ──────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Finishes a multipart upload once the browser has PUT every part directly
|
||||||
|
// (see presignUpload() above). Parts must carry the ETag each part's own PUT
|
||||||
|
// response returned — order doesn't matter here, they're sorted by
|
||||||
|
// PartNumber before submitting. Real credentials, not presigned: this is a
|
||||||
|
// small control-plane call, no file bytes involved.
|
||||||
|
//
|
||||||
|
// input: key, uploadId, parts — [{ partNumber, etag }, ...]
|
||||||
|
//
|
||||||
|
async function completeMultipartUpload(key, uploadId, parts) {
|
||||||
|
await s3.send(new CompleteMultipartUploadCommand({
|
||||||
|
Bucket: DEFAULT_BUCKET,
|
||||||
|
Key: key,
|
||||||
|
UploadId: uploadId,
|
||||||
|
MultipartUpload: {
|
||||||
|
Parts: parts
|
||||||
|
.slice()
|
||||||
|
.sort((a, b) => a.partNumber - b.partNumber)
|
||||||
|
.map(({ partNumber, etag }) => ({ PartNumber: partNumber, ETag: etag })),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── abortMultipartUpload ──────────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Cancels an in-progress multipart upload (a part permanently failed after
|
||||||
|
// retries, or the browser gave up) so it doesn't linger as orphaned storage
|
||||||
|
// forever. Real credentials, called server-side — this only ever happens on
|
||||||
|
// failure, no need for browser-direct access to it.
|
||||||
|
//
|
||||||
|
async function abortMultipartUpload(key, uploadId) {
|
||||||
|
await s3.send(new AbortMultipartUploadCommand({
|
||||||
|
Bucket: DEFAULT_BUCKET,
|
||||||
|
Key: key,
|
||||||
|
UploadId: uploadId,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── getFileMetadata ──────────────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Reads back what the browser actually uploaded via presignUpload() — used by
|
||||||
|
// the finalize step in place of multer's req.file (size, mimetype), since no
|
||||||
|
// buffer ever passes through this backend to read those from directly. ETag
|
||||||
|
// doubles as the object's checksum: for a single (non-multipart) unencrypted
|
||||||
|
// PutObjectCommand, S3-compatible ETag is exactly the MD5 of the body.
|
||||||
|
//
|
||||||
|
// input: key
|
||||||
|
// output: { size, mimetype, checksum } — checksum is the ETag with its
|
||||||
|
// surrounding quotes stripped, or null if the object has no ETag.
|
||||||
|
//
|
||||||
|
async function getFileMetadata(key) {
|
||||||
|
const result = await s3.send(new HeadObjectCommand({
|
||||||
|
Bucket: DEFAULT_BUCKET,
|
||||||
|
Key: key,
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
size: result.ContentLength ?? null,
|
||||||
|
mimetype: result.ContentType ?? null,
|
||||||
|
checksum: result.ETag ? result.ETag.replace(/"/g, "") : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
uploadFile, deleteFile, getSignedDownloadUrl, getPublicUrl, getObjectStream,
|
||||||
|
presignUpload, completeMultipartUpload, abortMultipartUpload,
|
||||||
|
getFileMetadata, buildPublicUrl, ping,
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user