kennethscoffee/starr-philproperties
1
0
Fork 0
mirror of https://github.com/rgrgogu/new_starr.git synced 2026-09-27 00:12:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

assets: go 15GB limit

Browse Source 
Signed-off-by: Kenneth Obsequio <k80308392@gmail.com>
This commit is contained in:
kennethobsequio
2026-08-01 12:07:39 +08:00
parent aad298bc54
commit 2bff5c030a
4 changed files with 347 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controllers/admin/assets.controller.js
+155 -172
View File
@@ -320,96 +320,68 @@ exports.getAsset = async (req, res) => {
// ─── UPLOAD (shared core) ──────────────────────────────────────────────────────
//
// ┌─────────────────────────────────────────────────────────────────────────┐
// │ TRANSACTION STRATEGY │
// │ PRESIGNED-UPLOAD STRATEGY │
// │ │
// │ Phase 1 — SLOW WORK (outside transaction): │
// │ • Input validation │
// │ • ffprobe metadata extraction │
// │ • Provider upload (chibi or s3) → track for rollback │
// │ • Thumbnail upload → track for rollback │
// │ │
// │ Phase 2 — FAST WORK (transaction open milliseconds only): │
// │ The browser already PUT the file's bytes straight to storage via a │
// │ presigned URL (see presignAssetUpload below) — this backend never │
// │ buffers or even touches them (this is what removes the old 500MB │
// │ multer-memoryStorage RAM ceiling entirely, regardless of file size). │
// │ Finalizing an asset from an already-uploaded object is just: │
// │ • HeadObjectCommand → real file_size/mime_type/checksum (=ETag) │
// │ • ffprobe by URL → video/audio metadata only, no download │
// │ • BEGIN → Asset.create() → COMMIT │
// │ │
// │ On any error: │
// │ • ROLLBACK transaction (if opened) │
// │ • rollbackUploads([{ key, provider }]) to clean orphans │
// │ On any error: rollbackUploads([{ key, provider }]) deletes the │
// │ already-uploaded object(s) — same cleanup as before, just always │
// │ covering both file + thumbnail upfront, since both already exist in │
// │ storage by the time this runs (the browser uploaded them first). │
// └─────────────────────────────────────────────────────────────────────────┘
//
// Shared by the single-file POST / and the multi-file POST /batch routes.
// Thumbnails are optional for both video and audio — a video/audio asset can
// land with thumbnail_url null and pick one up later via the existing
// "thumbnail-only" path in updateAsset().
//
async function createAssetFromUpload({ file, thumbFile, body, user, uploadId }) {
const uploadedFiles = []; // [{ key, provider }]
async function finalizeAssetFromStorage({ storage_key, thumbnail_storage_key, original_name, body, user }) {
const {
display_name,
description,
is_public = false,
storage_provider = "s3",
storage_bucket,
createdBy,
} = body;
const uploadedFiles = [{ key: storage_key, provider: storage_provider }];
if (thumbnail_storage_key) uploadedFiles.push({ key: thumbnail_storage_key, provider: storage_provider });
try {
if (!file) throw Object.assign(new Error("No file uploaded."), { status: 400 });
const {
display_name,
description,
is_public = false,
// TEMPORARY: defaulting to "s3" instead of "chibisafe" — zrok tunnel is
// dropping Chibisafe uploads (502s). Revert this default once off zrok/VPS migration.
storage_provider = "s3",
storage_bucket,
storage_key,
createdBy,
} = body;
if (!storage_key) throw Object.assign(new Error("storage_key is required."), { status: 400 });
if (!createdBy) throw Object.assign(new Error("createdBy is required."), { status: 400 });
const mime_type = file.mimetype;
const file_type = resolveFileType(mime_type);
const extension = resolveExtension(file.originalname);
const checksum = file.buffer ? resolveChecksum(file.buffer) : null;
const usesProvider = ["chibisafe", "s3"].includes(storage_provider);
if (usesProvider && !file.buffer) {
throw Object.assign(new Error("File buffer is required. Ensure multer uses memoryStorage."), { status: 400 });
const svc = getProvider(storage_provider);
if (!svc || !svc.getFileMetadata) {
throw Object.assign(new Error("Presigned uploads are only supported for S3 storage."), { status: 400 });
}
// ── Phase 1b: Upload main file ────────────────────────────────────────────
let file_url = null;
let storage_key_resolved = null;
if (usesProvider) {
const svc = getProvider(storage_provider);
// Real Express -> Garage progress, main file only (not the thumbnail —
// it's small enough that tracking it wouldn't add anything useful).
// Relayed live to the browser over SSE; see uploadProgress.service.js.
const onProgress = (storage_provider === "s3" && uploadId)
? ({ loaded, total }) => uploadProgress.publish(uploadId, {
phase: "storing",
loaded,
total,
pct: total ? Math.round((loaded / total) * 100) : 0,
})
: undefined;
const result = await svc.uploadFile({
buffer: file.buffer,
originalname: file.originalname,
mimetype: mime_type,
ownerType: file_type,
onProgress,
});
file_url = result.url;
storage_key_resolved = result.uuid;
uploadedFiles.push({ key: storage_key_resolved, provider: storage_provider });
} else {
file_url = storage_provider === "local"
? `/uploads/${file.filename}`
: body.file_url;
if (!file_url) throw Object.assign(new Error("file_url is required for non-local storage."), { status: 400 });
let meta;
try {
meta = await svc.getFileMetadata(storage_key);
} catch {
throw Object.assign(new Error("Uploaded file not found in storage — the upload may have failed or expired."), { status: 400 });
}
// ── Phase 1c: ffprobe + thumbnail ─────────────────────────────────────────
// S3's own Content-Type is authoritative when present, but browsers only
// send one automatically when the File object's own .type is non-empty —
// fall back to whatever the client reported at presign time, and finally
// to a generic default, rather than ever letting a NOT NULL column see
// null here (mime_type also drives file_type below, so a null here would
// misclassify the asset entirely, not just leave a field blank).
const mime_type = meta.mimetype || body.mimetype || "application/octet-stream";
const file_type = resolveFileType(mime_type);
const extension = resolveExtension(original_name || storage_key);
const file_url = await svc.buildPublicUrl(storage_key);
// ── ffprobe (video/audio only) ────────────────────────────────────────────
let width = null, height = null, resolution = null;
let duration = null, frame_rate = null, bitrate = null;
@@ -417,34 +389,18 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId })
let thumbnail_url = null;
if (file_type === "video" || file_type === "audio") {
const meta = await extractVideoMeta({ buffer: file.buffer, extension: extension || (file_type === "video" ? "mp4" : "mp3") });
width = meta.width;
height = meta.height;
resolution = meta.resolution;
duration = meta.duration;
frame_rate = meta.frame_rate;
bitrate = meta.bitrate;
video_codec = meta.video_codec;
audio_codec = meta.audio_codec;
const probeUrl = await svc.getSignedDownloadUrl(storage_key);
const videoMeta = await extractVideoMeta({ url: probeUrl });
width = videoMeta.width;
height = videoMeta.height;
resolution = videoMeta.resolution;
duration = videoMeta.duration;
frame_rate = videoMeta.frame_rate;
bitrate = videoMeta.bitrate;
video_codec = videoMeta.video_codec;
audio_codec = videoMeta.audio_codec;
if (usesProvider && thumbFile) {
if (!thumbFile.buffer) {
await rollbackUploads(uploadedFiles);
throw Object.assign(new Error("Thumbnail buffer is required."), { status: 400 });
}
const baseName = file.originalname.replace(/\.[^.]+$/, "");
const svc = getProvider(storage_provider);
const thumbResult = await svc.uploadFile({
buffer: thumbFile.buffer,
originalname: `thumb_${baseName}.${resolveExtension(thumbFile.originalname) || "jpg"}`,
mimetype: thumbFile.mimetype,
ownerType: "thumbnail",
});
thumbnail_url = thumbResult.url;
uploadedFiles.push({ key: thumbResult.uuid, provider: storage_provider });
} else if (!usesProvider) {
thumbnail_url = thumbFile?.filename ? `/uploads/${thumbFile.filename}` : null;
}
if (thumbnail_storage_key) thumbnail_url = await svc.buildPublicUrl(thumbnail_storage_key);
} else {
const parsedWidth = body.width ? parseInt(body.width) : null;
@@ -454,18 +410,18 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId })
resolution = resolveResolution(parsedWidth, parsedHeight);
}
// ── Phase 2: DB insert ────────────────────────────────────────────────────
// ── DB insert ──────────────────────────────────────────────────────────────
const t = await sequelize.transaction();
try {
const asset = await Asset.create({
original_name: file.originalname,
display_name: display_name || file.originalname,
original_name: original_name || storage_key,
display_name: display_name || original_name || storage_key,
file_url,
file_size: file.size,
file_size: meta.size,
mime_type,
extension,
checksum,
checksum: meta.checksum,
file_type,
width,
height,
@@ -479,7 +435,7 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId })
description,
storage_provider,
storage_bucket: storage_bucket || (storage_provider === "s3" ? process.env.S3_BUCKET : null) || null,
storage_key: storage_key_resolved || storage_key || file.filename || null,
storage_key,
is_public,
createdBy,
}, { transaction: t });
@@ -500,90 +456,117 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId })
}
}
exports.uploadAsset = async (req, res) => {
const { uploadId } = req.body;
// ─── PRESIGN UPLOAD ─────────────────────────────────────────────────────────
//
// Mints a short-lived presigned PUT URL so the browser can upload the file's
// bytes directly to storage — this backend never buffers them. Called once
// for the main file, and once more for a thumbnail if the admin picked one
// (see s3.service.js#presignUpload for the key-naming convention).
//
exports.presignAssetUpload = async (req, res) => {
try {
const file = req.files?.file?.[0];
const thumbFile = req.files?.thumbnail?.[0];
const { filename, mimetype, file_type, size = 0, storage_provider = "s3" } = req.body;
if (!filename) return R.error(res, "filename is required.", 400);
const asset = await createAssetFromUpload({ file, thumbFile, body: req.body, user: req.user, uploadId });
const svc = getProvider(storage_provider);
if (!svc || !svc.presignUpload) {
return R.error(res, "Presigned uploads are only supported for S3 storage.", 400);
}
const ownerType = file_type || resolveFileType(mimetype || "") || "document";
// Result is either { key, uploadUrl } or, above the single-PUT ceiling,
// { key, multipart: true, uploadId, partSize, parts } — see
// s3.service.js#presignUpload. The client branches on `multipart`.
const presigned = await svc.presignUpload(filename, ownerType, Number(size) || 0);
return R.success(res, "Presigned URL generated.", presigned);
} catch (err) {
console.error("[ASSET][PRESIGN]", err);
return R.error(res, "Could not generate upload URL.", 500);
}
};
// ─── COMPLETE / ABORT MULTIPART ─────────────────────────────────────────────
//
// Only used above presignAssetUpload's single-PUT ceiling (see
// s3.service.js's MULTIPART_THRESHOLD) — the browser PUTs every part
// directly, then calls complete-multipart with the ETags each part's PUT
// response returned. abort-multipart is the failure-path cleanup (a part
// exhausted its retries, or the admin cancelled) so an abandoned multipart
// upload doesn't linger as orphaned storage forever.
//
exports.completeMultipartAssetUpload = async (req, res) => {
try {
const { storage_key, uploadId, parts, storage_provider = "s3" } = req.body;
if (!storage_key || !uploadId || !Array.isArray(parts) || !parts.length) {
return R.error(res, "storage_key, uploadId, and parts are required.", 400);
}
const svc = getProvider(storage_provider);
if (!svc || !svc.completeMultipartUpload) {
return R.error(res, "Multipart uploads are only supported for S3 storage.", 400);
}
await svc.completeMultipartUpload(storage_key, uploadId, parts);
return R.success(res, "Multipart upload completed.", {});
} catch (err) {
console.error("[ASSET][COMPLETE MULTIPART]", err);
return R.error(res, "Could not complete multipart upload.", 500);
}
};
exports.abortMultipartAssetUpload = async (req, res) => {
try {
const { storage_key, uploadId, storage_provider = "s3" } = req.body;
if (!storage_key || !uploadId) return R.error(res, "storage_key and uploadId are required.", 400);
const svc = getProvider(storage_provider);
if (svc?.abortMultipartUpload) {
try {
await svc.abortMultipartUpload(storage_key, uploadId);
} catch (err) {
// Best-effort, same tolerance as rollbackUploads() — an already-gone
// or already-completed upload isn't worth failing the request over.
console.error(`[ASSET][ABORT MULTIPART] Failed to abort "${storage_key}":`, err.message);
}
}
return R.success(res, "Multipart upload aborted.", {});
} catch (err) {
console.error("[ASSET][ABORT MULTIPART]", err);
return R.error(res, "Could not abort multipart upload.", 500);
}
};
// ─── UPLOAD (finalize) ──────────────────────────────────────────────────────
//
// Called once the browser's direct-to-storage PUT(s) have completed. Body is
// plain JSON — no file bytes here, just the storage key(s) presignAssetUpload
// handed back plus asset metadata. Also the endpoint the bulk queue
// (UploadQueueContext.jsx) calls once per file, reusing this single-asset
// path instead of a separate batch endpoint.
//
exports.uploadAsset = async (req, res) => {
try {
const { storage_key, thumbnail_storage_key, original_name } = req.body;
const asset = await finalizeAssetFromStorage({ storage_key, thumbnail_storage_key, original_name, body: req.body, user: req.user });
invalidateListCache();
if (uploadId) uploadProgress.complete(uploadId, { phase: "done", pct: 100 });
return R.success(res, "Asset uploaded.", { data: asset }, 201);
} catch (err) {
console.error("[ASSET][UPLOAD]", err);
if (uploadId) uploadProgress.complete(uploadId, { phase: "error" });
if (err.status) return R.error(res, err.message, err.status, { detail: err.chibiBody });
return R.error(res, "Internal server error.", 500);
}
};
// ─── UPLOAD PROGRESS (SSE) ──────────────────────────────────────────────────
//
// Client opens this before POSTing the file, correlated by a client-generated
// uploadId sent as a form field on the upload request itself. Streams the
// real Express -> Garage httpUploadProgress events from s3.service.js's
// Upload — not a simulated or estimated number.
//
exports.streamUploadProgress = (req, res) => {
uploadProgress.subscribe(req.params.uploadId, res);
};
// Same broadcaster, reused as-is for document-conversion job progress — it's
// just a generic string-keyed SSE channel, nothing upload-specific about it.
exports.streamConvertProgress = (req, res) => {
uploadProgress.subscribe(req.params.jobId, res);
};
// ─── UPLOAD (batch) ─────────────────────────────────────────────────────────
//
// Accepts multiple files under the "files" field in one multipart request,
// all sharing the same is_public / storage_provider / createdBy. Each file
// is uploaded independently — one failing (bad codec, DB constraint, etc.)
// does not roll back the others. display_name defaults to the filename
// (minus extension) since there's no per-file metadata step in bulk mode.
// Videos land without a thumbnail (see createAssetFromUpload) — add one
// later via the existing "replace thumbnail" path on PATCH /:assetId.
//
exports.uploadAssetsBatch = async (req, res) => {
const files = req.files ?? [];
if (!files.length) return R.error(res, "No files uploaded.", 400);
// TEMPORARY: defaulting to "s3" instead of "chibisafe" — zrok tunnel is
// dropping Chibisafe uploads (502s). Revert this default once off zrok/VPS migration.
const { display_name, description, is_public = false, storage_provider = "s3", createdBy } = req.body;
if (!createdBy) return R.error(res, "createdBy is required.", 400);
const results = [];
for (const file of files) {
try {
const baseName = file.originalname.replace(/\.[^.]+$/, "");
const asset = await createAssetFromUpload({
file,
thumbFile: null,
body: {
display_name: files.length === 1 ? (display_name || baseName) : baseName,
description,
is_public,
storage_provider,
createdBy,
},
user: req.user,
});
results.push({ originalname: file.originalname, success: true, data: asset });
} catch (err) {
console.error("[ASSET][UPLOAD BATCH]", file.originalname, err.stack || err);
results.push({ originalname: file.originalname, success: false, message: err.status ? err.message : "Internal server error." });
}
}
invalidateListCache();
const createdCount = results.filter((r) => r.success).length;
return R.success(res, `${createdCount} of ${files.length} asset(s) uploaded.`, { results }, 201);
};
// ─── CONVERT TO MARKDOWN ────────────────────────────────────────────────────
//
// PDF/PPTX -> Markdown, text only (see services/documentConversion.service.js