diff --git a/controllers/admin/assets.controller.js b/controllers/admin/assets.controller.js index bcbcc3e..102750c 100644 --- a/controllers/admin/assets.controller.js +++ b/controllers/admin/assets.controller.js @@ -320,96 +320,68 @@ exports.getAsset = async (req, res) => { // ─── UPLOAD (shared core) ────────────────────────────────────────────────────── // // ┌─────────────────────────────────────────────────────────────────────────┐ -// │ TRANSACTION STRATEGY │ +// │ PRESIGNED-UPLOAD STRATEGY │ // │ │ -// │ Phase 1 — SLOW WORK (outside transaction): │ -// │ • Input validation │ -// │ • ffprobe metadata extraction │ -// │ • Provider upload (chibi or s3) → track for rollback │ -// │ • Thumbnail upload → track for rollback │ -// │ │ -// │ Phase 2 — FAST WORK (transaction open milliseconds only): │ +// │ The browser already PUT the file's bytes straight to storage via a │ +// │ presigned URL (see presignAssetUpload below) — this backend never │ +// │ buffers or even touches them (this is what removes the old 500MB │ +// │ multer-memoryStorage RAM ceiling entirely, regardless of file size). │ +// │ Finalizing an asset from an already-uploaded object is just: │ +// │ • HeadObjectCommand → real file_size/mime_type/checksum (=ETag) │ +// │ • ffprobe by URL → video/audio metadata only, no download │ // │ • BEGIN → Asset.create() → COMMIT │ // │ │ -// │ On any error: │ -// │ • ROLLBACK transaction (if opened) │ -// │ • rollbackUploads([{ key, provider }]) to clean orphans │ +// │ On any error: rollbackUploads([{ key, provider }]) deletes the │ +// │ already-uploaded object(s) — same cleanup as before, just always │ +// │ covering both file + thumbnail upfront, since both already exist in │ +// │ storage by the time this runs (the browser uploaded them first). │ // └─────────────────────────────────────────────────────────────────────────┘ // -// Shared by the single-file POST / and the multi-file POST /batch routes. // Thumbnails are optional for both video and audio — a video/audio asset can // land with thumbnail_url null and pick one up later via the existing // "thumbnail-only" path in updateAsset(). // -async function createAssetFromUpload({ file, thumbFile, body, user, uploadId }) { - const uploadedFiles = []; // [{ key, provider }] +async function finalizeAssetFromStorage({ storage_key, thumbnail_storage_key, original_name, body, user }) { + const { + display_name, + description, + is_public = false, + storage_provider = "s3", + storage_bucket, + createdBy, + } = body; + + const uploadedFiles = [{ key: storage_key, provider: storage_provider }]; + if (thumbnail_storage_key) uploadedFiles.push({ key: thumbnail_storage_key, provider: storage_provider }); try { - if (!file) throw Object.assign(new Error("No file uploaded."), { status: 400 }); - - const { - display_name, - description, - is_public = false, - // TEMPORARY: defaulting to "s3" instead of "chibisafe" — zrok tunnel is - // dropping Chibisafe uploads (502s). Revert this default once off zrok/VPS migration. - storage_provider = "s3", - storage_bucket, - storage_key, - createdBy, - } = body; - + if (!storage_key) throw Object.assign(new Error("storage_key is required."), { status: 400 }); if (!createdBy) throw Object.assign(new Error("createdBy is required."), { status: 400 }); - const mime_type = file.mimetype; - const file_type = resolveFileType(mime_type); - const extension = resolveExtension(file.originalname); - const checksum = file.buffer ? resolveChecksum(file.buffer) : null; - const usesProvider = ["chibisafe", "s3"].includes(storage_provider); - - if (usesProvider && !file.buffer) { - throw Object.assign(new Error("File buffer is required. Ensure multer uses memoryStorage."), { status: 400 }); + const svc = getProvider(storage_provider); + if (!svc || !svc.getFileMetadata) { + throw Object.assign(new Error("Presigned uploads are only supported for S3 storage."), { status: 400 }); } - // ── Phase 1b: Upload main file ──────────────────────────────────────────── - - let file_url = null; - let storage_key_resolved = null; - - if (usesProvider) { - const svc = getProvider(storage_provider); - // Real Express -> Garage progress, main file only (not the thumbnail — - // it's small enough that tracking it wouldn't add anything useful). - // Relayed live to the browser over SSE; see uploadProgress.service.js. - const onProgress = (storage_provider === "s3" && uploadId) - ? ({ loaded, total }) => uploadProgress.publish(uploadId, { - phase: "storing", - loaded, - total, - pct: total ? Math.round((loaded / total) * 100) : 0, - }) - : undefined; - - const result = await svc.uploadFile({ - buffer: file.buffer, - originalname: file.originalname, - mimetype: mime_type, - ownerType: file_type, - onProgress, - }); - file_url = result.url; - storage_key_resolved = result.uuid; - uploadedFiles.push({ key: storage_key_resolved, provider: storage_provider }); - - } else { - file_url = storage_provider === "local" - ? `/uploads/${file.filename}` - : body.file_url; - - if (!file_url) throw Object.assign(new Error("file_url is required for non-local storage."), { status: 400 }); + let meta; + try { + meta = await svc.getFileMetadata(storage_key); + } catch { + throw Object.assign(new Error("Uploaded file not found in storage — the upload may have failed or expired."), { status: 400 }); } - // ── Phase 1c: ffprobe + thumbnail ───────────────────────────────────────── + // S3's own Content-Type is authoritative when present, but browsers only + // send one automatically when the File object's own .type is non-empty — + // fall back to whatever the client reported at presign time, and finally + // to a generic default, rather than ever letting a NOT NULL column see + // null here (mime_type also drives file_type below, so a null here would + // misclassify the asset entirely, not just leave a field blank). + const mime_type = meta.mimetype || body.mimetype || "application/octet-stream"; + const file_type = resolveFileType(mime_type); + const extension = resolveExtension(original_name || storage_key); + const file_url = await svc.buildPublicUrl(storage_key); + + // ── ffprobe (video/audio only) ──────────────────────────────────────────── let width = null, height = null, resolution = null; let duration = null, frame_rate = null, bitrate = null; @@ -417,34 +389,18 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId }) let thumbnail_url = null; if (file_type === "video" || file_type === "audio") { - const meta = await extractVideoMeta({ buffer: file.buffer, extension: extension || (file_type === "video" ? "mp4" : "mp3") }); - width = meta.width; - height = meta.height; - resolution = meta.resolution; - duration = meta.duration; - frame_rate = meta.frame_rate; - bitrate = meta.bitrate; - video_codec = meta.video_codec; - audio_codec = meta.audio_codec; + const probeUrl = await svc.getSignedDownloadUrl(storage_key); + const videoMeta = await extractVideoMeta({ url: probeUrl }); + width = videoMeta.width; + height = videoMeta.height; + resolution = videoMeta.resolution; + duration = videoMeta.duration; + frame_rate = videoMeta.frame_rate; + bitrate = videoMeta.bitrate; + video_codec = videoMeta.video_codec; + audio_codec = videoMeta.audio_codec; - if (usesProvider && thumbFile) { - if (!thumbFile.buffer) { - await rollbackUploads(uploadedFiles); - throw Object.assign(new Error("Thumbnail buffer is required."), { status: 400 }); - } - const baseName = file.originalname.replace(/\.[^.]+$/, ""); - const svc = getProvider(storage_provider); - const thumbResult = await svc.uploadFile({ - buffer: thumbFile.buffer, - originalname: `thumb_${baseName}.${resolveExtension(thumbFile.originalname) || "jpg"}`, - mimetype: thumbFile.mimetype, - ownerType: "thumbnail", - }); - thumbnail_url = thumbResult.url; - uploadedFiles.push({ key: thumbResult.uuid, provider: storage_provider }); - } else if (!usesProvider) { - thumbnail_url = thumbFile?.filename ? `/uploads/${thumbFile.filename}` : null; - } + if (thumbnail_storage_key) thumbnail_url = await svc.buildPublicUrl(thumbnail_storage_key); } else { const parsedWidth = body.width ? parseInt(body.width) : null; @@ -454,18 +410,18 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId }) resolution = resolveResolution(parsedWidth, parsedHeight); } - // ── Phase 2: DB insert ──────────────────────────────────────────────────── + // ── DB insert ────────────────────────────────────────────────────────────── const t = await sequelize.transaction(); try { const asset = await Asset.create({ - original_name: file.originalname, - display_name: display_name || file.originalname, + original_name: original_name || storage_key, + display_name: display_name || original_name || storage_key, file_url, - file_size: file.size, + file_size: meta.size, mime_type, extension, - checksum, + checksum: meta.checksum, file_type, width, height, @@ -479,7 +435,7 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId }) description, storage_provider, storage_bucket: storage_bucket || (storage_provider === "s3" ? process.env.S3_BUCKET : null) || null, - storage_key: storage_key_resolved || storage_key || file.filename || null, + storage_key, is_public, createdBy, }, { transaction: t }); @@ -500,90 +456,117 @@ async function createAssetFromUpload({ file, thumbFile, body, user, uploadId }) } } -exports.uploadAsset = async (req, res) => { - const { uploadId } = req.body; +// ─── PRESIGN UPLOAD ───────────────────────────────────────────────────────── +// +// Mints a short-lived presigned PUT URL so the browser can upload the file's +// bytes directly to storage — this backend never buffers them. Called once +// for the main file, and once more for a thumbnail if the admin picked one +// (see s3.service.js#presignUpload for the key-naming convention). +// +exports.presignAssetUpload = async (req, res) => { try { - const file = req.files?.file?.[0]; - const thumbFile = req.files?.thumbnail?.[0]; + const { filename, mimetype, file_type, size = 0, storage_provider = "s3" } = req.body; + if (!filename) return R.error(res, "filename is required.", 400); - const asset = await createAssetFromUpload({ file, thumbFile, body: req.body, user: req.user, uploadId }); + const svc = getProvider(storage_provider); + if (!svc || !svc.presignUpload) { + return R.error(res, "Presigned uploads are only supported for S3 storage.", 400); + } + + const ownerType = file_type || resolveFileType(mimetype || "") || "document"; + // Result is either { key, uploadUrl } or, above the single-PUT ceiling, + // { key, multipart: true, uploadId, partSize, parts } — see + // s3.service.js#presignUpload. The client branches on `multipart`. + const presigned = await svc.presignUpload(filename, ownerType, Number(size) || 0); + return R.success(res, "Presigned URL generated.", presigned); + + } catch (err) { + console.error("[ASSET][PRESIGN]", err); + return R.error(res, "Could not generate upload URL.", 500); + } +}; + +// ─── COMPLETE / ABORT MULTIPART ───────────────────────────────────────────── +// +// Only used above presignAssetUpload's single-PUT ceiling (see +// s3.service.js's MULTIPART_THRESHOLD) — the browser PUTs every part +// directly, then calls complete-multipart with the ETags each part's PUT +// response returned. abort-multipart is the failure-path cleanup (a part +// exhausted its retries, or the admin cancelled) so an abandoned multipart +// upload doesn't linger as orphaned storage forever. +// +exports.completeMultipartAssetUpload = async (req, res) => { + try { + const { storage_key, uploadId, parts, storage_provider = "s3" } = req.body; + if (!storage_key || !uploadId || !Array.isArray(parts) || !parts.length) { + return R.error(res, "storage_key, uploadId, and parts are required.", 400); + } + + const svc = getProvider(storage_provider); + if (!svc || !svc.completeMultipartUpload) { + return R.error(res, "Multipart uploads are only supported for S3 storage.", 400); + } + + await svc.completeMultipartUpload(storage_key, uploadId, parts); + return R.success(res, "Multipart upload completed.", {}); + + } catch (err) { + console.error("[ASSET][COMPLETE MULTIPART]", err); + return R.error(res, "Could not complete multipart upload.", 500); + } +}; + +exports.abortMultipartAssetUpload = async (req, res) => { + try { + const { storage_key, uploadId, storage_provider = "s3" } = req.body; + if (!storage_key || !uploadId) return R.error(res, "storage_key and uploadId are required.", 400); + + const svc = getProvider(storage_provider); + if (svc?.abortMultipartUpload) { + try { + await svc.abortMultipartUpload(storage_key, uploadId); + } catch (err) { + // Best-effort, same tolerance as rollbackUploads() — an already-gone + // or already-completed upload isn't worth failing the request over. + console.error(`[ASSET][ABORT MULTIPART] Failed to abort "${storage_key}":`, err.message); + } + } + return R.success(res, "Multipart upload aborted.", {}); + + } catch (err) { + console.error("[ASSET][ABORT MULTIPART]", err); + return R.error(res, "Could not abort multipart upload.", 500); + } +}; + +// ─── UPLOAD (finalize) ────────────────────────────────────────────────────── +// +// Called once the browser's direct-to-storage PUT(s) have completed. Body is +// plain JSON — no file bytes here, just the storage key(s) presignAssetUpload +// handed back plus asset metadata. Also the endpoint the bulk queue +// (UploadQueueContext.jsx) calls once per file, reusing this single-asset +// path instead of a separate batch endpoint. +// +exports.uploadAsset = async (req, res) => { + try { + const { storage_key, thumbnail_storage_key, original_name } = req.body; + const asset = await finalizeAssetFromStorage({ storage_key, thumbnail_storage_key, original_name, body: req.body, user: req.user }); invalidateListCache(); - if (uploadId) uploadProgress.complete(uploadId, { phase: "done", pct: 100 }); return R.success(res, "Asset uploaded.", { data: asset }, 201); } catch (err) { console.error("[ASSET][UPLOAD]", err); - if (uploadId) uploadProgress.complete(uploadId, { phase: "error" }); if (err.status) return R.error(res, err.message, err.status, { detail: err.chibiBody }); return R.error(res, "Internal server error.", 500); } }; -// ─── UPLOAD PROGRESS (SSE) ────────────────────────────────────────────────── -// -// Client opens this before POSTing the file, correlated by a client-generated -// uploadId sent as a form field on the upload request itself. Streams the -// real Express -> Garage httpUploadProgress events from s3.service.js's -// Upload — not a simulated or estimated number. -// -exports.streamUploadProgress = (req, res) => { - uploadProgress.subscribe(req.params.uploadId, res); -}; - // Same broadcaster, reused as-is for document-conversion job progress — it's // just a generic string-keyed SSE channel, nothing upload-specific about it. exports.streamConvertProgress = (req, res) => { uploadProgress.subscribe(req.params.jobId, res); }; -// ─── UPLOAD (batch) ───────────────────────────────────────────────────────── -// -// Accepts multiple files under the "files" field in one multipart request, -// all sharing the same is_public / storage_provider / createdBy. Each file -// is uploaded independently — one failing (bad codec, DB constraint, etc.) -// does not roll back the others. display_name defaults to the filename -// (minus extension) since there's no per-file metadata step in bulk mode. -// Videos land without a thumbnail (see createAssetFromUpload) — add one -// later via the existing "replace thumbnail" path on PATCH /:assetId. -// -exports.uploadAssetsBatch = async (req, res) => { - const files = req.files ?? []; - if (!files.length) return R.error(res, "No files uploaded.", 400); - - // TEMPORARY: defaulting to "s3" instead of "chibisafe" — zrok tunnel is - // dropping Chibisafe uploads (502s). Revert this default once off zrok/VPS migration. - const { display_name, description, is_public = false, storage_provider = "s3", createdBy } = req.body; - if (!createdBy) return R.error(res, "createdBy is required.", 400); - - const results = []; - - for (const file of files) { - try { - const baseName = file.originalname.replace(/\.[^.]+$/, ""); - const asset = await createAssetFromUpload({ - file, - thumbFile: null, - body: { - display_name: files.length === 1 ? (display_name || baseName) : baseName, - description, - is_public, - storage_provider, - createdBy, - }, - user: req.user, - }); - results.push({ originalname: file.originalname, success: true, data: asset }); - } catch (err) { - console.error("[ASSET][UPLOAD BATCH]", file.originalname, err.stack || err); - results.push({ originalname: file.originalname, success: false, message: err.status ? err.message : "Internal server error." }); - } - } - - invalidateListCache(); - const createdCount = results.filter((r) => r.success).length; - return R.success(res, `${createdCount} of ${files.length} asset(s) uploaded.`, { results }, 201); -}; - // ─── CONVERT TO MARKDOWN ──────────────────────────────────────────────────── // // PDF/PPTX -> Markdown, text only (see services/documentConversion.service.js diff --git a/routes/admin/assets.routes.js b/routes/admin/assets.routes.js index ae4c788..93ea955 100644 --- a/routes/admin/assets.routes.js +++ b/routes/admin/assets.routes.js @@ -11,13 +11,20 @@ router.get('/field-values', controller.getAssetFieldValues); router.delete('/bulk', sensitiveOpsLimiter, controller.archiveAssets); router.delete('/bulk/permanent', sensitiveOpsLimiter, controller.permanentlyDeleteAssets); router.patch('/bulk-restore', controller.restoreAssets); -router.get('/upload-progress/:uploadId', controller.streamUploadProgress); router.get('/convert-progress/:jobId', controller.streamConvertProgress); // ─── Collection ─────────────────────────────────────────────────────────────── router.get('/', controller.getAssets); -router.post('/', handleUpload(upload.fields([{ name: 'file', maxCount: 1 }, { name: 'thumbnail', maxCount: 1 }])), controller.uploadAsset); -router.post('/batch', handleUpload(upload.array('files', 20)), controller.uploadAssetsBatch); +// Presigned-upload flow: the browser PUTs file bytes straight to storage, this +// backend never buffers them (see assets.controller.js). presign mints the +// upload URL(s); the POST / body that follows is plain JSON (no multer here). +router.post('/presign', controller.presignAssetUpload); +// Above presign's single-PUT ceiling only (see s3.service.js's +// MULTIPART_THRESHOLD) — the browser PUTs every part directly, then calls +// complete/abort here. Plain JSON, no file bytes. +router.post('/complete-multipart', controller.completeMultipartAssetUpload); +router.post('/abort-multipart', controller.abortMultipartAssetUpload); +router.post('/', controller.uploadAsset); // ─── Dynamic routes last ────────────────────────────────────────────────────── router.get('/:assetId', controller.getAsset); diff --git a/services/ffprobe.service.js b/services/ffprobe.service.js index d4dafe9..a4377a0 100644 --- a/services/ffprobe.service.js +++ b/services/ffprobe.service.js @@ -73,15 +73,43 @@ function probeFile(filePath) { }); } +// ffprobe/libavformat accepts http(s):// URLs directly — used for presigned +// -upload assets, where no buffer or local temp file exists at all (the +// browser PUT the bytes straight to storage, this backend never touched +// them). No default network timeout applies to a remote input the way it +// would to a local file, so this races the probe against an explicit one to +// avoid hanging the finalize request on a slow/stuck remote read. +function probeUrl(url, timeoutMs = 30_000) { + return Promise.race([ + new Promise((resolve, reject) => { + ffmpeg.ffprobe(url, (err, metadata) => { + if (err) return reject(err); + resolve(metadata); + }); + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error(`ffprobe timed out after ${timeoutMs}ms`)), timeoutMs) + ), + ]); +} + // ─── Public API ─────────────────────────────────────────────────────────────── /** - * Extract video metadata from a Buffer. + * Extract video metadata from either a Buffer or a remote URL. * Thumbnail is NOT generated here — the client uploads it as a separate file. * * @param {object} opts - * @param {Buffer} opts.buffer raw video bytes (multer memoryStorage) - * @param {string} opts.extension file extension without dot, e.g. "mp4" + * @param {Buffer} [opts.buffer] raw video bytes (multer memoryStorage) — + * written to a temp file and probed there. + * @param {string} [opts.extension] file extension without dot, e.g. "mp4" — + * only used with `buffer`. + * @param {string} [opts.url] a presigned GET URL (or any ffprobe/ + * libavformat-reachable address) to probe + * directly, with no local buffer or temp + * file at all — used for presigned-upload + * assets. Takes precedence over `buffer` + * when both are given. * * @returns {Promise} * @@ -95,11 +123,11 @@ function probeFile(filePath) { * @property {string|null} video_codec "H.264", "H.265", … * @property {string|null} audio_codec "AAC", "MP3", … */ -async function extractVideoMeta({ buffer, extension }) { - const tmpPath = writeTempFile(buffer, extension || "mp4"); +async function extractVideoMeta({ buffer, extension, url }) { + const tmpPath = url ? null : writeTempFile(buffer, extension || "mp4"); try { - const raw = await probeFile(tmpPath); + const raw = await (tmpPath ? probeFile(tmpPath) : probeUrl(url)); const videoStream = raw.streams?.find((s) => s.codec_type === "video") || {}; const audioStream = raw.streams?.find((s) => s.codec_type === "audio") || {}; @@ -135,7 +163,7 @@ async function extractVideoMeta({ buffer, extension }) { return { width, height, resolution, duration, frame_rate, bitrate, video_codec, audio_codec }; } finally { - cleanupTempFile(tmpPath); + if (tmpPath) cleanupTempFile(tmpPath); } } diff --git a/services/s3.service.js b/services/s3.service.js index 022650c..a071c91 100644 --- a/services/s3.service.js +++ b/services/s3.service.js @@ -22,7 +22,10 @@ // whenever S3_ENDPOINT isn't reachable from this machine — // see resolvePublicHost() below) -const { S3Client, DeleteObjectCommand, GetObjectCommand, HeadBucketCommand } = require("@aws-sdk/client-s3"); +const { + S3Client, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, HeadBucketCommand, PutObjectCommand, + CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, +} = require("@aws-sdk/client-s3"); const { Upload } = require("@aws-sdk/lib-storage"); const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); const crypto = require("crypto"); @@ -44,7 +47,14 @@ const credentials = { // WHEN_REQUIRED skips validation unless the operation mandates it (GetObject // never does) — safe here since Garage isn't computing/verifying per AWS spec // anyway. -const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED" }; +// +// requestChecksumCalculation: "WHEN_REQUIRED" — same fix, request side. SDK +// v3's default "WHEN_SUPPORTED" makes presignUpload()'s PutObjectCommand carry +// a signed x-amz-checksum-crc32 computed over an empty/phantom body (there's +// no real body yet at presign time), so Garage rejects the browser's actual +// PUT once real bytes arrive with "InvalidDigest ... expected Crc32([0,0,0,0])". +// PutObject doesn't require a checksum, so WHEN_REQUIRED just omits it. +const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED", requestChecksumCalculation: "WHEN_REQUIRED" }; // Internal client — uploads, deletes, direct streams from the server itself. // Always targets S3_ENDPOINT. When this backend is co-located with Garage, @@ -271,4 +281,138 @@ async function ping() { await client.send(new HeadBucketCommand({ Bucket: DEFAULT_BUCKET })); } -module.exports = { uploadFile, deleteFile, getSignedDownloadUrl, getPublicUrl, getObjectStream, ping }; \ No newline at end of file +// S3's own hard ceiling for a single (non-multipart) PUT — files above this +// need real multipart upload instead (see presignUpload() below). +const MULTIPART_THRESHOLD = 5 * 1024 ** 3; // 5GB + +// Fixed part size for multipart uploads — comfortably above S3's 5MB-per-part +// minimum, and keeps the part count reasonable even at the largest files this +// app expects (15GB / 50MB = 300 parts, well under S3's 10,000-part ceiling). +// No adaptive sizing needed at this scale. +const PART_SIZE = 50 * 1024 ** 2; // 50MB + +// ─── presignUpload ──────────────────────────────────────────────────────────── +// +// Presigned upload URL(s) for browser-direct uploads — the browser PUTs the +// file's bytes straight to storage; this backend never buffers them. Signed +// against the same public-facing host as getPublicUrl() (browsers can't +// reach S3_ENDPOINT when that's an internal-only address), and deliberately +// does NOT pin Content-Type on any signed command — the browser's actual +// Content-Type header (from the File object) would have to match whatever +// was signed exactly, and leaving it unsigned avoids that fragility. +// +// input: originalname, ownerType — same as uploadFile(), used to build the +// same {prefix}/{uuid}.{ext} key convention. +// size — total file size in bytes, decides single-PUT vs multipart. +// output: { key, uploadUrl } (size <= 5GB) +// or { key, multipart: true, uploadId, partSize, parts } (size > 5GB) +// parts: [{ partNumber, uploadUrl }, ...], one presigned UploadPart +// URL per part, all generated upfront in this one call. +// +async function presignUpload(originalname, ownerType = "image", size = 0) { + const key = buildKey(originalname, ownerType); + const client = getPublicClient(); + + if (size <= MULTIPART_THRESHOLD) { + // Generous expiry — even a file near the 5GB ceiling can take a long + // time to PUT on a slow connection, and the signature must still be + // valid when the browser actually gets around to sending it. + const uploadUrl = await getSignedUrl( + client, + new PutObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key }), + { expiresIn: 4 * 60 * 60 } // 4h + ); + return { key, uploadUrl }; + } + + // Above the single-PUT ceiling — the backend initiates the multipart + // session itself with real credentials (a lightweight control-plane call, + // no file bytes involved); only the individual part uploads, which do + // carry real bytes, get presigned for the browser. + const { UploadId: uploadId } = await s3.send(new CreateMultipartUploadCommand({ + Bucket: DEFAULT_BUCKET, + Key: key, + })); + + const partCount = Math.ceil(size / PART_SIZE); + const parts = []; + for (let partNumber = 1; partNumber <= partCount; partNumber++) { + const uploadUrl = await getSignedUrl( + client, + new UploadPartCommand({ Bucket: DEFAULT_BUCKET, Key: key, UploadId: uploadId, PartNumber: partNumber }), + { expiresIn: 24 * 60 * 60 } // 24h — a 15GB upload can genuinely take a while on a slow connection + ); + parts.push({ partNumber, uploadUrl }); + } + + return { key, multipart: true, uploadId, partSize: PART_SIZE, parts }; +} + +// ─── completeMultipartUpload ────────────────────────────────────────────────── +// +// Finishes a multipart upload once the browser has PUT every part directly +// (see presignUpload() above). Parts must carry the ETag each part's own PUT +// response returned — order doesn't matter here, they're sorted by +// PartNumber before submitting. Real credentials, not presigned: this is a +// small control-plane call, no file bytes involved. +// +// input: key, uploadId, parts — [{ partNumber, etag }, ...] +// +async function completeMultipartUpload(key, uploadId, parts) { + await s3.send(new CompleteMultipartUploadCommand({ + Bucket: DEFAULT_BUCKET, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts + .slice() + .sort((a, b) => a.partNumber - b.partNumber) + .map(({ partNumber, etag }) => ({ PartNumber: partNumber, ETag: etag })), + }, + })); +} + +// ─── abortMultipartUpload ────────────────────────────────────────────────────── +// +// Cancels an in-progress multipart upload (a part permanently failed after +// retries, or the browser gave up) so it doesn't linger as orphaned storage +// forever. Real credentials, called server-side — this only ever happens on +// failure, no need for browser-direct access to it. +// +async function abortMultipartUpload(key, uploadId) { + await s3.send(new AbortMultipartUploadCommand({ + Bucket: DEFAULT_BUCKET, + Key: key, + UploadId: uploadId, + })); +} + +// ─── getFileMetadata ────────────────────────────────────────────────────────── +// +// Reads back what the browser actually uploaded via presignUpload() — used by +// the finalize step in place of multer's req.file (size, mimetype), since no +// buffer ever passes through this backend to read those from directly. ETag +// doubles as the object's checksum: for a single (non-multipart) unencrypted +// PutObjectCommand, S3-compatible ETag is exactly the MD5 of the body. +// +// input: key +// output: { size, mimetype, checksum } — checksum is the ETag with its +// surrounding quotes stripped, or null if the object has no ETag. +// +async function getFileMetadata(key) { + const result = await s3.send(new HeadObjectCommand({ + Bucket: DEFAULT_BUCKET, + Key: key, + })); + return { + size: result.ContentLength ?? null, + mimetype: result.ContentType ?? null, + checksum: result.ETag ? result.ETag.replace(/"/g, "") : null, + }; +} + +module.exports = { + uploadFile, deleteFile, getSignedDownloadUrl, getPublicUrl, getObjectStream, + presignUpload, completeMultipartUpload, abortMultipartUpload, + getFileMetadata, buildPublicUrl, ping, +}; \ No newline at end of file