// services/s3.service.js // // S3-compatible storage service targeting Garage (self-hosted). // Exposes the same interface as chibisafe.service.js: // uploadFile({ buffer, originalname, mimetype, ownerType }) → { url, uuid } // deleteFile(key) // // Required .env vars: // S3_ENDPOINT – address this backend uses for uploads/deletes. // http://127.0.0.1:3900 when co-located with Garage. // Otherwise (backend runs on a different machine than // Garage) point this at a network-reachable address that // terminates on Garage — e.g. the same tunneled domain as // S3_PUBLIC_URL, since garage-anon-proxy re-signs any // non-presigned request with real credentials before // forwarding. Never leave this blank. // S3_REGION – garage // S3_ACCESS_KEY // S3_SECRET_KEY // S3_BUCKET – your-bucket-name // S3_PUBLIC_URL – https://cdn.yourdomain.com (used for browser-facing URLs // whenever S3_ENDPOINT isn't reachable from this machine — // see resolvePublicHost() below) const { S3Client, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, HeadBucketCommand, PutObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, } = require("@aws-sdk/client-s3"); const { Upload } = require("@aws-sdk/lib-storage"); const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); const crypto = require("crypto"); const path = require("path"); // ─── Clients ────────────────────────────────────────────────────────────────── const credentials = { accessKeyId: process.env.S3_ACCESS_KEY, secretAccessKey: process.env.S3_SECRET_KEY, }; // responseChecksumValidation: "WHEN_REQUIRED" — the AWS SDK v3 defaults to // "WHEN_SUPPORTED", which validates any response carrying an x-amz-checksum-* // header. Garage sends x-amz-checksum-crc32 on GetObject but its value doesn't // match what the SDK recomputes over the body, so every direct GetObjectCommand // (e.g. getObjectStream() below, used by the document-to-Markdown converter) // threw "Checksum mismatch" even though the bytes themselves were fine. // WHEN_REQUIRED skips validation unless the operation mandates it (GetObject // never does) — safe here since Garage isn't computing/verifying per AWS spec // anyway. // // requestChecksumCalculation: "WHEN_REQUIRED" — same fix, request side. SDK // v3's default "WHEN_SUPPORTED" makes presignUpload()'s PutObjectCommand carry // a signed x-amz-checksum-crc32 computed over an empty/phantom body (there's // no real body yet at presign time), so Garage rejects the browser's actual // PUT once real bytes arrive with "InvalidDigest ... expected Crc32([0,0,0,0])". // PutObject doesn't require a checksum, so WHEN_REQUIRED just omits it. const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED", requestChecksumCalculation: "WHEN_REQUIRED" }; // Internal client — uploads, deletes, direct streams from the server itself. // Always targets S3_ENDPOINT. When this backend is co-located with Garage, // that's a local address; when it isn't, S3_ENDPOINT must instead be a // network-reachable address that reaches Garage (e.g. the tunneled proxy // domain) — see the .env docs above. Do not assume co-location here. const s3 = new S3Client({ endpoint: process.env.S3_ENDPOINT, region: process.env.S3_REGION || "garage", credentials, forcePathStyle: true, ...CHECKSUM_CONFIG, }); const DEFAULT_BUCKET = process.env.S3_BUCKET; // ─── Public host resolution ─────────────────────────────────────────────────── // // URLs handed to browsers (file_url, presigned GET links) need a host reachable // from wherever the client sits. S3_ENDPOINT (e.g. 127.0.0.1:3900) only works // from the machine running Garage itself; S3_PUBLIC_URL is the externally // reachable address (tunnel/CDN/domain). // // This used to probe S3_ENDPOINT from the server and prefer it when reachable, // but that measures the wrong machine: Garage is always co-located with this // backend (see docker-compose.yml), so the probe was *always* reachable from // here and always resolved to 127.0.0.1 — even for browsers on other machines, // which then failed to connect to it. There is no way for the server to // determine what's reachable from the client by probing itself, so just trust // config: prefer S3_PUBLIC_URL whenever it's set, and only fall back to // S3_ENDPOINT for pure single-machine dev setups with no public URL at all. function resolvePublicHost() { return process.env.S3_PUBLIC_URL || process.env.S3_ENDPOINT || ""; } // Public client — built against whichever host resolvePublicHost() picks. function getPublicClient() { const endpoint = resolvePublicHost(); return new S3Client({ endpoint, region: process.env.S3_REGION || "garage", credentials, forcePathStyle: true, ...CHECKSUM_CONFIG, }); } // ─── Key prefix map ─────────────────────────────────────────────────────────── // // ownerType mirrors what the controller already passes to chibisafe // (image, video, audio, document, avatar, thumbnail). // Maps to folder prefixes inside the bucket. // const PREFIX_MAP = { image: "images", video: "videos", audio: "audios", document: "documents", avatar: "avatars", thumbnail: "thumbnails", badge: "badges", }; function resolvePrefix(ownerType = "image") { return PREFIX_MAP[ownerType] || "others"; } function resolveExtension(originalname = "") { return path.extname(originalname).replace(".", "").toLowerCase() || "bin"; } // ─── Helpers ────────────────────────────────────────────────────────────────── // Builds the S3 object key: {prefix}/{uuid}.{ext} // e.g. images/3f2a1b4c-uuid.jpg function buildKey(originalname, ownerType) { const prefix = resolvePrefix(ownerType); const ext = resolveExtension(originalname); const uuid = crypto.randomUUID(); return `${prefix}/${uuid}.${ext}`; } // Builds the public URL for a stored object. // Garage path-style: {host}/{bucket}/{key} // e.g. https://cdn.yourdomain.com/your-bucket/images/uuid.jpg async function buildPublicUrl(key, bucket = DEFAULT_BUCKET) { const host = (await resolvePublicHost()).replace(/\/$/, ""); return `${host}/${bucket}/${key}`; } // ─── uploadFile ─────────────────────────────────────────────────────────────── // // Matches chibisafe.service.js interface exactly. // input: { buffer, originalname, mimetype, ownerType? } // output: { url, uuid } ← uuid = the S3 key, stored as storage_key in DB // // onProgress?: ({ loaded, total }) => void — real bytes-sent-to-Garage events, // straight from the AWS SDK's own httpUploadProgress, not simulated. A plain // PutObjectCommand (what this used to be) has no progress API at all; Upload // auto-splits into multipart above its ~5MB partSize threshold, so large // files (the case this actually matters for) report genuine incremental // progress per part, while small ones just jump from 0 to 100 immediately. async function uploadFile({ buffer, originalname, mimetype, ownerType = "image", onProgress }) { if (!buffer) { throw Object.assign(new Error("File buffer is required for S3 uploads."), { status: 400 }); } const bucket = DEFAULT_BUCKET; const key = buildKey(originalname, ownerType); const uploader = new Upload({ client: s3, params: { Bucket: bucket, Key: key, Body: buffer, ContentType: mimetype }, }); if (onProgress) { uploader.on("httpUploadProgress", (progress) => { onProgress({ loaded: progress.loaded ?? 0, total: progress.total ?? buffer.length }); }); } await uploader.done(); return { url: await buildPublicUrl(key, bucket), uuid: key, // used as storage_key in DB — mirrors chibi_uuid usage }; } // ─── uploadStream ───────────────────────────────────────────────────────────── // // Same as uploadFile(), but Body is a Node Readable stream instead of a // Buffer — used by assetTranscode.service.js to push a remuxed video back to // storage straight off local disk, without ever holding the whole (possibly // multi-GB) file in this process's memory. Upload (lib-storage) auto-chunks // a stream body into multipart the same way it does a large Buffer. // // input: { stream, originalname, mimetype, ownerType? } // output: { url, uuid } ← uuid = the S3 key, stored as storage_key in DB // async function uploadStream({ stream, originalname, mimetype, ownerType = "video" }) { if (!stream) { throw Object.assign(new Error("A readable stream is required for S3 uploads."), { status: 400 }); } const bucket = DEFAULT_BUCKET; const key = buildKey(originalname, ownerType); const uploader = new Upload({ client: s3, params: { Bucket: bucket, Key: key, Body: stream, ContentType: mimetype }, }); await uploader.done(); return { url: await buildPublicUrl(key, bucket), uuid: key, }; } // ─── deleteFile ─────────────────────────────────────────────────────────────── // // Matches chibisafe.service.js interface. // input: key — the value stored in storage_key column (e.g. "images/uuid.jpg") // async function deleteFile(key) { if (!key) return; await s3.send(new DeleteObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key, })); } // ─── getSignedDownloadUrl ───────────────────────────────────────────────────── // // Generates a short-lived pre-signed GET URL against S3_ENDPOINT (internal — // this backend is always co-located with Garage, see docker-compose.yml). // For server-side reads only (e.g. media.controller.js streamAsset piping // bytes to the browser itself) — never hand this URL to a browser directly. // // Must NOT be signed against S3_PUBLIC_URL when Garage is in use: that host is // fronted by garage-anon-proxy, which re-signs every request itself (header-based // SigV4, real credentials) regardless of any query-string signature already // present. A presigned URL arriving there collides with the proxy's own // signature and Garage rejects the request (400 "Header `x-amz-date` should // be signed"). // // Uses the internal `s3` client (always S3_ENDPOINT) when available; falls back // to getPublicClient() for external S3-compatible services without Garage. // async function getSignedDownloadUrl(key, expiresInSeconds = 3600) { const client = process.env.S3_ENDPOINT ? s3 : getPublicClient(); return getSignedUrl( client, new GetObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key, }), { expiresIn: expiresInSeconds, } ); } // ─── getPublicUrl ───────────────────────────────────────────────────────────── // // Browser-facing pre-signed GET URL, signed against S3_PUBLIC_URL. Safe again // now that garage-anon-proxy detects a valid query-string SigV4 signature and // forwards it unmodified instead of re-signing on top of it (see // isPresignedRequest() in garage-anon-proxy/server.js) — that mismatch used to // produce Garage's 400 "Header `x-amz-date` should be signed". Use this // wherever a URL is handed directly to the browser (e.g. thumbnail previews). // // expiresInSeconds defaults to 4h to outlive the longest cache window a caller // hands this URL out under (client media token TTL — see media.controller.js), // so a cached thumbnail_url never outlives its own signature. // async function getPublicUrl(key, bucket = DEFAULT_BUCKET, expiresInSeconds = 4 * 60 * 60) { const client = getPublicClient(); return getSignedUrl( client, new GetObjectCommand({ Bucket: bucket, Key: key }), { expiresIn: expiresInSeconds } ); } // ─── getObjectStream ──────────────────────────────────────────────────────── // // Fetches an S3 object and returns its readable stream + metadata, for use // in a server-side download proxy (Content-Disposition: attachment). // // input: key — the storage_key (e.g. "images/uuid.jpg") // output: { stream, contentType, contentLength } // async function getObjectStream(key) { if (!key) { throw Object.assign(new Error("Storage key is required."), { status: 400 }); } const result = await s3.send(new GetObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key, })); return { stream: result.Body, // Node.js Readable stream contentType: result.ContentType, contentLength: result.ContentLength, }; } async function ping() { const client = await getPublicClient(); await client.send(new HeadBucketCommand({ Bucket: DEFAULT_BUCKET })); } // S3's own hard ceiling for a single (non-multipart) PUT — files above this // need real multipart upload instead (see presignUpload() below). const MULTIPART_THRESHOLD = 5 * 1024 ** 3; // 5GB // Fixed part size for multipart uploads — comfortably above S3's 5MB-per-part // minimum, and keeps the part count reasonable even at the largest files this // app expects (15GB / 50MB = 300 parts, well under S3's 10,000-part ceiling). // No adaptive sizing needed at this scale. const PART_SIZE = 50 * 1024 ** 2; // 50MB // ─── presignUpload ──────────────────────────────────────────────────────────── // // Presigned upload URL(s) for browser-direct uploads — the browser PUTs the // file's bytes straight to storage; this backend never buffers them. Signed // against the same public-facing host as getPublicUrl() (browsers can't // reach S3_ENDPOINT when that's an internal-only address), and deliberately // does NOT pin Content-Type on any signed command — the browser's actual // Content-Type header (from the File object) would have to match whatever // was signed exactly, and leaving it unsigned avoids that fragility. // // input: originalname, ownerType — same as uploadFile(), used to build the // same {prefix}/{uuid}.{ext} key convention. // size — total file size in bytes, decides single-PUT vs multipart. // output: { key, uploadUrl } (size <= 5GB) // or { key, multipart: true, uploadId, partSize, parts } (size > 5GB) // parts: [{ partNumber, uploadUrl }, ...], one presigned UploadPart // URL per part, all generated upfront in this one call. // async function presignUpload(originalname, ownerType = "image", size = 0) { const key = buildKey(originalname, ownerType); const client = getPublicClient(); if (size <= MULTIPART_THRESHOLD) { // Generous expiry — even a file near the 5GB ceiling can take a long // time to PUT on a slow connection, and the signature must still be // valid when the browser actually gets around to sending it. const uploadUrl = await getSignedUrl( client, new PutObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key }), { expiresIn: 4 * 60 * 60 } // 4h ); return { key, uploadUrl }; } // Above the single-PUT ceiling — the backend initiates the multipart // session itself with real credentials (a lightweight control-plane call, // no file bytes involved); only the individual part uploads, which do // carry real bytes, get presigned for the browser. const { UploadId: uploadId } = await s3.send(new CreateMultipartUploadCommand({ Bucket: DEFAULT_BUCKET, Key: key, })); const partCount = Math.ceil(size / PART_SIZE); const parts = []; for (let partNumber = 1; partNumber <= partCount; partNumber++) { const uploadUrl = await getSignedUrl( client, new UploadPartCommand({ Bucket: DEFAULT_BUCKET, Key: key, UploadId: uploadId, PartNumber: partNumber }), { expiresIn: 24 * 60 * 60 } // 24h — a 15GB upload can genuinely take a while on a slow connection ); parts.push({ partNumber, uploadUrl }); } return { key, multipart: true, uploadId, partSize: PART_SIZE, parts }; } // ─── completeMultipartUpload ────────────────────────────────────────────────── // // Finishes a multipart upload once the browser has PUT every part directly // (see presignUpload() above). Parts must carry the ETag each part's own PUT // response returned — order doesn't matter here, they're sorted by // PartNumber before submitting. Real credentials, not presigned: this is a // small control-plane call, no file bytes involved. // // input: key, uploadId, parts — [{ partNumber, etag }, ...] // async function completeMultipartUpload(key, uploadId, parts) { await s3.send(new CompleteMultipartUploadCommand({ Bucket: DEFAULT_BUCKET, Key: key, UploadId: uploadId, MultipartUpload: { Parts: parts .slice() .sort((a, b) => a.partNumber - b.partNumber) .map(({ partNumber, etag }) => ({ PartNumber: partNumber, ETag: etag })), }, })); } // ─── abortMultipartUpload ────────────────────────────────────────────────────── // // Cancels an in-progress multipart upload (a part permanently failed after // retries, or the browser gave up) so it doesn't linger as orphaned storage // forever. Real credentials, called server-side — this only ever happens on // failure, no need for browser-direct access to it. // async function abortMultipartUpload(key, uploadId) { await s3.send(new AbortMultipartUploadCommand({ Bucket: DEFAULT_BUCKET, Key: key, UploadId: uploadId, })); } // ─── getFileMetadata ────────────────────────────────────────────────────────── // // Reads back what the browser actually uploaded via presignUpload() — used by // the finalize step in place of multer's req.file (size, mimetype), since no // buffer ever passes through this backend to read those from directly. ETag // doubles as the object's checksum: for a single (non-multipart) unencrypted // PutObjectCommand, S3-compatible ETag is exactly the MD5 of the body. // // input: key // output: { size, mimetype, checksum } — checksum is the ETag with its // surrounding quotes stripped, or null if the object has no ETag. // async function getFileMetadata(key) { const result = await s3.send(new HeadObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key, })); return { size: result.ContentLength ?? null, mimetype: result.ContentType ?? null, checksum: result.ETag ? result.ETag.replace(/"/g, "") : null, }; } module.exports = { uploadFile, uploadStream, deleteFile, getSignedDownloadUrl, getPublicUrl, getObjectStream, presignUpload, completeMultipartUpload, abortMultipartUpload, getFileMetadata, buildPublicUrl, ping, };