chore: relocate backend into apps/api ahead of monorepo merge

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-27 00:12:54 +08:00 · 2026-08-24 12:20:40 +08:00
parent af44598df6
commit eaa6d2276a
388 changed files with 0 additions and 13998 deletions
@@ -0,0 +1,450 @@
+// services/s3.service.js
+//
+// S3-compatible storage service targeting Garage (self-hosted).
+// Exposes the same interface as chibisafe.service.js:
+//   uploadFile({ buffer, originalname, mimetype, ownerType }) → { url, uuid }
+//   deleteFile(key)
+//
+// Required .env vars:
+//   S3_ENDPOINT    – address this backend uses for uploads/deletes.
+//                    http://127.0.0.1:3900 when co-located with Garage.
+//                    Otherwise (backend runs on a different machine than
+//                    Garage) point this at a network-reachable address that
+//                    terminates on Garage — e.g. the same tunneled domain as
+//                    S3_PUBLIC_URL, since garage-anon-proxy re-signs any
+//                    non-presigned request with real credentials before
+//                    forwarding. Never leave this blank.
+//   S3_REGION      – garage
+//   S3_ACCESS_KEY
+//   S3_SECRET_KEY
+//   S3_BUCKET      – your-bucket-name
+//   S3_PUBLIC_URL  – https://cdn.yourdomain.com (used for browser-facing URLs
+//                    whenever S3_ENDPOINT isn't reachable from this machine —
+//                    see resolvePublicHost() below)
+
+const {
+  S3Client, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, HeadBucketCommand, PutObjectCommand,
+  CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand,
+} = require("@aws-sdk/client-s3");
+const { Upload } = require("@aws-sdk/lib-storage");
+const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
+const crypto = require("crypto");
+const path = require("path");
+
+// ─── Clients ──────────────────────────────────────────────────────────────────
+
+const credentials = {
+  accessKeyId:     process.env.S3_ACCESS_KEY,
+  secretAccessKey: process.env.S3_SECRET_KEY,
+};
+
+// responseChecksumValidation: "WHEN_REQUIRED" — the AWS SDK v3 defaults to
+// "WHEN_SUPPORTED", which validates any response carrying an x-amz-checksum-*
+// header. Garage sends x-amz-checksum-crc32 on GetObject but its value doesn't
+// match what the SDK recomputes over the body, so every direct GetObjectCommand
+// (e.g. getObjectStream() below, used by the document-to-Markdown converter)
+// threw "Checksum mismatch" even though the bytes themselves were fine.
+// WHEN_REQUIRED skips validation unless the operation mandates it (GetObject
+// never does) — safe here since Garage isn't computing/verifying per AWS spec
+// anyway.
+//
+// requestChecksumCalculation: "WHEN_REQUIRED" — same fix, request side. SDK
+// v3's default "WHEN_SUPPORTED" makes presignUpload()'s PutObjectCommand carry
+// a signed x-amz-checksum-crc32 computed over an empty/phantom body (there's
+// no real body yet at presign time), so Garage rejects the browser's actual
+// PUT once real bytes arrive with "InvalidDigest ... expected Crc32([0,0,0,0])".
+// PutObject doesn't require a checksum, so WHEN_REQUIRED just omits it.
+const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED", requestChecksumCalculation: "WHEN_REQUIRED" };
+
+// Internal client — uploads, deletes, direct streams from the server itself.
+// Always targets S3_ENDPOINT. When this backend is co-located with Garage,
+// that's a local address; when it isn't, S3_ENDPOINT must instead be a
+// network-reachable address that reaches Garage (e.g. the tunneled proxy
+// domain) — see the .env docs above. Do not assume co-location here.
+const s3 = new S3Client({
+  endpoint:       process.env.S3_ENDPOINT,
+  region:         process.env.S3_REGION || "garage",
+  credentials,
+  forcePathStyle: true,
+  ...CHECKSUM_CONFIG,
+});
+
+const DEFAULT_BUCKET = process.env.S3_BUCKET;
+
+// ─── Public host resolution ───────────────────────────────────────────────────
+//
+// URLs handed to browsers (file_url, presigned GET links) need a host reachable
+// from wherever the client sits. S3_ENDPOINT (e.g. 127.0.0.1:3900) only works
+// from the machine running Garage itself; S3_PUBLIC_URL is the externally
+// reachable address (tunnel/CDN/domain).
+//
+// This used to probe S3_ENDPOINT from the server and prefer it when reachable,
+// but that measures the wrong machine: Garage is always co-located with this
+// backend (see docker-compose.yml), so the probe was *always* reachable from
+// here and always resolved to 127.0.0.1 — even for browsers on other machines,
+// which then failed to connect to it. There is no way for the server to
+// determine what's reachable from the client by probing itself, so just trust
+// config: prefer S3_PUBLIC_URL whenever it's set, and only fall back to
+// S3_ENDPOINT for pure single-machine dev setups with no public URL at all.
+function resolvePublicHost() {
+  return process.env.S3_PUBLIC_URL || process.env.S3_ENDPOINT || "";
+}
+
+// Public client — built against whichever host resolvePublicHost() picks.
+function getPublicClient() {
+  const endpoint = resolvePublicHost();
+  return new S3Client({
+    endpoint,
+    region:         process.env.S3_REGION || "garage",
+    credentials,
+    forcePathStyle: true,
+    ...CHECKSUM_CONFIG,
+  });
+}
+
+// ─── Key prefix map ───────────────────────────────────────────────────────────
+//
+// ownerType mirrors what the controller already passes to chibisafe
+// (image, video, audio, document, avatar, thumbnail).
+// Maps to folder prefixes inside the bucket.
+//
+const PREFIX_MAP = {
+  image:     "images",
+  video:     "videos",
+  audio:     "audios",
+  document:  "documents",
+  avatar:    "avatars",
+  thumbnail: "thumbnails",
+  badge:     "badges",
+};
+
+function resolvePrefix(ownerType = "image") {
+  return PREFIX_MAP[ownerType] || "others";
+}
+
+function resolveExtension(originalname = "") {
+  return path.extname(originalname).replace(".", "").toLowerCase() || "bin";
+}
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+// Builds the S3 object key: {prefix}/{uuid}.{ext}
+// e.g. images/3f2a1b4c-uuid.jpg
+function buildKey(originalname, ownerType) {
+  const prefix = resolvePrefix(ownerType);
+  const ext    = resolveExtension(originalname);
+  const uuid   = crypto.randomUUID();
+  return `${prefix}/${uuid}.${ext}`;
+}
+
+// Builds the public URL for a stored object.
+// Garage path-style: {host}/{bucket}/{key}
+// e.g. https://cdn.yourdomain.com/your-bucket/images/uuid.jpg
+async function buildPublicUrl(key, bucket = DEFAULT_BUCKET) {
+  const host = (await resolvePublicHost()).replace(/\/$/, "");
+  return `${host}/${bucket}/${key}`;
+}
+
+// ─── uploadFile ───────────────────────────────────────────────────────────────
+//
+// Matches chibisafe.service.js interface exactly.
+// input:  { buffer, originalname, mimetype, ownerType? }
+// output: { url, uuid }  ← uuid = the S3 key, stored as storage_key in DB
+//
+// onProgress?: ({ loaded, total }) => void — real bytes-sent-to-Garage events,
+// straight from the AWS SDK's own httpUploadProgress, not simulated. A plain
+// PutObjectCommand (what this used to be) has no progress API at all; Upload
+// auto-splits into multipart above its ~5MB partSize threshold, so large
+// files (the case this actually matters for) report genuine incremental
+// progress per part, while small ones just jump from 0 to 100 immediately.
+async function uploadFile({ buffer, originalname, mimetype, ownerType = "image", onProgress }) {
+  if (!buffer) {
+    throw Object.assign(new Error("File buffer is required for S3 uploads."), { status: 400 });
+  }
+
+  const bucket = DEFAULT_BUCKET;
+  const key    = buildKey(originalname, ownerType);
+
+  const uploader = new Upload({
+    client: s3,
+    params: { Bucket: bucket, Key: key, Body: buffer, ContentType: mimetype },
+  });
+
+  if (onProgress) {
+    uploader.on("httpUploadProgress", (progress) => {
+      onProgress({ loaded: progress.loaded ?? 0, total: progress.total ?? buffer.length });
+    });
+  }
+
+  await uploader.done();
+
+  return {
+    url:  await buildPublicUrl(key, bucket),
+    uuid: key, // used as storage_key in DB — mirrors chibi_uuid usage
+  };
+}
+
+// ─── uploadStream ─────────────────────────────────────────────────────────────
+//
+// Same as uploadFile(), but Body is a Node Readable stream instead of a
+// Buffer — used by assetTranscode.service.js to push a remuxed video back to
+// storage straight off local disk, without ever holding the whole (possibly
+// multi-GB) file in this process's memory. Upload (lib-storage) auto-chunks
+// a stream body into multipart the same way it does a large Buffer.
+//
+// input:  { stream, originalname, mimetype, ownerType? }
+// output: { url, uuid }  ← uuid = the S3 key, stored as storage_key in DB
+//
+async function uploadStream({ stream, originalname, mimetype, ownerType = "video" }) {
+  if (!stream) {
+    throw Object.assign(new Error("A readable stream is required for S3 uploads."), { status: 400 });
+  }
+
+  const bucket = DEFAULT_BUCKET;
+  const key    = buildKey(originalname, ownerType);
+
+  const uploader = new Upload({
+    client: s3,
+    params: { Bucket: bucket, Key: key, Body: stream, ContentType: mimetype },
+  });
+
+  await uploader.done();
+
+  return {
+    url:  await buildPublicUrl(key, bucket),
+    uuid: key,
+  };
+}
+
+// ─── deleteFile ───────────────────────────────────────────────────────────────
+//
+// Matches chibisafe.service.js interface.
+// input: key — the value stored in storage_key column (e.g. "images/uuid.jpg")
+//
+async function deleteFile(key) {
+  if (!key) return;
+  await s3.send(new DeleteObjectCommand({
+    Bucket: DEFAULT_BUCKET,
+    Key:    key,
+  }));
+}
+
+// ─── getSignedDownloadUrl ─────────────────────────────────────────────────────
+//
+// Generates a short-lived pre-signed GET URL against S3_ENDPOINT (internal —
+// this backend is always co-located with Garage, see docker-compose.yml).
+// For server-side reads only (e.g. media.controller.js streamAsset piping
+// bytes to the browser itself) — never hand this URL to a browser directly.
+//
+// Must NOT be signed against S3_PUBLIC_URL when Garage is in use: that host is
+// fronted by garage-anon-proxy, which re-signs every request itself (header-based
+// SigV4, real credentials) regardless of any query-string signature already
+// present. A presigned URL arriving there collides with the proxy's own
+// signature and Garage rejects the request (400 "Header `x-amz-date` should
+// be signed").
+//
+// Uses the internal `s3` client (always S3_ENDPOINT) when available; falls back
+// to getPublicClient() for external S3-compatible services without Garage.
+//
+async function getSignedDownloadUrl(key, expiresInSeconds = 3600) {
+  const client = process.env.S3_ENDPOINT ? s3 : getPublicClient();
+
+  return getSignedUrl(
+    client,
+    new GetObjectCommand({
+      Bucket: DEFAULT_BUCKET,
+      Key: key,
+    }),
+    {
+      expiresIn: expiresInSeconds,
+    }
+  );
+}
+
+// ─── getPublicUrl ─────────────────────────────────────────────────────────────
+//
+// Browser-facing pre-signed GET URL, signed against S3_PUBLIC_URL. Safe again
+// now that garage-anon-proxy detects a valid query-string SigV4 signature and
+// forwards it unmodified instead of re-signing on top of it (see
+// isPresignedRequest() in garage-anon-proxy/server.js) — that mismatch used to
+// produce Garage's 400 "Header `x-amz-date` should be signed". Use this
+// wherever a URL is handed directly to the browser (e.g. thumbnail previews).
+//
+// expiresInSeconds defaults to 4h to outlive the longest cache window a caller
+// hands this URL out under (client media token TTL — see media.controller.js),
+// so a cached thumbnail_url never outlives its own signature.
+//
+async function getPublicUrl(key, bucket = DEFAULT_BUCKET, expiresInSeconds = 4 * 60 * 60) {
+  const client = getPublicClient();
+  return getSignedUrl(
+    client,
+    new GetObjectCommand({ Bucket: bucket, Key: key }),
+    { expiresIn: expiresInSeconds }
+  );
+}
+
+// ─── getObjectStream ────────────────────────────────────────────────────────
+//
+// Fetches an S3 object and returns its readable stream + metadata, for use
+// in a server-side download proxy (Content-Disposition: attachment).
+//
+// input:  key — the storage_key (e.g. "images/uuid.jpg")
+// output: { stream, contentType, contentLength }
+//
+async function getObjectStream(key) {
+    if (!key) {
+        throw Object.assign(new Error("Storage key is required."), { status: 400 });
+    }
+ 
+    const result = await s3.send(new GetObjectCommand({
+        Bucket: DEFAULT_BUCKET,
+        Key:    key,
+    }));
+ 
+    return {
+        stream:        result.Body,          // Node.js Readable stream
+        contentType:   result.ContentType,
+        contentLength: result.ContentLength,
+    };
+}
+
+async function ping() {
+  const client = await getPublicClient();
+  await client.send(new HeadBucketCommand({ Bucket: DEFAULT_BUCKET }));
+}
+
+// S3's own hard ceiling for a single (non-multipart) PUT — files above this
+// need real multipart upload instead (see presignUpload() below).
+const MULTIPART_THRESHOLD = 5 * 1024 ** 3; // 5GB
+
+// Fixed part size for multipart uploads — comfortably above S3's 5MB-per-part
+// minimum, and keeps the part count reasonable even at the largest files this
+// app expects (15GB / 50MB = 300 parts, well under S3's 10,000-part ceiling).
+// No adaptive sizing needed at this scale.
+const PART_SIZE = 50 * 1024 ** 2; // 50MB
+
+// ─── presignUpload ────────────────────────────────────────────────────────────
+//
+// Presigned upload URL(s) for browser-direct uploads — the browser PUTs the
+// file's bytes straight to storage; this backend never buffers them. Signed
+// against the same public-facing host as getPublicUrl() (browsers can't
+// reach S3_ENDPOINT when that's an internal-only address), and deliberately
+// does NOT pin Content-Type on any signed command — the browser's actual
+// Content-Type header (from the File object) would have to match whatever
+// was signed exactly, and leaving it unsigned avoids that fragility.
+//
+// input:  originalname, ownerType — same as uploadFile(), used to build the
+//         same {prefix}/{uuid}.{ext} key convention.
+//         size — total file size in bytes, decides single-PUT vs multipart.
+// output: { key, uploadUrl }                                    (size <= 5GB)
+//      or { key, multipart: true, uploadId, partSize, parts }    (size > 5GB)
+//         parts: [{ partNumber, uploadUrl }, ...], one presigned UploadPart
+//         URL per part, all generated upfront in this one call.
+//
+async function presignUpload(originalname, ownerType = "image", size = 0) {
+  const key = buildKey(originalname, ownerType);
+  const client = getPublicClient();
+
+  if (size <= MULTIPART_THRESHOLD) {
+    // Generous expiry — even a file near the 5GB ceiling can take a long
+    // time to PUT on a slow connection, and the signature must still be
+    // valid when the browser actually gets around to sending it.
+    const uploadUrl = await getSignedUrl(
+      client,
+      new PutObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key }),
+      { expiresIn: 4 * 60 * 60 } // 4h
+    );
+    return { key, uploadUrl };
+  }
+
+  // Above the single-PUT ceiling — the backend initiates the multipart
+  // session itself with real credentials (a lightweight control-plane call,
+  // no file bytes involved); only the individual part uploads, which do
+  // carry real bytes, get presigned for the browser.
+  const { UploadId: uploadId } = await s3.send(new CreateMultipartUploadCommand({
+    Bucket: DEFAULT_BUCKET,
+    Key:    key,
+  }));
+
+  const partCount = Math.ceil(size / PART_SIZE);
+  const parts = [];
+  for (let partNumber = 1; partNumber <= partCount; partNumber++) {
+    const uploadUrl = await getSignedUrl(
+      client,
+      new UploadPartCommand({ Bucket: DEFAULT_BUCKET, Key: key, UploadId: uploadId, PartNumber: partNumber }),
+      { expiresIn: 24 * 60 * 60 } // 24h — a 15GB upload can genuinely take a while on a slow connection
+    );
+    parts.push({ partNumber, uploadUrl });
+  }
+
+  return { key, multipart: true, uploadId, partSize: PART_SIZE, parts };
+}
+
+// ─── completeMultipartUpload ──────────────────────────────────────────────────
+//
+// Finishes a multipart upload once the browser has PUT every part directly
+// (see presignUpload() above). Parts must carry the ETag each part's own PUT
+// response returned — order doesn't matter here, they're sorted by
+// PartNumber before submitting. Real credentials, not presigned: this is a
+// small control-plane call, no file bytes involved.
+//
+// input: key, uploadId, parts — [{ partNumber, etag }, ...]
+//
+async function completeMultipartUpload(key, uploadId, parts) {
+  await s3.send(new CompleteMultipartUploadCommand({
+    Bucket:   DEFAULT_BUCKET,
+    Key:      key,
+    UploadId: uploadId,
+    MultipartUpload: {
+      Parts: parts
+        .slice()
+        .sort((a, b) => a.partNumber - b.partNumber)
+        .map(({ partNumber, etag }) => ({ PartNumber: partNumber, ETag: etag })),
+    },
+  }));
+}
+
+// ─── abortMultipartUpload ──────────────────────────────────────────────────────
+//
+// Cancels an in-progress multipart upload (a part permanently failed after
+// retries, or the browser gave up) so it doesn't linger as orphaned storage
+// forever. Real credentials, called server-side — this only ever happens on
+// failure, no need for browser-direct access to it.
+//
+async function abortMultipartUpload(key, uploadId) {
+  await s3.send(new AbortMultipartUploadCommand({
+    Bucket:   DEFAULT_BUCKET,
+    Key:      key,
+    UploadId: uploadId,
+  }));
+}
+
+// ─── getFileMetadata ──────────────────────────────────────────────────────────
+//
+// Reads back what the browser actually uploaded via presignUpload() — used by
+// the finalize step in place of multer's req.file (size, mimetype), since no
+// buffer ever passes through this backend to read those from directly. ETag
+// doubles as the object's checksum: for a single (non-multipart) unencrypted
+// PutObjectCommand, S3-compatible ETag is exactly the MD5 of the body.
+//
+// input:  key
+// output: { size, mimetype, checksum } — checksum is the ETag with its
+//         surrounding quotes stripped, or null if the object has no ETag.
+//
+async function getFileMetadata(key) {
+  const result = await s3.send(new HeadObjectCommand({
+    Bucket: DEFAULT_BUCKET,
+    Key:    key,
+  }));
+  return {
+    size:     result.ContentLength ?? null,
+    mimetype: result.ContentType ?? null,
+    checksum: result.ETag ? result.ETag.replace(/"/g, "") : null,
+  };
+}
+
+module.exports = {
+  uploadFile, uploadStream, deleteFile, getSignedDownloadUrl, getPublicUrl, getObjectStream,
+  presignUpload, completeMultipartUpload, abortMultipartUpload,
+  getFileMetadata, buildPublicUrl, ping,
+};