starr-philproperties/services/s3.service.js

// services/s3.service.js
//
// S3-compatible storage service targeting Garage (self-hosted).
// Exposes the same interface as chibisafe.service.js:
//   uploadFile({ buffer, originalname, mimetype, ownerType }) → { url, uuid }
//   deleteFile(key)
//
// Required .env vars:
//   S3_ENDPOINT    – address this backend uses for uploads/deletes.
//                    http://127.0.0.1:3900 when co-located with Garage.
//                    Otherwise (backend runs on a different machine than
//                    Garage) point this at a network-reachable address that
//                    terminates on Garage — e.g. the same tunneled domain as
//                    S3_PUBLIC_URL, since garage-anon-proxy re-signs any
//                    non-presigned request with real credentials before
//                    forwarding. Never leave this blank.
//   S3_REGION      – garage
//   S3_ACCESS_KEY
//   S3_SECRET_KEY
//   S3_BUCKET      – your-bucket-name
//   S3_PUBLIC_URL  – https://cdn.yourdomain.com (used for browser-facing URLs
//                    whenever S3_ENDPOINT isn't reachable from this machine —
//                    see resolvePublicHost() below)

const {
  S3Client, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, HeadBucketCommand, PutObjectCommand,
  CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand,
} = require("@aws-sdk/client-s3");
const { Upload } = require("@aws-sdk/lib-storage");
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
const crypto = require("crypto");
const path = require("path");

// ─── Clients ──────────────────────────────────────────────────────────────────

const credentials = {
  accessKeyId:     process.env.S3_ACCESS_KEY,
  secretAccessKey: process.env.S3_SECRET_KEY,
};

// responseChecksumValidation: "WHEN_REQUIRED" — the AWS SDK v3 defaults to
// "WHEN_SUPPORTED", which validates any response carrying an x-amz-checksum-*
// header. Garage sends x-amz-checksum-crc32 on GetObject but its value doesn't
// match what the SDK recomputes over the body, so every direct GetObjectCommand
// (e.g. getObjectStream() below, used by the document-to-Markdown converter)
// threw "Checksum mismatch" even though the bytes themselves were fine.
// WHEN_REQUIRED skips validation unless the operation mandates it (GetObject
// never does) — safe here since Garage isn't computing/verifying per AWS spec
// anyway.
//
// requestChecksumCalculation: "WHEN_REQUIRED" — same fix, request side. SDK
// v3's default "WHEN_SUPPORTED" makes presignUpload()'s PutObjectCommand carry
// a signed x-amz-checksum-crc32 computed over an empty/phantom body (there's
// no real body yet at presign time), so Garage rejects the browser's actual
// PUT once real bytes arrive with "InvalidDigest ... expected Crc32([0,0,0,0])".
// PutObject doesn't require a checksum, so WHEN_REQUIRED just omits it.
const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED", requestChecksumCalculation: "WHEN_REQUIRED" };

// Internal client — uploads, deletes, direct streams from the server itself.
// Always targets S3_ENDPOINT. When this backend is co-located with Garage,
// that's a local address; when it isn't, S3_ENDPOINT must instead be a
// network-reachable address that reaches Garage (e.g. the tunneled proxy
// domain) — see the .env docs above. Do not assume co-location here.
const s3 = new S3Client({
  endpoint:       process.env.S3_ENDPOINT,
  region:         process.env.S3_REGION || "garage",
  credentials,
  forcePathStyle: true,
  ...CHECKSUM_CONFIG,
});

const DEFAULT_BUCKET = process.env.S3_BUCKET;

// ─── Public host resolution ───────────────────────────────────────────────────
//
// URLs handed to browsers (file_url, presigned GET links) need a host reachable
// from wherever the client sits. S3_ENDPOINT (e.g. 127.0.0.1:3900) only works
// from the machine running Garage itself; S3_PUBLIC_URL is the externally
// reachable address (tunnel/CDN/domain).
//
// This used to probe S3_ENDPOINT from the server and prefer it when reachable,
// but that measures the wrong machine: Garage is always co-located with this
// backend (see docker-compose.yml), so the probe was *always* reachable from
// here and always resolved to 127.0.0.1 — even for browsers on other machines,
// which then failed to connect to it. There is no way for the server to
// determine what's reachable from the client by probing itself, so just trust
// config: prefer S3_PUBLIC_URL whenever it's set, and only fall back to
// S3_ENDPOINT for pure single-machine dev setups with no public URL at all.
function resolvePublicHost() {
  return process.env.S3_PUBLIC_URL || process.env.S3_ENDPOINT || "";
}

// Public client — built against whichever host resolvePublicHost() picks.
function getPublicClient() {
  const endpoint = resolvePublicHost();
  return new S3Client({
    endpoint,
    region:         process.env.S3_REGION || "garage",
    credentials,
    forcePathStyle: true,
    ...CHECKSUM_CONFIG,
  });
}

// ─── Key prefix map ───────────────────────────────────────────────────────────
//
// ownerType mirrors what the controller already passes to chibisafe
// (image, video, audio, document, avatar, thumbnail).
// Maps to folder prefixes inside the bucket.
//
const PREFIX_MAP = {
  image:     "images",
  video:     "videos",
  audio:     "audios",
  document:  "documents",
  avatar:    "avatars",
  thumbnail: "thumbnails",
  badge:     "badges",
};

function resolvePrefix(ownerType = "image") {
  return PREFIX_MAP[ownerType] || "others";
}

function resolveExtension(originalname = "") {
  return path.extname(originalname).replace(".", "").toLowerCase() || "bin";
}

// ─── Helpers ──────────────────────────────────────────────────────────────────

// Builds the S3 object key: {prefix}/{uuid}.{ext}
// e.g. images/3f2a1b4c-uuid.jpg
function buildKey(originalname, ownerType) {
  const prefix = resolvePrefix(ownerType);
  const ext    = resolveExtension(originalname);
  const uuid   = crypto.randomUUID();
  return `${prefix}/${uuid}.${ext}`;
}

// Builds the public URL for a stored object.
// Garage path-style: {host}/{bucket}/{key}
// e.g. https://cdn.yourdomain.com/your-bucket/images/uuid.jpg
async function buildPublicUrl(key, bucket = DEFAULT_BUCKET) {
  const host = (await resolvePublicHost()).replace(/\/$/, "");
  return `${host}/${bucket}/${key}`;
}

// ─── uploadFile ───────────────────────────────────────────────────────────────
//
// Matches chibisafe.service.js interface exactly.
// input:  { buffer, originalname, mimetype, ownerType? }
// output: { url, uuid }  ← uuid = the S3 key, stored as storage_key in DB
//
// onProgress?: ({ loaded, total }) => void — real bytes-sent-to-Garage events,
// straight from the AWS SDK's own httpUploadProgress, not simulated. A plain
// PutObjectCommand (what this used to be) has no progress API at all; Upload
// auto-splits into multipart above its ~5MB partSize threshold, so large
// files (the case this actually matters for) report genuine incremental
// progress per part, while small ones just jump from 0 to 100 immediately.
async function uploadFile({ buffer, originalname, mimetype, ownerType = "image", onProgress }) {
  if (!buffer) {
    throw Object.assign(new Error("File buffer is required for S3 uploads."), { status: 400 });
  }

  const bucket = DEFAULT_BUCKET;
  const key    = buildKey(originalname, ownerType);

  const uploader = new Upload({
    client: s3,
    params: { Bucket: bucket, Key: key, Body: buffer, ContentType: mimetype },
  });

  if (onProgress) {
    uploader.on("httpUploadProgress", (progress) => {
      onProgress({ loaded: progress.loaded ?? 0, total: progress.total ?? buffer.length });
    });
  }

  await uploader.done();

  return {
    url:  await buildPublicUrl(key, bucket),
    uuid: key, // used as storage_key in DB — mirrors chibi_uuid usage
  };
}

// ─── deleteFile ───────────────────────────────────────────────────────────────
//
// Matches chibisafe.service.js interface.
// input: key — the value stored in storage_key column (e.g. "images/uuid.jpg")
//
async function deleteFile(key) {
  if (!key) return;
  await s3.send(new DeleteObjectCommand({
    Bucket: DEFAULT_BUCKET,
    Key:    key,
  }));
}

// ─── getSignedDownloadUrl ─────────────────────────────────────────────────────
//
// Generates a short-lived pre-signed GET URL against S3_ENDPOINT (internal —
// this backend is always co-located with Garage, see docker-compose.yml).
// For server-side reads only (e.g. media.controller.js streamAsset piping
// bytes to the browser itself) — never hand this URL to a browser directly.
//
// Must NOT be signed against S3_PUBLIC_URL when Garage is in use: that host is
// fronted by garage-anon-proxy, which re-signs every request itself (header-based
// SigV4, real credentials) regardless of any query-string signature already
// present. A presigned URL arriving there collides with the proxy's own
// signature and Garage rejects the request (400 "Header `x-amz-date` should
// be signed").
//
// Uses the internal `s3` client (always S3_ENDPOINT) when available; falls back
// to getPublicClient() for external S3-compatible services without Garage.
//
async function getSignedDownloadUrl(key, expiresInSeconds = 3600) {
  const client = process.env.S3_ENDPOINT ? s3 : getPublicClient();

  return getSignedUrl(
    client,
    new GetObjectCommand({
      Bucket: DEFAULT_BUCKET,
      Key: key,
    }),
    {
      expiresIn: expiresInSeconds,
    }
  );
}

// ─── getPublicUrl ─────────────────────────────────────────────────────────────
//
// Browser-facing pre-signed GET URL, signed against S3_PUBLIC_URL. Safe again
// now that garage-anon-proxy detects a valid query-string SigV4 signature and
// forwards it unmodified instead of re-signing on top of it (see
// isPresignedRequest() in garage-anon-proxy/server.js) — that mismatch used to
// produce Garage's 400 "Header `x-amz-date` should be signed". Use this
// wherever a URL is handed directly to the browser (e.g. thumbnail previews).
//
// expiresInSeconds defaults to 4h to outlive the longest cache window a caller
// hands this URL out under (client media token TTL — see media.controller.js),
// so a cached thumbnail_url never outlives its own signature.
//
async function getPublicUrl(key, bucket = DEFAULT_BUCKET, expiresInSeconds = 4 * 60 * 60) {
  const client = getPublicClient();
  return getSignedUrl(
    client,
    new GetObjectCommand({ Bucket: bucket, Key: key }),
    { expiresIn: expiresInSeconds }
  );
}

// ─── getObjectStream ────────────────────────────────────────────────────────
//
// Fetches an S3 object and returns its readable stream + metadata, for use
// in a server-side download proxy (Content-Disposition: attachment).
//
// input:  key — the storage_key (e.g. "images/uuid.jpg")
// output: { stream, contentType, contentLength }
//
async function getObjectStream(key) {
    if (!key) {
        throw Object.assign(new Error("Storage key is required."), { status: 400 });
    }
 
    const result = await s3.send(new GetObjectCommand({
        Bucket: DEFAULT_BUCKET,
        Key:    key,
    }));
 
    return {
        stream:        result.Body,          // Node.js Readable stream
        contentType:   result.ContentType,
        contentLength: result.ContentLength,
    };
}

async function ping() {
  const client = await getPublicClient();
  await client.send(new HeadBucketCommand({ Bucket: DEFAULT_BUCKET }));
}

// S3's own hard ceiling for a single (non-multipart) PUT — files above this
// need real multipart upload instead (see presignUpload() below).
const MULTIPART_THRESHOLD = 5 * 1024 ** 3; // 5GB

// Fixed part size for multipart uploads — comfortably above S3's 5MB-per-part
// minimum, and keeps the part count reasonable even at the largest files this
// app expects (15GB / 50MB = 300 parts, well under S3's 10,000-part ceiling).
// No adaptive sizing needed at this scale.
const PART_SIZE = 50 * 1024 ** 2; // 50MB

// ─── presignUpload ────────────────────────────────────────────────────────────
//
// Presigned upload URL(s) for browser-direct uploads — the browser PUTs the
// file's bytes straight to storage; this backend never buffers them. Signed
// against the same public-facing host as getPublicUrl() (browsers can't
// reach S3_ENDPOINT when that's an internal-only address), and deliberately
// does NOT pin Content-Type on any signed command — the browser's actual
// Content-Type header (from the File object) would have to match whatever
// was signed exactly, and leaving it unsigned avoids that fragility.
//
// input:  originalname, ownerType — same as uploadFile(), used to build the
//         same {prefix}/{uuid}.{ext} key convention.
//         size — total file size in bytes, decides single-PUT vs multipart.
// output: { key, uploadUrl }                                    (size <= 5GB)
//      or { key, multipart: true, uploadId, partSize, parts }    (size > 5GB)
//         parts: [{ partNumber, uploadUrl }, ...], one presigned UploadPart
//         URL per part, all generated upfront in this one call.
//
async function presignUpload(originalname, ownerType = "image", size = 0) {
  const key = buildKey(originalname, ownerType);
  const client = getPublicClient();

  if (size <= MULTIPART_THRESHOLD) {
    // Generous expiry — even a file near the 5GB ceiling can take a long
    // time to PUT on a slow connection, and the signature must still be
    // valid when the browser actually gets around to sending it.
    const uploadUrl = await getSignedUrl(
      client,
      new PutObjectCommand({ Bucket: DEFAULT_BUCKET, Key: key }),
      { expiresIn: 4 * 60 * 60 } // 4h
    );
    return { key, uploadUrl };
  }

  // Above the single-PUT ceiling — the backend initiates the multipart
  // session itself with real credentials (a lightweight control-plane call,
  // no file bytes involved); only the individual part uploads, which do
  // carry real bytes, get presigned for the browser.
  const { UploadId: uploadId } = await s3.send(new CreateMultipartUploadCommand({
    Bucket: DEFAULT_BUCKET,
    Key:    key,
  }));

  const partCount = Math.ceil(size / PART_SIZE);
  const parts = [];
  for (let partNumber = 1; partNumber <= partCount; partNumber++) {
    const uploadUrl = await getSignedUrl(
      client,
      new UploadPartCommand({ Bucket: DEFAULT_BUCKET, Key: key, UploadId: uploadId, PartNumber: partNumber }),
      { expiresIn: 24 * 60 * 60 } // 24h — a 15GB upload can genuinely take a while on a slow connection
    );
    parts.push({ partNumber, uploadUrl });
  }

  return { key, multipart: true, uploadId, partSize: PART_SIZE, parts };
}

// ─── completeMultipartUpload ──────────────────────────────────────────────────
//
// Finishes a multipart upload once the browser has PUT every part directly
// (see presignUpload() above). Parts must carry the ETag each part's own PUT
// response returned — order doesn't matter here, they're sorted by
// PartNumber before submitting. Real credentials, not presigned: this is a
// small control-plane call, no file bytes involved.
//
// input: key, uploadId, parts — [{ partNumber, etag }, ...]
//
async function completeMultipartUpload(key, uploadId, parts) {
  await s3.send(new CompleteMultipartUploadCommand({
    Bucket:   DEFAULT_BUCKET,
    Key:      key,
    UploadId: uploadId,
    MultipartUpload: {
      Parts: parts
        .slice()
        .sort((a, b) => a.partNumber - b.partNumber)
        .map(({ partNumber, etag }) => ({ PartNumber: partNumber, ETag: etag })),
    },
  }));
}

// ─── abortMultipartUpload ──────────────────────────────────────────────────────
//
// Cancels an in-progress multipart upload (a part permanently failed after
// retries, or the browser gave up) so it doesn't linger as orphaned storage
// forever. Real credentials, called server-side — this only ever happens on
// failure, no need for browser-direct access to it.
//
async function abortMultipartUpload(key, uploadId) {
  await s3.send(new AbortMultipartUploadCommand({
    Bucket:   DEFAULT_BUCKET,
    Key:      key,
    UploadId: uploadId,
  }));
}

// ─── getFileMetadata ──────────────────────────────────────────────────────────
//
// Reads back what the browser actually uploaded via presignUpload() — used by
// the finalize step in place of multer's req.file (size, mimetype), since no
// buffer ever passes through this backend to read those from directly. ETag
// doubles as the object's checksum: for a single (non-multipart) unencrypted
// PutObjectCommand, S3-compatible ETag is exactly the MD5 of the body.
//
// input:  key
// output: { size, mimetype, checksum } — checksum is the ETag with its
//         surrounding quotes stripped, or null if the object has no ETag.
//
async function getFileMetadata(key) {
  const result = await s3.send(new HeadObjectCommand({
    Bucket: DEFAULT_BUCKET,
    Key:    key,
  }));
  return {
    size:     result.ContentLength ?? null,
    mimetype: result.ContentType ?? null,
    checksum: result.ETag ? result.ETag.replace(/"/g, "") : null,
  };
}

module.exports = {
  uploadFile, deleteFile, getSignedDownloadUrl, getPublicUrl, getObjectStream,
  presignUpload, completeMultipartUpload, abortMultipartUpload,
  getFileMetadata, buildPublicUrl, ping,
};