From b241220c15b86dd9a8d670ef89004d51421e3bbc Mon Sep 17 00:00:00 2001 From: Kenneth Obsequio Date: Sat, 18 Jul 2026 11:40:47 +0800 Subject: [PATCH] document to markdown Signed-off-by: Kenneth Obsequio --- .env-development | 10 ++ .env-production | 19 ++++ controllers/admin/assets.controller.js | 94 +++++++++++++++++ docker-compose.yml | 6 ++ package-lock.json | 75 ++++++++++++-- routes/admin/assets.routes.js | 2 + services/documentConversion.service.js | 135 +++++++++++++++++++++++++ services/s3.service.js | 13 +++ 8 files changed, 343 insertions(+), 11 deletions(-) create mode 100644 services/documentConversion.service.js diff --git a/.env-development b/.env-development index 31a60e9..c167a7e 100644 --- a/.env-development +++ b/.env-development @@ -94,5 +94,15 @@ CHIBISAFE_ALBUM_THUMBNAILS=CHANGE_ME_UUID CHIBISAFE_ALBUM_DOCUMENTS=CHANGE_ME_UUID CHIBISAFE_ALBUM_ARCHIVED=CHANGE_ME_UUID +# ── MarkItDown (document-import PDF/PPTX -> Markdown sidecar) ──────────────── +# Lives in the chibistar/ stack (chibistar/docker-compose.yml), not this +# repo's — `docker compose up -d markitdown` there. Reachable at +# 127.0.0.1:8000 since this app runs natively (npm run dev) during local dev, +# same host. +MARKITDOWN_BASE_URL=http://127.0.0.1:8000 +# Required even for local/loopback calls — the sidecar's /convert endpoint +# checks this on every request since it's also reachable publicly. +MARKITDOWN_SHARED_SECRET=CHANGE_ME + # ── CORS ────────────────────────────────────────────────────────────────────── ALLOWED_ORIGINS=http://localhost:5173,http://localhost:3024 diff --git a/.env-production b/.env-production index d49e7d4..07b93e9 100644 --- a/.env-production +++ b/.env-production @@ -125,6 +125,25 @@ CHIBISAFE_ALBUM_THUMBNAILS=CHANGE_ME_UUID CHIBISAFE_ALBUM_DOCUMENTS=CHANGE_ME_UUID CHIBISAFE_ALBUM_ARCHIVED=CHANGE_ME_UUID +# ── MarkItDown (document-import PDF/PPTX -> Markdown sidecar) ──────────────── +# Lives in the separate chibistar/ stack, not this one — not reachable by +# Docker service name. +# +# Self-hosted via this repo's docker-compose.yml, on the SAME machine as +# chibistar: reach it via host.docker.internal (see docker-compose.yml's +# extra_hosts on `backend`), which resolves to the host running chibistar's +# markitdown container (published to 127.0.0.1:8000 there). +MARKITDOWN_BASE_URL=http://host.docker.internal:8000 +# +# Deployed elsewhere (e.g. Render) — no shared network with chibistar, so +# use the public hostname instead (zrok reserved share, see +# chibistar/docker-compose.yml's zrok-markitdown service): +# MARKITDOWN_BASE_URL=https://markitdownstarr.share.zrok.io +# +# Required in both cases — /convert checks this on every request since it's +# reachable publicly, not just over loopback/the Docker network. +MARKITDOWN_SHARED_SECRET=CHANGE_ME + # ── CORS ────────────────────────────────────────────────────────────────────── # Comma-separated list of allowed origins. Must match FRONTEND_URL exactly. ALLOWED_ORIGINS=https://yourdomain.com diff --git a/controllers/admin/assets.controller.js b/controllers/admin/assets.controller.js index 25c5a73..23ad792 100644 --- a/controllers/admin/assets.controller.js +++ b/controllers/admin/assets.controller.js @@ -9,6 +9,7 @@ const s3 = require("../../services/s3.service"); const mediaToken = require("../../services/mediaToken.service"); const uploadProgress = require("../../services/uploadProgress.service"); const { extractVideoMeta } = require("../../services/ffprobe.service"); +const documentConversion = require("../../services/documentConversion.service"); const R = require('../../utils/response.util'); const { paginate } = require("../../utils/paginate.util"); const { adminExclude, jsonbSchemas, computedAttributes } = require("../../models/assets/assets.attributes"); @@ -60,6 +61,15 @@ function resolveChecksum(buffer) { return crypto.createHash("sha256").update(buffer).digest("hex"); } +function streamToBuffer(stream) { + return new Promise((resolve, reject) => { + const chunks = []; + stream.on("data", (chunk) => chunks.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(chunks))); + stream.on("error", reject); + }); +} + function resolveResolution(width, height) { if (!width || !height) return null; const h = Math.min(width, height); @@ -524,6 +534,12 @@ exports.streamUploadProgress = (req, res) => { uploadProgress.subscribe(req.params.uploadId, res); }; +// Same broadcaster, reused as-is for document-conversion job progress — it's +// just a generic string-keyed SSE channel, nothing upload-specific about it. +exports.streamConvertProgress = (req, res) => { + uploadProgress.subscribe(req.params.jobId, res); +}; + // ─── UPLOAD (batch) ───────────────────────────────────────────────────────── // // Accepts multiple files under the "files" field in one multipart request, @@ -573,6 +589,84 @@ exports.uploadAssetsBatch = async (req, res) => { return R.success(res, `${createdCount} of ${files.length} asset(s) uploaded.`, { results }, 201); }; +// ─── CONVERT TO MARKDOWN ──────────────────────────────────────────────────── +// +// PDF/PPTX -> Markdown, text only (see services/documentConversion.service.js +// for the compile/validate/automate stages and why OCR/images are out of +// scope). Nothing here is written to the database — the caller (the Lesson +// block builder) treats the response as a draft and only persists it if/when +// the admin explicitly inserts it into a block and saves the lesson page. +// +// Mirrors the upload flow's SSE progress pattern exactly: the actual work +// runs synchronously inside this request (uploadProgress.service.js is +// reused as-is, keyed by a client-generated jobId) while stage transitions +// are published for a live "Compiling / Validating / Generating" UI, same as +// AddAsset.jsx already renders for uploads. +// +const CONVERTIBLE_EXTENSIONS = new Set(["pdf", "pptx"]); +const MAX_CONVERT_SIZE_BYTES = 25 * 1024 * 1024; // 25MB — keeps this comfortably synchronous +// compile() calls the MarkItDown sidecar over HTTP now instead of running +// officeparser in-process, so this leaves a bit more room than the original +// 45s for network/queueing overhead — MarkItDown's own parsing is plain +// CPU-bound work, not ML inference, so it doesn't need much more than that. +const CONVERT_TIMEOUT_MS = 60_000; + +exports.convertAssetToMarkdown = async (req, res) => { + const { jobId } = req.body; + + try { + const { assetId } = req.params; + if (!assetId || assetId === "undefined") return R.error(res, "Invalid asset ID.", 400); + + const asset = await Asset.findOne({ where: { asset_id: assetId, ...notDeleted } }); + if (!asset) return R.error(res, "Asset not found.", 404); + + const extension = (asset.extension || "").toLowerCase(); + if (!CONVERTIBLE_EXTENSIONS.has(extension)) { + return R.error(res, "Only PDF and PPTX documents can be converted to Markdown.", 400); + } + if (asset.storage_provider !== "s3" || !asset.storage_key) { + return R.error(res, "This asset has no stored file to convert.", 400); + } + if (Number(asset.file_size) > MAX_CONVERT_SIZE_BYTES) { + return R.error(res, "File is too large to convert (25MB max).", 400); + } + + const publish = (phase) => { if (jobId) uploadProgress.publish(jobId, { phase }); }; + const abortController = new AbortController(); + const timeout = setTimeout(() => abortController.abort(), CONVERT_TIMEOUT_MS); + + let markdown, warnings, stats; + try { + publish("compiling"); + const { stream } = await s3.getObjectStream(asset.storage_key); + const buffer = await streamToBuffer(stream); + const ast = await documentConversion.compile(buffer, extension, { signal: abortController.signal }); + + publish("validating"); + const validated = documentConversion.validate(ast); + + publish("generating"); + const generated = await documentConversion.automate(ast, { signal: abortController.signal }); + + markdown = generated.markdown; + warnings = [...validated.warnings, ...generated.messages]; + stats = validated.stats; + } finally { + clearTimeout(timeout); + } + + if (jobId) uploadProgress.complete(jobId, { phase: "done" }); + return R.success(res, "Document converted.", { markdown, warnings, stats }); + + } catch (err) { + if (jobId) uploadProgress.complete(jobId, { phase: "error", message: err.message }); + console.error("[ASSET][CONVERT TO MARKDOWN]", err); + if (err.status) return R.error(res, err.message, err.status); + return R.error(res, "Internal server error.", 500); + } +}; + // ─── UPDATE ─────────────────────────────────────────────────────────────────── exports.updateAsset = async (req, res) => { diff --git a/docker-compose.yml b/docker-compose.yml index d3a5086..c268306 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,11 +29,17 @@ services: # ── Backend (Express) ─────────────────────────────────────────────────────── + # MarkItDown (PDF/PPTX -> Markdown sidecar for document import) lives in the + # separate chibistar/ stack, not here — extra_hosts lets this container + # reach its host-published port via host.docker.internal. Set + # MARKITDOWN_BASE_URL=http://host.docker.internal:8000 in .env for this to work. backend: build: . ports: - "3024:3024" env_file: .env + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: postgres: condition: service_healthy diff --git a/package-lock.json b/package-lock.json index c38e02b..6558c57 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,7 +68,6 @@ "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1085.0.tgz", "integrity": "sha512-O0xe8sR50AYkwxlvRRsV0qytEO2dtXQTQ1CF3YBBdE5xtVkbu27H0vGa1mjQi1/+fbYM80AWEIPai5jZmXyubw==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@aws-sdk/checksums": "^3.1000.16", "@aws-sdk/core": "^3.975.1", @@ -424,7 +423,6 @@ "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -977,13 +975,39 @@ "dev": true, "license": "MIT" }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "node_modules/@emnapi/core": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", + "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", "dev": true, "license": "MIT", "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -1488,7 +1512,6 @@ "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz", "integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==", "license": "MIT", - "peer": true, "dependencies": { "cluster-key-slot": "1.1.2", "generic-pool": "3.9.0", @@ -2057,6 +2080,40 @@ "node": ">=14.0.0" } }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.12.2.tgz", @@ -2508,7 +2565,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.38", "caniuse-lite": "^1.0.30001799", @@ -3578,7 +3634,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -3625,7 +3680,6 @@ "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.11.2.tgz", "integrity": "sha512-a7uwwfNTh1U60ssiIkuLFWHt4hAC5yxlLGU2VP0X4YNlyEDZAqF4tK3GD3NSitVBrCQmQ0++0uOyFOgC2y4DDw==", "license": "MIT", - "peer": true, "engines": { "node": ">= 14" }, @@ -6291,7 +6345,6 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", - "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", diff --git a/routes/admin/assets.routes.js b/routes/admin/assets.routes.js index 9dd0ae4..ae4c788 100644 --- a/routes/admin/assets.routes.js +++ b/routes/admin/assets.routes.js @@ -12,6 +12,7 @@ router.delete('/bulk', sensitiveOpsLimiter, controller.archiveAssets); router.delete('/bulk/permanent', sensitiveOpsLimiter, controller.permanentlyDeleteAssets); router.patch('/bulk-restore', controller.restoreAssets); router.get('/upload-progress/:uploadId', controller.streamUploadProgress); +router.get('/convert-progress/:jobId', controller.streamConvertProgress); // ─── Collection ─────────────────────────────────────────────────────────────── router.get('/', controller.getAssets); @@ -20,6 +21,7 @@ router.post('/batch', handleUpload(upload.array('files', 20)), controller.upload // ─── Dynamic routes last ────────────────────────────────────────────────────── router.get('/:assetId', controller.getAsset); +router.post('/:assetId/convert-to-markdown', sensitiveOpsLimiter, controller.convertAssetToMarkdown); router.patch('/:assetId', sensitiveOpsLimiter, handleUpload(upload.fields([{ name: 'file', maxCount: 1 }])), controller.updateAsset); router.patch('/:assetId/restore', sensitiveOpsLimiter, controller.restoreAsset); router.delete('/:assetId', sensitiveOpsLimiter, controller.archiveAsset); diff --git a/services/documentConversion.service.js b/services/documentConversion.service.js new file mode 100644 index 0000000..7b1ce2a --- /dev/null +++ b/services/documentConversion.service.js @@ -0,0 +1,135 @@ +// services/documentConversion.service.js +// +// PDF/PPTX -> Markdown via the MarkItDown sidecar (chibistar/markitdown-service), +// which does the actual parsing + Markdown generation. That's why this file no +// longer builds or walks an AST like the old officeparser-based version did — +// the sidecar already returns a Markdown string, so there's nothing left to +// serialize on this side. +// +// Three named stages, called in sequence by the controller (unchanged from +// before this swap — the controller and frontend don't know or care that the +// extraction backend changed): +// compile(buffer, extension) -> call the sidecar, get back markdown + a text length +// validate(result) -> confirm there's usable text before generating anything +// automate(result) -> light cleanup (frontmatter strip) of the sidecar's markdown +// +// Environment variables expected: +// MARKITDOWN_BASE_URL — base URL of the markitdown-service sidecar (see chibistar/docker-compose.yml) +// MARKITDOWN_SHARED_SECRET — sent as X-Shared-Secret; the sidecar is reachable +// publicly (markitdown.starr.philpro.orij.space), not +// just over loopback/the Docker network, so this is +// required in every environment, dev included. + +const axios = require("axios"); + +const MIN_EXTRACTED_TEXT_LENGTH = 20; + +// Backstop only — the controller's own AbortSignal (tied to its +// CONVERT_TIMEOUT_MS) is what actually bounds this request in practice. This +// exists so a hung sidecar can't wedge the request open indefinitely if that +// signal somehow never fires. MarkItDown does plain CPU-bound parsing (no ML +// inference), so this is a modest timeout, not the generous one a +// Docling-backed sidecar would need. +const MARKITDOWN_REQUEST_TIMEOUT_MS = 45_000; + +const BASE_URL = (process.env.MARKITDOWN_BASE_URL || "").replace(/\/$/, ""); +const SHARED_SECRET = process.env.MARKITDOWN_SHARED_SECRET || ""; + +const markitdown = axios.create({ + baseURL: BASE_URL, + timeout: MARKITDOWN_REQUEST_TIMEOUT_MS, + maxBodyLength: Infinity, + maxContentLength: Infinity, +}); + +// ─── compile ─────────────────────────────────────────────────────────────── +// +// PPTX is the more failure-prone input than PDF — OOXML slide decks exported +// by different PowerPoint/export-pipeline versions vary more than PDFs do — +// so PPTX failures get an explicit, format-aware message instead of a raw +// parser error. Same convention the sidecar itself follows for its own 422s. +// +async function compile(buffer, extension, { signal } = {}) { + const fileType = (extension || "").toLowerCase(); + + if (!BASE_URL) { + throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_BASE_URL missing)."), { status: 500, stage: "compile" }); + } + if (!SHARED_SECRET) { + throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_SHARED_SECRET missing)."), { status: 500, stage: "compile" }); + } + + try { + const res = await markitdown.post("/convert", buffer, { + headers: { + "Content-Type": "application/octet-stream", + "X-File-Extension": fileType, + "X-Shared-Secret": SHARED_SECRET, + }, + signal, + }); + return res.data; // { markdown, textLength, warnings } + } catch (err) { + // The sidecar reached us and rejected the file (422 bad/corrupt doc, 503 + // over its concurrency limit, etc.) — surface its own message as-is. + if (err.response) { + throw Object.assign(new Error(err.response.data?.detail || "Document conversion failed."), { + status: err.response.status, + stage: "compile", + cause: err, + }); + } + + // The sidecar never responded at all (down, unreachable, or the + // controller's/our own timeout fired) — fall back to a friendly, + // format-aware message since there's no sidecar-provided one to use. + const friendly = fileType === "pptx" + ? "Couldn't read this PowerPoint file. It may be corrupted, password-protected, or saved in a format this parser doesn't support — try re-exporting it from PowerPoint." + : "Couldn't read this PDF. It may be corrupted, password-protected, or scanned/image-only with no embedded text."; + + const timedOut = axios.isCancel(err) || err.code === "ECONNABORTED"; + throw Object.assign(new Error(friendly), { status: timedOut ? 504 : 502, stage: "compile", cause: err }); + } +} + +// ─── validate ────────────────────────────────────────────────────────────── +// +// Confirms the sidecar actually found usable text before anything is +// generated. A document with no extractable text (e.g. a slide deck that's +// entirely images/screenshots) is a hard stop here — OCR is out of scope, so +// there's no fallback to offer, and drafting an empty block would just be +// confusing. The length itself is computed sidecar-side — nothing here +// re-walks a tree anymore. +// +function validate(result) { + const warnings = result.warnings || []; + const extractedLength = result.textLength || 0; + + if (extractedLength < MIN_EXTRACTED_TEXT_LENGTH) { + throw Object.assign( + new Error("No readable text found in this document. This tool only extracts text (including hyperlinks) — image-only or scanned documents aren't supported."), + { status: 422, stage: "validate" }, + ); + } + + return { warnings, stats: { extractedLength } }; +} + +// ─── automate ────────────────────────────────────────────────────────────── +// +// MarkItDown already emits Markdown — this is just the same +// frontmatter-stripping cleanup the officeparser path used to need, kept +// as-is since some exporters still prepend a properties block. +// +function stripFrontmatter(markdown) { + return markdown.replace(/^---\n[\s\S]*?\n---\n+/, ""); +} + +async function automate(result) { + return { + markdown: stripFrontmatter(result.markdown || "").trim(), + messages: result.warnings || [], + }; +} + +module.exports = { compile, validate, automate }; diff --git a/services/s3.service.js b/services/s3.service.js index 9e5a43d..022650c 100644 --- a/services/s3.service.js +++ b/services/s3.service.js @@ -35,6 +35,17 @@ const credentials = { secretAccessKey: process.env.S3_SECRET_KEY, }; +// responseChecksumValidation: "WHEN_REQUIRED" — the AWS SDK v3 defaults to +// "WHEN_SUPPORTED", which validates any response carrying an x-amz-checksum-* +// header. Garage sends x-amz-checksum-crc32 on GetObject but its value doesn't +// match what the SDK recomputes over the body, so every direct GetObjectCommand +// (e.g. getObjectStream() below, used by the document-to-Markdown converter) +// threw "Checksum mismatch" even though the bytes themselves were fine. +// WHEN_REQUIRED skips validation unless the operation mandates it (GetObject +// never does) — safe here since Garage isn't computing/verifying per AWS spec +// anyway. +const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED" }; + // Internal client — uploads, deletes, direct streams from the server itself. // Always targets S3_ENDPOINT. When this backend is co-located with Garage, // that's a local address; when it isn't, S3_ENDPOINT must instead be a @@ -45,6 +56,7 @@ const s3 = new S3Client({ region: process.env.S3_REGION || "garage", credentials, forcePathStyle: true, + ...CHECKSUM_CONFIG, }); const DEFAULT_BUCKET = process.env.S3_BUCKET; @@ -76,6 +88,7 @@ function getPublicClient() { region: process.env.S3_REGION || "garage", credentials, forcePathStyle: true, + ...CHECKSUM_CONFIG, }); }