mirror of
https://github.com/rgrgogu/new_starr.git
synced 2026-09-27 00:12:54 +08:00
document to markdown
Signed-off-by: Kenneth Obsequio <k80308392@gmail.com>
This commit is contained in:
@@ -94,5 +94,15 @@ CHIBISAFE_ALBUM_THUMBNAILS=CHANGE_ME_UUID
|
||||
CHIBISAFE_ALBUM_DOCUMENTS=CHANGE_ME_UUID
|
||||
CHIBISAFE_ALBUM_ARCHIVED=CHANGE_ME_UUID
|
||||
|
||||
# ── MarkItDown (document-import PDF/PPTX -> Markdown sidecar) ────────────────
|
||||
# Lives in the chibistar/ stack (chibistar/docker-compose.yml), not this
|
||||
# repo's — `docker compose up -d markitdown` there. Reachable at
|
||||
# 127.0.0.1:8000 since this app runs natively (npm run dev) during local dev,
|
||||
# same host.
|
||||
MARKITDOWN_BASE_URL=http://127.0.0.1:8000
|
||||
# Required even for local/loopback calls — the sidecar's /convert endpoint
|
||||
# checks this on every request since it's also reachable publicly.
|
||||
MARKITDOWN_SHARED_SECRET=CHANGE_ME
|
||||
|
||||
# ── CORS ──────────────────────────────────────────────────────────────────────
|
||||
ALLOWED_ORIGINS=http://localhost:5173,http://localhost:3024
|
||||
|
||||
@@ -125,6 +125,25 @@ CHIBISAFE_ALBUM_THUMBNAILS=CHANGE_ME_UUID
|
||||
CHIBISAFE_ALBUM_DOCUMENTS=CHANGE_ME_UUID
|
||||
CHIBISAFE_ALBUM_ARCHIVED=CHANGE_ME_UUID
|
||||
|
||||
# ── MarkItDown (document-import PDF/PPTX -> Markdown sidecar) ────────────────
|
||||
# Lives in the separate chibistar/ stack, not this one — not reachable by
|
||||
# Docker service name.
|
||||
#
|
||||
# Self-hosted via this repo's docker-compose.yml, on the SAME machine as
|
||||
# chibistar: reach it via host.docker.internal (see docker-compose.yml's
|
||||
# extra_hosts on `backend`), which resolves to the host running chibistar's
|
||||
# markitdown container (published to 127.0.0.1:8000 there).
|
||||
MARKITDOWN_BASE_URL=http://host.docker.internal:8000
|
||||
#
|
||||
# Deployed elsewhere (e.g. Render) — no shared network with chibistar, so
|
||||
# use the public hostname instead (zrok reserved share, see
|
||||
# chibistar/docker-compose.yml's zrok-markitdown service):
|
||||
# MARKITDOWN_BASE_URL=https://markitdownstarr.share.zrok.io
|
||||
#
|
||||
# Required in both cases — /convert checks this on every request since it's
|
||||
# reachable publicly, not just over loopback/the Docker network.
|
||||
MARKITDOWN_SHARED_SECRET=CHANGE_ME
|
||||
|
||||
# ── CORS ──────────────────────────────────────────────────────────────────────
|
||||
# Comma-separated list of allowed origins. Must match FRONTEND_URL exactly.
|
||||
ALLOWED_ORIGINS=https://yourdomain.com
|
||||
|
||||
@@ -9,6 +9,7 @@ const s3 = require("../../services/s3.service");
|
||||
const mediaToken = require("../../services/mediaToken.service");
|
||||
const uploadProgress = require("../../services/uploadProgress.service");
|
||||
const { extractVideoMeta } = require("../../services/ffprobe.service");
|
||||
const documentConversion = require("../../services/documentConversion.service");
|
||||
const R = require('../../utils/response.util');
|
||||
const { paginate } = require("../../utils/paginate.util");
|
||||
const { adminExclude, jsonbSchemas, computedAttributes } = require("../../models/assets/assets.attributes");
|
||||
@@ -60,6 +61,15 @@ function resolveChecksum(buffer) {
|
||||
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
}
|
||||
|
||||
function streamToBuffer(stream) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
stream.on("data", (chunk) => chunks.push(chunk));
|
||||
stream.on("end", () => resolve(Buffer.concat(chunks)));
|
||||
stream.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function resolveResolution(width, height) {
|
||||
if (!width || !height) return null;
|
||||
const h = Math.min(width, height);
|
||||
@@ -524,6 +534,12 @@ exports.streamUploadProgress = (req, res) => {
|
||||
uploadProgress.subscribe(req.params.uploadId, res);
|
||||
};
|
||||
|
||||
// Same broadcaster, reused as-is for document-conversion job progress — it's
|
||||
// just a generic string-keyed SSE channel, nothing upload-specific about it.
|
||||
exports.streamConvertProgress = (req, res) => {
|
||||
uploadProgress.subscribe(req.params.jobId, res);
|
||||
};
|
||||
|
||||
// ─── UPLOAD (batch) ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// Accepts multiple files under the "files" field in one multipart request,
|
||||
@@ -573,6 +589,84 @@ exports.uploadAssetsBatch = async (req, res) => {
|
||||
return R.success(res, `${createdCount} of ${files.length} asset(s) uploaded.`, { results }, 201);
|
||||
};
|
||||
|
||||
// ─── CONVERT TO MARKDOWN ────────────────────────────────────────────────────
|
||||
//
|
||||
// PDF/PPTX -> Markdown, text only (see services/documentConversion.service.js
|
||||
// for the compile/validate/automate stages and why OCR/images are out of
|
||||
// scope). Nothing here is written to the database — the caller (the Lesson
|
||||
// block builder) treats the response as a draft and only persists it if/when
|
||||
// the admin explicitly inserts it into a block and saves the lesson page.
|
||||
//
|
||||
// Mirrors the upload flow's SSE progress pattern exactly: the actual work
|
||||
// runs synchronously inside this request (uploadProgress.service.js is
|
||||
// reused as-is, keyed by a client-generated jobId) while stage transitions
|
||||
// are published for a live "Compiling / Validating / Generating" UI, same as
|
||||
// AddAsset.jsx already renders for uploads.
|
||||
//
|
||||
const CONVERTIBLE_EXTENSIONS = new Set(["pdf", "pptx"]);
|
||||
const MAX_CONVERT_SIZE_BYTES = 25 * 1024 * 1024; // 25MB — keeps this comfortably synchronous
|
||||
// compile() calls the MarkItDown sidecar over HTTP now instead of running
|
||||
// officeparser in-process, so this leaves a bit more room than the original
|
||||
// 45s for network/queueing overhead — MarkItDown's own parsing is plain
|
||||
// CPU-bound work, not ML inference, so it doesn't need much more than that.
|
||||
const CONVERT_TIMEOUT_MS = 60_000;
|
||||
|
||||
exports.convertAssetToMarkdown = async (req, res) => {
|
||||
const { jobId } = req.body;
|
||||
|
||||
try {
|
||||
const { assetId } = req.params;
|
||||
if (!assetId || assetId === "undefined") return R.error(res, "Invalid asset ID.", 400);
|
||||
|
||||
const asset = await Asset.findOne({ where: { asset_id: assetId, ...notDeleted } });
|
||||
if (!asset) return R.error(res, "Asset not found.", 404);
|
||||
|
||||
const extension = (asset.extension || "").toLowerCase();
|
||||
if (!CONVERTIBLE_EXTENSIONS.has(extension)) {
|
||||
return R.error(res, "Only PDF and PPTX documents can be converted to Markdown.", 400);
|
||||
}
|
||||
if (asset.storage_provider !== "s3" || !asset.storage_key) {
|
||||
return R.error(res, "This asset has no stored file to convert.", 400);
|
||||
}
|
||||
if (Number(asset.file_size) > MAX_CONVERT_SIZE_BYTES) {
|
||||
return R.error(res, "File is too large to convert (25MB max).", 400);
|
||||
}
|
||||
|
||||
const publish = (phase) => { if (jobId) uploadProgress.publish(jobId, { phase }); };
|
||||
const abortController = new AbortController();
|
||||
const timeout = setTimeout(() => abortController.abort(), CONVERT_TIMEOUT_MS);
|
||||
|
||||
let markdown, warnings, stats;
|
||||
try {
|
||||
publish("compiling");
|
||||
const { stream } = await s3.getObjectStream(asset.storage_key);
|
||||
const buffer = await streamToBuffer(stream);
|
||||
const ast = await documentConversion.compile(buffer, extension, { signal: abortController.signal });
|
||||
|
||||
publish("validating");
|
||||
const validated = documentConversion.validate(ast);
|
||||
|
||||
publish("generating");
|
||||
const generated = await documentConversion.automate(ast, { signal: abortController.signal });
|
||||
|
||||
markdown = generated.markdown;
|
||||
warnings = [...validated.warnings, ...generated.messages];
|
||||
stats = validated.stats;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
if (jobId) uploadProgress.complete(jobId, { phase: "done" });
|
||||
return R.success(res, "Document converted.", { markdown, warnings, stats });
|
||||
|
||||
} catch (err) {
|
||||
if (jobId) uploadProgress.complete(jobId, { phase: "error", message: err.message });
|
||||
console.error("[ASSET][CONVERT TO MARKDOWN]", err);
|
||||
if (err.status) return R.error(res, err.message, err.status);
|
||||
return R.error(res, "Internal server error.", 500);
|
||||
}
|
||||
};
|
||||
|
||||
// ─── UPDATE ───────────────────────────────────────────────────────────────────
|
||||
|
||||
exports.updateAsset = async (req, res) => {
|
||||
|
||||
@@ -29,11 +29,17 @@
|
||||
services:
|
||||
|
||||
# ── Backend (Express) ───────────────────────────────────────────────────────
|
||||
# MarkItDown (PDF/PPTX -> Markdown sidecar for document import) lives in the
|
||||
# separate chibistar/ stack, not here — extra_hosts lets this container
|
||||
# reach its host-published port via host.docker.internal. Set
|
||||
# MARKITDOWN_BASE_URL=http://host.docker.internal:8000 in .env for this to work.
|
||||
backend:
|
||||
build: .
|
||||
ports:
|
||||
- "3024:3024"
|
||||
env_file: .env
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
Generated
+64
-11
@@ -68,7 +68,6 @@
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1085.0.tgz",
|
||||
"integrity": "sha512-O0xe8sR50AYkwxlvRRsV0qytEO2dtXQTQ1CF3YBBdE5xtVkbu27H0vGa1mjQi1/+fbYM80AWEIPai5jZmXyubw==",
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@aws-sdk/checksums": "^3.1000.16",
|
||||
"@aws-sdk/core": "^3.975.1",
|
||||
@@ -424,7 +423,6 @@
|
||||
"integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.29.7",
|
||||
"@babel/generator": "^7.29.7",
|
||||
@@ -977,13 +975,39 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@emnapi/wasi-threads": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz",
|
||||
"integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==",
|
||||
"node_modules/@emnapi/core": {
|
||||
"version": "1.11.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz",
|
||||
"integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@emnapi/wasi-threads": "1.2.2",
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@emnapi/runtime": {
|
||||
"version": "1.11.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz",
|
||||
"integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@emnapi/wasi-threads": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz",
|
||||
"integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
@@ -1488,7 +1512,6 @@
|
||||
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
|
||||
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"cluster-key-slot": "1.1.2",
|
||||
"generic-pool": "3.9.0",
|
||||
@@ -2057,6 +2080,40 @@
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/core": {
|
||||
"version": "1.10.0",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz",
|
||||
"integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/wasi-threads": "1.2.1",
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/runtime": {
|
||||
"version": "1.10.0",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz",
|
||||
"integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz",
|
||||
"integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@unrs/resolver-binding-win32-arm64-msvc": {
|
||||
"version": "1.12.2",
|
||||
"resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.12.2.tgz",
|
||||
@@ -2508,7 +2565,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"baseline-browser-mapping": "^2.10.38",
|
||||
"caniuse-lite": "^1.0.30001799",
|
||||
@@ -3578,7 +3634,6 @@
|
||||
"resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz",
|
||||
"integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"accepts": "~1.3.8",
|
||||
"array-flatten": "1.1.1",
|
||||
@@ -3625,7 +3680,6 @@
|
||||
"resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.11.2.tgz",
|
||||
"integrity": "sha512-a7uwwfNTh1U60ssiIkuLFWHt4hAC5yxlLGU2VP0X4YNlyEDZAqF4tK3GD3NSitVBrCQmQ0++0uOyFOgC2y4DDw==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
},
|
||||
@@ -6291,7 +6345,6 @@
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz",
|
||||
"integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"pg-connection-string": "^2.12.0",
|
||||
"pg-pool": "^3.13.0",
|
||||
|
||||
@@ -12,6 +12,7 @@ router.delete('/bulk', sensitiveOpsLimiter, controller.archiveAssets);
|
||||
router.delete('/bulk/permanent', sensitiveOpsLimiter, controller.permanentlyDeleteAssets);
|
||||
router.patch('/bulk-restore', controller.restoreAssets);
|
||||
router.get('/upload-progress/:uploadId', controller.streamUploadProgress);
|
||||
router.get('/convert-progress/:jobId', controller.streamConvertProgress);
|
||||
|
||||
// ─── Collection ───────────────────────────────────────────────────────────────
|
||||
router.get('/', controller.getAssets);
|
||||
@@ -20,6 +21,7 @@ router.post('/batch', handleUpload(upload.array('files', 20)), controller.upload
|
||||
|
||||
// ─── Dynamic routes last ──────────────────────────────────────────────────────
|
||||
router.get('/:assetId', controller.getAsset);
|
||||
router.post('/:assetId/convert-to-markdown', sensitiveOpsLimiter, controller.convertAssetToMarkdown);
|
||||
router.patch('/:assetId', sensitiveOpsLimiter, handleUpload(upload.fields([{ name: 'file', maxCount: 1 }])), controller.updateAsset);
|
||||
router.patch('/:assetId/restore', sensitiveOpsLimiter, controller.restoreAsset);
|
||||
router.delete('/:assetId', sensitiveOpsLimiter, controller.archiveAsset);
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
// services/documentConversion.service.js
|
||||
//
|
||||
// PDF/PPTX -> Markdown via the MarkItDown sidecar (chibistar/markitdown-service),
|
||||
// which does the actual parsing + Markdown generation. That's why this file no
|
||||
// longer builds or walks an AST like the old officeparser-based version did —
|
||||
// the sidecar already returns a Markdown string, so there's nothing left to
|
||||
// serialize on this side.
|
||||
//
|
||||
// Three named stages, called in sequence by the controller (unchanged from
|
||||
// before this swap — the controller and frontend don't know or care that the
|
||||
// extraction backend changed):
|
||||
// compile(buffer, extension) -> call the sidecar, get back markdown + a text length
|
||||
// validate(result) -> confirm there's usable text before generating anything
|
||||
// automate(result) -> light cleanup (frontmatter strip) of the sidecar's markdown
|
||||
//
|
||||
// Environment variables expected:
|
||||
// MARKITDOWN_BASE_URL — base URL of the markitdown-service sidecar (see chibistar/docker-compose.yml)
|
||||
// MARKITDOWN_SHARED_SECRET — sent as X-Shared-Secret; the sidecar is reachable
|
||||
// publicly (markitdown.starr.philpro.orij.space), not
|
||||
// just over loopback/the Docker network, so this is
|
||||
// required in every environment, dev included.
|
||||
|
||||
const axios = require("axios");
|
||||
|
||||
const MIN_EXTRACTED_TEXT_LENGTH = 20;
|
||||
|
||||
// Backstop only — the controller's own AbortSignal (tied to its
|
||||
// CONVERT_TIMEOUT_MS) is what actually bounds this request in practice. This
|
||||
// exists so a hung sidecar can't wedge the request open indefinitely if that
|
||||
// signal somehow never fires. MarkItDown does plain CPU-bound parsing (no ML
|
||||
// inference), so this is a modest timeout, not the generous one a
|
||||
// Docling-backed sidecar would need.
|
||||
const MARKITDOWN_REQUEST_TIMEOUT_MS = 45_000;
|
||||
|
||||
const BASE_URL = (process.env.MARKITDOWN_BASE_URL || "").replace(/\/$/, "");
|
||||
const SHARED_SECRET = process.env.MARKITDOWN_SHARED_SECRET || "";
|
||||
|
||||
const markitdown = axios.create({
|
||||
baseURL: BASE_URL,
|
||||
timeout: MARKITDOWN_REQUEST_TIMEOUT_MS,
|
||||
maxBodyLength: Infinity,
|
||||
maxContentLength: Infinity,
|
||||
});
|
||||
|
||||
// ─── compile ───────────────────────────────────────────────────────────────
|
||||
//
|
||||
// PPTX is the more failure-prone input than PDF — OOXML slide decks exported
|
||||
// by different PowerPoint/export-pipeline versions vary more than PDFs do —
|
||||
// so PPTX failures get an explicit, format-aware message instead of a raw
|
||||
// parser error. Same convention the sidecar itself follows for its own 422s.
|
||||
//
|
||||
async function compile(buffer, extension, { signal } = {}) {
|
||||
const fileType = (extension || "").toLowerCase();
|
||||
|
||||
if (!BASE_URL) {
|
||||
throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_BASE_URL missing)."), { status: 500, stage: "compile" });
|
||||
}
|
||||
if (!SHARED_SECRET) {
|
||||
throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_SHARED_SECRET missing)."), { status: 500, stage: "compile" });
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await markitdown.post("/convert", buffer, {
|
||||
headers: {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"X-File-Extension": fileType,
|
||||
"X-Shared-Secret": SHARED_SECRET,
|
||||
},
|
||||
signal,
|
||||
});
|
||||
return res.data; // { markdown, textLength, warnings }
|
||||
} catch (err) {
|
||||
// The sidecar reached us and rejected the file (422 bad/corrupt doc, 503
|
||||
// over its concurrency limit, etc.) — surface its own message as-is.
|
||||
if (err.response) {
|
||||
throw Object.assign(new Error(err.response.data?.detail || "Document conversion failed."), {
|
||||
status: err.response.status,
|
||||
stage: "compile",
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
|
||||
// The sidecar never responded at all (down, unreachable, or the
|
||||
// controller's/our own timeout fired) — fall back to a friendly,
|
||||
// format-aware message since there's no sidecar-provided one to use.
|
||||
const friendly = fileType === "pptx"
|
||||
? "Couldn't read this PowerPoint file. It may be corrupted, password-protected, or saved in a format this parser doesn't support — try re-exporting it from PowerPoint."
|
||||
: "Couldn't read this PDF. It may be corrupted, password-protected, or scanned/image-only with no embedded text.";
|
||||
|
||||
const timedOut = axios.isCancel(err) || err.code === "ECONNABORTED";
|
||||
throw Object.assign(new Error(friendly), { status: timedOut ? 504 : 502, stage: "compile", cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── validate ──────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Confirms the sidecar actually found usable text before anything is
|
||||
// generated. A document with no extractable text (e.g. a slide deck that's
|
||||
// entirely images/screenshots) is a hard stop here — OCR is out of scope, so
|
||||
// there's no fallback to offer, and drafting an empty block would just be
|
||||
// confusing. The length itself is computed sidecar-side — nothing here
|
||||
// re-walks a tree anymore.
|
||||
//
|
||||
function validate(result) {
|
||||
const warnings = result.warnings || [];
|
||||
const extractedLength = result.textLength || 0;
|
||||
|
||||
if (extractedLength < MIN_EXTRACTED_TEXT_LENGTH) {
|
||||
throw Object.assign(
|
||||
new Error("No readable text found in this document. This tool only extracts text (including hyperlinks) — image-only or scanned documents aren't supported."),
|
||||
{ status: 422, stage: "validate" },
|
||||
);
|
||||
}
|
||||
|
||||
return { warnings, stats: { extractedLength } };
|
||||
}
|
||||
|
||||
// ─── automate ──────────────────────────────────────────────────────────────
|
||||
//
|
||||
// MarkItDown already emits Markdown — this is just the same
|
||||
// frontmatter-stripping cleanup the officeparser path used to need, kept
|
||||
// as-is since some exporters still prepend a properties block.
|
||||
//
|
||||
function stripFrontmatter(markdown) {
|
||||
return markdown.replace(/^---\n[\s\S]*?\n---\n+/, "");
|
||||
}
|
||||
|
||||
async function automate(result) {
|
||||
return {
|
||||
markdown: stripFrontmatter(result.markdown || "").trim(),
|
||||
messages: result.warnings || [],
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { compile, validate, automate };
|
||||
@@ -35,6 +35,17 @@ const credentials = {
|
||||
secretAccessKey: process.env.S3_SECRET_KEY,
|
||||
};
|
||||
|
||||
// responseChecksumValidation: "WHEN_REQUIRED" — the AWS SDK v3 defaults to
|
||||
// "WHEN_SUPPORTED", which validates any response carrying an x-amz-checksum-*
|
||||
// header. Garage sends x-amz-checksum-crc32 on GetObject but its value doesn't
|
||||
// match what the SDK recomputes over the body, so every direct GetObjectCommand
|
||||
// (e.g. getObjectStream() below, used by the document-to-Markdown converter)
|
||||
// threw "Checksum mismatch" even though the bytes themselves were fine.
|
||||
// WHEN_REQUIRED skips validation unless the operation mandates it (GetObject
|
||||
// never does) — safe here since Garage isn't computing/verifying per AWS spec
|
||||
// anyway.
|
||||
const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED" };
|
||||
|
||||
// Internal client — uploads, deletes, direct streams from the server itself.
|
||||
// Always targets S3_ENDPOINT. When this backend is co-located with Garage,
|
||||
// that's a local address; when it isn't, S3_ENDPOINT must instead be a
|
||||
@@ -45,6 +56,7 @@ const s3 = new S3Client({
|
||||
region: process.env.S3_REGION || "garage",
|
||||
credentials,
|
||||
forcePathStyle: true,
|
||||
...CHECKSUM_CONFIG,
|
||||
});
|
||||
|
||||
const DEFAULT_BUCKET = process.env.S3_BUCKET;
|
||||
@@ -76,6 +88,7 @@ function getPublicClient() {
|
||||
region: process.env.S3_REGION || "garage",
|
||||
credentials,
|
||||
forcePathStyle: true,
|
||||
...CHECKSUM_CONFIG,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user