// services/documentConversion.service.js // // PDF/PPTX -> Markdown via the MarkItDown sidecar (chibistar/markitdown-service), // which does the actual parsing + Markdown generation. That's why this file no // longer builds or walks an AST like the old officeparser-based version did — // the sidecar already returns a Markdown string, so there's nothing left to // serialize on this side. // // Three named stages, called in sequence by the controller (unchanged from // before this swap — the controller and frontend don't know or care that the // extraction backend changed): // compile(buffer, extension) -> call the sidecar, get back markdown + a text length // validate(result) -> confirm there's usable text before generating anything // automate(result) -> light cleanup (frontmatter strip) of the sidecar's markdown // // Environment variables expected: // MARKITDOWN_BASE_URL — base URL of the markitdown-service sidecar (see chibistar/docker-compose.yml) // MARKITDOWN_SHARED_SECRET — sent as X-Shared-Secret; the sidecar is reachable // publicly (markitdown.starr.philpro.orij.space), not // just over loopback/the Docker network, so this is // required in every environment, dev included. const axios = require("axios"); const MIN_EXTRACTED_TEXT_LENGTH = 20; // Backstop only — the controller's own AbortSignal (tied to its // CONVERT_TIMEOUT_MS) is what actually bounds this request in practice. This // exists so a hung sidecar can't wedge the request open indefinitely if that // signal somehow never fires. MarkItDown does plain CPU-bound parsing (no ML // inference), so this is a modest timeout, not the generous one a // Docling-backed sidecar would need. const MARKITDOWN_REQUEST_TIMEOUT_MS = 45_000; const BASE_URL = (process.env.MARKITDOWN_BASE_URL || "").replace(/\/$/, ""); const SHARED_SECRET = process.env.MARKITDOWN_SHARED_SECRET || ""; const markitdown = axios.create({ baseURL: BASE_URL, timeout: MARKITDOWN_REQUEST_TIMEOUT_MS, maxBodyLength: Infinity, maxContentLength: Infinity, }); // ─── compile ─────────────────────────────────────────────────────────────── // // PPTX is the more failure-prone input than PDF — OOXML slide decks exported // by different PowerPoint/export-pipeline versions vary more than PDFs do — // so PPTX failures get an explicit, format-aware message instead of a raw // parser error. Same convention the sidecar itself follows for its own 422s. // async function compile(buffer, extension, { signal } = {}) { const fileType = (extension || "").toLowerCase(); if (!BASE_URL) { throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_BASE_URL missing)."), { status: 500, stage: "compile" }); } if (!SHARED_SECRET) { throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_SHARED_SECRET missing)."), { status: 500, stage: "compile" }); } try { const res = await markitdown.post("/convert", buffer, { headers: { "Content-Type": "application/octet-stream", "X-File-Extension": fileType, "X-Shared-Secret": SHARED_SECRET, }, signal, }); return res.data; // { markdown, textLength, warnings } } catch (err) { // The sidecar reached us and rejected the file (422 bad/corrupt doc, 503 // over its concurrency limit, etc.) — surface its own message as-is. if (err.response) { throw Object.assign(new Error(err.response.data?.detail || "Document conversion failed."), { status: err.response.status, stage: "compile", cause: err, }); } // The sidecar never responded at all (down, unreachable, or the // controller's/our own timeout fired) — fall back to a friendly, // format-aware message since there's no sidecar-provided one to use. const friendly = fileType === "pptx" ? "Couldn't read this PowerPoint file. It may be corrupted, password-protected, or saved in a format this parser doesn't support — try re-exporting it from PowerPoint." : "Couldn't read this PDF. It may be corrupted, password-protected, or scanned/image-only with no embedded text."; const timedOut = axios.isCancel(err) || err.code === "ECONNABORTED"; throw Object.assign(new Error(friendly), { status: timedOut ? 504 : 502, stage: "compile", cause: err }); } } // ─── validate ────────────────────────────────────────────────────────────── // // Confirms the sidecar actually found usable text before anything is // generated. A document with no extractable text (e.g. a slide deck that's // entirely images/screenshots) is a hard stop here — OCR is out of scope, so // there's no fallback to offer, and drafting an empty block would just be // confusing. The length itself is computed sidecar-side — nothing here // re-walks a tree anymore. // function validate(result) { const warnings = result.warnings || []; const extractedLength = result.textLength || 0; if (extractedLength < MIN_EXTRACTED_TEXT_LENGTH) { throw Object.assign( new Error("No readable text found in this document. This tool only extracts text (including hyperlinks) — image-only or scanned documents aren't supported."), { status: 422, stage: "validate" }, ); } return { warnings, stats: { extractedLength } }; } // ─── automate ────────────────────────────────────────────────────────────── // // MarkItDown already emits Markdown — this is just the same // frontmatter-stripping cleanup the officeparser path used to need, kept // as-is since some exporters still prepend a properties block. // function stripFrontmatter(markdown) { return markdown.replace(/^---\n[\s\S]*?\n---\n+/, ""); } async function automate(result) { return { markdown: stripFrontmatter(result.markdown || "").trim(), messages: result.warnings || [], }; } module.exports = { compile, validate, automate };