starr-philproperties/services/documentConversion.service.js

// services/documentConversion.service.js
//
// PDF/PPTX -> Markdown via the MarkItDown sidecar (chibistar/markitdown-service),
// which does the actual parsing + Markdown generation. That's why this file no
// longer builds or walks an AST like the old officeparser-based version did —
// the sidecar already returns a Markdown string, so there's nothing left to
// serialize on this side.
//
// Three named stages, called in sequence by the controller (unchanged from
// before this swap — the controller and frontend don't know or care that the
// extraction backend changed):
//   compile(buffer, extension)  -> call the sidecar, get back markdown + a text length
//   validate(result)            -> confirm there's usable text before generating anything
//   automate(result)            -> light cleanup (frontmatter strip) of the sidecar's markdown
//
// Environment variables expected:
//   MARKITDOWN_BASE_URL      — base URL of the markitdown-service sidecar (see chibistar/docker-compose.yml)
//   MARKITDOWN_SHARED_SECRET — sent as X-Shared-Secret; the sidecar is reachable
//                              publicly (markitdown.starr.philpro.orij.space), not
//                              just over loopback/the Docker network, so this is
//                              required in every environment, dev included.

const axios = require("axios");

const MIN_EXTRACTED_TEXT_LENGTH = 20;

// Backstop only — the controller's own AbortSignal (tied to its
// CONVERT_TIMEOUT_MS) is what actually bounds this request in practice. This
// exists so a hung sidecar can't wedge the request open indefinitely if that
// signal somehow never fires. MarkItDown does plain CPU-bound parsing (no ML
// inference), so this is a modest timeout, not the generous one a
// Docling-backed sidecar would need.
const MARKITDOWN_REQUEST_TIMEOUT_MS = 45_000;

const BASE_URL = (process.env.MARKITDOWN_BASE_URL || "").replace(/\/$/, "");
const SHARED_SECRET = process.env.MARKITDOWN_SHARED_SECRET || "";

const markitdown = axios.create({
  baseURL: BASE_URL,
  timeout: MARKITDOWN_REQUEST_TIMEOUT_MS,
  maxBodyLength: Infinity,
  maxContentLength: Infinity,
});

// ─── compile ───────────────────────────────────────────────────────────────
//
// PPTX is the more failure-prone input than PDF — OOXML slide decks exported
// by different PowerPoint/export-pipeline versions vary more than PDFs do —
// so PPTX failures get an explicit, format-aware message instead of a raw
// parser error. Same convention the sidecar itself follows for its own 422s.
//
async function compile(buffer, extension, { signal } = {}) {
  const fileType = (extension || "").toLowerCase();

  if (!BASE_URL) {
    throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_BASE_URL missing)."), { status: 500, stage: "compile" });
  }
  if (!SHARED_SECRET) {
    throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_SHARED_SECRET missing)."), { status: 500, stage: "compile" });
  }

  try {
    const res = await markitdown.post("/convert", buffer, {
      headers: {
        "Content-Type": "application/octet-stream",
        "X-File-Extension": fileType,
        "X-Shared-Secret": SHARED_SECRET,
      },
      signal,
    });
    return res.data; // { markdown, textLength, warnings }
  } catch (err) {
    // The sidecar reached us and rejected the file (422 bad/corrupt doc, 503
    // over its concurrency limit, etc.) — surface its own message as-is.
    if (err.response) {
      throw Object.assign(new Error(err.response.data?.detail || "Document conversion failed."), {
        status: err.response.status,
        stage: "compile",
        cause: err,
      });
    }

    // The sidecar never responded at all (down, unreachable, or the
    // controller's/our own timeout fired) — fall back to a friendly,
    // format-aware message since there's no sidecar-provided one to use.
    const friendly = fileType === "pptx"
      ? "Couldn't read this PowerPoint file. It may be corrupted, password-protected, or saved in a format this parser doesn't support — try re-exporting it from PowerPoint."
      : "Couldn't read this PDF. It may be corrupted, password-protected, or scanned/image-only with no embedded text.";

    const timedOut = axios.isCancel(err) || err.code === "ECONNABORTED";
    throw Object.assign(new Error(friendly), { status: timedOut ? 504 : 502, stage: "compile", cause: err });
  }
}

// ─── validate ──────────────────────────────────────────────────────────────
//
// Confirms the sidecar actually found usable text before anything is
// generated. A document with no extractable text (e.g. a slide deck that's
// entirely images/screenshots) is a hard stop here — OCR is out of scope, so
// there's no fallback to offer, and drafting an empty block would just be
// confusing. The length itself is computed sidecar-side — nothing here
// re-walks a tree anymore.
//
function validate(result) {
  const warnings = result.warnings || [];
  const extractedLength = result.textLength || 0;

  if (extractedLength < MIN_EXTRACTED_TEXT_LENGTH) {
    throw Object.assign(
      new Error("No readable text found in this document. This tool only extracts text (including hyperlinks) — image-only or scanned documents aren't supported."),
      { status: 422, stage: "validate" },
    );
  }

  return { warnings, stats: { extractedLength } };
}

// ─── automate ──────────────────────────────────────────────────────────────
//
// MarkItDown already emits Markdown — this is just the same
// frontmatter-stripping cleanup the officeparser path used to need, kept
// as-is since some exporters still prepend a properties block.
//
function stripFrontmatter(markdown) {
  return markdown.replace(/^---\n[\s\S]*?\n---\n+/, "");
}

async function automate(result) {
  return {
    markdown: stripFrontmatter(result.markdown || "").trim(),
    messages: result.warnings || [],
  };
}

module.exports = { compile, validate, automate };