document to markdown

Signed-off-by: Kenneth Obsequio <k80308392@gmail.com>

services/documentConversion.service.js

@@ -0,0 +1,135 @@
+// services/documentConversion.service.js
+//
+// PDF/PPTX -> Markdown via the MarkItDown sidecar (chibistar/markitdown-service),
+// which does the actual parsing + Markdown generation. That's why this file no
+// longer builds or walks an AST like the old officeparser-based version did —
+// the sidecar already returns a Markdown string, so there's nothing left to
+// serialize on this side.
+//
+// Three named stages, called in sequence by the controller (unchanged from
+// before this swap — the controller and frontend don't know or care that the
+// extraction backend changed):
+//   compile(buffer, extension)  -> call the sidecar, get back markdown + a text length
+//   validate(result)            -> confirm there's usable text before generating anything
+//   automate(result)            -> light cleanup (frontmatter strip) of the sidecar's markdown
+//
+// Environment variables expected:
+//   MARKITDOWN_BASE_URL      — base URL of the markitdown-service sidecar (see chibistar/docker-compose.yml)
+//   MARKITDOWN_SHARED_SECRET — sent as X-Shared-Secret; the sidecar is reachable
+//                              publicly (markitdown.starr.philpro.orij.space), not
+//                              just over loopback/the Docker network, so this is
+//                              required in every environment, dev included.
+
+const axios = require("axios");
+
+const MIN_EXTRACTED_TEXT_LENGTH = 20;
+
+// Backstop only — the controller's own AbortSignal (tied to its
+// CONVERT_TIMEOUT_MS) is what actually bounds this request in practice. This
+// exists so a hung sidecar can't wedge the request open indefinitely if that
+// signal somehow never fires. MarkItDown does plain CPU-bound parsing (no ML
+// inference), so this is a modest timeout, not the generous one a
+// Docling-backed sidecar would need.
+const MARKITDOWN_REQUEST_TIMEOUT_MS = 45_000;
+
+const BASE_URL = (process.env.MARKITDOWN_BASE_URL || "").replace(/\/$/, "");
+const SHARED_SECRET = process.env.MARKITDOWN_SHARED_SECRET || "";
+
+const markitdown = axios.create({
+  baseURL: BASE_URL,
+  timeout: MARKITDOWN_REQUEST_TIMEOUT_MS,
+  maxBodyLength: Infinity,
+  maxContentLength: Infinity,
+});
+
+// ─── compile ───────────────────────────────────────────────────────────────
+//
+// PPTX is the more failure-prone input than PDF — OOXML slide decks exported
+// by different PowerPoint/export-pipeline versions vary more than PDFs do —
+// so PPTX failures get an explicit, format-aware message instead of a raw
+// parser error. Same convention the sidecar itself follows for its own 422s.
+//
+async function compile(buffer, extension, { signal } = {}) {
+  const fileType = (extension || "").toLowerCase();
+
+  if (!BASE_URL) {
+    throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_BASE_URL missing)."), { status: 500, stage: "compile" });
+  }
+  if (!SHARED_SECRET) {
+    throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_SHARED_SECRET missing)."), { status: 500, stage: "compile" });
+  }
+
+  try {
+    const res = await markitdown.post("/convert", buffer, {
+      headers: {
+        "Content-Type": "application/octet-stream",
+        "X-File-Extension": fileType,
+        "X-Shared-Secret": SHARED_SECRET,
+      },
+      signal,
+    });
+    return res.data; // { markdown, textLength, warnings }
+  } catch (err) {
+    // The sidecar reached us and rejected the file (422 bad/corrupt doc, 503
+    // over its concurrency limit, etc.) — surface its own message as-is.
+    if (err.response) {
+      throw Object.assign(new Error(err.response.data?.detail || "Document conversion failed."), {
+        status: err.response.status,
+        stage: "compile",
+        cause: err,
+      });
+    }
+
+    // The sidecar never responded at all (down, unreachable, or the
+    // controller's/our own timeout fired) — fall back to a friendly,
+    // format-aware message since there's no sidecar-provided one to use.
+    const friendly = fileType === "pptx"
+      ? "Couldn't read this PowerPoint file. It may be corrupted, password-protected, or saved in a format this parser doesn't support — try re-exporting it from PowerPoint."
+      : "Couldn't read this PDF. It may be corrupted, password-protected, or scanned/image-only with no embedded text.";
+
+    const timedOut = axios.isCancel(err) || err.code === "ECONNABORTED";
+    throw Object.assign(new Error(friendly), { status: timedOut ? 504 : 502, stage: "compile", cause: err });
+  }
+}
+
+// ─── validate ──────────────────────────────────────────────────────────────
+//
+// Confirms the sidecar actually found usable text before anything is
+// generated. A document with no extractable text (e.g. a slide deck that's
+// entirely images/screenshots) is a hard stop here — OCR is out of scope, so
+// there's no fallback to offer, and drafting an empty block would just be
+// confusing. The length itself is computed sidecar-side — nothing here
+// re-walks a tree anymore.
+//
+function validate(result) {
+  const warnings = result.warnings || [];
+  const extractedLength = result.textLength || 0;
+
+  if (extractedLength < MIN_EXTRACTED_TEXT_LENGTH) {
+    throw Object.assign(
+      new Error("No readable text found in this document. This tool only extracts text (including hyperlinks) — image-only or scanned documents aren't supported."),
+      { status: 422, stage: "validate" },
+    );
+  }
+
+  return { warnings, stats: { extractedLength } };
+}
+
+// ─── automate ──────────────────────────────────────────────────────────────
+//
+// MarkItDown already emits Markdown — this is just the same
+// frontmatter-stripping cleanup the officeparser path used to need, kept
+// as-is since some exporters still prepend a properties block.
+//
+function stripFrontmatter(markdown) {
+  return markdown.replace(/^---\n[\s\S]*?\n---\n+/, "");
+}
+
+async function automate(result) {
+  return {
+    markdown: stripFrontmatter(result.markdown || "").trim(),
+    messages: result.warnings || [],
+  };
+}
+
+module.exports = { compile, validate, automate };

services/s3.service.js

@@ -35,6 +35,17 @@ const credentials = {
   secretAccessKey: process.env.S3_SECRET_KEY,
 };
 
+// responseChecksumValidation: "WHEN_REQUIRED" — the AWS SDK v3 defaults to
+// "WHEN_SUPPORTED", which validates any response carrying an x-amz-checksum-*
+// header. Garage sends x-amz-checksum-crc32 on GetObject but its value doesn't
+// match what the SDK recomputes over the body, so every direct GetObjectCommand
+// (e.g. getObjectStream() below, used by the document-to-Markdown converter)
+// threw "Checksum mismatch" even though the bytes themselves were fine.
+// WHEN_REQUIRED skips validation unless the operation mandates it (GetObject
+// never does) — safe here since Garage isn't computing/verifying per AWS spec
+// anyway.
+const CHECKSUM_CONFIG = { responseChecksumValidation: "WHEN_REQUIRED" };
+
 // Internal client — uploads, deletes, direct streams from the server itself.
 // Always targets S3_ENDPOINT. When this backend is co-located with Garage,
 // that's a local address; when it isn't, S3_ENDPOINT must instead be a
@@ -45,6 +56,7 @@ const s3 = new S3Client({
   region:         process.env.S3_REGION || "garage",
   credentials,
   forcePathStyle: true,
+  ...CHECKSUM_CONFIG,
 });
 
 const DEFAULT_BUCKET = process.env.S3_BUCKET;
@@ -76,6 +88,7 @@ function getPublicClient() {
     region:         process.env.S3_REGION || "garage",
     credentials,
     forcePathStyle: true,
+    ...CHECKSUM_CONFIG,
   });
 }