kennethscoffee/starr-philproperties
1
0
Fork 0
mirror of https://github.com/rgrgogu/new_starr.git synced 2026-09-27 00:12:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

added and fix some of things

Browse Source 
Signed-off-by: Kenneth Obsequio <k80308392@gmail.com>
This commit is contained in:
kennethobsequio
2026-08-03 12:03:02 +08:00
parent 6765c1faba
commit d246c09cdd
31 changed files with 724 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/documentConversion.service.js
-135
View File
@@ -1,135 +0,0 @@
// services/documentConversion.service.js
//
// PDF/PPTX -> Markdown via the MarkItDown sidecar (chibistar/markitdown-service),
// which does the actual parsing + Markdown generation. That's why this file no
// longer builds or walks an AST like the old officeparser-based version did —
// the sidecar already returns a Markdown string, so there's nothing left to
// serialize on this side.
//
// Three named stages, called in sequence by the controller (unchanged from
// before this swap — the controller and frontend don't know or care that the
// extraction backend changed):
// compile(buffer, extension) -> call the sidecar, get back markdown + a text length
// validate(result) -> confirm there's usable text before generating anything
// automate(result) -> light cleanup (frontmatter strip) of the sidecar's markdown
//
// Environment variables expected:
// MARKITDOWN_BASE_URL — base URL of the markitdown-service sidecar (see chibistar/docker-compose.yml)
// MARKITDOWN_SHARED_SECRET — sent as X-Shared-Secret; the sidecar is reachable
// publicly (markitdown.starr.philpro.orij.space), not
// just over loopback/the Docker network, so this is
// required in every environment, dev included.
const axios = require("axios");
const MIN_EXTRACTED_TEXT_LENGTH = 20;
// Backstop only — the controller's own AbortSignal (tied to its
// CONVERT_TIMEOUT_MS) is what actually bounds this request in practice. This
// exists so a hung sidecar can't wedge the request open indefinitely if that
// signal somehow never fires. MarkItDown does plain CPU-bound parsing (no ML
// inference), so this is a modest timeout, not the generous one a
// Docling-backed sidecar would need.
const MARKITDOWN_REQUEST_TIMEOUT_MS = 45_000;
const BASE_URL = (process.env.MARKITDOWN_BASE_URL || "").replace(/\/$/, "");
const SHARED_SECRET = process.env.MARKITDOWN_SHARED_SECRET || "";
const markitdown = axios.create({
baseURL: BASE_URL,
timeout: MARKITDOWN_REQUEST_TIMEOUT_MS,
maxBodyLength: Infinity,
maxContentLength: Infinity,
});
// ─── compile ───────────────────────────────────────────────────────────────
//
// PPTX is the more failure-prone input than PDF — OOXML slide decks exported
// by different PowerPoint/export-pipeline versions vary more than PDFs do —
// so PPTX failures get an explicit, format-aware message instead of a raw
// parser error. Same convention the sidecar itself follows for its own 422s.
//
async function compile(buffer, extension, { signal } = {}) {
const fileType = (extension || "").toLowerCase();
if (!BASE_URL) {
throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_BASE_URL missing)."), { status: 500, stage: "compile" });
}
if (!SHARED_SECRET) {
throw Object.assign(new Error("Document conversion is not configured (MARKITDOWN_SHARED_SECRET missing)."), { status: 500, stage: "compile" });
}
try {
const res = await markitdown.post("/convert", buffer, {
headers: {
"Content-Type": "application/octet-stream",
"X-File-Extension": fileType,
"X-Shared-Secret": SHARED_SECRET,
},
signal,
});
return res.data; // { markdown, textLength, warnings }
} catch (err) {
// The sidecar reached us and rejected the file (422 bad/corrupt doc, 503
// over its concurrency limit, etc.) — surface its own message as-is.
if (err.response) {
throw Object.assign(new Error(err.response.data?.detail || "Document conversion failed."), {
status: err.response.status,
stage: "compile",
cause: err,
});
}
// The sidecar never responded at all (down, unreachable, or the
// controller's/our own timeout fired) — fall back to a friendly,
// format-aware message since there's no sidecar-provided one to use.
const friendly = fileType === "pptx"
? "Couldn't read this PowerPoint file. It may be corrupted, password-protected, or saved in a format this parser doesn't support — try re-exporting it from PowerPoint."
: "Couldn't read this PDF. It may be corrupted, password-protected, or scanned/image-only with no embedded text.";
const timedOut = axios.isCancel(err) || err.code === "ECONNABORTED";
throw Object.assign(new Error(friendly), { status: timedOut ? 504 : 502, stage: "compile", cause: err });
}
}
// ─── validate ──────────────────────────────────────────────────────────────
//
// Confirms the sidecar actually found usable text before anything is
// generated. A document with no extractable text (e.g. a slide deck that's
// entirely images/screenshots) is a hard stop here — OCR is out of scope, so
// there's no fallback to offer, and drafting an empty block would just be
// confusing. The length itself is computed sidecar-side — nothing here
// re-walks a tree anymore.
//
function validate(result) {
const warnings = result.warnings || [];
const extractedLength = result.textLength || 0;
if (extractedLength < MIN_EXTRACTED_TEXT_LENGTH) {
throw Object.assign(
new Error("No readable text found in this document. This tool only extracts text (including hyperlinks) — image-only or scanned documents aren't supported."),
{ status: 422, stage: "validate" },
);
}
return { warnings, stats: { extractedLength } };
}
// ─── automate ──────────────────────────────────────────────────────────────
//
// MarkItDown already emits Markdown — this is just the same
// frontmatter-stripping cleanup the officeparser path used to need, kept
// as-is since some exporters still prepend a properties block.
//
function stripFrontmatter(markdown) {
return markdown.replace(/^---\n[\s\S]*?\n---\n+/, "");
}
async function automate(result) {
return {
markdown: stripFrontmatter(result.markdown || "").trim(),
messages: result.warnings || [],
};
}
module.exports = { compile, validate, automate };
services/ffmpeg.service.js
+49 -1
View File
@@ -99,4 +99,52 @@ function remuxToFaststartMp4(inputUrl, { timeoutMs = 30 * 60 * 1000 } = {}) {
});
}
module.exports = { needsRemux, remuxToFaststartMp4 };
// Grabs a single frame from a video as a JPEG thumbnail — used when a video
// asset lands with no client-provided thumbnail (see finalizeAssetFromStorage
// in assets.controller.js). Unlike remuxToFaststartMp4 above, this is cheap
// enough (one frame, not a full re-encode/copy) to run inline during the
// upload-finalize request rather than as a background job.
//
// input: presigned GET URL for the video, + its duration (seconds, from
// ffprobe) to pick a safe seek point.
// output: local filesystem path to the extracted .jpg (caller owns cleanup).
function extractFrameThumbnail(inputUrl, duration, { timeoutMs = 30 * 1000 } = {}) {
// Seek 1s in, or the midpoint for clips shorter than ~2s — avoids grabbing
// frame 0, which is often black/blank on screen recordings and slates.
const atSeconds = !duration || duration <= 0 ? 0 : Math.min(1, duration / 2);
const outputPath = path.join(os.tmpdir(), `thumb_${Date.now()}_${crypto.randomBytes(4).toString("hex")}.jpg`);
return new Promise((resolve, reject) => {
let settled = false;
const command = ffmpeg(inputUrl)
.seekInput(atSeconds) // input-side seek — fast, keyframe-based
.outputOptions(["-frames:v 1", "-q:v 2"]);
const timer = setTimeout(() => {
if (settled) return;
settled = true;
command.kill("SIGKILL");
fs.promises.unlink(outputPath).catch(() => {});
reject(new Error(`Thumbnail extraction timed out after ${timeoutMs}ms`));
}, timeoutMs);
command
.on("error", (err) => {
if (settled) return;
settled = true;
clearTimeout(timer);
fs.promises.unlink(outputPath).catch(() => {});
reject(err);
})
.on("end", () => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve(outputPath);
})
.save(outputPath);
});
}
module.exports = { needsRemux, remuxToFaststartMp4, extractFrameThumbnail };
services/uploadProgress.service.js
-52
View File
@@ -1,52 +0,0 @@
/***********************************************************************************************************************************************************************
* File Name: uploadProgress.service.js
* Type of Program: Service
* Description: In-memory SSE broadcaster for real upload progress on the
* Express -> S3 (Garage) leg of an asset upload. Keyed by a
* client-generated uploadId so the browser can open the stream
* before the upload request itself is even sent.
*
* No Redis — single-process only, same tradeoff already made by
* mediaToken.service.js's in-memory token cache. Fine for one
* instance; a second app instance would just never see progress
* for uploads routed to the other process.
*
* Author: Kenneth Obsequio (@lash0000)
***********************************************************************************************************************************************************************/
"use strict";
const clients = new Map(); // uploadId -> Response
function subscribe(uploadId, res) {
res.writeHead(200, {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache, no-transform",
Connection: "keep-alive",
"X-Accel-Buffering": "no", // disable proxy-side buffering (nginx-style intermediaries)
});
res.write(": connected\n\n");
clients.set(uploadId, res);
res.on("close", () => {
if (clients.get(uploadId) === res) clients.delete(uploadId);
});
}
// No-ops if nobody's subscribed (client never opened the stream, or already
// disconnected) — progress is a best-effort visual, never load-bearing for
// the actual upload.
function publish(uploadId, data) {
const res = clients.get(uploadId);
if (!res) return;
res.write(`data: ${JSON.stringify(data)}\n\n`);
}
function complete(uploadId, data = {}) {
const res = clients.get(uploadId);
if (!res) return;
res.write(`data: ${JSON.stringify({ ...data, done: true })}\n\n`);
res.end();
clients.delete(uploadId);
}
module.exports = { subscribe, publish, complete };