A fast, helpful, and open-source document parser
update output shape
Logan Markewich372b018db674
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rsindex cedf048..77e579a 100644--- a/crates/liteparse-napi/src/types.rs+++ b/crates/liteparse-napi/src/types.rs@@ -368,10 +368,12 @@ pub struct JsPageComplexityStats {pub image_block_count: u32,pub image_coverage: f64,pub largest_image_coverage: f64,+ pub full_page_image: bool,pub uncovered_vector_area: Option<f64>,pub is_garbled: bool,pub page_area: f64,pub needs_ocr: bool,+ pub reasons: Vec<String>,}impl JsPageComplexityStats {@@ -384,10 +386,16 @@ impl JsPageComplexityStats {image_block_count: stats.image_block_count as u32,image_coverage: stats.image_coverage as f64,largest_image_coverage: stats.largest_image_coverage as f64,+ full_page_image: stats.full_page_image,uncovered_vector_area: stats.uncovered_vector_area.map(|v| v as f64),is_garbled: stats.is_garbled,page_area: stats.page_area as f64,needs_ocr: stats.needs_ocr,+ reasons: stats+ .reasons+ .iter()+ .map(|r| r.as_str().to_string())+ .collect(),}}}diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rsindex 26985c3..2b0632e 100644--- a/crates/liteparse-python/src/lib.rs+++ b/crates/liteparse-python/src/lib.rs@@ -252,6 +252,8 @@ struct PyPageComplexityStats {#[pyo3(get)]largest_image_coverage: f32,#[pyo3(get)]+ full_page_image: bool,+ #[pyo3(get)]uncovered_vector_area: Option<f32>,#[pyo3(get)]is_garbled: bool,@@ -259,6 +261,8 @@ struct PyPageComplexityStats {page_area: f32,#[pyo3(get)]needs_ocr: bool,+ #[pyo3(get)]+ reasons: Vec<String>,}#[pymethods]@@ -281,10 +285,16 @@ impl PyPageComplexityStats {image_block_count: stats.image_block_count,image_coverage: stats.image_coverage,largest_image_coverage: stats.largest_image_coverage,+ full_page_image: stats.full_page_image,uncovered_vector_area: stats.uncovered_vector_area,is_garbled: stats.is_garbled,page_area: stats.page_area,needs_ocr: stats.needs_ocr,+ reasons: stats+ .reasons+ .iter()+ .map(|r| r.as_str().to_string())+ .collect(),}}}diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rsindex fc26dca..2733b04 100644--- a/crates/liteparse-wasm/src/lib.rs+++ b/crates/liteparse-wasm/src/lib.rs@@ -425,10 +425,12 @@ struct JsPageComplexityStats {image_block_count: usize,image_coverage: f32,largest_image_coverage: f32,+ full_page_image: bool,uncovered_vector_area: Option<f32>,is_garbled: bool,page_area: f32,needs_ocr: bool,+ reasons: Vec<String>,}impl JsPageComplexityStats {@@ -441,10 +443,16 @@ impl JsPageComplexityStats {image_block_count: stats.image_block_count,image_coverage: stats.image_coverage,largest_image_coverage: stats.largest_image_coverage,+ full_page_image: stats.full_page_image,uncovered_vector_area: stats.uncovered_vector_area,is_garbled: stats.is_garbled,page_area: stats.page_area,needs_ocr: stats.needs_ocr,+ reasons: stats+ .reasons+ .iter()+ .map(|r| r.as_str().to_string())+ .collect(),}}}diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rsindex 1652b23..f58ab3e 100644--- a/crates/liteparse/src/main.rs+++ b/crates/liteparse/src/main.rs@@ -225,9 +225,10 @@ struct IsComplexCommand {/// Input file pathfile: String,- /// Whether to output the result as JSON (true/false) or plain text (true/false)+ /// Emit dense, whitespace-free JSON instead of pretty-printed (still valid+ /// for `jq` and friends).#[arg(long)]- json: bool,+ compact: bool,/// Max pages to parse#[arg(long, default_value = "1000")]@@ -517,46 +518,31 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {let complex_pages = is_complex.iter().filter(|c| c.needs_ocr).count();- if cmd.json && !cmd.quiet {- let json = serde_json::to_string(&is_complex)?;- println!("{}", json);- } else if !cmd.quiet {+ // Always emit JSON on stdout so the command composes with `jq` and+ // friends without a flag. Pretty by default for human readability;+ // `--compact` drops the whitespace. Both parse identically.+ let json = if cmd.compact {+ serde_json::to_string(&is_complex)?+ } else {+ serde_json::to_string_pretty(&is_complex)?+ };+ println!("{}", json);++ // The human-readable verdict goes to stderr so it never pollutes the+ // JSON on stdout. The exit code below carries the same signal for+ // scripts that don't want to read either stream.+ if !cmd.quiet {let verdict = if complex_pages > 0 {"COMPLEX"} else {"SIMPLE"};- println!(+ eprintln!("{} — {}/{} page(s) need OCR",verdict,complex_pages,is_complex.len());- println!(- "{:>4} {:>7} {:>5} {:>6} {:>6} {:>7} {:>7} {:>5}",- "page", "text", "cov", "images", "imgcov", "garbled", "vector", "ocr"- );- for c in &is_complex {- let vector = match c.uncovered_vector_area {- Some(area) => format!("{:.0}", area),- None => "-".to_string(),- };- println!(- "{:>4} {:>7} {:>5.2} {:>6} {:>6.2} {:>7} {:>7} {:>5}",- c.page_number,- c.text_length,- c.text_coverage,- if c.has_substantial_images {- "yes"- } else {- "no"- },- c.image_coverage,- if c.is_garbled { "yes" } else { "no" },- vector,- if c.needs_ocr { "yes" } else { "no" },- );- }}// Exit non-zero when any page needs OCR, so the command is usable asdiff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rsindex 3d3534f..31ea997 100644--- a/crates/liteparse/src/ocr_merge.rs+++ b/crates/liteparse/src/ocr_merge.rs@@ -31,6 +31,49 @@ pub(crate) struct RenderedPage {pub height: u32,}+/// Why a page was flagged as needing more than the cheap text-only path.+/// Multiple reasons can apply to one page (e.g. a sparse page whose little+/// text is also garbled). Empty exactly when `needs_ocr` is false.+///+/// This is the discriminator a caller routes on: a scan goes to OCR, dense+/// vector text to a vector-aware pass, and so on. New variants will be added+/// as the routing function learns to recommend heavier pipelines (tables,+/// charts, LLM passes), so callers should treat unknown variants leniently.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+#[serde(rename_all = "kebab-case")]+pub enum ComplexityReason {+ /// A single raster covers essentially the whole page and there is little or+ /// no extractable text behind it — a scanned/photographed page.+ Scanned,+ /// Almost no extractable native text, and no full-page raster behind it+ /// (a blank page, or a near-empty cover/divider).+ NoText,+ /// Some real text, but it covers very little of the page — typically a+ /// figure-heavy page with only thin captions.+ SparseText,+ /// Substantial embedded raster figures sit alongside the native text.+ EmbeddedImages,+ /// The native text decodes to garbage (broken cmap / Type3 char-code+ /// fallback), so the visible glyphs and the extracted text disagree.+ Garbled,+ /// Text is painted as filled vector outlines, outside the text layer, so+ /// no native text items represent it.+ VectorText,+}++impl ComplexityReason {+ pub fn as_str(&self) -> &'static str {+ match self {+ ComplexityReason::Scanned => "scanned",+ ComplexityReason::NoText => "no-text",+ ComplexityReason::SparseText => "sparse-text",+ ComplexityReason::EmbeddedImages => "embedded-images",+ ComplexityReason::Garbled => "garbled",+ ComplexityReason::VectorText => "vector-text",+ }+ }+}+#[derive(Debug, Serialize)]pub struct PageComplexityStats {pub page_number: usize,@@ -49,13 +92,24 @@ pub struct PageComplexityStats {/// 1.0. Useful for telling a single full-bleed scan apart from many small/// inline figures that sum to a similar `image_coverage`.pub largest_image_coverage: f32,+ /// A single raster covering ≥90% of the page is present. Such full-page+ /// backgrounds are excluded from `image_coverage`/`largest_image_coverage`+ /// (they're not inline figures), so this flag is the only signal that+ /// distinguishes a scan from a genuinely blank page — both otherwise report+ /// no text and no counted images.+ pub full_page_image: bool,/// Filled vector-outline area not covered by native text, in pt². `None`/// when a cheaper predicate already flagged the page for OCR, so this/// expensive page-object walk was skipped (it wasn't the deciding signal).pub uncovered_vector_area: Option<f32>,pub is_garbled: bool,pub page_area: f32,+ /// Whether the page needs more than the cheap text-only path. Equivalent to+ /// `!reasons.is_empty()`; kept as a flat bool for the common predicate case+ /// and as the internal per-page OCR gate.pub needs_ocr: bool,+ /// Every reason the page was flagged, in no particular priority order.+ pub reasons: Vec<ComplexityReason>,}pub(crate) fn calculate_page_complexity(@@ -75,10 +129,21 @@ pub(crate) fn calculate_page_complexity(.filter(|item| !is_unusable_native(item)).map(|item| item.text.len()).sum();- let image_bounds = page_obj.image_bounds(MIN_IMAGE_SIZE_PT, MAX_IMAGE_PAGE_COVERAGE);+ // Collect every raster ≥ MIN_IMAGE_SIZE_PT, including full-page ones, so a+ // scan can be told apart from a blank page. The "counted" subset below then+ // drops full-page backgrounds, matching the old+ // `image_bounds(.., MAX_IMAGE_PAGE_COVERAGE)` behaviour for inline figures.+ let pw = page.page_width;+ let ph = page.page_height;+ let all_images = page_obj.image_bounds(MIN_IMAGE_SIZE_PT, f32::INFINITY);+ let is_full_page = |b: &ImageBounds| {+ b.width > pw * MAX_IMAGE_PAGE_COVERAGE && b.height > ph * MAX_IMAGE_PAGE_COVERAGE+ };+ let full_page_image = all_images.iter().any(is_full_page);+ let image_bounds: Vec<&ImageBounds> = all_images.iter().filter(|b| !is_full_page(b)).collect();let has_images = !image_bounds.is_empty();- let page_area = page.page_width * page.page_height;+ let page_area = pw * ph;let (image_area_sum, largest_image_area) =image_bounds@@ -112,7 +177,27 @@ pub(crate) fn calculate_page_complexity(// wide intra-cell whitespace) is spatially sparse but needs no OCR.let sparse_text = text_length < 2000 && text_coverage < 0.15;let is_garbled = page_is_garbled(page);- let mut needs_ocr = text_length < 20 || sparse_text || has_images || is_garbled;++ let mut reasons = Vec::new();+ if text_length < 20 {+ // Too little text to be the page's content. A full-page raster behind+ // it means a scan; otherwise it's effectively blank.+ reasons.push(if full_page_image {+ ComplexityReason::Scanned+ } else {+ ComplexityReason::NoText+ });+ } else if sparse_text {+ // There is real text, but it's too thin to be the whole page.+ reasons.push(ComplexityReason::SparseText);+ }+ if has_images {+ reasons.push(ComplexityReason::EmbeddedImages);+ }+ if is_garbled {+ reasons.push(ComplexityReason::Garbled);+ }+ let mut needs_ocr = !reasons.is_empty();// Text drawn as filled vector outlines lives outside the text layer// entirely: no text items, no image XObjects, so none of the cheap@@ -123,7 +208,10 @@ pub(crate) fn calculate_page_complexity(let uncovered_vector_area = if !needs_ocr {let path_bounds = page_obj.filled_path_bounds(3.0, 0.9);let uncovered = uncovered_path_area(&path_bounds, &page.text_items);- needs_ocr = uncovered >= UNCOVERED_VECTOR_AREA_THRESHOLD;+ if uncovered >= UNCOVERED_VECTOR_AREA_THRESHOLD {+ needs_ocr = true;+ reasons.push(ComplexityReason::VectorText);+ }Some(uncovered)} else {None@@ -137,10 +225,12 @@ pub(crate) fn calculate_page_complexity(image_block_count: image_bounds.len(),image_coverage,largest_image_coverage,+ full_page_image,uncovered_vector_area,is_garbled,page_area,needs_ocr,+ reasons,})}diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.tsindex 91b5f58..3673811 100644--- a/packages/node/src/cli.ts+++ b/packages/node/src/cli.ts@@ -147,7 +147,7 @@ program"Check if a document is 'complex' enough to require OCR or other advanced parsing",).argument("<file>", "Path to the document file")- .option("--json", "Output the result as JSON")+ .option("--compact", "Emit dense, whitespace-free JSON instead of pretty").option("--max-pages <n>", "Max pages to parse", parseInt).option("--target-pages <pages>",@@ -168,36 +168,23 @@ programconst complexPages = stats.filter((s) => s.needsOcr).length;- if (opts.json) {- process.stdout.write(JSON.stringify(stats));- process.stdout.write("\n");- } else {+ // Always emit JSON on stdout so the command composes with `jq` without a+ // flag. Pretty by default; `--compact` drops the whitespace. Both parse+ // identically for any reader.+ process.stdout.write(+ opts.compact+ ? JSON.stringify(stats)+ : JSON.stringify(stats, null, 2),+ );+ process.stdout.write("\n");++ // Human verdict goes to stderr so it never pollutes the JSON on stdout;+ // the exit code below carries the same signal for scripts.+ if (!opts.quiet) {const verdict = complexPages > 0 ? "COMPLEX" : "SIMPLE";- console.log(+ console.error(`${verdict} — ${complexPages}/${stats.length} page(s) need OCR`,);- console.log(- ["page", "text", "cov", "images", "garbled", "vector", "ocr"].join(- "\t",- ),- );- for (const s of stats) {- const vector =- s.uncoveredVectorArea !== undefined- ? s.uncoveredVectorArea.toFixed(0)- : "-";- console.log(- [- s.pageNumber,- s.textLength,- s.textCoverage.toFixed(2),- s.hasSubstantialImages ? "yes" : "no",- s.isGarbled ? "yes" : "no",- vector,- s.needsOcr ? "yes" : "no",- ].join("\t"),- );- }}// Exit non-zero when any page needs OCR, so the command is usable as adiff --git a/packages/node/src/lib.ts b/packages/node/src/lib.tsindex 2af2bad..8f72453 100644--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@ -133,8 +133,14 @@ export interface PageComplexityStats {imageBlockCount: number;/** Summed image-bbox area over page area, clamped to 1. */imageCoverage: number;- /** Largest single image's area over page area, clamped to 1. */+ /** Largest single *counted* image's area over page area, clamped to 1. */largestImageCoverage: number;+ /**+ * A single raster covers ≥90% of the page. Full-page backgrounds are excluded+ * from the image coverage fields, so this is the only signal that tells a scan+ * apart from a blank page — both otherwise report no text and no images.+ */+ fullPageImage: boolean;/*** Filled vector-outline area not covered by native text, in pt². `undefined`* when a cheaper signal already decided the page, so this walk was skipped.@@ -142,8 +148,14 @@ export interface PageComplexityStats {uncoveredVectorArea?: number;isGarbled: boolean;pageArea: number;- /** Verdict: whether this page should be sent through OCR. */+ /** Verdict: whether this page needs more than the cheap text-only path. */needsOcr: boolean;+ /**+ * Every reason the page was flagged (e.g. `"scanned"`, `"sparse-text"`,+ * `"garbled"`). Empty exactly when `needsOcr` is false. This is the value to+ * route on; new reasons may be added over time.+ */+ reasons: string[];}// ---------------------------------------------------------------------------@@ -248,10 +260,12 @@ export class LiteParse {imageBlockCount: s.imageBlockCount,imageCoverage: s.imageCoverage,largestImageCoverage: s.largestImageCoverage,+ fullPageImage: s.fullPageImage,uncoveredVectorArea: s.uncoveredVectorArea ?? undefined,isGarbled: s.isGarbled,pageArea: s.pageArea,needsOcr: s.needsOcr,+ reasons: s.reasons,}));}diff --git a/packages/node/src/native.ts b/packages/node/src/native.tsindex c692838..133ee9e 100644--- a/packages/node/src/native.ts+++ b/packages/node/src/native.ts@@ -110,10 +110,12 @@ export interface NativePageComplexityStats {imageBlockCount: number;imageCoverage: number;largestImageCoverage: number;+ fullPageImage: boolean;uncoveredVectorArea?: number;isGarbled: boolean;pageArea: number;needsOcr: boolean;+ reasons: string[];}export interface LiteParseNative {diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.pyindex 6fc7552..8531823 100644--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@ -216,10 +216,12 @@ class LiteParse:image_block_count=s.image_block_count,image_coverage=s.image_coverage,largest_image_coverage=s.largest_image_coverage,+ full_page_image=s.full_page_image,uncovered_vector_area=s.uncovered_vector_area,is_garbled=s.is_garbled,page_area=s.page_area,needs_ocr=s.needs_ocr,+ reasons=list(s.reasons),)for s in native_stats]diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.pyindex d75cede..57e8215 100644--- a/packages/python/liteparse/types.py+++ b/packages/python/liteparse/types.py@@ -81,10 +81,12 @@ class PageComplexityStats:image_block_count: intimage_coverage: floatlargest_image_coverage: float+ full_page_image: booluncovered_vector_area: Optional[float]is_garbled: boolpage_area: floatneeds_ocr: bool+ reasons: list[str]@dataclass