Skip to content

A fast, helpful, and open-source document parser

update output shape

Logan Markewich372b018db674
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs
index cedf048..77e579a 100644
--- a/crates/liteparse-napi/src/types.rs
+++ b/crates/liteparse-napi/src/types.rs
@@ -368,10 +368,12 @@ pub struct JsPageComplexityStats {
pub image_block_count: u32,
pub image_coverage: f64,
pub largest_image_coverage: f64,
+ pub full_page_image: bool,
pub uncovered_vector_area: Option<f64>,
pub is_garbled: bool,
pub page_area: f64,
pub needs_ocr: bool,
+ pub reasons: Vec<String>,
}
impl JsPageComplexityStats {
@@ -384,10 +386,16 @@ impl JsPageComplexityStats {
image_block_count: stats.image_block_count as u32,
image_coverage: stats.image_coverage as f64,
largest_image_coverage: stats.largest_image_coverage as f64,
+ full_page_image: stats.full_page_image,
uncovered_vector_area: stats.uncovered_vector_area.map(|v| v as f64),
is_garbled: stats.is_garbled,
page_area: stats.page_area as f64,
needs_ocr: stats.needs_ocr,
+ reasons: stats
+ .reasons
+ .iter()
+ .map(|r| r.as_str().to_string())
+ .collect(),
}
}
}
diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs
index 26985c3..2b0632e 100644
--- a/crates/liteparse-python/src/lib.rs
+++ b/crates/liteparse-python/src/lib.rs
@@ -252,6 +252,8 @@ struct PyPageComplexityStats {
#[pyo3(get)]
largest_image_coverage: f32,
#[pyo3(get)]
+ full_page_image: bool,
+ #[pyo3(get)]
uncovered_vector_area: Option<f32>,
#[pyo3(get)]
is_garbled: bool,
@@ -259,6 +261,8 @@ struct PyPageComplexityStats {
page_area: f32,
#[pyo3(get)]
needs_ocr: bool,
+ #[pyo3(get)]
+ reasons: Vec<String>,
}
#[pymethods]
@@ -281,10 +285,16 @@ impl PyPageComplexityStats {
image_block_count: stats.image_block_count,
image_coverage: stats.image_coverage,
largest_image_coverage: stats.largest_image_coverage,
+ full_page_image: stats.full_page_image,
uncovered_vector_area: stats.uncovered_vector_area,
is_garbled: stats.is_garbled,
page_area: stats.page_area,
needs_ocr: stats.needs_ocr,
+ reasons: stats
+ .reasons
+ .iter()
+ .map(|r| r.as_str().to_string())
+ .collect(),
}
}
}
diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs
index fc26dca..2733b04 100644
--- a/crates/liteparse-wasm/src/lib.rs
+++ b/crates/liteparse-wasm/src/lib.rs
@@ -425,10 +425,12 @@ struct JsPageComplexityStats {
image_block_count: usize,
image_coverage: f32,
largest_image_coverage: f32,
+ full_page_image: bool,
uncovered_vector_area: Option<f32>,
is_garbled: bool,
page_area: f32,
needs_ocr: bool,
+ reasons: Vec<String>,
}
impl JsPageComplexityStats {
@@ -441,10 +443,16 @@ impl JsPageComplexityStats {
image_block_count: stats.image_block_count,
image_coverage: stats.image_coverage,
largest_image_coverage: stats.largest_image_coverage,
+ full_page_image: stats.full_page_image,
uncovered_vector_area: stats.uncovered_vector_area,
is_garbled: stats.is_garbled,
page_area: stats.page_area,
needs_ocr: stats.needs_ocr,
+ reasons: stats
+ .reasons
+ .iter()
+ .map(|r| r.as_str().to_string())
+ .collect(),
}
}
}
diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rs
index 1652b23..f58ab3e 100644
--- a/crates/liteparse/src/main.rs
+++ b/crates/liteparse/src/main.rs
@@ -225,9 +225,10 @@ struct IsComplexCommand {
/// Input file path
file: String,
- /// Whether to output the result as JSON (true/false) or plain text (true/false)
+ /// Emit dense, whitespace-free JSON instead of pretty-printed (still valid
+ /// for `jq` and friends).
#[arg(long)]
- json: bool,
+ compact: bool,
/// Max pages to parse
#[arg(long, default_value = "1000")]
@@ -517,46 +518,31 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let complex_pages = is_complex.iter().filter(|c| c.needs_ocr).count();
- if cmd.json && !cmd.quiet {
- let json = serde_json::to_string(&is_complex)?;
- println!("{}", json);
- } else if !cmd.quiet {
+ // Always emit JSON on stdout so the command composes with `jq` and
+ // friends without a flag. Pretty by default for human readability;
+ // `--compact` drops the whitespace. Both parse identically.
+ let json = if cmd.compact {
+ serde_json::to_string(&is_complex)?
+ } else {
+ serde_json::to_string_pretty(&is_complex)?
+ };
+ println!("{}", json);
+
+ // The human-readable verdict goes to stderr so it never pollutes the
+ // JSON on stdout. The exit code below carries the same signal for
+ // scripts that don't want to read either stream.
+ if !cmd.quiet {
let verdict = if complex_pages > 0 {
"COMPLEX"
} else {
"SIMPLE"
};
- println!(
+ eprintln!(
"{} — {}/{} page(s) need OCR",
verdict,
complex_pages,
is_complex.len()
);
- println!(
- "{:>4} {:>7} {:>5} {:>6} {:>6} {:>7} {:>7} {:>5}",
- "page", "text", "cov", "images", "imgcov", "garbled", "vector", "ocr"
- );
- for c in &is_complex {
- let vector = match c.uncovered_vector_area {
- Some(area) => format!("{:.0}", area),
- None => "-".to_string(),
- };
- println!(
- "{:>4} {:>7} {:>5.2} {:>6} {:>6.2} {:>7} {:>7} {:>5}",
- c.page_number,
- c.text_length,
- c.text_coverage,
- if c.has_substantial_images {
- "yes"
- } else {
- "no"
- },
- c.image_coverage,
- if c.is_garbled { "yes" } else { "no" },
- vector,
- if c.needs_ocr { "yes" } else { "no" },
- );
- }
}
// Exit non-zero when any page needs OCR, so the command is usable as
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs
index 3d3534f..31ea997 100644
--- a/crates/liteparse/src/ocr_merge.rs
+++ b/crates/liteparse/src/ocr_merge.rs
@@ -31,6 +31,49 @@ pub(crate) struct RenderedPage {
pub height: u32,
}
+/// Why a page was flagged as needing more than the cheap text-only path.
+/// Multiple reasons can apply to one page (e.g. a sparse page whose little
+/// text is also garbled). Empty exactly when `needs_ocr` is false.
+///
+/// This is the discriminator a caller routes on: a scan goes to OCR, dense
+/// vector text to a vector-aware pass, and so on. New variants will be added
+/// as the routing function learns to recommend heavier pipelines (tables,
+/// charts, LLM passes), so callers should treat unknown variants leniently.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum ComplexityReason {
+ /// A single raster covers essentially the whole page and there is little or
+ /// no extractable text behind it — a scanned/photographed page.
+ Scanned,
+ /// Almost no extractable native text, and no full-page raster behind it
+ /// (a blank page, or a near-empty cover/divider).
+ NoText,
+ /// Some real text, but it covers very little of the page — typically a
+ /// figure-heavy page with only thin captions.
+ SparseText,
+ /// Substantial embedded raster figures sit alongside the native text.
+ EmbeddedImages,
+ /// The native text decodes to garbage (broken cmap / Type3 char-code
+ /// fallback), so the visible glyphs and the extracted text disagree.
+ Garbled,
+ /// Text is painted as filled vector outlines, outside the text layer, so
+ /// no native text items represent it.
+ VectorText,
+}
+
+impl ComplexityReason {
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ ComplexityReason::Scanned => "scanned",
+ ComplexityReason::NoText => "no-text",
+ ComplexityReason::SparseText => "sparse-text",
+ ComplexityReason::EmbeddedImages => "embedded-images",
+ ComplexityReason::Garbled => "garbled",
+ ComplexityReason::VectorText => "vector-text",
+ }
+ }
+}
+
#[derive(Debug, Serialize)]
pub struct PageComplexityStats {
pub page_number: usize,
@@ -49,13 +92,24 @@ pub struct PageComplexityStats {
/// 1.0. Useful for telling a single full-bleed scan apart from many small
/// inline figures that sum to a similar `image_coverage`.
pub largest_image_coverage: f32,
+ /// A single raster covering ≥90% of the page is present. Such full-page
+ /// backgrounds are excluded from `image_coverage`/`largest_image_coverage`
+ /// (they're not inline figures), so this flag is the only signal that
+ /// distinguishes a scan from a genuinely blank page — both otherwise report
+ /// no text and no counted images.
+ pub full_page_image: bool,
/// Filled vector-outline area not covered by native text, in pt². `None`
/// when a cheaper predicate already flagged the page for OCR, so this
/// expensive page-object walk was skipped (it wasn't the deciding signal).
pub uncovered_vector_area: Option<f32>,
pub is_garbled: bool,
pub page_area: f32,
+ /// Whether the page needs more than the cheap text-only path. Equivalent to
+ /// `!reasons.is_empty()`; kept as a flat bool for the common predicate case
+ /// and as the internal per-page OCR gate.
pub needs_ocr: bool,
+ /// Every reason the page was flagged, in no particular priority order.
+ pub reasons: Vec<ComplexityReason>,
}
pub(crate) fn calculate_page_complexity(
@@ -75,10 +129,21 @@ pub(crate) fn calculate_page_complexity(
.filter(|item| !is_unusable_native(item))
.map(|item| item.text.len())
.sum();
- let image_bounds = page_obj.image_bounds(MIN_IMAGE_SIZE_PT, MAX_IMAGE_PAGE_COVERAGE);
+ // Collect every raster ≥ MIN_IMAGE_SIZE_PT, including full-page ones, so a
+ // scan can be told apart from a blank page. The "counted" subset below then
+ // drops full-page backgrounds, matching the old
+ // `image_bounds(.., MAX_IMAGE_PAGE_COVERAGE)` behaviour for inline figures.
+ let pw = page.page_width;
+ let ph = page.page_height;
+ let all_images = page_obj.image_bounds(MIN_IMAGE_SIZE_PT, f32::INFINITY);
+ let is_full_page = |b: &ImageBounds| {
+ b.width > pw * MAX_IMAGE_PAGE_COVERAGE && b.height > ph * MAX_IMAGE_PAGE_COVERAGE
+ };
+ let full_page_image = all_images.iter().any(is_full_page);
+ let image_bounds: Vec<&ImageBounds> = all_images.iter().filter(|b| !is_full_page(b)).collect();
let has_images = !image_bounds.is_empty();
- let page_area = page.page_width * page.page_height;
+ let page_area = pw * ph;
let (image_area_sum, largest_image_area) =
image_bounds
@@ -112,7 +177,27 @@ pub(crate) fn calculate_page_complexity(
// wide intra-cell whitespace) is spatially sparse but needs no OCR.
let sparse_text = text_length < 2000 && text_coverage < 0.15;
let is_garbled = page_is_garbled(page);
- let mut needs_ocr = text_length < 20 || sparse_text || has_images || is_garbled;
+
+ let mut reasons = Vec::new();
+ if text_length < 20 {
+ // Too little text to be the page's content. A full-page raster behind
+ // it means a scan; otherwise it's effectively blank.
+ reasons.push(if full_page_image {
+ ComplexityReason::Scanned
+ } else {
+ ComplexityReason::NoText
+ });
+ } else if sparse_text {
+ // There is real text, but it's too thin to be the whole page.
+ reasons.push(ComplexityReason::SparseText);
+ }
+ if has_images {
+ reasons.push(ComplexityReason::EmbeddedImages);
+ }
+ if is_garbled {
+ reasons.push(ComplexityReason::Garbled);
+ }
+ let mut needs_ocr = !reasons.is_empty();
// Text drawn as filled vector outlines lives outside the text layer
// entirely: no text items, no image XObjects, so none of the cheap
@@ -123,7 +208,10 @@ pub(crate) fn calculate_page_complexity(
let uncovered_vector_area = if !needs_ocr {
let path_bounds = page_obj.filled_path_bounds(3.0, 0.9);
let uncovered = uncovered_path_area(&path_bounds, &page.text_items);
- needs_ocr = uncovered >= UNCOVERED_VECTOR_AREA_THRESHOLD;
+ if uncovered >= UNCOVERED_VECTOR_AREA_THRESHOLD {
+ needs_ocr = true;
+ reasons.push(ComplexityReason::VectorText);
+ }
Some(uncovered)
} else {
None
@@ -137,10 +225,12 @@ pub(crate) fn calculate_page_complexity(
image_block_count: image_bounds.len(),
image_coverage,
largest_image_coverage,
+ full_page_image,
uncovered_vector_area,
is_garbled,
page_area,
needs_ocr,
+ reasons,
})
}
diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.ts
index 91b5f58..3673811 100644
--- a/packages/node/src/cli.ts
+++ b/packages/node/src/cli.ts
@@ -147,7 +147,7 @@ program
"Check if a document is 'complex' enough to require OCR or other advanced parsing",
)
.argument("<file>", "Path to the document file")
- .option("--json", "Output the result as JSON")
+ .option("--compact", "Emit dense, whitespace-free JSON instead of pretty")
.option("--max-pages <n>", "Max pages to parse", parseInt)
.option(
"--target-pages <pages>",
@@ -168,36 +168,23 @@ program
const complexPages = stats.filter((s) => s.needsOcr).length;
- if (opts.json) {
- process.stdout.write(JSON.stringify(stats));
- process.stdout.write("\n");
- } else {
+ // Always emit JSON on stdout so the command composes with `jq` without a
+ // flag. Pretty by default; `--compact` drops the whitespace. Both parse
+ // identically for any reader.
+ process.stdout.write(
+ opts.compact
+ ? JSON.stringify(stats)
+ : JSON.stringify(stats, null, 2),
+ );
+ process.stdout.write("\n");
+
+ // Human verdict goes to stderr so it never pollutes the JSON on stdout;
+ // the exit code below carries the same signal for scripts.
+ if (!opts.quiet) {
const verdict = complexPages > 0 ? "COMPLEX" : "SIMPLE";
- console.log(
+ console.error(
`${verdict} — ${complexPages}/${stats.length} page(s) need OCR`,
);
- console.log(
- ["page", "text", "cov", "images", "garbled", "vector", "ocr"].join(
- "\t",
- ),
- );
- for (const s of stats) {
- const vector =
- s.uncoveredVectorArea !== undefined
- ? s.uncoveredVectorArea.toFixed(0)
- : "-";
- console.log(
- [
- s.pageNumber,
- s.textLength,
- s.textCoverage.toFixed(2),
- s.hasSubstantialImages ? "yes" : "no",
- s.isGarbled ? "yes" : "no",
- vector,
- s.needsOcr ? "yes" : "no",
- ].join("\t"),
- );
- }
}
// Exit non-zero when any page needs OCR, so the command is usable as a
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index 2af2bad..8f72453 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -133,8 +133,14 @@ export interface PageComplexityStats {
imageBlockCount: number;
/** Summed image-bbox area over page area, clamped to 1. */
imageCoverage: number;
- /** Largest single image's area over page area, clamped to 1. */
+ /** Largest single *counted* image's area over page area, clamped to 1. */
largestImageCoverage: number;
+ /**
+ * A single raster covers ≥90% of the page. Full-page backgrounds are excluded
+ * from the image coverage fields, so this is the only signal that tells a scan
+ * apart from a blank page — both otherwise report no text and no images.
+ */
+ fullPageImage: boolean;
/**
* Filled vector-outline area not covered by native text, in pt². `undefined`
* when a cheaper signal already decided the page, so this walk was skipped.
@@ -142,8 +148,14 @@ export interface PageComplexityStats {
uncoveredVectorArea?: number;
isGarbled: boolean;
pageArea: number;
- /** Verdict: whether this page should be sent through OCR. */
+ /** Verdict: whether this page needs more than the cheap text-only path. */
needsOcr: boolean;
+ /**
+ * Every reason the page was flagged (e.g. `"scanned"`, `"sparse-text"`,
+ * `"garbled"`). Empty exactly when `needsOcr` is false. This is the value to
+ * route on; new reasons may be added over time.
+ */
+ reasons: string[];
}
// ---------------------------------------------------------------------------
@@ -248,10 +260,12 @@ export class LiteParse {
imageBlockCount: s.imageBlockCount,
imageCoverage: s.imageCoverage,
largestImageCoverage: s.largestImageCoverage,
+ fullPageImage: s.fullPageImage,
uncoveredVectorArea: s.uncoveredVectorArea ?? undefined,
isGarbled: s.isGarbled,
pageArea: s.pageArea,
needsOcr: s.needsOcr,
+ reasons: s.reasons,
}));
}
diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts
index c692838..133ee9e 100644
--- a/packages/node/src/native.ts
+++ b/packages/node/src/native.ts
@@ -110,10 +110,12 @@ export interface NativePageComplexityStats {
imageBlockCount: number;
imageCoverage: number;
largestImageCoverage: number;
+ fullPageImage: boolean;
uncoveredVectorArea?: number;
isGarbled: boolean;
pageArea: number;
needsOcr: boolean;
+ reasons: string[];
}
export interface LiteParseNative {
diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py
index 6fc7552..8531823 100644
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@ -216,10 +216,12 @@ class LiteParse:
image_block_count=s.image_block_count,
image_coverage=s.image_coverage,
largest_image_coverage=s.largest_image_coverage,
+ full_page_image=s.full_page_image,
uncovered_vector_area=s.uncovered_vector_area,
is_garbled=s.is_garbled,
page_area=s.page_area,
needs_ocr=s.needs_ocr,
+ reasons=list(s.reasons),
)
for s in native_stats
]
diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py
index d75cede..57e8215 100644
--- a/packages/python/liteparse/types.py
+++ b/packages/python/liteparse/types.py
@@ -81,10 +81,12 @@ class PageComplexityStats:
image_block_count: int
image_coverage: float
largest_image_coverage: float
+ full_page_image: bool
uncovered_vector_area: Optional[float]
is_garbled: bool
page_area: float
needs_ocr: bool
+ reasons: list[str]
@dataclass