Skip to content

A fast, helpful, and open-source document parser

restore failure policy

Logan Markewichb53315375dbb
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs
index de142bc..97bdc36 100644
--- a/crates/liteparse-napi/src/types.rs
+++ b/crates/liteparse-napi/src/types.rs
@@ -95,12 +95,9 @@ pub struct JsLiteParseConfig {
/// Draw AcroForm field appearances into rendered rasters (screenshots and
/// OCR inputs). Runs the document's open/JS actions. Default false.
pub render_form_fields: Option<bool>,
- /// Whether an OCR failure on a page with insufficient native text
- /// stops parsing. Default `true`: report the error instead of silently
- /// returning a page with missing OCR text.
- /// Set `false` to return partial results and keep the available native
- /// text, for example when the application has its own OCR fallback.
- /// Other parsing errors can still stop parsing.
+ /// Whether a systemic OCR failure aborts the whole parse (default true).
+ /// Set false to keep already-recovered native text and return partial
+ /// results when OCR is unavailable, instead of rejecting.
pub ocr_failure_fatal: Option<bool>,
/// OCR request-hedging schedule (ms). Empty/unset = no hedging. Multiple
/// delays (e.g. `[0, 5000, 10000]`) fire duplicate requests per attempt and
diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rs
index 52a4156..369a0a1 100644
--- a/crates/liteparse/src/config.rs
+++ b/crates/liteparse/src/config.rs
@@ -113,12 +113,13 @@ pub struct LiteParseConfig {
/// scripts nor change raster bytes for form-bearing PDFs.
#[serde(default)]
pub render_form_fields: bool,
- /// Whether an OCR failure on a page with insufficient native text
- /// stops parsing. Default `true`: report the error instead of silently
- /// returning a page with missing OCR text.
- /// Set `false` to return partial results and keep the available native
- /// text, for example when the application has its own OCR fallback.
- /// Other parsing errors can still stop parsing.
+ /// Whether a systemic OCR failure (every OCR task failed *and* at least one
+ /// was a text-sparse page whose primary text source was OCR) aborts the
+ /// whole parse. Default `true`: surface the root cause instead of silently
+ /// emitting blank pages. Set `false` to keep already-recovered native text
+ /// and return partial results when OCR is unavailable — useful for callers
+ /// that prefer a degraded document over a hard failure (e.g. when the host
+ /// has its own OCR fallback or treats OCR as best-effort enrichment).
pub ocr_failure_fatal: bool,
/// OCR request-hedging schedule (milliseconds) for the HTTP OCR engine.
/// Empty (default) = no hedging. With multiple delays (e.g.
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index f2f410b..253b10f 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -798,32 +798,12 @@ impl LiteParse {
let render_options = ocr_render_options;
let ocr_input = repaired_input.as_ref().unwrap_or(validated_input);
let mut scan_start = 0usize;
- let page_indices: std::collections::HashMap<_, _> = pages
- .iter()
- .enumerate()
- .map(|(index, page)| (page.page_number, index))
- .collect();
let mut ocr_tasks =
stages::OcrWindow::new(engine, &self.config.ocr_language, self.config.num_workers);
loop {
+ // Reap completions before deciding whether to render so every
+ // available slot can be refilled immediately.
ocr_tasks.complete_ready();
- // Merge completed results before rendering more pages. If
- // `ocr_failure_fatal` requires parsing to stop, return before
- // starting more work. Otherwise, refill the available slots.
- for outcome in ocr_tasks.take_completed() {
- let index = *page_indices.get(&outcome.page_number).ok_or_else(|| {
- LiteParseError::Other(format!(
- "No page for OCR outcome {}",
- outcome.page_number
- ))
- })?;
- stages::merge_ocr(
- std::slice::from_mut(&mut pages[index]),
- vec![outcome],
- self.config.ocr_failure_fatal,
- self.config.effective_emit_word_boxes(),
- )?;
- }
if scan_start >= pages.len() || ocr_tasks.available_capacity() == 0 {
if !ocr_tasks.complete_one().await {
@@ -845,6 +825,17 @@ impl LiteParse {
ocr_tasks.submit(raster).await;
}
}
+
+ // Merge once, after every page has been recognized:
+ // `ocr_failure_fatal` only applies when *every* OCR task failed,
+ // which can't be decided from a single page's outcome.
+ ocr_tasks
+ .finish_and_merge(
+ &mut pages,
+ self.config.ocr_failure_fatal,
+ self.config.effective_emit_word_boxes(),
+ )
+ .await?;
}
let t_ocr = web_time::Instant::now();
log(&format!(
diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs
index 7afff97..1c78a34 100644
--- a/crates/liteparse/tests/integration_test.rs
+++ b/crates/liteparse/tests/integration_test.rs
@@ -1290,17 +1290,14 @@ async fn test_orientation_correction_rejects_non_cardinal_angle() {
}
}
-struct PartialFailureEngine {
- first_started: tokio::sync::Notify,
- failed: tokio::sync::Notify,
- release_first: tokio::sync::Notify,
- first_finished: tokio::sync::Notify,
- calls: std::sync::Mutex<Vec<u32>>,
+/// Fails recognition for the listed raster widths; succeeds otherwise.
+struct FailingWidthsEngine {
+ fail: Vec<u32>,
}
-impl liteparse::ocr::OcrEngine for PartialFailureEngine {
+impl liteparse::ocr::OcrEngine for FailingWidthsEngine {
fn name(&self) -> &str {
- "partial-failure"
+ "failing-widths"
}
fn recognize<'a, 'b: 'a, 'c: 'a>(
&'a self,
@@ -1320,15 +1317,8 @@ impl liteparse::ocr::OcrEngine for PartialFailureEngine {
>,
> {
Box::pin(async move {
- self.calls.lock().unwrap().push(width);
- if width == 200 {
- self.first_started.notify_one();
- self.release_first.notified().await;
- self.first_finished.notify_one();
- } else if width == 201 {
- self.first_started.notified().await;
- self.failed.notify_one();
- return Err("EXPECTED_PARTIAL_OCR_FAILURE".into());
+ if self.fail.contains(&width) {
+ return Err("EXPECTED_OCR_FAILURE".into());
}
Ok(vec![liteparse::ocr::OcrResult {
text: format!("SUCCESS_{width}"),
@@ -1340,74 +1330,61 @@ impl liteparse::ocr::OcrEngine for PartialFailureEngine {
}
}
+async fn parse_with_failing_widths(
+ fail: &[u32],
+ fatal: bool,
+) -> Result<liteparse::ParseResult, liteparse::LiteParseError> {
+ LiteParse::new(LiteParseConfig {
+ ocr_enabled: true,
+ num_workers: 2,
+ dpi: 72.0,
+ quiet: true,
+ ocr_failure_fatal: fatal,
+ ..Default::default()
+ })
+ .with_ocr_engine(std::sync::Arc::new(FailingWidthsEngine {
+ fail: fail.to_vec(),
+ }))
+ .parse_input(PdfInput::Bytes(blank_pdf(&[
+ (200, 200),
+ (201, 200),
+ (202, 200),
+ ])))
+ .await
+}
+
+// `ocr_failure_fatal` only aborts when *every* OCR task failed. One failed
+// page among successes must never abort the parse, regardless of the setting
+// or of the order in which concurrent recognitions complete.
#[tokio::test]
#[serial]
-async fn test_ocr_partial_failure_policy_with_active_job() {
- use std::{sync::Arc, time::Duration};
+async fn test_ocr_partial_failure_is_never_fatal() {
for fatal in [true, false] {
- let engine = Arc::new(PartialFailureEngine {
- first_started: tokio::sync::Notify::new(),
- failed: tokio::sync::Notify::new(),
- release_first: tokio::sync::Notify::new(),
- first_finished: tokio::sync::Notify::new(),
- calls: std::sync::Mutex::new(Vec::new()),
- });
- let parser = LiteParse::new(LiteParseConfig {
- ocr_enabled: true,
- num_workers: 2,
- dpi: 72.0,
- quiet: true,
- ocr_failure_fatal: fatal,
- ..Default::default()
- })
- .with_ocr_engine(engine.clone());
- let parse = tokio::spawn(async move {
- parser
- .parse_input(PdfInput::Bytes(blank_pdf(&[
- (200, 200),
- (201, 200),
- (202, 200),
- ])))
- .await
- });
- tokio::time::timeout(Duration::from_secs(5), engine.failed.notified())
- .await
- .unwrap();
- if !fatal {
- engine.release_first.notify_one();
- }
- let result = tokio::time::timeout(Duration::from_secs(5), parse).await;
- // Release the blocking job even if the assertion below fails.
- engine.release_first.notify_one();
- tokio::time::timeout(Duration::from_secs(5), engine.first_finished.notified())
+ let result = parse_with_failing_widths(&[201], fatal)
.await
- .unwrap();
- let result = result
- .expect(if fatal {
- "fatal OCR failure must return before the blocked job is released"
- } else {
- "nonfatal parsing must finish after the blocked job is released"
- })
- .unwrap();
- if fatal {
- assert!(
- result
- .err()
- .expect("parse must fail")
- .to_string()
- .contains("EXPECTED_PARTIAL_OCR_FAILURE")
- );
- assert!(!engine.calls.lock().unwrap().contains(&202));
- } else {
- let result = result.unwrap();
- assert_eq!(result.pages.len(), 3);
- assert!(result.pages[0].text.contains("SUCCESS_200"));
- assert!(result.pages[1].text.trim().is_empty());
- assert!(result.pages[2].text.contains("SUCCESS_202"));
- }
+ .unwrap_or_else(|e| panic!("fatal={fatal}: one failed page aborted the parse: {e}"));
+ assert_eq!(result.pages.len(), 3);
+ assert!(result.pages[0].text.contains("SUCCESS_200"));
+ assert!(result.pages[1].text.trim().is_empty());
+ assert!(result.pages[2].text.contains("SUCCESS_202"));
}
}
+#[tokio::test]
+#[serial]
+async fn test_ocr_systemic_failure_respects_fatal_flag() {
+ let all = [200, 201, 202];
+ match parse_with_failing_widths(&all, true).await {
+ Err(err) => assert!(err.to_string().contains("EXPECTED_OCR_FAILURE")),
+ Ok(_) => panic!("every page failing OCR must abort when fatal"),
+ }
+
+ let result = parse_with_failing_widths(&all, false)
+ .await
+ .expect("non-fatal mode must return partial results");
+ assert_eq!(result.pages.len(), 3);
+}
+
async fn diagonal_column_count(config: LiteParseConfig) -> usize {
let stats = LiteParse::new(LiteParseConfig {
quiet: true,
diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts
index 7bb16ba..f129b6f 100644
--- a/packages/node/native.d.ts
+++ b/packages/node/native.d.ts
@@ -105,12 +105,9 @@ export interface JsLiteParseConfig {
*/
renderFormFields?: boolean
/**
- * Whether an OCR failure on a page with insufficient native text
- * stops parsing. Default `true`: report the error instead of silently
- * returning a page with missing OCR text.
- * Set `false` to return partial results and keep the available native
- * text, for example when the application has its own OCR fallback.
- * Other parsing errors can still stop parsing.
+ * Whether a systemic OCR failure aborts the whole parse (default true).
+ * Set false to keep already-recovered native text and return partial
+ * results when OCR is unavailable, instead of rejecting.
*/
ocrFailureFatal?: boolean
/**
diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts
index 8d83978..5767161 100644
--- a/packages/node/src/lib.ts
+++ b/packages/node/src/lib.ts
@@ -95,12 +95,10 @@ export interface LiteParseConfig {
quiet: boolean;
numWorkers: number;
/**
- * Whether an OCR failure on a page with insufficient native text
- * stops parsing. Default `true`: report the error instead of silently
- * returning a page with missing OCR text.
- * Set `false` to return partial results and keep the available native
- * text, for example when the application has its own OCR fallback.
- * Other parsing errors can still stop parsing.
+ * Whether a systemic OCR failure (every OCR task failed and at least one was
+ * a text-sparse page) aborts the whole parse (default: true). Set false to
+ * keep already-recovered native text and return partial results instead of
+ * rejecting — for callers that prefer a degraded document over a hard failure.
*/
ocrFailureFatal: boolean;
/**
diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py
index f26a341..256afbc 100644
--- a/packages/python/liteparse/parser.py
+++ b/packages/python/liteparse/parser.py
@@ -550,12 +550,11 @@ class LiteParse:
the rendered Markdown.
extract_structure_tree: Include the tagged-PDF logical structure
tree as page-scoped structured data (default: False).
- ocr_failure_fatal: Whether an OCR failure on a page with insufficient
- native text stops parsing. Default True: report the error instead
- of silently returning a page with missing OCR text.
- Set False to return partial results and keep the available native
- text, for example when the application has its own OCR fallback.
- Other parsing errors can still stop parsing.
+ ocr_failure_fatal: Whether a systemic OCR failure (every OCR task
+ failed and at least one was a text-sparse page) aborts the whole
+ parse (default: True). Set False to keep already-recovered native
+ text and return partial results instead of raising — for callers
+ that prefer a degraded document over a hard failure.
ocr_hedge_delays_ms: Request-hedging schedule for HTTP OCR, in
milliseconds. Empty or single-element means no hedging (one
request per attempt — the default). With multiple delays (e.g.
diff --git a/packages/wasm/README.md b/packages/wasm/README.md
index 9258ba0..eb128e5 100644
--- a/packages/wasm/README.md
+++ b/packages/wasm/README.md
@@ -115,10 +115,10 @@ concurrent HTTP requests. Results stay assigned to their source pages when
jobs finish out of order. Active calls can keep up to `numWorkers` rendered
page rasters in memory, so use a small value for high-DPI documents.
-With `ocrFailureFatal: true` (the default), an OCR failure on a page with
-insufficient native text rejects `parse()`, even if other pages succeed.
-With `ocrFailureFatal: false`, parsing continues with the available text.
-If parsing fails, OCR calls that have already started can continue.
+A failed `recognize` call only affects its own page. With `ocrFailureFatal:
+true` (the default), `parse()` rejects only when every OCR call failed and at
+least one failed page had little native text. If parsing fails, OCR calls that
+have already started can continue.
## Building from source
diff --git a/scripts/browser-compat/wasm-test.html b/scripts/browser-compat/wasm-test.html
index fce6be3..89c66d1 100644
--- a/scripts/browser-compat/wasm-test.html
+++ b/scripts/browser-compat/wasm-test.html
@@ -289,36 +289,6 @@
groupedParser.free();
}
- status.textContent = "testing a ready failure before refill";
- const releaseReadyPair = deferred();
- let readyPairCalls = 0;
- const readyFailureParser = new LiteParse({
- ocrEnabled: true, dpi: 72, quiet: true, numWorkers: 2, ocrFailureFatal: true,
- ocrEngine: {
- async recognize(_imageData, width) {
- readyPairCalls += 1;
- if (readyPairCalls === 2) releaseReadyPair.resolve();
- await withDeadline(releaseReadyPair.promise, "The first two OCR pages did not start");
- if (width === 73) throw new Error("READY_OCR_FAILURE");
- return [{text: `READY_${width}`, bbox: [4, 4, width - 4, 24], confidence: 0.99}];
- },
- },
- });
- try {
- let failure;
- try {
- await readyFailureParser.parse(buildTestPdf(4));
- } catch (error) {
- failure = String(error);
- }
- if (!failure?.includes("READY_OCR_FAILURE") || readyPairCalls !== 2) {
- throw new Error(`A ready failure must stop refill: calls=${readyPairCalls}, error=${failure}`);
- }
- } finally {
- releaseReadyPair.resolve();
- readyFailureParser.free();
- }
-
status.textContent = "testing more workers than pages";
const allPagesStarted = deferred();
let largePoolCalls = 0;
@@ -390,84 +360,51 @@
}
status.textContent = "testing OCR failure policy";
+ // `ocrFailureFatal` only rejects when every OCR call failed. One failed
+ // page among successes never rejects, in either mode.
let failureCases = 0;
for (const numWorkers of [1, 2]) {
for (const fatal of [true, false]) {
- const secondStarted = deferred();
- const thirdStarted = deferred();
- const failed = deferred();
- const releaseThird = deferred();
- const thirdFinished = deferred();
- const calls = [];
- let thirdActive = false;
- const failureParser = new LiteParse({
- ocrEnabled: true, dpi: 72, quiet: true, outputFormat: "text",
- numWorkers, ocrFailureFatal: fatal,
- ocrEngine: {
- async recognize(_imageData, width) {
- calls.push(width);
- if (width === 72 && numWorkers === 2) {
- await secondStarted.promise;
- }
- if (width === 73) {
- secondStarted.resolve();
- if (numWorkers === 2) await thirdStarted.promise;
- failed.resolve();
- throw new Error("EXPECTED_OCR_PAGE_FAILURE");
+ for (const failAll of [false, true]) {
+ const failureParser = new LiteParse({
+ ocrEnabled: true, dpi: 72, quiet: true, outputFormat: "text",
+ numWorkers, ocrFailureFatal: fatal,
+ ocrEngine: {
+ async recognize(_imageData, width) {
+ if (failAll || width === 73) throw new Error("EXPECTED_OCR_PAGE_FAILURE");
+ return [{
+ text: `OCR_SUCCESS_${width}`,
+ bbox: [4, 4, width - 4, 24], confidence: 0.99,
+ }];
+ },
+ },
+ });
+ try {
+ const outcome = await withDeadline(
+ failureParser.parse(buildTestPdf(3)).then(
+ (result) => ({ result }), (error) => ({ error }),
+ ),
+ "OCR failure policy did not settle parse()",
+ );
+ const label = `numWorkers=${numWorkers} fatal=${fatal} failAll=${failAll}`;
+ if (failAll && fatal) {
+ if (!String(outcome.error).includes("EXPECTED_OCR_PAGE_FAILURE")) {
+ throw new Error(`Expected a fatal OCR error (${label})`);
}
- if (width === 74) {
- thirdActive = true;
- thirdStarted.resolve();
- try {
- await releaseThird.promise;
- } finally {
- thirdActive = false;
- thirdFinished.resolve();
- }
+ } else {
+ if (outcome.error) throw new Error(`Unexpected rejection (${label}): ${outcome.error}`);
+ const pages = outcome.result.pages;
+ if (pages.length !== 3) throw new Error(`Expected 3 pages (${label})`);
+ if (!failAll && (!pages[0].text.includes("OCR_SUCCESS_72") ||
+ pages[1].text.trim() !== "" || !pages[2].text.includes("OCR_SUCCESS_74"))) {
+ throw new Error(`Partial OCR failure did not preserve successful pages (${label})`);
}
- return [{
- text: `OCR_SUCCESS_${width}`,
- bbox: [4, 4, width - 4, 24], confidence: 0.99,
- }];
- },
- },
- });
- const settled = failureParser.parse(buildTestPdf(3)).then(
- (result) => ({ result }), (error) => ({ error }),
- );
- try {
- await withDeadline(failed.promise, "The failing OCR page did not start");
- if (!fatal) releaseThird.resolve();
- const outcome = await withDeadline(settled, "OCR failure policy did not settle parse()");
- if (fatal) {
- if (!String(outcome.error).includes("EXPECTED_OCR_PAGE_FAILURE")) {
- throw new Error(`Expected a fatal OCR error with ${numWorkers} worker(s)`);
}
- if (numWorkers === 1 && calls.join(",") !== "72,73") {
- throw new Error("Serial OCR started another page after a fatal error");
- }
- if (numWorkers === 2 && !thirdActive) {
- throw new Error("Fatal OCR did not reject while another call was active");
- }
- } else {
- if (outcome.error) throw outcome.error;
- const pages = outcome.result.pages;
- if (pages.length !== 3 || !pages[0].text.includes("OCR_SUCCESS_72") ||
- pages[1].text.trim() !== "" || !pages[2].text.includes("OCR_SUCCESS_74")) {
- throw new Error("Nonfatal OCR did not preserve successful pages");
- }
- }
- } finally {
- secondStarted.resolve();
- thirdStarted.resolve();
- releaseThird.resolve();
- await withDeadline(settled, "OCR cleanup did not settle parse()");
- if (calls.includes(74)) {
- await withDeadline(thirdFinished.promise, "The active OCR call did not finish");
+ } finally {
+ failureParser.free();
}
- failureParser.free();
+ failureCases += 1;
}
- failureCases += 1;
}
}
@@ -479,7 +416,7 @@
resultEl.setAttribute("data-ocr-bytes", ocrByteLength);
resultEl.setAttribute("data-ocr-max-concurrency", maxActiveOcrCalls);
resultEl.setAttribute("data-ocr-failure-cases", failureCases);
- resultEl.textContent = `OK: ${pageCount} pages, ${textLength} chars, OCR callback, concurrency, worker limits, grouped completion, ready failure, native text refill, and ${failureCases} failure cases passed`;
+ resultEl.textContent = `OK: ${pageCount} pages, ${textLength} chars, OCR callback, concurrency, worker limits, grouped completion, native text refill, and ${failureCases} failure cases passed`;
} catch (err) {
status.textContent = "error";
errorEl.style.display = "block";