A fast, helpful, and open-source document parser
restore failure policy
Logan Markewichb53315375dbb
diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rsindex de142bc..97bdc36 100644--- a/crates/liteparse-napi/src/types.rs+++ b/crates/liteparse-napi/src/types.rs@@ -95,12 +95,9 @@ pub struct JsLiteParseConfig {/// Draw AcroForm field appearances into rendered rasters (screenshots and/// OCR inputs). Runs the document's open/JS actions. Default false.pub render_form_fields: Option<bool>,- /// Whether an OCR failure on a page with insufficient native text- /// stops parsing. Default `true`: report the error instead of silently- /// returning a page with missing OCR text.- /// Set `false` to return partial results and keep the available native- /// text, for example when the application has its own OCR fallback.- /// Other parsing errors can still stop parsing.+ /// Whether a systemic OCR failure aborts the whole parse (default true).+ /// Set false to keep already-recovered native text and return partial+ /// results when OCR is unavailable, instead of rejecting.pub ocr_failure_fatal: Option<bool>,/// OCR request-hedging schedule (ms). Empty/unset = no hedging. Multiple/// delays (e.g. `[0, 5000, 10000]`) fire duplicate requests per attempt anddiff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rsindex 52a4156..369a0a1 100644--- a/crates/liteparse/src/config.rs+++ b/crates/liteparse/src/config.rs@@ -113,12 +113,13 @@ pub struct LiteParseConfig {/// scripts nor change raster bytes for form-bearing PDFs.#[serde(default)]pub render_form_fields: bool,- /// Whether an OCR failure on a page with insufficient native text- /// stops parsing. Default `true`: report the error instead of silently- /// returning a page with missing OCR text.- /// Set `false` to return partial results and keep the available native- /// text, for example when the application has its own OCR fallback.- /// Other parsing errors can still stop parsing.+ /// Whether a systemic OCR failure (every OCR task failed *and* at least one+ /// was a text-sparse page whose primary text source was OCR) aborts the+ /// whole parse. Default `true`: surface the root cause instead of silently+ /// emitting blank pages. Set `false` to keep already-recovered native text+ /// and return partial results when OCR is unavailable — useful for callers+ /// that prefer a degraded document over a hard failure (e.g. when the host+ /// has its own OCR fallback or treats OCR as best-effort enrichment).pub ocr_failure_fatal: bool,/// OCR request-hedging schedule (milliseconds) for the HTTP OCR engine./// Empty (default) = no hedging. With multiple delays (e.g.diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex f2f410b..253b10f 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -798,32 +798,12 @@ impl LiteParse {let render_options = ocr_render_options;let ocr_input = repaired_input.as_ref().unwrap_or(validated_input);let mut scan_start = 0usize;- let page_indices: std::collections::HashMap<_, _> = pages- .iter()- .enumerate()- .map(|(index, page)| (page.page_number, index))- .collect();let mut ocr_tasks =stages::OcrWindow::new(engine, &self.config.ocr_language, self.config.num_workers);loop {+ // Reap completions before deciding whether to render so every+ // available slot can be refilled immediately.ocr_tasks.complete_ready();- // Merge completed results before rendering more pages. If- // `ocr_failure_fatal` requires parsing to stop, return before- // starting more work. Otherwise, refill the available slots.- for outcome in ocr_tasks.take_completed() {- let index = *page_indices.get(&outcome.page_number).ok_or_else(|| {- LiteParseError::Other(format!(- "No page for OCR outcome {}",- outcome.page_number- ))- })?;- stages::merge_ocr(- std::slice::from_mut(&mut pages[index]),- vec![outcome],- self.config.ocr_failure_fatal,- self.config.effective_emit_word_boxes(),- )?;- }if scan_start >= pages.len() || ocr_tasks.available_capacity() == 0 {if !ocr_tasks.complete_one().await {@@ -845,6 +825,17 @@ impl LiteParse {ocr_tasks.submit(raster).await;}}++ // Merge once, after every page has been recognized:+ // `ocr_failure_fatal` only applies when *every* OCR task failed,+ // which can't be decided from a single page's outcome.+ ocr_tasks+ .finish_and_merge(+ &mut pages,+ self.config.ocr_failure_fatal,+ self.config.effective_emit_word_boxes(),+ )+ .await?;}let t_ocr = web_time::Instant::now();log(&format!(diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rsindex 7afff97..1c78a34 100644--- a/crates/liteparse/tests/integration_test.rs+++ b/crates/liteparse/tests/integration_test.rs@@ -1290,17 +1290,14 @@ async fn test_orientation_correction_rejects_non_cardinal_angle() {}}-struct PartialFailureEngine {- first_started: tokio::sync::Notify,- failed: tokio::sync::Notify,- release_first: tokio::sync::Notify,- first_finished: tokio::sync::Notify,- calls: std::sync::Mutex<Vec<u32>>,+/// Fails recognition for the listed raster widths; succeeds otherwise.+struct FailingWidthsEngine {+ fail: Vec<u32>,}-impl liteparse::ocr::OcrEngine for PartialFailureEngine {+impl liteparse::ocr::OcrEngine for FailingWidthsEngine {fn name(&self) -> &str {- "partial-failure"+ "failing-widths"}fn recognize<'a, 'b: 'a, 'c: 'a>(&'a self,@@ -1320,15 +1317,8 @@ impl liteparse::ocr::OcrEngine for PartialFailureEngine {>,> {Box::pin(async move {- self.calls.lock().unwrap().push(width);- if width == 200 {- self.first_started.notify_one();- self.release_first.notified().await;- self.first_finished.notify_one();- } else if width == 201 {- self.first_started.notified().await;- self.failed.notify_one();- return Err("EXPECTED_PARTIAL_OCR_FAILURE".into());+ if self.fail.contains(&width) {+ return Err("EXPECTED_OCR_FAILURE".into());}Ok(vec![liteparse::ocr::OcrResult {text: format!("SUCCESS_{width}"),@@ -1340,74 +1330,61 @@ impl liteparse::ocr::OcrEngine for PartialFailureEngine {}}+async fn parse_with_failing_widths(+ fail: &[u32],+ fatal: bool,+) -> Result<liteparse::ParseResult, liteparse::LiteParseError> {+ LiteParse::new(LiteParseConfig {+ ocr_enabled: true,+ num_workers: 2,+ dpi: 72.0,+ quiet: true,+ ocr_failure_fatal: fatal,+ ..Default::default()+ })+ .with_ocr_engine(std::sync::Arc::new(FailingWidthsEngine {+ fail: fail.to_vec(),+ }))+ .parse_input(PdfInput::Bytes(blank_pdf(&[+ (200, 200),+ (201, 200),+ (202, 200),+ ])))+ .await+}++// `ocr_failure_fatal` only aborts when *every* OCR task failed. One failed+// page among successes must never abort the parse, regardless of the setting+// or of the order in which concurrent recognitions complete.#[tokio::test]#[serial]-async fn test_ocr_partial_failure_policy_with_active_job() {- use std::{sync::Arc, time::Duration};+async fn test_ocr_partial_failure_is_never_fatal() {for fatal in [true, false] {- let engine = Arc::new(PartialFailureEngine {- first_started: tokio::sync::Notify::new(),- failed: tokio::sync::Notify::new(),- release_first: tokio::sync::Notify::new(),- first_finished: tokio::sync::Notify::new(),- calls: std::sync::Mutex::new(Vec::new()),- });- let parser = LiteParse::new(LiteParseConfig {- ocr_enabled: true,- num_workers: 2,- dpi: 72.0,- quiet: true,- ocr_failure_fatal: fatal,- ..Default::default()- })- .with_ocr_engine(engine.clone());- let parse = tokio::spawn(async move {- parser- .parse_input(PdfInput::Bytes(blank_pdf(&[- (200, 200),- (201, 200),- (202, 200),- ])))- .await- });- tokio::time::timeout(Duration::from_secs(5), engine.failed.notified())- .await- .unwrap();- if !fatal {- engine.release_first.notify_one();- }- let result = tokio::time::timeout(Duration::from_secs(5), parse).await;- // Release the blocking job even if the assertion below fails.- engine.release_first.notify_one();- tokio::time::timeout(Duration::from_secs(5), engine.first_finished.notified())+ let result = parse_with_failing_widths(&[201], fatal).await- .unwrap();- let result = result- .expect(if fatal {- "fatal OCR failure must return before the blocked job is released"- } else {- "nonfatal parsing must finish after the blocked job is released"- })- .unwrap();- if fatal {- assert!(- result- .err()- .expect("parse must fail")- .to_string()- .contains("EXPECTED_PARTIAL_OCR_FAILURE")- );- assert!(!engine.calls.lock().unwrap().contains(&202));- } else {- let result = result.unwrap();- assert_eq!(result.pages.len(), 3);- assert!(result.pages[0].text.contains("SUCCESS_200"));- assert!(result.pages[1].text.trim().is_empty());- assert!(result.pages[2].text.contains("SUCCESS_202"));- }+ .unwrap_or_else(|e| panic!("fatal={fatal}: one failed page aborted the parse: {e}"));+ assert_eq!(result.pages.len(), 3);+ assert!(result.pages[0].text.contains("SUCCESS_200"));+ assert!(result.pages[1].text.trim().is_empty());+ assert!(result.pages[2].text.contains("SUCCESS_202"));}}+#[tokio::test]+#[serial]+async fn test_ocr_systemic_failure_respects_fatal_flag() {+ let all = [200, 201, 202];+ match parse_with_failing_widths(&all, true).await {+ Err(err) => assert!(err.to_string().contains("EXPECTED_OCR_FAILURE")),+ Ok(_) => panic!("every page failing OCR must abort when fatal"),+ }++ let result = parse_with_failing_widths(&all, false)+ .await+ .expect("non-fatal mode must return partial results");+ assert_eq!(result.pages.len(), 3);+}+async fn diagonal_column_count(config: LiteParseConfig) -> usize {let stats = LiteParse::new(LiteParseConfig {quiet: true,diff --git a/packages/node/native.d.ts b/packages/node/native.d.tsindex 7bb16ba..f129b6f 100644--- a/packages/node/native.d.ts+++ b/packages/node/native.d.ts@@ -105,12 +105,9 @@ export interface JsLiteParseConfig {*/renderFormFields?: boolean/**- * Whether an OCR failure on a page with insufficient native text- * stops parsing. Default `true`: report the error instead of silently- * returning a page with missing OCR text.- * Set `false` to return partial results and keep the available native- * text, for example when the application has its own OCR fallback.- * Other parsing errors can still stop parsing.+ * Whether a systemic OCR failure aborts the whole parse (default true).+ * Set false to keep already-recovered native text and return partial+ * results when OCR is unavailable, instead of rejecting.*/ocrFailureFatal?: boolean/**diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.tsindex 8d83978..5767161 100644--- a/packages/node/src/lib.ts+++ b/packages/node/src/lib.ts@@ -95,12 +95,10 @@ export interface LiteParseConfig {quiet: boolean;numWorkers: number;/**- * Whether an OCR failure on a page with insufficient native text- * stops parsing. Default `true`: report the error instead of silently- * returning a page with missing OCR text.- * Set `false` to return partial results and keep the available native- * text, for example when the application has its own OCR fallback.- * Other parsing errors can still stop parsing.+ * Whether a systemic OCR failure (every OCR task failed and at least one was+ * a text-sparse page) aborts the whole parse (default: true). Set false to+ * keep already-recovered native text and return partial results instead of+ * rejecting — for callers that prefer a degraded document over a hard failure.*/ocrFailureFatal: boolean;/**diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.pyindex f26a341..256afbc 100644--- a/packages/python/liteparse/parser.py+++ b/packages/python/liteparse/parser.py@@ -550,12 +550,11 @@ class LiteParse:the rendered Markdown.extract_structure_tree: Include the tagged-PDF logical structuretree as page-scoped structured data (default: False).- ocr_failure_fatal: Whether an OCR failure on a page with insufficient- native text stops parsing. Default True: report the error instead- of silently returning a page with missing OCR text.- Set False to return partial results and keep the available native- text, for example when the application has its own OCR fallback.- Other parsing errors can still stop parsing.+ ocr_failure_fatal: Whether a systemic OCR failure (every OCR task+ failed and at least one was a text-sparse page) aborts the whole+ parse (default: True). Set False to keep already-recovered native+ text and return partial results instead of raising — for callers+ that prefer a degraded document over a hard failure.ocr_hedge_delays_ms: Request-hedging schedule for HTTP OCR, inmilliseconds. Empty or single-element means no hedging (onerequest per attempt — the default). With multiple delays (e.g.diff --git a/packages/wasm/README.md b/packages/wasm/README.mdindex 9258ba0..eb128e5 100644--- a/packages/wasm/README.md+++ b/packages/wasm/README.md@@ -115,10 +115,10 @@ concurrent HTTP requests. Results stay assigned to their source pages whenjobs finish out of order. Active calls can keep up to `numWorkers` renderedpage rasters in memory, so use a small value for high-DPI documents.-With `ocrFailureFatal: true` (the default), an OCR failure on a page with-insufficient native text rejects `parse()`, even if other pages succeed.-With `ocrFailureFatal: false`, parsing continues with the available text.-If parsing fails, OCR calls that have already started can continue.+A failed `recognize` call only affects its own page. With `ocrFailureFatal:+true` (the default), `parse()` rejects only when every OCR call failed and at+least one failed page had little native text. If parsing fails, OCR calls that+have already started can continue.## Building from sourcediff --git a/scripts/browser-compat/wasm-test.html b/scripts/browser-compat/wasm-test.htmlindex fce6be3..89c66d1 100644--- a/scripts/browser-compat/wasm-test.html+++ b/scripts/browser-compat/wasm-test.html@@ -289,36 +289,6 @@groupedParser.free();}- status.textContent = "testing a ready failure before refill";- const releaseReadyPair = deferred();- let readyPairCalls = 0;- const readyFailureParser = new LiteParse({- ocrEnabled: true, dpi: 72, quiet: true, numWorkers: 2, ocrFailureFatal: true,- ocrEngine: {- async recognize(_imageData, width) {- readyPairCalls += 1;- if (readyPairCalls === 2) releaseReadyPair.resolve();- await withDeadline(releaseReadyPair.promise, "The first two OCR pages did not start");- if (width === 73) throw new Error("READY_OCR_FAILURE");- return [{text: `READY_${width}`, bbox: [4, 4, width - 4, 24], confidence: 0.99}];- },- },- });- try {- let failure;- try {- await readyFailureParser.parse(buildTestPdf(4));- } catch (error) {- failure = String(error);- }- if (!failure?.includes("READY_OCR_FAILURE") || readyPairCalls !== 2) {- throw new Error(`A ready failure must stop refill: calls=${readyPairCalls}, error=${failure}`);- }- } finally {- releaseReadyPair.resolve();- readyFailureParser.free();- }-status.textContent = "testing more workers than pages";const allPagesStarted = deferred();let largePoolCalls = 0;@@ -390,84 +360,51 @@}status.textContent = "testing OCR failure policy";+ // `ocrFailureFatal` only rejects when every OCR call failed. One failed+ // page among successes never rejects, in either mode.let failureCases = 0;for (const numWorkers of [1, 2]) {for (const fatal of [true, false]) {- const secondStarted = deferred();- const thirdStarted = deferred();- const failed = deferred();- const releaseThird = deferred();- const thirdFinished = deferred();- const calls = [];- let thirdActive = false;- const failureParser = new LiteParse({- ocrEnabled: true, dpi: 72, quiet: true, outputFormat: "text",- numWorkers, ocrFailureFatal: fatal,- ocrEngine: {- async recognize(_imageData, width) {- calls.push(width);- if (width === 72 && numWorkers === 2) {- await secondStarted.promise;- }- if (width === 73) {- secondStarted.resolve();- if (numWorkers === 2) await thirdStarted.promise;- failed.resolve();- throw new Error("EXPECTED_OCR_PAGE_FAILURE");+ for (const failAll of [false, true]) {+ const failureParser = new LiteParse({+ ocrEnabled: true, dpi: 72, quiet: true, outputFormat: "text",+ numWorkers, ocrFailureFatal: fatal,+ ocrEngine: {+ async recognize(_imageData, width) {+ if (failAll || width === 73) throw new Error("EXPECTED_OCR_PAGE_FAILURE");+ return [{+ text: `OCR_SUCCESS_${width}`,+ bbox: [4, 4, width - 4, 24], confidence: 0.99,+ }];+ },+ },+ });+ try {+ const outcome = await withDeadline(+ failureParser.parse(buildTestPdf(3)).then(+ (result) => ({ result }), (error) => ({ error }),+ ),+ "OCR failure policy did not settle parse()",+ );+ const label = `numWorkers=${numWorkers} fatal=${fatal} failAll=${failAll}`;+ if (failAll && fatal) {+ if (!String(outcome.error).includes("EXPECTED_OCR_PAGE_FAILURE")) {+ throw new Error(`Expected a fatal OCR error (${label})`);}- if (width === 74) {- thirdActive = true;- thirdStarted.resolve();- try {- await releaseThird.promise;- } finally {- thirdActive = false;- thirdFinished.resolve();- }+ } else {+ if (outcome.error) throw new Error(`Unexpected rejection (${label}): ${outcome.error}`);+ const pages = outcome.result.pages;+ if (pages.length !== 3) throw new Error(`Expected 3 pages (${label})`);+ if (!failAll && (!pages[0].text.includes("OCR_SUCCESS_72") ||+ pages[1].text.trim() !== "" || !pages[2].text.includes("OCR_SUCCESS_74"))) {+ throw new Error(`Partial OCR failure did not preserve successful pages (${label})`);}- return [{- text: `OCR_SUCCESS_${width}`,- bbox: [4, 4, width - 4, 24], confidence: 0.99,- }];- },- },- });- const settled = failureParser.parse(buildTestPdf(3)).then(- (result) => ({ result }), (error) => ({ error }),- );- try {- await withDeadline(failed.promise, "The failing OCR page did not start");- if (!fatal) releaseThird.resolve();- const outcome = await withDeadline(settled, "OCR failure policy did not settle parse()");- if (fatal) {- if (!String(outcome.error).includes("EXPECTED_OCR_PAGE_FAILURE")) {- throw new Error(`Expected a fatal OCR error with ${numWorkers} worker(s)`);}- if (numWorkers === 1 && calls.join(",") !== "72,73") {- throw new Error("Serial OCR started another page after a fatal error");- }- if (numWorkers === 2 && !thirdActive) {- throw new Error("Fatal OCR did not reject while another call was active");- }- } else {- if (outcome.error) throw outcome.error;- const pages = outcome.result.pages;- if (pages.length !== 3 || !pages[0].text.includes("OCR_SUCCESS_72") ||- pages[1].text.trim() !== "" || !pages[2].text.includes("OCR_SUCCESS_74")) {- throw new Error("Nonfatal OCR did not preserve successful pages");- }- }- } finally {- secondStarted.resolve();- thirdStarted.resolve();- releaseThird.resolve();- await withDeadline(settled, "OCR cleanup did not settle parse()");- if (calls.includes(74)) {- await withDeadline(thirdFinished.promise, "The active OCR call did not finish");+ } finally {+ failureParser.free();}- failureParser.free();+ failureCases += 1;}- failureCases += 1;}}@@ -479,7 +416,7 @@resultEl.setAttribute("data-ocr-bytes", ocrByteLength);resultEl.setAttribute("data-ocr-max-concurrency", maxActiveOcrCalls);resultEl.setAttribute("data-ocr-failure-cases", failureCases);- resultEl.textContent = `OK: ${pageCount} pages, ${textLength} chars, OCR callback, concurrency, worker limits, grouped completion, ready failure, native text refill, and ${failureCases} failure cases passed`;+ resultEl.textContent = `OK: ${pageCount} pages, ${textLength} chars, OCR callback, concurrency, worker limits, grouped completion, native text refill, and ${failureCases} failure cases passed`;} catch (err) {status.textContent = "error";errorEl.style.display = "block";