A fast, helpful, and open-source document parser
feat: detect annotations in complexity calculation
Logan Markewich0413ea6856e0
diff --git a/README.md b/README.mdindex 70cd2ce..ff0a8a0 100644--- a/README.md+++ b/README.md@@ -295,7 +295,7 @@ lit is-complex document.pdf --compact | jq '[.[] | select(.needs_ocr) | .page_nuIt always prints per-page JSON to **stdout**, a human-readable verdict to **stderr**, andexits **non-zero when any page needs OCR**. Each page carries a `needs_ocr` verdict and alist of `reasons` (`scanned`, `no-text`, `sparse-text`, `embedded-images`, `garbled`,-`vector-text`).+`vector-text`, `annotation-text`).### Batch Parsingdiff --git a/crates/liteparse/README.md b/crates/liteparse/README.mdindex aded3ff..9fb3a48 100644--- a/crates/liteparse/README.md+++ b/crates/liteparse/README.md@@ -146,7 +146,7 @@ if pages.iter().any(|p| p.needs_ocr) {````reasons` is a `Vec<ComplexityReason>` (`Scanned`, `NoText`, `SparseText`,-`EmbeddedImages`, `Garbled`, `VectorText`); new variants may be added over time, so match+`EmbeddedImages`, `Garbled`, `VectorText`, `AnnotationText`); new variants may be added over time, so matchleniently.## Custom OCR Enginediff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rsindex 5e5def2..bbafe13 100644--- a/crates/liteparse/src/ocr_merge.rs+++ b/crates/liteparse/src/ocr_merge.rs@@ -57,7 +57,9 @@ pub enum ComplexityReason {/// no extractable text behind it — a scanned/photographed page.Scanned,/// Almost no extractable native text, and no full-page raster behind it- /// (a blank page, or a near-empty cover/divider).+ /// (a blank page, or a near-empty cover/divider). Check for a companion+ /// `AnnotationText` before treating the page as blank — it means the text+ /// is there, just outside the extractable surface.NoText,/// Some real text, but it covers very little of the page — typically a/// figure-heavy page with only thin captions.@@ -70,6 +72,13 @@ pub enum ComplexityReason {/// Text is painted as filled vector outlines, outside the text layer, so/// no native text items represent it.VectorText,+ /// The page's own content stream has (almost) no text, but an annotation+ /// paints text through its appearance stream. PDFium extracts the content+ /// stream only, so this text is invisible to the text-only path even+ /// though the page renders — and OCR does recover it. Always accompanied+ /// by `NoText` or `Scanned`; it tells a genuinely blank page apart from+ /// one whose body text simply lives out of reach.+ AnnotationText,}impl ComplexityReason {@@ -81,6 +90,7 @@ impl ComplexityReason {ComplexityReason::EmbeddedImages => "embedded-images",ComplexityReason::Garbled => "garbled",ComplexityReason::VectorText => "vector-text",+ ComplexityReason::AnnotationText => "annotation-text",}}}@@ -205,6 +215,13 @@ pub(crate) fn calculate_page_complexity(} else {ComplexityReason::NoText});+ // ...unless the text is there and simply out of PDFium's reach: text+ // painted by an annotation appearance stream renders (so OCR gets it)+ // but is never tokenized by the text API. Only checked on pages that+ // already look empty, so the annotation walk stays off the hot path.+ if page_obj.has_annotation_text() {+ reasons.push(ComplexityReason::AnnotationText);+ }} else if sparse_text {// There is real text, but it's too thin to be the whole page.reasons.push(ComplexityReason::SparseText);diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rsindex 186f2bd..15380d6 100644--- a/crates/liteparse/tests/integration_test.rs+++ b/crates/liteparse/tests/integration_test.rs@@ -1,6 +1,7 @@use std::path::Path;use liteparse::conversion::convert_data_to_pdf;+use liteparse::ocr_merge::ComplexityReason;use liteparse::types::PdfInput;use liteparse::{LiteParse, LiteParseConfig};use serial_test::serial;@@ -225,3 +226,29 @@ async fn test_concurrent_parse_does_not_crash() {// 16 tasks × 1 page eachassert_eq!(total, 16);}++/// A page whose only text is painted by an annotation's `/AP /N` appearance+/// stream extracts as empty (PDFium tokenizes the page content stream only),+/// so it must be distinguishable from a genuinely blank page. See issue #378.+#[tokio::test]+#[serial]+async fn test_annotation_text_complexity_reason() {+ let lit = LiteParse::new(LiteParseConfig::default());+ let stats = lit+ .is_complex(PdfInput::Path(+ "../../integration_tests_data/annotation_text.pdf".into(),+ ))+ .await+ .expect("is_complex should succeed");++ assert_eq!(stats.len(), 1);+ let page = &stats[0];+ assert_eq!(page.text_length, 0, "annotation text is not extractable");+ assert!(page.needs_ocr);+ assert!(page.reasons.contains(&ComplexityReason::NoText));+ assert!(+ page.reasons.contains(&ComplexityReason::AnnotationText),+ "expected annotation-text, got {:?}",+ page.reasons+ );+}diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rsindex b1fb7bb..be24f03 100644--- a/crates/pdfium-sys/src/dynamic.rs+++ b/crates/pdfium-sys/src/dynamic.rs@@ -366,6 +366,10 @@ pub struct PdfiumBindings {pub FPDFAnnot_GetLinkedAnnot:unsafe extern "C" fn(FPDF_ANNOTATION, FPDF_BYTESTRING) -> FPDF_ANNOTATION,pub FPDFAnnot_GetObjNum: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int,+ pub FPDFAnnot_GetFlags: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int,+ pub FPDFAnnot_GetObjectCount: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int,+ pub FPDFAnnot_GetObject:+ unsafe extern "C" fn(FPDF_ANNOTATION, std::os::raw::c_int) -> FPDF_PAGEOBJECT,pub FPDFAnnot_GetFormFieldFlags:unsafe extern "C" fn(FPDF_FORMHANDLE, FPDF_ANNOTATION) -> std::os::raw::c_int,pub FPDFAnnot_GetFormFieldName: unsafe extern "C" fn(@@ -629,6 +633,9 @@ impl PdfiumBindings {FPDFAnnot_GetLink: load_fn!(lib, "FPDFAnnot_GetLink"),FPDFAnnot_GetLinkedAnnot: load_fn!(lib, "FPDFAnnot_GetLinkedAnnot"),FPDFAnnot_GetObjNum: load_fn!(lib, "FPDFAnnot_GetObjNum"),+ FPDFAnnot_GetFlags: load_fn!(lib, "FPDFAnnot_GetFlags"),+ FPDFAnnot_GetObjectCount: load_fn!(lib, "FPDFAnnot_GetObjectCount"),+ FPDFAnnot_GetObject: load_fn!(lib, "FPDFAnnot_GetObject"),FPDFAnnot_GetFormFieldFlags: load_fn!(lib, "FPDFAnnot_GetFormFieldFlags"),FPDFAnnot_GetFormFieldName: load_fn!(lib, "FPDFAnnot_GetFormFieldName"),FPDFAnnot_GetFormFieldAlternateName: load_fn!(diff --git a/crates/pdfium/src/page.rs b/crates/pdfium/src/page.rsindex 46a1058..a967d39 100644--- a/crates/pdfium/src/page.rs+++ b/crates/pdfium/src/page.rs@@ -890,6 +890,51 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> {out}+ /// Whether any visible annotation paints text through its `/AP /N`+ /// appearance stream.+ ///+ /// PDFium's text API tokenizes the page content stream only — text drawn+ /// by an annotation appearance is rendered but never extracted. Pages+ /// authored that way (a common anti-copy / production-tool pattern) look+ /// identical to a blank page through [`Page::text_items`], so callers need+ /// this to tell "nothing here" from "content OCR can recover".+ ///+ /// Stops at the first text object found; hidden and popup annotations are+ /// skipped because they are not painted in the render OCR would see.+ pub fn has_annotation_text(&self) -> bool {+ let count = unsafe { ffi!(FPDFPage_GetAnnotCount(self.handle)) };+ for index in 0..count {+ let annot = unsafe { ffi!(FPDFPage_GetAnnot(self.handle, index)) };+ if annot.is_null() {+ continue;+ }+ let subtype = unsafe { ffi!(FPDFAnnot_GetSubtype(annot)) };+ let flags = unsafe { ffi!(FPDFAnnot_GetFlags(annot)) };+ let hidden = flags & pdfium_sys::FPDF_ANNOT_FLAG_HIDDEN as i32 != 0;+ let mut found = false;+ if !hidden && subtype != pdfium_sys::FPDF_ANNOT_POPUP as i32 {+ let object_count = unsafe { ffi!(FPDFAnnot_GetObjectCount(annot)) };+ for object_index in 0..object_count {+ let object = unsafe { ffi!(FPDFAnnot_GetObject(annot, object_index)) };+ if object.is_null() {+ continue;+ }+ if unsafe { ffi!(FPDFPageObj_GetType(object)) }+ == pdfium_sys::FPDF_PAGEOBJ_TEXT as i32+ {+ found = true;+ break;+ }+ }+ }+ unsafe { ffi!(FPDFPage_CloseAnnot(annot)) };+ if found {+ return true;+ }+ }+ false+ }+/// Enumerate AcroForm widget annotations and resolve their field values/// through PDFium's form-fill environment.pub fn form_fields(diff --git a/integration_tests_data/annotation_text.pdf b/integration_tests_data/annotation_text.pdfnew file mode 100644index 0000000..f282afd--- /dev/null+++ b/integration_tests_data/annotation_text.pdf@@ -0,0 +1,80 @@+%PDF-1.3+%����+1 0 obj+<<+/Producer (pypdf)+>>+endobj+2 0 obj+<<+/Type /Pages+/Count 1+/Kids [ 4 0 R ]+>>+endobj+3 0 obj+<<+/Type /Catalog+/Pages 2 0 R+>>+endobj+4 0 obj+<<+/Type /Page+/Resources <<+>>+/MediaBox [ 0.0 0.0 612 792 ]+/Parent 2 0 R+/Annots [ 6 0 R ]+>>+endobj+5 0 obj+<<+/Type /XObject+/Subtype /Form+/BBox [ 0.0 0.0 612 792 ]+/Resources <<+/Font <<+/Helv <<+/Type /Font+/Subtype /Type1+/BaseFont /Helvetica+>>+>>+>>+/Length 115+>>+stream+BT /Helv 12 Tf 50 700 Td (Text painted only in an annotation appearance stream, not the page content stream.) Tj ET+endstream+endobj+6 0 obj+<<+/Type /Annot+/Subtype /FreeText+/Rect [ 0.0 0.0 612 792 ]+/Contents (synthetic annotation\055only text)+/AP <<+/N 5 0 R+>>+/F 4+>>+endobj+xref+0 7+0000000000 65535 f+0000000015 00000 n+0000000054 00000 n+0000000113 00000 n+0000000162 00000 n+0000000274 00000 n+0000000586 00000 n+trailer+<<+/Size 7+/Root 3 0 R+/Info 1 0 R+>>+startxref+735+%%EOFdiff --git a/packages/node/README.md b/packages/node/README.mdindex 94552cb..12aeeb8 100644--- a/packages/node/README.md+++ b/packages/node/README.md@@ -159,7 +159,7 @@ for (const page of pages.filter((p) => p.needsOcr)) {````reasons` is one of `"scanned"`, `"no-text"`, `"sparse-text"`, `"embedded-images"`,-`"garbled"`, or `"vector-text"`. Raw bytes work here too.+`"garbled"`, `"vector-text"`, or `"annotation-text"`. Raw bytes work here too.## Supported Formatsdiff --git a/packages/python/README.md b/packages/python/README.mdindex 208dec7..434b5f2 100644--- a/packages/python/README.md+++ b/packages/python/README.md@@ -140,7 +140,7 @@ for page in pages:````reasons` is one of `"scanned"`, `"no-text"`, `"sparse-text"`, `"embedded-images"`,-`"garbled"`, or `"vector-text"`. Raw bytes work here too.+`"garbled"`, `"vector-text"`, or `"annotation-text"`. Raw bytes work here too.## Supported Formatsdiff --git a/packages/wasm/README.md b/packages/wasm/README.mdindex 0c0b14d..503e3f4 100644--- a/packages/wasm/README.md+++ b/packages/wasm/README.md@@ -52,7 +52,7 @@ if (pages.some((p) => p.needsOcr)) {````reasons` is one of `"scanned"`, `"no-text"`, `"sparse-text"`, `"embedded-images"`,-`"garbled"`, or `"vector-text"`.+`"garbled"`, `"vector-text"`, or `"annotation-text"`.## Config options