Skip to content

A fast, helpful, and open-source document parser

feat: detect annotations in complexity calculation

Logan Markewich0413ea6856e0
diff --git a/README.md b/README.md
index 70cd2ce..ff0a8a0 100644
--- a/README.md
+++ b/README.md
@@ -295,7 +295,7 @@ lit is-complex document.pdf --compact | jq '[.[] | select(.needs_ocr) | .page_nu
It always prints per-page JSON to **stdout**, a human-readable verdict to **stderr**, and
exits **non-zero when any page needs OCR**. Each page carries a `needs_ocr` verdict and a
list of `reasons` (`scanned`, `no-text`, `sparse-text`, `embedded-images`, `garbled`,
-`vector-text`).
+`vector-text`, `annotation-text`).
### Batch Parsing
diff --git a/crates/liteparse/README.md b/crates/liteparse/README.md
index aded3ff..9fb3a48 100644
--- a/crates/liteparse/README.md
+++ b/crates/liteparse/README.md
@@ -146,7 +146,7 @@ if pages.iter().any(|p| p.needs_ocr) {
```
`reasons` is a `Vec<ComplexityReason>` (`Scanned`, `NoText`, `SparseText`,
-`EmbeddedImages`, `Garbled`, `VectorText`); new variants may be added over time, so match
+`EmbeddedImages`, `Garbled`, `VectorText`, `AnnotationText`); new variants may be added over time, so match
leniently.
## Custom OCR Engine
diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs
index 5e5def2..bbafe13 100644
--- a/crates/liteparse/src/ocr_merge.rs
+++ b/crates/liteparse/src/ocr_merge.rs
@@ -57,7 +57,9 @@ pub enum ComplexityReason {
/// no extractable text behind it — a scanned/photographed page.
Scanned,
/// Almost no extractable native text, and no full-page raster behind it
- /// (a blank page, or a near-empty cover/divider).
+ /// (a blank page, or a near-empty cover/divider). Check for a companion
+ /// `AnnotationText` before treating the page as blank — it means the text
+ /// is there, just outside the extractable surface.
NoText,
/// Some real text, but it covers very little of the page — typically a
/// figure-heavy page with only thin captions.
@@ -70,6 +72,13 @@ pub enum ComplexityReason {
/// Text is painted as filled vector outlines, outside the text layer, so
/// no native text items represent it.
VectorText,
+ /// The page's own content stream has (almost) no text, but an annotation
+ /// paints text through its appearance stream. PDFium extracts the content
+ /// stream only, so this text is invisible to the text-only path even
+ /// though the page renders — and OCR does recover it. Always accompanied
+ /// by `NoText` or `Scanned`; it tells a genuinely blank page apart from
+ /// one whose body text simply lives out of reach.
+ AnnotationText,
}
impl ComplexityReason {
@@ -81,6 +90,7 @@ impl ComplexityReason {
ComplexityReason::EmbeddedImages => "embedded-images",
ComplexityReason::Garbled => "garbled",
ComplexityReason::VectorText => "vector-text",
+ ComplexityReason::AnnotationText => "annotation-text",
}
}
}
@@ -205,6 +215,13 @@ pub(crate) fn calculate_page_complexity(
} else {
ComplexityReason::NoText
});
+ // ...unless the text is there and simply out of PDFium's reach: text
+ // painted by an annotation appearance stream renders (so OCR gets it)
+ // but is never tokenized by the text API. Only checked on pages that
+ // already look empty, so the annotation walk stays off the hot path.
+ if page_obj.has_annotation_text() {
+ reasons.push(ComplexityReason::AnnotationText);
+ }
} else if sparse_text {
// There is real text, but it's too thin to be the whole page.
reasons.push(ComplexityReason::SparseText);
diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs
index 186f2bd..15380d6 100644
--- a/crates/liteparse/tests/integration_test.rs
+++ b/crates/liteparse/tests/integration_test.rs
@@ -1,6 +1,7 @@
use std::path::Path;
use liteparse::conversion::convert_data_to_pdf;
+use liteparse::ocr_merge::ComplexityReason;
use liteparse::types::PdfInput;
use liteparse::{LiteParse, LiteParseConfig};
use serial_test::serial;
@@ -225,3 +226,29 @@ async fn test_concurrent_parse_does_not_crash() {
// 16 tasks × 1 page each
assert_eq!(total, 16);
}
+
+/// A page whose only text is painted by an annotation's `/AP /N` appearance
+/// stream extracts as empty (PDFium tokenizes the page content stream only),
+/// so it must be distinguishable from a genuinely blank page. See issue #378.
+#[tokio::test]
+#[serial]
+async fn test_annotation_text_complexity_reason() {
+ let lit = LiteParse::new(LiteParseConfig::default());
+ let stats = lit
+ .is_complex(PdfInput::Path(
+ "../../integration_tests_data/annotation_text.pdf".into(),
+ ))
+ .await
+ .expect("is_complex should succeed");
+
+ assert_eq!(stats.len(), 1);
+ let page = &stats[0];
+ assert_eq!(page.text_length, 0, "annotation text is not extractable");
+ assert!(page.needs_ocr);
+ assert!(page.reasons.contains(&ComplexityReason::NoText));
+ assert!(
+ page.reasons.contains(&ComplexityReason::AnnotationText),
+ "expected annotation-text, got {:?}",
+ page.reasons
+ );
+}
diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rs
index b1fb7bb..be24f03 100644
--- a/crates/pdfium-sys/src/dynamic.rs
+++ b/crates/pdfium-sys/src/dynamic.rs
@@ -366,6 +366,10 @@ pub struct PdfiumBindings {
pub FPDFAnnot_GetLinkedAnnot:
unsafe extern "C" fn(FPDF_ANNOTATION, FPDF_BYTESTRING) -> FPDF_ANNOTATION,
pub FPDFAnnot_GetObjNum: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int,
+ pub FPDFAnnot_GetFlags: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int,
+ pub FPDFAnnot_GetObjectCount: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int,
+ pub FPDFAnnot_GetObject:
+ unsafe extern "C" fn(FPDF_ANNOTATION, std::os::raw::c_int) -> FPDF_PAGEOBJECT,
pub FPDFAnnot_GetFormFieldFlags:
unsafe extern "C" fn(FPDF_FORMHANDLE, FPDF_ANNOTATION) -> std::os::raw::c_int,
pub FPDFAnnot_GetFormFieldName: unsafe extern "C" fn(
@@ -629,6 +633,9 @@ impl PdfiumBindings {
FPDFAnnot_GetLink: load_fn!(lib, "FPDFAnnot_GetLink"),
FPDFAnnot_GetLinkedAnnot: load_fn!(lib, "FPDFAnnot_GetLinkedAnnot"),
FPDFAnnot_GetObjNum: load_fn!(lib, "FPDFAnnot_GetObjNum"),
+ FPDFAnnot_GetFlags: load_fn!(lib, "FPDFAnnot_GetFlags"),
+ FPDFAnnot_GetObjectCount: load_fn!(lib, "FPDFAnnot_GetObjectCount"),
+ FPDFAnnot_GetObject: load_fn!(lib, "FPDFAnnot_GetObject"),
FPDFAnnot_GetFormFieldFlags: load_fn!(lib, "FPDFAnnot_GetFormFieldFlags"),
FPDFAnnot_GetFormFieldName: load_fn!(lib, "FPDFAnnot_GetFormFieldName"),
FPDFAnnot_GetFormFieldAlternateName: load_fn!(
diff --git a/crates/pdfium/src/page.rs b/crates/pdfium/src/page.rs
index 46a1058..a967d39 100644
--- a/crates/pdfium/src/page.rs
+++ b/crates/pdfium/src/page.rs
@@ -890,6 +890,51 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> {
out
}
+ /// Whether any visible annotation paints text through its `/AP /N`
+ /// appearance stream.
+ ///
+ /// PDFium's text API tokenizes the page content stream only — text drawn
+ /// by an annotation appearance is rendered but never extracted. Pages
+ /// authored that way (a common anti-copy / production-tool pattern) look
+ /// identical to a blank page through [`Page::text_items`], so callers need
+ /// this to tell "nothing here" from "content OCR can recover".
+ ///
+ /// Stops at the first text object found; hidden and popup annotations are
+ /// skipped because they are not painted in the render OCR would see.
+ pub fn has_annotation_text(&self) -> bool {
+ let count = unsafe { ffi!(FPDFPage_GetAnnotCount(self.handle)) };
+ for index in 0..count {
+ let annot = unsafe { ffi!(FPDFPage_GetAnnot(self.handle, index)) };
+ if annot.is_null() {
+ continue;
+ }
+ let subtype = unsafe { ffi!(FPDFAnnot_GetSubtype(annot)) };
+ let flags = unsafe { ffi!(FPDFAnnot_GetFlags(annot)) };
+ let hidden = flags & pdfium_sys::FPDF_ANNOT_FLAG_HIDDEN as i32 != 0;
+ let mut found = false;
+ if !hidden && subtype != pdfium_sys::FPDF_ANNOT_POPUP as i32 {
+ let object_count = unsafe { ffi!(FPDFAnnot_GetObjectCount(annot)) };
+ for object_index in 0..object_count {
+ let object = unsafe { ffi!(FPDFAnnot_GetObject(annot, object_index)) };
+ if object.is_null() {
+ continue;
+ }
+ if unsafe { ffi!(FPDFPageObj_GetType(object)) }
+ == pdfium_sys::FPDF_PAGEOBJ_TEXT as i32
+ {
+ found = true;
+ break;
+ }
+ }
+ }
+ unsafe { ffi!(FPDFPage_CloseAnnot(annot)) };
+ if found {
+ return true;
+ }
+ }
+ false
+ }
+
/// Enumerate AcroForm widget annotations and resolve their field values
/// through PDFium's form-fill environment.
pub fn form_fields(
diff --git a/integration_tests_data/annotation_text.pdf b/integration_tests_data/annotation_text.pdf
new file mode 100644
index 0000000..f282afd
--- /dev/null
+++ b/integration_tests_data/annotation_text.pdf
@@ -0,0 +1,80 @@
+%PDF-1.3
+%����
+1 0 obj
+<<
+/Producer (pypdf)
+>>
+endobj
+2 0 obj
+<<
+/Type /Pages
+/Count 1
+/Kids [ 4 0 R ]
+>>
+endobj
+3 0 obj
+<<
+/Type /Catalog
+/Pages 2 0 R
+>>
+endobj
+4 0 obj
+<<
+/Type /Page
+/Resources <<
+>>
+/MediaBox [ 0.0 0.0 612 792 ]
+/Parent 2 0 R
+/Annots [ 6 0 R ]
+>>
+endobj
+5 0 obj
+<<
+/Type /XObject
+/Subtype /Form
+/BBox [ 0.0 0.0 612 792 ]
+/Resources <<
+/Font <<
+/Helv <<
+/Type /Font
+/Subtype /Type1
+/BaseFont /Helvetica
+>>
+>>
+>>
+/Length 115
+>>
+stream
+BT /Helv 12 Tf 50 700 Td (Text painted only in an annotation appearance stream, not the page content stream.) Tj ET
+endstream
+endobj
+6 0 obj
+<<
+/Type /Annot
+/Subtype /FreeText
+/Rect [ 0.0 0.0 612 792 ]
+/Contents (synthetic annotation\055only text)
+/AP <<
+/N 5 0 R
+>>
+/F 4
+>>
+endobj
+xref
+0 7
+0000000000 65535 f
+0000000015 00000 n
+0000000054 00000 n
+0000000113 00000 n
+0000000162 00000 n
+0000000274 00000 n
+0000000586 00000 n
+trailer
+<<
+/Size 7
+/Root 3 0 R
+/Info 1 0 R
+>>
+startxref
+735
+%%EOF
diff --git a/packages/node/README.md b/packages/node/README.md
index 94552cb..12aeeb8 100644
--- a/packages/node/README.md
+++ b/packages/node/README.md
@@ -159,7 +159,7 @@ for (const page of pages.filter((p) => p.needsOcr)) {
```
`reasons` is one of `"scanned"`, `"no-text"`, `"sparse-text"`, `"embedded-images"`,
-`"garbled"`, or `"vector-text"`. Raw bytes work here too.
+`"garbled"`, `"vector-text"`, or `"annotation-text"`. Raw bytes work here too.
## Supported Formats
diff --git a/packages/python/README.md b/packages/python/README.md
index 208dec7..434b5f2 100644
--- a/packages/python/README.md
+++ b/packages/python/README.md
@@ -140,7 +140,7 @@ for page in pages:
```
`reasons` is one of `"scanned"`, `"no-text"`, `"sparse-text"`, `"embedded-images"`,
-`"garbled"`, or `"vector-text"`. Raw bytes work here too.
+`"garbled"`, `"vector-text"`, or `"annotation-text"`. Raw bytes work here too.
## Supported Formats
diff --git a/packages/wasm/README.md b/packages/wasm/README.md
index 0c0b14d..503e3f4 100644
--- a/packages/wasm/README.md
+++ b/packages/wasm/README.md
@@ -52,7 +52,7 @@ if (pages.some((p) => p.needsOcr)) {
```
`reasons` is one of `"scanned"`, `"no-text"`, `"sparse-text"`, `"embedded-images"`,
-`"garbled"`, or `"vector-text"`.
+`"garbled"`, `"vector-text"`, or `"annotation-text"`.
## Config options