A fast, helpful, and open-source document parser
fix: extract visible AcroForm values as text
Neel Patel8ad92a3d02fb
diff --git a/.gitignore b/.gitignoreindex a620ebc..aee1078 100644--- a/.gitignore+++ b/.gitignore@@ -45,6 +45,7 @@ temp/!integration_tests_data/receipt.png!integration_tests_data/sample3.docx.doc!integration_tests_data/sample.pdf+!integration_tests_data/filled_acroform.pdfdeno.json# Python filesdiff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex c6cdf16..09efcf1 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -66,7 +66,8 @@ pub(crate) fn extract_pages_from_document(/// raster image object to bytes (when `output_options.extract_images` is true). Returned/// `ExtractedImage`s carry the same ids the markdown emitter will reference,/// so callers can match them up by id. When image extraction is disabled the-/// returned image vec is always empty.+/// returned image vec is always empty. The final boolean reports whether a+/// page was flattened, which mutates the open PDFium document.pub(crate) fn extract_pages_and_images(document: &Document,target_pages: Option<&[u32]>,@@ -74,12 +75,13 @@ pub(crate) fn extract_pages_and_images(extract_links: bool,glyph_resolver: Option<&dyn crate::GlyphResolver>,output_options: ExtractionOutputOptions,-) -> Result<(Vec<LitePage>, Vec<ExtractedImage>, u32), LiteParseError> {+) -> Result<(Vec<LitePage>, Vec<ExtractedImage>, u32, bool), LiteParseError> {let page_count = document.page_count();let mut pages = Vec::new();let mut images: Vec<ExtractedImage> = Vec::new();let mut image_cache = ImageCache::default();let mut image_error_count = 0u32;+ let mut flattened_form_widgets = false;let form_environment = output_options.extract_form_fields.then(|| document.form_environment())@@ -99,24 +101,21 @@ pub(crate) fn extract_pages_and_images(}let page = document.page(page_index)?;- let text_page = page.text()?;+ let page_width = page.width();+ let page_height = page.height();let view_box = page.view_box().unwrap_or(RectF {left: 0.0,- top: page.height(),- right: page.width(),+ top: page_height,+ right: page_width,bottom: 0.0,});- let mut text_items = extract_page_text_items(- &page,- &text_page,- &view_box,- glyph_resolver,- output_options.emit_word_boxes,- output_options.extract_text_metadata,- )?;- if extract_links {- assign_links(&mut text_items, &page.links(&view_box));- }+ // Once a qualifying widget is found, PDFium flattens every visible+ // annotation on the page. Collect every annotation-backed output first.+ let links = if extract_links {+ page.links(&view_box)+ } else {+ Vec::new()+ };// Computed when emitted (`extract_content_bounds`) or needed// internally by the white-fill heuristic (`extract_vector_graphics`).let content_bounds = (output_options.extract_content_bounds@@ -131,7 +130,6 @@ pub(crate) fn extract_pages_and_images(let vector_graphics = output_options.extract_vector_graphics.then(|| build_vector_graphics(&paths, content_bounds.as_ref()));- assign_strikethrough(&mut text_items, &graphics);let struct_nodes = extract_page_struct_nodes(&page, &view_box);let extracted_refs =extract_page_image_refs(&page, page_number, output_options.extract_images);@@ -195,10 +193,36 @@ pub(crate) fn extract_pages_and_images(}}+ // PDFium's text API reads only the page content stream. Filled form+ // values commonly live in widget appearance streams, so flatten the+ // page's visible annotations in memory and reload before extracting text.+ // This does not initialize the form environment or execute document JS.+ let extract_text = |page: &Page| -> Result<Vec<TextItem>, LiteParseError> {+ let text_page = page.text()?;+ extract_page_text_items(+ page,+ &text_page,+ &view_box,+ glyph_resolver,+ output_options.emit_word_boxes,+ output_options.extract_text_metadata,+ )+ };+ let mut text_items = if page.has_form_widget_text() && page.flatten_for_display() {+ flattened_form_widgets = true;+ drop(page);+ let flattened_page = document.page(page_index)?;+ extract_text(&flattened_page)?+ } else {+ extract_text(&page)?+ };+ assign_links(&mut text_items, &links);+ assign_strikethrough(&mut text_items, &graphics);+pages.push(LitePage {page_number: page_number as usize,- page_width: page.width(),- page_height: page.height(),+ page_width,+ page_height,content_bounds: output_options.extract_content_bounds.then_some(content_bounds)@@ -214,7 +238,7 @@ pub(crate) fn extract_pages_and_images(});}- Ok((pages, images, image_error_count))+ Ok((pages, images, image_error_count, flattened_form_widgets))}#[derive(Debug, Clone, Copy, Default)]diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rsindex 5f21718..278ea0e 100644--- a/crates/liteparse/src/parser.rs+++ b/crates/liteparse/src/parser.rs@@ -283,7 +283,7 @@ impl LiteParse {let lib = Library::init();let document = extract::load_document_from_input(&lib, &validated_input, password)?;- let (pages, _, _) = extract::extract_pages_and_images(+ let (pages, _, _, flattened_form_widgets) = extract::extract_pages_and_images(&document,target_pages.as_deref(),self.config.max_pages,@@ -291,6 +291,13 @@ impl LiteParse {self.glyph_resolver.as_deref(),extract::ExtractionOutputOptions::default(),)?;+ // Form flattening promotes appearances into page content by+ // mutating the PDFium document. Complexity still needs the+ // original annotations, so reopen the input when that happened.+ let pristine_document = flattened_form_widgets+ .then(|| extract::load_document_from_input(&lib, &validated_input, password))+ .transpose()?;+ let complexity_document = pristine_document.as_ref().unwrap_or(&document);let t_extract = web_time::Instant::now();log(&format!("[liteparse] extract: {:.1}ms ({} pages)",@@ -301,7 +308,7 @@ impl LiteParse {let page_complexities = pages.iter().map(|page| {- let page_obj = document.page((page.page_number - 1) as i32)?;+ let page_obj = complexity_document.page((page.page_number - 1) as i32)?;ocr_merge::calculate_page_complexity(page, &page_obj)}).collect::<Result<Vec<_>, _>>()?;@@ -499,28 +506,39 @@ impl LiteParse {.collect::<Vec<_>>()});let outline = extract::extract_outline(&document);- let (pages, images, image_error_count) = extract::extract_pages_and_images(- &document,- target_pages.as_deref(),- self.config.max_pages,- self.config.extract_links- && self.config.output_format == crate::config::OutputFormat::Markdown,- self.glyph_resolver.as_deref(),- extract::ExtractionOutputOptions {- extract_content_bounds: self.config.extract_content_bounds,- extract_images: self.config.effective_extract_images(),- // The markdown table detector splits PDFium's merged- // multi-cell runs on real word geometry, so it needs word- // boxes even when the caller didn't ask for them.- emit_word_boxes: self.config.emit_word_boxes- || self.config.output_format == crate::config::OutputFormat::Markdown,- extract_text_metadata: self.config.extract_text_metadata,- extract_vector_graphics: self.config.extract_vector_graphics,- extract_annotations: self.config.extract_annotations,- extract_form_fields: self.config.extract_form_fields,- extract_structure_tree: self.config.extract_structure_tree,- },- )?;+ let (pages, images, image_error_count, flattened_form_widgets) =+ extract::extract_pages_and_images(+ &document,+ target_pages.as_deref(),+ self.config.max_pages,+ self.config.extract_links+ && self.config.output_format == crate::config::OutputFormat::Markdown,+ self.glyph_resolver.as_deref(),+ extract::ExtractionOutputOptions {+ extract_content_bounds: self.config.extract_content_bounds,+ extract_images: self.config.effective_extract_images(),+ // The markdown table detector splits PDFium's merged+ // multi-cell runs on real word geometry, so it needs word+ // boxes even when the caller didn't ask for them.+ emit_word_boxes: self.config.emit_word_boxes+ || self.config.output_format == crate::config::OutputFormat::Markdown,+ extract_text_metadata: self.config.extract_text_metadata,+ extract_vector_graphics: self.config.extract_vector_graphics,+ extract_annotations: self.config.extract_annotations,+ extract_form_fields: self.config.extract_form_fields,+ extract_structure_tree: self.config.extract_structure_tree,+ },+ )?;+ // Flattening is deliberately limited to text extraction. OCR and+ // complexity must retain the original annotations/widgets so the+ // opt-in form renderer can still run document actions and paint+ // computed field appearances.+ let needs_pristine_document = flattened_form_widgets+ && (self.config.ocr_enabled || self.config.include_complexity);+ let pristine_document = needs_pristine_document+ .then(|| extract::load_document_from_input(&lib, document_input, password))+ .transpose()?;+ let analysis_document = pristine_document.as_ref().unwrap_or(&document);let t_extract = web_time::Instant::now();log(&format!("[liteparse] extract: {:.1}ms ({} pages)",@@ -529,7 +547,7 @@ impl LiteParse {));let rendered = if self.config.ocr_enabled {let r = ocr_merge::render_pages_for_ocr(- &document,+ analysis_document,&pages,self.config.dpi,ocr_grayscale,@@ -552,7 +570,7 @@ impl LiteParse {pages.iter().map(|page| {- let page_obj = document.page((page.page_number - 1) as i32)?;+ let page_obj = analysis_document.page((page.page_number - 1) as i32)?;ocr_merge::calculate_page_complexity(page, &page_obj)}).collect::<Result<Vec<_>, _>>()?diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rsindex 71e40dc..db9c749 100644--- a/crates/liteparse/tests/integration_test.rs+++ b/crates/liteparse/tests/integration_test.rs@@ -1,5 +1,6 @@use std::path::Path;+use liteparse::config::OutputFormat;use liteparse::conversion::convert_data_to_pdf;use liteparse::ocr_merge::ComplexityReason;use liteparse::types::PdfInput;@@ -286,3 +287,96 @@ async fn test_annotation_text_complexity_reason() {page.reasons);}++/// Filled AcroForm values are visible page content even though PDFium's text+/// API does not expose widget appearance streams until they are flattened.+#[tokio::test]+#[serial]+async fn test_filled_acroform_values_are_extracted_as_text() {+ let lit = LiteParse::new(LiteParseConfig {+ ocr_enabled: false,+ output_format: OutputFormat::Markdown,+ ..Default::default()+ });+ let parsed = lit+ .parse("../../integration_tests_data/filled_acroform.pdf")+ .await+ .expect("filled form should parse");++ for expected in ["ACROFORM-CUSTOMER-7319", "2026-07-28", "50.00"] {+ assert_eq!(+ parsed.text.matches(expected).count(),+ 1,+ "visible form value should appear exactly once: {expected}"+ );+ assert!(+ parsed.pages[0]+ .text_items+ .iter()+ .any(|item| item.text.contains(expected)),+ "form value should be a positioned text item: {expected}"+ );+ }+ assert!(+ !parsed.text.contains("DEFAULT-ONLY-SHOULD-NOT-APPEAR"),+ "an unpainted default choice must not be treated as a filled value"+ );+ assert!(+ parsed.pages[0].form_fields.is_none(),+ "default text extraction must not enable structured form metadata"+ );++ let with_metadata = LiteParse::new(LiteParseConfig {+ ocr_enabled: false,+ output_format: OutputFormat::Markdown,+ extract_annotations: true,+ extract_form_fields: true,+ ..Default::default()+ })+ .parse("../../integration_tests_data/filled_acroform.pdf")+ .await+ .expect("filled form should parse with structured metadata");++ assert_eq!(+ with_metadata.pages[0].annotations.as_ref().unwrap().len(),+ 5+ );+ let fields = with_metadata.pages[0].form_fields.as_ref().unwrap();+ assert_eq!(fields.len(), 5);+ assert!(fields.iter().any(|field| {+ field.name.as_deref() == Some("customer_name")+ && field.value.as_deref() == Some("ACROFORM-CUSTOMER-7319")+ }));+ assert!(fields.iter().any(|field| {+ field.name.as_deref() == Some("default_only_choice")+ && field.value.as_deref() == Some("DEFAULT-ONLY-SHOULD-NOT-APPEAR")+ }));+ assert_eq!(+ with_metadata.pages[1].annotations.as_ref().unwrap().len(),+ 1+ );+ let second_page_fields = with_metadata.pages[1].form_fields.as_ref().unwrap();+ assert_eq!(second_page_fields.len(), 1);+ assert_eq!(+ second_page_fields[0].name.as_deref(),+ Some("complexity_sentinel")+ );+ assert_eq!(second_page_fields[0].value.as_deref(), Some("OK"));++ let complexity = LiteParse::new(LiteParseConfig {+ ocr_enabled: false,+ ..Default::default()+ })+ .is_complex(PdfInput::Path(+ "../../integration_tests_data/filled_acroform.pdf".into(),+ ))+ .await+ .expect("complexity analysis should use an unmodified document");+ assert_eq!(complexity.len(), 2);+ assert!(+ complexity[1]+ .reasons+ .contains(&ComplexityReason::AnnotationText),+ "the pristine second page must retain its widget annotation"+ );+}diff --git a/crates/pdfium-sys/bindings.rs b/crates/pdfium-sys/bindings.rsindex ed73e49..e6f46c1 100644--- a/crates/pdfium-sys/bindings.rs+++ b/crates/pdfium-sys/bindings.rs@@ -99,6 +99,11 @@ pub const FPDF_FORMFIELD_LISTBOX: u32 = 5;pub const FPDF_FORMFIELD_TEXTFIELD: u32 = 6;pub const FPDF_FORMFIELD_SIGNATURE: u32 = 7;pub const FPDF_FORMFIELD_COUNT: u32 = 8;+pub const FLATTEN_FAIL: u32 = 0;+pub const FLATTEN_SUCCESS: u32 = 1;+pub const FLATTEN_NOTHINGTODO: u32 = 2;+pub const FLAT_NORMALDISPLAY: u32 = 0;+pub const FLAT_PRINT: u32 = 1;pub const FPDF_ANNOT_UNKNOWN: u32 = 0;pub const FPDF_ANNOT_TEXT: u32 = 1;pub const FPDF_ANNOT_LINK: u32 = 2;@@ -1055,6 +1060,12 @@ pub struct FPDF_IMAGEOBJ_METADATA {pub colorspace: ::std::os::raw::c_int,pub marked_content_id: ::std::os::raw::c_int,}+unsafe extern "C" {+ pub fn FPDFPage_Flatten(+ page: FPDF_PAGE,+ nFlag: ::std::os::raw::c_int,+ ) -> ::std::os::raw::c_int;+}unsafe extern "C" {pub fn FPDF_CreateNewDocument() -> FPDF_DOCUMENT;}diff --git a/crates/pdfium-sys/build.rs b/crates/pdfium-sys/build.rsindex 7cfdbb4..dc507b4 100644--- a/crates/pdfium-sys/build.rs+++ b/crates/pdfium-sys/build.rs@@ -277,6 +277,7 @@ fn run_bindgen(include_dir: &Path) {.allowlist_type("FPDF.*").allowlist_type("FS_.*").allowlist_var("FPDF.*")+ .allowlist_var("FLAT.*").derive_debug(true).derive_default(true).layout_tests(false)diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rsindex 606954f..1fa2f60 100644--- a/crates/pdfium-sys/src/dynamic.rs+++ b/crates/pdfium-sys/src/dynamic.rs@@ -116,6 +116,8 @@ pub struct PdfiumBindings {pub FPDF_GetPageHeightF: unsafe extern "C" fn(FPDF_PAGE) -> f32,pub FPDF_GetPageBoundingBox: unsafe extern "C" fn(FPDF_PAGE, *mut FS_RECTF) -> FPDF_BOOL,pub FPDFPage_GetRotation: unsafe extern "C" fn(FPDF_PAGE) -> std::os::raw::c_int,+ pub FPDFPage_Flatten:+ unsafe extern "C" fn(FPDF_PAGE, std::os::raw::c_int) -> std::os::raw::c_int,pub FPDF_PageToDevice: unsafe extern "C" fn(FPDF_PAGE,std::os::raw::c_int,@@ -570,6 +572,7 @@ impl PdfiumBindings {FPDF_GetPageHeightF: load_fn!(lib, "FPDF_GetPageHeightF"),FPDF_GetPageBoundingBox: load_fn!(lib, "FPDF_GetPageBoundingBox"),FPDFPage_GetRotation: load_fn!(lib, "FPDFPage_GetRotation"),+ FPDFPage_Flatten: load_fn!(lib, "FPDFPage_Flatten"),FPDF_PageToDevice: load_fn!(lib, "FPDF_PageToDevice"),FPDFPage_CountObjects: load_fn!(lib, "FPDFPage_CountObjects"),FPDFPage_GetObject: load_fn!(lib, "FPDFPage_GetObject"),diff --git a/crates/pdfium-sys/wrapper.h b/crates/pdfium-sys/wrapper.hindex 46d184d..10cbd3f 100644--- a/crates/pdfium-sys/wrapper.h+++ b/crates/pdfium-sys/wrapper.h@@ -6,3 +6,4 @@#include "fpdf_structtree.h"#include "fpdf_transformpage.h"#include "fpdf_signature.h"+#include "fpdf_flatten.h"diff --git a/crates/pdfium/src/page.rs b/crates/pdfium/src/page.rsindex a967d39..805e799 100644--- a/crates/pdfium/src/page.rs+++ b/crates/pdfium/src/page.rs@@ -909,24 +909,28 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> {continue;}let subtype = unsafe { ffi!(FPDFAnnot_GetSubtype(annot)) };- let flags = unsafe { ffi!(FPDFAnnot_GetFlags(annot)) };- let hidden = flags & pdfium_sys::FPDF_ANNOT_FLAG_HIDDEN as i32 != 0;- let mut found = false;- if !hidden && subtype != pdfium_sys::FPDF_ANNOT_POPUP as i32 {- let object_count = unsafe { ffi!(FPDFAnnot_GetObjectCount(annot)) };- for object_index in 0..object_count {- let object = unsafe { ffi!(FPDFAnnot_GetObject(annot, object_index)) };- if object.is_null() {- continue;- }- if unsafe { ffi!(FPDFPageObj_GetType(object)) }- == pdfium_sys::FPDF_PAGEOBJ_TEXT as i32- {- found = true;- break;- }- }+ let found =+ subtype != pdfium_sys::FPDF_ANNOT_POPUP as i32 && annotation_paints_text(annot);+ unsafe { ffi!(FPDFPage_CloseAnnot(annot)) };+ if found {+ return true;+ }+ }+ false+ }++ /// Whether a visible AcroForm widget paints text through its appearance.+ /// PDFium's page text API omits these glyphs until the page is flattened.+ pub fn has_form_widget_text(&self) -> bool {+ let count = unsafe { ffi!(FPDFPage_GetAnnotCount(self.handle)) };+ for index in 0..count {+ let annot = unsafe { ffi!(FPDFPage_GetAnnot(self.handle, index)) };+ if annot.is_null() {+ continue;}+ let found = unsafe { ffi!(FPDFAnnot_GetSubtype(annot)) }+ == pdfium_sys::FPDF_ANNOT_WIDGET as i32+ && annotation_paints_text(annot);unsafe { ffi!(FPDFPage_CloseAnnot(annot)) };if found {return true;@@ -935,6 +939,17 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> {false}+ /// Promote visible annotation and form appearances into page content.+ /// Returns true only when PDFium changed the page.+ pub fn flatten_for_display(&self) -> bool {+ unsafe {+ ffi!(FPDFPage_Flatten(+ self.handle,+ pdfium_sys::FLAT_NORMALDISPLAY as i32+ )) == pdfium_sys::FLATTEN_SUCCESS as i32+ }+ }+/// Enumerate AcroForm widget annotations and resolve their field values/// through PDFium's form-fill environment.pub fn form_fields(@@ -1087,6 +1102,23 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> {}}+fn annotation_paints_text(annot: pdfium_sys::FPDF_ANNOTATION) -> bool {+ let flags = unsafe { ffi!(FPDFAnnot_GetFlags(annot)) };+ let suppressed = pdfium_sys::FPDF_ANNOT_FLAG_INVISIBLE+ | pdfium_sys::FPDF_ANNOT_FLAG_HIDDEN+ | pdfium_sys::FPDF_ANNOT_FLAG_NOVIEW;+ if flags & suppressed as i32 != 0 {+ return false;+ }++ let object_count = unsafe { ffi!(FPDFAnnot_GetObjectCount(annot)) };+ (0..object_count).any(|object_index| {+ let object = unsafe { ffi!(FPDFAnnot_GetObject(annot, object_index)) };+ !object.is_null()+ && unsafe { ffi!(FPDFPageObj_GetType(object)) } == pdfium_sys::FPDF_PAGEOBJ_TEXT as i32+ })+}+fn form_field_type_name(field_type: i32) -> &'static str {match field_type as u32 {pdfium_sys::FPDF_FORMFIELD_PUSHBUTTON => "pushbutton",diff --git a/integration_tests_data/filled_acroform.pdf b/integration_tests_data/filled_acroform.pdfnew file mode 100644index 0000000..0bcd365Binary files /dev/null and b/integration_tests_data/filled_acroform.pdf differ