A fast, helpful, and open-source document parser
fix: exclude glyphs hidden by rectangular PDF clips
Pierre-Loic doulcetaed33ebc0849
diff --git a/AGENTS.md b/AGENTS.mdindex 5bb3424..137e84d 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -124,6 +124,7 @@ Rather than implementing format parsers, LiteParse converts office file formatsKey files in `crates/liteparse/src/`:- `projection.rs` — Layout reconstruction (most complex)- `extract.rs` — Raw text item extraction from PDFium+- `text_clip.rs` — Shared glyph visibility: PDFium clip points already include the object matrix; compose only ancestor form matrices and inherit ancestor clips.- `ocr_merge.rs` — Merging OCR and native text### Adding CLI Optionsdiff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex acd8b3a..00cc7ca 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -1512,6 +1512,7 @@ fn extract_page_text_items(eprintln!("[extract-debug] char_count={char_count}, skip_invisible={skip_invisible}");}+ let clips = crate::text_clip::TextClip::new(page);let page_rotation = page.rotation();let vp_xform = page.viewport_transform(view_box);let mut items: Vec<TextItem> = Vec::new();@@ -1529,6 +1530,10 @@ fn extract_page_text_items(ch: &ch,rec: char_chunks.as_mut().and_then(|chunks| chunks.record(i)),};+ if clips.hides(&cv) {+ seg.flush(&mut items);+ continue;+ }let unicode = cv.unicode();let is_generated = cv.is_generated();diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rsindex 2a4f499..1719d80 100644--- a/crates/liteparse/src/lib.rs+++ b/crates/liteparse/src/lib.rs@@ -28,6 +28,7 @@ pub mod parser;pub mod raw_text;pub mod search;pub mod stages;+mod text_clip;pub mod types;// ── Internal modules (available for binding crates, hidden from docs) ──diff --git a/crates/liteparse/src/raw_text.rs b/crates/liteparse/src/raw_text.rsindex 2ed6e3a..7d6d97c 100644--- a/crates/liteparse/src/raw_text.rs+++ b/crates/liteparse/src/raw_text.rs@@ -4,9 +4,9 @@//! [`crate::extract`]. It applies no layout heuristics — no gap-based line or word//! merging, no invisible-text skip, no ligature expansion, no punctuation//! normalisation, no dedup — so its output is a stable function of what pdfium-//! reports and a consumer can build its own segmentation on top. Every glyph pdfium-//! reports lands in exactly one item, in page order, and a glyph is never dropped on-//! its own.+//! reports and a consumer can build its own segmentation on top. Glyphs hidden by+//! known rectangular clips are omitted before grouping; partially clipped glyphs+//! are kept when their loose-box centre is inside. Unknown clips preserve text.//!//! Rules://!@@ -157,6 +157,7 @@ pub fn extract_raw_text_items(let char_count = text_page.char_count();let page_rotation = page.rotation();let mut chunks = CharInfoChunks::new(text_page);+ let clips = crate::text_clip::TextClip::new(page);let mut items = Vec::new();let mut run: Vec<Glyph> = Vec::new();// Outline shapes repeat across a page (the same glyph at the same size and@@ -186,6 +187,10 @@ pub fn extract_raw_text_items(ch: &ch,rec: chunks.as_mut().and_then(|chunks| chunks.record(i)),};+ if clips.hides(&cv) {+ flush(&mut run, &mut items);+ continue;+ }let glyph = load_glyph(page, view_box, page_rotation, &cv, i);if let Some(first) = run.first()diff --git a/crates/liteparse/src/text_clip.rs b/crates/liteparse/src/text_clip.rsnew file mode 100644index 0000000..74ef9c6--- /dev/null+++ b/crates/liteparse/src/text_clip.rs@@ -0,0 +1,193 @@+//! Conservative glyph visibility under rectangular PDF graphics-state clips.+//! Clip points already include the object's CTM; only ancestor form matrices+//! remain. Unknown paths/transforms preserve text, including OCR text layers.++use std::collections::HashMap;++use pdfium::{Matrix, Page, PageObject, PageObjectKind, RawPathSegment, RectF, SegmentKind};++use crate::extract::CharView;++const IDENTITY: Matrix = Matrix {+ a: 1.0,+ b: 0.0,+ c: 0.0,+ d: 1.0,+ e: 0.0,+ f: 0.0,+};++/// Only objects whose complete path stack is understood have an entry.+/// Bounds are y-up page coordinates; inverted bounds mean an empty intersection.+pub(crate) struct TextClip {+ objects: HashMap<usize, RectF>,+}++impl TextClip {+ pub(crate) fn new(page: &Page) -> Self {+ let mut clips = Self {+ objects: HashMap::new(),+ };+ for index in 0..page.object_count() {+ if let Some(object) = page.object(index) {+ clips.walk(&object, &IDENTITY, None, 0);+ }+ }+ clips+ }++ fn walk(+ &mut self,+ object: &PageObject,+ parent: &Matrix,+ inherited: Option<RectF>,+ depth: usize,+ ) {+ if depth > 64 {+ return;+ }+ let Some(paths) = object.clip_paths() else {+ return;+ };+ let mut bounds = inherited;+ for path in paths {+ let Some(rect) = rectangle(&path, parent) else {+ return;+ };+ bounds = Some(match bounds {+ None => rect,+ Some(old) => RectF {+ left: old.left.max(rect.left),+ right: old.right.min(rect.right),+ bottom: old.bottom.max(rect.bottom),+ top: old.top.min(rect.top),+ },+ });+ }+ match object.kind() {+ PageObjectKind::Text => {+ if let Some(bounds) = bounds {+ self.objects.insert(object.id(), bounds);+ }+ }+ PageObjectKind::Form => {+ let Some(matrix) = object.matrix() else {+ return;+ };+ let matrix = compose(parent, &matrix);+ if !valid_matrix(&matrix) {+ return;+ }+ let Some(count) = object.form_object_count() else {+ return;+ };+ for index in 0..count {+ if let Some(child) = object.form_object(index) {+ self.walk(&child, &matrix, bounds, depth + 1);+ }+ }+ }+ _ => {}+ }+ }++ /// Generated separators have no reliable source object and are preserved.+ /// Partially clipped glyphs survive when their loose-box centre is inside.+ pub(crate) fn hides(&self, cv: &CharView<'_, '_>) -> bool {+ if cv.is_generated() {+ return false;+ }+ let Some(bounds) = cv+ .text_object()+ .and_then(|obj| self.objects.get(&(obj as usize)))+ else {+ return false;+ };+ let Some(glyph) = cv.loose_char_box().or_else(|| cv.strict_char_box()) else {+ return false;+ };+ if ![glyph.left, glyph.right, glyph.bottom, glyph.top]+ .iter()+ .all(|v| v.is_finite())+ || glyph.left > glyph.right+ || glyph.bottom > glyph.top+ {+ return false;+ }+ let x = f64::from(glyph.left) * 0.5 + f64::from(glyph.right) * 0.5;+ let y = f64::from(glyph.bottom) * 0.5 + f64::from(glyph.top) * 0.5;+ // A tiny tolerance prevents float rounding at a clip edge losing a glyph.+ const EPS: f64 = 0.001;+ bounds.left > bounds.right+ || bounds.bottom > bounds.top+ || x < f64::from(bounds.left) - EPS+ || x > f64::from(bounds.right) + EPS+ || y < f64::from(bounds.bottom) - EPS+ || y > f64::from(bounds.top) + EPS+ }+}++fn compose(p: &Matrix, c: &Matrix) -> Matrix {+ Matrix {+ a: p.a * c.a + p.c * c.b,+ b: p.b * c.a + p.d * c.b,+ c: p.a * c.c + p.c * c.d,+ d: p.b * c.c + p.d * c.d,+ e: p.a * c.e + p.c * c.f + p.e,+ f: p.b * c.e + p.d * c.f + p.f,+ }+}++fn valid_matrix(m: &Matrix) -> bool {+ [m.a, m.b, m.c, m.d, m.e, m.f].iter().all(|v| v.is_finite())+ && (f64::from(m.a) * f64::from(m.d) - f64::from(m.b) * f64::from(m.c)).abs() > 1e-12+}++/// Accept exactly one closed rectangle, never the bounding box of a compound+/// path (which can contain holes). Rotated/sheared non-axis-aligned clips fail open.+fn rectangle(path: &[RawPathSegment], parent: &Matrix) -> Option<RectF> {+ if !matches!(path.len(), 4 | 5) || !path.last()?.close || !valid_matrix(parent) {+ return None;+ }+ let mut points = Vec::with_capacity(path.len());+ for (i, segment) in path.iter().enumerate() {+ let expected = if i == 0 {+ SegmentKind::MoveTo+ } else {+ SegmentKind::LineTo+ };+ if segment.kind != Some(expected) || (i + 1 != path.len() && segment.close) {+ return None;+ }+ let (x, y) = segment.point?;+ let point = (+ parent.a * x + parent.c * y + parent.e,+ parent.b * x + parent.d * y + parent.f,+ );+ if !point.0.is_finite() || !point.1.is_finite() {+ return None;+ }+ points.push(point);+ }+ if points.len() == 5 && points.pop()? != points[0] {+ return None;+ }+ let mut horizontal = [false; 4];+ for i in 0..4 {+ let a = points[i];+ let b = points[(i + 1) % 4];+ horizontal[i] = a.1 == b.1 && a.0 != b.0;+ if !(horizontal[i] || a.0 == b.0 && a.1 != b.1) {+ return None;+ }+ }+ if (0..4).any(|i| horizontal[i] == horizontal[(i + 1) % 4]) {+ return None;+ }+ Some(RectF {+ left: points.iter().map(|p| p.0).fold(f32::INFINITY, f32::min),+ right: points.iter().map(|p| p.0).fold(f32::NEG_INFINITY, f32::max),+ bottom: points.iter().map(|p| p.1).fold(f32::INFINITY, f32::min),+ top: points.iter().map(|p| p.1).fold(f32::NEG_INFINITY, f32::max),+ })+}diff --git a/crates/liteparse/tests/clipped_text.rs b/crates/liteparse/tests/clipped_text.rsnew file mode 100644index 0000000..0da5c91--- /dev/null+++ b/crates/liteparse/tests/clipped_text.rs@@ -0,0 +1,185 @@+//! Synthetic PDFs keep clipping regressions independent of customer documents.+use liteparse::{RawTextItem, extract_raw_text_items, stages};+use pdfium::Library;++fn pdf(content: &str, forms: &[&str], rotation: i32) -> Vec<u8> {+ let mut objects = vec![+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),+ format!(+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Rotate {rotation} /Resources << /Font << /F1 4 0 R >> /XObject << /Outer 6 0 R >> >> /Contents 5 0 R >>"+ ),+ "<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>".to_string(),+ stream("", content),+ ];+ objects.extend(forms.iter().map(|s| s.to_string()));+ let mut data = b"%PDF-1.7\n".to_vec();+ let mut offsets = Vec::new();+ for (i, object) in objects.iter().enumerate() {+ offsets.push(data.len());+ data.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes());+ }+ let xref = data.len();+ data.extend_from_slice(+ format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes(),+ );+ for offset in offsets {+ data.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());+ }+ data.extend_from_slice(+ format!(+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",+ objects.len() + 1+ )+ .as_bytes(),+ );+ data+}++fn stream(dict: &str, content: &str) -> String {+ format!(+ "<< {dict} /Length {} >>\nstream\n{content}\nendstream",+ content.len()+ )+}++fn extract(bytes: &[u8]) -> (Vec<RawTextItem>, String) {+ let lib = Library::init();+ let doc = lib.load_document_from_bytes(bytes, None).unwrap();+ let page = doc.page(0).unwrap();+ let text = page.text().unwrap();+ let raw = extract_raw_text_items(&page, &text, &page.view_box().unwrap(), None);+ let standard = stages::extract(&doc, &stages::ExtractRequest::default()).unwrap();+ let standard = standard.pages[0]+ .text_items+ .iter()+ .map(|i| i.text.as_str())+ .collect::<Vec<_>>()+ .join(" ");+ (raw, standard)+}++fn assert_text(bytes: &[u8], expected: &str) {+ let (raw, standard) = extract(bytes);+ let raw: String = raw.iter().map(|i| i.text.as_str()).collect();+ // PDFium inserts separators between source objects. Compare real characters.+ let compact = |s: &str| s.chars().filter(|c| !c.is_whitespace()).collect::<String>();+ assert_eq!(compact(&raw), compact(expected), "raw: {raw:?}");+ assert_eq!(+ compact(&standard),+ compact(expected),+ "standard: {standard:?}"+ );+}++#[test]+fn clipped_overflow_does_not_leak_into_next_row() {+ let content = "q 100 200 70 8 re W* n BT /F1 6 Tf 105 201 Td (VISIBLE) Tj 0 -7 Td (42) Tj ET Q BT /F1 6 Tf 105 194 Td (NEXT) Tj ET";+ assert_text(&pdf(content, &[], 0), "VISIBLE NEXT");+}++#[test]+fn horizontal_partial_glyphs_use_centre_and_page_rotation_does_not_matter() {+ for rotation in [0, 90, 180, 270] {+ for (edge, expected) in [(26, "ABC"), (24, "AB")] {+ let content = format!(+ "q 10 10 {} 30 re W n BT /F1 10 Tf 10 20 Td (ABCDE) Tj ET Q",+ edge - 10+ );+ assert_text(&pdf(&content, &[], rotation), expected);+ }+ }+}++#[test]+fn nested_forms_compose_transforms_and_inherit_outer_clips() {+ let inner = stream(+ "/Type /XObject /Subtype /Form /BBox [0 0 100 100] /Resources << /Font << /F1 4 0 R >> >>",+ "0 0 25 80 re W n BT /F1 10 Tf 5 20 Td (ABCDE) Tj ET",+ );+ let outer = stream(+ "/Type /XObject /Subtype /Form /BBox [0 0 200 200] /Resources << /XObject << /Inner 7 0 R >> >>",+ "2 0 0 2 10 15 cm /Inner Do",+ );+ let content = "q 100 100 35 300 re W n 1 0 0 1 100 100 cm /Outer Do Q";+ assert_text(&pdf(content, &[&outer, &inner], 0), "A");+}++#[test]+fn form_bbox_clips_text_without_explicit_clip_operator() {+ let form = stream(+ "/Type /XObject /Subtype /Form /BBox [0 0 18 100] /Resources << /Font << /F1 4 0 R >> >>",+ "BT /F1 10 Tf 0 20 Td (ABCDE) Tj ET",+ );+ assert_text(&pdf("1 0 0 1 100 100 cm /Outer Do", &[&form], 0), "ABC");+}++#[test]+fn stacked_clips_intersect_and_restore_does_not_leak() {+ let content = "q 0 0 100 100 re W n 10 10 14 30 re W* n BT /F1 10 Tf 10 20 Td (ABCDE) Tj ET Q BT /F1 10 Tf 100 100 Td (RESTORED) Tj ET";+ assert_text(&pdf(content, &[], 0), "AB RESTORED");+}++#[test]+fn unsupported_clips_preserve_source_text() {+ for path in [+ "0 0 m 1 0 l 1 1 l h", // triangle, all text outside+ "0 0 1 1 re 2 2 1 1 re", // compound even-odd path+ "0 0 m 1 0 1 1 0 1 c h", // curve+ ] {+ let content = format!("q {path} W* n BT /F1 10 Tf 10 20 Td (KEEP) Tj ET Q");+ assert_text(&pdf(&content, &[], 0), "KEEP");+ }+}++#[test]+fn unclipped_raw_items_are_identical_with_redundant_clip() {+ let text = "BT /F1 10 Tf 10 20 Td (UNCHANGED WORDS) Tj ET";+ let plain = extract(&pdf(text, &[], 0));+ let clipped = extract(&pdf(&format!("q 0 0 600 800 re W n {text} Q"), &[], 0));+ assert_eq!(plain, clipped);+}++#[test]+fn invisible_ocr_text_without_path_clips_is_preserved() {+ assert_text(+ &pdf("BT /F1 10 Tf 3 Tr 10 20 Td (OCR) Tj ET", &[], 0),+ "OCR",+ );+}++#[test]+fn disjoint_clips_hide_all_glyphs() {+ assert_text(+ &pdf(+ "q 0 0 5 5 re W n 10 10 5 5 re W n BT /F1 10 Tf 10 20 Td (HIDDEN) Tj ET Q",+ &[],+ 0,+ ),+ "",+ );+}++#[test]+fn content_matrix_is_not_applied_to_clip_twice() {+ assert_text(+ &pdf(+ "q 2 0 0 2 100 100 cm 0 0 14 40 re W n BT /F1 10 Tf 0 20 Td (ABCDE) Tj ET Q",+ &[],+ 0,+ ),+ "AB",+ );+}++#[test]+fn unsupported_parent_clip_preserves_form_text() {+ let form = stream(+ "/Type /XObject /Subtype /Form /BBox [0 0 100 100] /Resources << /Font << /F1 4 0 R >> >>",+ "BT /F1 10 Tf 10 20 Td (KEEP) Tj ET",+ );+ assert_text(+ &pdf("0 0 m 1 0 l 1 1 l h W n /Outer Do", &[&form], 0),+ "KEEP",+ );+}diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rsindex 9a242f9..7b08234 100644--- a/crates/pdfium-sys/src/dynamic.rs+++ b/crates/pdfium-sys/src/dynamic.rs@@ -200,6 +200,17 @@ pub struct PdfiumBindings {*mut std::os::raw::c_uint,*mut std::os::raw::c_uint,) -> FPDF_BOOL,+ pub FPDFPageObj_GetClipPath: Option<unsafe extern "C" fn(FPDF_PAGEOBJECT) -> FPDF_CLIPPATH>,+ pub FPDFClipPath_CountPaths: Option<unsafe extern "C" fn(FPDF_CLIPPATH) -> std::os::raw::c_int>,+ pub FPDFClipPath_CountPathSegments:+ Option<unsafe extern "C" fn(FPDF_CLIPPATH, std::os::raw::c_int) -> std::os::raw::c_int>,+ pub FPDFClipPath_GetPathSegment: Option<+ unsafe extern "C" fn(+ FPDF_CLIPPATH,+ std::os::raw::c_int,+ std::os::raw::c_int,+ ) -> FPDF_PATHSEGMENT,+ >,pub FPDFPageObj_GetMatrix: unsafe extern "C" fn(FPDF_PAGEOBJECT, *mut FS_MATRIX) -> FPDF_BOOL,pub FPDFPageObj_GetStrokeColor: unsafe extern "C" fn(FPDF_PAGEOBJECT,@@ -647,6 +658,10 @@ impl PdfiumBindings {FPDFImageObj_GetImageFilter: load_fn!(lib, "FPDFImageObj_GetImageFilter"),FPDFImageObj_GetImageMetadata: load_fn!(lib, "FPDFImageObj_GetImageMetadata"),FPDFImageObj_GetImagePixelSize: load_fn!(lib, "FPDFImageObj_GetImagePixelSize"),+ FPDFPageObj_GetClipPath: load_fn_opt!(lib, "FPDFPageObj_GetClipPath"),+ FPDFClipPath_CountPaths: load_fn_opt!(lib, "FPDFClipPath_CountPaths"),+ FPDFClipPath_CountPathSegments: load_fn_opt!(lib, "FPDFClipPath_CountPathSegments"),+ FPDFClipPath_GetPathSegment: load_fn_opt!(lib, "FPDFClipPath_GetPathSegment"),FPDFPageObj_GetMatrix: load_fn!(lib, "FPDFPageObj_GetMatrix"),FPDFPageObj_GetStrokeColor: load_fn!(lib, "FPDFPageObj_GetStrokeColor"),FPDFPageObj_GetFillColor: load_fn!(lib, "FPDFPageObj_GetFillColor"),diff --git a/crates/pdfium/src/page_object.rs b/crates/pdfium/src/page_object.rsindex ddf1ac9..10806b5 100644--- a/crates/pdfium/src/page_object.rs+++ b/crates/pdfium/src/page_object.rs@@ -120,6 +120,78 @@ impl<'page, 'lib> PageObject<'page, 'lib> {}}+ /// Clip paths in the containing form's coordinates (page coordinates at the+ /// top level). The object's own matrix is already applied to clip points.+ /// No clip is an empty list; unavailable APIs, failed reads, or more than+ /// 1024 segments return `None`. Text-based clipping is not exposed by PDFium.+ pub fn clip_paths(&self) -> Option<Vec<Vec<RawPathSegment>>> {+ #[cfg(not(target_arch = "wasm32"))]+ let (get_clip, count_paths, count_segments, get_segment) = {+ let api = pdfium_sys::dynamic::pdfium();+ (+ api.FPDFPageObj_GetClipPath?,+ api.FPDFClipPath_CountPaths?,+ api.FPDFClipPath_CountPathSegments?,+ api.FPDFClipPath_GetPathSegment?,+ )+ };+ #[cfg(target_arch = "wasm32")]+ let (get_clip, count_paths, count_segments, get_segment) = (+ pdfium_sys::FPDFPageObj_GetClipPath,+ pdfium_sys::FPDFClipPath_CountPaths,+ pdfium_sys::FPDFClipPath_CountPathSegments,+ pdfium_sys::FPDFClipPath_GetPathSegment,+ );+ let clip = unsafe { get_clip(self.handle) };+ if clip.is_null() {+ return None;+ }+ let count = unsafe { count_paths(clip) };+ // PDFium returns -1 for an object with no clip-path reference.+ if count == -1 {+ return Some(Vec::new());+ }+ if !(0..=1024).contains(&count) {+ return None;+ }+ let mut paths = Vec::new();+ let mut budget = 1024;+ for path in 0..count {+ let count = unsafe { count_segments(clip, path) };+ if count < 0 || count > budget {+ return None;+ }+ budget -= count;+ let mut segments = Vec::new();+ for index in 0..count {+ let segment = unsafe { get_segment(clip, path, index) };+ if segment.is_null() {+ return None;+ }+ let kind = match unsafe { ffi!(FPDFPathSegment_GetType(segment)) } {+ t if t == pdfium_sys::FPDF_SEGMENT_MOVETO as i32 => Some(SegmentKind::MoveTo),+ t if t == pdfium_sys::FPDF_SEGMENT_LINETO as i32 => Some(SegmentKind::LineTo),+ t if t == pdfium_sys::FPDF_SEGMENT_BEZIERTO as i32 => {+ Some(SegmentKind::BezierTo)+ }+ _ => None,+ };+ let mut x = 0.0;+ let mut y = 0.0;+ if unsafe { ffi!(FPDFPathSegment_GetPoint(segment, &mut x, &mut y)) } == 0 {+ return None;+ }+ segments.push(RawPathSegment {+ kind,+ point: Some((x, y)),+ close: unsafe { ffi!(FPDFPathSegment_GetClose(segment)) } != 0,+ });+ }+ paths.push(segments);+ }+ Some(paths)+ }+/// The object's transformation matrix (`FPDFPageObj_GetMatrix`). For an object/// inside a form this is relative to the form, not the page. `None` when pdfium/// reports none — normal for a shading object without a clip path.