Skip to content

A fast, helpful, and open-source document parser

fix: remove some O(n^2) complexities around bboxes

Logan Markewich7d5e7ddd6e03
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index 693756f..d84d2ef 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -13,6 +13,15 @@ use pdfium::{
SegmentKind, TextPage,
};
+/// Dedup spatial-grid cell size bounds (pt). The cell tracks the typical item
+/// footprint so a cell holds O(1) non-overlapping items.
+const DEDUP_MIN_CELL_SIZE: f32 = 8.0;
+const DEDUP_MAX_CELL_SIZE: f32 = 256.0;
+/// Items spanning more grid cells than this (full-page watermarks,
+/// row-spanning leaders) skip the grid and go to a side list that every item
+/// checks against.
+const DEDUP_MAX_CELLS_PER_ITEM: i64 = 64;
+
/// Open a PDF from path or bytes with an optional password.
///
/// The returned [`Document`] borrows from the provided [`Library`], which
@@ -1780,123 +1789,201 @@ fn extract_page_text_items(
/// Remove duplicate text items: exact text matches with any bbox overlap,
/// and near-duplicates (different text) with high bbox overlap (>50% area).
+/// Pair predicate for [`dedup_overlapping_items`]: should the *earlier* item
+/// `i` be dropped in favor of the later item `j` (later = painted on top)?
+///
+/// Callers must already have filtered out diagonal items (see the comment in
+/// `dedup_overlapping_items`).
+fn dedup_pair_drops_earlier(items: &[TextItem], i: usize, j: usize, debug: bool) -> bool {
+ let a = &items[i];
+ let b = &items[j];
+
+ // Compute intersection area
+ let ix_left = a.x.max(b.x);
+ let ix_right = (a.x + a.width).min(b.x + b.width);
+ let iy_top = a.y.max(b.y);
+ let iy_bottom = (a.y + a.height).min(b.y + b.height);
+
+ if ix_left >= ix_right || iy_top >= iy_bottom {
+ return false; // no overlap
+ }
+
+ let intersection = (ix_right - ix_left) * (iy_bottom - iy_top);
+ let area_a = a.width * a.height;
+ let area_b = b.width * b.height;
+ let smaller_area = area_a.min(area_b);
+
+ // Strong overlap: >50% of the smaller item is covered. Guards against
+ // dropping legitimate repeats of the same word elsewhere on the page —
+ // true duplicate stamps overlap essentially 100%, unrelated repeats
+ // share at most a sliver of slack loose-box area.
+ if !(smaller_area > 0.0 && intersection / smaller_area > 0.5) {
+ return false;
+ }
+
+ if a.text == b.text {
+ if debug {
+ eprintln!(
+ "[extract-debug] DEDUP exact-match drop i={i} text='{}' at ({:.1},{:.1} {}x{}) in favor of j={j} at ({:.1},{:.1} {}x{}) overlap_ratio={:.2}",
+ a.text,
+ a.x,
+ a.y,
+ a.width,
+ a.height,
+ b.x,
+ b.y,
+ b.width,
+ b.height,
+ intersection / smaller_area
+ );
+ }
+ return true;
+ }
+
+ // Different text but strong overlap: likely overpainted text layers
+ // (e.g. old/new branding); keep the later one (on top in paint order).
+ // Skip when sizes differ wildly (area ratio > 5x) — a small cell value
+ // inside a row-spanning dotted leader is separate content, not a layer.
+ let larger_area = area_a.max(area_b);
+ if larger_area / smaller_area > 5.0 {
+ if debug {
+ eprintln!(
+ "[extract-debug] DEDUP skip (area ratio {:.1}x) i={i} text='{}' j={j} text='{}'",
+ larger_area / smaller_area,
+ a.text,
+ b.text
+ );
+ }
+ return false;
+ }
+ if debug {
+ eprintln!(
+ "[extract-debug] DEDUP overlap drop i={i} text='{}' at ({:.1},{:.1} {}x{}) in favor of j={j} text='{}' at ({:.1},{:.1} {}x{}) overlap_ratio={:.2}",
+ a.text,
+ a.x,
+ a.y,
+ a.width,
+ a.height,
+ b.text,
+ b.x,
+ b.y,
+ b.width,
+ b.height,
+ intersection / smaller_area
+ );
+ }
+ true
+}
+
+/// Remove items an overlapping later item duplicates or overpaints.
+///
+/// An item is dropped iff *some later item* passes
+/// [`dedup_pair_drops_earlier`] — drops only ever hit the earlier item of a
+/// pair, so the result is independent of comparison order and the search can
+/// consult a uniform spatial grid: only items whose bounding boxes can
+/// intersect are ever compared. This must stay near-linear — single-page CAD
+/// exports and receipt ribbons reach 10⁵–10⁶ items, and this pass runs while
+/// holding the process-global PDFium lock.
+///
+/// Diagonal (non-right-angle) text never participates: its *loose*
+/// axis-aligned bounding box — the hull of a rotated glyph run — is far
+/// larger than the ink, so two stacked lines of the same skewed block report
+/// heavy bbox overlap even though the glyphs never touch. True duplicate
+/// stamps are upright and still handled.
fn dedup_overlapping_items(items: &mut Vec<TextItem>, debug: bool) {
if items.len() < 2 {
return;
}
- let mut keep = vec![true; items.len()];
- for i in 0..items.len() {
- if !keep[i] {
+ let upright: Vec<u32> = (0..items.len())
+ .filter(|&i| !is_diagonal_rotation(items[i].rotation))
+ .map(|i| i as u32)
+ .collect();
+ if upright.len() < 2 {
+ return;
+ }
+
+ let (sum_w, sum_h) = upright.iter().fold((0.0f64, 0.0f64), |(w, h), &i| {
+ let it = &items[i as usize];
+ (w + it.width.max(0.0) as f64, h + it.height.max(0.0) as f64)
+ });
+ let avg_dim = ((sum_w + sum_h) / (2 * upright.len()) as f64) as f32;
+ let cell = avg_dim.clamp(DEDUP_MIN_CELL_SIZE, DEDUP_MAX_CELL_SIZE);
+ let cell_range = |it: &TextItem| -> (i32, i32, i32, i32) {
+ (
+ (it.x / cell).floor() as i32,
+ ((it.x + it.width) / cell).floor() as i32,
+ (it.y / cell).floor() as i32,
+ ((it.y + it.height) / cell).floor() as i32,
+ )
+ };
+
+ let mut grid: std::collections::HashMap<(i32, i32), Vec<u32>> =
+ std::collections::HashMap::new();
+ let mut oversized: Vec<u32> = Vec::new();
+ for &idx in &upright {
+ let (cx0, cx1, cy0, cy1) = cell_range(&items[idx as usize]);
+ let cells = (cx1 as i64 - cx0 as i64 + 1) * (cy1 as i64 - cy0 as i64 + 1);
+ if cells > DEDUP_MAX_CELLS_PER_ITEM {
+ oversized.push(idx);
continue;
}
- for j in (i + 1)..items.len() {
- if !keep[j] {
- continue;
+ for cx in cx0..=cx1 {
+ for cy in cy0..=cy1 {
+ grid.entry((cx, cy)).or_default().push(idx);
}
+ }
+ }
- let a = &items[i];
- let b = &items[j];
-
- // Diagonal (non-right-angle) text has a *loose* axis-aligned
- // bounding box — the hull of a rotated glyph run is far larger than
- // the ink, so two stacked lines of the same skewed block (e.g.
- // "Paris has the" above "eiffel tower" at 51°) report heavy bbox
- // overlap even though the glyphs never touch. Dedup keys off AABB
- // overlap, so it would wrongly drop one of those lines. Skip the
- // comparison entirely when either item is diagonal; true duplicate
- // stamps are upright and still handled below.
- if is_diagonal_rotation(a.rotation) || is_diagonal_rotation(b.rotation) {
- continue;
+ let mut keep = vec![true; items.len()];
+ // Generation stamps so a candidate sharing several cells with the current
+ // item is only tested once.
+ let mut last_seen = vec![u32::MAX; items.len()];
+ for (generation, &i) in upright.iter().enumerate() {
+ let i = i as usize;
+ let generation = generation as u32;
+ last_seen[i] = generation;
+ let mut check = |j: u32, keep_i: &mut bool| -> bool {
+ let j = j as usize;
+ if j <= i || last_seen[j] == generation {
+ return false;
}
-
- // Compute intersection area
- let ix_left = a.x.max(b.x);
- let ix_right = (a.x + a.width).min(b.x + b.width);
- let iy_top = a.y.max(b.y);
- let iy_bottom = (a.y + a.height).min(b.y + b.height);
-
- if ix_left >= ix_right || iy_top >= iy_bottom {
- continue; // no overlap
+ last_seen[j] = generation;
+ if dedup_pair_drops_earlier(items, i, j, debug) {
+ *keep_i = false;
+ return true; // i is gone, move to next i
}
-
- let intersection = (ix_right - ix_left) * (iy_bottom - iy_top);
- let area_a = a.width * a.height;
- let area_b = b.width * b.height;
- let smaller_area = area_a.min(area_b);
-
- if items[i].text == items[j].text {
- // Exact text match: require strong bounding-box overlap before
- // dedup. The same word routinely appears more than once on a
- // page in different positions; firing on any overlap would drop
- // a legitimate occurrence when two identical words' bboxes share
- // even a sliver of area (e.g. one column's word vertically
- // adjacent to another column's identical word with a slack
- // loose-box), corrupting that line.
- //
- // Require ≥50% overlap of the smaller item — same threshold
- // as the non-exact branch. True duplicate stamps overlap
- // essentially 100%; unrelated repeats overlap 0%.
- let strong_overlap = smaller_area > 0.0 && intersection / smaller_area > 0.5;
- if !strong_overlap {
- continue;
- }
- if debug {
- eprintln!(
- "[extract-debug] DEDUP exact-match drop i={i} text='{}' at ({:.1},{:.1} {}x{}) in favor of j={j} at ({:.1},{:.1} {}x{}) overlap_ratio={:.2}",
- items[i].text,
- items[i].x,
- items[i].y,
- items[i].width,
- items[i].height,
- items[j].x,
- items[j].y,
- items[j].width,
- items[j].height,
- intersection / smaller_area
- );
+ false
+ };
+ let (cx0, cx1, cy0, cy1) = cell_range(&items[i]);
+ let cells = (cx1 as i64 - cx0 as i64 + 1) * (cy1 as i64 - cy0 as i64 + 1);
+ 'search: {
+ // An oversized item's cell walk would be huge; its overlap
+ // partners are found by the exhaustive pass below instead.
+ if cells <= DEDUP_MAX_CELLS_PER_ITEM {
+ for cx in cx0..=cx1 {
+ for cy in cy0..=cy1 {
+ let Some(bucket) = grid.get(&(cx, cy)) else {
+ continue;
+ };
+ for &j in bucket {
+ if check(j, &mut keep[i]) {
+ break 'search;
+ }
+ }
+ }
}
- keep[i] = false;
- break; // i is gone, move to next i
- } else if smaller_area > 0.0 && intersection / smaller_area > 0.5 {
- // Different text but >50% overlap of the smaller item:
- // likely overlapping text layers (e.g. old/new branding).
- // Keep the later one (rendered on top in PDF paint order).
- //
- // However, skip dedup when the items have very different sizes
- // (area ratio > 5x). This happens when a small cell value sits
- // inside a row-spanning element like a dotted leader — these are
- // separate content, not overlapping layers.
- let larger_area = area_a.max(area_b);
- if larger_area / smaller_area > 5.0 {
- if debug {
- eprintln!(
- "[extract-debug] DEDUP skip (area ratio {:.1}x) i={i} text='{}' j={j} text='{}'",
- larger_area / smaller_area,
- items[i].text,
- items[j].text
- );
+ for &j in &oversized {
+ if check(j, &mut keep[i]) {
+ break 'search;
}
- continue;
}
- if debug {
- eprintln!(
- "[extract-debug] DEDUP overlap drop i={i} text='{}' at ({:.1},{:.1} {}x{}) in favor of j={j} text='{}' at ({:.1},{:.1} {}x{}) overlap_ratio={:.2}",
- items[i].text,
- items[i].x,
- items[i].y,
- items[i].width,
- items[i].height,
- items[j].text,
- items[j].x,
- items[j].y,
- items[j].width,
- items[j].height,
- intersection / smaller_area
- );
+ } else {
+ for &j in &upright {
+ if check(j, &mut keep[i]) {
+ break 'search;
+ }
}
- keep[i] = false;
- break; // i is gone, move to next i
}
}
}
@@ -3098,6 +3185,117 @@ mod tests {
assert_eq!(items.len(), 2);
}
+ /// Exhaustive-search oracle: applies the dedup rule ("drop an item iff
+ /// some later upright item passes the pair predicate") by checking every
+ /// pair. The grid-backed search must match this on any layout.
+ fn dedup_overlapping_items_exhaustive(items: &mut Vec<TextItem>) {
+ if items.len() < 2 {
+ return;
+ }
+ let mut keep = vec![true; items.len()];
+ for i in 0..items.len() {
+ for j in (i + 1)..items.len() {
+ if is_diagonal_rotation(items[i].rotation)
+ || is_diagonal_rotation(items[j].rotation)
+ {
+ continue;
+ }
+ if dedup_pair_drops_earlier(items, i, j, false) {
+ keep[i] = false;
+ break;
+ }
+ }
+ }
+ let mut idx = 0;
+ items.retain(|_| {
+ let k = keep[idx];
+ idx += 1;
+ k
+ });
+ }
+
+ #[test]
+ fn dedup_grid_matches_exhaustive_on_random_layouts() {
+ // Deterministic LCG so failures reproduce.
+ let mut state = 0x2545F4914F6CDD1Du64;
+ let mut next = move || {
+ state = state
+ .wrapping_mul(6364136223846793005)
+ .wrapping_add(1442695040888963407);
+ (state >> 33) as f32 / (1u64 << 31) as f32
+ };
+ for round in 0..50 {
+ let n = 20 + (round * 17) % 300;
+ let mut items: Vec<TextItem> = (0..n)
+ .map(|k| {
+ // Cluster positions so overlaps are common, vary size so
+ // both the area-ratio skip and the oversized side-list
+ // trigger, and reuse a small text alphabet so exact-match
+ // dedup fires.
+ let big = next() < 0.05;
+ TextItem {
+ rotation: if next() < 0.1 { 51.0 } else { 0.0 },
+ ..ti(
+ ["a", "b", "cc", "ddd"][k % 4],
+ next() * 200.0,
+ next() * 3000.0,
+ if big { 500.0 } else { 4.0 + next() * 30.0 },
+ if big { 2000.0 } else { 4.0 + next() * 12.0 },
+ )
+ }
+ })
+ .collect();
+ let mut expected = items.clone();
+ dedup_overlapping_items_exhaustive(&mut expected);
+ dedup_overlapping_items(&mut items, false);
+ let got: Vec<_> = items
+ .iter()
+ .map(|it| (it.text.clone(), it.x, it.y))
+ .collect();
+ let want: Vec<_> = expected
+ .iter()
+ .map(|it| (it.text.clone(), it.x, it.y))
+ .collect();
+ assert_eq!(
+ got, want,
+ "grid dedup diverged from oracle on round {round}"
+ );
+ }
+ }
+
+ #[test]
+ fn dedup_scales_to_ribbon_pages() {
+ // Single-page CAD exports / receipt ribbons put 10⁵–10⁶ items on one
+ // page; dedup must stay effectively linear there. A hang here (CI
+ // timeout) means quadratic behavior regressed.
+ let n: usize = 200_000;
+ let mut items: Vec<TextItem> = (0..n)
+ .map(|k| {
+ // Every 10th item is a near-exact restamp of its predecessor
+ // (slightly nudged, same text) so the exact-match dedup path
+ // fires; everything else is a disjoint grid cell.
+ let dup = k % 10 == 0 && k > 0;
+ let base = if dup { k - 1 } else { k };
+ let col = (base % 40) as f32;
+ let row = (base / 40) as f32;
+ ti(
+ "cell",
+ col * 5.0 + if dup { 0.3 } else { 0.0 },
+ row * 8.0,
+ 4.0,
+ 6.0,
+ )
+ })
+ .collect();
+ dedup_overlapping_items(&mut items, false);
+ let dups = (n - 1) / 10;
+ assert!(
+ items.len() <= n - dups && items.len() >= n - 2 * dups,
+ "expected ~{dups} restamped predecessors dropped, got {} of {n} items left",
+ items.len()
+ );
+ }
+
#[test]
fn dedup_noop_for_empty_or_single() {
let mut empty: Vec<TextItem> = vec![];
diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs
index f0ed986..339a3ce 100644
--- a/crates/liteparse/src/projection.rs
+++ b/crates/liteparse/src/projection.rs
@@ -820,100 +820,101 @@ fn form_lines(
*line = merged_line;
}
- // Merge overlapping lines when there is no horizontal bbox overlap.
- let mut i = 1usize;
- while i < lines.len() {
- let (previous_min_y, previous_max_y) = {
- let previous = &lines[i - 1];
- let min_y = previous
- .iter()
- .map(|v| v.item.y)
- .fold(f32::INFINITY, |a, b| a.min(b));
- let max_y = previous
- .iter()
- .map(|v| v.item.y + v.item.height)
- .fold(f32::NEG_INFINITY, |a, b| a.max(b));
- (min_y, max_y)
- };
-
- let (current_min_y, current_max_y) = {
- let current = &lines[i];
- let min_y = current
- .iter()
- .map(|v| v.item.y)
- .fold(f32::INFINITY, |a, b| a.min(b));
- let max_y = current
- .iter()
- .map(|v| v.item.y + v.item.height)
- .fold(f32::NEG_INFINITY, |a, b| a.max(b));
- (min_y, max_y)
- };
-
- // Do the two lines overlap vertically?
- let lines_overlap = previous_max_y > current_min_y && previous_min_y < current_max_y;
-
- if lines_overlap {
- let bbox_overlap = {
- let previous = &lines[i - 1];
- let current = &lines[i];
- current.iter().any(|bbox| {
+ // Merge overlapping lines when there is no horizontal bbox overlap,
+ // folding into an accumulator whose last line keeps absorbing successors.
+ // Ribbon pages reach 10⁵+ lines, so this must stay linear: the x-sort of
+ // a merged line is deferred until it stops absorbing (no check here
+ // depends on item order) and its y-range is maintained incrementally.
+ let y_range = |line: &[ProjectedTextItem]| -> (f32, f32) {
+ line.iter()
+ .fold((f32::INFINITY, f32::NEG_INFINITY), |acc, v| {
+ (acc.0.min(v.item.y), acc.1.max(v.item.y + v.item.height))
+ })
+ };
+ let mut merged: Vec<Vec<ProjectedTextItem>> = Vec::with_capacity(lines.len());
+ let mut prev_range = (f32::INFINITY, f32::NEG_INFINITY);
+ let mut prev_absorbed = false;
+ for mut current in lines {
+ let (current_min_y, current_max_y) = y_range(&current);
+ if let Some(previous) = merged.last_mut() {
+ let (previous_min_y, previous_max_y) = prev_range;
+
+ // Do the two lines overlap vertically?
+ let lines_overlap = previous_max_y > current_min_y && previous_min_y < current_max_y;
+
+ if lines_overlap {
+ let bbox_overlap = current.iter().any(|bbox| {
previous.iter().any(|prev_bbox| {
(bbox.item.x >= prev_bbox.item.x
&& bbox.item.x <= prev_bbox.item.x + prev_bbox.item.width)
|| (prev_bbox.item.x >= bbox.item.x
&& prev_bbox.item.x <= bbox.item.x + bbox.item.width)
})
- })
- };
+ });
- if !bbox_overlap {
- if dbg_lines {
- let pj: String = lines[i - 1]
- .iter()
- .map(|x| x.item.text.trim())
- .collect::<Vec<_>>()
- .join("|");
- let cj: String = lines[i]
- .iter()
- .map(|x| x.item.text.trim())
- .collect::<Vec<_>>()
- .join("|");
- eprintln!(
- "[form_lines:overlap-merge] py={previous_min_y:.1}..{previous_max_y:.1} cy={current_min_y:.1}..{current_max_y:.1}\n prev={pj:.60}\n cur ={cj:.60}"
+ if !bbox_overlap {
+ if dbg_lines {
+ let pj: String = previous
+ .iter()
+ .map(|x| x.item.text.trim())
+ .collect::<Vec<_>>()
+ .join("|");
+ let cj: String = current
+ .iter()
+ .map(|x| x.item.text.trim())
+ .collect::<Vec<_>>()
+ .join("|");
+ eprintln!(
+ "[form_lines:overlap-merge] py={previous_min_y:.1}..{previous_max_y:.1} cy={current_min_y:.1}..{current_max_y:.1}\n prev={pj:.60}\n cur ={cj:.60}"
+ );
+ }
+ previous.append(&mut current);
+ prev_range = (
+ previous_min_y.min(current_min_y),
+ previous_max_y.max(current_max_y),
);
+ prev_absorbed = true;
+ continue;
}
- let mut current = lines.remove(i);
- lines[i - 1].append(&mut current);
- lines[i - 1].sort_by(|a, b| a.item.x.total_cmp(&b.item.x));
- continue;
+ }
+ if prev_absorbed {
+ previous.sort_by(|a, b| a.item.x.total_cmp(&b.item.x));
}
}
-
- i += 1;
+ merged.push(current);
+ prev_range = (current_min_y, current_max_y);
+ prev_absorbed = false;
+ }
+ if prev_absorbed && let Some(previous) = merged.last_mut() {
+ previous.sort_by(|a, b| a.item.x.total_cmp(&b.item.x));
}
+ let lines = merged;
dump_lines("after_overlap_merge", &lines);
- // Insert blank lines for vertical gaps between lines.
- let mut i = 1;
- while i < lines.len() {
- let prev_metrics = representative_line_metrics(&lines[i - 1], median_height);
- let cur_metrics = representative_line_metrics(&lines[i], median_height);
- let y_delta = cur_metrics.0 - prev_metrics.1; // cur_top - prev_bottom
- let reference_height = median_height.max(prev_metrics.2.min(cur_metrics.2));
-
- if y_delta > reference_height {
- let num_blank = ((y_delta / reference_height).round() as usize).saturating_sub(1);
- let to_insert = num_blank.clamp(1, 10);
- for _ in 0..to_insert {
- lines.insert(i, Vec::new());
- i += 1;
+ // Insert blank lines for vertical gaps between lines. Gaps are measured
+ // between consecutive content lines only; inserted blanks never
+ // participate.
+ let mut with_blanks: Vec<Vec<ProjectedTextItem>> = Vec::with_capacity(lines.len());
+ let mut prev_metrics: Option<(f32, f32, f32)> = None;
+ for line in lines {
+ let cur_metrics = representative_line_metrics(&line, median_height);
+ if let Some(prev_metrics) = prev_metrics {
+ let y_delta = cur_metrics.0 - prev_metrics.1; // cur_top - prev_bottom
+ let reference_height = median_height.max(prev_metrics.2.min(cur_metrics.2));
+
+ if y_delta > reference_height {
+ let num_blank = ((y_delta / reference_height).round() as usize).saturating_sub(1);
+ for _ in 0..num_blank.clamp(1, 10) {
+ with_blanks.push(Vec::new());
+ }
}
}
- i += 1;
+ prev_metrics = Some(cur_metrics);
+ with_blanks.push(line);
}
- lines
+ with_blanks
}
/// Returns (top, bottom, height) for representative items in a line,