A fast, helpful, and open-source document parser
tables: read rowspans off the rules so merged cells stop reading as empty
Logan Markewich87d54ef7a4d5
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rsindex 81b1ff5..2cb9465 100644--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@ -69,6 +69,10 @@ pub(super) struct SpanPiece<'a> {pub(super) text: String,/// The run this piece came from — for bold/font lookups.pub(super) span: &'a TextItem,+ /// The piece's own word boxes, in reading order, but **only** when they+ /// were verified to reconstruct `text` exactly. Empty otherwise, so a+ /// consumer can treat non-empty as "real geometry for every word here".+ pub(super) words: Vec<&'a crate::types::WordBox>,}/// Split one PDFium run into column pieces at its internal gutters.@@ -87,31 +91,22 @@ pub(super) struct SpanPiece<'a> {////// Returns a single piece covering the whole run when it has no word boxes,/// fewer than three words, or no bimodal gap.-pub(super) fn split_span_at_gutters(span: &TextItem) -> Vec<SpanPiece<'_>> {- let whole = || {- vec![SpanPiece {- x: span.x,- end_x: span.x + span.width.max(0.0),- text: collapse_whitespace(span.text.trim()),- span,- }]- };- // Only words that lie within this run's own x-extent count: projection can- // append a merged neighbour's word boxes onto a run without re-slicing the- // text, so `words` is a superset in that case.+/// A run's own word boxes, in reading order, but only when they reconstruct the+/// run's text exactly. Anything less and the boxes can't be used to slice the+/// text, because the split points wouldn't correspond to it.+///+/// The x-filter matters: projection can append a merged neighbour's word boxes+/// onto a run without re-slicing the text, making `span.words` a superset.+pub(super) fn verified_words(span: &TextItem) -> Option<Vec<&crate::types::WordBox>> {let x1 = span.x + span.width.max(0.0);let words: Vec<&crate::types::WordBox> = span.words.iter().filter(|w| !w.text.trim().is_empty() && w.x >= span.x - 1.0 && w.x <= x1 + 1.0).collect();- // Two words give exactly one gap, which is trivially "the largest" — no- // ratio to test against, so there is no evidence of bimodality.- if words.len() < 3 {- return whole();+ if words.is_empty() {+ return None;}- // The word list must actually reconstruct the run's text, or the split- // points would not correspond to the text we are about to slice.let rebuilt = collapse_whitespace(&words.iter()@@ -119,7 +114,26 @@ pub(super) fn split_span_at_gutters(span: &TextItem) -> Vec<SpanPiece<'_>> {.collect::<Vec<_>>().join(" "),);- if rebuilt != collapse_whitespace(span.text.trim()) {+ (rebuilt == collapse_whitespace(span.text.trim())).then_some(words)+}++pub(super) fn split_span_at_gutters(span: &TextItem) -> Vec<SpanPiece<'_>> {+ let verified_list = verified_words(span);+ let words: Vec<&crate::types::WordBox> = verified_list.clone().unwrap_or_default();+ let verified = verified_list.is_some();+ let whole = || {+ vec![SpanPiece {+ x: span.x,+ end_x: span.x + span.width.max(0.0),+ text: collapse_whitespace(span.text.trim()),+ span,+ words: if verified { words.clone() } else { Vec::new() },+ }]+ };+ // Two words give exactly one gap, which is trivially "the largest" — no+ // ratio to test against, so there is no evidence of bimodality. The piece+ // still carries its words, so a track-driven split can use them later.+ if words.len() < 3 || !verified {return whole();}@@ -173,7 +187,10 @@ pub(super) fn split_span_at_gutters(span: &TextItem) -> Vec<SpanPiece<'_>> {pieces}-fn piece_from_words<'a>(words: &[&crate::types::WordBox], span: &'a TextItem) -> SpanPiece<'a> {+fn piece_from_words<'a>(+ words: &[&'a crate::types::WordBox],+ span: &'a TextItem,+) -> SpanPiece<'a> {let x = words.first().map_or(span.x, |w| w.x);let end_x = words.last()@@ -189,6 +206,7 @@ fn piece_from_words<'a>(words: &[&crate::types::WordBox], span: &'a TextItem) ->.join(" "),),span,+ words: words.to_vec(),}}@@ -636,12 +654,12 @@ fn cells_from_raw_items_with_tracks(}}_ => {- let pieces = split_text_at_x_anchors(- &span.text,- span.x,- span.end_x - span.x,- &covered[1..].iter().map(|&i| tracks[i]).collect::<Vec<_>>(),- )?;+ let anchors: Vec<f32> = covered[1..].iter().map(|&i| tracks[i]).collect();+ // Real word x's first; the character-index estimate only when+ // this piece has no verified word geometry to cut on.+ let pieces = split_words_at_x_anchors(&span.words, &anchors).or_else(|| {+ split_text_at_x_anchors(&span.text, span.x, span.end_x - span.x, &anchors)+ })?;let bold = is_bold_item(span.span);for (idx, piece) in covered.iter().zip(pieces.iter()) {if piece.is_empty() {@@ -697,6 +715,55 @@ fn is_value_like(text: &str) -> bool {/// usable whitespace boundary (e.g. unbroken text like a long hex string)./// Pieces may be empty strings — callers that require non-empty pieces must/// check.+/// Split a word sequence into `anchors.len() + 1` pieces using the words' own+/// x positions: each anchor picks the inter-word boundary whose following word+/// starts closest to it.+///+/// This is the geometric counterpart of [`split_text_at_x_anchors`], which can+/// only *interpolate* an x from a character index and so assumes every glyph is+/// the same width — false for any proportional font, and the reason a split can+/// land one character off (`10 .1%`). Prefer this whenever the piece carries+/// verified word boxes.+///+/// Returns `None` when there are too few words to host every anchor, when two+/// anchors want the same boundary, or when an anchor's nearest boundary is+/// further away than the track tolerance — a miss that large means no word+/// actually starts at this column, so the caller should fall back rather than+/// cut where the geometry says nothing happens.+fn split_words_at_x_anchors(+ words: &[&crate::types::WordBox],+ anchors: &[f32],+) -> Option<Vec<String>> {+ if anchors.is_empty() || words.len() < anchors.len() + 1 {+ return None;+ }+ let mut cuts: Vec<usize> = Vec::with_capacity(anchors.len());+ for &target in anchors {+ let (k, dist) = (1..words.len())+ .filter(|k| !cuts.contains(k))+ .map(|k| (k, (words[k].x - target).abs()))+ .min_by(|a, b| a.1.total_cmp(&b.1))?;+ if dist > TABLE_TRACK_TOLERANCE_PT {+ return None;+ }+ cuts.push(k);+ }+ cuts.sort_unstable();+ let mut pieces: Vec<String> = Vec::with_capacity(cuts.len() + 1);+ let mut prev = 0usize;+ for &k in cuts.iter().chain(std::iter::once(&words.len())) {+ pieces.push(collapse_whitespace(+ &words[prev..k]+ .iter()+ .map(|w| w.text.trim())+ .collect::<Vec<_>>()+ .join(" "),+ ));+ prev = k;+ }+ Some(pieces)+}+fn split_text_at_x_anchors(text: &str,x0: f32,@@ -2986,6 +3053,11 @@ struct CellGrid {/// Per-row flag: the row contains an alpha-dominant multi-column span (a/// group-header label, as opposed to a mostly-digit merged data run).row_alpha_spanner: Vec<bool>,+ /// Per-cell flag: the rules say this cell is vertically merged with the one+ /// above it, so it is empty by construction. See `count_rowspan_cells`.+ /// Travels with the row/column collapses so the density gate forgives the+ /// empties that are actually here, not ones from a discarded row.+ spanned: Vec<Vec<bool>>,}impl CellGrid {@@ -2996,6 +3068,7 @@ impl CellGrid {has_text: vec![vec![false; n_cols]; n_rows],repl: vec![vec![String::new(); n_cols]; n_rows],row_alpha_spanner: vec![false; n_rows],+ spanned: vec![vec![false; n_cols]; n_rows],}}@@ -3042,6 +3115,7 @@ impl CellGrid {self.has_text = filter_by(std::mem::take(&mut self.has_text), keep);self.repl = filter_by(std::mem::take(&mut self.repl), keep);self.row_alpha_spanner = filter_by(std::mem::take(&mut self.row_alpha_spanner), keep);+ self.spanned = filter_by(std::mem::take(&mut self.spanned), keep);}/// Keep only columns `c` where `keep[c]`; the per-cell layers filter, the@@ -3063,6 +3137,10 @@ impl CellGrid {.into_iter().map(|row| filter_by(row, keep)).collect();+ self.spanned = std::mem::take(&mut self.spanned)+ .into_iter()+ .map(|row| filter_by(row, keep))+ .collect();}/// Drop "phantom rows" produced by stacked thin border-strip rects (some@@ -3267,7 +3345,18 @@ fn assign_cells(// nearest each crossed boundary x (xs[k] is column k's left// boundary, which is exactly the split target).let covered: Vec<usize> = (c_lo..=c_hi).collect();- if let Some(pieces) = split_span_at_anchors(span, &covered, xs) {+ // Prefer real word geometry: each word lands in the ruled column+ // its own centre falls in, which is exact. `split_span_at_anchors`+ // has to guess an x from a character index, so on a proportional+ // font it can cut a word or two off from the true boundary.+ if let Some(by_word) = bucket_words_into_columns(span, xs) {+ for (col, text) in by_word {+ grid.push_text(row, col, &text);+ if !line.all_bold {+ grid.is_bold[row][col] = false;+ }+ }+ } else if let Some(pieces) = split_span_at_anchors(span, &covered, xs) {for (k, piece) in pieces.iter().enumerate() {grid.push_text(row, c_lo + k, piece);if !line.all_bold {@@ -3460,7 +3549,11 @@ fn merge_stacked_header(}/// Density gate: a grid that is mostly empty cells is rejected unless it shows-/// strong table evidence. Three escape hatches keep real tables:+/// strong table evidence. Cells the rules say are vertically merged into the+/// cell above (`spanned`) don't count as empty at all — they are empty by+/// construction, and a rowspan label column ("1. Embodying sustainability+/// values" beside three competence rows) otherwise reads as a sparse grid and+/// dies here. Beyond that, three escape hatches keep real tables:/// - **col0 spine**: a filled, short-text first column (a label column)./// - **long-prose table**: a large (≥5×3) grid with a bold header band covering/// ≥3 columns and a dense (≥70%-fill) inner description column — a@@ -3474,6 +3567,59 @@ fn merge_stacked_header(////// Returns `true` to keep the table, `false` to reject it.///+/// Mark the grid cells that are *vertically merged* with the cell above them,+/// read straight off the rules: a cell whose top boundary carries no horizontal+/// stroke across its own column is the continuation of a rowspan.+///+/// This is the geometric explanation for legitimately-empty cells. Every+/// interior `ys` boundary exists because *some* horizontal segment sits there,+/// so a column only scores when that particular rule stops short of it — which+/// is exactly how a PDF draws a merged cell. A fully-ruled grid scores zero.+fn rowspan_mask(+ hs: &[HSeg],+ h_indices: &[usize],+ vs: &[VSeg],+ v_indices: &[usize],+ xs: &[f32],+ ys: &[f32],+) -> Vec<Vec<bool>> {+ let tol = TABLE_GRID_CLUSTER_PT;+ let mut mask = vec![vec![false; xs.len() - 1]; ys.len() - 1];+ for (r, &y) in ys[1..ys.len() - 1].iter().enumerate() {+ // Row `r + 1` is the one below boundary `ys[r + 1]`.+ let row = r + 1;+ // Some vertical rule must run *through* the boundary. That is what+ // distinguishes "the grid continues, this one cell is merged" from+ // "the grid ended here" — a page-frame component whose stray rules+ // cross unruled prose has no vertical continuing past them, and+ // forgiving its empties would turn body text into a two-column table.+ let grid_continues = v_indices+ .iter()+ .map(|&i| &vs[i])+ .any(|v| v.y_min <= y - tol && v.y_max >= y + tol);+ if !grid_continues {+ continue;+ }+ let at_y: Vec<&HSeg> = h_indices+ .iter()+ .map(|&i| &hs[i])+ .filter(|h| (h.y - y).abs() <= tol)+ .collect();+ for c in 0..xs.len() - 1 {+ // Test the column's *centre*, not containment of the whole column:+ // `xs` comes from clustered and gutter-collapsed boundaries, so the+ // outer ones can sit tens of points off the drawn border, and cell+ // rules are inset by their padding. The centre is the one point+ // that is unambiguously inside this cell.+ let cx = (xs[c] + xs[c + 1]) * 0.5;+ if !at_y.iter().any(|h| h.x_min <= cx && h.x_max >= cx) {+ mask[row][c] = true;+ }+ }+ }+ mask+}+fn passes_density_gate(cells: &[Vec<String>],cell_has_text: &[Vec<bool>],@@ -3481,13 +3627,25 @@ fn passes_density_gate(n_rows: usize,n_cols: usize,flattened: bool,+ spanned: &[Vec<bool>],dbg: bool,) -> bool {let total = n_rows * n_cols;- let empty_count = cell_has_text- .iter()- .flatten()- .filter(|filled| !**filled)+ // A spanned cell only excuses itself when the cell it is merged into+ // actually holds text. Walk up the run of merged cells to its head: an+ // empty head means nothing was merged here, just an unruled hole — the+ // shape a chart's vertical gridlines make, where forgiving the empties+ // would turn a plot into a 27-column table.+ let merged_into_text = |r: usize, c: usize| {+ let mut i = r;+ while i > 0 && spanned[i][c] {+ i -= 1;+ }+ cell_has_text[i][c]+ };+ let empty_count = (0..n_rows)+ .flat_map(|r| (0..n_cols).map(move |c| (r, c)))+ .filter(|&(r, c)| !cell_has_text[r][c] && !(spanned[r][c] && merged_into_text(r, c))).count();let empty_frac = (empty_count as f32) / (total as f32);if empty_frac <= TABLE_MAX_EMPTY_CELL_FRACTION {@@ -3724,6 +3882,7 @@ fn build_ruled_table(// too many spans straddle interior column boundaries (decorative box, not// a table).let (mut grid, consumed_indices) = assign_cells(lines, &xs, &ys, dbg)?;+ grid.spanned = rowspan_mask(hs, h_indices, vs, v_indices, &xs, &ys);// Collapse phantom rows (text-less thin border-strip rects) then phantom// columns (text-less narrow border strips). A real table with one phantom@@ -3757,6 +3916,7 @@ fn build_ruled_table(has_text: cell_has_text,repl: cells_repl,row_alpha_spanner,+ spanned,} = grid;let flattened_header = flatten_header_band(@@ -3780,6 +3940,18 @@ fn build_ruled_table(dbg,);+ // `merge_stacked_header` folds the top `k` rows into one, so realign the+ // rowspan mask: the new first row is the merged header (never a rowspan+ // continuation), and every later row keeps its own mask.+ let spanned = if spanned.len() > n_rows {+ let dropped = spanned.len() - n_rows;+ let mut m = vec![vec![false; n_cols]; 1];+ m.extend(spanned[dropped + 1..].iter().cloned());+ m+ } else {+ spanned+ };+if !passes_density_gate(&cells,&cell_has_text,@@ -3787,6 +3959,7 @@ fn build_ruled_table(n_rows,n_cols,flattened_header.is_some(),+ &spanned,dbg,) {return None;@@ -3877,6 +4050,37 @@ fn dedup_close(v: &mut Vec<f32>, tol: f32) {/// Find the bucket index `i` such that `boundaries[i] <= val < boundaries[i+1]`./// Returns `None` if `val` is outside the boundaries.+/// Distribute a multi-column run's words into ruled columns by each word's own+/// centre, returning `(column, text)` pairs in column order.+///+/// This replaces guessing a split point from a character index: with word boxes+/// the answer is simply which cell each word sits in. Returns `None` when the+/// run has no verified word geometry, or when every word lands in one column —+/// in that case the caller's whole-span fallbacks are the honest answer, since+/// the run only *overlaps* the neighbouring cell (padding, an overhanging+/// descender) rather than genuinely spanning it.+fn bucket_words_into_columns(span: &TextItem, xs: &[f32]) -> Option<Vec<(usize, String)>> {+ let words = verified_words(span)?;+ let mut out: Vec<(usize, String)> = Vec::new();+ for w in words {+ let cx = (w.x + w.width.max(0.0) * 0.5).clamp(xs[0], *xs.last()?);+ let col = find_bucket(xs, cx)?;+ match out.last_mut() {+ Some((c, text)) if *c == col => {+ text.push(' ');+ text.push_str(w.text.trim());+ }+ _ => out.push((col, w.text.trim().to_string())),+ }+ }+ // Words must not zig-zag back into an earlier column, and the run must+ // really occupy more than one.+ if out.len() < 2 || out.windows(2).any(|p| p[1].0 <= p[0].0) {+ return None;+ }+ Some(out)+}+fn find_bucket(boundaries: &[f32], val: f32) -> Option<usize> {if boundaries.len() < 2 || val < boundaries[0] || val > *boundaries.last().unwrap() {return None;