A fast, helpful, and open-source document parser
Detect running prose across ruled-table columns
Artie Pesh-Imam40fdda7cf50c
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rsindex 34fecb7..638a00b 100644--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@ -3818,10 +3818,224 @@ impl CellGrid {/// consumed line indices, or `None` if no line landed inside the grid or too/// many spans straddle interior boundaries (a decorative box over prose, not a/// table).+#[derive(Clone, Copy)]+struct ProseBoundaryCrossing {+ crossed: bool,+ via: &'static str,+ gap_width: Option<f32>,+}++type ProseXRange = (f32, f32);++fn running_prose_sentence_shape(line: &ProjectedLine) -> bool {+ let text = line.text.trim();+ let char_count = text.chars().count();+ let word_count = text.split_whitespace().count();+ let non_space_count = text.chars().filter(|ch| !ch.is_whitespace()).count();+ let alpha_count = text.chars().filter(|ch| ch.is_alphabetic()).count();+ char_count >= 90+ && word_count >= 12+ && non_space_count > 0+ && alpha_count as f32 >= non_space_count as f32 * 0.6+ && text.chars().any(|ch| matches!(ch, '.' | '!' | '?'))+}++fn prose_line_x_ranges(line: &ProjectedLine) -> (Vec<ProseXRange>, Vec<ProseXRange>) {+ let mut words = Vec::new();+ let mut item_only = Vec::new();+ for span in line+ .spans+ .iter()+ .filter(|span| !span.text.trim().is_empty())+ {+ if let Some(span_words) = verified_words(span) {+ words.extend(span_words.into_iter().filter_map(|word| {+ let x1 = word.x + word.width;+ (word.width > 0.0 && x1.is_finite()).then_some((word.x, x1))+ }));+ } else if span.width > 0.0 {+ item_only.push((span.x, span.x + span.width));+ }+ }+ words.sort_by(|a, b| a.0.total_cmp(&b.0));+ (words, item_only)+}++fn prose_boundary_crossing(+ boundary: f32,+ word_ranges: &[ProseXRange],+ item_only_ranges: &[ProseXRange],+ gap_limit: f32,+) -> ProseBoundaryCrossing {+ const EDGE_MARGIN_PT: f32 = 1.5;+ if word_ranges+ .iter()+ .any(|&(x0, x1)| boundary >= x0 + EDGE_MARGIN_PT && boundary <= x1 - EDGE_MARGIN_PT)+ {+ return ProseBoundaryCrossing {+ crossed: true,+ via: "word-box",+ gap_width: None,+ };+ }+ for pair in word_ranges.windows(2) {+ let gap_width = pair[1].0 - pair[0].1;+ if gap_width >= 0.0 && boundary >= pair[0].1 && boundary <= pair[1].0 {+ return ProseBoundaryCrossing {+ crossed: gap_width < gap_limit,+ via: if gap_width < gap_limit {+ "inter-word-gap"+ } else {+ "wide-gap"+ },+ gap_width: Some(gap_width),+ };+ }+ }+ if word_ranges+ .iter()+ .any(|&(x0, x1)| boundary > x0 && boundary < x1)+ {+ return ProseBoundaryCrossing {+ crossed: false,+ via: "word-edge-margin",+ gap_width: None,+ };+ }+ if item_only_ranges+ .iter()+ .any(|&(x0, x1)| boundary >= x0 + EDGE_MARGIN_PT && boundary <= x1 - EDGE_MARGIN_PT)+ {+ return ProseBoundaryCrossing {+ crossed: true,+ via: "item-box",+ gap_width: None,+ };+ }+ if item_only_ranges+ .iter()+ .any(|&(x0, x1)| boundary > x0 && boundary < x1)+ {+ return ProseBoundaryCrossing {+ crossed: false,+ via: "item-edge-margin",+ gap_width: None,+ };+ }+ ProseBoundaryCrossing {+ crossed: false,+ via: "none",+ gap_width: None,+ }+}++fn prose_line_font_height(line: &ProjectedLine) -> f32 {+ let mut heights: Vec<f32> = line+ .spans+ .iter()+ .filter_map(|span| span.font_height.or(span.font_size))+ .filter(|height| height.is_finite() && *height > 0.0)+ .collect();+ heights.sort_by(f32::total_cmp);+ heights+ .get(heights.len() / 2)+ .copied()+ .or_else(|| {+ (line.dominant_font_size.is_finite() && line.dominant_font_size > 0.0)+ .then_some(line.dominant_font_size)+ })+ .unwrap_or_else(|| line.bbox.height.max(1.0))+}++fn should_stop_at_running_prose(+ line: &ProjectedLine,+ xs: &[f32],+ row: usize,+ idx: usize,+ dbg: bool,+) -> bool {+ if !running_prose_sentence_shape(line) {+ return false;+ }+ let boundaries: Vec<f32> = xs+ .iter()+ .copied()+ .skip(1)+ .take(xs.len().saturating_sub(2))+ .collect();+ let (word_ranges, item_only_ranges) = prose_line_x_ranges(line);+ let geometry = match (word_ranges.is_empty(), item_only_ranges.is_empty()) {+ (false, false) => "verified-word-boxes with item-box fallback",+ (false, true) => "verified-word-boxes",+ (true, false) => "item-level boxes (no verified word boxes)",+ (true, true) => "no word or item boxes",+ };+ let line_x0 = line.bbox.x;+ let line_x1 = line.bbox.x + line.bbox.width;+ let font_height = prose_line_font_height(line);+ let gap_limit = font_height * 0.5;+ if boundaries.is_empty() {+ if dbg {+ eprintln!(+ "[ruled] prose candidate line idx={idx} row={row} x={line_x0:.1}..{line_x1:.1} geometry={geometry}: no interior column boundaries; skip stop"+ );+ }+ return false;+ }+ if dbg {+ eprintln!(+ "[ruled] prose candidate line idx={idx} row={row} x={line_x0:.1}..{line_x1:.1} font-height={font_height:.1} geometry={geometry} interior-boundaries={}",+ boundaries.len()+ );+ }+ let mut in_extent = 0usize;+ let mut crossed = 0usize;+ for boundary in boundaries {+ let within_line = boundary >= line_x0 && boundary <= line_x1;+ let decision = if within_line {+ in_extent += 1;+ prose_boundary_crossing(boundary, &word_ranges, &item_only_ranges, gap_limit)+ } else {+ ProseBoundaryCrossing {+ crossed: false,+ via: "outside-line",+ gap_width: None,+ }+ };+ if decision.crossed {+ crossed += 1;+ }+ if dbg {+ let gap_width = decision+ .gap_width+ .map_or_else(|| "n/a".to_string(), |gap| format!("{gap:.1}"));+ eprintln!(+ "[ruled] prose boundary idx={idx} x={boundary:.1} in_extent={} crossed={} via={} gap_width={gap_width} gap_limit={gap_limit:.1}",+ if within_line { "yes" } else { "no" },+ if decision.crossed { "yes" } else { "no" },+ decision.via+ );+ }+ }+ let ratio = if in_extent == 0 {+ 0.0+ } else {+ crossed as f32 / in_extent as f32+ };+ let stop = in_extent >= 2 && crossed >= 2 && ratio >= 0.67;+ if dbg {+ eprintln!(+ "[ruled] prose boundary decision idx={idx} crossed={crossed} in_extent={in_extent} ratio={ratio:.2} stop={stop}"+ );+ }+ stop+}+fn assign_cells(lines: &[ProjectedLine],xs: &[f32],ys: &[f32],+ stop_at_running_prose: bool,dbg: bool,) -> Option<(CellGrid, Vec<usize>)> {let n_rows = ys.len() - 1;@@ -3861,6 +4075,15 @@ fn assign_cells(if line.text.trim().is_empty() {continue;}+ if stop_at_running_prose && should_stop_at_running_prose(line, xs, row, idx, dbg) {+ if dbg {+ eprintln!(+ "[ruled] stop at running prose line idx={idx} row={row}: {:?}",+ line.text.chars().take(120).collect::<String>()+ );+ }+ break;+ }// Only consume the line when its text sits inside the grid// horizontally — a line poking past the grid edge belongs (at least@@ -4674,7 +4897,8 @@ fn build_ruled_table_from(// Assign text to cells, then reject if no lines landed inside the grid or// too many spans straddle interior column boundaries (decorative box, not// a table).- let (mut grid, consumed_indices) = assign_cells(lines, &xs, &ys, dbg)?;+ let (mut grid, consumed_indices) =+ assign_cells(lines, &xs, &ys, pass == RuledPass::Global, dbg)?;let straddle_frac = grid.straddle_frac;grid.spanned = rowspan_mask(hs, h_indices, vs, v_kept, &xs, &ys);@@ -5330,6 +5554,157 @@ mod tests {use super::super::test_helpers::{line, line_with_spans, rect_borders, stroke};use super::*;+ fn prose_test_words<'a>(+ words: &'a [&'a str],+ start_x: f32,+ char_width: f32,+ word_gap: f32,+ ) -> Vec<(&'a str, f32, f32)> {+ let mut x = start_x;+ words+ .iter()+ .map(|word| {+ let width = word.chars().count() as f32 * char_width;+ let positioned = (*word, x, width);+ x += width + word_gap;+ positioned+ })+ .collect()+ }++ fn prose_test_span(words: &[(&str, f32, f32)], font_size: f32) -> crate::types::TextItem {+ let x = words[0].1;+ let end_x = words.last().map(|(_, x, width)| x + width).unwrap_or(x);+ crate::types::TextItem {+ text: words+ .iter()+ .map(|(word, _, _)| *word)+ .collect::<Vec<_>>()+ .join(" "),+ x,+ y: 0.0,+ width: end_x - x,+ height: font_size,+ font_size: Some(font_size),+ words: words+ .iter()+ .map(|(text, x, width)| crate::types::WordBox {+ text: (*text).into(),+ x: *x,+ y: 0.0,+ width: *width,+ height: font_size,+ })+ .collect(),+ ..Default::default()+ }+ }++ fn prose_test_line(+ text: &str,+ spans: Vec<crate::types::TextItem>,+ font_size: f32,+ ) -> ProjectedLine {+ let x0 = spans+ .iter()+ .map(|span| span.x)+ .fold(f32::INFINITY, f32::min);+ let x1 = spans+ .iter()+ .map(|span| span.x + span.width)+ .fold(f32::NEG_INFINITY, f32::max);+ let mut line = line(text, x0, 0.0, font_size, font_size);+ line.bbox = crate::types::Rect {+ x: x0,+ y: 0.0,+ width: x1 - x0,+ height: font_size,+ };+ line.spans = spans;+ line+ }++ #[test]+ fn justified_prose_crossing_three_ruled_columns_stops() {+ let text = "During the year ended September 30, 2025, compensation expense increased $1.3 billion, or 2.8%, compared";+ let words: Vec<&str> = text.split_whitespace().collect();+ let positions = prose_test_words(&words, 10.0, 3.0, 2.0);+ let line = prose_test_line(text, vec![prose_test_span(&positions, 10.0)], 10.0);+ let first_gap = positions[5].1 + positions[5].2 + 1.0;+ let second_gap = positions[10].1 + positions[10].2 + 1.0;+ let xs = [0.0, first_gap, second_gap, 330.0];++ assert!(running_prose_sentence_shape(&line));+ assert!(should_stop_at_running_prose(&line, &xs, 0, 25, false));+ }++ #[test]+ fn first_rent_comps_row_with_wide_cell_gaps_continues() {+ let left_words = ["1", "Ashley", "Gables", "Buckhead", "530", "E", "Paces"];+ let middle_words = ["Ferry", "Rd", "NE", "2019", "Residential"];+ let right_words = ["94%", "325", "0.6", "Miles", "$2,443"];+ let left_positions = prose_test_words(&left_words, 10.0, 2.0, 2.0);+ let middle_positions = prose_test_words(&middle_words, 120.0, 2.0, 2.0);+ let right_positions = prose_test_words(&right_words, 220.0, 2.0, 2.0);+ let text = "1 Ashley Gables Buckhead 530 E Paces Ferry Rd NE 2019 Residential 94% 325 0.6 Miles $2,443";+ let line = prose_test_line(+ text,+ vec![+ prose_test_span(&left_positions, 10.0),+ prose_test_span(&middle_positions, 10.0),+ prose_test_span(&right_positions, 10.0),+ ],+ 10.0,+ );+ let xs = [0.0, 100.0, 200.0, 300.0];+ let ys = [0.0, 20.0];++ assert!(running_prose_sentence_shape(&line));+ let (_, consumed) = assign_cells(&[line], &xs, &ys, true, false).unwrap();+ assert_eq!(+ consumed,+ [0],+ "the first data row has no prior assigned cells"+ );+ }++ #[test]+ fn long_left_column_label_with_right_column_numbers_continues() {+ let label = "The subject property benefits from recent renovations and convenient access to neighborhood services.";+ let text = format!("{label} $1,250 2025");+ let spans = vec![+ crate::types::TextItem {+ text: label.into(),+ x: 10.0,+ y: 0.0,+ width: 270.0,+ height: 10.0,+ ..Default::default()+ },+ crate::types::TextItem {+ text: "$1,250".into(),+ x: 330.0,+ y: 0.0,+ width: 30.0,+ height: 10.0,+ ..Default::default()+ },+ crate::types::TextItem {+ text: "2025".into(),+ x: 520.0,+ y: 0.0,+ width: 24.0,+ height: 10.0,+ ..Default::default()+ },+ ];+ let line = prose_test_line(&text, spans, 10.0);+ let xs = [0.0, 300.0, 500.0, 680.0];++ assert!(running_prose_sentence_shape(&line));+ assert!(!should_stop_at_running_prose(&line, &xs, 0, 0, false));+ }+/// One PDFium run whose words sit at the given (x, width) positions.fn run_with_words(words: &[(&str, f32, f32)]) -> TextItem {let x = words[0].1;