A fast, helpful, and open-source document parser
Merge remote-tracking branch 'origin/main' into fix/markdown-dropped-text
Logan Markewich384479e7eb29
diff --cc crates/liteparse/src/markdown_layout/tables.rsindex 5787dab,0c497f6..bce2221--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@@ -1274,11 -1339,10 +1348,14 @@@ fn try_detect_table(lines: &[ProjectedL.fold(f32::NEG_INFINITY, f32::max);let tracks_right_edge = track_max_x + TABLE_TRACK_TOLERANCE_PT.max(8.0);+ // Lines and cells the run steps over. They sit inside `start..end`, which+ // the classifier skips, so they're handed back as interstitials rather+ // than dropped; `true` marks a whole stepped-over line.+ let mut stepped_over: Vec<(usize, bool, Cell)> = Vec::new();let mut j = start_idx + 1;+ // Leading half of the next row's wrapped label, prepended when that row is+ // pushed (see the wrap path below).+ let mut pending_prefix: Vec<TableCell> = Vec::new();while j < lines.len() {// Skip lines that sit entirely to the right of the table's column// tracks — almost certainly content from a different page column.@@@ -1429,10 -1504,26 +1531,27 @@@.filter(|(c, r)| !cell_aligns_track(c, **r)).count();if misaligned > 1 {+ pending_prefix.clear();break;}+ if !pending_prefix.is_empty() {+ let prefix = pending_prefix+ .iter()+ .map(|c| c.text.trim())+ .filter(|t| !t.is_empty())+ .collect::<Vec<_>>()+ .join(" ");+ if !prefix.is_empty() && !cells.is_empty() {+ if !cells[0].text.is_empty() {+ cells[0].text = format!("{prefix} {}", cells[0].text);+ } else {+ cells[0].text = prefix;+ }+ }+ pending_prefix.clear();+ }rows.push((j, &lines[j], cells));+ stepped_over.extend(trimmed.into_iter().map(|cell| (j, false, cell)));j += 1;}@@@ -5205,9 -5749,243 +5802,243 @@@ pub(super) fn escape_table_cell(s: &str#[cfg(test)]mod tests {- use super::super::test_helpers::{line, line_with_spans, rect_borders, stroke};+ use super::super::test_helpers::{line, line_with_spans, page, rect_borders, stroke};use super::*;+ fn prose_test_words<'a>(+ words: &'a [&'a str],+ start_x: f32,+ char_width: f32,+ word_gap: f32,+ ) -> Vec<(&'a str, f32, f32)> {+ let mut x = start_x;+ words+ .iter()+ .map(|word| {+ let width = word.chars().count() as f32 * char_width;+ let positioned = (*word, x, width);+ x += width + word_gap;+ positioned+ })+ .collect()+ }++ fn prose_test_span(words: &[(&str, f32, f32)], font_size: f32) -> crate::types::TextItem {+ let x = words[0].1;+ let end_x = words.last().map(|(_, x, width)| x + width).unwrap_or(x);+ crate::types::TextItem {+ text: words+ .iter()+ .map(|(word, _, _)| *word)+ .collect::<Vec<_>>()+ .join(" "),+ x,+ y: 0.0,+ width: end_x - x,+ height: font_size,+ font_size: Some(font_size),+ words: words+ .iter()+ .map(|(text, x, width)| crate::types::WordBox {+ text: (*text).into(),+ x: *x,+ y: 0.0,+ width: *width,+ height: font_size,+ })+ .collect(),+ ..Default::default()+ }+ }++ fn prose_test_line(+ text: &str,+ spans: Vec<crate::types::TextItem>,+ font_size: f32,+ ) -> ProjectedLine {+ let x0 = spans+ .iter()+ .map(|span| span.x)+ .fold(f32::INFINITY, f32::min);+ let x1 = spans+ .iter()+ .map(|span| span.x + span.width)+ .fold(f32::NEG_INFINITY, f32::max);+ let mut line = line(text, x0, 0.0, font_size, font_size);+ line.bbox = crate::types::Rect {+ x: x0,+ y: 0.0,+ width: x1 - x0,+ height: font_size,+ };+ line.spans = spans;+ line+ }++ #[test]+ fn justified_prose_crossing_three_ruled_columns_stops() {+ let text = "During the year ended September 30, 2025, compensation expense increased $1.3 billion, or 2.8%, compared";+ let words: Vec<&str> = text.split_whitespace().collect();+ let positions = prose_test_words(&words, 10.0, 3.0, 2.0);+ let line = prose_test_line(text, vec![prose_test_span(&positions, 10.0)], 10.0);+ let first_gap = positions[5].1 + positions[5].2 + 1.0;+ let second_gap = positions[10].1 + positions[10].2 + 1.0;+ let xs = [0.0, first_gap, second_gap, 330.0];++ assert!(running_prose_sentence_shape(&line));+ assert!(should_stop_at_running_prose(&line, &xs, 0, 25, false));+ }++ #[test]+ fn first_rent_comps_row_with_wide_cell_gaps_continues() {+ let left_words = ["1", "Ashley", "Gables", "Buckhead", "530", "E", "Paces"];+ let middle_words = ["Ferry", "Rd", "NE", "2019", "Residential"];+ let right_words = ["94%", "325", "0.6", "Miles", "$2,443"];+ let left_positions = prose_test_words(&left_words, 10.0, 2.0, 2.0);+ let middle_positions = prose_test_words(&middle_words, 120.0, 2.0, 2.0);+ let right_positions = prose_test_words(&right_words, 220.0, 2.0, 2.0);+ let text = "1 Ashley Gables Buckhead 530 E Paces Ferry Rd NE 2019 Residential 94% 325 0.6 Miles $2,443";+ let line = prose_test_line(+ text,+ vec![+ prose_test_span(&left_positions, 10.0),+ prose_test_span(&middle_positions, 10.0),+ prose_test_span(&right_positions, 10.0),+ ],+ 10.0,+ );+ let xs = [0.0, 100.0, 200.0, 300.0];+ let ys = [0.0, 20.0];++ assert!(running_prose_sentence_shape(&line));+ let (_, consumed) = assign_cells(&[line], &xs, &ys, true, false).unwrap();+ assert_eq!(+ consumed,+ [0],+ "the first data row has no prior assigned cells"+ );+ }++ fn at_y(mut line: ProjectedLine, y: f32) -> ProjectedLine {+ line.bbox.y = y;+ for span in &mut line.spans {+ span.y = y;+ for word in &mut span.words {+ word.y = y;+ }+ }+ line+ }++ fn data_row(cells: &[&str], y: f32) -> ProjectedLine {+ let spans: Vec<_> = cells+ .iter()+ .enumerate()+ .map(|(c, text)| {+ let words = [*text];+ prose_test_span(+ &prose_test_words(&words, 10.0 + 100.0 * c as f32, 5.0, 2.0),+ 10.0,+ )+ })+ .collect();+ at_y(prose_test_line(&cells.join(" "), spans, 10.0), y)+ }++ fn prose_row(y: f32) -> ProjectedLine {+ let text = "During the year ended September 30, 2025, compensation expense increased $1.3 billion, or 2.8%, compared";+ let words: Vec<&str> = text.split_whitespace().collect();+ let positions = prose_test_words(&words, 10.0, 3.0, 2.0);+ at_y(+ prose_test_line(text, vec![prose_test_span(&positions, 10.0)], 10.0),+ y,+ )+ }++ #[test]+ fn merged_note_row_mid_table_keeps_rows_below() {+ let lines = vec![+ data_row(&["Cash", "1,000", "900"], 5.0),+ prose_row(25.0),+ data_row(&["Debt", "500", "600"], 45.0),+ data_row(&["Equity", "500", "300"], 65.0),+ ];+ let xs = [0.0, 100.0, 200.0, 330.0];+ let ys = [0.0, 20.0, 40.0, 60.0, 80.0];++ assert!(should_stop_at_running_prose(&lines[1], &xs, 1, 1, false));+ let (_, consumed) = assign_cells(&lines, &xs, &ys, true, false).unwrap();+ assert_eq!(consumed, [0, 1, 2, 3]);+ }++ #[test]+ fn prose_paragraph_between_tables_stops() {+ let lines = vec![+ data_row(&["Cash", "1,000", "900"], 5.0),+ prose_row(25.0),+ prose_row(45.0),+ data_row(&["Debt", "500", "600"], 65.0),+ data_row(&["Equity", "500", "300"], 85.0),+ ];+ let xs = [0.0, 100.0, 200.0, 330.0];+ let ys = [0.0, 20.0, 40.0, 60.0, 80.0, 100.0];++ let (_, consumed) = assign_cells(&lines, &xs, &ys, true, false).unwrap();+ assert_eq!(consumed, [0]);+ }++ #[test]+ fn running_prose_below_table_still_stops() {+ let lines = vec![+ data_row(&["Cash", "1,000", "900"], 5.0),+ data_row(&["Debt", "500", "600"], 25.0),+ prose_row(45.0),+ prose_row(65.0),+ ];+ let xs = [0.0, 100.0, 200.0, 330.0];+ let ys = [0.0, 20.0, 40.0, 60.0, 80.0];++ let (_, consumed) = assign_cells(&lines, &xs, &ys, true, false).unwrap();+ assert_eq!(consumed, [0, 1]);+ }++ #[test]+ fn long_left_column_label_with_right_column_numbers_continues() {+ let label = "The subject property benefits from recent renovations and convenient access to neighborhood services.";+ let text = format!("{label} $1,250 2025");+ let spans = vec![+ crate::types::TextItem {+ text: label.into(),+ x: 10.0,+ y: 0.0,+ width: 270.0,+ height: 10.0,+ ..Default::default()+ },+ crate::types::TextItem {+ text: "$1,250".into(),+ x: 330.0,+ y: 0.0,+ width: 30.0,+ height: 10.0,+ ..Default::default()+ },+ crate::types::TextItem {+ text: "2025".into(),+ x: 520.0,+ y: 0.0,+ width: 24.0,+ height: 10.0,+ ..Default::default()+ },+ ];+ let line = prose_test_line(&text, spans, 10.0);+ let xs = [0.0, 300.0, 500.0, 680.0];++ assert!(running_prose_sentence_shape(&line));+ assert!(!should_stop_at_running_prose(&line, &xs, 0, 0, false));+ }+/// One PDFium run whose words sit at the given (x, width) positions.fn run_with_words(words: &[(&str, f32, f32)]) -> TextItem {let x = words[0].1;