Skip to content

A fast, helpful, and open-source document parser

Merge remote-tracking branch 'origin/main' into fix/markdown-dropped-text

Logan Markewich384479e7eb29
diff --cc crates/liteparse/src/markdown_layout/tables.rs
index 5787dab,0c497f6..bce2221
--- a/crates/liteparse/src/markdown_layout/tables.rs
+++ b/crates/liteparse/src/markdown_layout/tables.rs
@@@ -1274,11 -1339,10 +1348,14 @@@ fn try_detect_table(lines: &[ProjectedL
.fold(f32::NEG_INFINITY, f32::max);
let tracks_right_edge = track_max_x + TABLE_TRACK_TOLERANCE_PT.max(8.0);
+ // Lines and cells the run steps over. They sit inside `start..end`, which
+ // the classifier skips, so they're handed back as interstitials rather
+ // than dropped; `true` marks a whole stepped-over line.
+ let mut stepped_over: Vec<(usize, bool, Cell)> = Vec::new();
let mut j = start_idx + 1;
+ // Leading half of the next row's wrapped label, prepended when that row is
+ // pushed (see the wrap path below).
+ let mut pending_prefix: Vec<TableCell> = Vec::new();
while j < lines.len() {
// Skip lines that sit entirely to the right of the table's column
// tracks — almost certainly content from a different page column.
@@@ -1429,10 -1504,26 +1531,27 @@@
.filter(|(c, r)| !cell_aligns_track(c, **r))
.count();
if misaligned > 1 {
+ pending_prefix.clear();
break;
}
+ if !pending_prefix.is_empty() {
+ let prefix = pending_prefix
+ .iter()
+ .map(|c| c.text.trim())
+ .filter(|t| !t.is_empty())
+ .collect::<Vec<_>>()
+ .join(" ");
+ if !prefix.is_empty() && !cells.is_empty() {
+ if !cells[0].text.is_empty() {
+ cells[0].text = format!("{prefix} {}", cells[0].text);
+ } else {
+ cells[0].text = prefix;
+ }
+ }
+ pending_prefix.clear();
+ }
rows.push((j, &lines[j], cells));
+ stepped_over.extend(trimmed.into_iter().map(|cell| (j, false, cell)));
j += 1;
}
@@@ -5205,9 -5749,243 +5802,243 @@@ pub(super) fn escape_table_cell(s: &str
#[cfg(test)]
mod tests {
- use super::super::test_helpers::{line, line_with_spans, rect_borders, stroke};
+ use super::super::test_helpers::{line, line_with_spans, page, rect_borders, stroke};
use super::*;
+ fn prose_test_words<'a>(
+ words: &'a [&'a str],
+ start_x: f32,
+ char_width: f32,
+ word_gap: f32,
+ ) -> Vec<(&'a str, f32, f32)> {
+ let mut x = start_x;
+ words
+ .iter()
+ .map(|word| {
+ let width = word.chars().count() as f32 * char_width;
+ let positioned = (*word, x, width);
+ x += width + word_gap;
+ positioned
+ })
+ .collect()
+ }
+
+ fn prose_test_span(words: &[(&str, f32, f32)], font_size: f32) -> crate::types::TextItem {
+ let x = words[0].1;
+ let end_x = words.last().map(|(_, x, width)| x + width).unwrap_or(x);
+ crate::types::TextItem {
+ text: words
+ .iter()
+ .map(|(word, _, _)| *word)
+ .collect::<Vec<_>>()
+ .join(" "),
+ x,
+ y: 0.0,
+ width: end_x - x,
+ height: font_size,
+ font_size: Some(font_size),
+ words: words
+ .iter()
+ .map(|(text, x, width)| crate::types::WordBox {
+ text: (*text).into(),
+ x: *x,
+ y: 0.0,
+ width: *width,
+ height: font_size,
+ })
+ .collect(),
+ ..Default::default()
+ }
+ }
+
+ fn prose_test_line(
+ text: &str,
+ spans: Vec<crate::types::TextItem>,
+ font_size: f32,
+ ) -> ProjectedLine {
+ let x0 = spans
+ .iter()
+ .map(|span| span.x)
+ .fold(f32::INFINITY, f32::min);
+ let x1 = spans
+ .iter()
+ .map(|span| span.x + span.width)
+ .fold(f32::NEG_INFINITY, f32::max);
+ let mut line = line(text, x0, 0.0, font_size, font_size);
+ line.bbox = crate::types::Rect {
+ x: x0,
+ y: 0.0,
+ width: x1 - x0,
+ height: font_size,
+ };
+ line.spans = spans;
+ line
+ }
+
+ #[test]
+ fn justified_prose_crossing_three_ruled_columns_stops() {
+ let text = "During the year ended September 30, 2025, compensation expense increased $1.3 billion, or 2.8%, compared";
+ let words: Vec<&str> = text.split_whitespace().collect();
+ let positions = prose_test_words(&words, 10.0, 3.0, 2.0);
+ let line = prose_test_line(text, vec![prose_test_span(&positions, 10.0)], 10.0);
+ let first_gap = positions[5].1 + positions[5].2 + 1.0;
+ let second_gap = positions[10].1 + positions[10].2 + 1.0;
+ let xs = [0.0, first_gap, second_gap, 330.0];
+
+ assert!(running_prose_sentence_shape(&line));
+ assert!(should_stop_at_running_prose(&line, &xs, 0, 25, false));
+ }
+
+ #[test]
+ fn first_rent_comps_row_with_wide_cell_gaps_continues() {
+ let left_words = ["1", "Ashley", "Gables", "Buckhead", "530", "E", "Paces"];
+ let middle_words = ["Ferry", "Rd", "NE", "2019", "Residential"];
+ let right_words = ["94%", "325", "0.6", "Miles", "$2,443"];
+ let left_positions = prose_test_words(&left_words, 10.0, 2.0, 2.0);
+ let middle_positions = prose_test_words(&middle_words, 120.0, 2.0, 2.0);
+ let right_positions = prose_test_words(&right_words, 220.0, 2.0, 2.0);
+ let text = "1 Ashley Gables Buckhead 530 E Paces Ferry Rd NE 2019 Residential 94% 325 0.6 Miles $2,443";
+ let line = prose_test_line(
+ text,
+ vec![
+ prose_test_span(&left_positions, 10.0),
+ prose_test_span(&middle_positions, 10.0),
+ prose_test_span(&right_positions, 10.0),
+ ],
+ 10.0,
+ );
+ let xs = [0.0, 100.0, 200.0, 300.0];
+ let ys = [0.0, 20.0];
+
+ assert!(running_prose_sentence_shape(&line));
+ let (_, consumed) = assign_cells(&[line], &xs, &ys, true, false).unwrap();
+ assert_eq!(
+ consumed,
+ [0],
+ "the first data row has no prior assigned cells"
+ );
+ }
+
+ fn at_y(mut line: ProjectedLine, y: f32) -> ProjectedLine {
+ line.bbox.y = y;
+ for span in &mut line.spans {
+ span.y = y;
+ for word in &mut span.words {
+ word.y = y;
+ }
+ }
+ line
+ }
+
+ fn data_row(cells: &[&str], y: f32) -> ProjectedLine {
+ let spans: Vec<_> = cells
+ .iter()
+ .enumerate()
+ .map(|(c, text)| {
+ let words = [*text];
+ prose_test_span(
+ &prose_test_words(&words, 10.0 + 100.0 * c as f32, 5.0, 2.0),
+ 10.0,
+ )
+ })
+ .collect();
+ at_y(prose_test_line(&cells.join(" "), spans, 10.0), y)
+ }
+
+ fn prose_row(y: f32) -> ProjectedLine {
+ let text = "During the year ended September 30, 2025, compensation expense increased $1.3 billion, or 2.8%, compared";
+ let words: Vec<&str> = text.split_whitespace().collect();
+ let positions = prose_test_words(&words, 10.0, 3.0, 2.0);
+ at_y(
+ prose_test_line(text, vec![prose_test_span(&positions, 10.0)], 10.0),
+ y,
+ )
+ }
+
+ #[test]
+ fn merged_note_row_mid_table_keeps_rows_below() {
+ let lines = vec![
+ data_row(&["Cash", "1,000", "900"], 5.0),
+ prose_row(25.0),
+ data_row(&["Debt", "500", "600"], 45.0),
+ data_row(&["Equity", "500", "300"], 65.0),
+ ];
+ let xs = [0.0, 100.0, 200.0, 330.0];
+ let ys = [0.0, 20.0, 40.0, 60.0, 80.0];
+
+ assert!(should_stop_at_running_prose(&lines[1], &xs, 1, 1, false));
+ let (_, consumed) = assign_cells(&lines, &xs, &ys, true, false).unwrap();
+ assert_eq!(consumed, [0, 1, 2, 3]);
+ }
+
+ #[test]
+ fn prose_paragraph_between_tables_stops() {
+ let lines = vec![
+ data_row(&["Cash", "1,000", "900"], 5.0),
+ prose_row(25.0),
+ prose_row(45.0),
+ data_row(&["Debt", "500", "600"], 65.0),
+ data_row(&["Equity", "500", "300"], 85.0),
+ ];
+ let xs = [0.0, 100.0, 200.0, 330.0];
+ let ys = [0.0, 20.0, 40.0, 60.0, 80.0, 100.0];
+
+ let (_, consumed) = assign_cells(&lines, &xs, &ys, true, false).unwrap();
+ assert_eq!(consumed, [0]);
+ }
+
+ #[test]
+ fn running_prose_below_table_still_stops() {
+ let lines = vec![
+ data_row(&["Cash", "1,000", "900"], 5.0),
+ data_row(&["Debt", "500", "600"], 25.0),
+ prose_row(45.0),
+ prose_row(65.0),
+ ];
+ let xs = [0.0, 100.0, 200.0, 330.0];
+ let ys = [0.0, 20.0, 40.0, 60.0, 80.0];
+
+ let (_, consumed) = assign_cells(&lines, &xs, &ys, true, false).unwrap();
+ assert_eq!(consumed, [0, 1]);
+ }
+
+ #[test]
+ fn long_left_column_label_with_right_column_numbers_continues() {
+ let label = "The subject property benefits from recent renovations and convenient access to neighborhood services.";
+ let text = format!("{label} $1,250 2025");
+ let spans = vec![
+ crate::types::TextItem {
+ text: label.into(),
+ x: 10.0,
+ y: 0.0,
+ width: 270.0,
+ height: 10.0,
+ ..Default::default()
+ },
+ crate::types::TextItem {
+ text: "$1,250".into(),
+ x: 330.0,
+ y: 0.0,
+ width: 30.0,
+ height: 10.0,
+ ..Default::default()
+ },
+ crate::types::TextItem {
+ text: "2025".into(),
+ x: 520.0,
+ y: 0.0,
+ width: 24.0,
+ height: 10.0,
+ ..Default::default()
+ },
+ ];
+ let line = prose_test_line(&text, spans, 10.0);
+ let xs = [0.0, 300.0, 500.0, 680.0];
+
+ assert!(running_prose_sentence_shape(&line));
+ assert!(!should_stop_at_running_prose(&line, &xs, 0, 0, false));
+ }
+
/// One PDFium run whose words sit at the given (x, width) positions.
fn run_with_words(words: &[(&str, f32, f32)]) -> TextItem {
let x = words[0].1;