A fast, helpful, and open-source document parser
fix(markdown): read a table run that keeps under 70% of its span as text
ceamkrier14001ae40f6d
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rsindex c2f5e13..299e7b9 100644--- a/crates/liteparse/src/markdown_layout/classify.rs+++ b/crates/liteparse/src/markdown_layout/classify.rs@@ -553,7 +553,14 @@ fn classify_region(let ruled_runs = detect_ruled_tables(lines, rule_segments, page.page_width, page.page_height);let borderless_runs = precomputed_tables.unwrap_or_else(|| detect_tables_banded(lines, rule_segments, page.page_height));- let table_runs = merge_table_runs(ruled_runs, borderless_runs);+ // A table that carries under 70% of the text of the lines it spans loses+ // the rest, since the loop below skips the span. Handing back lines no+ // cell took a word from cannot help when every line shares a word with+ // the cells, so such a run is read as text: every value kept, no grid.+ let table_runs: Vec<_> = merge_table_runs(ruled_runs, borderless_runs)+ .into_iter()+ .filter(|run| super::tables::run_text_kept(run, lines).is_none_or(|kept| kept >= 0.7))+ .collect();// Region-wide pre-pass: which line indices carry a lettered/roman marker// (`a.`, `i.`) that belongs to a confirmed sequential list run. Only thesediff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rsindex 3a4be49..426144f 100644--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@ -808,6 +808,33 @@ fn is_alpha_dominant(text: &str) -> bool {letters > digits}+/// Share of the non-space characters of the lines a run spans that its cells+/// and interstitials carry, or `None` for a run that is not a pipe table. The+/// classifier skips the whole span, so whatever the table leaves out is+/// dropped from the markdown. Header replication can push it above 1.+pub(super) fn run_text_kept(run: &TableRun, lines: &[ProjectedLine]) -> Option<f32> {+ let Block::Table { header, rows } = &run.block else {+ return None;+ };+ let chars = |t: &str| t.chars().filter(|c| !c.is_whitespace()).count();+ let span: usize = lines+ .get(run.start..run.end)?+ .iter()+ .map(|l| chars(&l.text))+ .sum();+ if span == 0 {+ return None;+ }+ let kept: usize = header+ .iter()+ .flatten()+ .chain(rows.iter().flatten())+ .chain(&run.interstitials)+ .map(|c| chars(&c.text))+ .sum();+ Some(kept as f32 / span as f32)+}+/// Lines inside a run's span that none of its cells or interstitials took a/// word from. The classifier skips the span, so these would be dropped. A/// line that shares even one word with the table stays with it, so nothing@@ -7757,6 +7784,32 @@ mod tests {assert!(md.contains(r"Dose \*as written\*"), "{md}");}+ #[test]+ fn run_text_kept_counts_what_the_table_carries() {+ use super::super::test_helpers::line;+ let lines = vec![+ line("Model Capacity", 10.0, 10.0, 12.0, 10.0),+ line("50D 5,000 @ 24 2,270 @ 600 26-79", 10.0, 25.0, 12.0, 10.0),+ ];+ let run = |rows: Vec<Vec<Cell>>| TableRun {+ interstitials: Vec::new(),+ start: 0,+ end: 2,+ body_start: 0,+ bbox: None,+ block: Block::Table { header: None, rows },+ };+ let whole = run(vec![+ vec!["Model".into(), "Capacity".into()],+ vec!["50D".into(), "5,000 @ 24 2,270 @ 600 26-79".into()],+ ]);+ assert_eq!(run_text_kept(&whole, &lines), Some(1.0));+ // Only the model column made it into the cells: the capacities are+ // inside the span the classifier skips, so they would be lost.+ let partial = run(vec![vec!["Model".into()], vec!["50D".into()]]);+ assert!(run_text_kept(&partial, &lines).unwrap() < 0.7);+ }+#[test]fn lines_left_out_names_lines_no_cell_took() {use super::super::test_helpers::line;