Skip to content

A fast, helpful, and open-source document parser

fix(markdown): read a table run that keeps under 70% of its span as text

ceamkrier14001ae40f6d
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rs
index c2f5e13..299e7b9 100644
--- a/crates/liteparse/src/markdown_layout/classify.rs
+++ b/crates/liteparse/src/markdown_layout/classify.rs
@@ -553,7 +553,14 @@ fn classify_region(
let ruled_runs = detect_ruled_tables(lines, rule_segments, page.page_width, page.page_height);
let borderless_runs = precomputed_tables
.unwrap_or_else(|| detect_tables_banded(lines, rule_segments, page.page_height));
- let table_runs = merge_table_runs(ruled_runs, borderless_runs);
+ // A table that carries under 70% of the text of the lines it spans loses
+ // the rest, since the loop below skips the span. Handing back lines no
+ // cell took a word from cannot help when every line shares a word with
+ // the cells, so such a run is read as text: every value kept, no grid.
+ let table_runs: Vec<_> = merge_table_runs(ruled_runs, borderless_runs)
+ .into_iter()
+ .filter(|run| super::tables::run_text_kept(run, lines).is_none_or(|kept| kept >= 0.7))
+ .collect();
// Region-wide pre-pass: which line indices carry a lettered/roman marker
// (`a.`, `i.`) that belongs to a confirmed sequential list run. Only these
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs
index 3a4be49..426144f 100644
--- a/crates/liteparse/src/markdown_layout/tables.rs
+++ b/crates/liteparse/src/markdown_layout/tables.rs
@@ -808,6 +808,33 @@ fn is_alpha_dominant(text: &str) -> bool {
letters > digits
}
+/// Share of the non-space characters of the lines a run spans that its cells
+/// and interstitials carry, or `None` for a run that is not a pipe table. The
+/// classifier skips the whole span, so whatever the table leaves out is
+/// dropped from the markdown. Header replication can push it above 1.
+pub(super) fn run_text_kept(run: &TableRun, lines: &[ProjectedLine]) -> Option<f32> {
+ let Block::Table { header, rows } = &run.block else {
+ return None;
+ };
+ let chars = |t: &str| t.chars().filter(|c| !c.is_whitespace()).count();
+ let span: usize = lines
+ .get(run.start..run.end)?
+ .iter()
+ .map(|l| chars(&l.text))
+ .sum();
+ if span == 0 {
+ return None;
+ }
+ let kept: usize = header
+ .iter()
+ .flatten()
+ .chain(rows.iter().flatten())
+ .chain(&run.interstitials)
+ .map(|c| chars(&c.text))
+ .sum();
+ Some(kept as f32 / span as f32)
+}
+
/// Lines inside a run's span that none of its cells or interstitials took a
/// word from. The classifier skips the span, so these would be dropped. A
/// line that shares even one word with the table stays with it, so nothing
@@ -7757,6 +7784,32 @@ mod tests {
assert!(md.contains(r"Dose \*as written\*"), "{md}");
}
+ #[test]
+ fn run_text_kept_counts_what_the_table_carries() {
+ use super::super::test_helpers::line;
+ let lines = vec![
+ line("Model Capacity", 10.0, 10.0, 12.0, 10.0),
+ line("50D 5,000 @ 24 2,270 @ 600 26-79", 10.0, 25.0, 12.0, 10.0),
+ ];
+ let run = |rows: Vec<Vec<Cell>>| TableRun {
+ interstitials: Vec::new(),
+ start: 0,
+ end: 2,
+ body_start: 0,
+ bbox: None,
+ block: Block::Table { header: None, rows },
+ };
+ let whole = run(vec![
+ vec!["Model".into(), "Capacity".into()],
+ vec!["50D".into(), "5,000 @ 24 2,270 @ 600 26-79".into()],
+ ]);
+ assert_eq!(run_text_kept(&whole, &lines), Some(1.0));
+ // Only the model column made it into the cells: the capacities are
+ // inside the span the classifier skips, so they would be lost.
+ let partial = run(vec![vec!["Model".into()], vec!["50D".into()]]);
+ assert!(run_text_kept(&partial, &lines).unwrap() < 0.7);
+ }
+
#[test]
fn lines_left_out_names_lines_no_cell_took() {
use super::super::test_helpers::line;