A fast, helpful, and open-source document parser
fix: try to properly absorb table headers
Logan Markewichde8dad898ade
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rsindex 7db5b52..fad8caf 100644--- a/crates/liteparse/src/markdown_layout/classify.rs+++ b/crates/liteparse/src/markdown_layout/classify.rs@@ -568,6 +568,23 @@ fn classify_region(state.flush_code(&mut blocks);state.reset_list();let run = table_iter.next().unwrap();+ // Prose lines the run stepped over between its header and body+ // sit inside `[run.start, run.end)`, which this loop skips — emit+ // each as its own paragraph ahead of the table so the text isn't+ // dropped.+ for note in run.interstitials {+ if note.text.is_empty() {+ continue;+ }+ blocks.push(PositionedBlock::new(+ Block::Paragraph {+ text: note.text,+ bold: false,+ italic: false,+ },+ note.bbox,+ ));+ }blocks.push(PositionedBlock::new(run.block, run.bbox));idx = run.end;continue;diff --git a/crates/liteparse/src/markdown_layout/cross_region.rs b/crates/liteparse/src/markdown_layout/cross_region.rsindex ef08646..4ce24df 100644--- a/crates/liteparse/src/markdown_layout/cross_region.rs+++ b/crates/liteparse/src/markdown_layout/cross_region.rs@@ -458,6 +458,7 @@ fn try_two_col_direct(clusters: &[Cluster], merged_len: usize, tol: f32) -> Optibody_start: if header.is_some() { 1 } else { 0 },bbox,block: Block::Table { header, rows: body },+ interstitials: Vec::new(),}])}diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rsindex 8bf3fc3..ac7efe4 100644--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@ -272,6 +272,12 @@ pub(super) struct TableRun {/// re-extraction pass must not re-bin those header lines as body rows.pub(super) body_start: usize,pub(super) block: Block,+ /// Free-standing prose lines the run stepped over between its absorbed+ /// header and body rows (e.g. a note interleaved under the header). They+ /// are inside the run's `[start, end)` span, so the classifier never+ /// visits them — the consumer must emit each as its own paragraph ahead+ /// of the table or the text is silently dropped. Top-to-bottom order.+ pub(super) interstitials: Vec<Cell>,/// Region the whole table occupies. For ruled tables this is the drawn/// grid; for the borderless detectors it is the union of the lines the run/// consumed. `None` when the run was built without geometry (tests, and@@ -924,8 +930,10 @@ fn finalize_table_run(// 1 when the bold-first-row promotion consumes rows[0].let first_row = &rows[0].2;let bold_header_qualifies = absorbed.is_none() && bold_first_row_eligible;- let (run_start, header, row_start) = match absorbed {- Some((hstart, header_texts)) => (hstart, Some(header_texts), 0),+ let (run_start, header, row_start, interstitials) = match absorbed {+ Some((hstart, header_texts, interstitials)) => {+ (hstart, Some(header_texts), 0, interstitials)+ }None if bold_header_qualifies => (start_idx,Some(@@ -935,8 +943,9 @@ fn finalize_table_run(.collect(),),1,+ Vec::new(),),- None => (start_idx, None, 0),+ None => (start_idx, None, 0, Vec::new()),};let body_rows: Vec<Vec<Cell>> = rows[row_start..].iter()@@ -967,6 +976,7 @@ fn finalize_table_run(header,rows: body_rows,},+ interstitials,})}@@ -1419,6 +1429,7 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->body_start: start_idx,bbox: lines_bbox(lines, start_idx, end),block: Block::GridFallback { lines: raw },+ interstitials: Vec::new(),});}@@ -1448,12 +1459,20 @@ fn absorb_header_lines(track_ranges: &[(f32, f32)],column_count: usize,floor: usize,-) -> Option<(usize, Vec<Cell>)> {+) -> Option<(usize, Vec<Cell>, Vec<Cell>)> {let dbgt = *super::flags::DEBUG_TABLE;// Line index is kept alongside the cells so the assembled header cells can// report the band they were read from — a wrapped header spans every line// that contributed to it, not just the last one.let mut absorbed: Vec<(usize, Vec<TableCell>)> = Vec::new();+ // A single free-text line interleaved between the header and the first+ // body row (a note, a caption) used to end absorption here, leaving the+ // table headerless — the markdown emitter then promotes the first *data*+ // row into the header slot and that row disappears when parsed. Step over+ // at most one such line and keep walking; it only commits if a genuine+ // header (≥2 cells, all track-aligned) is found directly above, otherwise+ // absorption fails as before and the line stays outside the run.+ let mut skipped: Vec<usize> = Vec::new();let mut j = start_idx;while j > floor {let cand = j - 1;@@ -1475,7 +1494,13 @@ fn absorb_header_lines(// a merged-run header is a single line, whereas letting recovery// track-align line after line lets absorption climb an entire// paragraph, one plausible-looking row at a time.- if absorbed.is_empty() && cells.len() < column_count {+ // Also gated on no interstitial skip having happened: skip + recovery+ // compose into two speculative rescues, and together they manufacture+ // headers out of wrapped document titles (a title line above a+ // stepped-over caption shreds into plausible short cells and then+ // suppresses the legitimate bold-first-row promotion). A post-skip+ // header must split into multiple cells on its own.+ if absorbed.is_empty() && skipped.is_empty() && cells.len() < column_count {const HEADER_MAX_CELL_CHARS: usize = 30;let tracks: Vec<f32> = track_ranges.iter().map(|r| r.0).collect();if let Some(recovered) = recover_merged_cell(cells.clone(), &tracks)@@ -1500,6 +1525,24 @@ fn absorb_header_lines(// A header line must carry at least two cells (a single cell is a// title/caption, not a header) and sit tight above the row below it.if cells.len() < 2 {+ // Interstitial skip: a single-cell prose line sitting tight above+ // the body may have a real header directly above it. Restricted to+ // one line, before any absorption (the note sits against the body,+ // never inside a wrapped header band), and the line must be+ // adjacent to the row below it so absorption can't jump gaps.+ if absorbed.is_empty()+ && skipped.is_empty()+ && cells.len() == 1+ && cand > floor+ && table_rows_adjacent(&lines[cand], &lines[j])+ {+ if dbgt {+ eprintln!("[tbl-absorb skip @{cand}] single-cell interstitial, continuing up");+ }+ skipped.push(cand);+ j = cand;+ continue;+ }break;}if !table_rows_adjacent(&lines[cand], &lines[j]) {@@ -1520,6 +1563,15 @@ fn absorb_header_lines(if absorbed.is_empty() {return None;}+ // Skips only commit when a header was found above them — otherwise the+ // run keeps its old start and the stepped-over line is classified+ // normally. Each committed skip is handed back as owned text + box so the+ // consumer can emit it as a paragraph (the line index lands inside the+ // run span, where the classifier never looks).+ let interstitials: Vec<Cell> = skipped+ .iter()+ .map(|&idx| Cell::located(lines[idx].text.trim().to_string(), lines[idx].bbox.clone()))+ .collect();// Collected bottom-up; reverse so text reads top-to-bottom per column.absorbed.reverse();let mut header = vec![Cell::default(); column_count];@@ -1535,7 +1587,24 @@ fn absorb_header_lines(Rect::extend(&mut header[idx].bbox, &cell_rect(c, &lines[*line_idx]));}}- Some((j, header))+ // A header reached via a skip is held to a coverage bar: it must name at+ // least a third of the columns. A sparse group-label line (2 cells over an+ // 11-column body) absorbed across the wrapped remainder of its own band+ // would otherwise displace the fully-populated real header row below it+ // into the body. Failing the bar fails the whole absorption — the run+ // falls back to its pre-skip behavior.+ if !skipped.is_empty() {+ let nonempty = header.iter().filter(|c| !c.text.is_empty()).count();+ if nonempty * 3 < column_count {+ if dbgt {+ eprintln!(+ "[tbl-absorb reject] post-skip header names {nonempty}/{column_count} columns"+ );+ }+ return None;+ }+ }+ Some((j, header, interstitials))}/// Scan `lines` once and return all detected tabular regions (sorted by@@ -2141,6 +2210,7 @@ fn try_detect_description_list(lines: &[ProjectedLine], start_idx: usize) -> Optheader: None,rows: body,},+ interstitials: Vec::new(),})}@@ -2705,6 +2775,11 @@ fn build_union_table(start,end: window_end,body_start: window_start,+ // Fragments' own interstitials are not carried over: the union span+ // starts at the first fragment's body and `union_header_from_above`+ // stops at prose, so any stepped-over line lands outside the new span+ // and is classified normally — preserved as a paragraph, not dropped.+ interstitials: Vec::new(),bbox: lines_bbox(lines, start, window_end),block: Block::Table { header, rows },})@@ -2943,6 +3018,12 @@ fn try_merge_pair(a: &TableRun, b: &TableRun, lines: &[ProjectedLine]) -> Option// between them.bbox: lines_bbox(lines, a.start, b.end),block: Block::Table { header, rows },+ interstitials: a+ .interstitials+ .iter()+ .chain(&b.interstitials)+ .cloned()+ .collect(),});}@@ -3009,6 +3090,12 @@ fn try_merge_pair(a: &TableRun, b: &TableRun, lines: &[ProjectedLine]) -> Optionheader: Some(merged_header),rows: merged_rows,},+ interstitials: a+ .interstitials+ .iter()+ .chain(&b.interstitials)+ .cloned()+ .collect(),});}@@ -4525,6 +4612,7 @@ fn build_ruled_table_from(start,end,body_start: start,+ interstitials: Vec::new(),// A ruled table's region is the grid the generator actually drew.bbox: Some(bbox),block: Block::Table {@@ -5629,6 +5717,55 @@ mod tests {}}+ #[test]+ fn absorbs_header_over_interleaved_note_line() {+ // Issue #395: a free-text note between the header and the first body+ // row blocked header absorption, leaving the table headerless — the+ // pipe-table emitter then promotes the first DATA row into the header+ // slot and that row is lost to downstream markdown parsers. The walk+ // must step over one such line, absorb the real header, and hand the+ // note back as an interstitial paragraph.+ // Geometry mirrors the issue's PDF: the note sits a wider-than-row+ // gap below the header (so the forward walk can't wrap it into the+ // header row) but still within absorption adjacency, and tight above+ // the first body row.+ let lines = vec![+ line_with_spans(+ &[("Name", 50.0), ("Score", 150.0), ("Ref", 250.0)],+ 100.0,+ 10.0,+ ),+ line_with_spans(&[("Shipment was delayed in transit.", 80.0)], 122.0, 10.0),+ line_with_spans(&[("A", 50.0), ("1", 150.0), ("2", 250.0)], 136.0, 10.0),+ line_with_spans(&[("B", 50.0), ("3", 150.0), ("4", 250.0)], 150.0, 10.0),+ line_with_spans(&[("C", 50.0), ("5", 150.0), ("6", 250.0)], 164.0, 10.0),+ ];+ let runs = detect_tables(&lines);+ assert_eq!(runs.len(), 1);+ let run = &runs[0];+ assert_eq!(run.start, 0, "header must be absorbed across the note");+ assert_eq!(run.end, 5);+ assert_eq!(+ run.interstitials+ .iter()+ .map(|c| c.text.as_str())+ .collect::<Vec<_>>(),+ vec!["Shipment was delayed in transit."],+ "the stepped-over note must be preserved for paragraph emission"+ );+ match &run.block {+ Block::Table { header, rows } => {+ let header = header.as_ref().expect("header should be present");+ assert_eq!(+ header,+ &vec!["Name".to_string(), "Score".to_string(), "Ref".to_string()]+ );+ assert_eq!(rows.len(), 3, "no data row may be consumed as header");+ }+ other => panic!("expected Block::Table, got {other:?}"),+ }+ }+#[test]fn does_not_absorb_single_cell_title_above_body() {// A one-cell title/caption above a table is NOT a header row and must@@ -5903,6 +6040,7 @@ mod tests {#[test]fn merge_prefers_ruled_when_overlapping() {let ruled = vec![TableRun {+ interstitials: Vec::new(),start: 5,end: 10,body_start: 5,@@ -5913,6 +6051,7 @@ mod tests {},}];let borderless = vec![TableRun {+ interstitials: Vec::new(),start: 6,end: 11,body_start: 6,@@ -5965,6 +6104,7 @@ mod tests {three_col_line("b3", 70.0),];let a = TableRun {+ interstitials: Vec::new(),start: 0,end: 2,body_start: 0,@@ -5975,6 +6115,7 @@ mod tests {},};let b = TableRun {+ interstitials: Vec::new(),start: 2,end: 5,body_start: 2,@@ -6015,6 +6156,7 @@ mod tests {four_col_line("body", 70.0),];let a = TableRun {+ interstitials: Vec::new(),start: 0,end: 2,body_start: 0,@@ -6028,6 +6170,7 @@ mod tests {},};let b = TableRun {+ interstitials: Vec::new(),start: 2,end: 5,body_start: 2,@@ -6069,6 +6212,7 @@ mod tests {three_col_line("b2", 215.0),];let a = TableRun {+ interstitials: Vec::new(),start: 0,end: 2,body_start: 0,@@ -6079,6 +6223,7 @@ mod tests {},};let b = TableRun {+ interstitials: Vec::new(),start: 2,end: 4,body_start: 2,@@ -6104,6 +6249,7 @@ mod tests {.chain((0..3).map(|i| four_col_line("y", 160.0 + i as f32 * 15.0))).collect();let a = TableRun {+ interstitials: Vec::new(),start: 0,end: 10,body_start: 0,@@ -6116,6 +6262,7 @@ mod tests {},};let b = TableRun {+ interstitials: Vec::new(),start: 10,end: 13,body_start: 10,@@ -6162,6 +6309,7 @@ mod tests {),];let a = TableRun {+ interstitials: Vec::new(),start: 0,end: 2,body_start: 0,@@ -6175,6 +6323,7 @@ mod tests {},};let b = TableRun {+ interstitials: Vec::new(),start: 2,end: 4,body_start: 2,@@ -6199,6 +6348,7 @@ mod tests {three_col_line("c", 40.0),];let a = TableRun {+ interstitials: Vec::new(),start: 0,end: 2,body_start: 0,@@ -6212,6 +6362,7 @@ mod tests {},};let b = TableRun {+ interstitials: Vec::new(),start: 2,end: 3,body_start: 2,@@ -6245,6 +6396,7 @@ mod tests {three_col_line("b", 70.0),];let a = TableRun {+ interstitials: Vec::new(),start: 0,end: 2,body_start: 0,@@ -6255,6 +6407,7 @@ mod tests {},};let b = TableRun {+ interstitials: Vec::new(),start: 3,end: 5,body_start: 3,@@ -6285,6 +6438,7 @@ mod tests {four_col_line("body", 85.0),];let a = TableRun {+ interstitials: Vec::new(),start: 0,end: 2,body_start: 0,@@ -6298,6 +6452,7 @@ mod tests {},};let b = TableRun {+ interstitials: Vec::new(),start: 3,end: 6,body_start: 3,