A fast, helpful, and open-source document parser
clean up part 2
Logan Markewichb8d3654a4380
diff --git a/crates/liteparse/src/markdown_layout/blocks.rs b/crates/liteparse/src/markdown_layout/blocks.rsindex 2e289c7..3cc4931 100644--- a/crates/liteparse/src/markdown_layout/blocks.rs+++ b/crates/liteparse/src/markdown_layout/blocks.rs@@ -1,5 +1,5 @@use super::inline::escape_inline;-use super::paragraphs::ParaAccum;+use super::paragraphs::{ParaAccum, is_soft_hyphen_break};use super::tables::escape_table_cell;/// Coarse block representation: the output of page classification, consumed by@@ -113,15 +113,12 @@ pub fn render_blocks(blocks: &[Block]) -> String {italic: false,},) = (&blocks[i - 1], block)- && out.ends_with('-')- && out- .chars()- .rev()- .nth(1)- .is_some_and(|p| p.is_ascii_alphabetic())- && text.chars().next().is_some_and(|c| c.is_ascii_lowercase())+ && is_soft_hyphen_break(&out, text){- out.pop();+ while out.ends_with(|c: char| c.is_whitespace()) {+ out.pop();+ }+ out.pop(); // the soft hyphenout.push_str(text);continue;}diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rsindex 3977c25..d027f24 100644--- a/crates/liteparse/src/markdown_layout/classify.rs+++ b/crates/liteparse/src/markdown_layout/classify.rs@@ -13,6 +13,7 @@ use super::inline::{use super::lists::{LIST_INDENT_STEP_PT, parse_list_marker};use super::paragraphs::{ParaAccum, append_to_paragraph, collapse_whitespace, continues_heading, continues_paragraph,+ ends_hyphenated, is_soft_hyphen_break,};use super::repetition::is_header_or_footer;use super::tables::{detect_ruled_tables, detect_tables, merge_table_runs};@@ -452,7 +453,7 @@ fn classify_region(.iter().map(|run| {let top_line = &lines[run.start];- let row_h = top_line.bbox.height.max(8.0);+ let row_h = top_line.bbox.height.max(super::MIN_ROW_HEIGHT_PT);let top = top_line.bbox.y - row_h * TABLE_HR_SUPPRESS_HEADROOM_ROWS;let last = &lines[run.end.saturating_sub(1).max(run.start)];let bot = last.bbox.y + last.bbox.height;@@ -621,10 +622,7 @@ fn classify_region(// line is its continuation regardless of capitalization// ("…SOLAR 10.7 Billion-" / "Parameter Model: We have…").let prev_hyphen_wrap = !*super::flags::DISABLE_HEADING_GUARDS- && prev.is_some_and(|p| {- let t = p.text.trim_end();- t.ends_with('-') && t.chars().rev().nth(1).is_some_and(|c| c.is_alphanumeric())- });+ && prev.is_some_and(|p| ends_hyphenated(&p.text));!((starts_lower || prev_hyphen_wrap)&& prev.is_some_and(|p| continues_paragraph(p, line)))});@@ -974,16 +972,9 @@ fn stitch_regions(blocks: Vec<Block>, region_starts: &[usize]) -> Vec<Block> {.chars().next().is_some_and(|c| c.is_lowercase());- // Hyphen-splice path: bare `<letter>-` mid-word break across the+ // Hyphen-splice path: a mid-word soft hyphen break across the// leaf boundary. Drop the hyphen, join with no separator.- let hyphen_splice = prev_trim.ends_with('-')- && prev_trim- .chars()- .rev()- .nth(1)- .is_some_and(|c| c.is_ascii_alphabetic())- && starts_lower;- if hyphen_splice {+ if is_soft_hyphen_break(prev_text, cur_text) {// Pop the trailing `-` (it may have whitespace after it from// a previous trim, so re-trim).while prev_text.ends_with(|c: char| c.is_whitespace()) {diff --git a/crates/liteparse/src/markdown_layout/headings.rs b/crates/liteparse/src/markdown_layout/headings.rsindex 7fccdd7..964f013 100644--- a/crates/liteparse/src/markdown_layout/headings.rs+++ b/crates/liteparse/src/markdown_layout/headings.rs@@ -763,7 +763,7 @@ pub(super) fn outline_heading_level(if normalized_line.is_empty() {return None;}- let row_h = line.bbox.height.max(8.0);+ let row_h = line.bbox.height.max(super::MIN_ROW_HEIGHT_PT);let y_tolerance = row_h * 1.5;for entry in outline {let normalized_title = normalize_outline_text(&entry.title);diff --git a/crates/liteparse/src/markdown_layout/mod.rs b/crates/liteparse/src/markdown_layout/mod.rsindex 97c1ef4..5bbff91 100644--- a/crates/liteparse/src/markdown_layout/mod.rs+++ b/crates/liteparse/src/markdown_layout/mod.rs@@ -24,5 +24,12 @@ pub use headings::{build_heading_map, compute_body_size};pub use repetition::{compute_header_footer_set, detect_single_page_chrome};pub use tables::detect_table_rects;+/// Minimum plausible text-row height in points. Floors a `bbox.height` before+/// it is multiplied into a band / tolerance window, so a degenerate near-zero+/// glyph box can't collapse that window to nothing. Shared by the table+/// HR-suppress headroom (`classify.rs`) and outline y-tolerance+/// (`headings.rs`) so the two stay in sync.+const MIN_ROW_HEIGHT_PT: f32 = 8.0;+#[cfg(test)]pub(crate) mod test_helpers;diff --git a/crates/liteparse/src/markdown_layout/paragraphs.rs b/crates/liteparse/src/markdown_layout/paragraphs.rsindex b749806..768f048 100644--- a/crates/liteparse/src/markdown_layout/paragraphs.rs+++ b/crates/liteparse/src/markdown_layout/paragraphs.rs@@ -53,11 +53,40 @@ pub(super) fn collapse_whitespace(s: &str) -> String {out}+/// True when `text` ends with a mid-word soft hyphen — a trailing `-` whose+/// preceding character is a letter. Distinguishes a wrapped word (`architec-`)+/// from a dangling list marker (`-`) or a numeric range (`5-`), neither of+/// which should be rejoined. Unicode-aware, so accented stems (`café-`)+/// qualify. Trims trailing whitespace first.+///+/// Single source of truth for the "ends open on a hyphen" test; the heading+/// continuation guard, region stitching, and block joining all key off this.+/// (Note: `stitch_regions` has an unrelated local `ends_open` that tests+/// sentence-terminal punctuation — different concept, hence the distinct name.)+pub(super) fn ends_hyphenated(text: &str) -> bool {+ let t = text.trim_end();+ t.ends_with('-') && t.chars().rev().nth(1).is_some_and(|c| c.is_alphabetic())+}++/// True when `prev` ends with a soft hyphen (see [`ends_hyphenated`]) and+/// `next` begins with a lowercase letter — the signal that a single word was+/// split across a line / block / region break and should be rejoined with the+/// hyphen dropped and no separator (`architec-` + `ture` → `architecture`). A+/// capitalized continuation (`well-` + `Known`) is left as a real compound.+pub(super) fn is_soft_hyphen_break(prev: &str, next: &str) -> bool {+ ends_hyphenated(prev)+ && next+ .trim_start()+ .chars()+ .next()+ .is_some_and(|c| c.is_lowercase())+}+/// Append `to_append` onto `prev`, de-hyphenating across the boundary. When-/// `prev` ends with `-` and `check` (the plain text of the continuation) starts-/// with an ASCII lowercase letter, the trailing hyphen is dropped and the text-/// concatenated directly (`co-` + `operate` → `cooperate`); otherwise the join-/// is a single space.+/// the boundary is a soft hyphen break (see [`is_soft_hyphen_break`], tested+/// against `check` — the plain text of the continuation), the trailing hyphen+/// is dropped and the text concatenated directly (`co-` + `operate` →+/// `cooperate`); otherwise the join is a single space.////// `check` and `to_append` are separate so a caller tracking a styled `inline`/// representation can test the condition against the raw text while appending@@ -70,9 +99,12 @@ pub(super) fn dehyphenate_join(prev: &mut String, check: &str, to_append: &str)prev.push_str(to_append);return;}- if prev.ends_with('-') && check.chars().next().is_some_and(|c| c.is_ascii_lowercase()) {- prev.pop();- prev.push_str(to_append);+ if is_soft_hyphen_break(prev, check) {+ while prev.ends_with(|c: char| c.is_whitespace()) {+ prev.pop();+ }+ prev.pop(); // the soft hyphen+ prev.push_str(to_append.trim_start());} else {prev.push(' ');prev.push_str(to_append);diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rsindex 784a343..a3e76ac 100644--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@ -18,6 +18,19 @@ const TABLE_CELL_GAP_FONT_MULTIPLIER: f32 = 1.0;/// track when extending a candidate table run.const TABLE_TRACK_TOLERANCE_PT: f32 = 6.0;+/// Minimum gap (in multiples of the dominant font size) required between two+/// adjacent column tracks for them to count as distinct columns. The two+/// callers — inferred-track detection and the ruled-grid track derivation —+/// must use the same value or one stage can fuse columns the other split.+const TABLE_MIN_TRACK_GAP_FONT_MULT: f32 = 1.5;+/// Absolute floor (points) on the inter-track gap, paired with+/// `TABLE_MIN_TRACK_GAP_FONT_MULT` so tiny-font pages keep a sane minimum.+const TABLE_MIN_TRACK_GAP_FLOOR_PT: f32 = 12.0;++/// Minimum fraction of a row's cells that must carry text for that row to+/// anchor the header/body split in ruled-grid collapse.+const TABLE_ROW_MIN_FILL: f32 = 0.9;+/// Floor for the sparse-new-row path: a partial-cell line whose bottom-gap/// exceeds this fraction qualifies as a real new row (with empty cells at/// missing tracks) instead of being treated as a wrap continuation. Below@@ -671,7 +684,8 @@ fn try_detect_table_inferred(} else {lines[start_idx].bbox.height.max(1.0)};- let min_track_gap = (font_size * 1.5).max(12.0);+ let min_track_gap =+ (font_size * TABLE_MIN_TRACK_GAP_FONT_MULT).max(TABLE_MIN_TRACK_GAP_FLOOR_PT);let min_gap = tracks.windows(2).map(|w| w[1] - w[0])@@ -858,9 +872,17 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->let line_height = prev_line.bbox.height.max(lines[j].bbox.height).max(1.0);let centroid_dy = lines[j].bbox.y - prev_y_top;let bottom_gap = lines[j].bbox.y - prev_bottom;- let all_align_track = cells+ // Map each cell to its column track once. `match_track_idx`+ // returns `Some` exactly when `cell_aligns_track` would return+ // `true`, so `mapping.len() == cells.len()` is the same+ // "every cell aligns to a track" predicate — and the mapping is+ // already in hand for the paths below, no separate unwrap whose+ // safety depends on the two functions staying in lockstep.+ let mapping: Vec<usize> = cells.iter()- .all(|c| track_ranges.iter().any(|r| cell_aligns_track(c, *r)));+ .filter_map(|c| match_track_idx(c, &track_ranges))+ .collect();+ let all_align_track = mapping.len() == cells.len();// Sparse-new-row path runs FIRST. When the line sits a clear// inter-row gap below the previous row AND its cells map to// distinct tracks, treat it as a new row with empty cells at@@ -871,10 +893,6 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->&& cells.len() >= 2&& bottom_gap >= line_height * TABLE_SPARSE_ROW_MIN_BOTTOM_GAP_FRAC{- let mapping: Vec<usize> = cells- .iter()- .map(|c| match_track_idx(c, &track_ranges).unwrap())- .collect();let mut distinct = mapping.clone();distinct.sort_unstable();distinct.dedup();@@ -899,13 +917,11 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->// row, multi-line cell continuation.if centroid_dy <= line_height * 1.5 && all_align_track {let prev_cells = &mut rows.last_mut().unwrap().2;- for c in &cells {- if let Some(idx) = match_track_idx(c, &track_ranges) {- if !prev_cells[idx].text.is_empty() && !c.text.is_empty() {- prev_cells[idx].text.push(' ');- }- prev_cells[idx].text.push_str(&c.text);+ for (c, &idx) in cells.iter().zip(&mapping) {+ if !prev_cells[idx].text.is_empty() && !c.text.is_empty() {+ prev_cells[idx].text.push(' ');}+ prev_cells[idx].text.push_str(&c.text);}j += 1;continue;@@ -1896,7 +1912,8 @@ fn build_union_table(.collect();font_sizes.sort_by(f32::total_cmp);let median_font = font_sizes[font_sizes.len() / 2];- let min_track_gap = (median_font * 1.5).max(12.0);+ let min_track_gap =+ (median_font * TABLE_MIN_TRACK_GAP_FONT_MULT).max(TABLE_MIN_TRACK_GAP_FLOOR_PT);let mut coalesced: Vec<(f32, usize)> = Vec::with_capacity(supported.len());for (t, n) in supported {match coalesced.last_mut() {@@ -2983,48 +3000,50 @@ fn build_ruled_table(let flattened_header: Option<(Vec<String>, usize)> = {let row_fill =|r: usize| cell_has_text[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;- (0..n_rows).find(|r| row_fill(*r) >= 0.9).and_then(|b| {- let nonempty = cell_has_text[b].iter().filter(|t| **t).count();- let alpha_cells = (0..n_cols)- .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))- .count();- // The bottom header layer may carry digit labels (years like- // "2024") but never measurement values. A decimal / % / $ /- // dash-placeholder / comma-grouped number in the anchor row means- // it's the first DATA row of a table whose real header just- // missed the 0.9-fill anchor (colspan header covering <90% of- // columns) — folding it would eat a data row (DS5795A_page4).- let has_value_cell =- (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));- let qualifies = (1..=3).contains(&b)- && b + 1 < n_rows- && (0..b).any(|r| row_alpha_spanner[r])- && alpha_cells * 2 >= nonempty- && !has_value_cell;- if !qualifies {- return None;- }- let header: Vec<String> = (0..n_cols)- .map(|c| {- let mut parts: Vec<&str> = Vec::new();- for row in cells_repl.iter().take(b + 1) {- let s = row[c].as_str();- if s.is_empty() || parts.last() == Some(&s) {- continue;+ (0..n_rows)+ .find(|r| row_fill(*r) >= TABLE_ROW_MIN_FILL)+ .and_then(|b| {+ let nonempty = cell_has_text[b].iter().filter(|t| **t).count();+ let alpha_cells = (0..n_cols)+ .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))+ .count();+ // The bottom header layer may carry digit labels (years like+ // "2024") but never measurement values. A decimal / % / $ /+ // dash-placeholder / comma-grouped number in the anchor row means+ // it's the first DATA row of a table whose real header just+ // missed the 0.9-fill anchor (colspan header covering <90% of+ // columns) — folding it would eat a data row (DS5795A_page4).+ let has_value_cell =+ (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));+ let qualifies = (1..=3).contains(&b)+ && b + 1 < n_rows+ && (0..b).any(|r| row_alpha_spanner[r])+ && alpha_cells * 2 >= nonempty+ && !has_value_cell;+ if !qualifies {+ return None;+ }+ let header: Vec<String> = (0..n_cols)+ .map(|c| {+ let mut parts: Vec<&str> = Vec::new();+ for row in cells_repl.iter().take(b + 1) {+ let s = row[c].as_str();+ if s.is_empty() || parts.last() == Some(&s) {+ continue;+ }+ parts.push(s);}- parts.push(s);- }- parts.join(" ")- })- .collect();- if header.iter().all(|h| h.is_empty()) {- return None;- }- if dbg {- eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");- }- Some((header, b + 1))- })+ parts.join(" ")+ })+ .collect();+ if header.iter().all(|h| h.is_empty()) {+ return None;+ }+ if dbg {+ eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");+ }+ Some((header, b + 1))+ })};// Stacked-header merge (Mode 5): some generators rule every text baseline@@ -3039,7 +3058,7 @@ fn build_ruled_table(};// Anchor on the first fully-dense row (the first real data row).let k = (0..n_rows)- .find(|r| row_fill(*r, &cell_has_text) >= 0.9)+ .find(|r| row_fill(*r, &cell_has_text) >= TABLE_ROW_MIN_FILL).unwrap_or(0);let union_cols = (0..n_cols).filter(|c| (0..k).any(|r| cell_has_text[r][*c]))