Skip to content

A fast, helpful, and open-source document parser

clean up part 2

Logan Markewichb8d3654a4380
diff --git a/crates/liteparse/src/markdown_layout/blocks.rs b/crates/liteparse/src/markdown_layout/blocks.rs
index 2e289c7..3cc4931 100644
--- a/crates/liteparse/src/markdown_layout/blocks.rs
+++ b/crates/liteparse/src/markdown_layout/blocks.rs
@@ -1,5 +1,5 @@
use super::inline::escape_inline;
-use super::paragraphs::ParaAccum;
+use super::paragraphs::{ParaAccum, is_soft_hyphen_break};
use super::tables::escape_table_cell;
/// Coarse block representation: the output of page classification, consumed by
@@ -113,15 +113,12 @@ pub fn render_blocks(blocks: &[Block]) -> String {
italic: false,
},
) = (&blocks[i - 1], block)
- && out.ends_with('-')
- && out
- .chars()
- .rev()
- .nth(1)
- .is_some_and(|p| p.is_ascii_alphabetic())
- && text.chars().next().is_some_and(|c| c.is_ascii_lowercase())
+ && is_soft_hyphen_break(&out, text)
{
- out.pop();
+ while out.ends_with(|c: char| c.is_whitespace()) {
+ out.pop();
+ }
+ out.pop(); // the soft hyphen
out.push_str(text);
continue;
}
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rs
index 3977c25..d027f24 100644
--- a/crates/liteparse/src/markdown_layout/classify.rs
+++ b/crates/liteparse/src/markdown_layout/classify.rs
@@ -13,6 +13,7 @@ use super::inline::{
use super::lists::{LIST_INDENT_STEP_PT, parse_list_marker};
use super::paragraphs::{
ParaAccum, append_to_paragraph, collapse_whitespace, continues_heading, continues_paragraph,
+ ends_hyphenated, is_soft_hyphen_break,
};
use super::repetition::is_header_or_footer;
use super::tables::{detect_ruled_tables, detect_tables, merge_table_runs};
@@ -452,7 +453,7 @@ fn classify_region(
.iter()
.map(|run| {
let top_line = &lines[run.start];
- let row_h = top_line.bbox.height.max(8.0);
+ let row_h = top_line.bbox.height.max(super::MIN_ROW_HEIGHT_PT);
let top = top_line.bbox.y - row_h * TABLE_HR_SUPPRESS_HEADROOM_ROWS;
let last = &lines[run.end.saturating_sub(1).max(run.start)];
let bot = last.bbox.y + last.bbox.height;
@@ -621,10 +622,7 @@ fn classify_region(
// line is its continuation regardless of capitalization
// ("…SOLAR 10.7 Billion-" / "Parameter Model: We have…").
let prev_hyphen_wrap = !*super::flags::DISABLE_HEADING_GUARDS
- && prev.is_some_and(|p| {
- let t = p.text.trim_end();
- t.ends_with('-') && t.chars().rev().nth(1).is_some_and(|c| c.is_alphanumeric())
- });
+ && prev.is_some_and(|p| ends_hyphenated(&p.text));
!((starts_lower || prev_hyphen_wrap)
&& prev.is_some_and(|p| continues_paragraph(p, line)))
});
@@ -974,16 +972,9 @@ fn stitch_regions(blocks: Vec<Block>, region_starts: &[usize]) -> Vec<Block> {
.chars()
.next()
.is_some_and(|c| c.is_lowercase());
- // Hyphen-splice path: bare `<letter>-` mid-word break across the
+ // Hyphen-splice path: a mid-word soft hyphen break across the
// leaf boundary. Drop the hyphen, join with no separator.
- let hyphen_splice = prev_trim.ends_with('-')
- && prev_trim
- .chars()
- .rev()
- .nth(1)
- .is_some_and(|c| c.is_ascii_alphabetic())
- && starts_lower;
- if hyphen_splice {
+ if is_soft_hyphen_break(prev_text, cur_text) {
// Pop the trailing `-` (it may have whitespace after it from
// a previous trim, so re-trim).
while prev_text.ends_with(|c: char| c.is_whitespace()) {
diff --git a/crates/liteparse/src/markdown_layout/headings.rs b/crates/liteparse/src/markdown_layout/headings.rs
index 7fccdd7..964f013 100644
--- a/crates/liteparse/src/markdown_layout/headings.rs
+++ b/crates/liteparse/src/markdown_layout/headings.rs
@@ -763,7 +763,7 @@ pub(super) fn outline_heading_level(
if normalized_line.is_empty() {
return None;
}
- let row_h = line.bbox.height.max(8.0);
+ let row_h = line.bbox.height.max(super::MIN_ROW_HEIGHT_PT);
let y_tolerance = row_h * 1.5;
for entry in outline {
let normalized_title = normalize_outline_text(&entry.title);
diff --git a/crates/liteparse/src/markdown_layout/mod.rs b/crates/liteparse/src/markdown_layout/mod.rs
index 97c1ef4..5bbff91 100644
--- a/crates/liteparse/src/markdown_layout/mod.rs
+++ b/crates/liteparse/src/markdown_layout/mod.rs
@@ -24,5 +24,12 @@ pub use headings::{build_heading_map, compute_body_size};
pub use repetition::{compute_header_footer_set, detect_single_page_chrome};
pub use tables::detect_table_rects;
+/// Minimum plausible text-row height in points. Floors a `bbox.height` before
+/// it is multiplied into a band / tolerance window, so a degenerate near-zero
+/// glyph box can't collapse that window to nothing. Shared by the table
+/// HR-suppress headroom (`classify.rs`) and outline y-tolerance
+/// (`headings.rs`) so the two stay in sync.
+const MIN_ROW_HEIGHT_PT: f32 = 8.0;
+
#[cfg(test)]
pub(crate) mod test_helpers;
diff --git a/crates/liteparse/src/markdown_layout/paragraphs.rs b/crates/liteparse/src/markdown_layout/paragraphs.rs
index b749806..768f048 100644
--- a/crates/liteparse/src/markdown_layout/paragraphs.rs
+++ b/crates/liteparse/src/markdown_layout/paragraphs.rs
@@ -53,11 +53,40 @@ pub(super) fn collapse_whitespace(s: &str) -> String {
out
}
+/// True when `text` ends with a mid-word soft hyphen — a trailing `-` whose
+/// preceding character is a letter. Distinguishes a wrapped word (`architec-`)
+/// from a dangling list marker (`-`) or a numeric range (`5-`), neither of
+/// which should be rejoined. Unicode-aware, so accented stems (`café-`)
+/// qualify. Trims trailing whitespace first.
+///
+/// Single source of truth for the "ends open on a hyphen" test; the heading
+/// continuation guard, region stitching, and block joining all key off this.
+/// (Note: `stitch_regions` has an unrelated local `ends_open` that tests
+/// sentence-terminal punctuation — different concept, hence the distinct name.)
+pub(super) fn ends_hyphenated(text: &str) -> bool {
+ let t = text.trim_end();
+ t.ends_with('-') && t.chars().rev().nth(1).is_some_and(|c| c.is_alphabetic())
+}
+
+/// True when `prev` ends with a soft hyphen (see [`ends_hyphenated`]) and
+/// `next` begins with a lowercase letter — the signal that a single word was
+/// split across a line / block / region break and should be rejoined with the
+/// hyphen dropped and no separator (`architec-` + `ture` → `architecture`). A
+/// capitalized continuation (`well-` + `Known`) is left as a real compound.
+pub(super) fn is_soft_hyphen_break(prev: &str, next: &str) -> bool {
+ ends_hyphenated(prev)
+ && next
+ .trim_start()
+ .chars()
+ .next()
+ .is_some_and(|c| c.is_lowercase())
+}
+
/// Append `to_append` onto `prev`, de-hyphenating across the boundary. When
-/// `prev` ends with `-` and `check` (the plain text of the continuation) starts
-/// with an ASCII lowercase letter, the trailing hyphen is dropped and the text
-/// concatenated directly (`co-` + `operate` → `cooperate`); otherwise the join
-/// is a single space.
+/// the boundary is a soft hyphen break (see [`is_soft_hyphen_break`], tested
+/// against `check` — the plain text of the continuation), the trailing hyphen
+/// is dropped and the text concatenated directly (`co-` + `operate` →
+/// `cooperate`); otherwise the join is a single space.
///
/// `check` and `to_append` are separate so a caller tracking a styled `inline`
/// representation can test the condition against the raw text while appending
@@ -70,9 +99,12 @@ pub(super) fn dehyphenate_join(prev: &mut String, check: &str, to_append: &str)
prev.push_str(to_append);
return;
}
- if prev.ends_with('-') && check.chars().next().is_some_and(|c| c.is_ascii_lowercase()) {
- prev.pop();
- prev.push_str(to_append);
+ if is_soft_hyphen_break(prev, check) {
+ while prev.ends_with(|c: char| c.is_whitespace()) {
+ prev.pop();
+ }
+ prev.pop(); // the soft hyphen
+ prev.push_str(to_append.trim_start());
} else {
prev.push(' ');
prev.push_str(to_append);
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs
index 784a343..a3e76ac 100644
--- a/crates/liteparse/src/markdown_layout/tables.rs
+++ b/crates/liteparse/src/markdown_layout/tables.rs
@@ -18,6 +18,19 @@ const TABLE_CELL_GAP_FONT_MULTIPLIER: f32 = 1.0;
/// track when extending a candidate table run.
const TABLE_TRACK_TOLERANCE_PT: f32 = 6.0;
+/// Minimum gap (in multiples of the dominant font size) required between two
+/// adjacent column tracks for them to count as distinct columns. The two
+/// callers — inferred-track detection and the ruled-grid track derivation —
+/// must use the same value or one stage can fuse columns the other split.
+const TABLE_MIN_TRACK_GAP_FONT_MULT: f32 = 1.5;
+/// Absolute floor (points) on the inter-track gap, paired with
+/// `TABLE_MIN_TRACK_GAP_FONT_MULT` so tiny-font pages keep a sane minimum.
+const TABLE_MIN_TRACK_GAP_FLOOR_PT: f32 = 12.0;
+
+/// Minimum fraction of a row's cells that must carry text for that row to
+/// anchor the header/body split in ruled-grid collapse.
+const TABLE_ROW_MIN_FILL: f32 = 0.9;
+
/// Floor for the sparse-new-row path: a partial-cell line whose bottom-gap
/// exceeds this fraction qualifies as a real new row (with empty cells at
/// missing tracks) instead of being treated as a wrap continuation. Below
@@ -671,7 +684,8 @@ fn try_detect_table_inferred(
} else {
lines[start_idx].bbox.height.max(1.0)
};
- let min_track_gap = (font_size * 1.5).max(12.0);
+ let min_track_gap =
+ (font_size * TABLE_MIN_TRACK_GAP_FONT_MULT).max(TABLE_MIN_TRACK_GAP_FLOOR_PT);
let min_gap = tracks
.windows(2)
.map(|w| w[1] - w[0])
@@ -858,9 +872,17 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->
let line_height = prev_line.bbox.height.max(lines[j].bbox.height).max(1.0);
let centroid_dy = lines[j].bbox.y - prev_y_top;
let bottom_gap = lines[j].bbox.y - prev_bottom;
- let all_align_track = cells
+ // Map each cell to its column track once. `match_track_idx`
+ // returns `Some` exactly when `cell_aligns_track` would return
+ // `true`, so `mapping.len() == cells.len()` is the same
+ // "every cell aligns to a track" predicate — and the mapping is
+ // already in hand for the paths below, no separate unwrap whose
+ // safety depends on the two functions staying in lockstep.
+ let mapping: Vec<usize> = cells
.iter()
- .all(|c| track_ranges.iter().any(|r| cell_aligns_track(c, *r)));
+ .filter_map(|c| match_track_idx(c, &track_ranges))
+ .collect();
+ let all_align_track = mapping.len() == cells.len();
// Sparse-new-row path runs FIRST. When the line sits a clear
// inter-row gap below the previous row AND its cells map to
// distinct tracks, treat it as a new row with empty cells at
@@ -871,10 +893,6 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->
&& cells.len() >= 2
&& bottom_gap >= line_height * TABLE_SPARSE_ROW_MIN_BOTTOM_GAP_FRAC
{
- let mapping: Vec<usize> = cells
- .iter()
- .map(|c| match_track_idx(c, &track_ranges).unwrap())
- .collect();
let mut distinct = mapping.clone();
distinct.sort_unstable();
distinct.dedup();
@@ -899,13 +917,11 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->
// row, multi-line cell continuation.
if centroid_dy <= line_height * 1.5 && all_align_track {
let prev_cells = &mut rows.last_mut().unwrap().2;
- for c in &cells {
- if let Some(idx) = match_track_idx(c, &track_ranges) {
- if !prev_cells[idx].text.is_empty() && !c.text.is_empty() {
- prev_cells[idx].text.push(' ');
- }
- prev_cells[idx].text.push_str(&c.text);
+ for (c, &idx) in cells.iter().zip(&mapping) {
+ if !prev_cells[idx].text.is_empty() && !c.text.is_empty() {
+ prev_cells[idx].text.push(' ');
}
+ prev_cells[idx].text.push_str(&c.text);
}
j += 1;
continue;
@@ -1896,7 +1912,8 @@ fn build_union_table(
.collect();
font_sizes.sort_by(f32::total_cmp);
let median_font = font_sizes[font_sizes.len() / 2];
- let min_track_gap = (median_font * 1.5).max(12.0);
+ let min_track_gap =
+ (median_font * TABLE_MIN_TRACK_GAP_FONT_MULT).max(TABLE_MIN_TRACK_GAP_FLOOR_PT);
let mut coalesced: Vec<(f32, usize)> = Vec::with_capacity(supported.len());
for (t, n) in supported {
match coalesced.last_mut() {
@@ -2983,48 +3000,50 @@ fn build_ruled_table(
let flattened_header: Option<(Vec<String>, usize)> = {
let row_fill =
|r: usize| cell_has_text[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;
- (0..n_rows).find(|r| row_fill(*r) >= 0.9).and_then(|b| {
- let nonempty = cell_has_text[b].iter().filter(|t| **t).count();
- let alpha_cells = (0..n_cols)
- .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))
- .count();
- // The bottom header layer may carry digit labels (years like
- // "2024") but never measurement values. A decimal / % / $ /
- // dash-placeholder / comma-grouped number in the anchor row means
- // it's the first DATA row of a table whose real header just
- // missed the 0.9-fill anchor (colspan header covering <90% of
- // columns) — folding it would eat a data row (DS5795A_page4).
- let has_value_cell =
- (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));
- let qualifies = (1..=3).contains(&b)
- && b + 1 < n_rows
- && (0..b).any(|r| row_alpha_spanner[r])
- && alpha_cells * 2 >= nonempty
- && !has_value_cell;
- if !qualifies {
- return None;
- }
- let header: Vec<String> = (0..n_cols)
- .map(|c| {
- let mut parts: Vec<&str> = Vec::new();
- for row in cells_repl.iter().take(b + 1) {
- let s = row[c].as_str();
- if s.is_empty() || parts.last() == Some(&s) {
- continue;
+ (0..n_rows)
+ .find(|r| row_fill(*r) >= TABLE_ROW_MIN_FILL)
+ .and_then(|b| {
+ let nonempty = cell_has_text[b].iter().filter(|t| **t).count();
+ let alpha_cells = (0..n_cols)
+ .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))
+ .count();
+ // The bottom header layer may carry digit labels (years like
+ // "2024") but never measurement values. A decimal / % / $ /
+ // dash-placeholder / comma-grouped number in the anchor row means
+ // it's the first DATA row of a table whose real header just
+ // missed the 0.9-fill anchor (colspan header covering <90% of
+ // columns) — folding it would eat a data row (DS5795A_page4).
+ let has_value_cell =
+ (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));
+ let qualifies = (1..=3).contains(&b)
+ && b + 1 < n_rows
+ && (0..b).any(|r| row_alpha_spanner[r])
+ && alpha_cells * 2 >= nonempty
+ && !has_value_cell;
+ if !qualifies {
+ return None;
+ }
+ let header: Vec<String> = (0..n_cols)
+ .map(|c| {
+ let mut parts: Vec<&str> = Vec::new();
+ for row in cells_repl.iter().take(b + 1) {
+ let s = row[c].as_str();
+ if s.is_empty() || parts.last() == Some(&s) {
+ continue;
+ }
+ parts.push(s);
}
- parts.push(s);
- }
- parts.join(" ")
- })
- .collect();
- if header.iter().all(|h| h.is_empty()) {
- return None;
- }
- if dbg {
- eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");
- }
- Some((header, b + 1))
- })
+ parts.join(" ")
+ })
+ .collect();
+ if header.iter().all(|h| h.is_empty()) {
+ return None;
+ }
+ if dbg {
+ eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");
+ }
+ Some((header, b + 1))
+ })
};
// Stacked-header merge (Mode 5): some generators rule every text baseline
@@ -3039,7 +3058,7 @@ fn build_ruled_table(
};
// Anchor on the first fully-dense row (the first real data row).
let k = (0..n_rows)
- .find(|r| row_fill(*r, &cell_has_text) >= 0.9)
+ .find(|r| row_fill(*r, &cell_has_text) >= TABLE_ROW_MIN_FILL)
.unwrap_or(0);
let union_cols = (0..n_cols)
.filter(|c| (0..k).any(|r| cell_has_text[r][*c]))