Skip to content

A fast, helpful, and open-source document parser

wip

Logan Markewich65bf40b1f624
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rs
index ad5dc37..b0f5ee2 100644
--- a/crates/liteparse/src/markdown_layout/classify.rs
+++ b/crates/liteparse/src/markdown_layout/classify.rs
@@ -78,7 +78,50 @@ pub fn classify_page_with_filters(
// own title ("Contents", "Table of Contents") is shorter without a
// trailing number — it falls through the size/bold heuristics with no
// special handling needed.
- let toc_page = page_is_toc(page);
+ // Fallback TOC detection: when projection truncates the trailing page
+ // numbers on TOC entries (common when entries use dot-leaders that the
+ // projection eats), `page_is_toc` misses. If a `Contents` / `Table of
+ // Contents` title is the first text on the page, treat the rest as TOC
+ // entries even without page-number tails. Guard: don't fire if the page
+ // has a substantial body paragraph (≥3 lines of ≥80 chars each) — that's
+ // a real content page that just happens to mention "contents" near the
+ // top.
+ let toc_page = page_is_toc(page) || {
+ let mut saw_title = false;
+ let mut long_lines = 0usize;
+ for line in &page.projected_lines {
+ if is_rotated_line(line) {
+ continue;
+ }
+ let t = line.text.trim();
+ if t.is_empty() {
+ continue;
+ }
+ if !saw_title {
+ if is_toc_title(t) {
+ saw_title = true;
+ continue;
+ }
+ if t.chars().count() > 40 {
+ break;
+ }
+ } else if t.chars().count() >= 80 {
+ // Dot-leader fragments ("`. . . . . . . .`") are part of
+ // TOC layout, not body paragraphs — skip them.
+ let alpha = t.chars().filter(|c| c.is_alphabetic()).count();
+ let alpha_ratio = alpha as f32 / t.chars().count() as f32;
+ if alpha_ratio < 0.3 {
+ continue;
+ }
+ long_lines += 1;
+ if long_lines >= 3 {
+ saw_title = false;
+ break;
+ }
+ }
+ }
+ saw_title
+ };
// Strip running header/footer lines up-front so they don't leak into
// table detection (a repeating two-column footer would otherwise look
@@ -246,6 +289,11 @@ fn classify_region(
let mut last_list_item_idx: Option<usize> = None;
let mut last_list_line: Option<usize> = None;
let mut heading_run: Option<(u8, usize)> = None;
+ // Track whether we've already emitted a TOC title on this page. Once seen,
+ // any subsequent line matching `is_toc_title` (e.g. "Index", "List of
+ // Figures") is a TOC entry pointing at that chapter, not another title,
+ // and must stay suppressed.
+ let mut toc_title_emitted = false;
let flush_paragraph = |blocks: &mut Vec<Block>, p: Option<ParaAccum>| {
if let Some(acc) = p
@@ -408,13 +456,23 @@ fn classify_region(
// Priority chain: tagged-PDF struct tree → outline → font-size map.
let tagged_level = struct_heading_level(line, &page.struct_nodes);
- let outline_level =
- tagged_level.or_else(|| outline_heading_level(line, page.page_height, outline, text));
// Caption lines ("Figure 7", "Table 3.") are routinely set in a
// distinct (and slightly larger) font that lands them in the
// font-size heading map. Suppress font-size promotion for them;
// outline / struct-tree signals still win since those are explicit.
- let toc_suppress = toc_page && !is_toc_title(text);
+ let is_first_toc_title = is_toc_title(text) && !toc_title_emitted;
+ let toc_suppress = toc_page && !is_first_toc_title;
+ // Outline entries on a TOC page are the TOC itself — every entry
+ // prefix-matches an outline target ("Introduction", "Part I", ...).
+ // Promoting them to `##` shreds the GT MHS structure (which has just
+ // `# Contents`). Tagged-PDF struct levels are explicit and still win.
+ let outline_level = tagged_level.or_else(|| {
+ if toc_suppress {
+ None
+ } else {
+ outline_heading_level(line, page.page_height, outline, text)
+ }
+ });
let size_level = if is_caption_line(text) || toc_suppress {
None
} else {
@@ -443,17 +501,79 @@ fn classify_region(
// can run 100+ chars — but a 200-char citation is unambiguously
// not a heading.
let size_level = size_level.filter(|_| text.chars().count() <= HEADING_MAX_TEXT_CHARS);
+ // Attribution lines under figures/charts are never section headings,
+ // even when the chart caption font is slightly above body size.
+ let size_level =
+ size_level.filter(|_| !crate::markdown_layout::headings::is_attribution_line(text));
+ // Run-in label guard: a size-promoted line that contains a mid-line
+ // ". " followed by ≥3 more words AND ends with a mid-word "-" wrap
+ // is almost always a paragraph leading with a bold run-in label
+ // ("Base model. Any n-layer transformer architec-"), not a section
+ // heading. Both signals must fire — a heading occasionally has either
+ // on its own (e.g. abbreviation periods, intentional hyphen).
+ let size_level = size_level.filter(|_| {
+ let has_mid_sentence = text.contains(". ");
+ let mid_word_wrap = text.trim_end().ends_with('-');
+ !(has_mid_sentence && mid_word_wrap)
+ });
// A standalone "Contents" / "Table of Contents" / "Index" line is
// almost always a real H1, even when `page_is_toc` is false — many
// TOCs list entries without inline trailing page numbers, so the
// page-level detector misses them. `is_toc_title` matches the *entire*
// trimmed line against an exact whitelist, so it can't fire mid-prose.
- let toc_title_level = if is_toc_title(text) { Some(1u8) } else { None };
+ let toc_title_level = if is_first_toc_title { Some(1u8) } else { None };
+ if is_first_toc_title {
+ toc_title_emitted = true;
+ }
let level = outline_level
.or(size_level)
.or(toc_title_level)
.map(|l| l.clamp(1, MAX_HEADING_LEVELS as u8));
let mut demoted_heading = false;
+ // Unconditional wrap-continuation merge: when the previous block is a
+ // live heading_run and this line continues it (same font, same region,
+ // tight gap), merge regardless of whether this line itself qualifies
+ // as a heading. Catches body-size wrap continuations of bold headings
+ // ("Cellular Cycle\nand Replication") where line 2 has the same
+ // sub-body size and would otherwise emit as a stray bold paragraph.
+ // Gate the unconditional merge: only fold body-sized continuations that
+ // *look* like the rest of a heading, not body prose. `continues_heading`
+ // only enforces structural agreement (font/bold/region/gap), so a body
+ // paragraph that happens to share those traits (e.g. a heading followed
+ // by an all-bold body paragraph) would otherwise be absorbed.
+ // Reject the candidate if it starts a list item (`l.`, `1.`, `· `…) or
+ // contains a mid-sentence period — both are unambiguous prose signals
+ // and never appear inside a real wrapped heading.
+ let cont_looks_like_heading = parse_list_marker(text).is_none() && !text.contains(". ");
+ if level.is_none()
+ && cont_looks_like_heading
+ && let Some((run_level, run_idx)) = heading_run.as_ref()
+ && continues_heading(&lines[*run_idx], line)
+ && let Some(Block::Heading {
+ level: last_level,
+ text: htext,
+ }) = blocks.last_mut()
+ && *last_level == *run_level
+ {
+ let combined_chars = htext.chars().count() + 1 + text.chars().count();
+ if combined_chars <= HEADING_MAX_TEXT_CHARS {
+ let run_level = *run_level;
+ if debug {
+ eprintln!(
+ "[MD heading-wrap-uncond] merge h{} '{}' <- '{}' (prev_idx={} cur_idx={} combined={})",
+ run_level,
+ htext.chars().take(40).collect::<String>(),
+ text.chars().take(60).collect::<String>(),
+ run_idx,
+ line_idx,
+ combined_chars
+ );
+ }
+ append_inline_continuation(htext, text, &collapse_whitespace(text));
+ heading_run = Some((run_level, line_idx));
+ continue;
+ }
+ }
if let Some(level) = level {
// Merge a wrapped continuation into the heading directly above when
// it flows as one block: same level, the heading is still the last
@@ -501,6 +621,15 @@ fn classify_region(
list_base_indent = None;
last_list_item_idx = None;
last_list_line = None;
+ if debug {
+ eprintln!(
+ "[MD heading-emit size/outline] h{} idx={} '{}' size={:.2}",
+ level,
+ line_idx,
+ text.chars().take(80).collect::<String>(),
+ line.dominant_font_size,
+ );
+ }
blocks.push(Block::Heading {
level,
text: collapse_whitespace(text),
@@ -602,10 +731,22 @@ fn classify_region(
// have. With an empty heading_map this lands on H1; with a full
// 6-level map it caps at H6.
let level = (heading_map.len() as u8 + 1).clamp(1, MAX_HEADING_LEVELS as u8);
+ if debug {
+ eprintln!(
+ "[MD heading-emit bold] h{} idx={} '{}' size={:.2}",
+ level,
+ line_idx,
+ text.chars().take(80).collect::<String>(),
+ line.dominant_font_size,
+ );
+ }
blocks.push(Block::Heading {
level,
text: collapse_whitespace(text),
});
+ // Arm the heading_run so a wrapped continuation on the next line
+ // merges into this heading instead of emitting as a second one.
+ heading_run = Some((level, line_idx));
continue;
}
diff --git a/crates/liteparse/src/markdown_layout/headings.rs b/crates/liteparse/src/markdown_layout/headings.rs
index 5a57e25..68b08b3 100644
--- a/crates/liteparse/src/markdown_layout/headings.rs
+++ b/crates/liteparse/src/markdown_layout/headings.rs
@@ -38,6 +38,88 @@ pub(super) const BOLD_HEADING_MAX_CHARS: usize = 80;
/// multi-clause sentences that exceed any reasonable label length.
pub(super) const HEADING_MAX_TEXT_CHARS: usize = 140;
+/// Recognize a section-number prefix like "1", "1.5", "A.2", "Sec. 2",
+/// "Ch. 3", "§4". Used to exempt numbered subsection headings from the
+/// bold-heading run-in guard (which would otherwise reject "1.5. Migrant
+/// Workers..." because of the embedded ". " after the section number).
+pub(super) fn is_section_number_prefix(s: &str) -> bool {
+ let t = s.trim();
+ if t.is_empty() {
+ return false;
+ }
+ // Strip optional "Sec", "Ch", "Chapter", "§" lead-in.
+ let stripped = t
+ .strip_prefix('§')
+ .or_else(|| {
+ for lead in [
+ "Sec.", "Sec", "Ch.", "Ch", "Chapter", "Chap.", "Chap", "Part",
+ ] {
+ if let Some(rest) = t.strip_prefix(lead) {
+ return Some(rest.trim_start());
+ }
+ }
+ None
+ })
+ .unwrap_or(t);
+ // Remaining must be a dotted numeric / alphanumeric section identifier.
+ // Examples accepted: "1", "1.5", "1.5.2", "A.2", "IV".
+ if stripped.is_empty() {
+ // "§" alone counts as a section marker context.
+ return true;
+ }
+ let mut saw_digit = false;
+ let mut prev_dot = true;
+ for c in stripped.chars() {
+ if c.is_ascii_digit() {
+ saw_digit = true;
+ prev_dot = false;
+ } else if c.is_ascii_uppercase() && !prev_dot {
+ // Allow a leading letter ("A.2") but not letters mid-segment.
+ return false;
+ } else if c.is_ascii_uppercase() {
+ prev_dot = false;
+ } else if c == '.' {
+ if prev_dot {
+ return false;
+ }
+ prev_dot = true;
+ } else {
+ return false;
+ }
+ }
+ saw_digit
+}
+
+/// Recognize an attribution / annotation prefix like "Source:",
+/// "Note:", "Adapted from", "Reproduced from", "Image:" — these
+/// commonly appear as isolated bold-styled lines beneath charts and
+/// figures, but they are never section headings.
+pub(super) fn is_attribution_line(text: &str) -> bool {
+ let t = text.trim_start();
+ const PREFIXES: &[&str] = &[
+ "Source:",
+ "Sources:",
+ "Note:",
+ "Notes:",
+ "Adapted from",
+ "Reproduced from",
+ "Reprinted from",
+ "Image:",
+ "Image source:",
+ "Photo:",
+ "Photo credit:",
+ "Credit:",
+ "Caption:",
+ ];
+ for p in PREFIXES {
+ if t.len() >= p.len() && t.is_char_boundary(p.len()) && t[..p.len()].eq_ignore_ascii_case(p)
+ {
+ return true;
+ }
+ }
+ false
+}
+
/// Recognize caption-prefix lines like "Figure 7", "Fig. 12.", "Table 3:",
/// "Tab. 5", "Equation (4)" — these routinely render in a slightly distinct
/// font/size that lands them in the heading_map and gets them promoted to a
@@ -273,6 +355,11 @@ pub(super) fn looks_like_bold_heading(
if is_caption_line(text) {
return false;
}
+ // Attribution lines ("Source: …", "Note: …", "Adapted from …") commonly
+ // appear as isolated bold lines beneath charts/figures. Never headings.
+ if is_attribution_line(text) {
+ return false;
+ }
// Accept a line whose spans are all bold (italic may vary) and non-mono.
// A strict single-style check would reject headings that mix bold and
// bold-italic spans (e.g. "**4** ***Foo*** **Bar**"), which are common
@@ -288,6 +375,43 @@ pub(super) fn looks_like_bold_heading(
if text.ends_with('.') {
return false;
}
+ // Run-in label: a bold line whose first sentence ends in ". " followed
+ // by body prose (multi-word continuation that doesn't look like a
+ // sub-numbered heading) is a paragraph lead, not a section heading.
+ // Tight constraints to avoid rejecting numbered subsection headings like
+ // "1.5. Migrant Workers" or "Sec. 2. Method":
+ // - ≥3 space-separated words after the break
+ // - line ends with mid-word "-" (wrap continuation) OR is >50 chars
+ // - first char after the break is uppercase ASCII letter
+ if let Some(pos) = text.find(". ") {
+ let before = &text[..pos];
+ // Section-number prefix exemption: when the segment before ". " is a
+ // numbered section identifier (e.g. "1", "1.5", "A.2", "Sec. 2",
+ // "Ch. 3", "§4"), this is a numbered subsection heading like
+ // "1.5. Migrant Workers..." — the period is part of the section
+ // number, not a sentence terminator. Skip the run-in rejection.
+ let is_section_number = is_section_number_prefix(before);
+ let after = text[pos + 2..].trim();
+ let starts_upper = after.chars().next().is_some_and(|c| c.is_ascii_uppercase());
+ let word_count = after.split_whitespace().count();
+ let ends_hyphen = text.trim_end().ends_with('-');
+ if !is_section_number
+ && starts_upper
+ && ((word_count >= 2 && ends_hyphen) || (word_count >= 3 && text.chars().count() > 50))
+ {
+ if std::env::var("LITEPARSE_DEBUG_MD").is_ok() {
+ eprintln!(
+ "[MD bold-heading REJECT run-in] '{}' (pos={} word_count={} ends_hyphen={} len={})",
+ text.chars().take(80).collect::<String>(),
+ pos,
+ word_count,
+ ends_hyphen,
+ text.chars().count()
+ );
+ }
+ return false;
+ }
+ }
// Block headings are capitalized. A bold line starting lowercase is almost
// always a stray bold word or a borderless table cell, not a heading.
if text.chars().next().is_some_and(|c| c.is_lowercase()) {
@@ -332,7 +456,13 @@ pub(super) fn looks_like_bold_heading(
}
match next {
None => true,
- Some(n) => !continues_paragraph(line, n),
+ // A wrapped multi-line bold heading ("Cellular Cycle\nand Replication")
+ // would otherwise be rejected here because line 1's next line continues
+ // the paragraph. Accept the case where `next` is itself an all-bold
+ // line — the `heading_run` merge in classify.rs will absorb the
+ // continuation. Without this, wrapped bold-only headings silently emit
+ // as bold paragraphs and don't reach the heading hierarchy at all.
+ Some(n) => !continues_paragraph(line, n) || line_all_bold(n),
}
}
@@ -664,6 +794,15 @@ pub(super) fn outline_heading_level(
if line_len > max_line_len || sentence_breaks >= 2 {
continue;
}
+ // Run-in label guard: a single ". " sentence break is OK on a heading
+ // only when the line is roughly title-shaped. When the line carries
+ // substantial body text past the title (e.g. "Base model. Any n-layer
+ // transformer architec-"), it's a paragraph leading with a bold
+ // run-in label, not a section heading. Title is the *prefix* of the
+ // matched line, so excess body length = line_len - title_len.
+ if sentence_breaks >= 1 && line_len > title_len + 15 {
+ continue;
+ }
if normalized_line.starts_with(&normalized_title)
|| normalized_title.starts_with(&normalized_line)
{
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs
index 8a806a5..11debbf 100644
--- a/crates/liteparse/src/markdown_layout/tables.rs
+++ b/crates/liteparse/src/markdown_layout/tables.rs
@@ -2219,22 +2219,6 @@ fn build_ruled_table(
.count();
let empty_frac = (empty_count as f32) / (total as f32);
if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION {
- // Fill-in-blank forms (docs 119/120/180) have a column of row labels
- // OR a row of column headers but most body cells empty by design.
- // Two filters together distinguish real forms from layout wrappers
- // that happen to have a dense column (doc 130's right-side prose
- // column trips a naive max-fill check):
- // 1. The dense band must be col 0 or row 0 — that's where labels
- // and headers actually live; a dense column anywhere else is
- // almost always body prose, not a structural label spine.
- // 2. Spine cells must be short (≤ TABLE_SPINE_MAX_CELL_CHARS) —
- // labels are 1-5 words, not multi-sentence prose.
- // Restrict to the col-0 case only. row-0 spines turn out to be too
- // permissive — doc 051 has a multi-line header that lands across all
- // columns (row0_fill = 1.0) but its body is badly fragmented, so the
- // row-0 signal misleads. Col-0 label columns are the much stronger
- // fill-in-form signature: rare in body prose, near-universal in
- // forms.
let col0_fill = (0..n_rows).filter(|r| cell_has_text[*r][0]).count() as f32 / n_rows as f32;
let col0_max_chars = (0..n_rows)
.filter(|r| cell_has_text[*r][0])
@@ -2243,7 +2227,49 @@ fn build_ruled_table(
.unwrap_or(0);
let col0_spine =
col0_fill >= TABLE_SPINE_FILL_FRACTION && col0_max_chars <= TABLE_SPINE_MAX_CELL_CHARS;
- if !col0_spine || empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION_WITH_SPINE {
+ // Long-prose table bypass: a large grid (≥5×3) with a structurally
+ // bold row 0 header AND a dense description column (any col with
+ // ≥70% fill rate) is almost certainly a multi-line legal/reference
+ // table where the description column wraps over many empty
+ // continuation rows. Density arbitration in `merge_table_runs`
+ // prevents a decorative grid that happens to match these criteria
+ // from beating a real overlapping borderless table.
+ // Reproducer: docs 088/089/090 multi-page legal report.
+ // Header may span multiple visual rows (the grid detector slices on
+ // each text baseline). Treat the first ≤4 rows as the header band
+ // and require their *union* to cover most columns AND be all-bold.
+ let header_band = n_rows.min(4);
+ let mut header_cols_covered = vec![false; n_cols];
+ let mut header_all_bold = true;
+ for r in 0..header_band {
+ for c in 0..n_cols {
+ if cell_has_text[r][c] {
+ header_cols_covered[c] = true;
+ if !cell_is_bold[r][c] {
+ header_all_bold = false;
+ }
+ }
+ }
+ }
+ let header_coverage = header_cols_covered.iter().filter(|t| **t).count();
+ let dense_inner_col = (1..n_cols).any(|c| {
+ let col_fill =
+ (0..n_rows).filter(|r| cell_has_text[*r][c]).count() as f32 / n_rows as f32;
+ col_fill >= TABLE_SPINE_FILL_FRACTION
+ });
+ // Header coverage doesn't need to span every column — wide-cell
+ // legal tables often spread the header across many visual baselines
+ // and only a few columns land in the top-4-rows band. Require ≥3
+ // columns covered as evidence of a real header, not just a title.
+ let long_prose_table = n_rows >= 5
+ && n_cols >= 3
+ && header_coverage >= 3
+ && header_all_bold
+ && dense_inner_col;
+ if !col0_spine && !long_prose_table {
+ return None;
+ }
+ if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION_WITH_SPINE && !long_prose_table {
return None;
}
}
@@ -2392,19 +2418,39 @@ pub(super) fn detect_ruled_tables(
out
}
+/// Count filled (non-empty) cells in a TableRun. GridFallback returns 0 so
+/// it never beats a real Table in density comparisons.
+fn run_filled_cells(run: &TableRun) -> usize {
+ match &run.block {
+ Block::Table { header, rows } => {
+ let header_filled = header
+ .as_ref()
+ .map(|h| h.iter().filter(|c| !c.trim().is_empty()).count())
+ .unwrap_or(0);
+ let body_filled: usize = rows
+ .iter()
+ .flat_map(|r| r.iter())
+ .filter(|c| !c.trim().is_empty())
+ .count();
+ header_filled + body_filled
+ }
+ _ => 0,
+ }
+}
+
/// Merge ruled-grid runs with borderless runs into a single sorted list. When
-/// ranges overlap the ruled run wins (path-based geometry is strictly stronger
-/// than text-alignment heuristics) — overlapping borderless runs are dropped.
+/// ranges overlap the ruled run normally wins (path-based geometry is a
+/// stronger signal than text-alignment heuristics), with two exceptions:
+/// 1. A single-column ruled run yields to a multi-column borderless run
+/// covering the same range (vertical separators may be implicit; doc 078).
+/// 2. A sparse ruled run yields to a denser borderless run — decorative
+/// vector boxes around titles / callout banners produce ruled "tables"
+/// with few filled cells; when a borderless detector finds a much denser
+/// real table in the same region, prefer it (doc 051).
pub(super) fn merge_table_runs(
mut ruled: Vec<TableRun>,
borderless: Vec<TableRun>,
) -> Vec<TableRun> {
- // A ruled run normally beats an overlapping borderless run (path geometry
- // is a stronger signal than text alignment). But a 1-column ruled run is
- // ambiguous — it can come from a real single-column table (doc 149's
- // stacked cards) OR from a multi-column table whose vertical separators
- // are implicit and only the top/bottom rules were drawn (doc 078). Yield
- // to a multi-column borderless run that covers the same range.
let mut kept: Vec<TableRun> = Vec::with_capacity(ruled.len());
for r in ruled.drain(..) {
let is_one_col = matches!(&r.block, Block::Table { rows, .. } if rows.first().map(|row| row.len()).unwrap_or(0) <= 1);
@@ -2420,6 +2466,21 @@ pub(super) fn merge_table_runs(
continue;
}
}
+ // Density check: if a borderless run overlaps and carries
+ // substantially more filled cells, the ruled run is most likely
+ // a decorative grid (page chrome, title banner) wrapping the real
+ // table the borderless detector already found.
+ let ruled_density = run_filled_cells(&r);
+ let beaten_by_density = borderless.iter().any(|b| {
+ let overlaps = !(b.end <= r.start || b.start >= r.end);
+ if !overlaps {
+ return false;
+ }
+ run_filled_cells(b) >= ruled_density * 2 + 4
+ });
+ if beaten_by_density {
+ continue;
+ }
kept.push(r);
}
for b in borderless {
diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs
index ed338ea..9fbb1cb 100644
--- a/crates/liteparse/src/projection.rs
+++ b/crates/liteparse/src/projection.rs
@@ -570,7 +570,30 @@ fn form_lines(
// Cap the line height to a reasonable multiple of median text height.
let proposed_min_y = current_line_min_y.min(item.item.y);
let proposed_max_y = current_line_max_y.max(item.item.y + item.item.height);
- let too_tall = (proposed_max_y - proposed_min_y) > median_height * 1.8;
+ // too_tall threshold: prefer the smallest *trusted* item height
+ // already in the current line as the line-height baseline. PDFium
+ // reports inflated em-box heights for items with matrix-baked
+ // sizing (fs=1.0); using the global `median_height` then permits
+ // a real heading row to absorb the next visual row below it.
+ // Falls back to median_height when no smaller anchor is available.
+ let line_anchor_h = current_line
+ .iter()
+ .filter_map(|i| {
+ if matches!(i.item.font_size, Some(fs) if fs > 1.5)
+ || i.item.height < median_height
+ {
+ Some(i.item.height)
+ } else {
+ None
+ }
+ })
+ .fold(f32::INFINITY, f32::min);
+ let too_tall_threshold = if line_anchor_h.is_finite() {
+ (line_anchor_h * 1.8).max(median_height * 0.6)
+ } else {
+ median_height * 1.8
+ };
+ let too_tall = (proposed_max_y - proposed_min_y) > too_tall_threshold;
if !line_collide
&& !margin_mismatch
@@ -3386,7 +3409,18 @@ fn xy_find_column_cut(
// numbered marker, etc.) carry only a few lines.
let strong_before = peaks.len();
let raw_counts: Vec<(f32, usize)> = peaks.clone();
- peaks.retain(|(_, c)| *c >= XY_COLUMN_MIN_LINES_PER_PEAK);
+ // Adaptive minimum: scale with how many line-starts the band actually has.
+ // The static `XY_COLUMN_MIN_LINES_PER_PEAK = 10` floor is meant to drop
+ // indented-paragraph false peaks (a handful of lines) on a large region,
+ // but on smaller regions it can drop legitimate column peaks too. For a
+ // body region with ~36 lines split across 2 columns, each peak only
+ // carries 5–6 line-starts after the histogram spreads them across the
+ // page width — well under 10. Use `max(line_starts / 6, 4)`: still rules
+ // out the tiny indent peaks, but a 2-column region with ~5 lines/peak now
+ // qualifies. Capped by the original `XY_COLUMN_MIN_LINES_PER_PEAK` to
+ // preserve behavior on large regions.
+ let adaptive_min = (line_starts.len() / 6).clamp(4, XY_COLUMN_MIN_LINES_PER_PEAK);
+ peaks.retain(|(_, c)| *c >= adaptive_min);
if peaks.len() < 2 {
if dbg {
let preview: Vec<String> = raw_counts
@@ -3395,8 +3429,7 @@ fn xy_find_column_cut(
.map(|(x, c)| format!("({x:.0}:{c})"))
.collect();
eprintln!(
- "[xy col-reject] peaks<2 after min_per_peak={} (raw={strong_before} strong={}): {}",
- XY_COLUMN_MIN_LINES_PER_PEAK,
+ "[xy col-reject] peaks<2 after min_per_peak={adaptive_min} (raw={strong_before} strong={}): {}",
peaks.len(),
preview.join(",")
);
@@ -4218,14 +4251,32 @@ pub(crate) fn build_projected_lines(
const Y_BAND_HEIGHT_CAP: f32 = 24.0;
for idx in sorted {
let y = items[idx].item.y;
- let h = items[idx].item.height.clamp(1.0, Y_BAND_HEIGHT_CAP);
+ let raw_h = items[idx].item.height;
+ let h = raw_h.clamp(1.0, Y_BAND_HEIGHT_CAP);
if current.is_empty() {
current.push(idx);
current_y = y;
current_h = h;
continue;
}
- let same = (y - current_y).abs() < current_h.max(h) * 0.5;
+ // Use the SMALLER of the two heights for the y-band tolerance —
+ // matching baselines (same row) differ by ≪ either glyph height,
+ // while the next-row-down baseline differs by ≈ the smaller
+ // glyph's full height. Using `.max()` here is fooled by
+ // matrix-inflated em-boxes: a 16pt heading row "swallows" a
+ // 50pt-em-box body item on the row below because the tolerance
+ // becomes 25pt. Reproducer: doc 122 ("III. Electrophorese" at
+ // y=286.2 vs "Reagents:" at y=295.9, diff=9.7pt).
+ // When the incoming item's raw bbox is wildly taller than the
+ // current line's anchor (>2× and >Y_BAND_HEIGHT_CAP), this is
+ // an em-box-inflated body item trying to crash into a heading
+ // row. Halve the tolerance further so a small y-offset (4–7pt)
+ // doesn't fuse them. Reproducer: doc 122 "Load the Gel"
+ // (h=14.9, y=435.6) vs next-row "Use" (h=55.6, y=442.0,
+ // diff=6.4pt vs 14.9*0.5=7.45 tolerance — too permissive).
+ let height_mismatch = raw_h > Y_BAND_HEIGHT_CAP && raw_h > current_h * 2.0;
+ let tol_factor = if height_mismatch { 0.3 } else { 0.5 };
+ let same = (y - current_y).abs() < current_h.min(h) * tol_factor;
if same {
current.push(idx);
current_y = current_y.min(y);