Skip to content

A fast, helpful, and open-source document parser

wip

Logan Markewich647cdfb894ce
diff --git a/MARKDOWN_PROGRESS.md b/MARKDOWN_PROGRESS.md
index 79968c9..ef35987 100644
--- a/MARKDOWN_PROGRESS.md
+++ b/MARKDOWN_PROGRESS.md
@@ -2,6 +2,8 @@
Tracks build-order progress against [MARKDOWN_PLAN.md](MARKDOWN_PLAN.md). Update as steps complete.
+> **2026-06-09 union-track cluster merge — ParseBench grits_con 0.4308 → 0.4389, TRM composite 0.2887 → 0.2947, unmatched predicted tables −26%; olmOCR & opendataloader regression-clean.** Triage showed 394 of ParseBench's 425 unmatched predicted tables were fragments of real tables (runs broken at column-count changes, stranded PDFium-merged rows, 56% of emitted tables headerless). New `merge_fragmented_table_clusters` post-pass in `markdown_layout/tables.rs` re-derives union column tracks across adjacent table runs and re-extracts every line against them; `union_header_from_above` bins stacked/colspan headers per-span with replication. Track-compatibility gate (≥75% of narrower run's tracks must map into wider run's) prevents fusing genuinely separate stacked tables. Env kill-switch `LITEPARSE_DISABLE_CLUSTER_MERGE=1`. Full details + tried-and-reverted notes (gap-based wrap merge has no signal; opendataloader TEDS baseline correction 0.5917 → 0.568) in FAILURE_MODES.md fix log. Next table levers: multi-line-cell row merging (needs ruled/label-anchor signal, not gaps) and cross-region fragments (V-cut sliced tables).
+
> **2026-06-04 inferred-tracks table detector — olmOCR-bench table_tests 38.8% → 39.5% (+0.7pp, +7 tests), Overall 35.1% → 35.2%; opendataloader & ParseBench unchanged.** Targeted the dominant `wrong_neighbors` failure on financial/earnings tables (e.g. Apple/Wiley `a769f9a3…pg2`, `fa18a15c…`, hundreds of similar narrow-numeric tables): the existing `try_detect_table` derives column tracks from the FIRST row's `split_cells` output, which uses `TABLE_CELL_GAP_FONT_MULTIPLIER × font_size` (= 14pt at 14pt body) as the inter-span gap threshold. The Apple Revenue Q3 row has `$448 → $427` and `$427 → 7%` gaps of 13.9pt — *just under* the threshold — so its 5 distinct PDFium items collapse to 3 cells (`$448 $427 7%` lumped). Every row in the Revenue table fails identically, so the table establishes as 3-col, and the cross-run merge can't recover (`merge_consecutive_table_runs` only handles `|A|+1 == |B|`).
>
> **New `try_detect_table_inferred` (`markdown_layout/tables.rs`).** Runs in `detect_tables` *before* `try_detect_table`. Steps:
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index 0772aa9..5d6e002 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -1,4 +1,5 @@
use crate::error::LiteParseError;
+use crate::glyph_names::resolve_glyph_name;
use crate::render::encode_png;
use crate::types::{
ExtractedImage, GraphicPrimitive, ImageRef, OutlineTarget, Page as LitePage, PdfInput, Rect,
@@ -411,20 +412,17 @@ fn extract_page_text_items(
let vp_xform = page.viewport_transform(view_box);
let mut items: Vec<TextItem> = Vec::new();
let mut seg = SegmentBuilder::new();
+ let garbage_fonts = detect_garbage_unicode_fonts(text_page, char_count);
+ let mut glyph_decoder = GlyphDecoder::new(
+ std::env::var("LITEPARSE_DEBUG_GLYPH").is_ok(),
+ garbage_fonts,
+ );
for i in 0..char_count {
let ch = text_page.char_at_unchecked(i);
let unicode = ch.unicode();
let is_generated = ch.is_generated();
- // Skip null / invalid sentinels
- if unicode == 0 || unicode == 0xFFFE || unicode == 0xFFFF {
- if debug {
- eprintln!("[extract-debug] i={i} SKIP sentinel unicode=0x{unicode:04X}");
- }
- continue;
- }
-
// Skip invisible text (render mode 3) only when the page also has visible text.
// If all text is invisible, it's likely an OCR text layer and we should keep it.
if skip_invisible && ch.text_render_mode() == Some(3) {
@@ -437,26 +435,47 @@ fn extract_page_text_items(
continue;
}
+ // Glyph-name recovery: when the font's unicode mapping is missing or
+ // untrusted, resolve the charcode's PostScript glyph name instead.
+ let decoded: Option<&str> = if is_generated {
+ None
+ } else {
+ glyph_decoder.decode(&ch, unicode)
+ };
+
+ // Skip null / invalid sentinels (unless the glyph name recovered them)
+ if decoded.is_none() && (unicode == 0 || unicode == 0xFFFE || unicode == 0xFFFF) {
+ if debug {
+ eprintln!("[extract-debug] i={i} SKIP sentinel unicode=0x{unicode:04X}");
+ }
+ continue;
+ }
+
// Map to a Rust char, with special-case replacements.
// Some PDF fonts encode ligatures as control characters; expand them.
// We use the first char for segment decisions, then append trailing chars.
- let (c, ligature_tail): (char, &str) = match unicode {
- 0x02 => ('-', ""), // STX → hyphen (common in some PDF encodings)
- 0x1A => ('f', "f"), // ff ligature
- 0x1B => ('f', "t"), // ft ligature
- 0x1C => ('f', "i"), // fi ligature
- 0x1D => ('T', "h"), // Th ligature
- 0x1E => ('f', "fi"), // ffi ligature
- 0x1F => ('f', "l"), // fl ligature
- _ => match char::from_u32(unicode) {
- Some(ch_mapped) => (ch_mapped, ""),
- None => {
- if debug {
- eprintln!("[extract-debug] i={i} SKIP invalid unicode=0x{unicode:04X}");
+ let (c, ligature_tail): (char, &str) = if let Some(s) = decoded {
+ let mut it = s.chars();
+ (it.next().unwrap(), it.as_str())
+ } else {
+ match unicode {
+ 0x02 => ('-', ""), // STX → hyphen (common in some PDF encodings)
+ 0x1A => ('f', "f"), // ff ligature
+ 0x1B => ('f', "t"), // ft ligature
+ 0x1C => ('f', "i"), // fi ligature
+ 0x1D => ('T', "h"), // Th ligature
+ 0x1E => ('f', "fi"), // ffi ligature
+ 0x1F => ('f', "l"), // fl ligature
+ _ => match char::from_u32(unicode) {
+ Some(ch_mapped) => (ch_mapped, ""),
+ None => {
+ if debug {
+ eprintln!("[extract-debug] i={i} SKIP invalid unicode=0x{unicode:04X}");
+ }
+ continue;
}
- continue;
- }
- },
+ },
+ }
};
let c = normalize_punct(c);
@@ -849,6 +868,226 @@ fn color_to_argb_hex(c: &pdfium::Color) -> String {
format!("{:02x}{:02x}{:02x}{:02x}", c.a, c.r, c.g, c.b)
}
+/// Per-page glyph-name-based unicode recovery (fork API).
+///
+/// When a font has no /ToUnicode CMap, PDFium derives unicode from the
+/// encoding alone — garbage for custom/Identity encodings (Mode 10 glyph
+/// soup), and guessed control-code expansions for ligatures (Mode 16). The
+/// PostScript glyph name the font assigns to the charcode (from /Encoding
+/// /Differences or the embedded font program) resolved against the Adobe
+/// Glyph List is the authoritative signal in both cases.
+struct GlyphDecoder {
+ fonts: std::collections::HashMap<usize, FontGlyphInfo>,
+ /// Chars arrive in runs per text object; cache the last object's font key
+ /// to skip the FPDFTextObj_GetFont FFI call on the common path.
+ last_obj: usize,
+ last_key: usize,
+ /// Font handles whose /ToUnicode the prescan flagged as garbage (high
+ /// fraction of control/PUA unicodes across the page).
+ garbage_fonts: std::collections::HashSet<usize>,
+ debug: bool,
+ /// Kill-switch for A/B benching: LITEPARSE_DISABLE_GLYPH_NAMES=1
+ disabled: bool,
+}
+
+struct FontGlyphInfo {
+ font: Font,
+ /// No /ToUnicode and no standard base encoding (or the prescan flagged
+ /// the ToUnicode as garbage): PDFium's unicode values for this font are
+ /// untrusted, so every charcode gets a recovery try.
+ untrusted: bool,
+ /// charcode → resolved replacement text (None = unrecoverable)
+ cache: std::collections::HashMap<u32, Option<String>>,
+ /// Lazily-built glyph_index → unicode map from the embedded font
+ /// program's cmap table (None = not yet built, Some(None) = unavailable).
+ reverse_cmap: Option<Option<std::collections::HashMap<u32, u32>>>,
+}
+
+impl GlyphDecoder {
+ fn new(debug: bool, garbage_fonts: std::collections::HashSet<usize>) -> Self {
+ Self {
+ fonts: std::collections::HashMap::new(),
+ garbage_fonts,
+ last_obj: 0,
+ last_key: 0,
+ debug,
+ disabled: std::env::var("LITEPARSE_DISABLE_GLYPH_NAMES").is_ok(),
+ }
+ }
+
+ /// Returns replacement text for this char when its glyph name resolves
+ /// and the current unicode is suspicious (control/PUA/sentinel/map-error)
+ /// or the font's unicode mapping is untrusted altogether.
+ fn decode(&mut self, ch: &pdfium::TextChar, unicode: u32) -> Option<&str> {
+ if self.disabled {
+ return None;
+ }
+ let cheap_suspicious = matches!(unicode, 0 | 0xFFFE | 0xFFFF)
+ || (unicode < 0x20 && !matches!(unicode, 0x09 | 0x0A | 0x0D))
+ || (0xE000..=0xF8FF).contains(&unicode);
+
+ let obj_ptr = ch.text_object()?;
+ let obj = obj_ptr as usize;
+ let key = if obj == self.last_obj {
+ self.last_key
+ } else {
+ let font = unsafe { Font::from_text_object(obj_ptr) }?;
+ let key = font.handle() as usize;
+ let debug = self.debug;
+ let garbage = self.garbage_fonts.contains(&key);
+ self.fonts.entry(key).or_insert_with(|| {
+ let has_to_unicode = font.has_to_unicode();
+ let encoding = font.encoding();
+ let untrusted = garbage
+ || (!has_to_unicode
+ && !matches!(
+ encoding.as_deref(),
+ Some("WinAnsiEncoding")
+ | Some("MacRomanEncoding")
+ | Some("MacExpertEncoding")
+ | Some("StandardEncoding")
+ ));
+ if debug {
+ eprintln!(
+ "[glyph] font={:?} to_unicode={} encoding={:?} garbage={} untrusted={}",
+ font.base_name(),
+ has_to_unicode,
+ encoding,
+ garbage,
+ untrusted
+ );
+ }
+ FontGlyphInfo {
+ font,
+ untrusted,
+ cache: std::collections::HashMap::new(),
+ reverse_cmap: None,
+ }
+ });
+ self.last_obj = obj;
+ self.last_key = key;
+ key
+ };
+ let info = self.fonts.get_mut(&key)?;
+
+ // map-error FFI check is the expensive part of "suspicious"; only
+ // consult it when the cheap checks and font trust don't decide.
+ if !info.untrusted && !cheap_suspicious && !ch.has_unicode_map_error() {
+ return None;
+ }
+ let debug = self.debug;
+
+ let char_code = ch.char_code();
+ let FontGlyphInfo {
+ font,
+ cache,
+ reverse_cmap,
+ ..
+ } = info;
+ cache
+ .entry(char_code)
+ .or_insert_with(|| {
+ let name = font.char_glyph_name(char_code);
+ let resolved = name
+ .as_deref()
+ .and_then(resolve_glyph_name)
+ .filter(|r| r.chars().all(|c| !c.is_control()));
+ // Fallback: reverse-map the glyph index through the embedded
+ // font program's own cmap table.
+ let resolved = resolved.or_else(|| {
+ let glyph = font.char_glyph_index(char_code)?;
+ let map = reverse_cmap
+ .get_or_insert_with(|| {
+ let data = font.font_data();
+ let map = data.as_deref().and_then(crate::font_cmap::reverse_cmap);
+ if debug {
+ eprintln!(
+ "[glyph] reverse_cmap build: data={:?} bytes, entries={:?}",
+ data.as_ref().map(|d| d.len()),
+ map.as_ref().map(|m| m.len())
+ );
+ }
+ map
+ })
+ .as_ref()?;
+ let u = *map.get(&glyph)?;
+ if (0xE000..=0xF8FF).contains(&u) {
+ return None;
+ }
+ // Synthetic subset cmaps just echo the charcode back
+ // (charcode-identity, not semantic unicode). A recovery
+ // that "resolves" to the charcode itself is that
+ // signature, not a real mapping — keep PDFium's value.
+ if u == char_code && u != unicode {
+ return None;
+ }
+ let c = char::from_u32(u).filter(|c| !c.is_control())?;
+ Some(match crate::glyph_names::presentation_form_expansion(c) {
+ Some(s) => s.to_string(),
+ None => c.to_string(),
+ })
+ });
+ if debug {
+ eprintln!(
+ "[glyph] cc=0x{char_code:04X} unicode=0x{unicode:04X} name={name:?} -> {resolved:?}"
+ );
+ }
+ resolved
+ })
+ .as_deref()
+ }
+}
+
+/// Prescan: flag fonts whose /ToUnicode maps a high fraction of chars into
+/// control/PUA/sentinel codepoints — a structurally present but garbage CMap
+/// (e.g. `text_simple__spd`). Chars from flagged fonts get glyph-name /
+/// reverse-cmap recovery even when their individual unicode looks plausible.
+fn detect_garbage_unicode_fonts(
+ text_page: &TextPage,
+ char_count: i32,
+) -> std::collections::HashSet<usize> {
+ let mut counts: std::collections::HashMap<usize, (u32, u32)> = std::collections::HashMap::new();
+ let mut last_obj: usize = 0;
+ let mut last_key: usize = 0;
+ for i in 0..char_count {
+ let ch = text_page.char_at_unchecked(i);
+ if ch.is_generated() {
+ continue;
+ }
+ let unicode = ch.unicode();
+ if matches!(unicode, 0x09 | 0x0A | 0x0D | 0x20) {
+ continue;
+ }
+ let Some(obj_ptr) = ch.text_object() else {
+ continue;
+ };
+ let obj = obj_ptr as usize;
+ let key = if obj == last_obj {
+ last_key
+ } else {
+ let Some(font) = (unsafe { Font::from_text_object(obj_ptr) }) else {
+ continue;
+ };
+ last_obj = obj;
+ last_key = font.handle() as usize;
+ last_key
+ };
+ let entry = counts.entry(key).or_insert((0, 0));
+ entry.0 += 1;
+ let suspicious = matches!(unicode, 0 | 0xFFFE | 0xFFFF)
+ || unicode < 0x20
+ || (0xE000..=0xF8FF).contains(&unicode);
+ if suspicious {
+ entry.1 += 1;
+ }
+ }
+ counts
+ .into_iter()
+ .filter(|&(_, (total, suspicious))| total >= 20 && suspicious * 10 >= total)
+ .map(|(key, _)| key)
+ .collect()
+}
+
/// Accumulates characters into a single TextItem segment.
struct SegmentBuilder {
text: String,
diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rs
index 72c68b6..ecb1913 100644
--- a/crates/liteparse/src/lib.rs
+++ b/crates/liteparse/src/lib.rs
@@ -27,6 +27,10 @@ pub mod extract;
#[doc(hidden)]
pub mod figure_cluster;
#[doc(hidden)]
+pub mod font_cmap;
+#[doc(hidden)]
+pub mod glyph_names;
+#[doc(hidden)]
pub mod markdown_layout;
#[doc(hidden)]
pub mod ocr;
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs
index 11debbf..05ee051 100644
--- a/crates/liteparse/src/markdown_layout/tables.rs
+++ b/crates/liteparse/src/markdown_layout/tables.rs
@@ -54,6 +54,10 @@ pub(super) struct TableCell {
pub(super) struct TableRun {
pub(super) start: usize,
pub(super) end: usize,
+ /// First line index of the table's *body* rows. Differs from `start` when
+ /// header lines above the body were absorbed into the run; the cluster
+ /// re-extraction pass must not re-bin those header lines as body rows.
+ pub(super) body_start: usize,
pub(super) block: Block,
}
@@ -525,6 +529,35 @@ fn cells_from_raw_items_with_tracks(
Some(cells)
}
+/// Letters strictly outnumber digits — discriminates word-like labels
+/// (group headers) from merged numeric runs ("$448 $427 7%").
+fn is_alpha_dominant(text: &str) -> bool {
+ let letters = text.chars().filter(|c| c.is_alphabetic()).count();
+ let digits = text.chars().filter(|c| c.is_ascii_digit()).count();
+ letters > digits
+}
+
+/// Measurement-value shape: a decimal number ("2.28"), currency/percent,
+/// comma-grouped number ("1,240"), or a dash placeholder ("--" / "—").
+/// Bare integers (years, codes) deliberately do NOT match — they appear in
+/// legitimate header layers.
+fn is_value_like(text: &str) -> bool {
+ let t = text.trim();
+ if t.is_empty() {
+ return false;
+ }
+ if t.chars().all(|c| matches!(c, '-' | '—' | '–')) {
+ return true;
+ }
+ if t.contains('$') || t.contains('%') || t.contains('±') {
+ return true;
+ }
+ let chars: Vec<char> = t.chars().collect();
+ chars
+ .windows(3)
+ .any(|w| w[0].is_ascii_digit() && (w[1] == '.' || w[1] == ',') && w[2].is_ascii_digit())
+}
+
/// Split a multi-track-spanning span's text into one piece per covered track
/// by picking whitespace positions whose linearly-interpolated x is closest
/// to each subsequent anchor. Returns `Some(pieces)` of length
@@ -762,6 +795,7 @@ fn try_detect_table_inferred(
Some(TableRun {
start: run_start,
end,
+ body_start: start_idx,
block: Block::Table {
header,
rows: body_rows,
@@ -942,6 +976,7 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->
return Some(TableRun {
start: start_idx,
end,
+ body_start: start_idx,
block: Block::GridFallback { lines: raw },
});
}
@@ -978,9 +1013,16 @@ fn try_detect_table(lines: &[ProjectedLine], start_idx: usize, floor: usize) ->
return None;
}
+ if std::env::var("LITEPARSE_DEBUG_TABLE").is_ok() {
+ eprintln!(
+ "[tbl-detect @{start_idx}..{end}] cols={column_count} header={header:?} rows={}",
+ body_rows.len()
+ );
+ }
Some(TableRun {
start: run_start,
end,
+ body_start: start_idx,
block: Block::Table {
header,
rows: body_rows,
@@ -999,11 +1041,23 @@ fn absorb_header_lines(
column_count: usize,
floor: usize,
) -> Option<(usize, Vec<String>)> {
+ let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
let mut absorbed: Vec<Vec<TableCell>> = Vec::new();
let mut j = start_idx;
while j > floor {
let cand = j - 1;
let cells = split_cells(&lines[cand]);
+ if dbgt {
+ let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect();
+ eprintln!(
+ "[tbl-absorb cand @{cand} {:?}] cells={texts:?} extents={:?}",
+ lines[cand].text.chars().take(40).collect::<String>(),
+ cells
+ .iter()
+ .map(|c| (c.start_x as i32, c.end_x as i32))
+ .collect::<Vec<_>>()
+ );
+ }
// A header line must carry at least two cells (a single cell is a
// title/caption, not a header) and sit tight above the row below it.
if cells.len() < 2 {
@@ -1068,7 +1122,17 @@ pub(super) fn detect_tables(lines: &[ProjectedLine]) -> Vec<TableRun> {
i += 1;
}
}
- merge_consecutive_table_runs(out, lines)
+ let mut merged = merge_consecutive_table_runs(out, lines);
+ // Iterate to fixpoint (bounded): a merged cluster is itself a table run
+ // that may now sit adjacent to the next fragment cluster.
+ for _ in 0..4 {
+ let before = merged.len();
+ merged = merge_fragmented_table_clusters(merged, lines);
+ if merged.len() == before {
+ break;
+ }
+ }
+ merged
}
// ── Description-list 2-column table detector ──────────────────────────────
@@ -1377,6 +1441,7 @@ fn try_detect_description_list(lines: &[ProjectedLine], start_idx: usize) -> Opt
Some(TableRun {
start: start_idx,
end: j,
+ body_start: start_idx,
block: Block::Table {
header: None,
rows: body,
@@ -1477,6 +1542,460 @@ fn merge_consecutive_table_runs(runs: Vec<TableRun>, lines: &[ProjectedLine]) ->
out
}
+// ── Fragmented-cluster re-extraction ───────────────────────────────────────
+//
+// `try_detect_table` closes a run whenever the per-row cell count changes, so
+// one real table with sparse rows (row-label column present on some rows
+// only, empty value columns on others) fragments into several runs at
+// different column counts, with stranded single rows between them. The pairs
+// of fragments rarely satisfy `try_merge_pair`'s same-cols / |A|+1==|B|
+// cases. This pass takes the opposite approach: instead of mapping fragment
+// columns onto each other, it re-derives the *union* column track set from
+// the raw PDFium span positions across the whole cluster and re-extracts
+// every line against those tracks. Sparse rows get empty cells, merged spans
+// split at track anchors, and the cluster emits as one table.
+//
+// Only fires when ≥2 table runs are already y-adjacent — prose never enters
+// this path, so the false-positive surface is limited to "two genuinely
+// separate stacked tables", which the both-complete guard rejects.
+
+/// Max non-table lines between two runs for them to join one cluster.
+const TABLE_CLUSTER_MAX_INTERSTITIAL_LINES: usize = 2;
+
+/// Abort the cluster merge when more than this fraction of the window's
+/// lines can't be binned into the union tracks.
+const TABLE_CLUSTER_MAX_FAILED_ROW_FRAC: f32 = 0.3;
+
+/// Max header lines walked above the cluster body by the union header pass.
+const TABLE_CLUSTER_MAX_HEADER_LINES: usize = 4;
+
+fn merge_fragmented_table_clusters(runs: Vec<TableRun>, lines: &[ProjectedLine]) -> Vec<TableRun> {
+ if runs.len() < 2 || std::env::var("LITEPARSE_DISABLE_CLUSTER_MERGE").is_ok() {
+ return runs;
+ }
+ let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
+ let mut out: Vec<TableRun> = Vec::with_capacity(runs.len());
+ let mut i = 0;
+ while i < runs.len() {
+ let mut j = i + 1;
+ while j < runs.len() && cluster_adjacent(&runs[j - 1], &runs[j], lines) {
+ j += 1;
+ }
+ if j - i >= 2 {
+ let floor = out.last().map(|r| r.end).unwrap_or(0);
+ if let Some(merged) = build_union_table(&runs[i..j], lines, floor) {
+ if dbgt {
+ eprintln!(
+ "[tbl-cluster] merged {} runs (lines {}..{}) into one table",
+ j - i,
+ merged.start,
+ merged.end
+ );
+ }
+ out.push(merged);
+ i = j;
+ continue;
+ } else if dbgt {
+ eprintln!(
+ "[tbl-cluster] union build failed for {} runs @{}..{}",
+ j - i,
+ runs[i].start,
+ runs[j - 1].end
+ );
+ }
+ }
+ out.push(runs[i].clone());
+ i += 1;
+ }
+ out
+}
+
+fn cluster_adjacent(a: &TableRun, b: &TableRun, lines: &[ProjectedLine]) -> bool {
+ let (a_header, a_rows_len) = match &a.block {
+ Block::Table { header, rows } => (header.is_some(), rows.len()),
+ _ => return false,
+ };
+ let (b_header, b_rows_len) = match &b.block {
+ Block::Table { header, rows } => (header.is_some(), rows.len()),
+ _ => return false,
+ };
+ if b.start.saturating_sub(a.end) > TABLE_CLUSTER_MAX_INTERSTITIAL_LINES {
+ return false;
+ }
+ if a.end == 0 || a.end > lines.len() || b.start >= lines.len() {
+ return false;
+ }
+ let a_last = &lines[a.end - 1];
+ let b_first = &lines[b.start];
+ let line_height = a_last.bbox.height.max(b_first.bbox.height).max(1.0);
+ let gap = b_first.bbox.y - (a_last.bbox.y + a_last.bbox.height);
+ if gap > line_height * TABLE_MERGE_MAX_Y_GAP_LINES || gap < -line_height {
+ return false;
+ }
+ // Two complete-looking tables (both with explicit headers and real bodies)
+ // separated by a visible gap are most likely genuinely separate tables.
+ let both_complete = a_header && b_header && a_rows_len >= 3 && b_rows_len >= 3;
+ if both_complete && gap > line_height {
+ return false;
+ }
+ // Track compatibility: fragments of one table share column geometry (the
+ // narrower fragment's tracks are a subset of the wider one's), while two
+ // genuinely different stacked tables (e.g. a 2-col label/value list above
+ // a 4-col transaction table) do not. Without this gate the union merge
+ // fuses them and shreds both. Require ≥75% of the narrower run's tracks
+ // to align to the wider run's tracks.
+ let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
+ let (Some(a_tracks), Some(b_tracks)) = (run_body_tracks(a, lines), run_body_tracks(b, lines))
+ else {
+ // Inferred-path runs often have no line whose split_cells count
+ // matches the declared column count, so tracks can't be re-derived.
+ // Stay permissive — build_union_table's own soundness guards (width
+ // check, failed-row fraction) still gate the actual merge.
+ return true;
+ };
+ let (narrow, wide) = if a_tracks.len() <= b_tracks.len() {
+ (&a_tracks, &b_tracks)
+ } else {
+ (&b_tracks, &a_tracks)
+ };
+ let matched = narrow
+ .iter()
+ .filter(|&&t| {
+ wide.iter()
+ .any(|&w| subset_match_score(t, w, TABLE_SUBSET_TRACK_TOLERANCE_PT).is_some())
+ })
+ .count();
+ let ok = (matched as f32) >= (narrow.len() as f32) * 0.75;
+ if !ok && dbgt {
+ eprintln!(
+ "[tbl-cluster] adjacency reject @{}..{}: tracks {}/{} matched (narrow=[{}] wide=[{}])",
+ a.start,
+ b.end,
+ matched,
+ narrow.len(),
+ narrow
+ .iter()
+ .map(|t| format!("{:.0}-{:.0}", t.0, t.1))
+ .collect::<Vec<_>>()
+ .join(","),
+ wide.iter()
+ .map(|t| format!("{:.0}-{:.0}", t.0, t.1))
+ .collect::<Vec<_>>()
+ .join(",")
+ );
+ }
+ ok
+}
+
+/// Union column tracks across a window: every row with ≥2 gap-split cells
+/// contributes its cell start-x positions; positions cluster at
+/// `TABLE_TRACK_TOLERANCE_PT`. Cells (not raw spans) are the unit here —
+/// multi-word cell content emits several PDFium spans whose x positions are
+/// word starts, not column starts, and would shred the track set.
+fn union_tracks_in_window(lines: &[ProjectedLine], start: usize, end: usize) -> Vec<(f32, usize)> {
+ let mut xs: Vec<f32> = Vec::new();
+ for line in &lines[start..end.min(lines.len())] {
+ let cells = split_cells(line);
+ if cells.len() >= 2 {
+ xs.extend(cells.iter().map(|c| c.start_x));
+ }
+ }
+ xs.sort_by(f32::total_cmp);
+ let mut clusters: Vec<(f32, usize)> = Vec::new();
+ let mut current_sum = 0.0f32;
+ let mut current_count = 0usize;
+ let mut current_anchor = f32::NEG_INFINITY;
+ for &x in &xs {
+ // Looser tolerance than the per-run paths: center-aligned columns
+ // jitter their cell start-x by the half-width difference of their
+ // content (±10pt is routine); near-duplicate tracks get coalesced
+ // by the caller.
+ if current_count == 0 || (x - current_anchor).abs() <= TABLE_SUBSET_TRACK_TOLERANCE_PT {
+ current_sum += x;
+ current_count += 1;
+ current_anchor = current_sum / current_count as f32;
+ } else {
+ clusters.push((current_sum / current_count as f32, current_count));
+ current_sum = x;
+ current_count = 1;
+ current_anchor = x;
+ }
+ }
+ if current_count > 0 {
+ clusters.push((current_sum / current_count as f32, current_count));
+ }
+ clusters
+}
+
+/// Bin a line into the union tracks via its gap-split cells when the raw-span
+/// path fails: each cell maps to the nearest track (by start-x, within the
+/// loose subset tolerance) and all cells must map to distinct tracks. Unlike
+/// the body-detection paths this accepts 1-cell rows — inside a confirmed
+/// table cluster a lone label at a track is a sparse row (e.g. a row-label
+/// city with every value column empty), not prose.
+fn sparse_row_via_cells(line: &ProjectedLine, tracks: &[f32]) -> Option<Vec<String>> {
+ let cells = split_cells(line);
+ if cells.is_empty() {
+ return None;
+ }
+ let tol = TABLE_SUBSET_TRACK_TOLERANCE_PT;
+ let mut mapping: Vec<usize> = Vec::with_capacity(cells.len());
+ for c in &cells {
+ let (idx, d) = tracks
+ .iter()
+ .enumerate()
+ .map(|(i, &t)| (i, (c.start_x - t).abs()))
+ .min_by(|a, b| a.1.total_cmp(&b.1))?;
+ if d > tol {
+ return None;
+ }
+ mapping.push(idx);
+ }
+ let mut distinct = mapping.clone();
+ distinct.sort_unstable();
+ distinct.dedup();
+ if distinct.len() != mapping.len() {
+ return None;
+ }
+ let mut row = vec![String::new(); tracks.len()];
+ for (c, &idx) in cells.iter().zip(&mapping) {
+ row[idx] = c.text.clone();
+ }
+ Some(row)
+}
+
+/// Walk upward from the cluster's body start, binning each candidate header
+/// line's raw spans into the union tracks. A span covering multiple tracks
+/// (a group header like `EFETIVO` spanning its `ENFERMARIA`/`QUARTO`
+/// sub-columns) replicates its text across every covered track — the pipe
+/// table flattening of a colspan'd grid header. Layers stack top-to-bottom
+/// per column. Returns `(new_start, header)`.
+fn union_header_from_above(
+ lines: &[ProjectedLine],
+ body_start: usize,
+ floor: usize,
+ tracks: &[f32],
+) -> Option<(usize, Vec<String>)> {
+ let tol = TABLE_TRACK_TOLERANCE_PT;
+ // Fallback assignment for centered header cells whose extent doesn't
+ // reach the track anchor: nearest track within half the local gap.
+ let assign_nearest = |x_center: f32| -> Option<usize> {
+ let (idx, d) = tracks
+ .iter()
+ .enumerate()
+ .map(|(i, &t)| (i, (x_center - t).abs()))
+ .min_by(|a, b| a.1.total_cmp(&b.1))?;
+ let local_gap = if idx + 1 < tracks.len() {
+ tracks[idx + 1] - tracks[idx]
+ } else if idx > 0 {
+ tracks[idx] - tracks[idx - 1]
+ } else {
+ f32::INFINITY
+ };
+ if d <= (local_gap * 0.5).max(TABLE_SUBSET_TRACK_TOLERANCE_PT) {
+ Some(idx)
+ } else {
+ None
+ }
+ };
+ let mut layers: Vec<Vec<String>> = Vec::new();
+ let mut j = body_start;
+ while j > floor && layers.len() < TABLE_CLUSTER_MAX_HEADER_LINES {
+ let cand = j - 1;
+ if !table_rows_adjacent(&lines[cand], &lines[j]) {
+ break;
+ }
+ let spans: Vec<&TextItem> = lines[cand]
+ .spans
+ .iter()
+ .filter(|s| !s.text.trim().is_empty())
+ .collect();
+ if spans.len() < 2 {
+ break;
+ }
+ let mut layer = vec![String::new(); tracks.len()];
+ let mut ok = true;
+ for s in &spans {
+ let x0 = s.x;
+ let x1 = s.x + s.width.max(0.0);
+ let covered: Vec<usize> = tracks
+ .iter()
+ .enumerate()
+ .filter(|&(_, &t)| t >= x0 - tol && t <= x1 + tol)
+ .map(|(i, _)| i)
+ .collect();
+ let targets: Vec<usize> = if !covered.is_empty() {
+ covered
+ } else if let Some(idx) = assign_nearest((x0 + x1) * 0.5) {
+ vec![idx]
+ } else {
+ ok = false;
+ break;
+ };
+ for idx in targets {
+ let dst = &mut layer[idx];
+ if !dst.is_empty() {
+ dst.push(' ');
+ }
+ dst.push_str(s.text.trim());
+ }
+ }
+ if !ok || layer.iter().filter(|t| !t.is_empty()).count() < 2 {
+ break;
+ }
+ layers.push(layer);
+ j = cand;
+ }
+ if layers.is_empty() {
+ return None;
+ }
+ layers.reverse();
+ let header: Vec<String> = (0..tracks.len())
+ .map(|col| {
+ let mut parts: Vec<&str> = Vec::new();
+ for layer in &layers {
+ let s = layer[col].as_str();
+ if s.is_empty() || parts.last() == Some(&s) {
+ continue;
+ }
+ parts.push(s);
+ }
+ parts.join(" ")
+ })
+ .collect();
+ if header.iter().all(|h| h.is_empty()) {
+ return None;
+ }
+ Some((j, header))
+}
+
+/// Re-extract a cluster of adjacent table runs as one table against the
+/// union track set. Returns `None` (leaving the original runs untouched)
+/// when the union tracks are unsound or too many lines fail to bin.
+fn build_union_table(
+ cluster: &[TableRun],
+ lines: &[ProjectedLine],
+ floor: usize,
+) -> Option<TableRun> {
+ let dbgt = std::env::var("LITEPARSE_DEBUG_TABLE").is_ok();
+ let window_start = cluster.first()?.body_start;
+ let window_end = cluster.last()?.end.min(lines.len());
+ if window_start >= window_end {
+ return None;
+ }
+ let mut supported = union_tracks_in_window(lines, window_start, window_end);
+ // Drop low-support tracks: a column that only 1-2 cells across the whole
+ // cluster ever start at is noise (wrapped-cell continuation indents,
+ // intra-cell splits on widely-tracked text), not a real column.
+ let window_len = window_end - window_start;
+ let min_support = 2.max(window_len / 10);
+ supported.retain(|&(_, n)| n >= min_support);
+ // Coalesce adjacent tracks closer than a real column gutter — these are
+ // start-x jitter of one center-aligned column (content of different
+ // widths), not two columns. Real word-gap prose would coalesce down to a
+ // handful of wide tracks and then fail the width check below.
+ let mut font_sizes: Vec<f32> = lines[window_start..window_end]
+ .iter()
+ .map(|l| {
+ if l.dominant_font_size > 0.0 {
+ l.dominant_font_size
+ } else {
+ l.bbox.height.max(1.0)
+ }
+ })
+ .collect();
+ font_sizes.sort_by(f32::total_cmp);
+ let median_font = font_sizes[font_sizes.len() / 2];
+ let min_track_gap = (median_font * 1.5).max(12.0);
+ let mut coalesced: Vec<(f32, usize)> = Vec::with_capacity(supported.len());
+ for (t, n) in supported {
+ match coalesced.last_mut() {
+ Some((last, last_n)) if t - *last < min_track_gap => {
+ // Support-weighted midpoint keeps the anchor near the
+ // dominant alignment.
+ *last = (*last * *last_n as f32 + t * n as f32) / (*last_n + n) as f32;
+ *last_n += n;
+ }
+ _ => coalesced.push((t, n)),
+ }
+ }
+ let tracks: Vec<f32> = coalesced.into_iter().map(|(t, _)| t).collect();
+ if tracks.len() < TABLE_MIN_COLUMNS {
+ return None;
+ }
+ // The union must be at least as wide as the widest fragment — otherwise
+ // re-extraction would lose columns the per-run detection already found
+ // (and a word-gap prose union that coalesced to nothing lands here too).
+ let max_run_cols = cluster.iter().filter_map(run_column_count).max()?;
+ if tracks.len() < max_run_cols {
+ if dbgt {
+ eprintln!(
+ "[tbl-cluster] reject: {} tracks < widest fragment {max_run_cols} ([{}])",
+ tracks.len(),
+ tracks
+ .iter()
+ .map(|t| format!("{t:.0}"))
+ .collect::<Vec<_>>()
+ .join(",")
+ );
+ }
+ return None;
+ }
+
+ // NOTE: no vertical wrap-merge here. Gap-based merging of multi-line
+ // cells was tried and reverted — inside uniformly-leaded tables the gap
+ // between a wrapped cell line and a genuine next row is identical, so
+ // any threshold either shreds multi-line cells (no merge) or fuses
+ // adjacent logical rows (merge). Re-extracted clusters keep one row per
+ // line; multi-line-cell recovery needs a stronger signal (ruled-grid row
+ // boundaries, or label-column row anchoring).
+ let mut rows: Vec<Vec<String>> = Vec::new();
+ let mut failed_count = 0usize;
+ for idx in window_start..window_end {
+ let line = &lines[idx];
+ if let Some(cells) = cells_from_raw_items_with_tracks(line, &tracks) {
+ if cells.iter().any(|c| !c.text.is_empty()) {
+ rows.push(cells.into_iter().map(|c| c.text).collect());
+ }
+ } else if let Some(row) = sparse_row_via_cells(line, &tracks) {
+ rows.push(row);
+ } else {
+ // The line couldn't bin but still carries content — keep its
+ // full text as a column-0 row rather than dropping it. Counted
+ // toward the abort threshold below.
+ failed_count += 1;
+ let text = line.text.trim();
+ if !text.is_empty() {
+ let mut row = vec![String::new(); tracks.len()];
+ row[0] = collapse_whitespace(text);
+ rows.push(row);
+ }
+ }
+ }
+ let window_len = window_end - window_start;
+ if (failed_count as f32) > (window_len as f32) * TABLE_CLUSTER_MAX_FAILED_ROW_FRAC {
+ if dbgt {
+ eprintln!("[tbl-cluster] reject: {failed_count}/{window_len} lines unbinnable");
+ }
+ return None;
+ }
+ if rows.len() < TABLE_MIN_ROWS {
+ return None;
+ }
+
+ let absorbed = union_header_from_above(lines, window_start, floor, &tracks);
+ let (start, header) = match absorbed {
+ Some((hstart, header)) => (hstart, Some(header)),
+ None => (window_start, None),
+ };
+ Some(TableRun {
+ start,
+ end: window_end,
+ body_start: window_start,
+ block: Block::Table { header, rows },
+ })
+}
+
fn run_column_count(run: &TableRun) -> Option<usize> {
match &run.block {
Block::Table { header, rows } => header
@@ -1698,6 +2217,7 @@ fn try_merge_pair(a: &TableRun, b: &TableRun, lines: &[ProjectedLine]) -> Option
return Some(TableRun {
start: a.start,
end: b.end,
+ body_start: a.body_start,
block: Block::Table { header, rows },
});
}
@@ -1747,6 +2267,9 @@ fn try_merge_pair(a: &TableRun, b: &TableRun, lines: &[ProjectedLine]) -> Option
return Some(TableRun {
start: a.start,
end: b.end,
+ // A was folded into B's header, so the merged table's body
+ // begins where B's body did.
+ body_start: b.body_start,
block: Block::Table {
header: Some(merged_header),
rows: merged_rows,
@@ -1802,6 +2325,12 @@ const TABLE_GRID_CLUSTER_PT: f32 = 2.0;
/// individual segments with small gaps.
const TABLE_CROSS_TOLERANCE_PT: f32 = 3.0;
+/// Cluster tolerance for ruled column boundaries. Wider than
+/// `TABLE_GRID_CLUSTER_PT` because adjacent cell-border rects draw paired
+/// edges 4-6pt apart; real columns are ≥ ~10pt wide so the mean-centered
+/// merge cannot eat a genuine narrow column.
+const TABLE_COL_BOUNDARY_CLUSTER_PT: f32 = 6.0;
+
/// Reject ruled-table candidates whose empty-cell fraction exceeds this.
/// NOTE: this can't be loosened to recover blank worksheets/forms — a real
/// sparse table (doc 180, a 4-col Version History, ~75% empty) and a spurious
@@ -2021,6 +2550,7 @@ fn build_ruled_table(
page_width: f32,
page_height: f32,
) -> Option<TableRun> {
+ let dbg = std::env::var("LITEPARSE_DEBUG_RULED").is_ok();
// Distinct row y-coords (cluster again — multiple H lines may share a y).
let mut ys: Vec<f32> = h_indices.iter().map(|&i| hs[i].y).collect();
ys.sort_by(|a, b| a.total_cmp(b));
@@ -2028,7 +2558,18 @@ fn build_ruled_table(
let mut xs: Vec<f32> = v_indices.iter().map(|&i| vs[i].x).collect();
xs.sort_by(|a, b| a.total_cmp(b));
- dedup_close(&mut xs, TABLE_GRID_CLUSTER_PT);
+ // Coarser, mean-centered clustering for column boundaries: cell-border
+ // rects contribute paired edges 4-6pt apart that would otherwise become
+ // phantom 5pt "columns" the span splitter then shreds text into.
+ cluster_boundaries(&mut xs, TABLE_COL_BOUNDARY_CLUSTER_PT);
+ if dbg {
+ eprintln!(
+ "[ruled] component: ys={:?} xs={:?} ({} lines in scope)",
+ ys,
+ xs,
+ lines.len()
+ );
+ }
// Need ≥2 row boundaries (1 row) and ≥2 column boundaries (1 col); but
// a 1×1 grid is just a callout box, so also require ≥1 inner divider
@@ -2036,6 +2577,13 @@ fn build_ruled_table(
// are accepted when row evidence is strong enough — extra guards apply
// below after the empty-row collapse.
if ys.len() < 3 || xs.len() < 2 {
+ if dbg {
+ eprintln!(
+ "[ruled] REJECT grid-too-small ys={} xs={}",
+ ys.len(),
+ xs.len()
+ );
+ }
return None;
}
@@ -2052,46 +2600,171 @@ fn build_ruled_table(
if page_width > 0.0 && page_height > 0.0 {
let coverage = (bbox.width / page_width) * (bbox.height / page_height);
if coverage > TABLE_MAX_PAGE_COVERAGE {
+ if dbg {
+ eprintln!("[ruled] REJECT page-coverage {coverage:.2}");
+ }
return None;
}
}
- // Assign each text line to its cell by centroid.
+ // Assign text to cells. A projected line frequently spans several ruled
+ // columns (one baseline = one line), so binning whole lines by centroid
+ // lumps an entire row into one cell and leaves the rest empty — the
+ // empty-cell filter then rejects the real table. Instead bin each line's
+ // raw spans by span center; spans whose x-extent crosses one or more
+ // interior column boundaries are split at the whitespace nearest each
+ // crossed boundary (same interpolation as the inferred-track path).
let mut cells: Vec<Vec<String>> = vec![vec![String::new(); n_cols]; n_rows];
let mut cell_is_bold: Vec<Vec<bool>> = vec![vec![true; n_cols]; n_rows];
let mut cell_has_text: Vec<Vec<bool>> = vec![vec![false; n_cols]; n_rows];
+ // Parallel grid where a span crossing ≥2 column boundaries contributes
+ // its FULL text to every covered cell (colspan semantics) instead of the
+ // whitespace split. Only consulted for header-band flattening — a group
+ // header like "North America" centered over its Revenue/Units sub-columns
+ // must replicate, while the same geometry in a data row is a merged run
+ // that should split.
+ let mut cells_repl: Vec<Vec<String>> = vec![vec![String::new(); n_cols]; n_rows];
+ let mut row_alpha_spanner: Vec<bool> = vec![false; n_rows];
let mut consumed_indices: Vec<usize> = Vec::new();
+ const GRID_X_SLACK_PT: f32 = 6.0;
+
+ let mut push_cell = |cells: &mut Vec<Vec<String>>,
+ cell_has_text: &mut Vec<Vec<bool>>,
+ row: usize,
+ col: usize,
+ txt: &str| {
+ let txt = txt.trim();
+ if txt.is_empty() {
+ return;
+ }
+ if !cells[row][col].is_empty() {
+ cells[row][col].push(' ');
+ }
+ cells[row][col].push_str(txt);
+ cell_has_text[row][col] = true;
+ };
for (idx, line) in lines.iter().enumerate() {
- let cx = line.bbox.x + line.bbox.width * 0.5;
let cy = line.bbox.y + line.bbox.height * 0.5;
- if cy < ys[0] || cy > ys[n_rows] || cx < xs[0] || cx > xs[n_cols] {
+ if cy < ys[0] || cy > ys[n_rows] {
continue;
}
+ // Line-level row bucket. Used by the no-spans fallback and as the
+ // default when a span's own y doesn't resolve. Spans bin by their own
+ // baseline below — a projected line can merge text from several ruled
+ // rows (e.g. a multi-baseline wrapped header emitted as one line).
let row = match find_bucket(&ys, cy) {
Some(r) => r,
None => continue,
};
- let col = match find_bucket(&xs, cx) {
- Some(c) => c,
- None => continue,
- };
- let txt = line.text.trim();
- if txt.is_empty() {
+ if line.text.trim().is_empty() {
continue;
}
- if !cells[row][col].is_empty() {
- cells[row][col].push(' ');
+
+ // Only consume the line when its text sits inside the grid
+ // horizontally — a line poking past the grid edge belongs (at least
+ // partly) to surrounding prose, and consuming it would lose text.
+ let line_x0 = line.bbox.x;
+ let line_x1 = line.bbox.x + line.bbox.width;
+ if line_x0 < xs[0] - GRID_X_SLACK_PT || line_x1 > xs[n_cols] + GRID_X_SLACK_PT {
+ if dbg {
+ eprintln!(
+ "[ruled] skip-overhang row={row} x={line_x0:.0}..{line_x1:.0} grid={:.0}..{:.0} text={:?}",
+ xs[0],
+ xs[n_cols],
+ &line.text.chars().take(60).collect::<String>()
+ );
+ }
+ continue;
}
- cells[row][col].push_str(txt);
- cell_has_text[row][col] = true;
- if !line.all_bold {
- cell_is_bold[row][col] = false;
+
+ let text_spans: Vec<&TextItem> = line
+ .spans
+ .iter()
+ .filter(|s| !s.text.trim().is_empty())
+ .collect();
+ let push_repl = |cells_repl: &mut Vec<Vec<String>>, row: usize, col: usize, txt: &str| {
+ let txt = txt.trim();
+ if txt.is_empty() {
+ return;
+ }
+ let dst = &mut cells_repl[row][col];
+ if !dst.is_empty() {
+ dst.push(' ');
+ }
+ dst.push_str(txt);
+ };
+
+ if text_spans.is_empty() {
+ // No raw text spans (synthetic/OCR lines): old whole-line centroid path.
+ let cx = line.bbox.x + line.bbox.width * 0.5;
+ if let Some(col) = find_bucket(&xs, cx.clamp(xs[0], xs[n_cols])) {
+ push_cell(&mut cells, &mut cell_has_text, row, col, &line.text);
+ push_repl(&mut cells_repl, row, col, &line.text);
+ if !line.all_bold {
+ cell_is_bold[row][col] = false;
+ }
+ consumed_indices.push(idx);
+ }
+ continue;
+ }
+
+ for span in text_spans {
+ // Per-span row: a span carries its own baseline y, which may sit
+ // in a different ruled row than the line centroid.
+ let span_cy = span.y + span.height * 0.5;
+ let row = find_bucket(&ys, span_cy.clamp(ys[0], ys[n_rows])).unwrap_or(row);
+ let sx0 = (span.x).clamp(xs[0], xs[n_cols]);
+ let sx1 = (span.x + span.width).clamp(xs[0], xs[n_cols]);
+ let c_lo = find_bucket(&xs, sx0).unwrap_or(0);
+ let c_hi = find_bucket(&xs, sx1).unwrap_or(n_cols - 1);
+ if c_lo == c_hi {
+ push_cell(&mut cells, &mut cell_has_text, row, c_lo, &span.text);
+ push_repl(&mut cells_repl, row, c_lo, &span.text);
+ if !line.all_bold {
+ cell_is_bold[row][c_lo] = false;
+ }
+ continue;
+ }
+ // Multi-column span: replicate the full text into every covered
+ // cell of the colspan-semantics grid, and flag the row when the
+ // label is alpha-dominant (group headers are words; merged data
+ // runs are mostly digits).
+ for col in c_lo..=c_hi {
+ push_repl(&mut cells_repl, row, col, &span.text);
+ }
+ if is_alpha_dominant(&span.text) {
+ row_alpha_spanner[row] = true;
+ }
+ // Span crosses interior boundaries: split at the whitespace
+ // nearest each crossed boundary x (xs[k] is column k's left
+ // boundary, which is exactly the split target).
+ let covered: Vec<usize> = (c_lo..=c_hi).collect();
+ if let Some(pieces) = split_span_at_anchors(span, &covered, &xs) {
+ for (k, piece) in pieces.iter().enumerate() {
+ push_cell(&mut cells, &mut cell_has_text, row, c_lo + k, piece);
+ if !line.all_bold {
+ cell_is_bold[row][c_lo + k] = false;
+ }
+ }
+ } else {
+ // No whitespace to split on — assign whole span by center.
+ let cx = (span.x + span.width * 0.5).clamp(xs[0], xs[n_cols]);
+ if let Some(col) = find_bucket(&xs, cx) {
+ push_cell(&mut cells, &mut cell_has_text, row, col, &span.text);
+ if !line.all_bold {
+ cell_is_bold[row][col] = false;
+ }
+ }
+ }
}
consumed_indices.push(idx);
}
if consumed_indices.is_empty() {
+ if dbg {
+ eprintln!("[ruled] REJECT no-lines-consumed");
+ }
return None;
}
@@ -2122,6 +2795,10 @@ fn build_ruled_table(
has_text || row_heights[r] >= median_h * 0.8
})
.collect();
+ let kept_row_heights: Vec<f32> = (0..n_rows)
+ .filter(|r| keep[*r])
+ .map(|r| row_heights[r])
+ .collect();
let cells: Vec<Vec<String>> = (0..n_rows)
.filter(|r| keep[*r])
.map(|r| cells[r].clone())
@@ -2134,8 +2811,19 @@ fn build_ruled_table(
.filter(|r| keep[*r])
.map(|r| cell_is_bold[r].clone())
.collect();
+ let cells_repl: Vec<Vec<String>> = (0..n_rows)
+ .filter(|r| keep[*r])
+ .map(|r| cells_repl[r].clone())
+ .collect();
+ let row_alpha_spanner: Vec<bool> = (0..n_rows)
+ .filter(|r| keep[*r])
+ .map(|r| row_alpha_spanner[r])
+ .collect();
let n_rows = cells.len();
if n_rows < 2 {
+ if dbg {
+ eprintln!("[ruled] REJECT rows-after-collapse {n_rows}");
+ }
return None;
}
@@ -2201,6 +2889,16 @@ fn build_ruled_table(
.collect()
})
.collect();
+ let cells_repl: Vec<Vec<String>> = cells_repl
+ .into_iter()
+ .map(|row| {
+ row.into_iter()
+ .enumerate()
+ .filter(|(c, _)| keep_col[*c])
+ .map(|(_, v)| v)
+ .collect()
+ })
+ .collect();
let n_cols = cells.first().map(|r| r.len()).unwrap_or(0);
if n_cols == 0 {
return None;
@@ -2211,6 +2909,130 @@ fn build_ruled_table(
return None;
}
+ // Colspan header-band flattening (Mode 5 / TRM): a sparse top row whose
+ // alpha spanning label covers several columns ("North America" over its
+ // Revenue/Units sub-columns) followed by a dense label row is a stacked
+ // header. Flatten rows 0..=b from the colspan-semantics grid into one
+ // header row (per-column top-to-bottom join), so each column carries its
+ // full layer chain ("North America Revenue") — that chain is what
+ // header-keyed evals (and readers) need.
+ let flattened_header: Option<(Vec<String>, usize)> = {
+ let row_fill =
+ |r: usize| cell_has_text[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;
+ (0..n_rows).find(|r| row_fill(*r) >= 0.9).and_then(|b| {
+ let nonempty = cell_has_text[b].iter().filter(|t| **t).count();
+ let alpha_cells = (0..n_cols)
+ .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))
+ .count();
+ // The bottom header layer may carry digit labels (years like
+ // "2024") but never measurement values. A decimal / % / $ /
+ // dash-placeholder / comma-grouped number in the anchor row means
+ // it's the first DATA row of a table whose real header just
+ // missed the 0.9-fill anchor (colspan header covering <90% of
+ // columns) — folding it would eat a data row (DS5795A_page4).
+ let has_value_cell =
+ (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));
+ let qualifies = (1..=3).contains(&b)
+ && b + 1 < n_rows
+ && (0..b).any(|r| row_alpha_spanner[r])
+ && alpha_cells * 2 >= nonempty
+ && !has_value_cell;
+ if !qualifies {
+ return None;
+ }
+ let header: Vec<String> = (0..n_cols)
+ .map(|c| {
+ let mut parts: Vec<&str> = Vec::new();
+ for row in cells_repl.iter().take(b + 1) {
+ let s = row[c].as_str();
+ if s.is_empty() || parts.last() == Some(&s) {
+ continue;
+ }
+ parts.push(s);
+ }
+ parts.join(" ")
+ })
+ .collect();
+ if header.iter().all(|h| h.is_empty()) {
+ return None;
+ }
+ if dbg {
+ eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");
+ }
+ Some((header, b + 1))
+ })
+ };
+
+ // Stacked-header merge (Mode 5): some generators rule every text baseline
+ // of a wrapped header cell, slicing one logical header row into several
+ // thin sparse rows ("Rated" / "Voltage" / "(VDC)" each in its own band).
+ // When the top k≥2 rows are individually sparse but their union covers
+ // most columns AND a dense data row follows, collapse them into a single
+ // header row (per-column top-to-bottom join).
+ let (cells, cell_has_text, cell_is_bold, n_rows, merged_stacked_header) = {
+ let row_fill = |r: usize, has: &Vec<Vec<bool>>| {
+ has[r].iter().filter(|t| **t).count() as f32 / n_cols as f32
+ };
+ // Anchor on the first fully-dense row (the first real data row).
+ let k = (0..n_rows)
+ .find(|r| row_fill(*r, &cell_has_text) >= 0.9)
+ .unwrap_or(0);
+ let union_cols = (0..n_cols)
+ .filter(|c| (0..k).any(|r| cell_has_text[r][*c]))
+ .count();
+ // Tightness: the band rows must be noticeably shorter than the data
+ // rows below — ruled-per-baseline header bands sit at text leading
+ // (~7pt) while real rows carry cell padding. Without this, a table
+ // whose first body rows are legitimately sparse would get merged.
+ let band_tight = if k >= 2 && k < kept_row_heights.len() {
+ let mut below: Vec<f32> = kept_row_heights[k..].to_vec();
+ below.sort_by(|a, b| a.total_cmp(b));
+ let median_below = below[below.len() / 2];
+ kept_row_heights[..k]
+ .iter()
+ .all(|h| *h <= 0.75 * median_below)
+ } else {
+ false
+ };
+ if flattened_header.is_none()
+ && k >= 2
+ && k < n_rows
+ && band_tight
+ && union_cols as f32 >= 0.7 * n_cols as f32
+ {
+ if dbg {
+ eprintln!("[ruled] stacked-header merge: top {k} rows → 1");
+ }
+ let mut merged_row = vec![String::new(); n_cols];
+ let mut merged_has = vec![false; n_cols];
+ let mut merged_bold = vec![true; n_cols];
+ for r in 0..k {
+ for c in 0..n_cols {
+ if cell_has_text[r][c] {
+ if !merged_row[c].is_empty() {
+ merged_row[c].push(' ');
+ }
+ merged_row[c].push_str(&cells[r][c]);
+ merged_has[c] = true;
+ if !cell_is_bold[r][c] {
+ merged_bold[c] = false;
+ }
+ }
+ }
+ }
+ let mut new_cells = vec![merged_row];
+ let mut new_has = vec![merged_has];
+ let mut new_bold = vec![merged_bold];
+ new_cells.extend(cells[k..].iter().cloned());
+ new_has.extend(cell_has_text[k..].iter().cloned());
+ new_bold.extend(cell_is_bold[k..].iter().cloned());
+ let nr = new_cells.len();
+ (new_cells, new_has, new_bold, nr, true)
+ } else {
+ (cells, cell_has_text, cell_is_bold, n_rows, false)
+ }
+ };
+
let total = n_rows * n_cols;
let empty_count = cell_has_text
.iter()
@@ -2266,26 +3088,43 @@ fn build_ruled_table(
&& header_coverage >= 3
&& header_all_bold
&& dense_inner_col;
- if !col0_spine && !long_prose_table {
+ // A fired colspan header flatten is table evidence on par with a
+ // label spine: it required a ≥0.9-fill alpha label row under an
+ // alpha spanning group label — decorative grids don't have that.
+ // Still subject to the WITH_SPINE ceiling below.
+ let flattened_evidence = flattened_header.is_some();
+ if !col0_spine && !long_prose_table && !flattened_evidence {
+ if dbg {
+ let fills: Vec<usize> = (0..n_rows)
+ .map(|r| cell_has_text[r].iter().filter(|t| **t).count())
+ .collect();
+ eprintln!(
+ "[ruled] REJECT empty-frac {empty_frac:.2} ({n_rows}x{n_cols}, no spine/long-prose) row_fills={fills:?}"
+ );
+ }
return None;
}
if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION_WITH_SPINE && !long_prose_table {
+ if dbg {
+ eprintln!("[ruled] REJECT empty-frac-with-spine {empty_frac:.2}");
+ }
return None;
}
}
- // Header = first row iff every non-empty cell in it is bold.
- let header_qualifies = cell_has_text[0]
- .iter()
- .zip(cell_is_bold[0].iter())
- .all(|(has, bold)| !has || *bold)
- && cell_has_text[0].iter().any(|has| *has);
- let header = if header_qualifies {
- Some(cells[0].clone())
- } else {
- None
+ // Header preference order: flattened colspan band (carries the full
+ // per-column layer chain) > merged stacked-header band > bold first row.
+ let header_qualifies = merged_stacked_header
+ || (cell_has_text[0]
+ .iter()
+ .zip(cell_is_bold[0].iter())
+ .all(|(has, bold)| !has || *bold)
+ && cell_has_text[0].iter().any(|has| *has));
+ let (header, body_start) = match flattened_header {
+ Some((h, bs)) => (Some(h), bs),
+ None if header_qualifies => (Some(cells[0].clone()), 1),
+ None => (None, 0),
};
- let body_start = if header.is_some() { 1 } else { 0 };
let body_rows: Vec<Vec<String>> = cells[body_start..].to_vec();
if body_rows.is_empty() {
return None;
@@ -2298,6 +3137,7 @@ fn build_ruled_table(
Some(TableRun {
start,
end,
+ body_start: start,
block: Block::Table {
header,
rows: body_rows,
@@ -2305,6 +3145,35 @@ fn build_ruled_table(
})
}
+/// Single-link cluster a sorted Vec of boundary coordinates, replacing each
+/// chain of entries (adjacent gap ≤ `tol`) with its mean. Unlike
+/// `dedup_close` (keep-first), the mean centers the boundary between the
+/// paired edges that cell-border rects produce (left border strip + right
+/// border strip of adjacent cells, typically 4-6pt apart), so the split
+/// target lands between cells rather than inside one.
+fn cluster_boundaries(v: &mut Vec<f32>, tol: f32) {
+ if v.len() < 2 {
+ return;
+ }
+ let mut out: Vec<f32> = Vec::with_capacity(v.len());
+ let mut cluster_sum = v[0];
+ let mut cluster_n = 1usize;
+ let mut last = v[0];
+ for &x in v.iter().skip(1) {
+ if x - last <= tol {
+ cluster_sum += x;
+ cluster_n += 1;
+ } else {
+ out.push(cluster_sum / cluster_n as f32);
+ cluster_sum = x;
+ cluster_n = 1;
+ }
+ last = x;
+ }
+ out.push(cluster_sum / cluster_n as f32);
+ *v = out;
+}
+
/// In-place dedup of a sorted Vec, collapsing entries within `tol` to the
/// first of each cluster.
fn dedup_close(v: &mut Vec<f32>, tol: f32) {
@@ -2792,6 +3661,7 @@ mod tests {
let ruled = vec![TableRun {
start: 5,
end: 10,
+ body_start: 5,
block: Block::Table {
header: None,
rows: vec![vec!["ruled".into()]],
@@ -2800,6 +3670,7 @@ mod tests {
let borderless = vec![TableRun {
start: 6,
end: 11,
+ body_start: 6,
block: Block::GridFallback {
lines: vec!["bl".into()],
},
@@ -2850,6 +3721,7 @@ mod tests {
let a = TableRun {
start: 0,
end: 2,
+ body_start: 0,
block: Block::Table {
header: Some(vec!["A".into(), "B".into(), "C".into()]),
rows: vec![vec!["1".into(), "2".into(), "3".into()]],
@@ -2858,6 +3730,7 @@ mod tests {
let b = TableRun {
start: 2,
end: 5,
+ body_start: 2,
block: Block::Table {
header: None,
rows: vec![
@@ -2896,6 +3769,7 @@ mod tests {
let a = TableRun {
start: 0,
end: 2,
+ body_start: 0,
block: Block::Table {
header: None,
rows: vec![
@@ -2907,6 +3781,7 @@ mod tests {
let b = TableRun {
start: 2,
end: 5,
+ body_start: 2,
block: Block::Table {
header: None,
rows: vec![
@@ -2946,6 +3821,7 @@ mod tests {
let a = TableRun {
start: 0,
end: 2,
+ body_start: 0,
block: Block::Table {
header: Some(vec!["A".into(), "B".into(), "C".into()]),
rows: vec![vec!["1".into(), "2".into(), "3".into()]],
@@ -2954,6 +3830,7 @@ mod tests {
let b = TableRun {
start: 2,
end: 4,
+ body_start: 2,
block: Block::Table {
header: None,
rows: vec![
@@ -2977,6 +3854,7 @@ mod tests {
let a = TableRun {
start: 0,
end: 10,
+ body_start: 0,
block: Block::Table {
header: None,
rows: (0..10)
@@ -2987,6 +3865,7 @@ mod tests {
let b = TableRun {
start: 10,
end: 13,
+ body_start: 10,
block: Block::Table {
header: None,
rows: (0..3)
@@ -3031,6 +3910,7 @@ mod tests {
let a = TableRun {
start: 0,
end: 2,
+ body_start: 0,
block: Block::Table {
header: None,
rows: vec![
@@ -3042,6 +3922,7 @@ mod tests {
let b = TableRun {
start: 2,
end: 4,
+ body_start: 2,
block: Block::Table {
header: None,
rows: vec![
@@ -3064,6 +3945,7 @@ mod tests {
let a = TableRun {
start: 0,
end: 2,
+ body_start: 0,
block: Block::Table {
header: None,
rows: vec![
@@ -3075,6 +3957,7 @@ mod tests {
let b = TableRun {
start: 2,
end: 3,
+ body_start: 2,
block: Block::GridFallback {
lines: vec!["fallback".into()],
},
@@ -3106,6 +3989,7 @@ mod tests {
let a = TableRun {
start: 0,
end: 2,
+ body_start: 0,
block: Block::Table {
header: Some(vec!["A".into(), "B".into(), "C".into()]),
rows: vec![vec!["1".into(), "2".into(), "3".into()]],
@@ -3114,6 +3998,7 @@ mod tests {
let b = TableRun {
start: 3,
end: 5,
+ body_start: 3,
block: Block::Table {
header: None,
rows: vec![
@@ -3142,6 +4027,7 @@ mod tests {
let a = TableRun {
start: 0,
end: 2,
+ body_start: 0,
block: Block::Table {
header: None,
rows: vec![
@@ -3153,6 +4039,7 @@ mod tests {
let b = TableRun {
start: 3,
end: 6,
+ body_start: 3,
block: Block::Table {
header: None,
rows: vec![
diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs
index af61a1e..9d7c507 100644
--- a/crates/liteparse/src/projection.rs
+++ b/crates/liteparse/src/projection.rs
@@ -2971,7 +2971,7 @@ const XY_COLUMN_PEAK_BALANCE_RATIO: f32 = 0.4;
/// detector) on sparse tabular layouts.
const XY_COLUMN_MIN_FILL: f32 = 0.55;
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy)]
struct CutCandidate {
axis: CutAxis,
/// Coordinate of the split (y for horizontal cut, x for vertical).
@@ -3309,6 +3309,7 @@ fn xy_find_column_cut(
idxs: &[usize],
bbox: &Rect,
median_h: f32,
+ tabular: &mut bool,
) -> Option<CutCandidate> {
let dbg = std::env::var("LITEPARSE_DEBUG_XY").is_ok();
macro_rules! reject {
@@ -3598,9 +3599,11 @@ fn xy_find_column_cut(
}
const MIN_FILL_HARD_FLOOR: f32 = 0.38;
if min_fill.is_finite() && min_fill < MIN_FILL_HARD_FLOOR {
+ *tabular = true;
reject!("N-column min fill {min_fill:.2} < {MIN_FILL_HARD_FLOOR} (tabular column)");
}
if avg_fill.is_finite() && avg_fill < XY_COLUMN_MIN_FILL {
+ *tabular = true;
reject!(
"N-column avg fill {avg_fill:.2} < {} (tabular)",
XY_COLUMN_MIN_FILL
@@ -3724,7 +3727,10 @@ fn column_cut_x(
figures: &[Rect],
) -> Option<f32> {
xy_find_best_cut(items, idxs, bbox, CutAxis::Vertical, median_h, figures)
- .or_else(|| xy_find_column_cut(items, idxs, bbox, median_h))
+ .or_else(|| {
+ let mut tabular = false;
+ xy_find_column_cut(items, idxs, bbox, median_h, &mut tabular)
+ })
.map(|c| c.position)
}
@@ -3871,8 +3877,15 @@ fn xy_cut_rec(
// figure-straddle layouts) where the column structure is real but the
// density projection can't see it. If either density cut found a
// valley, trust it instead of firing the histogram fallback.
+ // Run the histogram probe unconditionally: besides being a fallback cut
+ // source, its fill gate is the region-level table detector — when it
+ // rejects with "tabular", density V-cuts must not slice the table's
+ // inter-column gutters either (the dominant column-major-table failure:
+ // SERFF_CA page2024 et al).
+ let mut region_tabular = false;
+ let column_full = xy_find_column_cut(items, &idxs, &bbox, median_h, &mut region_tabular);
let column = if v.is_none() && h.is_none() {
- xy_find_column_cut(items, &idxs, &bbox, median_h)
+ column_full
} else {
None
};
@@ -3894,7 +3907,7 @@ fn xy_cut_rec(
// Also probe what the column histogram WOULD return if we ran it
// unconditionally — this is the diagnostic that tells us whether
// reordering V/H/column priority would unlock more column splits.
- let column_probe = xy_find_column_cut(items, &idxs, &bbox, median_h);
+ let column_probe = column_full;
let pad = " ".repeat(depth as usize);
let fmt = |c: &Option<CutCandidate>| {
c.as_ref()
@@ -3952,6 +3965,18 @@ fn xy_cut_rec(
// margin (e.g. peeling a 1-line footer) trips the min-lines guard and
// collapses the whole region to a single leaf — masking the real
// column structure the V-cut would have revealed.
+ // When the histogram fill gate classified this region as tabular, a
+ // density V valley here is a table gutter, not a column boundary —
+ // suppress it so the table reaches the table detectors row-major.
+ let v = if region_tabular && std::env::var("LITEPARSE_DISABLE_TABULAR_V_SUPPRESS").is_err() {
+ if debug_xy && v.is_some() {
+ let pad = " ".repeat(depth as usize);
+ eprintln!("[xy d={depth}]{pad} -> SUPPRESS density V (region tabular)");
+ }
+ None
+ } else {
+ v
+ };
if prefer_h_over_v {
if let Some(hc) = h {
candidates.push(hc);
@@ -3967,14 +3992,14 @@ fn xy_cut_rec(
candidates.push(hc);
}
}
- // Final fallback: unconditional column-histogram probe. Skipped when
- // an existing candidate is already a histogram-derived column cut
- // (score = 1.0e9 from `xy_find_column_cut`).
+ // Final fallback: re-use the unconditional column-histogram probe.
+ // Skipped when an existing candidate is already a histogram-derived
+ // column cut (score = 1.0e9 from `xy_find_column_cut`).
if !candidates
.iter()
.any(|c| c.axis == CutAxis::Vertical && (c.score - 1.0e9).abs() < 1.0)
{
- if let Some(cp) = xy_find_column_cut(items, &idxs, &bbox, median_h) {
+ if let Some(cp) = column_full {
candidates.push(cp);
}
}
diff --git a/crates/pdfium-sys/bindings.rs b/crates/pdfium-sys/bindings.rs
index 14931a4..8ff4e3e 100644
--- a/crates/pdfium-sys/bindings.rs
+++ b/crates/pdfium-sys/bindings.rs
@@ -1169,6 +1169,12 @@ unsafe extern "C" {
name: FPDF_BYTESTRING,
) -> FPDF_PAGEOBJECTMARK;
}
+unsafe extern "C" {
+ pub fn FPDFPageObj_AddExistingMark(
+ page_object: FPDF_PAGEOBJECT,
+ mark: FPDF_PAGEOBJECTMARK,
+ ) -> FPDF_BOOL;
+}
unsafe extern "C" {
pub fn FPDFPageObj_RemoveMark(
page_object: FPDF_PAGEOBJECT,
@@ -1584,6 +1590,9 @@ unsafe extern "C" {
unsafe extern "C" {
pub fn FPDFTextObj_GetFontSize(text: FPDF_PAGEOBJECT, size: *mut f32) -> FPDF_BOOL;
}
+unsafe extern "C" {
+ pub fn FPDFTextObj_SetFontSize(text: FPDF_PAGEOBJECT, size: f32) -> FPDF_BOOL;
+}
unsafe extern "C" {
pub fn FPDFFont_Close(font: FPDF_FONT);
}
@@ -1692,6 +1701,27 @@ unsafe extern "C" {
font_size: f32,
) -> FPDF_GLYPHPATH;
}
+unsafe extern "C" {
+ pub fn FPDFFont_HasToUnicode(font: FPDF_FONT) -> FPDF_BOOL;
+}
+unsafe extern "C" {
+ pub fn FPDFFont_GetCharGlyphName(
+ font: FPDF_FONT,
+ char_code: u32,
+ buffer: *mut ::std::os::raw::c_char,
+ length: ::std::os::raw::c_ulong,
+ ) -> ::std::os::raw::c_ulong;
+}
+unsafe extern "C" {
+ pub fn FPDFFont_GetEncoding(
+ font: FPDF_FONT,
+ buffer: *mut ::std::os::raw::c_char,
+ length: ::std::os::raw::c_ulong,
+ ) -> ::std::os::raw::c_ulong;
+}
+unsafe extern "C" {
+ pub fn FPDFFont_GetCharGlyphIndex(font: FPDF_FONT, char_code: u32) -> ::std::os::raw::c_int;
+}
unsafe extern "C" {
pub fn FPDFGlyphPath_CountGlyphSegments(glyphpath: FPDF_GLYPHPATH) -> ::std::os::raw::c_int;
}
diff --git a/crates/pdfium-sys/build.rs b/crates/pdfium-sys/build.rs
index a230d94..1db5263 100644
--- a/crates/pdfium-sys/build.rs
+++ b/crates/pdfium-sys/build.rs
@@ -3,7 +3,7 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
-const PDFIUM_RELEASE_TAG: &str = "chromium/7870";
+const PDFIUM_RELEASE_TAG: &str = "chromium/7884";
const PDFIUM_RELEASE_URL: &str = "https://github.com/run-llama/pdfium-binaries/releases/download";
fn main() {
diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rs
index 8121a13..c5dbb15 100644
--- a/crates/pdfium-sys/src/dynamic.rs
+++ b/crates/pdfium-sys/src/dynamic.rs
@@ -93,6 +93,9 @@ pub struct PdfiumBindings {
*mut std::os::raw::c_int,
*mut FPDF_BOOL,
) -> FPDF_BOOL,
+ pub FPDFFormObj_CountObjects: unsafe extern "C" fn(FPDF_PAGEOBJECT) -> std::os::raw::c_int,
+ pub FPDFFormObj_GetObject:
+ unsafe extern "C" fn(FPDF_PAGEOBJECT, std::os::raw::c_ulong) -> FPDF_PAGEOBJECT,
pub FPDFPath_CountSegments: unsafe extern "C" fn(FPDF_PAGEOBJECT) -> std::os::raw::c_int,
pub FPDFPath_GetPathSegment:
unsafe extern "C" fn(FPDF_PAGEOBJECT, std::os::raw::c_int) -> FPDF_PATHSEGMENT,
@@ -230,6 +233,21 @@ pub struct PdfiumBindings {
pub FPDFFont_GetGlyphWidth: unsafe extern "C" fn(FPDF_FONT, u32, f32, *mut f32) -> FPDF_BOOL,
pub FPDFFont_GetGlyphWidthFromCharCode:
unsafe extern "C" fn(FPDF_FONT, u32, f32, *mut f32) -> FPDF_BOOL,
+ pub FPDFFont_HasToUnicode: unsafe extern "C" fn(FPDF_FONT) -> FPDF_BOOL,
+ pub FPDFFont_GetCharGlyphName: unsafe extern "C" fn(
+ FPDF_FONT,
+ u32,
+ *mut std::os::raw::c_char,
+ std::os::raw::c_ulong,
+ ) -> std::os::raw::c_ulong,
+ pub FPDFFont_GetEncoding: unsafe extern "C" fn(
+ FPDF_FONT,
+ *mut std::os::raw::c_char,
+ std::os::raw::c_ulong,
+ ) -> std::os::raw::c_ulong,
+ pub FPDFFont_GetCharGlyphIndex: unsafe extern "C" fn(FPDF_FONT, u32) -> std::os::raw::c_int,
+ pub FPDFFont_GetFontData:
+ unsafe extern "C" fn(FPDF_FONT, *mut u8, usize, *mut usize) -> FPDF_BOOL,
// -- Outline (bookmarks) --
pub FPDFBookmark_GetFirstChild:
@@ -324,6 +342,8 @@ impl PdfiumBindings {
FPDFPageObj_GetFillColor: load_fn!(lib, "FPDFPageObj_GetFillColor"),
FPDFPageObj_GetStrokeWidth: load_fn!(lib, "FPDFPageObj_GetStrokeWidth"),
FPDFPath_GetDrawMode: load_fn!(lib, "FPDFPath_GetDrawMode"),
+ FPDFFormObj_CountObjects: load_fn!(lib, "FPDFFormObj_CountObjects"),
+ FPDFFormObj_GetObject: load_fn!(lib, "FPDFFormObj_GetObject"),
FPDFPath_CountSegments: load_fn!(lib, "FPDFPath_CountSegments"),
FPDFPath_GetPathSegment: load_fn!(lib, "FPDFPath_GetPathSegment"),
FPDFPathSegment_GetPoint: load_fn!(lib, "FPDFPathSegment_GetPoint"),
@@ -367,6 +387,11 @@ impl PdfiumBindings {
FPDFFont_GetDescent: load_fn!(lib, "FPDFFont_GetDescent"),
FPDFFont_GetGlyphWidth: load_fn!(lib, "FPDFFont_GetGlyphWidth"),
FPDFFont_GetGlyphWidthFromCharCode: load_fn!(lib, "FPDFFont_GetGlyphWidthFromCharCode"),
+ FPDFFont_HasToUnicode: load_fn!(lib, "FPDFFont_HasToUnicode"),
+ FPDFFont_GetCharGlyphName: load_fn!(lib, "FPDFFont_GetCharGlyphName"),
+ FPDFFont_GetEncoding: load_fn!(lib, "FPDFFont_GetEncoding"),
+ FPDFFont_GetCharGlyphIndex: load_fn!(lib, "FPDFFont_GetCharGlyphIndex"),
+ FPDFFont_GetFontData: load_fn!(lib, "FPDFFont_GetFontData"),
FPDFBookmark_GetFirstChild: load_fn!(lib, "FPDFBookmark_GetFirstChild"),
FPDFBookmark_GetNextSibling: load_fn!(lib, "FPDFBookmark_GetNextSibling"),
diff --git a/crates/pdfium/src/font.rs b/crates/pdfium/src/font.rs
index 7180881..181c9be 100644
--- a/crates/pdfium/src/font.rs
+++ b/crates/pdfium/src/font.rs
@@ -130,4 +130,114 @@ impl Font {
};
if ok != 0 { Some(width) } else { None }
}
+
+ /// Whether the font dictionary defines a /ToUnicode CMap. When false, the
+ /// unicode values PDFium reports for this font's chars are derived from
+ /// the encoding alone and may be garbage for custom/Identity encodings.
+ pub fn has_to_unicode(&self) -> bool {
+ unsafe { ffi!(FPDFFont_HasToUnicode(self.handle)) != 0 }
+ }
+
+ /// Get the font's /Encoding name ("WinAnsiEncoding", "Identity-H", ...),
+ /// the /BaseEncoding name when /Encoding is a dict, or "Custom" for
+ /// font-private encodings.
+ pub fn encoding(&self) -> Option<String> {
+ let len =
+ unsafe { ffi!(FPDFFont_GetEncoding(self.handle, std::ptr::null_mut(), 0)) } as usize;
+ if len == 0 {
+ return None;
+ }
+ let mut buf: Vec<u8> = vec![0; len];
+ let written = unsafe {
+ ffi!(FPDFFont_GetEncoding(
+ self.handle,
+ buf.as_mut_ptr() as *mut std::ffi::c_char,
+ len as u64 as _,
+ ))
+ } as usize;
+ if written == 0 {
+ return None;
+ }
+ let str_len = if buf[written - 1] == 0 {
+ written - 1
+ } else {
+ written
+ };
+ Some(String::from_utf8_lossy(&buf[..str_len]).into_owned())
+ }
+
+ /// Get the PostScript glyph name the font assigns to a raw char code
+ /// (from /Encoding /Differences, falling back to the embedded font
+ /// program's glyph name table). Resolve against the Adobe Glyph List to
+ /// recover unicode when /ToUnicode is missing.
+ pub fn char_glyph_name(&self, char_code: u32) -> Option<String> {
+ let len = unsafe {
+ ffi!(FPDFFont_GetCharGlyphName(
+ self.handle,
+ char_code,
+ std::ptr::null_mut(),
+ 0
+ ))
+ } as usize;
+ if len == 0 {
+ return None;
+ }
+ let mut buf: Vec<u8> = vec![0; len];
+ let written = unsafe {
+ ffi!(FPDFFont_GetCharGlyphName(
+ self.handle,
+ char_code,
+ buf.as_mut_ptr() as *mut std::ffi::c_char,
+ len as u64 as _,
+ ))
+ } as usize;
+ if written == 0 {
+ return None;
+ }
+ let str_len = if buf[written - 1] == 0 {
+ written - 1
+ } else {
+ written
+ };
+ Some(String::from_utf8_lossy(&buf[..str_len]).into_owned())
+ }
+
+ /// Get the glyph index in the embedded font program for a raw char code.
+ /// Pair with glyph-path rendering for a per-glyph OCR fallback.
+ pub fn char_glyph_index(&self, char_code: u32) -> Option<u32> {
+ let idx = unsafe { ffi!(FPDFFont_GetCharGlyphIndex(self.handle, char_code)) };
+ if idx >= 0 { Some(idx as u32) } else { None }
+ }
+
+ /// Get the embedded font program bytes (decompressed FontFile/2/3 stream,
+ /// or the substitute font data for non-embedded fonts).
+ pub fn font_data(&self) -> Option<Vec<u8>> {
+ let mut size: usize = 0;
+ let ok = unsafe {
+ ffi!(FPDFFont_GetFontData(
+ self.handle,
+ std::ptr::null_mut(),
+ 0,
+ &mut size
+ ))
+ };
+ if ok == 0 || size == 0 {
+ return None;
+ }
+ let mut buf: Vec<u8> = vec![0; size];
+ let mut written: usize = 0;
+ let ok = unsafe {
+ ffi!(FPDFFont_GetFontData(
+ self.handle,
+ buf.as_mut_ptr(),
+ buf.len(),
+ &mut written
+ ))
+ };
+ if ok == 0 || written == 0 {
+ return None;
+ }
+ buf.truncate(written);
+ Some(buf)
+ }
}
diff --git a/crates/pdfium/src/page.rs b/crates/pdfium/src/page.rs
index 97777dd..acb37b3 100644
--- a/crates/pdfium/src/page.rs
+++ b/crates/pdfium/src/page.rs
@@ -279,127 +279,201 @@ impl Page<'_> {
/// Enumerate vector path objects on this page. Segment points are
/// transformed into viewport space (top-left origin, 72 DPI) by composing
- /// the object's matrix with the page→viewport transform.
+ /// the object's matrix with the page→viewport transform. Recurses into
+ /// Form XObjects (composing each form's matrix) — table rules and other
+ /// vector art are frequently wrapped in a form container, invisible to a
+ /// top-level-only walk.
pub fn path_objects(&self, view_box: &RectF) -> Vec<PathObject> {
let vp = self.viewport_transform(view_box);
let obj_count = unsafe { ffi!(FPDFPage_CountObjects(self.handle)) };
let mut out = Vec::new();
+ let identity = pdfium_sys::FS_MATRIX {
+ a: 1.0,
+ b: 0.0,
+ c: 0.0,
+ d: 1.0,
+ e: 0.0,
+ f: 0.0,
+ };
for i in 0..obj_count {
let obj = unsafe { ffi!(FPDFPage_GetObject(self.handle, i)) };
if obj.is_null() {
continue;
}
- let obj_type = unsafe { ffi!(FPDFPageObj_GetType(obj)) };
- if obj_type != pdfium_sys::FPDF_PAGEOBJ_PATH as i32 {
- continue;
- }
+ collect_path_objects(obj, &identity, &vp, 0, &mut out);
+ }
+ out
+ }
+}
- // Object → page matrix. Defaults to identity when not available.
- let mut m = pdfium_sys::FS_MATRIX {
- a: 1.0,
- b: 0.0,
- c: 0.0,
- d: 1.0,
- e: 0.0,
- f: 0.0,
- };
- unsafe { ffi!(FPDFPageObj_GetMatrix(obj, &mut m)) };
-
- // Bounds are reported in page space already accounting for the
- // matrix — just lift to viewport.
- let mut left = 0.0f32;
- let mut bottom = 0.0f32;
- let mut right = 0.0f32;
- let mut top = 0.0f32;
- let ok = unsafe {
- ffi!(FPDFPageObj_GetBounds(
- obj,
- &mut left,
- &mut bottom,
- &mut right,
- &mut top
- ))
- };
- if ok == 0 {
+const FS_IDENTITY: pdfium_sys::FS_MATRIX = pdfium_sys::FS_MATRIX {
+ a: 1.0,
+ b: 0.0,
+ c: 0.0,
+ d: 1.0,
+ e: 0.0,
+ f: 0.0,
+};
+
+/// Compose two affine matrices: `result(p) = outer(inner(p))`.
+fn compose_matrix(
+ outer: &pdfium_sys::FS_MATRIX,
+ inner: &pdfium_sys::FS_MATRIX,
+) -> pdfium_sys::FS_MATRIX {
+ pdfium_sys::FS_MATRIX {
+ a: outer.a * inner.a + outer.c * inner.b,
+ b: outer.b * inner.a + outer.d * inner.b,
+ c: outer.a * inner.c + outer.c * inner.d,
+ d: outer.b * inner.c + outer.d * inner.d,
+ e: outer.a * inner.e + outer.c * inner.f + outer.e,
+ f: outer.b * inner.e + outer.d * inner.f + outer.f,
+ }
+}
+
+/// Recursively collect path objects, descending into Form XObjects. `parent`
+/// is the accumulated form matrix mapping this object's content space into
+/// page space (identity at the top level).
+fn collect_path_objects(
+ obj: pdfium_sys::FPDF_PAGEOBJECT,
+ parent: &pdfium_sys::FS_MATRIX,
+ vp: &ViewportTransform,
+ depth: usize,
+ out: &mut Vec<PathObject>,
+) {
+ const MAX_FORM_DEPTH: usize = 6;
+ let obj_type = unsafe { ffi!(FPDFPageObj_GetType(obj)) };
+
+ if obj_type == pdfium_sys::FPDF_PAGEOBJ_FORM as i32 {
+ if depth >= MAX_FORM_DEPTH {
+ return;
+ }
+ let mut fm = FS_IDENTITY;
+ unsafe { ffi!(FPDFPageObj_GetMatrix(obj, &mut fm)) };
+ let combined = compose_matrix(parent, &fm);
+ let n = unsafe { ffi!(FPDFFormObj_CountObjects(obj)) };
+ for i in 0..n {
+ let child = unsafe { ffi!(FPDFFormObj_GetObject(obj, i as std::os::raw::c_ulong)) };
+ if child.is_null() {
continue;
}
- let bbox = vp.transform_bounds(&RectF {
- left,
- top,
- right,
- bottom,
- });
-
- // Draw mode → is_filled / is_stroked.
- let mut fill_mode = 0i32;
- let mut stroke_bool = 0i32;
- let dm_ok =
- unsafe { ffi!(FPDFPath_GetDrawMode(obj, &mut fill_mode, &mut stroke_bool)) };
- let (is_filled, is_stroked) = if dm_ok != 0 {
- (
- fill_mode != pdfium_sys::FPDF_FILLMODE_NONE as i32,
- stroke_bool != 0,
- )
- } else {
- (false, false)
- };
-
- // Colors are reported as RGBA channels in 0..=255 cuint.
- let stroke_color = read_color(|r, g, b, a| unsafe {
- ffi!(FPDFPageObj_GetStrokeColor(obj, r, g, b, a))
- });
- let fill_color =
- read_color(|r, g, b, a| unsafe { ffi!(FPDFPageObj_GetFillColor(obj, r, g, b, a)) });
-
- let mut stroke_width = 0.0f32;
- unsafe { ffi!(FPDFPageObj_GetStrokeWidth(obj, &mut stroke_width)) };
-
- // Walk segments. Points are in the object's local coords; apply
- // matrix → page, then viewport transform.
- let n_segs = unsafe { ffi!(FPDFPath_CountSegments(obj)) };
- let mut segments = Vec::with_capacity(n_segs.max(0) as usize);
- for si in 0..n_segs {
- let seg = unsafe { ffi!(FPDFPath_GetPathSegment(obj, si)) };
- if seg.is_null() {
- continue;
- }
- let mut sx = 0.0f32;
- let mut sy = 0.0f32;
- let pt_ok = unsafe { ffi!(FPDFPathSegment_GetPoint(seg, &mut sx, &mut sy)) };
- if pt_ok == 0 {
- continue;
- }
- let ty = unsafe { ffi!(FPDFPathSegment_GetType(seg)) };
- let close = unsafe { ffi!(FPDFPathSegment_GetClose(seg)) } != 0;
- let kind = match ty as u32 {
- pdfium_sys::FPDF_SEGMENT_MOVETO => SegmentKind::MoveTo,
- pdfium_sys::FPDF_SEGMENT_LINETO => SegmentKind::LineTo,
- pdfium_sys::FPDF_SEGMENT_BEZIERTO => SegmentKind::BezierTo,
- _ => continue,
- };
+ collect_path_objects(child, &combined, vp, depth + 1, out);
+ }
+ return;
+ }
- // Apply object matrix (FS_MATRIX is column-major a/b/c/d/e/f
- // matching the PDF text-matrix convention used elsewhere).
- let page_x = m.a * sx + m.c * sy + m.e;
- let page_y = m.b * sx + m.d * sy + m.f;
- let (x, y) = vp.transform_point(page_x, page_y);
- segments.push(PathSegment { kind, x, y, close });
- }
+ if obj_type != pdfium_sys::FPDF_PAGEOBJ_PATH as i32 {
+ return;
+ }
- out.push(PathObject {
- bbox,
- stroke_color,
- fill_color,
- stroke_width,
- is_stroked,
- is_filled,
- segments,
- });
+ // Object → content-space matrix, composed with the accumulated form
+ // matrix to reach page space.
+ let mut m = FS_IDENTITY;
+ unsafe { ffi!(FPDFPageObj_GetMatrix(obj, &mut m)) };
+ let m = compose_matrix(parent, &m);
+
+ // GetBounds reports bounds in the object's content-stream space (its own
+ // matrix applied, ancestor form matrices not). Lift the corners through
+ // the parent matrix, then to viewport.
+ let mut left = 0.0f32;
+ let mut bottom = 0.0f32;
+ let mut right = 0.0f32;
+ let mut top = 0.0f32;
+ let ok = unsafe {
+ ffi!(FPDFPageObj_GetBounds(
+ obj,
+ &mut left,
+ &mut bottom,
+ &mut right,
+ &mut top
+ ))
+ };
+ if ok == 0 {
+ return;
+ }
+ let corners = [(left, bottom), (left, top), (right, bottom), (right, top)];
+ let mut min_x = f32::INFINITY;
+ let mut min_y = f32::INFINITY;
+ let mut max_x = f32::NEG_INFINITY;
+ let mut max_y = f32::NEG_INFINITY;
+ for (x, y) in corners {
+ let px = parent.a * x + parent.c * y + parent.e;
+ let py = parent.b * x + parent.d * y + parent.f;
+ min_x = min_x.min(px);
+ max_x = max_x.max(px);
+ min_y = min_y.min(py);
+ max_y = max_y.max(py);
+ }
+ let bbox = vp.transform_bounds(&RectF {
+ left: min_x,
+ top: max_y,
+ right: max_x,
+ bottom: min_y,
+ });
+
+ // Draw mode → is_filled / is_stroked.
+ let mut fill_mode = 0i32;
+ let mut stroke_bool = 0i32;
+ let dm_ok = unsafe { ffi!(FPDFPath_GetDrawMode(obj, &mut fill_mode, &mut stroke_bool)) };
+ let (is_filled, is_stroked) = if dm_ok != 0 {
+ (
+ fill_mode != pdfium_sys::FPDF_FILLMODE_NONE as i32,
+ stroke_bool != 0,
+ )
+ } else {
+ (false, false)
+ };
+
+ // Colors are reported as RGBA channels in 0..=255 cuint.
+ let stroke_color =
+ read_color(|r, g, b, a| unsafe { ffi!(FPDFPageObj_GetStrokeColor(obj, r, g, b, a)) });
+ let fill_color =
+ read_color(|r, g, b, a| unsafe { ffi!(FPDFPageObj_GetFillColor(obj, r, g, b, a)) });
+
+ let mut stroke_width = 0.0f32;
+ unsafe { ffi!(FPDFPageObj_GetStrokeWidth(obj, &mut stroke_width)) };
+
+ // Walk segments. Points are in the object's local coords; apply the
+ // composed matrix → page, then viewport transform.
+ let n_segs = unsafe { ffi!(FPDFPath_CountSegments(obj)) };
+ let mut segments = Vec::with_capacity(n_segs.max(0) as usize);
+ for si in 0..n_segs {
+ let seg = unsafe { ffi!(FPDFPath_GetPathSegment(obj, si)) };
+ if seg.is_null() {
+ continue;
+ }
+ let mut sx = 0.0f32;
+ let mut sy = 0.0f32;
+ let pt_ok = unsafe { ffi!(FPDFPathSegment_GetPoint(seg, &mut sx, &mut sy)) };
+ if pt_ok == 0 {
+ continue;
}
+ let ty = unsafe { ffi!(FPDFPathSegment_GetType(seg)) };
+ let close = unsafe { ffi!(FPDFPathSegment_GetClose(seg)) } != 0;
+ let kind = match ty as u32 {
+ pdfium_sys::FPDF_SEGMENT_MOVETO => SegmentKind::MoveTo,
+ pdfium_sys::FPDF_SEGMENT_LINETO => SegmentKind::LineTo,
+ pdfium_sys::FPDF_SEGMENT_BEZIERTO => SegmentKind::BezierTo,
+ _ => continue,
+ };
- out
+ // Apply the composed matrix (FS_MATRIX is column-major a/b/c/d/e/f
+ // matching the PDF text-matrix convention used elsewhere).
+ let page_x = m.a * sx + m.c * sy + m.e;
+ let page_y = m.b * sx + m.d * sy + m.f;
+ let (x, y) = vp.transform_point(page_x, page_y);
+ segments.push(PathSegment { kind, x, y, close });
}
+
+ out.push(PathObject {
+ bbox,
+ stroke_color,
+ fill_color,
+ stroke_width,
+ is_stroked,
+ is_filled,
+ segments,
+ });
}
/// Helper: call a PDFium getter for RGBA color channels and pack into our `Color`.