Skip to content

A fast, helpful, and open-source document parser

final clean up

Logan Markewichabe987282953
diff --git a/crates/liteparse/src/font_cmap.rs b/crates/liteparse/src/font_cmap.rs
new file mode 100644
index 0000000..8fe25dc
--- /dev/null
+++ b/crates/liteparse/src/font_cmap.rs
@@ -0,0 +1,219 @@
+//! Reverse-cmap recovery: parse an embedded sfnt (TrueType/OpenType) font
+//! program's `cmap` table and invert it to a glyph-index → unicode map.
+//!
+//! Used when a font's /ToUnicode is missing or garbage AND its glyph names
+//! are unavailable (typical of CID TrueType subsets). The embedded font's own
+//! character map is the last structural source of truth tying glyphs back to
+//! unicode. Pure Rust, no deps; only formats 4, 12, 6 and 0 are parsed (these
+//! cover essentially all real-world fonts).
+
+use std::collections::HashMap;
+
+/// Build glyph_index → unicode from an sfnt font program. Returns None when
+/// the data is not sfnt (e.g. bare CFF) or has no usable cmap subtable.
+pub fn reverse_cmap(data: &[u8]) -> Option<HashMap<u32, u32>> {
+ let font = sfnt_slice(data)?;
+ let cmap = find_table(font, b"cmap")?;
+
+ let num_subtables = read_u16(cmap, 2)? as usize;
+ // Pick the best unicode subtable: full-repertoire (fmt 12) beats BMP (fmt 4).
+ let mut best: Option<(u32, u32)> = None; // (score, offset)
+ for i in 0..num_subtables {
+ let rec = 4 + i * 8;
+ let platform = read_u16(cmap, rec)?;
+ let encoding = read_u16(cmap, rec + 2)?;
+ let offset = read_u32(cmap, rec + 4)?;
+ // Only true unicode subtables. Mac Roman (1,0) and symbol (3,0) cmaps
+ // encode charcodes, not unicode — reversing them echoes garbage (e.g.
+ // Wingdings (1,0) maps the checkmark glyph back to 'ü').
+ let score = match (platform, encoding) {
+ (3, 10) | (0, 4) | (0, 6) => 4, // UCS-4
+ (3, 1) | (0, 0..=3) => 3, // BMP
+ _ => 0,
+ };
+ if score > 0 && best.is_none_or(|(s, _)| score > s) {
+ best = Some((score, offset));
+ }
+ }
+ let (_, offset) = best?;
+ let sub = cmap.get(offset as usize..)?;
+
+ let mut map: HashMap<u32, u32> = HashMap::new();
+ let mut add = |glyph: u32, unicode: u32| {
+ if glyph == 0 || unicode == 0 {
+ return;
+ }
+ // On collision prefer non-PUA, then the smaller codepoint (ASCII /
+ // canonical forms over compatibility duplicates).
+ match map.entry(glyph) {
+ std::collections::hash_map::Entry::Vacant(e) => {
+ e.insert(unicode);
+ }
+ std::collections::hash_map::Entry::Occupied(mut e) => {
+ let cur = *e.get();
+ let cur_pua = (0xE000..=0xF8FF).contains(&cur);
+ let new_pua = (0xE000..=0xF8FF).contains(&unicode);
+ if (cur_pua && !new_pua) || (cur_pua == new_pua && unicode < cur) {
+ e.insert(unicode);
+ }
+ }
+ }
+ };
+
+ match read_u16(sub, 0)? {
+ 0 => {
+ // Byte encoding table: 256 glyph ids at offset 6
+ for code in 0..256u32 {
+ let g = *sub.get(6 + code as usize)? as u32;
+ add(g, code);
+ }
+ }
+ 4 => {
+ let seg_count = (read_u16(sub, 6)? / 2) as usize;
+ let end_codes = 14;
+ let start_codes = end_codes + seg_count * 2 + 2;
+ let id_deltas = start_codes + seg_count * 2;
+ let id_range_offsets = id_deltas + seg_count * 2;
+ for seg in 0..seg_count {
+ let end = read_u16(sub, end_codes + seg * 2)? as u32;
+ let start = read_u16(sub, start_codes + seg * 2)? as u32;
+ let delta = read_u16(sub, id_deltas + seg * 2)? as u32;
+ let range_offset = read_u16(sub, id_range_offsets + seg * 2)? as usize;
+ if start == 0xFFFF && end == 0xFFFF {
+ continue;
+ }
+ for code in start..=end.min(0xFFFE) {
+ let glyph = if range_offset == 0 {
+ (code + delta) & 0xFFFF
+ } else {
+ // glyphIdArray indexing relative to this rangeOffset slot
+ let slot =
+ id_range_offsets + seg * 2 + range_offset + (code - start) as usize * 2;
+ let g = read_u16(sub, slot)? as u32;
+ if g == 0 { 0 } else { (g + delta) & 0xFFFF }
+ };
+ add(glyph, code);
+ }
+ }
+ }
+ 6 => {
+ let first = read_u16(sub, 6)? as u32;
+ let count = read_u16(sub, 8)? as usize;
+ for i in 0..count {
+ let g = read_u16(sub, 10 + i * 2)? as u32;
+ add(g, first + i as u32);
+ }
+ }
+ 12 => {
+ let n_groups = read_u32(sub, 12)? as usize;
+ for i in 0..n_groups.min(100_000) {
+ let rec = 16 + i * 12;
+ let start = read_u32(sub, rec)?;
+ let end = read_u32(sub, rec + 4)?;
+ let start_glyph = read_u32(sub, rec + 8)?;
+ if end < start || end - start > 0x10000 {
+ continue;
+ }
+ for off in 0..=(end - start) {
+ add(start_glyph + off, start + off);
+ }
+ }
+ }
+ _ => return None,
+ }
+
+ if map.is_empty() { None } else { Some(map) }
+}
+
+/// Resolve TTC wrappers and validate the sfnt magic.
+fn sfnt_slice(data: &[u8]) -> Option<&[u8]> {
+ let tag = data.get(..4)?;
+ if tag == b"ttcf" {
+ let first = read_u32(data, 12)? as usize;
+ let sub = data.get(first..)?;
+ let m = sub.get(..4)?;
+ return (m == [0, 1, 0, 0] || m == b"OTTO" || m == b"true").then_some(sub);
+ }
+ (tag == [0, 1, 0, 0] || tag == b"OTTO" || tag == b"true").then_some(data)
+}
+
+fn find_table<'a>(font: &'a [u8], tag: &[u8; 4]) -> Option<&'a [u8]> {
+ let num_tables = read_u16(font, 4)? as usize;
+ for i in 0..num_tables {
+ let rec = 12 + i * 16;
+ if font.get(rec..rec + 4)? == tag {
+ let offset = read_u32(font, rec + 8)? as usize;
+ let length = read_u32(font, rec + 12)? as usize;
+ return font.get(offset..offset.checked_add(length)?);
+ }
+ }
+ None
+}
+
+fn read_u16(data: &[u8], offset: usize) -> Option<u16> {
+ let b = data.get(offset..offset + 2)?;
+ Some(u16::from_be_bytes([b[0], b[1]]))
+}
+
+fn read_u32(data: &[u8], offset: usize) -> Option<u32> {
+ let b = data.get(offset..offset + 4)?;
+ Some(u32::from_be_bytes([b[0], b[1], b[2], b[3]]))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Build a minimal sfnt with a single format-4 cmap subtable mapping
+ /// 'A'..='C' (0x41..0x43) to glyphs 10..12.
+ fn minimal_format4_font() -> Vec<u8> {
+ // format 4 subtable: one real segment + terminator segment
+ let mut sub: Vec<u8> = Vec::new();
+ sub.extend([0, 4]); // format
+ sub.extend([0, 0]); // length (unchecked)
+ sub.extend([0, 0]); // language
+ sub.extend([0, 4]); // segCountX2 = 4 (2 segments)
+ sub.extend([0, 0, 0, 0, 0, 0]); // searchRange/entrySelector/rangeShift
+ sub.extend([0x00, 0x43, 0xFF, 0xFF]); // endCodes
+ sub.extend([0, 0]); // reservedPad
+ sub.extend([0x00, 0x41, 0xFF, 0xFF]); // startCodes
+ // idDelta: glyph = code + delta mod 65536; 10 - 0x41 = -55 = 0xFFC9
+ sub.extend([0xFF, 0xC9, 0x00, 0x01]); // idDeltas
+ sub.extend([0, 0, 0, 0]); // idRangeOffsets
+
+ let mut cmap: Vec<u8> = Vec::new();
+ cmap.extend([0, 0]); // version
+ cmap.extend([0, 1]); // numTables
+ cmap.extend([0, 3, 0, 1]); // platform 3, encoding 1
+ cmap.extend(12u32.to_be_bytes()); // subtable offset
+ cmap.extend(&sub);
+
+ let mut font: Vec<u8> = Vec::new();
+ font.extend([0, 1, 0, 0]); // sfnt version
+ font.extend([0, 1]); // numTables
+ font.extend([0, 0, 0, 0, 0, 0]); // search fields
+ font.extend(b"cmap");
+ font.extend([0, 0, 0, 0]); // checksum
+ font.extend(28u32.to_be_bytes()); // offset (12 + 16)
+ font.extend((cmap.len() as u32).to_be_bytes());
+ font.extend(&cmap);
+ font
+ }
+
+ #[test]
+ fn parses_format4_and_inverts() {
+ let font = minimal_format4_font();
+ let map = reverse_cmap(&font).unwrap();
+ assert_eq!(map.get(&10), Some(&0x41)); // A
+ assert_eq!(map.get(&11), Some(&0x42)); // B
+ assert_eq!(map.get(&12), Some(&0x43)); // C
+ assert!(!map.contains_key(&0));
+ }
+
+ #[test]
+ fn rejects_non_sfnt() {
+ assert!(reverse_cmap(b"%!PS-AdobeFont").is_none());
+ assert!(reverse_cmap(&[1, 0, 0, 0]).is_none()); // bare CFF header
+ assert!(reverse_cmap(&[]).is_none());
+ }
+}
diff --git a/crates/liteparse/src/glyph_names.rs b/crates/liteparse/src/glyph_names.rs
new file mode 100644
index 0000000..cee16d3
--- /dev/null
+++ b/crates/liteparse/src/glyph_names.rs
@@ -0,0 +1,490 @@
+//! PostScript glyph-name → Unicode resolution (Adobe Glyph List conventions).
+//!
+//! Used to recover text when a font has no usable /ToUnicode CMap. Glyph names
+//! come from the PDF's /Encoding /Differences array or the embedded font
+//! program (via the fork's `FPDFFont_GetCharGlyphName`).
+//!
+//! Implements the AGL "reverse mapping" algorithm:
+//! 1. Strip any variant suffix after the first '.' ("fi.alt" → "fi").
+//! 2. Split on '_' and resolve each component ("f_i" → "fi").
+//! 3. Per component: `uni` + 4n hex digits, `u` + 4-6 hex digits, or a lookup
+//! in the static AGL subset table below.
+
+/// Resolve a glyph name to its Unicode string. Returns None when the name is
+/// meaningless (subset-generated names like "g42", "cid1234") or unknown.
+pub fn resolve_glyph_name(name: &str) -> Option<String> {
+ let base = name.split('.').next()?;
+ if base.is_empty() {
+ return None;
+ }
+ let mut out = String::new();
+ for component in base.split('_') {
+ for c in resolve_component(component)?.chars() {
+ match presentation_form_expansion(c) {
+ Some(s) => out.push_str(s),
+ None => out.push(c),
+ }
+ }
+ }
+ if out.is_empty() { None } else { Some(out) }
+}
+
+/// ASCII expansion for ligature presentation forms, matching the existing
+/// extraction-time ligature handling.
+pub fn presentation_form_expansion(c: char) -> Option<&'static str> {
+ match c {
+ '\u{FB00}' => Some("ff"),
+ '\u{FB01}' => Some("fi"),
+ '\u{FB02}' => Some("fl"),
+ '\u{FB03}' => Some("ffi"),
+ '\u{FB04}' => Some("ffl"),
+ '\u{FB05}' | '\u{FB06}' => Some("st"),
+ _ => None,
+ }
+}
+
+fn resolve_component(comp: &str) -> Option<String> {
+ if comp.is_empty() {
+ return None;
+ }
+
+ // uniXXXX (one or more groups of exactly 4 uppercase hex digits)
+ if let Some(hex) = comp.strip_prefix("uni") {
+ if !hex.is_empty() && hex.len() % 4 == 0 && hex.bytes().all(is_agl_hex_digit) {
+ let mut s = String::new();
+ for group in hex.as_bytes().chunks(4) {
+ let v = u32::from_str_radix(std::str::from_utf8(group).ok()?, 16).ok()?;
+ // Surrogate range is invalid scalar values
+ s.push(char::from_u32(v)?);
+ }
+ return Some(s);
+ }
+ }
+
+ // uXXXX / uXXXXX / uXXXXXX
+ if let Some(hex) = comp.strip_prefix('u') {
+ if (4..=6).contains(&hex.len()) && hex.bytes().all(is_agl_hex_digit) {
+ let v = u32::from_str_radix(hex, 16).ok()?;
+ return char::from_u32(v).map(String::from);
+ }
+ }
+
+ // Static table lookup
+ if let Ok(idx) = AGL_SUBSET.binary_search_by(|(n, _)| (*n).cmp(comp)) {
+ return Some(AGL_SUBSET[idx].1.to_string());
+ }
+
+ None
+}
+
+fn is_agl_hex_digit(b: u8) -> bool {
+ b.is_ascii_digit() || (b'A'..=b'F').contains(&b)
+}
+
+/// Curated subset of the Adobe Glyph List, sorted by name for binary search.
+/// Covers ASCII, Latin-1/Latin Extended accents, ligatures, common punctuation
+/// and symbols, and Greek — the ranges that show up in real-world Differences
+/// arrays and embedded Latin font programs.
+static AGL_SUBSET: &[(&str, &str)] = &[
+ ("A", "A"),
+ ("AE", "Æ"),
+ ("Aacute", "Á"),
+ ("Abreve", "Ă"),
+ ("Acircumflex", "Â"),
+ ("Adieresis", "Ä"),
+ ("Agrave", "À"),
+ ("Alpha", "Α"),
+ ("Amacron", "Ā"),
+ ("Aogonek", "Ą"),
+ ("Aring", "Å"),
+ ("Atilde", "Ã"),
+ ("B", "B"),
+ ("Beta", "Β"),
+ ("C", "C"),
+ ("Cacute", "Ć"),
+ ("Ccaron", "Č"),
+ ("Ccedilla", "Ç"),
+ ("Chi", "Χ"),
+ ("D", "D"),
+ ("Dcaron", "Ď"),
+ ("Dcroat", "Đ"),
+ ("Delta", "Δ"),
+ ("E", "E"),
+ ("Eacute", "É"),
+ ("Ecaron", "Ě"),
+ ("Ecircumflex", "Ê"),
+ ("Edieresis", "Ë"),
+ ("Egrave", "È"),
+ ("Emacron", "Ē"),
+ ("Eogonek", "Ę"),
+ ("Epsilon", "Ε"),
+ ("Eta", "Η"),
+ ("Eth", "Ð"),
+ ("Euro", "€"),
+ ("F", "F"),
+ ("G", "G"),
+ ("Gamma", "Γ"),
+ ("Gbreve", "Ğ"),
+ ("H", "H"),
+ ("I", "I"),
+ ("Iacute", "Í"),
+ ("Icircumflex", "Î"),
+ ("Idieresis", "Ï"),
+ ("Idotaccent", "İ"),
+ ("Igrave", "Ì"),
+ ("Imacron", "Ī"),
+ ("Iota", "Ι"),
+ ("J", "J"),
+ ("K", "K"),
+ ("Kappa", "Κ"),
+ ("L", "L"),
+ ("Lacute", "Ĺ"),
+ ("Lambda", "Λ"),
+ ("Lcaron", "Ľ"),
+ ("Lslash", "Ł"),
+ ("M", "M"),
+ ("Mu", "Μ"),
+ ("N", "N"),
+ ("Nacute", "Ń"),
+ ("Ncaron", "Ň"),
+ ("Ntilde", "Ñ"),
+ ("Nu", "Ν"),
+ ("O", "O"),
+ ("OE", "Œ"),
+ ("Oacute", "Ó"),
+ ("Ocircumflex", "Ô"),
+ ("Odieresis", "Ö"),
+ ("Ograve", "Ò"),
+ ("Ohungarumlaut", "Ő"),
+ ("Omacron", "Ō"),
+ ("Omega", "Ω"),
+ ("Omicron", "Ο"),
+ ("Oslash", "Ø"),
+ ("Otilde", "Õ"),
+ ("P", "P"),
+ ("Phi", "Φ"),
+ ("Pi", "Π"),
+ ("Psi", "Ψ"),
+ ("Q", "Q"),
+ ("R", "R"),
+ ("Racute", "Ŕ"),
+ ("Rcaron", "Ř"),
+ ("Rho", "Ρ"),
+ ("S", "S"),
+ ("Sacute", "Ś"),
+ ("Scaron", "Š"),
+ ("Scedilla", "Ş"),
+ ("Sigma", "Σ"),
+ ("T", "T"),
+ ("Tau", "Τ"),
+ ("Tbar", "Ŧ"),
+ ("Tcaron", "Ť"),
+ ("Theta", "Θ"),
+ ("Thorn", "Þ"),
+ ("U", "U"),
+ ("Uacute", "Ú"),
+ ("Ucircumflex", "Û"),
+ ("Udieresis", "Ü"),
+ ("Ugrave", "Ù"),
+ ("Uhungarumlaut", "Ű"),
+ ("Umacron", "Ū"),
+ ("Uogonek", "Ų"),
+ ("Upsilon", "Υ"),
+ ("Uring", "Ů"),
+ ("V", "V"),
+ ("W", "W"),
+ ("X", "X"),
+ ("Xi", "Ξ"),
+ ("Y", "Y"),
+ ("Yacute", "Ý"),
+ ("Ydieresis", "Ÿ"),
+ ("Z", "Z"),
+ ("Zacute", "Ź"),
+ ("Zcaron", "Ž"),
+ ("Zdotaccent", "Ż"),
+ ("Zeta", "Ζ"),
+ ("a", "a"),
+ ("aacute", "á"),
+ ("abreve", "ă"),
+ ("acircumflex", "â"),
+ ("acute", "´"),
+ ("adieresis", "ä"),
+ ("ae", "æ"),
+ ("agrave", "à"),
+ ("alpha", "α"),
+ ("amacron", "ā"),
+ ("ampersand", "&"),
+ ("aogonek", "ą"),
+ ("approxequal", "≈"),
+ ("aring", "å"),
+ ("asciicircum", "^"),
+ ("asciitilde", "~"),
+ ("asterisk", "*"),
+ ("at", "@"),
+ ("atilde", "ã"),
+ ("b", "b"),
+ ("backslash", "\\"),
+ ("bar", "|"),
+ ("beta", "β"),
+ ("braceleft", "{"),
+ ("braceright", "}"),
+ ("bracketleft", "["),
+ ("bracketright", "]"),
+ ("breve", "˘"),
+ ("brokenbar", "¦"),
+ ("bullet", "•"),
+ ("c", "c"),
+ ("cacute", "ć"),
+ ("caron", "ˇ"),
+ ("ccaron", "č"),
+ ("ccedilla", "ç"),
+ ("cedilla", "¸"),
+ ("cent", "¢"),
+ ("chi", "χ"),
+ ("circumflex", "ˆ"),
+ ("colon", ":"),
+ ("comma", ","),
+ ("copyright", "©"),
+ ("currency", "¤"),
+ ("d", "d"),
+ ("dagger", "†"),
+ ("daggerdbl", "‡"),
+ ("dcaron", "ď"),
+ ("dcroat", "đ"),
+ ("degree", "°"),
+ ("delta", "δ"),
+ ("dieresis", "¨"),
+ ("divide", "÷"),
+ ("dollar", "$"),
+ ("dotaccent", "˙"),
+ ("dotlessi", "ı"),
+ ("e", "e"),
+ ("eacute", "é"),
+ ("ecaron", "ě"),
+ ("ecircumflex", "ê"),
+ ("edieresis", "ë"),
+ ("egrave", "è"),
+ ("eight", "8"),
+ ("ellipsis", "…"),
+ ("emacron", "ē"),
+ ("emdash", "—"),
+ ("endash", "–"),
+ ("eogonek", "ę"),
+ ("epsilon", "ε"),
+ ("equal", "="),
+ ("eta", "η"),
+ ("eth", "ð"),
+ ("exclam", "!"),
+ ("exclamdown", "¡"),
+ ("f", "f"),
+ ("ff", "ff"),
+ ("ffi", "ffi"),
+ ("ffl", "ffl"),
+ ("fi", "fi"),
+ ("five", "5"),
+ ("fl", "fl"),
+ ("florin", "ƒ"),
+ ("four", "4"),
+ ("fraction", "⁄"),
+ ("g", "g"),
+ ("gamma", "γ"),
+ ("gbreve", "ğ"),
+ ("germandbls", "ß"),
+ ("grave", "`"),
+ ("greater", ">"),
+ ("greaterequal", "≥"),
+ ("guillemotleft", "«"),
+ ("guillemotright", "»"),
+ ("guilsinglleft", "‹"),
+ ("guilsinglright", "›"),
+ ("h", "h"),
+ ("hungarumlaut", "˝"),
+ ("hyphen", "-"),
+ ("i", "i"),
+ ("iacute", "í"),
+ ("icircumflex", "î"),
+ ("idieresis", "ï"),
+ ("igrave", "ì"),
+ ("imacron", "ī"),
+ ("infinity", "∞"),
+ ("iota", "ι"),
+ ("j", "j"),
+ ("k", "k"),
+ ("kappa", "κ"),
+ ("l", "l"),
+ ("lacute", "ĺ"),
+ ("lambda", "λ"),
+ ("lcaron", "ľ"),
+ ("less", "<"),
+ ("lessequal", "≤"),
+ ("logicalnot", "¬"),
+ ("lslash", "ł"),
+ ("m", "m"),
+ ("macron", "¯"),
+ ("minus", "−"),
+ ("mu", "μ"),
+ ("multiply", "×"),
+ ("n", "n"),
+ ("nacute", "ń"),
+ ("ncaron", "ň"),
+ ("nine", "9"),
+ ("notequal", "≠"),
+ ("ntilde", "ñ"),
+ ("nu", "ν"),
+ ("numbersign", "#"),
+ ("o", "o"),
+ ("oacute", "ó"),
+ ("ocircumflex", "ô"),
+ ("odieresis", "ö"),
+ ("oe", "œ"),
+ ("ogonek", "˛"),
+ ("ograve", "ò"),
+ ("ohungarumlaut", "ő"),
+ ("omacron", "ō"),
+ ("omega", "ω"),
+ ("omicron", "ο"),
+ ("one", "1"),
+ ("onehalf", "½"),
+ ("onequarter", "¼"),
+ ("onesuperior", "¹"),
+ ("ordfeminine", "ª"),
+ ("ordmasculine", "º"),
+ ("oslash", "ø"),
+ ("otilde", "õ"),
+ ("p", "p"),
+ ("paragraph", "¶"),
+ ("parenleft", "("),
+ ("parenright", ")"),
+ ("partialdiff", "∂"),
+ ("percent", "%"),
+ ("period", "."),
+ ("periodcentered", "·"),
+ ("perthousand", "‰"),
+ ("phi", "φ"),
+ ("pi", "π"),
+ ("plus", "+"),
+ ("plusminus", "±"),
+ ("psi", "ψ"),
+ ("q", "q"),
+ ("question", "?"),
+ ("questiondown", "¿"),
+ ("quotedbl", "\""),
+ ("quotedblbase", "„"),
+ ("quotedblleft", "“"),
+ ("quotedblright", "”"),
+ ("quoteleft", "‘"),
+ ("quoteright", "’"),
+ ("quotesinglbase", "‚"),
+ ("quotesingle", "'"),
+ ("r", "r"),
+ ("racute", "ŕ"),
+ ("radical", "√"),
+ ("rcaron", "ř"),
+ ("registered", "®"),
+ ("rho", "ρ"),
+ ("ring", "˚"),
+ ("s", "s"),
+ ("sacute", "ś"),
+ ("scaron", "š"),
+ ("scedilla", "ş"),
+ ("section", "§"),
+ ("semicolon", ";"),
+ ("seven", "7"),
+ ("sigma", "σ"),
+ ("sigma1", "ς"),
+ ("six", "6"),
+ ("slash", "/"),
+ ("space", " "),
+ ("sterling", "£"),
+ ("summation", "∑"),
+ ("t", "t"),
+ ("tau", "τ"),
+ ("tbar", "ŧ"),
+ ("tcaron", "ť"),
+ ("theta", "θ"),
+ ("thorn", "þ"),
+ ("three", "3"),
+ ("threequarters", "¾"),
+ ("threesuperior", "³"),
+ ("tilde", "˜"),
+ ("trademark", "™"),
+ ("two", "2"),
+ ("twosuperior", "²"),
+ ("u", "u"),
+ ("uacute", "ú"),
+ ("ucircumflex", "û"),
+ ("udieresis", "ü"),
+ ("ugrave", "ù"),
+ ("uhungarumlaut", "ű"),
+ ("umacron", "ū"),
+ ("underscore", "_"),
+ ("uogonek", "ų"),
+ ("upsilon", "υ"),
+ ("uring", "ů"),
+ ("v", "v"),
+ ("w", "w"),
+ ("x", "x"),
+ ("xi", "ξ"),
+ ("y", "y"),
+ ("yacute", "ý"),
+ ("ydieresis", "ÿ"),
+ ("yen", "¥"),
+ ("z", "z"),
+ ("zacute", "ź"),
+ ("zcaron", "ž"),
+ ("zdotaccent", "ż"),
+ ("zero", "0"),
+ ("zeta", "ζ"),
+];
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn agl_subset_is_sorted_and_unique() {
+ for w in AGL_SUBSET.windows(2) {
+ assert!(w[0].0 < w[1].0, "{} >= {}", w[0].0, w[1].0);
+ }
+ }
+
+ #[test]
+ fn resolves_plain_names() {
+ assert_eq!(resolve_glyph_name("germandbls").as_deref(), Some("ß"));
+ assert_eq!(resolve_glyph_name("adieresis").as_deref(), Some("ä"));
+ assert_eq!(resolve_glyph_name("fl").as_deref(), Some("fl"));
+ assert_eq!(resolve_glyph_name("ffi").as_deref(), Some("ffi"));
+ assert_eq!(resolve_glyph_name("A").as_deref(), Some("A"));
+ assert_eq!(resolve_glyph_name("space").as_deref(), Some(" "));
+ }
+
+ #[test]
+ fn resolves_uni_and_u_forms() {
+ assert_eq!(resolve_glyph_name("uni0041").as_deref(), Some("A"));
+ assert_eq!(resolve_glyph_name("uniFB01").as_deref(), Some("fi"));
+ assert_eq!(resolve_glyph_name("uni00410042").as_deref(), Some("AB"));
+ assert_eq!(resolve_glyph_name("u1F600").as_deref(), Some("😀"));
+ assert_eq!(resolve_glyph_name("u0041").as_deref(), Some("A"));
+ // lowercase hex is not AGL-valid
+ assert_eq!(resolve_glyph_name("uni00e9"), None);
+ // surrogate halves are invalid
+ assert_eq!(resolve_glyph_name("uniD800"), None);
+ }
+
+ #[test]
+ fn resolves_compounds_and_suffixes() {
+ assert_eq!(resolve_glyph_name("f_i").as_deref(), Some("fi"));
+ assert_eq!(resolve_glyph_name("f_f_l").as_deref(), Some("ffl"));
+ assert_eq!(resolve_glyph_name("fi.alt").as_deref(), Some("fi"));
+ assert_eq!(resolve_glyph_name("uni0041.sc").as_deref(), Some("A"));
+ }
+
+ #[test]
+ fn rejects_meaningless_names() {
+ assert_eq!(resolve_glyph_name("g42"), None);
+ assert_eq!(resolve_glyph_name("cid1234"), None);
+ assert_eq!(resolve_glyph_name("glyph7"), None);
+ assert_eq!(resolve_glyph_name(""), None);
+ assert_eq!(resolve_glyph_name(".notdef"), None);
+ }
+}
diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rs
index d027f24..9f400c7 100644
--- a/crates/liteparse/src/markdown_layout/classify.rs
+++ b/crates/liteparse/src/markdown_layout/classify.rs
@@ -395,6 +395,73 @@ pub fn classify_page_with_filters(
stitch_regions(blocks, &region_boundaries)
}
+/// Mutable per-line flow state threaded through `classify_region`: the active
+/// paragraph accumulator, the active code run, and the current list run. The
+/// flush/reset/emit methods live here so the (previously 8-parameter) closures
+/// and the six copy-pasted "reset list state" blocks collapse to method calls;
+/// methods on the struct also sidestep the borrow-checker friction of threading
+/// five `&mut Option<…>` cells through a free closure.
+#[derive(Default)]
+struct FlowState {
+ paragraph: Option<ParaAccum>,
+ code: Option<Vec<String>>,
+ list_base_indent: Option<f32>,
+ last_list_item_idx: Option<usize>,
+ last_list_line: Option<usize>,
+}
+
+impl FlowState {
+ /// Drop the active list run. A heading, table, code block, or paragraph
+ /// break all terminate the current list.
+ fn reset_list(&mut self) {
+ self.list_base_indent = None;
+ self.last_list_item_idx = None;
+ self.last_list_line = None;
+ }
+
+ /// Emit the active paragraph (if it has non-blank content) and clear it.
+ fn flush_paragraph(&mut self, blocks: &mut Vec<Block>) {
+ if let Some(acc) = self.paragraph.take()
+ && !acc.raw.trim().is_empty()
+ {
+ blocks.push(paragraph_from_accum(acc));
+ }
+ }
+
+ /// Emit the active code run (if non-empty) and clear it.
+ fn flush_code(&mut self, blocks: &mut Vec<Block>) {
+ if let Some(lines) = self.code.take()
+ && !lines.is_empty()
+ {
+ blocks.push(Block::CodeBlock { lines });
+ }
+ }
+
+ /// Emit any interruptions whose y is at or above `before_y`, flushing the
+ /// active paragraph/code/list state first so each lands as its own block.
+ fn emit_before(
+ &mut self,
+ blocks: &mut Vec<Block>,
+ iter: &mut std::iter::Peekable<std::vec::IntoIter<(f32, Interruption)>>,
+ before_y: f32,
+ ) {
+ while let Some((y, _)) = iter.peek() {
+ if *y > before_y {
+ break;
+ }
+ let (_, kind) = iter.next().unwrap();
+ self.flush_paragraph(blocks);
+ self.flush_code(blocks);
+ self.reset_list();
+ blocks.push(match kind {
+ Interruption::Hr => Block::HorizontalRule,
+ Interruption::Figure(r) => Block::Figure { id: r.id },
+ Interruption::Table(b) => b,
+ });
+ }
+ }
+}
+
/// Classify the lines of a single xy_cut leaf into a sequence of blocks. All
/// per-line state (active paragraph, code run, list run, heading_run) is local
/// to this call — nothing crosses a leaf boundary except through the explicit
@@ -413,11 +480,7 @@ fn classify_region(
precomputed_tables: Option<Vec<super::tables::TableRun>>,
) -> Vec<Block> {
let mut blocks: Vec<Block> = Vec::new();
- let mut paragraph: Option<ParaAccum> = None;
- let mut code: Option<Vec<String>> = None;
- let mut list_base_indent: Option<f32> = None;
- let mut last_list_item_idx: Option<usize> = None;
- let mut last_list_line: Option<usize> = None;
+ let mut state = FlowState::default();
let mut heading_run: Option<(u8, usize)> = None;
// Track whether we've already emitted a TOC title on this page. Once seen,
// any subsequent line matching `is_toc_title` (e.g. "Index", "List of
@@ -425,21 +488,6 @@ fn classify_region(
// and must stay suppressed.
let mut toc_title_emitted = false;
- let flush_paragraph = |blocks: &mut Vec<Block>, p: Option<ParaAccum>| {
- if let Some(acc) = p
- && !acc.raw.trim().is_empty()
- {
- blocks.push(paragraph_from_accum(acc));
- }
- };
- let flush_code = |blocks: &mut Vec<Block>, c: Option<Vec<String>>| {
- if let Some(lines) = c
- && !lines.is_empty()
- {
- blocks.push(Block::CodeBlock { lines });
- }
- };
-
// Per-region table detection. Indices are local to `lines`. Page-level
// graphics are still consulted for ruled-table detection because path
// objects are page-coordinate; the detector intersects them against the
@@ -475,34 +523,6 @@ fn classify_region(
region_interruptions.sort_by(|a, b| a.0.total_cmp(&b.0));
let mut interruptions = region_interruptions.into_iter().peekable();
- // Emit any interruptions whose y is at or above `before_y`, flushing the
- // active paragraph/code/list state first so each lands as its own block.
- let emit_before = |blocks: &mut Vec<Block>,
- paragraph: &mut Option<ParaAccum>,
- code: &mut Option<Vec<String>>,
- list_base: &mut Option<f32>,
- last_item: &mut Option<usize>,
- last_line: &mut Option<usize>,
- iter: &mut std::iter::Peekable<std::vec::IntoIter<(f32, Interruption)>>,
- before_y: f32| {
- while let Some((y, _)) = iter.peek() {
- if *y > before_y {
- break;
- }
- let (_, kind) = iter.next().unwrap();
- flush_paragraph(blocks, paragraph.take());
- flush_code(blocks, code.take());
- *list_base = None;
- *last_item = None;
- *last_line = None;
- blocks.push(match kind {
- Interruption::Hr => Block::HorizontalRule,
- Interruption::Figure(r) => Block::Figure { id: r.id },
- Interruption::Table(b) => b,
- });
- }
- };
-
let mut idx = 0;
while idx < lines.len() {
if let Some(run) = table_iter.peek()
@@ -510,21 +530,10 @@ fn classify_region(
{
// Flush any interruptions above this table's top edge first.
let table_top = lines[run.start].bbox.y;
- emit_before(
- &mut blocks,
- &mut paragraph,
- &mut code,
- &mut list_base_indent,
- &mut last_list_item_idx,
- &mut last_list_line,
- &mut interruptions,
- table_top,
- );
- flush_paragraph(&mut blocks, paragraph.take());
- flush_code(&mut blocks, code.take());
- list_base_indent = None;
- last_list_item_idx = None;
- last_list_line = None;
+ state.emit_before(&mut blocks, &mut interruptions, table_top);
+ state.flush_paragraph(&mut blocks);
+ state.flush_code(&mut blocks);
+ state.reset_list();
let run = table_iter.next().unwrap();
blocks.push(run.block);
idx = run.end;
@@ -533,16 +542,7 @@ fn classify_region(
let line_idx = idx;
let line = &lines[line_idx];
// Emit any interruptions that fall above this line.
- emit_before(
- &mut blocks,
- &mut paragraph,
- &mut code,
- &mut list_base_indent,
- &mut last_list_item_idx,
- &mut last_list_line,
- &mut interruptions,
- line.bbox.y,
- );
+ state.emit_before(&mut blocks, &mut interruptions, line.bbox.y);
idx += 1;
let text = line.text.trim();
if text.is_empty() {
@@ -571,16 +571,16 @@ fn classify_region(
// still becomes code. Mono content also wouldn't make a useful
// heading.
if line.all_mono {
- flush_paragraph(&mut blocks, paragraph.take());
- list_base_indent = None;
- last_list_item_idx = None;
- last_list_line = None;
- code.get_or_insert_with(Vec::new)
+ state.flush_paragraph(&mut blocks);
+ state.reset_list();
+ state
+ .code
+ .get_or_insert_with(Vec::new)
.push(line.text.trim_end().to_string());
continue;
}
// Any non-mono line ends the current code block (if any).
- flush_code(&mut blocks, code.take());
+ state.flush_code(&mut blocks);
// Priority chain: tagged-PDF struct tree → outline → font-size map.
let tagged_level = struct_heading_level(line, &page.struct_nodes);
@@ -614,10 +614,11 @@ fn classify_region(
// Outline / struct-tree levels are explicit and bypass this guard.
let size_level = size_level.filter(|_| {
let starts_lower = text.chars().next().is_some_and(|c| c.is_lowercase());
- let prev = paragraph
+ let prev = state
+ .paragraph
.as_ref()
.map(|p| &p.last)
- .or(last_list_line.map(|i| &lines[i]));
+ .or(state.last_list_line.map(|i| &lines[i]));
// A previous line ending in a mid-word hyphen wrap means this
// line is its continuation regardless of capitalization
// ("…SOLAR 10.7 Billion-" / "Parameter Model: We have…").
@@ -736,7 +737,7 @@ fn classify_region(
if combined_chars > HEADING_MAX_TEXT_CHARS {
let demoted = std::mem::take(htext);
blocks.pop();
- paragraph = Some(ParaAccum {
+ state.paragraph = Some(ParaAccum {
raw: demoted.clone(),
inline: demoted,
last: lines[*run_idx].clone(),
@@ -751,10 +752,8 @@ fn classify_region(
}
}
if !demoted_heading {
- flush_paragraph(&mut blocks, paragraph.take());
- list_base_indent = None;
- last_list_item_idx = None;
- last_list_line = None;
+ state.flush_paragraph(&mut blocks);
+ state.reset_list();
if debug {
eprintln!(
"[MD heading-emit size/outline] h{} idx={} '{}' size={:.2}",
@@ -787,16 +786,15 @@ fn classify_region(
&& looks_like_numbered_bold_heading(
line,
rest,
- paragraph
+ state
+ .paragraph
.as_ref()
.map(|p| &p.last)
- .or(last_list_line.map(|i| &lines[i])),
+ .or(state.last_list_line.map(|i| &lines[i])),
)
{
- flush_paragraph(&mut blocks, paragraph.take());
- list_base_indent = None;
- last_list_item_idx = None;
- last_list_line = None;
+ state.flush_paragraph(&mut blocks);
+ state.reset_list();
let level = (heading_map.len() as u8 + 1).clamp(1, MAX_HEADING_LEVELS as u8);
blocks.push(Block::Heading {
level,
@@ -804,13 +802,13 @@ fn classify_region(
});
continue;
}
- flush_paragraph(&mut blocks, paragraph.take());
- let base = *list_base_indent.get_or_insert(line.indent_x);
+ state.flush_paragraph(&mut blocks);
+ let base = *state.list_base_indent.get_or_insert(line.indent_x);
let level = (((line.indent_x - base) / LIST_INDENT_STEP_PT)
.round()
.max(0.0)) as u8;
- last_list_item_idx = Some(blocks.len());
- last_list_line = Some(line_idx);
+ state.last_list_item_idx = Some(blocks.len());
+ state.last_list_line = Some(line_idx);
// Render the list-item text via the inline pipeline so per-span
// emphasis surfaces. We strip the marker from `rest` (already
// done by `parse_list_marker`), but emphasis lives on `line.spans`,
@@ -831,8 +829,8 @@ fn classify_region(
// Continuation of a list item: same gap/font rules as paragraphs.
// Footnote-style continuations often left-flush below the marker's
// hanging indent, so we don't require indent ≥ marker indent.
- if let Some(item_idx) = last_list_item_idx
- && let Some(prev_idx) = last_list_line
+ if let Some(item_idx) = state.last_list_item_idx
+ && let Some(prev_idx) = state.last_list_line
&& continues_paragraph(&lines[prev_idx], line)
&& let Some(Block::ListItem {
text: prev_text, ..
@@ -842,7 +840,7 @@ fn classify_region(
// inline-styled continuation.
let cont_inline = render_line_inline(line);
append_inline_continuation(prev_text, text, &cont_inline);
- last_list_line = Some(line_idx);
+ state.last_list_line = Some(line_idx);
continue;
}
@@ -851,16 +849,15 @@ fn classify_region(
// "1 Introduction"); without this rule they look just like a bold
// first sentence of a paragraph. Runs after list-marker detection so
// bold bullet items stay as list items.
- let prev_for_gap = paragraph
+ let prev_for_gap = state
+ .paragraph
.as_ref()
.map(|p| &p.last)
- .or(last_list_line.map(|i| &lines[i]));
+ .or(state.last_list_line.map(|i| &lines[i]));
let next_for_gap = lines.get(idx);
if !toc_suppress && looks_like_bold_heading(line, prev_for_gap, next_for_gap) {
- flush_paragraph(&mut blocks, paragraph.take());
- list_base_indent = None;
- last_list_item_idx = None;
- last_list_line = None;
+ state.flush_paragraph(&mut blocks);
+ state.reset_list();
// Level: one deeper than the deepest size-based level we already
// have. With an empty heading_map this lands on H1; with a full
// 6-level map it caps at H6.
@@ -884,19 +881,17 @@ fn classify_region(
continue;
}
- match paragraph.as_mut() {
+ match state.paragraph.as_mut() {
Some(acc) if continues_paragraph(&acc.last, line) => {
append_to_paragraph(acc, line);
}
_ => {
- flush_paragraph(&mut blocks, paragraph.take());
- list_base_indent = None;
- last_list_item_idx = None;
- last_list_line = None;
+ state.flush_paragraph(&mut blocks);
+ state.reset_list();
let inline = render_line_inline(line);
let raw = collapse_whitespace(text);
let uniform = line_uniform_style(line).map(|s| (s.bold, s.italic));
- paragraph = Some(ParaAccum {
+ state.paragraph = Some(ParaAccum {
raw,
inline,
last: line.clone(),
@@ -906,19 +901,10 @@ fn classify_region(
}
}
- flush_paragraph(&mut blocks, paragraph.take());
- flush_code(&mut blocks, code.take());
+ state.flush_paragraph(&mut blocks);
+ state.flush_code(&mut blocks);
// Flush any trailing interruptions that sat below the last text line.
- emit_before(
- &mut blocks,
- &mut paragraph,
- &mut code,
- &mut list_base_indent,
- &mut last_list_item_idx,
- &mut last_list_line,
- &mut interruptions,
- f32::INFINITY,
- );
+ state.emit_before(&mut blocks, &mut interruptions, f32::INFINITY);
blocks
}
diff --git a/crates/liteparse/src/markdown_layout/cross_region.rs b/crates/liteparse/src/markdown_layout/cross_region.rs
new file mode 100644
index 0000000..d67d76f
--- /dev/null
+++ b/crates/liteparse/src/markdown_layout/cross_region.rs
@@ -0,0 +1,767 @@
+//! Cross-region table re-merge.
+//!
+//! When the XY-cut commits a V-cut *through* a table (the column gutter of the
+//! table looks like a layout gutter), the table's lines land in two or more
+//! sibling leaves and no per-region detector ever sees the rows whole. This
+//! pass runs before region grouping: it finds side-by-side leaf groups whose
+//! baselines align across the cut, fuses the same-baseline lines back into
+//! single rows, and validates that the fused zone actually classifies as a
+//! table. Only on successful validation are the original lines replaced — a
+//! failed candidate leaves the page untouched, so the V-cut's wins on
+//! multi-column prose are preserved.
+
+use super::blocks::Block;
+use super::tables::{TableRun, detect_tables};
+use crate::types::ProjectedLine;
+
+/// Minimum fraction of the smaller side's y-band that must overlap the other
+/// side's. Side-by-side table halves overlap almost fully; a sidebar next to
+/// the tail of a column does not.
+const CR_MIN_Y_OVERLAP_FRAC: f32 = 0.5;
+/// Minimum horizontal gap between the two sides (the sliced gutter).
+const CR_MIN_COL_GAP_PT: f32 = 8.0;
+/// Baseline-cluster tolerance as a fraction of the median line height.
+const CR_ROW_TOL_FACTOR: f32 = 0.6;
+/// At least this many baselines must align across the cut.
+const CR_MIN_ALIGNED_ROWS: usize = 3;
+/// ...and they must account for this fraction of the sparser side's lines.
+const CR_MIN_ALIGNED_FRAC: f32 = 0.4;
+/// A side "looks like prose" when it has at least this many lines...
+const CR_PROSE_MIN_LINES: usize = 3;
+/// ...whose median width fills this fraction of the side's width...
+const CR_PROSE_WIDTH_FRAC: f32 = 0.7;
+/// ...and whose summed line heights fill this fraction of the y-band.
+const CR_PROSE_VFILL: f32 = 0.55;
+/// A prose side is dominated by single-cell lines: when this fraction or more
+/// of a side's lines split into ≥2 cells, it's a sliced table half (row
+/// fragments with internal column gaps), not flowing text.
+const CR_PROSE_MAX_MULTI_CELL_FRAC: f32 = 0.3;
+/// Validation A: the detected table run(s) must cover this fraction of the
+/// fused lines.
+const CR_VALIDATE_MIN_COVERAGE: f32 = 0.6;
+/// Validation A: reject a detected run when more than this fraction of its
+/// non-empty cells run ≥ `CR_LONG_CELL_WORDS` words. Fused newspaper /
+/// reference columns align into convincing grids, but their "cells" are
+/// sentence fragments; real table cells are short.
+const CR_VALIDATE_MAX_LONG_CELL_FRAC: f32 = 0.3;
+const CR_LONG_CELL_WORDS: usize = 5;
+/// Direct 2-col path: max chars for a left-column label cell.
+const CR_LABEL_MAX_CHARS: usize = 60;
+/// Direct 2-col path: minimum row count.
+const CR_TWO_COL_MIN_ROWS: usize = 3;
+/// A left-side line continues the previous row's label when its top is within
+/// this multiple of line height of the previous label line's bottom.
+const CR_LABEL_WRAP_GAP_FACTOR: f32 = 1.5;
+
+/// A validated cross-region merge: replace `lines[start..end]` with `merged`
+/// (all sharing one synthetic region path) and hand `runs` (indices local to
+/// `merged`) to the region classifier so the table emission is guaranteed.
+pub(super) struct CrossRegionMerge {
+ pub(super) start: usize,
+ pub(super) end: usize,
+ pub(super) merged: Vec<ProjectedLine>,
+ pub(super) runs: Vec<TableRun>,
+}
+
+struct Leaf {
+ start: usize,
+ end: usize,
+ x0: f32,
+ x1: f32,
+ y0: f32,
+ y1: f32,
+}
+
+/// Maximum x-overlap between sibling V-cut slices, as a fraction of the
+/// narrower side's width.
+const CR_SIBLING_MAX_X_OVERLAP_FRAC: f32 = 0.4;
+
+/// True when two leaves are direct children of the same region-tree node and
+/// overlap only modestly in x — the signature of a V-cut whose slices were
+/// widened by straddling rows.
+fn sibling_v_cut_pair(a: &Leaf, b: &Leaf, lines: &[ProjectedLine]) -> bool {
+ let pa = &lines[a.start].region_path;
+ let pb = &lines[b.start].region_path;
+ if pa.len() != pb.len() || pa.is_empty() || pa[..pa.len() - 1] != pb[..pb.len() - 1] {
+ return false;
+ }
+ let x_overlap = a.x1.min(b.x1) - a.x0.max(b.x0);
+ let narrow = (a.x1 - a.x0).min(b.x1 - b.x0).max(1.0);
+ x_overlap < CR_SIBLING_MAX_X_OVERLAP_FRAC * narrow
+}
+
+fn build_leaves(lines: &[ProjectedLine]) -> Vec<Leaf> {
+ let mut leaves = Vec::new();
+ let mut s = 0;
+ while s < lines.len() {
+ let path = &lines[s].region_path;
+ let mut e = s + 1;
+ while e < lines.len() && lines[e].region_path == *path {
+ e += 1;
+ }
+ let (mut x0, mut x1) = (f32::INFINITY, f32::NEG_INFINITY);
+ let (mut y0, mut y1) = (f32::INFINITY, f32::NEG_INFINITY);
+ for l in &lines[s..e] {
+ x0 = x0.min(l.bbox.x);
+ x1 = x1.max(l.bbox.x + l.bbox.width);
+ y0 = y0.min(l.bbox.y);
+ y1 = y1.max(l.bbox.y + l.bbox.height);
+ }
+ leaves.push(Leaf {
+ start: s,
+ end: e,
+ x0,
+ x1,
+ y0,
+ y1,
+ });
+ s = e;
+ }
+ leaves
+}
+
+fn median(mut v: Vec<f32>) -> f32 {
+ if v.is_empty() {
+ return 0.0;
+ }
+ v.sort_by(|a, b| a.total_cmp(b));
+ v[v.len() / 2]
+}
+
+/// Longest common prefix of the region paths of all lines in the set.
+fn common_path_prefix(lines: &[&ProjectedLine]) -> Vec<u16> {
+ let mut prefix: Vec<u16> = lines[0].region_path.clone();
+ for l in &lines[1..] {
+ let n = prefix
+ .iter()
+ .zip(l.region_path.iter())
+ .take_while(|(a, b)| a == b)
+ .count();
+ prefix.truncate(n);
+ }
+ prefix
+}
+
+/// Does this side's line set look like a column of flowing prose? Dense
+/// vertical fill + lines that fill the column width. A table fragment has
+/// short, sparse lines on at least one side.
+fn side_prose_metrics(side: &[&ProjectedLine], side_x0: f32, side_x1: f32) -> (f32, f32) {
+ let side_width = (side_x1 - side_x0).max(1.0);
+ let med_width = median(side.iter().map(|l| l.bbox.width).collect());
+ let (mut y0, mut y1) = (f32::INFINITY, f32::NEG_INFINITY);
+ let mut sum_h = 0.0f32;
+ for l in side {
+ y0 = y0.min(l.bbox.y);
+ y1 = y1.max(l.bbox.y + l.bbox.height);
+ sum_h += l.bbox.height;
+ }
+ (med_width / side_width, sum_h / (y1 - y0).max(1.0))
+}
+
+fn side_looks_like_prose(side: &[&ProjectedLine], side_x0: f32, side_x1: f32) -> bool {
+ if side.len() < CR_PROSE_MIN_LINES {
+ return false;
+ }
+ let (width_fill, vfill) = side_prose_metrics(side, side_x0, side_x1);
+ if width_fill < CR_PROSE_WIDTH_FRAC || vfill < CR_PROSE_VFILL {
+ return false;
+ }
+ // A sliced *wide* table's row-halves also fill their side's width and
+ // y-band, but they carry internal cell gaps; flowing prose doesn't.
+ let multi_cell = side
+ .iter()
+ .filter(|l| super::tables::split_cells(l).len() >= 2)
+ .count();
+ (multi_cell as f32) < CR_PROSE_MAX_MULTI_CELL_FRAC * side.len() as f32
+}
+
+struct Cluster<'a> {
+ left: Vec<&'a ProjectedLine>,
+ right: Vec<&'a ProjectedLine>,
+}
+
+impl<'a> Cluster<'a> {
+ fn members(&self) -> impl Iterator<Item = &&'a ProjectedLine> {
+ self.left.iter().chain(self.right.iter())
+ }
+}
+
+/// Cluster the set's lines into baseline rows: lines whose y-centers fall
+/// within `tol` of the cluster seed share a row.
+fn cluster_rows<'a>(set: &[(&'a ProjectedLine, bool)], tol: f32) -> Vec<Cluster<'a>> {
+ let mut sorted: Vec<&(&ProjectedLine, bool)> = set.iter().collect();
+ sorted.sort_by(|a, b| {
+ let ya = a.0.bbox.y + a.0.bbox.height * 0.5;
+ let yb = b.0.bbox.y + b.0.bbox.height * 0.5;
+ ya.total_cmp(&yb)
+ });
+ let mut clusters: Vec<(f32, Cluster)> = Vec::new();
+ for (line, is_left) in sorted {
+ let yc = line.bbox.y + line.bbox.height * 0.5;
+ match clusters.last_mut() {
+ Some((seed_y, c)) if (yc - *seed_y).abs() <= tol => {
+ if *is_left {
+ c.left.push(line);
+ } else {
+ c.right.push(line);
+ }
+ }
+ _ => {
+ let mut c = Cluster {
+ left: Vec::new(),
+ right: Vec::new(),
+ };
+ if *is_left {
+ c.left.push(line);
+ } else {
+ c.right.push(line);
+ }
+ clusters.push((yc, c));
+ }
+ }
+ }
+ clusters.into_iter().map(|(_, c)| c).collect()
+}
+
+/// Fuse one baseline cluster into a single `ProjectedLine`: spans concatenate
+/// (cell boundaries fall out of the span gaps in `split_cells`), text joins
+/// with gap-proportional spacing, styles AND together.
+fn fuse_cluster(cluster: &Cluster, path: &[u16]) -> ProjectedLine {
+ let mut members: Vec<&ProjectedLine> = cluster.members().copied().collect();
+ members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x));
+
+ let first = members[0];
+ let mut fused = first.clone();
+ fused.region_path = path.to_vec();
+ for m in &members[1..] {
+ let prev_right = fused.bbox.x + fused.bbox.width;
+ let gap = (m.bbox.x - prev_right).max(0.0);
+ let approx_char_w = (fused.dominant_font_size * 0.5).max(2.0);
+ let n_spaces = ((gap / approx_char_w) as usize).clamp(2, 24);
+ fused.text.push_str(&" ".repeat(n_spaces));
+ fused.text.push_str(&m.text);
+ fused.spans.extend(m.spans.iter().cloned());
+ let x1 = (fused.bbox.x + fused.bbox.width).max(m.bbox.x + m.bbox.width);
+ let y1 = (fused.bbox.y + fused.bbox.height).max(m.bbox.y + m.bbox.height);
+ fused.bbox.x = fused.bbox.x.min(m.bbox.x);
+ fused.bbox.y = fused.bbox.y.min(m.bbox.y);
+ fused.bbox.width = x1 - fused.bbox.x;
+ fused.bbox.height = y1 - fused.bbox.y;
+ fused.all_bold &= m.all_bold;
+ fused.all_italic &= m.all_italic;
+ fused.all_mono &= m.all_mono;
+ fused.all_strike &= m.all_strike;
+ fused.font_size_is_estimated |= m.font_size_is_estimated;
+ }
+ fused.spans.sort_by(|a, b| a.x.total_cmp(&b.x));
+ fused
+}
+
+/// Validation A: the fused zone re-classifies as a table via the standard
+/// detectors, covering most of its lines.
+fn validate_via_detectors(merged: &[ProjectedLine]) -> Option<Vec<TableRun>> {
+ let runs = detect_tables(merged);
+ let covered: usize = runs
+ .iter()
+ .filter(|r| match &r.block {
+ Block::Table { header, rows } => {
+ let cols = header
+ .as_ref()
+ .map(|h| h.len())
+ .or_else(|| rows.first().map(|row| row.len()))
+ .unwrap_or(0);
+ if cols < 2 {
+ return false;
+ }
+ let (mut cells, mut long) = (0usize, 0usize);
+ for row in rows {
+ for cell in row {
+ let t = cell.trim();
+ if t.is_empty() {
+ continue;
+ }
+ cells += 1;
+ if t.split_whitespace().count() >= CR_LONG_CELL_WORDS {
+ long += 1;
+ }
+ }
+ }
+ cells > 0 && (long as f32) <= CR_VALIDATE_MAX_LONG_CELL_FRAC * cells as f32
+ }
+ _ => false,
+ })
+ .map(|r| r.end - r.start)
+ .sum();
+ if covered as f32 >= CR_VALIDATE_MIN_COVERAGE * merged.len() as f32 {
+ Some(runs)
+ } else {
+ None
+ }
+}
+
+/// Validation B: direct 2-column construction for label/description shapes
+/// the standard detectors reject (e.g. labels with list-marker prefixes —
+/// `is_label_like` is intentionally strict in normal flow, but here the V-cut
+/// geometry has already established two columns). Rows are anchored on
+/// left-side lines; a left line within wrap distance of the previous one
+/// continues the label cell, right-side lines append to the current row's
+/// description cell.
+fn try_two_col_direct(clusters: &[Cluster], merged_len: usize, tol: f32) -> Option<Vec<TableRun>> {
+ let mut rows: Vec<(String, String, bool)> = Vec::new(); // (left, right, bold)
+ let mut last_label_bottom: Option<f32> = None;
+
+ for cluster in clusters {
+ if !cluster.left.is_empty() {
+ let text = cluster
+ .left
+ .iter()
+ .map(|l| l.text.trim())
+ .collect::<Vec<_>>()
+ .join(" ");
+ let top = cluster
+ .left
+ .iter()
+ .map(|l| l.bbox.y)
+ .fold(f32::INFINITY, f32::min);
+ let bottom = cluster
+ .left
+ .iter()
+ .map(|l| l.bbox.y + l.bbox.height)
+ .fold(f32::NEG_INFINITY, f32::max);
+ let h = (bottom - top).max(1.0);
+ let wraps = match (last_label_bottom, rows.last()) {
+ (Some(lb), Some(_)) => top - lb <= CR_LABEL_WRAP_GAP_FACTOR * h.min(tol * 2.0),
+ _ => false,
+ };
+ if wraps {
+ let row = rows.last_mut().unwrap();
+ if !row.0.is_empty() {
+ row.0.push(' ');
+ }
+ row.0.push_str(&text);
+ } else {
+ let bold = cluster.left.iter().all(|l| l.all_bold);
+ rows.push((text, String::new(), bold));
+ }
+ last_label_bottom = Some(bottom);
+ }
+ if !cluster.right.is_empty() {
+ let text = cluster
+ .right
+ .iter()
+ .map(|l| l.text.trim())
+ .collect::<Vec<_>>()
+ .join(" ");
+ if rows.is_empty() {
+ rows.push((String::new(), String::new(), false));
+ }
+ let row = rows.last_mut().unwrap();
+ if !row.1.is_empty() {
+ row.1.push(' ');
+ }
+ row.1.push_str(&text);
+ }
+ }
+
+ if rows.len() < CR_TWO_COL_MIN_ROWS {
+ return None;
+ }
+ // Labels must stay label-shaped after wrap joins; a left side of long
+ // sentences is prose the anti-prose gate missed.
+ if rows
+ .iter()
+ .any(|(l, _, _)| l.chars().count() > CR_LABEL_MAX_CHARS)
+ {
+ return None;
+ }
+ let both = rows
+ .iter()
+ .filter(|(l, r, _)| !l.is_empty() && !r.is_empty())
+ .count();
+ if both < 2 {
+ return None;
+ }
+
+ // First row promotes to header when bold or when both cells are short
+ // header-ish words ("Area" / "Competence").
+ let header = {
+ let (l, r, bold) = &rows[0];
+ if !l.is_empty()
+ && !r.is_empty()
+ && (*bold || (l.chars().count() <= 20 && r.chars().count() <= 20))
+ {
+ Some(vec![l.clone(), r.clone()])
+ } else {
+ None
+ }
+ };
+ let body: Vec<Vec<String>> = rows
+ .iter()
+ .skip(if header.is_some() { 1 } else { 0 })
+ .map(|(l, r, _)| vec![l.clone(), r.clone()])
+ .collect();
+ if body.len() < 2 {
+ return None;
+ }
+ Some(vec![TableRun {
+ start: 0,
+ end: merged_len,
+ body_start: if header.is_some() { 1 } else { 0 },
+ block: Block::Table { header, rows: body },
+ }])
+}
+
+/// Find one validated cross-region table merge on this page, or `None`.
+/// Callers may re-invoke on the spliced result to catch multiple sliced
+/// tables per page.
+pub(super) fn find_cross_region_table_merge(lines: &[ProjectedLine]) -> Option<CrossRegionMerge> {
+ if *super::flags::DISABLE_CROSS_REGION_TABLES {
+ return None;
+ }
+ let debug = *super::flags::DEBUG_CROSS_REGION;
+ let leaves = build_leaves(lines);
+ if debug {
+ for (k, l) in leaves.iter().enumerate() {
+ eprintln!(
+ "[cross-region] leaf {k}: lines [{},{}) x=[{:.1},{:.1}] y=[{:.1},{:.1}] path={:?}",
+ l.start, l.end, l.x0, l.x1, l.y0, l.y1, lines[l.start].region_path
+ );
+ }
+ }
+ if leaves.len() < 2 {
+ return None;
+ }
+
+ for i in 0..leaves.len() {
+ for j in (i + 1)..leaves.len() {
+ let (a, b) = (&leaves[i], &leaves[j]);
+ let (left, right) = if a.x1 <= b.x0 - CR_MIN_COL_GAP_PT {
+ (a, b)
+ } else if b.x1 <= a.x0 - CR_MIN_COL_GAP_PT {
+ (b, a)
+ } else if sibling_v_cut_pair(a, b, lines) {
+ // Direct siblings of one region-tree node that overlap in y
+ // can only come from a V-cut. Their bbox x-extents may overlap
+ // (table rows straddling the cut widen the leaf), so the
+ // disjointness test above misses them.
+ if a.x0 <= b.x0 { (a, b) } else { (b, a) }
+ } else {
+ continue;
+ };
+ let ov = left.y1.min(right.y1) - left.y0.max(right.y0);
+ let min_h = (left.y1 - left.y0).min(right.y1 - right.y0).max(1.0);
+ if ov < CR_MIN_Y_OVERLAP_FRAC * min_h {
+ continue;
+ }
+ // For disjoint sides this is the gutter midpoint; for overlapping
+ // sibling slices it's the midpoint of the contested band.
+ let cut_x = (left.x1.min(right.x1) + right.x0.max(left.x0)) * 0.5;
+ // Union band: a tall side may pair with the first of several
+ // stacked leaves on the other side; the extension below pulls the
+ // rest in, so the band must cover both sides fully.
+ let band = (left.y0.min(right.y0) - 2.0, left.y1.max(right.y1) + 2.0);
+
+ // Extend the pair to every leaf living inside the overlap band and
+ // entirely on one side of the cut (a side may itself be H-cut into
+ // multiple leaves — doc 146's Area column is).
+ let mut set_leaves: Vec<usize> = Vec::new();
+ for (k, leaf) in leaves.iter().enumerate() {
+ let leaf_h = (leaf.y1 - leaf.y0).max(1.0);
+ let leaf_ov = leaf.y1.min(band.1) - leaf.y0.max(band.0);
+ if leaf_ov < 0.5 * leaf_h {
+ continue;
+ }
+ // The leaf must sit predominantly on one side of the cut:
+ // its overhang across the cut stays within the straddle
+ // allowance (widened slices), measured against its own width.
+ let width = (leaf.x1 - leaf.x0).max(1.0);
+ let center = (leaf.x0 + leaf.x1) * 0.5;
+ let overhang = if center < cut_x {
+ (leaf.x1 - cut_x).max(0.0)
+ } else {
+ (cut_x - leaf.x0).max(0.0)
+ };
+ if overhang <= CR_SIBLING_MAX_X_OVERLAP_FRAC * width {
+ set_leaves.push(k);
+ }
+ }
+ if set_leaves.len() < 2 {
+ continue;
+ }
+
+ // The set's line ranges must tile a contiguous range — a foreign
+ // leaf interleaved in reading order means this isn't a clean
+ // sliced zone.
+ let lo = set_leaves.iter().map(|&k| leaves[k].start).min().unwrap();
+ let hi = set_leaves.iter().map(|&k| leaves[k].end).max().unwrap();
+ let covered: usize = set_leaves
+ .iter()
+ .map(|&k| leaves[k].end - leaves[k].start)
+ .sum();
+ if covered != hi - lo {
+ continue;
+ }
+
+ // Split the set's lines into sides of the cut.
+ let mut set: Vec<(&ProjectedLine, bool)> = Vec::new();
+ for &k in &set_leaves {
+ let leaf = &leaves[k];
+ let is_left = (leaf.x0 + leaf.x1) * 0.5 < cut_x;
+ for l in &lines[leaf.start..leaf.end] {
+ set.push((l, is_left));
+ }
+ }
+ let left_lines: Vec<&ProjectedLine> =
+ set.iter().filter(|(_, s)| *s).map(|(l, _)| *l).collect();
+ let right_lines: Vec<&ProjectedLine> =
+ set.iter().filter(|(_, s)| !*s).map(|(l, _)| *l).collect();
+ if left_lines.is_empty() || right_lines.is_empty() {
+ continue;
+ }
+
+ // Anti-prose gate: two side-by-side prose columns must stay split.
+ let lx0 = left_lines
+ .iter()
+ .map(|l| l.bbox.x)
+ .fold(f32::INFINITY, f32::min);
+ let rx0 = right_lines
+ .iter()
+ .map(|l| l.bbox.x)
+ .fold(f32::INFINITY, f32::min);
+ let lx1 = left_lines
+ .iter()
+ .map(|l| l.bbox.x + l.bbox.width)
+ .fold(f32::NEG_INFINITY, f32::max);
+ let rx1 = right_lines
+ .iter()
+ .map(|l| l.bbox.x + l.bbox.width)
+ .fold(f32::NEG_INFINITY, f32::max);
+ if debug {
+ let (lw, lv) = side_prose_metrics(&left_lines, lx0, lx1);
+ let (rw, rv) = side_prose_metrics(&right_lines, rx0, rx1);
+ eprintln!(
+ "[cross-region] candidate cut@{cut_x:.1}: left n={} wfill={lw:.2} vfill={lv:.2} | right n={} wfill={rw:.2} vfill={rv:.2}",
+ left_lines.len(),
+ right_lines.len()
+ );
+ }
+ // Either side reading as flowing prose means the V-cut was right:
+ // a body column next to a sidebar / second column must stay split.
+ if side_looks_like_prose(&left_lines, lx0, lx1)
+ || side_looks_like_prose(&right_lines, rx0, rx1)
+ {
+ if debug {
+ eprintln!("[cross-region] reject cut@{cut_x:.1}: a side is prose-like");
+ }
+ continue;
+ }
+
+ // Row alignment across the cut.
+ let tol = CR_ROW_TOL_FACTOR
+ * median(set.iter().map(|(l, _)| l.bbox.height).collect()).max(1.0);
+ let clusters = cluster_rows(&set, tol);
+ let aligned = clusters
+ .iter()
+ .filter(|c| !c.left.is_empty() && !c.right.is_empty())
+ .count();
+ let min_side = left_lines.len().min(right_lines.len());
+ if aligned < CR_MIN_ALIGNED_ROWS
+ || (aligned as f32) < CR_MIN_ALIGNED_FRAC * min_side as f32
+ {
+ if debug {
+ eprintln!(
+ "[cross-region] reject cut@{cut_x:.1}: aligned={aligned} of min_side={min_side}"
+ );
+ }
+ continue;
+ }
+
+ // Fuse and validate.
+ let all: Vec<&ProjectedLine> = set.iter().map(|(l, _)| *l).collect();
+ let path = common_path_prefix(&all);
+ let merged: Vec<ProjectedLine> =
+ clusters.iter().map(|c| fuse_cluster(c, &path)).collect();
+
+ let runs = validate_via_detectors(&merged)
+ .or_else(|| try_two_col_direct(&clusters, merged.len(), tol));
+ let Some(runs) = runs else {
+ if debug {
+ eprintln!(
+ "[cross-region] reject cut@{cut_x:.1}: fused zone failed table validation"
+ );
+ }
+ continue;
+ };
+ if debug {
+ eprintln!(
+ "[cross-region] MERGE cut@{cut_x:.1}: lines [{lo},{hi}) -> {} fused rows, {} run(s)",
+ merged.len(),
+ runs.len()
+ );
+ }
+ return Some(CrossRegionMerge {
+ start: lo,
+ end: hi,
+ merged,
+ runs,
+ });
+ }
+ }
+ None
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::markdown_layout::test_helpers::{line, line_with_spans};
+
+ fn at_region(mut l: ProjectedLine, path: &[u16]) -> ProjectedLine {
+ l.region_path = path.to_vec();
+ l
+ }
+
+ /// A 3-col numeric table sliced by a V-cut: label column in one leaf,
+ /// two numeric columns in the other. Validation A (standard detectors on
+ /// the fused rows) should accept.
+ #[test]
+ fn sliced_numeric_table_is_merged() {
+ let mut lines = Vec::new();
+ // Left leaf: row labels.
+ for (i, label) in ["MANILA", "CEBU", "CAGAYAN DE ORO", "SUBIC"]
+ .iter()
+ .enumerate()
+ {
+ let y = 100.0 + i as f32 * 20.0;
+ lines.push(at_region(
+ line_with_spans(&[(label, 50.0)], y, 10.0),
+ &[0, 0],
+ ));
+ }
+ // Right leaf: two numeric columns per row.
+ for (i, (a, b)) in [
+ ("2454", "6125"),
+ ("1138", "79500"),
+ ("958", "13196"),
+ ("313", "136"),
+ ]
+ .iter()
+ .enumerate()
+ {
+ let y = 100.0 + i as f32 * 20.0;
+ lines.push(at_region(
+ line_with_spans(&[(a, 200.0), (b, 280.0)], y, 10.0),
+ &[0, 1],
+ ));
+ }
+ let m = find_cross_region_table_merge(&lines).expect("should merge");
+ assert_eq!((m.start, m.end), (0, 8));
+ assert_eq!(m.merged.len(), 4);
+ assert!(!m.runs.is_empty());
+ // Every fused row carries the label and both numbers.
+ assert!(m.merged[0].text.contains("MANILA"));
+ assert!(m.merged[0].text.contains("2454"));
+ assert!(m.merged[0].text.contains("6125"));
+ }
+
+ /// Two side-by-side prose columns with coincidentally aligned baselines
+ /// must NOT merge — that's the layout the V-cut exists to protect.
+ #[test]
+ fn two_column_prose_is_not_merged() {
+ let mut lines = Vec::new();
+ for i in 0..8 {
+ let y = 100.0 + i as f32 * 12.0;
+ lines.push(at_region(
+ line("the quick brown fox jumps over it", 50.0, y, 10.0, 10.0),
+ &[0, 0],
+ ));
+ }
+ for i in 0..8 {
+ let y = 100.0 + i as f32 * 12.0;
+ lines.push(at_region(
+ line("a second column of body text here", 250.0, y, 10.0, 10.0),
+ &[0, 1],
+ ));
+ }
+ assert!(find_cross_region_table_merge(&lines).is_none());
+ }
+
+ /// Label / description shape with list-marker labels (the doc-146 shape):
+ /// standard detectors reject it, the direct 2-col path accepts.
+ #[test]
+ fn sliced_label_description_table_uses_direct_path() {
+ let mut lines = Vec::new();
+ lines.push(at_region(line("Area", 50.0, 100.0, 10.0, 10.0), &[0, 0]));
+ for (i, label) in [
+ "1. Embodying values",
+ "2. Embracing complexity",
+ "3. Envisioning futures",
+ ]
+ .iter()
+ .enumerate()
+ {
+ let y = 150.0 + i as f32 * 50.0;
+ lines.push(at_region(line(label, 50.0, y, 10.0, 10.0), &[0, 0]));
+ }
+ lines.push(at_region(
+ line("Competence", 250.0, 100.0, 10.0, 10.0),
+ &[0, 1],
+ ));
+ for (i, desc) in [
+ "1.1 Valuing sustainability 1.2 Supporting fairness",
+ "2.1 Systems thinking 2.2 Critical thinking",
+ "3.1 Futures literacy 3.2 Adaptability",
+ ]
+ .iter()
+ .enumerate()
+ {
+ let y = 150.0 + i as f32 * 50.0;
+ lines.push(at_region(line(desc, 250.0, y, 10.0, 10.0), &[0, 1]));
+ }
+ let m = find_cross_region_table_merge(&lines).expect("should merge via direct path");
+ assert_eq!(m.runs.len(), 1);
+ match &m.runs[0].block {
+ Block::Table { header, rows } => {
+ assert_eq!(
+ header.as_deref(),
+ Some(&["Area".to_string(), "Competence".to_string()][..])
+ );
+ assert_eq!(rows.len(), 3);
+ assert!(rows[0][1].contains("1.1 Valuing sustainability"));
+ }
+ other => panic!("expected table, got {other:?}"),
+ }
+ }
+
+ /// A single region (no V-cut) never merges.
+ #[test]
+ fn single_leaf_is_untouched() {
+ let lines: Vec<ProjectedLine> = (0..6)
+ .map(|i| {
+ at_region(
+ line("text", 50.0, 100.0 + i as f32 * 12.0, 10.0, 10.0),
+ &[0],
+ )
+ })
+ .collect();
+ assert!(find_cross_region_table_merge(&lines).is_none());
+ }
+
+ /// Vertically stacked leaves (H-cut siblings) never pair: no y-overlap.
+ #[test]
+ fn stacked_leaves_do_not_pair() {
+ let mut lines = Vec::new();
+ for i in 0..4 {
+ lines.push(at_region(
+ line("alpha beta", 50.0, 100.0 + i as f32 * 12.0, 10.0, 10.0),
+ &[0],
+ ));
+ }
+ for i in 0..4 {
+ lines.push(at_region(
+ line("gamma delta", 50.0, 300.0 + i as f32 * 12.0, 10.0, 10.0),
+ &[1],
+ ));
+ }
+ assert!(find_cross_region_table_merge(&lines).is_none());
+ }
+}
diff --git a/crates/liteparse/src/markdown_layout/flags.rs b/crates/liteparse/src/markdown_layout/flags.rs
new file mode 100644
index 0000000..7dc61d6
--- /dev/null
+++ b/crates/liteparse/src/markdown_layout/flags.rs
@@ -0,0 +1,30 @@
+//! Process-wide debug / kill-switch flags, read once from the environment.
+//!
+//! Each flag mirrors a `LITEPARSE_*` env var. `env::var` scans the process
+//! environment table and allocates on a miss, so per-line hot paths cache the
+//! lookup in a `LazyLock` instead of re-reading on every call. Each static is
+//! `true` when the variable is **set** (matching the historical
+//! `env::var(..).is_ok()`); "disable" call sites negate it.
+
+use std::sync::LazyLock;
+
+macro_rules! env_set_flag {
+ ($(#[$m:meta])* $name:ident, $var:literal) => {
+ $(#[$m])*
+ pub(super) static $name: LazyLock<bool> =
+ LazyLock::new(|| std::env::var($var).is_ok());
+ };
+}
+
+env_set_flag!(DEBUG_MD, "LITEPARSE_DEBUG_MD");
+env_set_flag!(DEBUG_TABLE, "LITEPARSE_DEBUG_TABLE");
+env_set_flag!(DEBUG_RULED, "LITEPARSE_DEBUG_RULED");
+env_set_flag!(DEBUG_CROSS_REGION, "LITEPARSE_DEBUG_CROSS_REGION");
+env_set_flag!(DISABLE_GLOBAL_RULED, "LITEPARSE_DISABLE_GLOBAL_RULED");
+env_set_flag!(DISABLE_HEADING_GUARDS, "LITEPARSE_DISABLE_HEADING_GUARDS");
+env_set_flag!(
+ DISABLE_CROSS_REGION_TABLES,
+ "LITEPARSE_DISABLE_CROSS_REGION_TABLES"
+);
+env_set_flag!(DISABLE_CLUSTER_MERGE, "LITEPARSE_DISABLE_CLUSTER_MERGE");
+env_set_flag!(DISABLE_STRADDLE_GUARD, "LITEPARSE_DISABLE_STRADDLE_GUARD");
diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs
index 9597f52..e56261a 100644
--- a/crates/liteparse/src/markdown_layout/tables.rs
+++ b/crates/liteparse/src/markdown_layout/tables.rs
@@ -2527,121 +2527,218 @@ fn find_grid_components(hs: &[HSeg], vs: &[VSeg]) -> Vec<(Vec<usize>, Vec<usize>
comps
}
-/// Build a `TableRun` for one ruled-grid component. Returns `None` if the
-/// resulting grid is too small (< 2 cols or < 2 rows), covers nearly the
-/// whole page (likely the page border), or is mostly empty cells.
-fn build_ruled_table(
- hs: &[HSeg],
- vs: &[VSeg],
- h_indices: &[usize],
- v_indices: &[usize],
- lines: &[ProjectedLine],
- page_width: f32,
- page_height: f32,
- global: bool,
-) -> Option<(TableRun, Vec<usize>)> {
- let dbg = *super::flags::DEBUG_RULED;
- let mut xs: Vec<f32> = v_indices.iter().map(|&i| vs[i].x).collect();
- xs.sort_by(|a, b| a.total_cmp(b));
- // Coarser, mean-centered clustering for column boundaries: cell-border
- // rects contribute paired edges 4-6pt apart that would otherwise become
- // phantom 5pt "columns" the span splitter then shreds text into.
- cluster_boundaries(&mut xs, TABLE_COL_BOUNDARY_CLUSTER_PT);
+/// Which ruled-table detection pass is running. The global pass reassembles a
+/// grid from page-level graphics (lines arrive in xy_cut leaf order and grid
+/// rules may include short decorative horizontals), so it applies extra
+/// filtering the per-region pass does not need.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum RuledPass {
+ PerRegion,
+ Global,
+}
- // Distinct row y-coords (cluster again — multiple H lines may share a y).
- // In the global pass, first drop horizontal rules that span only a small
- // fraction of the column extent (see `RULED_HLINE_MIN_COVERAGE`).
- let raw_ys = |idxs: &[usize]| {
- let mut v: Vec<f32> = idxs.iter().map(|&i| hs[i].y).collect();
- v.sort_by(|a, b| a.total_cmp(b));
- dedup_close(&mut v, TABLE_GRID_CLUSTER_PT);
- v
- };
- let ys: Vec<f32> = if global && xs.len() >= 2 {
- let col_lo = xs[0];
- let col_hi = xs[xs.len() - 1];
- let extent = (col_hi - col_lo).max(1.0);
- let kept: Vec<usize> = h_indices
- .iter()
- .copied()
- .filter(|&i| {
- let h = &hs[i];
- let ov = (h.x_max.min(col_hi) - h.x_min.max(col_lo)).max(0.0);
- ov / extent >= RULED_HLINE_MIN_COVERAGE
- })
- .collect();
- let filtered = raw_ys(&kept);
- if filtered.len() >= 3 {
- filtered
- } else {
- raw_ys(h_indices)
+/// Filter a vector down to the elements whose `keep` flag is set. `keep` must
+/// be at least as long as `items`.
+fn filter_by<T>(items: Vec<T>, keep: &[bool]) -> Vec<T> {
+ items
+ .into_iter()
+ .zip(keep.iter())
+ .filter(|(_, k)| **k)
+ .map(|(v, _)| v)
+ .collect()
+}
+
+/// The per-cell grids `build_ruled_table` fills, collapses, and reads as one
+/// unit. Bundling them lets the row/column-collapse passes filter every layer
+/// through a single `retain_rows`/`retain_cols` call instead of five parallel
+/// `filter().map().collect()` chains — removing the "forgot to filter one
+/// array" bug class.
+struct CellGrid {
+ /// Rendered cell text (each multi-column span whitespace-split at the
+ /// crossed column boundaries).
+ text: Vec<Vec<String>>,
+ /// Per-cell bold flag — starts `true`, cleared when any non-bold span
+ /// lands in the cell.
+ is_bold: Vec<Vec<bool>>,
+ /// Per-cell "carries text" flag.
+ has_text: Vec<Vec<bool>>,
+ /// Colspan-semantics grid: a span crossing ≥2 column boundaries replicates
+ /// its FULL text into every covered cell instead of being split. Consulted
+ /// only for header-band flattening — a group header like "North America"
+ /// centered over its sub-columns must replicate, while the same geometry in
+ /// a data row is a merged run that should split.
+ repl: Vec<Vec<String>>,
+ /// Per-row flag: the row contains an alpha-dominant multi-column span (a
+ /// group-header label, as opposed to a mostly-digit merged data run).
+ row_alpha_spanner: Vec<bool>,
+}
+
+impl CellGrid {
+ fn new(n_rows: usize, n_cols: usize) -> Self {
+ CellGrid {
+ text: vec![vec![String::new(); n_cols]; n_rows],
+ is_bold: vec![vec![true; n_cols]; n_rows],
+ has_text: vec![vec![false; n_cols]; n_rows],
+ repl: vec![vec![String::new(); n_cols]; n_rows],
+ row_alpha_spanner: vec![false; n_rows],
}
- } else {
- raw_ys(h_indices)
- };
- if dbg {
- eprintln!(
- "[ruled] component: ys={:?} xs={:?} ({} lines in scope)",
- ys,
- xs,
- lines.len()
- );
}
- // Need ≥2 row boundaries (1 row) and ≥2 column boundaries (1 col); but
- // a 1×1 grid is just a callout box, so also require ≥1 inner divider
- // (i.e. ys.len() ≥ 3 for ≥2 rows). Single-column tables (`xs.len() == 2`)
- // are accepted when row evidence is strong enough — extra guards apply
- // below after the empty-row collapse.
- if ys.len() < 3 || xs.len() < 2 {
- if dbg {
- eprintln!(
- "[ruled] REJECT grid-too-small ys={} xs={}",
- ys.len(),
- xs.len()
- );
- }
- return None;
+ fn n_rows(&self) -> usize {
+ self.text.len()
}
- let n_rows = ys.len() - 1;
- let n_cols = xs.len() - 1;
- let bbox = crate::types::Rect {
- x: xs[0],
- y: ys[0],
- width: xs[n_cols] - xs[0],
- height: ys[n_rows] - ys[0],
- };
+ fn n_cols(&self) -> usize {
+ self.text.first().map(|r| r.len()).unwrap_or(0)
+ }
- // Reject page-border-as-table.
- if page_width > 0.0 && page_height > 0.0 {
- let coverage = (bbox.width / page_width) * (bbox.height / page_height);
- if coverage > TABLE_MAX_PAGE_COVERAGE {
- if dbg {
- eprintln!("[ruled] REJECT page-coverage {coverage:.2}");
- }
- return None;
+ /// Append trimmed text to a cell, marking it as text-bearing. No-op for
+ /// blank input.
+ fn push_text(&mut self, row: usize, col: usize, txt: &str) {
+ let txt = txt.trim();
+ if txt.is_empty() {
+ return;
}
+ if !self.text[row][col].is_empty() {
+ self.text[row][col].push(' ');
+ }
+ self.text[row][col].push_str(txt);
+ self.has_text[row][col] = true;
+ }
+
+ /// Append trimmed text to the colspan-semantics (`repl`) cell. No-op for
+ /// blank input.
+ fn push_repl(&mut self, row: usize, col: usize, txt: &str) {
+ let txt = txt.trim();
+ if txt.is_empty() {
+ return;
+ }
+ let dst = &mut self.repl[row][col];
+ if !dst.is_empty() {
+ dst.push(' ');
+ }
+ dst.push_str(txt);
+ }
+
+ /// Keep only rows `r` where `keep[r]`; every layer filters identically.
+ fn retain_rows(&mut self, keep: &[bool]) {
+ self.text = filter_by(std::mem::take(&mut self.text), keep);
+ self.is_bold = filter_by(std::mem::take(&mut self.is_bold), keep);
+ self.has_text = filter_by(std::mem::take(&mut self.has_text), keep);
+ self.repl = filter_by(std::mem::take(&mut self.repl), keep);
+ self.row_alpha_spanner = filter_by(std::mem::take(&mut self.row_alpha_spanner), keep);
+ }
+
+ /// Keep only columns `c` where `keep[c]`; the per-cell layers filter, the
+ /// per-row `row_alpha_spanner` is unaffected.
+ fn retain_cols(&mut self, keep: &[bool]) {
+ self.text = std::mem::take(&mut self.text)
+ .into_iter()
+ .map(|row| filter_by(row, keep))
+ .collect();
+ self.is_bold = std::mem::take(&mut self.is_bold)
+ .into_iter()
+ .map(|row| filter_by(row, keep))
+ .collect();
+ self.has_text = std::mem::take(&mut self.has_text)
+ .into_iter()
+ .map(|row| filter_by(row, keep))
+ .collect();
+ self.repl = std::mem::take(&mut self.repl)
+ .into_iter()
+ .map(|row| filter_by(row, keep))
+ .collect();
}
- // Assign text to cells. A projected line frequently spans several ruled
- // columns (one baseline = one line), so binning whole lines by centroid
- // lumps an entire row into one cell and leaves the rest empty — the
- // empty-cell filter then rejects the real table. Instead bin each line's
- // raw spans by span center; spans whose x-extent crosses one or more
- // interior column boundaries are split at the whitespace nearest each
- // crossed boundary (same interpolation as the inferred-track path).
- let mut cells: Vec<Vec<String>> = vec![vec![String::new(); n_cols]; n_rows];
- let mut cell_is_bold: Vec<Vec<bool>> = vec![vec![true; n_cols]; n_rows];
- let mut cell_has_text: Vec<Vec<bool>> = vec![vec![false; n_cols]; n_rows];
- // Parallel grid where a span crossing ≥2 column boundaries contributes
- // its FULL text to every covered cell (colspan semantics) instead of the
- // whitespace split. Only consulted for header-band flattening — a group
- // header like "North America" centered over its Revenue/Units sub-columns
- // must replicate, while the same geometry in a data row is a merged run
- // that should split.
- let mut cells_repl: Vec<Vec<String>> = vec![vec![String::new(); n_cols]; n_rows];
- let mut row_alpha_spanner: Vec<bool> = vec![false; n_rows];
+ /// Drop "phantom rows" produced by stacked thin border-strip rects (doc 149
+ /// rules each visual row as top-strip ~1pt / body ~22pt / bottom-strip ~5pt,
+ /// each surviving the 2pt clustering as its own grid row). A row is dropped
+ /// iff it has no text in any cell AND its height is < 80% of the median
+ /// non-empty row height — the height gate preserves real fill-in-the-blank
+ /// forms whose empty body rows are full height. `ys` are the row boundaries
+ /// (length `n_rows + 1`). Returns the kept rows' heights.
+ fn collapse_phantom_rows(&mut self, ys: &[f32]) -> Vec<f32> {
+ let n_rows = self.n_rows();
+ let row_heights: Vec<f32> = (0..n_rows).map(|r| ys[r + 1] - ys[r]).collect();
+ let nonempty_heights: Vec<f32> = (0..n_rows)
+ .filter(|r| self.has_text[*r].iter().any(|t| *t))
+ .map(|r| row_heights[r])
+ .collect();
+ let median_h = if !nonempty_heights.is_empty() {
+ let mut s = nonempty_heights.clone();
+ s.sort_by(|a, b| a.total_cmp(b));
+ s[s.len() / 2]
+ } else {
+ let mut s = row_heights.clone();
+ s.sort_by(|a, b| a.total_cmp(b));
+ s[s.len() / 2]
+ };
+ let keep: Vec<bool> = (0..n_rows)
+ .map(|r| {
+ let has_text = self.has_text[r].iter().any(|t| *t);
+ has_text || row_heights[r] >= median_h * 0.8
+ })
+ .collect();
+ let kept_row_heights: Vec<f32> = (0..n_rows)
+ .filter(|r| keep[*r])
+ .map(|r| row_heights[r])
+ .collect();
+ self.retain_rows(&keep);
+ kept_row_heights
+ }
+
+ /// Mirror `collapse_phantom_rows` on columns: some ruled tables (doc 149)
+ /// draw left/right borders as thin strip rects ~5pt wide, which become
+ /// phantom text-less columns. Drop a column iff it is both empty AND
+ /// narrower than 30% of the median text-bearing column. `xs` are the column
+ /// boundaries (length `n_cols + 1`).
+ fn collapse_phantom_cols(&mut self, xs: &[f32]) {
+ let n_rows = self.n_rows();
+ let n_cols = self.n_cols();
+ let col_widths: Vec<f32> = (0..n_cols).map(|c| xs[c + 1] - xs[c]).collect();
+ let nonempty_col_widths: Vec<f32> = (0..n_cols)
+ .filter(|c| (0..n_rows).any(|r| self.has_text[r][*c]))
+ .map(|c| col_widths[c])
+ .collect();
+ let median_w = if !nonempty_col_widths.is_empty() {
+ let mut s = nonempty_col_widths.clone();
+ s.sort_by(|a, b| a.total_cmp(b));
+ s[s.len() / 2]
+ } else {
+ let mut s = col_widths.clone();
+ s.sort_by(|a, b| a.total_cmp(b));
+ s[s.len() / 2]
+ };
+ let keep_col: Vec<bool> = (0..n_cols)
+ .map(|c| {
+ let has_text = (0..n_rows).any(|r| self.has_text[r][c]);
+ has_text || col_widths[c] >= median_w * 0.3
+ })
+ .collect();
+ self.retain_cols(&keep_col);
+ }
+}
+
+/// Bin each line's text into the grid cells defined by column boundaries `xs`
+/// and row boundaries `ys`. A projected line frequently spans several ruled
+/// columns (one baseline = one line), so binning whole lines by centroid would
+/// lump an entire row into one cell and leave the rest empty — the empty-cell
+/// filter would then reject the real table. Instead bin each line's raw spans
+/// by span center; spans whose x-extent crosses one or more interior column
+/// boundaries are split at the whitespace nearest each crossed boundary (same
+/// interpolation as the inferred-track path). Returns the filled grid and the
+/// consumed line indices, or `None` if no line landed inside the grid or too
+/// many spans straddle interior boundaries (a decorative box over prose, not a
+/// table).
+fn assign_cells(
+ lines: &[ProjectedLine],
+ xs: &[f32],
+ ys: &[f32],
+ dbg: bool,
+) -> Option<(CellGrid, Vec<usize>)> {
+ let n_rows = ys.len() - 1;
+ let n_cols = xs.len() - 1;
+ let mut grid = CellGrid::new(n_rows, n_cols);
let mut consumed_indices: Vec<usize> = Vec::new();
const GRID_X_SLACK_PT: f32 = 6.0;
// Straddle census: spans that cross an interior column boundary by a
@@ -2652,22 +2749,6 @@ fn build_ruled_table(
let mut span_total = 0usize;
let mut span_straddle = 0usize;
- let push_cell = |cells: &mut Vec<Vec<String>>,
- cell_has_text: &mut Vec<Vec<bool>>,
- row: usize,
- col: usize,
- txt: &str| {
- let txt = txt.trim();
- if txt.is_empty() {
- return;
- }
- if !cells[row][col].is_empty() {
- cells[row][col].push(' ');
- }
- cells[row][col].push_str(txt);
- cell_has_text[row][col] = true;
- };
-
// Iterate lines top-to-bottom. The global pass receives lines in xy_cut
// leaf order (column-by-column), which scrambles a multi-line cell's text
// when concatenated in array order; y-sorted iteration restores reading
@@ -2685,7 +2766,7 @@ fn build_ruled_table(
// default when a span's own y doesn't resolve. Spans bin by their own
// baseline below — a projected line can merge text from several ruled
// rows (e.g. a multi-baseline wrapped header emitted as one line).
- let row = match find_bucket(&ys, cy) {
+ let row = match find_bucket(ys, cy) {
Some(r) => r,
None => continue,
};
@@ -2716,26 +2797,15 @@ fn build_ruled_table(
.filter(|s| !s.text.trim().is_empty())
.collect();
text_spans.sort_by(|a, b| a.x.total_cmp(&b.x));
- let push_repl = |cells_repl: &mut Vec<Vec<String>>, row: usize, col: usize, txt: &str| {
- let txt = txt.trim();
- if txt.is_empty() {
- return;
- }
- let dst = &mut cells_repl[row][col];
- if !dst.is_empty() {
- dst.push(' ');
- }
- dst.push_str(txt);
- };
if text_spans.is_empty() {
// No raw text spans (synthetic/OCR lines): old whole-line centroid path.
let cx = line.bbox.x + line.bbox.width * 0.5;
- if let Some(col) = find_bucket(&xs, cx.clamp(xs[0], xs[n_cols])) {
- push_cell(&mut cells, &mut cell_has_text, row, col, &line.text);
- push_repl(&mut cells_repl, row, col, &line.text);
+ if let Some(col) = find_bucket(xs, cx.clamp(xs[0], xs[n_cols])) {
+ grid.push_text(row, col, &line.text);
+ grid.push_repl(row, col, &line.text);
if !line.all_bold {
- cell_is_bold[row][col] = false;
+ grid.is_bold[row][col] = false;
}
consumed_indices.push(idx);
}
@@ -2746,24 +2816,24 @@ fn build_ruled_table(
// Per-span row: a span carries its own baseline y, which may sit
// in a different ruled row than the line centroid.
let span_cy = span.y + span.height * 0.5;
- let row = find_bucket(&ys, span_cy.clamp(ys[0], ys[n_rows])).unwrap_or(row);
+ let row = find_bucket(ys, span_cy.clamp(ys[0], ys[n_rows])).unwrap_or(row);
let sx0 = (span.x).clamp(xs[0], xs[n_cols]);
let sx1 = (span.x + span.width).clamp(xs[0], xs[n_cols]);
- let c_lo = find_bucket(&xs, sx0).unwrap_or(0);
- let c_hi = find_bucket(&xs, sx1).unwrap_or(n_cols - 1);
+ let c_lo = find_bucket(xs, sx0).unwrap_or(0);
+ let c_hi = find_bucket(xs, sx1).unwrap_or(n_cols - 1);
span_total += 1;
{
let m0 = (span.x + STRADDLE_MARGIN_PT).clamp(xs[0], xs[n_cols]);
let m1 = (span.x + span.width - STRADDLE_MARGIN_PT).clamp(xs[0], xs[n_cols]);
- if m1 > m0 && find_bucket(&xs, m0) != find_bucket(&xs, m1) {
+ if m1 > m0 && find_bucket(xs, m0) != find_bucket(xs, m1) {
span_straddle += 1;
}
}
if c_lo == c_hi {
- push_cell(&mut cells, &mut cell_has_text, row, c_lo, &span.text);
- push_repl(&mut cells_repl, row, c_lo, &span.text);
+ grid.push_text(row, c_lo, &span.text);
+ grid.push_repl(row, c_lo, &span.text);
if !line.all_bold {
- cell_is_bold[row][c_lo] = false;
+ grid.is_bold[row][c_lo] = false;
}
continue;
}
@@ -2772,29 +2842,29 @@ fn build_ruled_table(
// label is alpha-dominant (group headers are words; merged data
// runs are mostly digits).
for col in c_lo..=c_hi {
- push_repl(&mut cells_repl, row, col, &span.text);
+ grid.push_repl(row, col, &span.text);
}
if is_alpha_dominant(&span.text) {
- row_alpha_spanner[row] = true;
+ grid.row_alpha_spanner[row] = true;
}
// Span crosses interior boundaries: split at the whitespace
// nearest each crossed boundary x (xs[k] is column k's left
// boundary, which is exactly the split target).
let covered: Vec<usize> = (c_lo..=c_hi).collect();
- if let Some(pieces) = split_span_at_anchors(span, &covered, &xs) {
+ if let Some(pieces) = split_span_at_anchors(span, &covered, xs) {
for (k, piece) in pieces.iter().enumerate() {
- push_cell(&mut cells, &mut cell_has_text, row, c_lo + k, piece);
+ grid.push_text(row, c_lo + k, piece);
if !line.all_bold {
- cell_is_bold[row][c_lo + k] = false;
+ grid.is_bold[row][c_lo + k] = false;
}
}
} else {
// No whitespace to split on — assign whole span by center.
let cx = (span.x + span.width * 0.5).clamp(xs[0], xs[n_cols]);
- if let Some(col) = find_bucket(&xs, cx) {
- push_cell(&mut cells, &mut cell_has_text, row, col, &span.text);
+ if let Some(col) = find_bucket(xs, cx) {
+ grid.push_text(row, col, &span.text);
if !line.all_bold {
- cell_is_bold[row][col] = false;
+ grid.is_bold[row][col] = false;
}
}
}
@@ -2824,273 +2894,176 @@ fn build_ruled_table(
return None;
}
- // Collapse "phantom rows" produced by stacked thin border-strip rects
- // (doc 149 draws each visual table row as: top border strip ~1pt, body
- // rect ~22pt, bottom border strip ~5pt — each contributes y-coords that
- // survive the 2pt clustering as separate grid rows). Rule: drop a row
- // iff (a) it has no text in any cell AND (b) its height is < 50% of the
- // median non-empty row height. The height gate preserves real
- // fill-in-the-blank forms where empty body rows are full-height.
- let row_heights: Vec<f32> = (0..n_rows).map(|r| ys[r + 1] - ys[r]).collect();
- let nonempty_heights: Vec<f32> = (0..n_rows)
- .filter(|r| cell_has_text[*r].iter().any(|t| *t))
- .map(|r| row_heights[r])
- .collect();
- let median_h = if !nonempty_heights.is_empty() {
- let mut s = nonempty_heights.clone();
- s.sort_by(|a, b| a.total_cmp(b));
- s[s.len() / 2]
- } else {
- let mut s = row_heights.clone();
- s.sort_by(|a, b| a.total_cmp(b));
- s[s.len() / 2]
- };
- let keep: Vec<bool> = (0..n_rows)
- .map(|r| {
- let has_text = cell_has_text[r].iter().any(|t| *t);
- has_text || row_heights[r] >= median_h * 0.8
+ Some((grid, consumed_indices))
+}
+
+/// Colspan header-band flattening (Mode 5 / TRM): a sparse top row whose alpha
+/// spanning label covers several columns ("North America" over its
+/// Revenue/Units sub-columns) followed by a dense label row is a stacked
+/// header. Flatten rows `0..=b` from the colspan-semantics (`repl`) grid into
+/// one header row (per-column top-to-bottom join), so each column carries its
+/// full layer chain ("North America Revenue") — that chain is what header-keyed
+/// evals (and readers) need. Returns `(header_row, body_start)` or `None`.
+fn flatten_header_band(
+ cells: &[Vec<String>],
+ cell_has_text: &[Vec<bool>],
+ cells_repl: &[Vec<String>],
+ row_alpha_spanner: &[bool],
+ n_rows: usize,
+ n_cols: usize,
+ dbg: bool,
+) -> Option<(Vec<String>, usize)> {
+ let row_fill =
+ |r: usize| cell_has_text[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;
+ (0..n_rows)
+ .find(|r| row_fill(*r) >= TABLE_ROW_MIN_FILL)
+ .and_then(|b| {
+ let nonempty = cell_has_text[b].iter().filter(|t| **t).count();
+ let alpha_cells = (0..n_cols)
+ .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))
+ .count();
+ // The bottom header layer may carry digit labels (years like
+ // "2024") but never measurement values. A decimal / % / $ /
+ // dash-placeholder / comma-grouped number in the anchor row means
+ // it's the first DATA row of a table whose real header just
+ // missed the 0.9-fill anchor (colspan header covering <90% of
+ // columns) — folding it would eat a data row (DS5795A_page4).
+ let has_value_cell =
+ (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));
+ let qualifies = (1..=3).contains(&b)
+ && b + 1 < n_rows
+ && (0..b).any(|r| row_alpha_spanner[r])
+ && alpha_cells * 2 >= nonempty
+ && !has_value_cell;
+ if !qualifies {
+ return None;
+ }
+ let header: Vec<String> = (0..n_cols)
+ .map(|c| {
+ let mut parts: Vec<&str> = Vec::new();
+ for row in cells_repl.iter().take(b + 1) {
+ let s = row[c].as_str();
+ if s.is_empty() || parts.last() == Some(&s) {
+ continue;
+ }
+ parts.push(s);
+ }
+ parts.join(" ")
+ })
+ .collect();
+ if header.iter().all(|h| h.is_empty()) {
+ return None;
+ }
+ if dbg {
+ eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");
+ }
+ Some((header, b + 1))
})
- .collect();
- let kept_row_heights: Vec<f32> = (0..n_rows)
- .filter(|r| keep[*r])
- .map(|r| row_heights[r])
- .collect();
- let cells: Vec<Vec<String>> = (0..n_rows)
- .filter(|r| keep[*r])
- .map(|r| cells[r].clone())
- .collect();
- let cell_has_text: Vec<Vec<bool>> = (0..n_rows)
- .filter(|r| keep[*r])
- .map(|r| cell_has_text[r].clone())
- .collect();
- let cell_is_bold: Vec<Vec<bool>> = (0..n_rows)
- .filter(|r| keep[*r])
- .map(|r| cell_is_bold[r].clone())
- .collect();
- let cells_repl: Vec<Vec<String>> = (0..n_rows)
- .filter(|r| keep[*r])
- .map(|r| cells_repl[r].clone())
- .collect();
- let row_alpha_spanner: Vec<bool> = (0..n_rows)
- .filter(|r| keep[*r])
- .map(|r| row_alpha_spanner[r])
- .collect();
- let n_rows = cells.len();
- if n_rows < 2 {
- if dbg {
- eprintln!("[ruled] REJECT rows-after-collapse {n_rows}");
- }
- return None;
- }
+}
- // Mirror the row-collapse rule on columns. Some ruled tables (doc 149)
- // draw their left/right borders as thin strip rects 5pt wide — those
- // become phantom columns with no text. Drop columns that are both empty
- // AND noticeably narrower than the median text-bearing column.
- let col_widths: Vec<f32> = (0..n_cols).map(|c| xs[c + 1] - xs[c]).collect();
- let nonempty_col_widths: Vec<f32> = (0..n_cols)
- .filter(|c| (0..n_rows).any(|r| cell_has_text[r][*c]))
- .map(|c| col_widths[c])
- .collect();
- let median_w = if !nonempty_col_widths.is_empty() {
- let mut s = nonempty_col_widths.clone();
- s.sort_by(|a, b| a.total_cmp(b));
- s[s.len() / 2]
+/// Stacked-header merge (Mode 5): some generators rule every text baseline of a
+/// wrapped header cell, slicing one logical header row into several thin sparse
+/// rows ("Rated" / "Voltage" / "(VDC)" each in its own band). When the top
+/// `k ≥ 2` rows are individually sparse but their union covers most columns AND
+/// a dense data row follows, collapse them into a single header row (per-column
+/// top-to-bottom join). `flattened` is whether `flatten_header_band` already
+/// fired (the two are mutually exclusive). Returns the possibly-merged grid
+/// plus a flag for whether the merge happened.
+#[allow(clippy::type_complexity)]
+fn merge_stacked_header(
+ cells: Vec<Vec<String>>,
+ cell_has_text: Vec<Vec<bool>>,
+ cell_is_bold: Vec<Vec<bool>>,
+ n_rows: usize,
+ n_cols: usize,
+ kept_row_heights: &[f32],
+ flattened: bool,
+ dbg: bool,
+) -> (
+ Vec<Vec<String>>,
+ Vec<Vec<bool>>,
+ Vec<Vec<bool>>,
+ usize,
+ bool,
+) {
+ let row_fill =
+ |r: usize, has: &[Vec<bool>]| has[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;
+ // Anchor on the first fully-dense row (the first real data row).
+ let k = (0..n_rows)
+ .find(|r| row_fill(*r, &cell_has_text) >= TABLE_ROW_MIN_FILL)
+ .unwrap_or(0);
+ let union_cols = (0..n_cols)
+ .filter(|c| (0..k).any(|r| cell_has_text[r][*c]))
+ .count();
+ // Tightness: the band rows must be noticeably shorter than the data
+ // rows below — ruled-per-baseline header bands sit at text leading
+ // (~7pt) while real rows carry cell padding. Without this, a table
+ // whose first body rows are legitimately sparse would get merged.
+ let band_tight = if k >= 2 && k < kept_row_heights.len() {
+ let mut below: Vec<f32> = kept_row_heights[k..].to_vec();
+ below.sort_by(|a, b| a.total_cmp(b));
+ let median_below = below[below.len() / 2];
+ kept_row_heights[..k]
+ .iter()
+ .all(|h| *h <= 0.75 * median_below)
} else {
- let mut s = col_widths.clone();
- s.sort_by(|a, b| a.total_cmp(b));
- s[s.len() / 2]
+ false
};
- let keep_col: Vec<bool> = (0..n_cols)
- .map(|c| {
- let has_text = (0..n_rows).any(|r| cell_has_text[r][c]);
- has_text || col_widths[c] >= median_w * 0.3
- })
- .collect();
- // Cap how aggressively we drop columns. A real table with one phantom
- // border-strip column (doc 149: 5pt-wide left border) drops exactly one.
- // Anything more than that is almost certainly a chart whose vertical
- // grid-lines got merged with text data (doc 078: a chart's 18 V-lines
- // straddle a table to its left/right and collapse to 1 col), so the
- // "table" is bogus — bail and let the borderless detector handle it.
- // Note: keep_col already only drops columns where both (a) no text AND
- // (b) width < 30% of median text-bearing column.
- let cells: Vec<Vec<String>> = cells
- .into_iter()
- .map(|row| {
- row.into_iter()
- .enumerate()
- .filter(|(c, _)| keep_col[*c])
- .map(|(_, v)| v)
- .collect()
- })
- .collect();
- let cell_has_text: Vec<Vec<bool>> = cell_has_text
- .into_iter()
- .map(|row| {
- row.into_iter()
- .enumerate()
- .filter(|(c, _)| keep_col[*c])
- .map(|(_, v)| v)
- .collect()
- })
- .collect();
- let cell_is_bold: Vec<Vec<bool>> = cell_is_bold
- .into_iter()
- .map(|row| {
- row.into_iter()
- .enumerate()
- .filter(|(c, _)| keep_col[*c])
- .map(|(_, v)| v)
- .collect()
- })
- .collect();
- let cells_repl: Vec<Vec<String>> = cells_repl
- .into_iter()
- .map(|row| {
- row.into_iter()
- .enumerate()
- .filter(|(c, _)| keep_col[*c])
- .map(|(_, v)| v)
- .collect()
- })
- .collect();
- let n_cols = cells.first().map(|r| r.len()).unwrap_or(0);
- if n_cols == 0 {
- return None;
- }
- // Single-column tables are ambiguous (could be a captioned card) — require
- // ≥3 rows of geometric + textual evidence.
- if n_cols == 1 && n_rows < 3 {
- return None;
+ if flattened || k < 2 || k >= n_rows || !band_tight || (union_cols as f32) < 0.7 * n_cols as f32
+ {
+ return (cells, cell_has_text, cell_is_bold, n_rows, false);
}
-
- // Colspan header-band flattening (Mode 5 / TRM): a sparse top row whose
- // alpha spanning label covers several columns ("North America" over its
- // Revenue/Units sub-columns) followed by a dense label row is a stacked
- // header. Flatten rows 0..=b from the colspan-semantics grid into one
- // header row (per-column top-to-bottom join), so each column carries its
- // full layer chain ("North America Revenue") — that chain is what
- // header-keyed evals (and readers) need.
- let flattened_header: Option<(Vec<String>, usize)> = {
- let row_fill =
- |r: usize| cell_has_text[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;
- (0..n_rows)
- .find(|r| row_fill(*r) >= TABLE_ROW_MIN_FILL)
- .and_then(|b| {
- let nonempty = cell_has_text[b].iter().filter(|t| **t).count();
- let alpha_cells = (0..n_cols)
- .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))
- .count();
- // The bottom header layer may carry digit labels (years like
- // "2024") but never measurement values. A decimal / % / $ /
- // dash-placeholder / comma-grouped number in the anchor row means
- // it's the first DATA row of a table whose real header just
- // missed the 0.9-fill anchor (colspan header covering <90% of
- // columns) — folding it would eat a data row (DS5795A_page4).
- let has_value_cell =
- (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));
- let qualifies = (1..=3).contains(&b)
- && b + 1 < n_rows
- && (0..b).any(|r| row_alpha_spanner[r])
- && alpha_cells * 2 >= nonempty
- && !has_value_cell;
- if !qualifies {
- return None;
- }
- let header: Vec<String> = (0..n_cols)
- .map(|c| {
- let mut parts: Vec<&str> = Vec::new();
- for row in cells_repl.iter().take(b + 1) {
- let s = row[c].as_str();
- if s.is_empty() || parts.last() == Some(&s) {
- continue;
- }
- parts.push(s);
- }
- parts.join(" ")
- })
- .collect();
- if header.iter().all(|h| h.is_empty()) {
- return None;
- }
- if dbg {
- eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");
+ if dbg {
+ eprintln!("[ruled] stacked-header merge: top {k} rows → 1");
+ }
+ let mut merged_row = vec![String::new(); n_cols];
+ let mut merged_has = vec![false; n_cols];
+ let mut merged_bold = vec![true; n_cols];
+ for r in 0..k {
+ for c in 0..n_cols {
+ if cell_has_text[r][c] {
+ if !merged_row[c].is_empty() {
+ merged_row[c].push(' ');
}
- Some((header, b + 1))
- })
- };
-
- // Stacked-header merge (Mode 5): some generators rule every text baseline
- // of a wrapped header cell, slicing one logical header row into several
- // thin sparse rows ("Rated" / "Voltage" / "(VDC)" each in its own band).
- // When the top k≥2 rows are individually sparse but their union covers
- // most columns AND a dense data row follows, collapse them into a single
- // header row (per-column top-to-bottom join).
- let (cells, cell_has_text, cell_is_bold, n_rows, merged_stacked_header) = {
- let row_fill = |r: usize, has: &Vec<Vec<bool>>| {
- has[r].iter().filter(|t| **t).count() as f32 / n_cols as f32
- };
- // Anchor on the first fully-dense row (the first real data row).
- let k = (0..n_rows)
- .find(|r| row_fill(*r, &cell_has_text) >= TABLE_ROW_MIN_FILL)
- .unwrap_or(0);
- let union_cols = (0..n_cols)
- .filter(|c| (0..k).any(|r| cell_has_text[r][*c]))
- .count();
- // Tightness: the band rows must be noticeably shorter than the data
- // rows below — ruled-per-baseline header bands sit at text leading
- // (~7pt) while real rows carry cell padding. Without this, a table
- // whose first body rows are legitimately sparse would get merged.
- let band_tight = if k >= 2 && k < kept_row_heights.len() {
- let mut below: Vec<f32> = kept_row_heights[k..].to_vec();
- below.sort_by(|a, b| a.total_cmp(b));
- let median_below = below[below.len() / 2];
- kept_row_heights[..k]
- .iter()
- .all(|h| *h <= 0.75 * median_below)
- } else {
- false
- };
- if flattened_header.is_none()
- && k >= 2
- && k < n_rows
- && band_tight
- && union_cols as f32 >= 0.7 * n_cols as f32
- {
- if dbg {
- eprintln!("[ruled] stacked-header merge: top {k} rows → 1");
- }
- let mut merged_row = vec![String::new(); n_cols];
- let mut merged_has = vec![false; n_cols];
- let mut merged_bold = vec![true; n_cols];
- for r in 0..k {
- for c in 0..n_cols {
- if cell_has_text[r][c] {
- if !merged_row[c].is_empty() {
- merged_row[c].push(' ');
- }
- merged_row[c].push_str(&cells[r][c]);
- merged_has[c] = true;
- if !cell_is_bold[r][c] {
- merged_bold[c] = false;
- }
- }
+ merged_row[c].push_str(&cells[r][c]);
+ merged_has[c] = true;
+ if !cell_is_bold[r][c] {
+ merged_bold[c] = false;
}
}
- let mut new_cells = vec![merged_row];
- let mut new_has = vec![merged_has];
- let mut new_bold = vec![merged_bold];
- new_cells.extend(cells[k..].iter().cloned());
- new_has.extend(cell_has_text[k..].iter().cloned());
- new_bold.extend(cell_is_bold[k..].iter().cloned());
- let nr = new_cells.len();
- (new_cells, new_has, new_bold, nr, true)
- } else {
- (cells, cell_has_text, cell_is_bold, n_rows, false)
}
- };
+ }
+ let mut new_cells = vec![merged_row];
+ let mut new_has = vec![merged_has];
+ let mut new_bold = vec![merged_bold];
+ new_cells.extend(cells[k..].iter().cloned());
+ new_has.extend(cell_has_text[k..].iter().cloned());
+ new_bold.extend(cell_is_bold[k..].iter().cloned());
+ let nr = new_cells.len();
+ (new_cells, new_has, new_bold, nr, true)
+}
+/// Density gate: a grid that is mostly empty cells is rejected unless it shows
+/// strong table evidence. Three escape hatches keep real tables:
+/// - **col0 spine**: a filled, short-text first column (a label column).
+/// - **long-prose table**: a large (≥5×3) grid with a bold header band covering
+/// ≥3 columns and a dense (≥70%-fill) inner description column — a
+/// multi-line legal/reference table whose description wraps over many empty
+/// continuation rows (docs 088/089/090).
+/// - **flattened header**: a fired colspan header flatten is table evidence on
+/// par with a spine.
+/// With a spine but above the higher WITH_SPINE ceiling, still reject (unless a
+/// long-prose table). `flattened` is whether `flatten_header_band` fired.
+/// Returns `true` to keep the table, `false` to reject it.
+fn passes_density_gate(
+ cells: &[Vec<String>],
+ cell_has_text: &[Vec<bool>],
+ cell_is_bold: &[Vec<bool>],
+ n_rows: usize,
+ n_cols: usize,
+ flattened: bool,
+ dbg: bool,
+) -> bool {
let total = n_rows * n_cols;
let empty_count = cell_has_text
.iter()
@@ -3098,78 +3071,233 @@ fn build_ruled_table(
.filter(|filled| !**filled)
.count();
let empty_frac = (empty_count as f32) / (total as f32);
- if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION {
- let col0_fill = (0..n_rows).filter(|r| cell_has_text[*r][0]).count() as f32 / n_rows as f32;
- let col0_max_chars = (0..n_rows)
- .filter(|r| cell_has_text[*r][0])
- .map(|r| cells[r][0].len())
- .max()
- .unwrap_or(0);
- let col0_spine =
- col0_fill >= TABLE_SPINE_FILL_FRACTION && col0_max_chars <= TABLE_SPINE_MAX_CELL_CHARS;
- // Long-prose table bypass: a large grid (≥5×3) with a structurally
- // bold row 0 header AND a dense description column (any col with
- // ≥70% fill rate) is almost certainly a multi-line legal/reference
- // table where the description column wraps over many empty
- // continuation rows. Density arbitration in `merge_table_runs`
- // prevents a decorative grid that happens to match these criteria
- // from beating a real overlapping borderless table.
- // Reproducer: docs 088/089/090 multi-page legal report.
- // Header may span multiple visual rows (the grid detector slices on
- // each text baseline). Treat the first ≤4 rows as the header band
- // and require their *union* to cover most columns AND be all-bold.
- let header_band = n_rows.min(4);
- let mut header_cols_covered = vec![false; n_cols];
- let mut header_all_bold = true;
- for r in 0..header_band {
- for c in 0..n_cols {
- if cell_has_text[r][c] {
- header_cols_covered[c] = true;
- if !cell_is_bold[r][c] {
- header_all_bold = false;
- }
+ if empty_frac <= TABLE_MAX_EMPTY_CELL_FRACTION {
+ return true;
+ }
+ let col0_fill = (0..n_rows).filter(|r| cell_has_text[*r][0]).count() as f32 / n_rows as f32;
+ let col0_max_chars = (0..n_rows)
+ .filter(|r| cell_has_text[*r][0])
+ .map(|r| cells[r][0].len())
+ .max()
+ .unwrap_or(0);
+ let col0_spine =
+ col0_fill >= TABLE_SPINE_FILL_FRACTION && col0_max_chars <= TABLE_SPINE_MAX_CELL_CHARS;
+ // Header may span multiple visual rows (the grid detector slices on each
+ // text baseline). Treat the first ≤4 rows as the header band and require
+ // their *union* to cover most columns AND be all-bold.
+ let header_band = n_rows.min(4);
+ let mut header_cols_covered = vec![false; n_cols];
+ let mut header_all_bold = true;
+ for r in 0..header_band {
+ for c in 0..n_cols {
+ if cell_has_text[r][c] {
+ header_cols_covered[c] = true;
+ if !cell_is_bold[r][c] {
+ header_all_bold = false;
}
}
}
- let header_coverage = header_cols_covered.iter().filter(|t| **t).count();
- let dense_inner_col = (1..n_cols).any(|c| {
- let col_fill =
- (0..n_rows).filter(|r| cell_has_text[*r][c]).count() as f32 / n_rows as f32;
- col_fill >= TABLE_SPINE_FILL_FRACTION
- });
- // Header coverage doesn't need to span every column — wide-cell
- // legal tables often spread the header across many visual baselines
- // and only a few columns land in the top-4-rows band. Require ≥3
- // columns covered as evidence of a real header, not just a title.
- let long_prose_table = n_rows >= 5
- && n_cols >= 3
- && header_coverage >= 3
- && header_all_bold
- && dense_inner_col;
- // A fired colspan header flatten is table evidence on par with a
- // label spine: it required a ≥0.9-fill alpha label row under an
- // alpha spanning group label — decorative grids don't have that.
- // Still subject to the WITH_SPINE ceiling below.
- let flattened_evidence = flattened_header.is_some();
- if !col0_spine && !long_prose_table && !flattened_evidence {
- if dbg {
- let fills: Vec<usize> = (0..n_rows)
- .map(|r| cell_has_text[r].iter().filter(|t| **t).count())
- .collect();
- eprintln!(
- "[ruled] REJECT empty-frac {empty_frac:.2} ({n_rows}x{n_cols}, no spine/long-prose) row_fills={fills:?}"
- );
- }
- return None;
+ }
+ let header_coverage = header_cols_covered.iter().filter(|t| **t).count();
+ let dense_inner_col = (1..n_cols).any(|c| {
+ let col_fill = (0..n_rows).filter(|r| cell_has_text[*r][c]).count() as f32 / n_rows as f32;
+ col_fill >= TABLE_SPINE_FILL_FRACTION
+ });
+ // Header coverage doesn't need to span every column — wide-cell legal
+ // tables often spread the header across many visual baselines and only a
+ // few columns land in the top-4-rows band. Require ≥3 columns covered as
+ // evidence of a real header, not just a title.
+ let long_prose_table =
+ n_rows >= 5 && n_cols >= 3 && header_coverage >= 3 && header_all_bold && dense_inner_col;
+ if !col0_spine && !long_prose_table && !flattened {
+ if dbg {
+ let fills: Vec<usize> = (0..n_rows)
+ .map(|r| cell_has_text[r].iter().filter(|t| **t).count())
+ .collect();
+ eprintln!(
+ "[ruled] REJECT empty-frac {empty_frac:.2} ({n_rows}x{n_cols}, no spine/long-prose) row_fills={fills:?}"
+ );
+ }
+ return false;
+ }
+ if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION_WITH_SPINE && !long_prose_table {
+ if dbg {
+ eprintln!("[ruled] REJECT empty-frac-with-spine {empty_frac:.2}");
}
- if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION_WITH_SPINE && !long_prose_table {
+ return false;
+ }
+ true
+}
+
+/// Build a `TableRun` for one ruled-grid component. Returns `None` if the
+/// resulting grid is too small (< 2 cols or < 2 rows), covers nearly the
+/// whole page (likely the page border), or is mostly empty cells.
+fn build_ruled_table(
+ hs: &[HSeg],
+ vs: &[VSeg],
+ h_indices: &[usize],
+ v_indices: &[usize],
+ lines: &[ProjectedLine],
+ page_width: f32,
+ page_height: f32,
+ pass: RuledPass,
+) -> Option<(TableRun, Vec<usize>)> {
+ let dbg = *super::flags::DEBUG_RULED;
+ let mut xs: Vec<f32> = v_indices.iter().map(|&i| vs[i].x).collect();
+ xs.sort_by(|a, b| a.total_cmp(b));
+ // Coarser, mean-centered clustering for column boundaries: cell-border
+ // rects contribute paired edges 4-6pt apart that would otherwise become
+ // phantom 5pt "columns" the span splitter then shreds text into.
+ cluster_boundaries(&mut xs, TABLE_COL_BOUNDARY_CLUSTER_PT);
+
+ // Distinct row y-coords (cluster again — multiple H lines may share a y).
+ // In the global pass, first drop horizontal rules that span only a small
+ // fraction of the column extent (see `RULED_HLINE_MIN_COVERAGE`).
+ let raw_ys = |idxs: &[usize]| {
+ let mut v: Vec<f32> = idxs.iter().map(|&i| hs[i].y).collect();
+ v.sort_by(|a, b| a.total_cmp(b));
+ dedup_close(&mut v, TABLE_GRID_CLUSTER_PT);
+ v
+ };
+ let ys: Vec<f32> = if pass == RuledPass::Global && xs.len() >= 2 {
+ let col_lo = xs[0];
+ let col_hi = xs[xs.len() - 1];
+ let extent = (col_hi - col_lo).max(1.0);
+ let kept: Vec<usize> = h_indices
+ .iter()
+ .copied()
+ .filter(|&i| {
+ let h = &hs[i];
+ let ov = (h.x_max.min(col_hi) - h.x_min.max(col_lo)).max(0.0);
+ ov / extent >= RULED_HLINE_MIN_COVERAGE
+ })
+ .collect();
+ let filtered = raw_ys(&kept);
+ if filtered.len() >= 3 {
+ filtered
+ } else {
+ raw_ys(h_indices)
+ }
+ } else {
+ raw_ys(h_indices)
+ };
+ if dbg {
+ eprintln!(
+ "[ruled] component: ys={:?} xs={:?} ({} lines in scope)",
+ ys,
+ xs,
+ lines.len()
+ );
+ }
+
+ // Need ≥2 row boundaries (1 row) and ≥2 column boundaries (1 col); but
+ // a 1×1 grid is just a callout box, so also require ≥1 inner divider
+ // (i.e. ys.len() ≥ 3 for ≥2 rows). Single-column tables (`xs.len() == 2`)
+ // are accepted when row evidence is strong enough — extra guards apply
+ // below after the empty-row collapse.
+ if ys.len() < 3 || xs.len() < 2 {
+ if dbg {
+ eprintln!(
+ "[ruled] REJECT grid-too-small ys={} xs={}",
+ ys.len(),
+ xs.len()
+ );
+ }
+ return None;
+ }
+
+ let n_rows = ys.len() - 1;
+ let n_cols = xs.len() - 1;
+ let bbox = crate::types::Rect {
+ x: xs[0],
+ y: ys[0],
+ width: xs[n_cols] - xs[0],
+ height: ys[n_rows] - ys[0],
+ };
+
+ // Reject page-border-as-table.
+ if page_width > 0.0 && page_height > 0.0 {
+ let coverage = (bbox.width / page_width) * (bbox.height / page_height);
+ if coverage > TABLE_MAX_PAGE_COVERAGE {
if dbg {
- eprintln!("[ruled] REJECT empty-frac-with-spine {empty_frac:.2}");
+ eprintln!("[ruled] REJECT page-coverage {coverage:.2}");
}
return None;
}
}
+ // Assign text to cells, then reject if no lines landed inside the grid or
+ // too many spans straddle interior column boundaries (decorative box, not
+ // a table).
+ let (mut grid, consumed_indices) = assign_cells(lines, &xs, &ys, dbg)?;
+
+ // Collapse phantom rows (text-less thin border-strip rects) then phantom
+ // columns (text-less narrow border strips). A real table with one phantom
+ // border-strip column (doc 149) drops exactly one; a chart whose vertical
+ // grid-lines merged with text data (doc 078) collapses to 1 col and is
+ // rejected below, letting the borderless detector handle it.
+ let kept_row_heights = grid.collapse_phantom_rows(&ys);
+ let n_rows = grid.n_rows();
+ if n_rows < 2 {
+ if dbg {
+ eprintln!("[ruled] REJECT rows-after-collapse {n_rows}");
+ }
+ return None;
+ }
+ grid.collapse_phantom_cols(&xs);
+ let n_cols = grid.n_cols();
+ if n_cols == 0 {
+ return None;
+ }
+ // Single-column tables are ambiguous (could be a captioned card) — require
+ // ≥3 rows of geometric + textual evidence.
+ if n_cols == 1 && n_rows < 3 {
+ return None;
+ }
+
+ // Hand the collapsed grids back to plain locals for the header-detection
+ // stages below.
+ let CellGrid {
+ text: cells,
+ is_bold: cell_is_bold,
+ has_text: cell_has_text,
+ repl: cells_repl,
+ row_alpha_spanner,
+ } = grid;
+
+ let flattened_header = flatten_header_band(
+ &cells,
+ &cell_has_text,
+ &cells_repl,
+ &row_alpha_spanner,
+ n_rows,
+ n_cols,
+ dbg,
+ );
+
+ let (cells, cell_has_text, cell_is_bold, n_rows, merged_stacked_header) = merge_stacked_header(
+ cells,
+ cell_has_text,
+ cell_is_bold,
+ n_rows,
+ n_cols,
+ &kept_row_heights,
+ flattened_header.is_some(),
+ dbg,
+ );
+
+ if !passes_density_gate(
+ &cells,
+ &cell_has_text,
+ &cell_is_bold,
+ n_rows,
+ n_cols,
+ flattened_header.is_some(),
+ dbg,
+ ) {
+ return None;
+ }
+
// Header preference order: flattened colspan band (carries the full
// per-column layer chain) > merged stacked-header band > bold first row.
let header_qualifies = merged_stacked_header
@@ -3328,7 +3456,7 @@ fn detect_ruled_tables_impl(
graphics: &[GraphicPrimitive],
page_width: f32,
page_height: f32,
- global: bool,
+ pass: RuledPass,
) -> Vec<(TableRun, Vec<usize>)> {
let (hs, vs) = extract_h_v_segments(graphics);
let hs = cluster_h_segments(hs);
@@ -3347,7 +3475,7 @@ fn detect_ruled_tables_impl(
lines,
page_width,
page_height,
- global,
+ pass,
) {
out.push(run);
}
@@ -3364,10 +3492,16 @@ pub(super) fn detect_ruled_tables(
page_width: f32,
page_height: f32,
) -> Vec<TableRun> {
- detect_ruled_tables_impl(lines, graphics, page_width, page_height, false)
- .into_iter()
- .map(|(r, _)| r)
- .collect()
+ detect_ruled_tables_impl(
+ lines,
+ graphics,
+ page_width,
+ page_height,
+ RuledPass::PerRegion,
+ )
+ .into_iter()
+ .map(|(r, _)| r)
+ .collect()
}
/// Page-level ruled-table detection over *all* lines. A table whose rows
@@ -3381,7 +3515,7 @@ pub(super) fn detect_ruled_tables_global(
page_width: f32,
page_height: f32,
) -> Vec<(TableRun, Vec<usize>)> {
- detect_ruled_tables_impl(lines, graphics, page_width, page_height, true)
+ detect_ruled_tables_impl(lines, graphics, page_width, page_height, RuledPass::Global)
}
/// Count filled (non-empty) cells in a TableRun. GridFallback returns 0 so