A fast, helpful, and open-source document parser
final clean up
Logan Markewichabe987282953
diff --git a/crates/liteparse/src/font_cmap.rs b/crates/liteparse/src/font_cmap.rsnew file mode 100644index 0000000..8fe25dc--- /dev/null+++ b/crates/liteparse/src/font_cmap.rs@@ -0,0 +1,219 @@+//! Reverse-cmap recovery: parse an embedded sfnt (TrueType/OpenType) font+//! program's `cmap` table and invert it to a glyph-index → unicode map.+//!+//! Used when a font's /ToUnicode is missing or garbage AND its glyph names+//! are unavailable (typical of CID TrueType subsets). The embedded font's own+//! character map is the last structural source of truth tying glyphs back to+//! unicode. Pure Rust, no deps; only formats 4, 12, 6 and 0 are parsed (these+//! cover essentially all real-world fonts).++use std::collections::HashMap;++/// Build glyph_index → unicode from an sfnt font program. Returns None when+/// the data is not sfnt (e.g. bare CFF) or has no usable cmap subtable.+pub fn reverse_cmap(data: &[u8]) -> Option<HashMap<u32, u32>> {+ let font = sfnt_slice(data)?;+ let cmap = find_table(font, b"cmap")?;++ let num_subtables = read_u16(cmap, 2)? as usize;+ // Pick the best unicode subtable: full-repertoire (fmt 12) beats BMP (fmt 4).+ let mut best: Option<(u32, u32)> = None; // (score, offset)+ for i in 0..num_subtables {+ let rec = 4 + i * 8;+ let platform = read_u16(cmap, rec)?;+ let encoding = read_u16(cmap, rec + 2)?;+ let offset = read_u32(cmap, rec + 4)?;+ // Only true unicode subtables. Mac Roman (1,0) and symbol (3,0) cmaps+ // encode charcodes, not unicode — reversing them echoes garbage (e.g.+ // Wingdings (1,0) maps the checkmark glyph back to 'ü').+ let score = match (platform, encoding) {+ (3, 10) | (0, 4) | (0, 6) => 4, // UCS-4+ (3, 1) | (0, 0..=3) => 3, // BMP+ _ => 0,+ };+ if score > 0 && best.is_none_or(|(s, _)| score > s) {+ best = Some((score, offset));+ }+ }+ let (_, offset) = best?;+ let sub = cmap.get(offset as usize..)?;++ let mut map: HashMap<u32, u32> = HashMap::new();+ let mut add = |glyph: u32, unicode: u32| {+ if glyph == 0 || unicode == 0 {+ return;+ }+ // On collision prefer non-PUA, then the smaller codepoint (ASCII /+ // canonical forms over compatibility duplicates).+ match map.entry(glyph) {+ std::collections::hash_map::Entry::Vacant(e) => {+ e.insert(unicode);+ }+ std::collections::hash_map::Entry::Occupied(mut e) => {+ let cur = *e.get();+ let cur_pua = (0xE000..=0xF8FF).contains(&cur);+ let new_pua = (0xE000..=0xF8FF).contains(&unicode);+ if (cur_pua && !new_pua) || (cur_pua == new_pua && unicode < cur) {+ e.insert(unicode);+ }+ }+ }+ };++ match read_u16(sub, 0)? {+ 0 => {+ // Byte encoding table: 256 glyph ids at offset 6+ for code in 0..256u32 {+ let g = *sub.get(6 + code as usize)? as u32;+ add(g, code);+ }+ }+ 4 => {+ let seg_count = (read_u16(sub, 6)? / 2) as usize;+ let end_codes = 14;+ let start_codes = end_codes + seg_count * 2 + 2;+ let id_deltas = start_codes + seg_count * 2;+ let id_range_offsets = id_deltas + seg_count * 2;+ for seg in 0..seg_count {+ let end = read_u16(sub, end_codes + seg * 2)? as u32;+ let start = read_u16(sub, start_codes + seg * 2)? as u32;+ let delta = read_u16(sub, id_deltas + seg * 2)? as u32;+ let range_offset = read_u16(sub, id_range_offsets + seg * 2)? as usize;+ if start == 0xFFFF && end == 0xFFFF {+ continue;+ }+ for code in start..=end.min(0xFFFE) {+ let glyph = if range_offset == 0 {+ (code + delta) & 0xFFFF+ } else {+ // glyphIdArray indexing relative to this rangeOffset slot+ let slot =+ id_range_offsets + seg * 2 + range_offset + (code - start) as usize * 2;+ let g = read_u16(sub, slot)? as u32;+ if g == 0 { 0 } else { (g + delta) & 0xFFFF }+ };+ add(glyph, code);+ }+ }+ }+ 6 => {+ let first = read_u16(sub, 6)? as u32;+ let count = read_u16(sub, 8)? as usize;+ for i in 0..count {+ let g = read_u16(sub, 10 + i * 2)? as u32;+ add(g, first + i as u32);+ }+ }+ 12 => {+ let n_groups = read_u32(sub, 12)? as usize;+ for i in 0..n_groups.min(100_000) {+ let rec = 16 + i * 12;+ let start = read_u32(sub, rec)?;+ let end = read_u32(sub, rec + 4)?;+ let start_glyph = read_u32(sub, rec + 8)?;+ if end < start || end - start > 0x10000 {+ continue;+ }+ for off in 0..=(end - start) {+ add(start_glyph + off, start + off);+ }+ }+ }+ _ => return None,+ }++ if map.is_empty() { None } else { Some(map) }+}++/// Resolve TTC wrappers and validate the sfnt magic.+fn sfnt_slice(data: &[u8]) -> Option<&[u8]> {+ let tag = data.get(..4)?;+ if tag == b"ttcf" {+ let first = read_u32(data, 12)? as usize;+ let sub = data.get(first..)?;+ let m = sub.get(..4)?;+ return (m == [0, 1, 0, 0] || m == b"OTTO" || m == b"true").then_some(sub);+ }+ (tag == [0, 1, 0, 0] || tag == b"OTTO" || tag == b"true").then_some(data)+}++fn find_table<'a>(font: &'a [u8], tag: &[u8; 4]) -> Option<&'a [u8]> {+ let num_tables = read_u16(font, 4)? as usize;+ for i in 0..num_tables {+ let rec = 12 + i * 16;+ if font.get(rec..rec + 4)? == tag {+ let offset = read_u32(font, rec + 8)? as usize;+ let length = read_u32(font, rec + 12)? as usize;+ return font.get(offset..offset.checked_add(length)?);+ }+ }+ None+}++fn read_u16(data: &[u8], offset: usize) -> Option<u16> {+ let b = data.get(offset..offset + 2)?;+ Some(u16::from_be_bytes([b[0], b[1]]))+}++fn read_u32(data: &[u8], offset: usize) -> Option<u32> {+ let b = data.get(offset..offset + 4)?;+ Some(u32::from_be_bytes([b[0], b[1], b[2], b[3]]))+}++#[cfg(test)]+mod tests {+ use super::*;++ /// Build a minimal sfnt with a single format-4 cmap subtable mapping+ /// 'A'..='C' (0x41..0x43) to glyphs 10..12.+ fn minimal_format4_font() -> Vec<u8> {+ // format 4 subtable: one real segment + terminator segment+ let mut sub: Vec<u8> = Vec::new();+ sub.extend([0, 4]); // format+ sub.extend([0, 0]); // length (unchecked)+ sub.extend([0, 0]); // language+ sub.extend([0, 4]); // segCountX2 = 4 (2 segments)+ sub.extend([0, 0, 0, 0, 0, 0]); // searchRange/entrySelector/rangeShift+ sub.extend([0x00, 0x43, 0xFF, 0xFF]); // endCodes+ sub.extend([0, 0]); // reservedPad+ sub.extend([0x00, 0x41, 0xFF, 0xFF]); // startCodes+ // idDelta: glyph = code + delta mod 65536; 10 - 0x41 = -55 = 0xFFC9+ sub.extend([0xFF, 0xC9, 0x00, 0x01]); // idDeltas+ sub.extend([0, 0, 0, 0]); // idRangeOffsets++ let mut cmap: Vec<u8> = Vec::new();+ cmap.extend([0, 0]); // version+ cmap.extend([0, 1]); // numTables+ cmap.extend([0, 3, 0, 1]); // platform 3, encoding 1+ cmap.extend(12u32.to_be_bytes()); // subtable offset+ cmap.extend(&sub);++ let mut font: Vec<u8> = Vec::new();+ font.extend([0, 1, 0, 0]); // sfnt version+ font.extend([0, 1]); // numTables+ font.extend([0, 0, 0, 0, 0, 0]); // search fields+ font.extend(b"cmap");+ font.extend([0, 0, 0, 0]); // checksum+ font.extend(28u32.to_be_bytes()); // offset (12 + 16)+ font.extend((cmap.len() as u32).to_be_bytes());+ font.extend(&cmap);+ font+ }++ #[test]+ fn parses_format4_and_inverts() {+ let font = minimal_format4_font();+ let map = reverse_cmap(&font).unwrap();+ assert_eq!(map.get(&10), Some(&0x41)); // A+ assert_eq!(map.get(&11), Some(&0x42)); // B+ assert_eq!(map.get(&12), Some(&0x43)); // C+ assert!(!map.contains_key(&0));+ }++ #[test]+ fn rejects_non_sfnt() {+ assert!(reverse_cmap(b"%!PS-AdobeFont").is_none());+ assert!(reverse_cmap(&[1, 0, 0, 0]).is_none()); // bare CFF header+ assert!(reverse_cmap(&[]).is_none());+ }+}diff --git a/crates/liteparse/src/glyph_names.rs b/crates/liteparse/src/glyph_names.rsnew file mode 100644index 0000000..cee16d3--- /dev/null+++ b/crates/liteparse/src/glyph_names.rs@@ -0,0 +1,490 @@+//! PostScript glyph-name → Unicode resolution (Adobe Glyph List conventions).+//!+//! Used to recover text when a font has no usable /ToUnicode CMap. Glyph names+//! come from the PDF's /Encoding /Differences array or the embedded font+//! program (via the fork's `FPDFFont_GetCharGlyphName`).+//!+//! Implements the AGL "reverse mapping" algorithm:+//! 1. Strip any variant suffix after the first '.' ("fi.alt" → "fi").+//! 2. Split on '_' and resolve each component ("f_i" → "fi").+//! 3. Per component: `uni` + 4n hex digits, `u` + 4-6 hex digits, or a lookup+//! in the static AGL subset table below.++/// Resolve a glyph name to its Unicode string. Returns None when the name is+/// meaningless (subset-generated names like "g42", "cid1234") or unknown.+pub fn resolve_glyph_name(name: &str) -> Option<String> {+ let base = name.split('.').next()?;+ if base.is_empty() {+ return None;+ }+ let mut out = String::new();+ for component in base.split('_') {+ for c in resolve_component(component)?.chars() {+ match presentation_form_expansion(c) {+ Some(s) => out.push_str(s),+ None => out.push(c),+ }+ }+ }+ if out.is_empty() { None } else { Some(out) }+}++/// ASCII expansion for ligature presentation forms, matching the existing+/// extraction-time ligature handling.+pub fn presentation_form_expansion(c: char) -> Option<&'static str> {+ match c {+ '\u{FB00}' => Some("ff"),+ '\u{FB01}' => Some("fi"),+ '\u{FB02}' => Some("fl"),+ '\u{FB03}' => Some("ffi"),+ '\u{FB04}' => Some("ffl"),+ '\u{FB05}' | '\u{FB06}' => Some("st"),+ _ => None,+ }+}++fn resolve_component(comp: &str) -> Option<String> {+ if comp.is_empty() {+ return None;+ }++ // uniXXXX (one or more groups of exactly 4 uppercase hex digits)+ if let Some(hex) = comp.strip_prefix("uni") {+ if !hex.is_empty() && hex.len() % 4 == 0 && hex.bytes().all(is_agl_hex_digit) {+ let mut s = String::new();+ for group in hex.as_bytes().chunks(4) {+ let v = u32::from_str_radix(std::str::from_utf8(group).ok()?, 16).ok()?;+ // Surrogate range is invalid scalar values+ s.push(char::from_u32(v)?);+ }+ return Some(s);+ }+ }++ // uXXXX / uXXXXX / uXXXXXX+ if let Some(hex) = comp.strip_prefix('u') {+ if (4..=6).contains(&hex.len()) && hex.bytes().all(is_agl_hex_digit) {+ let v = u32::from_str_radix(hex, 16).ok()?;+ return char::from_u32(v).map(String::from);+ }+ }++ // Static table lookup+ if let Ok(idx) = AGL_SUBSET.binary_search_by(|(n, _)| (*n).cmp(comp)) {+ return Some(AGL_SUBSET[idx].1.to_string());+ }++ None+}++fn is_agl_hex_digit(b: u8) -> bool {+ b.is_ascii_digit() || (b'A'..=b'F').contains(&b)+}++/// Curated subset of the Adobe Glyph List, sorted by name for binary search.+/// Covers ASCII, Latin-1/Latin Extended accents, ligatures, common punctuation+/// and symbols, and Greek — the ranges that show up in real-world Differences+/// arrays and embedded Latin font programs.+static AGL_SUBSET: &[(&str, &str)] = &[+ ("A", "A"),+ ("AE", "Æ"),+ ("Aacute", "Á"),+ ("Abreve", "Ă"),+ ("Acircumflex", "Â"),+ ("Adieresis", "Ä"),+ ("Agrave", "À"),+ ("Alpha", "Α"),+ ("Amacron", "Ā"),+ ("Aogonek", "Ą"),+ ("Aring", "Å"),+ ("Atilde", "Ã"),+ ("B", "B"),+ ("Beta", "Β"),+ ("C", "C"),+ ("Cacute", "Ć"),+ ("Ccaron", "Č"),+ ("Ccedilla", "Ç"),+ ("Chi", "Χ"),+ ("D", "D"),+ ("Dcaron", "Ď"),+ ("Dcroat", "Đ"),+ ("Delta", "Δ"),+ ("E", "E"),+ ("Eacute", "É"),+ ("Ecaron", "Ě"),+ ("Ecircumflex", "Ê"),+ ("Edieresis", "Ë"),+ ("Egrave", "È"),+ ("Emacron", "Ē"),+ ("Eogonek", "Ę"),+ ("Epsilon", "Ε"),+ ("Eta", "Η"),+ ("Eth", "Ð"),+ ("Euro", "€"),+ ("F", "F"),+ ("G", "G"),+ ("Gamma", "Γ"),+ ("Gbreve", "Ğ"),+ ("H", "H"),+ ("I", "I"),+ ("Iacute", "Í"),+ ("Icircumflex", "Î"),+ ("Idieresis", "Ï"),+ ("Idotaccent", "İ"),+ ("Igrave", "Ì"),+ ("Imacron", "Ī"),+ ("Iota", "Ι"),+ ("J", "J"),+ ("K", "K"),+ ("Kappa", "Κ"),+ ("L", "L"),+ ("Lacute", "Ĺ"),+ ("Lambda", "Λ"),+ ("Lcaron", "Ľ"),+ ("Lslash", "Ł"),+ ("M", "M"),+ ("Mu", "Μ"),+ ("N", "N"),+ ("Nacute", "Ń"),+ ("Ncaron", "Ň"),+ ("Ntilde", "Ñ"),+ ("Nu", "Ν"),+ ("O", "O"),+ ("OE", "Œ"),+ ("Oacute", "Ó"),+ ("Ocircumflex", "Ô"),+ ("Odieresis", "Ö"),+ ("Ograve", "Ò"),+ ("Ohungarumlaut", "Ő"),+ ("Omacron", "Ō"),+ ("Omega", "Ω"),+ ("Omicron", "Ο"),+ ("Oslash", "Ø"),+ ("Otilde", "Õ"),+ ("P", "P"),+ ("Phi", "Φ"),+ ("Pi", "Π"),+ ("Psi", "Ψ"),+ ("Q", "Q"),+ ("R", "R"),+ ("Racute", "Ŕ"),+ ("Rcaron", "Ř"),+ ("Rho", "Ρ"),+ ("S", "S"),+ ("Sacute", "Ś"),+ ("Scaron", "Š"),+ ("Scedilla", "Ş"),+ ("Sigma", "Σ"),+ ("T", "T"),+ ("Tau", "Τ"),+ ("Tbar", "Ŧ"),+ ("Tcaron", "Ť"),+ ("Theta", "Θ"),+ ("Thorn", "Þ"),+ ("U", "U"),+ ("Uacute", "Ú"),+ ("Ucircumflex", "Û"),+ ("Udieresis", "Ü"),+ ("Ugrave", "Ù"),+ ("Uhungarumlaut", "Ű"),+ ("Umacron", "Ū"),+ ("Uogonek", "Ų"),+ ("Upsilon", "Υ"),+ ("Uring", "Ů"),+ ("V", "V"),+ ("W", "W"),+ ("X", "X"),+ ("Xi", "Ξ"),+ ("Y", "Y"),+ ("Yacute", "Ý"),+ ("Ydieresis", "Ÿ"),+ ("Z", "Z"),+ ("Zacute", "Ź"),+ ("Zcaron", "Ž"),+ ("Zdotaccent", "Ż"),+ ("Zeta", "Ζ"),+ ("a", "a"),+ ("aacute", "á"),+ ("abreve", "ă"),+ ("acircumflex", "â"),+ ("acute", "´"),+ ("adieresis", "ä"),+ ("ae", "æ"),+ ("agrave", "à"),+ ("alpha", "α"),+ ("amacron", "ā"),+ ("ampersand", "&"),+ ("aogonek", "ą"),+ ("approxequal", "≈"),+ ("aring", "å"),+ ("asciicircum", "^"),+ ("asciitilde", "~"),+ ("asterisk", "*"),+ ("at", "@"),+ ("atilde", "ã"),+ ("b", "b"),+ ("backslash", "\\"),+ ("bar", "|"),+ ("beta", "β"),+ ("braceleft", "{"),+ ("braceright", "}"),+ ("bracketleft", "["),+ ("bracketright", "]"),+ ("breve", "˘"),+ ("brokenbar", "¦"),+ ("bullet", "•"),+ ("c", "c"),+ ("cacute", "ć"),+ ("caron", "ˇ"),+ ("ccaron", "č"),+ ("ccedilla", "ç"),+ ("cedilla", "¸"),+ ("cent", "¢"),+ ("chi", "χ"),+ ("circumflex", "ˆ"),+ ("colon", ":"),+ ("comma", ","),+ ("copyright", "©"),+ ("currency", "¤"),+ ("d", "d"),+ ("dagger", "†"),+ ("daggerdbl", "‡"),+ ("dcaron", "ď"),+ ("dcroat", "đ"),+ ("degree", "°"),+ ("delta", "δ"),+ ("dieresis", "¨"),+ ("divide", "÷"),+ ("dollar", "$"),+ ("dotaccent", "˙"),+ ("dotlessi", "ı"),+ ("e", "e"),+ ("eacute", "é"),+ ("ecaron", "ě"),+ ("ecircumflex", "ê"),+ ("edieresis", "ë"),+ ("egrave", "è"),+ ("eight", "8"),+ ("ellipsis", "…"),+ ("emacron", "ē"),+ ("emdash", "—"),+ ("endash", "–"),+ ("eogonek", "ę"),+ ("epsilon", "ε"),+ ("equal", "="),+ ("eta", "η"),+ ("eth", "ð"),+ ("exclam", "!"),+ ("exclamdown", "¡"),+ ("f", "f"),+ ("ff", "ff"),+ ("ffi", "ffi"),+ ("ffl", "ffl"),+ ("fi", "fi"),+ ("five", "5"),+ ("fl", "fl"),+ ("florin", "ƒ"),+ ("four", "4"),+ ("fraction", "⁄"),+ ("g", "g"),+ ("gamma", "γ"),+ ("gbreve", "ğ"),+ ("germandbls", "ß"),+ ("grave", "`"),+ ("greater", ">"),+ ("greaterequal", "≥"),+ ("guillemotleft", "«"),+ ("guillemotright", "»"),+ ("guilsinglleft", "‹"),+ ("guilsinglright", "›"),+ ("h", "h"),+ ("hungarumlaut", "˝"),+ ("hyphen", "-"),+ ("i", "i"),+ ("iacute", "í"),+ ("icircumflex", "î"),+ ("idieresis", "ï"),+ ("igrave", "ì"),+ ("imacron", "ī"),+ ("infinity", "∞"),+ ("iota", "ι"),+ ("j", "j"),+ ("k", "k"),+ ("kappa", "κ"),+ ("l", "l"),+ ("lacute", "ĺ"),+ ("lambda", "λ"),+ ("lcaron", "ľ"),+ ("less", "<"),+ ("lessequal", "≤"),+ ("logicalnot", "¬"),+ ("lslash", "ł"),+ ("m", "m"),+ ("macron", "¯"),+ ("minus", "−"),+ ("mu", "μ"),+ ("multiply", "×"),+ ("n", "n"),+ ("nacute", "ń"),+ ("ncaron", "ň"),+ ("nine", "9"),+ ("notequal", "≠"),+ ("ntilde", "ñ"),+ ("nu", "ν"),+ ("numbersign", "#"),+ ("o", "o"),+ ("oacute", "ó"),+ ("ocircumflex", "ô"),+ ("odieresis", "ö"),+ ("oe", "œ"),+ ("ogonek", "˛"),+ ("ograve", "ò"),+ ("ohungarumlaut", "ő"),+ ("omacron", "ō"),+ ("omega", "ω"),+ ("omicron", "ο"),+ ("one", "1"),+ ("onehalf", "½"),+ ("onequarter", "¼"),+ ("onesuperior", "¹"),+ ("ordfeminine", "ª"),+ ("ordmasculine", "º"),+ ("oslash", "ø"),+ ("otilde", "õ"),+ ("p", "p"),+ ("paragraph", "¶"),+ ("parenleft", "("),+ ("parenright", ")"),+ ("partialdiff", "∂"),+ ("percent", "%"),+ ("period", "."),+ ("periodcentered", "·"),+ ("perthousand", "‰"),+ ("phi", "φ"),+ ("pi", "π"),+ ("plus", "+"),+ ("plusminus", "±"),+ ("psi", "ψ"),+ ("q", "q"),+ ("question", "?"),+ ("questiondown", "¿"),+ ("quotedbl", "\""),+ ("quotedblbase", "„"),+ ("quotedblleft", "“"),+ ("quotedblright", "”"),+ ("quoteleft", "‘"),+ ("quoteright", "’"),+ ("quotesinglbase", "‚"),+ ("quotesingle", "'"),+ ("r", "r"),+ ("racute", "ŕ"),+ ("radical", "√"),+ ("rcaron", "ř"),+ ("registered", "®"),+ ("rho", "ρ"),+ ("ring", "˚"),+ ("s", "s"),+ ("sacute", "ś"),+ ("scaron", "š"),+ ("scedilla", "ş"),+ ("section", "§"),+ ("semicolon", ";"),+ ("seven", "7"),+ ("sigma", "σ"),+ ("sigma1", "ς"),+ ("six", "6"),+ ("slash", "/"),+ ("space", " "),+ ("sterling", "£"),+ ("summation", "∑"),+ ("t", "t"),+ ("tau", "τ"),+ ("tbar", "ŧ"),+ ("tcaron", "ť"),+ ("theta", "θ"),+ ("thorn", "þ"),+ ("three", "3"),+ ("threequarters", "¾"),+ ("threesuperior", "³"),+ ("tilde", "˜"),+ ("trademark", "™"),+ ("two", "2"),+ ("twosuperior", "²"),+ ("u", "u"),+ ("uacute", "ú"),+ ("ucircumflex", "û"),+ ("udieresis", "ü"),+ ("ugrave", "ù"),+ ("uhungarumlaut", "ű"),+ ("umacron", "ū"),+ ("underscore", "_"),+ ("uogonek", "ų"),+ ("upsilon", "υ"),+ ("uring", "ů"),+ ("v", "v"),+ ("w", "w"),+ ("x", "x"),+ ("xi", "ξ"),+ ("y", "y"),+ ("yacute", "ý"),+ ("ydieresis", "ÿ"),+ ("yen", "¥"),+ ("z", "z"),+ ("zacute", "ź"),+ ("zcaron", "ž"),+ ("zdotaccent", "ż"),+ ("zero", "0"),+ ("zeta", "ζ"),+];++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn agl_subset_is_sorted_and_unique() {+ for w in AGL_SUBSET.windows(2) {+ assert!(w[0].0 < w[1].0, "{} >= {}", w[0].0, w[1].0);+ }+ }++ #[test]+ fn resolves_plain_names() {+ assert_eq!(resolve_glyph_name("germandbls").as_deref(), Some("ß"));+ assert_eq!(resolve_glyph_name("adieresis").as_deref(), Some("ä"));+ assert_eq!(resolve_glyph_name("fl").as_deref(), Some("fl"));+ assert_eq!(resolve_glyph_name("ffi").as_deref(), Some("ffi"));+ assert_eq!(resolve_glyph_name("A").as_deref(), Some("A"));+ assert_eq!(resolve_glyph_name("space").as_deref(), Some(" "));+ }++ #[test]+ fn resolves_uni_and_u_forms() {+ assert_eq!(resolve_glyph_name("uni0041").as_deref(), Some("A"));+ assert_eq!(resolve_glyph_name("uniFB01").as_deref(), Some("fi"));+ assert_eq!(resolve_glyph_name("uni00410042").as_deref(), Some("AB"));+ assert_eq!(resolve_glyph_name("u1F600").as_deref(), Some("😀"));+ assert_eq!(resolve_glyph_name("u0041").as_deref(), Some("A"));+ // lowercase hex is not AGL-valid+ assert_eq!(resolve_glyph_name("uni00e9"), None);+ // surrogate halves are invalid+ assert_eq!(resolve_glyph_name("uniD800"), None);+ }++ #[test]+ fn resolves_compounds_and_suffixes() {+ assert_eq!(resolve_glyph_name("f_i").as_deref(), Some("fi"));+ assert_eq!(resolve_glyph_name("f_f_l").as_deref(), Some("ffl"));+ assert_eq!(resolve_glyph_name("fi.alt").as_deref(), Some("fi"));+ assert_eq!(resolve_glyph_name("uni0041.sc").as_deref(), Some("A"));+ }++ #[test]+ fn rejects_meaningless_names() {+ assert_eq!(resolve_glyph_name("g42"), None);+ assert_eq!(resolve_glyph_name("cid1234"), None);+ assert_eq!(resolve_glyph_name("glyph7"), None);+ assert_eq!(resolve_glyph_name(""), None);+ assert_eq!(resolve_glyph_name(".notdef"), None);+ }+}diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rsindex d027f24..9f400c7 100644--- a/crates/liteparse/src/markdown_layout/classify.rs+++ b/crates/liteparse/src/markdown_layout/classify.rs@@ -395,6 +395,73 @@ pub fn classify_page_with_filters(stitch_regions(blocks, ®ion_boundaries)}+/// Mutable per-line flow state threaded through `classify_region`: the active+/// paragraph accumulator, the active code run, and the current list run. The+/// flush/reset/emit methods live here so the (previously 8-parameter) closures+/// and the six copy-pasted "reset list state" blocks collapse to method calls;+/// methods on the struct also sidestep the borrow-checker friction of threading+/// five `&mut Option<…>` cells through a free closure.+#[derive(Default)]+struct FlowState {+ paragraph: Option<ParaAccum>,+ code: Option<Vec<String>>,+ list_base_indent: Option<f32>,+ last_list_item_idx: Option<usize>,+ last_list_line: Option<usize>,+}++impl FlowState {+ /// Drop the active list run. A heading, table, code block, or paragraph+ /// break all terminate the current list.+ fn reset_list(&mut self) {+ self.list_base_indent = None;+ self.last_list_item_idx = None;+ self.last_list_line = None;+ }++ /// Emit the active paragraph (if it has non-blank content) and clear it.+ fn flush_paragraph(&mut self, blocks: &mut Vec<Block>) {+ if let Some(acc) = self.paragraph.take()+ && !acc.raw.trim().is_empty()+ {+ blocks.push(paragraph_from_accum(acc));+ }+ }++ /// Emit the active code run (if non-empty) and clear it.+ fn flush_code(&mut self, blocks: &mut Vec<Block>) {+ if let Some(lines) = self.code.take()+ && !lines.is_empty()+ {+ blocks.push(Block::CodeBlock { lines });+ }+ }++ /// Emit any interruptions whose y is at or above `before_y`, flushing the+ /// active paragraph/code/list state first so each lands as its own block.+ fn emit_before(+ &mut self,+ blocks: &mut Vec<Block>,+ iter: &mut std::iter::Peekable<std::vec::IntoIter<(f32, Interruption)>>,+ before_y: f32,+ ) {+ while let Some((y, _)) = iter.peek() {+ if *y > before_y {+ break;+ }+ let (_, kind) = iter.next().unwrap();+ self.flush_paragraph(blocks);+ self.flush_code(blocks);+ self.reset_list();+ blocks.push(match kind {+ Interruption::Hr => Block::HorizontalRule,+ Interruption::Figure(r) => Block::Figure { id: r.id },+ Interruption::Table(b) => b,+ });+ }+ }+}+/// Classify the lines of a single xy_cut leaf into a sequence of blocks. All/// per-line state (active paragraph, code run, list run, heading_run) is local/// to this call — nothing crosses a leaf boundary except through the explicit@@ -413,11 +480,7 @@ fn classify_region(precomputed_tables: Option<Vec<super::tables::TableRun>>,) -> Vec<Block> {let mut blocks: Vec<Block> = Vec::new();- let mut paragraph: Option<ParaAccum> = None;- let mut code: Option<Vec<String>> = None;- let mut list_base_indent: Option<f32> = None;- let mut last_list_item_idx: Option<usize> = None;- let mut last_list_line: Option<usize> = None;+ let mut state = FlowState::default();let mut heading_run: Option<(u8, usize)> = None;// Track whether we've already emitted a TOC title on this page. Once seen,// any subsequent line matching `is_toc_title` (e.g. "Index", "List of@@ -425,21 +488,6 @@ fn classify_region(// and must stay suppressed.let mut toc_title_emitted = false;- let flush_paragraph = |blocks: &mut Vec<Block>, p: Option<ParaAccum>| {- if let Some(acc) = p- && !acc.raw.trim().is_empty()- {- blocks.push(paragraph_from_accum(acc));- }- };- let flush_code = |blocks: &mut Vec<Block>, c: Option<Vec<String>>| {- if let Some(lines) = c- && !lines.is_empty()- {- blocks.push(Block::CodeBlock { lines });- }- };-// Per-region table detection. Indices are local to `lines`. Page-level// graphics are still consulted for ruled-table detection because path// objects are page-coordinate; the detector intersects them against the@@ -475,34 +523,6 @@ fn classify_region(region_interruptions.sort_by(|a, b| a.0.total_cmp(&b.0));let mut interruptions = region_interruptions.into_iter().peekable();- // Emit any interruptions whose y is at or above `before_y`, flushing the- // active paragraph/code/list state first so each lands as its own block.- let emit_before = |blocks: &mut Vec<Block>,- paragraph: &mut Option<ParaAccum>,- code: &mut Option<Vec<String>>,- list_base: &mut Option<f32>,- last_item: &mut Option<usize>,- last_line: &mut Option<usize>,- iter: &mut std::iter::Peekable<std::vec::IntoIter<(f32, Interruption)>>,- before_y: f32| {- while let Some((y, _)) = iter.peek() {- if *y > before_y {- break;- }- let (_, kind) = iter.next().unwrap();- flush_paragraph(blocks, paragraph.take());- flush_code(blocks, code.take());- *list_base = None;- *last_item = None;- *last_line = None;- blocks.push(match kind {- Interruption::Hr => Block::HorizontalRule,- Interruption::Figure(r) => Block::Figure { id: r.id },- Interruption::Table(b) => b,- });- }- };-let mut idx = 0;while idx < lines.len() {if let Some(run) = table_iter.peek()@@ -510,21 +530,10 @@ fn classify_region({// Flush any interruptions above this table's top edge first.let table_top = lines[run.start].bbox.y;- emit_before(- &mut blocks,- &mut paragraph,- &mut code,- &mut list_base_indent,- &mut last_list_item_idx,- &mut last_list_line,- &mut interruptions,- table_top,- );- flush_paragraph(&mut blocks, paragraph.take());- flush_code(&mut blocks, code.take());- list_base_indent = None;- last_list_item_idx = None;- last_list_line = None;+ state.emit_before(&mut blocks, &mut interruptions, table_top);+ state.flush_paragraph(&mut blocks);+ state.flush_code(&mut blocks);+ state.reset_list();let run = table_iter.next().unwrap();blocks.push(run.block);idx = run.end;@@ -533,16 +542,7 @@ fn classify_region(let line_idx = idx;let line = &lines[line_idx];// Emit any interruptions that fall above this line.- emit_before(- &mut blocks,- &mut paragraph,- &mut code,- &mut list_base_indent,- &mut last_list_item_idx,- &mut last_list_line,- &mut interruptions,- line.bbox.y,- );+ state.emit_before(&mut blocks, &mut interruptions, line.bbox.y);idx += 1;let text = line.text.trim();if text.is_empty() {@@ -571,16 +571,16 @@ fn classify_region(// still becomes code. Mono content also wouldn't make a useful// heading.if line.all_mono {- flush_paragraph(&mut blocks, paragraph.take());- list_base_indent = None;- last_list_item_idx = None;- last_list_line = None;- code.get_or_insert_with(Vec::new)+ state.flush_paragraph(&mut blocks);+ state.reset_list();+ state+ .code+ .get_or_insert_with(Vec::new).push(line.text.trim_end().to_string());continue;}// Any non-mono line ends the current code block (if any).- flush_code(&mut blocks, code.take());+ state.flush_code(&mut blocks);// Priority chain: tagged-PDF struct tree → outline → font-size map.let tagged_level = struct_heading_level(line, &page.struct_nodes);@@ -614,10 +614,11 @@ fn classify_region(// Outline / struct-tree levels are explicit and bypass this guard.let size_level = size_level.filter(|_| {let starts_lower = text.chars().next().is_some_and(|c| c.is_lowercase());- let prev = paragraph+ let prev = state+ .paragraph.as_ref().map(|p| &p.last)- .or(last_list_line.map(|i| &lines[i]));+ .or(state.last_list_line.map(|i| &lines[i]));// A previous line ending in a mid-word hyphen wrap means this// line is its continuation regardless of capitalization// ("…SOLAR 10.7 Billion-" / "Parameter Model: We have…").@@ -736,7 +737,7 @@ fn classify_region(if combined_chars > HEADING_MAX_TEXT_CHARS {let demoted = std::mem::take(htext);blocks.pop();- paragraph = Some(ParaAccum {+ state.paragraph = Some(ParaAccum {raw: demoted.clone(),inline: demoted,last: lines[*run_idx].clone(),@@ -751,10 +752,8 @@ fn classify_region(}}if !demoted_heading {- flush_paragraph(&mut blocks, paragraph.take());- list_base_indent = None;- last_list_item_idx = None;- last_list_line = None;+ state.flush_paragraph(&mut blocks);+ state.reset_list();if debug {eprintln!("[MD heading-emit size/outline] h{} idx={} '{}' size={:.2}",@@ -787,16 +786,15 @@ fn classify_region(&& looks_like_numbered_bold_heading(line,rest,- paragraph+ state+ .paragraph.as_ref().map(|p| &p.last)- .or(last_list_line.map(|i| &lines[i])),+ .or(state.last_list_line.map(|i| &lines[i])),){- flush_paragraph(&mut blocks, paragraph.take());- list_base_indent = None;- last_list_item_idx = None;- last_list_line = None;+ state.flush_paragraph(&mut blocks);+ state.reset_list();let level = (heading_map.len() as u8 + 1).clamp(1, MAX_HEADING_LEVELS as u8);blocks.push(Block::Heading {level,@@ -804,13 +802,13 @@ fn classify_region(});continue;}- flush_paragraph(&mut blocks, paragraph.take());- let base = *list_base_indent.get_or_insert(line.indent_x);+ state.flush_paragraph(&mut blocks);+ let base = *state.list_base_indent.get_or_insert(line.indent_x);let level = (((line.indent_x - base) / LIST_INDENT_STEP_PT).round().max(0.0)) as u8;- last_list_item_idx = Some(blocks.len());- last_list_line = Some(line_idx);+ state.last_list_item_idx = Some(blocks.len());+ state.last_list_line = Some(line_idx);// Render the list-item text via the inline pipeline so per-span// emphasis surfaces. We strip the marker from `rest` (already// done by `parse_list_marker`), but emphasis lives on `line.spans`,@@ -831,8 +829,8 @@ fn classify_region(// Continuation of a list item: same gap/font rules as paragraphs.// Footnote-style continuations often left-flush below the marker's// hanging indent, so we don't require indent ≥ marker indent.- if let Some(item_idx) = last_list_item_idx- && let Some(prev_idx) = last_list_line+ if let Some(item_idx) = state.last_list_item_idx+ && let Some(prev_idx) = state.last_list_line&& continues_paragraph(&lines[prev_idx], line)&& let Some(Block::ListItem {text: prev_text, ..@@ -842,7 +840,7 @@ fn classify_region(// inline-styled continuation.let cont_inline = render_line_inline(line);append_inline_continuation(prev_text, text, &cont_inline);- last_list_line = Some(line_idx);+ state.last_list_line = Some(line_idx);continue;}@@ -851,16 +849,15 @@ fn classify_region(// "1 Introduction"); without this rule they look just like a bold// first sentence of a paragraph. Runs after list-marker detection so// bold bullet items stay as list items.- let prev_for_gap = paragraph+ let prev_for_gap = state+ .paragraph.as_ref().map(|p| &p.last)- .or(last_list_line.map(|i| &lines[i]));+ .or(state.last_list_line.map(|i| &lines[i]));let next_for_gap = lines.get(idx);if !toc_suppress && looks_like_bold_heading(line, prev_for_gap, next_for_gap) {- flush_paragraph(&mut blocks, paragraph.take());- list_base_indent = None;- last_list_item_idx = None;- last_list_line = None;+ state.flush_paragraph(&mut blocks);+ state.reset_list();// Level: one deeper than the deepest size-based level we already// have. With an empty heading_map this lands on H1; with a full// 6-level map it caps at H6.@@ -884,19 +881,17 @@ fn classify_region(continue;}- match paragraph.as_mut() {+ match state.paragraph.as_mut() {Some(acc) if continues_paragraph(&acc.last, line) => {append_to_paragraph(acc, line);}_ => {- flush_paragraph(&mut blocks, paragraph.take());- list_base_indent = None;- last_list_item_idx = None;- last_list_line = None;+ state.flush_paragraph(&mut blocks);+ state.reset_list();let inline = render_line_inline(line);let raw = collapse_whitespace(text);let uniform = line_uniform_style(line).map(|s| (s.bold, s.italic));- paragraph = Some(ParaAccum {+ state.paragraph = Some(ParaAccum {raw,inline,last: line.clone(),@@ -906,19 +901,10 @@ fn classify_region(}}- flush_paragraph(&mut blocks, paragraph.take());- flush_code(&mut blocks, code.take());+ state.flush_paragraph(&mut blocks);+ state.flush_code(&mut blocks);// Flush any trailing interruptions that sat below the last text line.- emit_before(- &mut blocks,- &mut paragraph,- &mut code,- &mut list_base_indent,- &mut last_list_item_idx,- &mut last_list_line,- &mut interruptions,- f32::INFINITY,- );+ state.emit_before(&mut blocks, &mut interruptions, f32::INFINITY);blocks}diff --git a/crates/liteparse/src/markdown_layout/cross_region.rs b/crates/liteparse/src/markdown_layout/cross_region.rsnew file mode 100644index 0000000..d67d76f--- /dev/null+++ b/crates/liteparse/src/markdown_layout/cross_region.rs@@ -0,0 +1,767 @@+//! Cross-region table re-merge.+//!+//! When the XY-cut commits a V-cut *through* a table (the column gutter of the+//! table looks like a layout gutter), the table's lines land in two or more+//! sibling leaves and no per-region detector ever sees the rows whole. This+//! pass runs before region grouping: it finds side-by-side leaf groups whose+//! baselines align across the cut, fuses the same-baseline lines back into+//! single rows, and validates that the fused zone actually classifies as a+//! table. Only on successful validation are the original lines replaced — a+//! failed candidate leaves the page untouched, so the V-cut's wins on+//! multi-column prose are preserved.++use super::blocks::Block;+use super::tables::{TableRun, detect_tables};+use crate::types::ProjectedLine;++/// Minimum fraction of the smaller side's y-band that must overlap the other+/// side's. Side-by-side table halves overlap almost fully; a sidebar next to+/// the tail of a column does not.+const CR_MIN_Y_OVERLAP_FRAC: f32 = 0.5;+/// Minimum horizontal gap between the two sides (the sliced gutter).+const CR_MIN_COL_GAP_PT: f32 = 8.0;+/// Baseline-cluster tolerance as a fraction of the median line height.+const CR_ROW_TOL_FACTOR: f32 = 0.6;+/// At least this many baselines must align across the cut.+const CR_MIN_ALIGNED_ROWS: usize = 3;+/// ...and they must account for this fraction of the sparser side's lines.+const CR_MIN_ALIGNED_FRAC: f32 = 0.4;+/// A side "looks like prose" when it has at least this many lines...+const CR_PROSE_MIN_LINES: usize = 3;+/// ...whose median width fills this fraction of the side's width...+const CR_PROSE_WIDTH_FRAC: f32 = 0.7;+/// ...and whose summed line heights fill this fraction of the y-band.+const CR_PROSE_VFILL: f32 = 0.55;+/// A prose side is dominated by single-cell lines: when this fraction or more+/// of a side's lines split into ≥2 cells, it's a sliced table half (row+/// fragments with internal column gaps), not flowing text.+const CR_PROSE_MAX_MULTI_CELL_FRAC: f32 = 0.3;+/// Validation A: the detected table run(s) must cover this fraction of the+/// fused lines.+const CR_VALIDATE_MIN_COVERAGE: f32 = 0.6;+/// Validation A: reject a detected run when more than this fraction of its+/// non-empty cells run ≥ `CR_LONG_CELL_WORDS` words. Fused newspaper /+/// reference columns align into convincing grids, but their "cells" are+/// sentence fragments; real table cells are short.+const CR_VALIDATE_MAX_LONG_CELL_FRAC: f32 = 0.3;+const CR_LONG_CELL_WORDS: usize = 5;+/// Direct 2-col path: max chars for a left-column label cell.+const CR_LABEL_MAX_CHARS: usize = 60;+/// Direct 2-col path: minimum row count.+const CR_TWO_COL_MIN_ROWS: usize = 3;+/// A left-side line continues the previous row's label when its top is within+/// this multiple of line height of the previous label line's bottom.+const CR_LABEL_WRAP_GAP_FACTOR: f32 = 1.5;++/// A validated cross-region merge: replace `lines[start..end]` with `merged`+/// (all sharing one synthetic region path) and hand `runs` (indices local to+/// `merged`) to the region classifier so the table emission is guaranteed.+pub(super) struct CrossRegionMerge {+ pub(super) start: usize,+ pub(super) end: usize,+ pub(super) merged: Vec<ProjectedLine>,+ pub(super) runs: Vec<TableRun>,+}++struct Leaf {+ start: usize,+ end: usize,+ x0: f32,+ x1: f32,+ y0: f32,+ y1: f32,+}++/// Maximum x-overlap between sibling V-cut slices, as a fraction of the+/// narrower side's width.+const CR_SIBLING_MAX_X_OVERLAP_FRAC: f32 = 0.4;++/// True when two leaves are direct children of the same region-tree node and+/// overlap only modestly in x — the signature of a V-cut whose slices were+/// widened by straddling rows.+fn sibling_v_cut_pair(a: &Leaf, b: &Leaf, lines: &[ProjectedLine]) -> bool {+ let pa = &lines[a.start].region_path;+ let pb = &lines[b.start].region_path;+ if pa.len() != pb.len() || pa.is_empty() || pa[..pa.len() - 1] != pb[..pb.len() - 1] {+ return false;+ }+ let x_overlap = a.x1.min(b.x1) - a.x0.max(b.x0);+ let narrow = (a.x1 - a.x0).min(b.x1 - b.x0).max(1.0);+ x_overlap < CR_SIBLING_MAX_X_OVERLAP_FRAC * narrow+}++fn build_leaves(lines: &[ProjectedLine]) -> Vec<Leaf> {+ let mut leaves = Vec::new();+ let mut s = 0;+ while s < lines.len() {+ let path = &lines[s].region_path;+ let mut e = s + 1;+ while e < lines.len() && lines[e].region_path == *path {+ e += 1;+ }+ let (mut x0, mut x1) = (f32::INFINITY, f32::NEG_INFINITY);+ let (mut y0, mut y1) = (f32::INFINITY, f32::NEG_INFINITY);+ for l in &lines[s..e] {+ x0 = x0.min(l.bbox.x);+ x1 = x1.max(l.bbox.x + l.bbox.width);+ y0 = y0.min(l.bbox.y);+ y1 = y1.max(l.bbox.y + l.bbox.height);+ }+ leaves.push(Leaf {+ start: s,+ end: e,+ x0,+ x1,+ y0,+ y1,+ });+ s = e;+ }+ leaves+}++fn median(mut v: Vec<f32>) -> f32 {+ if v.is_empty() {+ return 0.0;+ }+ v.sort_by(|a, b| a.total_cmp(b));+ v[v.len() / 2]+}++/// Longest common prefix of the region paths of all lines in the set.+fn common_path_prefix(lines: &[&ProjectedLine]) -> Vec<u16> {+ let mut prefix: Vec<u16> = lines[0].region_path.clone();+ for l in &lines[1..] {+ let n = prefix+ .iter()+ .zip(l.region_path.iter())+ .take_while(|(a, b)| a == b)+ .count();+ prefix.truncate(n);+ }+ prefix+}++/// Does this side's line set look like a column of flowing prose? Dense+/// vertical fill + lines that fill the column width. A table fragment has+/// short, sparse lines on at least one side.+fn side_prose_metrics(side: &[&ProjectedLine], side_x0: f32, side_x1: f32) -> (f32, f32) {+ let side_width = (side_x1 - side_x0).max(1.0);+ let med_width = median(side.iter().map(|l| l.bbox.width).collect());+ let (mut y0, mut y1) = (f32::INFINITY, f32::NEG_INFINITY);+ let mut sum_h = 0.0f32;+ for l in side {+ y0 = y0.min(l.bbox.y);+ y1 = y1.max(l.bbox.y + l.bbox.height);+ sum_h += l.bbox.height;+ }+ (med_width / side_width, sum_h / (y1 - y0).max(1.0))+}++fn side_looks_like_prose(side: &[&ProjectedLine], side_x0: f32, side_x1: f32) -> bool {+ if side.len() < CR_PROSE_MIN_LINES {+ return false;+ }+ let (width_fill, vfill) = side_prose_metrics(side, side_x0, side_x1);+ if width_fill < CR_PROSE_WIDTH_FRAC || vfill < CR_PROSE_VFILL {+ return false;+ }+ // A sliced *wide* table's row-halves also fill their side's width and+ // y-band, but they carry internal cell gaps; flowing prose doesn't.+ let multi_cell = side+ .iter()+ .filter(|l| super::tables::split_cells(l).len() >= 2)+ .count();+ (multi_cell as f32) < CR_PROSE_MAX_MULTI_CELL_FRAC * side.len() as f32+}++struct Cluster<'a> {+ left: Vec<&'a ProjectedLine>,+ right: Vec<&'a ProjectedLine>,+}++impl<'a> Cluster<'a> {+ fn members(&self) -> impl Iterator<Item = &&'a ProjectedLine> {+ self.left.iter().chain(self.right.iter())+ }+}++/// Cluster the set's lines into baseline rows: lines whose y-centers fall+/// within `tol` of the cluster seed share a row.+fn cluster_rows<'a>(set: &[(&'a ProjectedLine, bool)], tol: f32) -> Vec<Cluster<'a>> {+ let mut sorted: Vec<&(&ProjectedLine, bool)> = set.iter().collect();+ sorted.sort_by(|a, b| {+ let ya = a.0.bbox.y + a.0.bbox.height * 0.5;+ let yb = b.0.bbox.y + b.0.bbox.height * 0.5;+ ya.total_cmp(&yb)+ });+ let mut clusters: Vec<(f32, Cluster)> = Vec::new();+ for (line, is_left) in sorted {+ let yc = line.bbox.y + line.bbox.height * 0.5;+ match clusters.last_mut() {+ Some((seed_y, c)) if (yc - *seed_y).abs() <= tol => {+ if *is_left {+ c.left.push(line);+ } else {+ c.right.push(line);+ }+ }+ _ => {+ let mut c = Cluster {+ left: Vec::new(),+ right: Vec::new(),+ };+ if *is_left {+ c.left.push(line);+ } else {+ c.right.push(line);+ }+ clusters.push((yc, c));+ }+ }+ }+ clusters.into_iter().map(|(_, c)| c).collect()+}++/// Fuse one baseline cluster into a single `ProjectedLine`: spans concatenate+/// (cell boundaries fall out of the span gaps in `split_cells`), text joins+/// with gap-proportional spacing, styles AND together.+fn fuse_cluster(cluster: &Cluster, path: &[u16]) -> ProjectedLine {+ let mut members: Vec<&ProjectedLine> = cluster.members().copied().collect();+ members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x));++ let first = members[0];+ let mut fused = first.clone();+ fused.region_path = path.to_vec();+ for m in &members[1..] {+ let prev_right = fused.bbox.x + fused.bbox.width;+ let gap = (m.bbox.x - prev_right).max(0.0);+ let approx_char_w = (fused.dominant_font_size * 0.5).max(2.0);+ let n_spaces = ((gap / approx_char_w) as usize).clamp(2, 24);+ fused.text.push_str(&" ".repeat(n_spaces));+ fused.text.push_str(&m.text);+ fused.spans.extend(m.spans.iter().cloned());+ let x1 = (fused.bbox.x + fused.bbox.width).max(m.bbox.x + m.bbox.width);+ let y1 = (fused.bbox.y + fused.bbox.height).max(m.bbox.y + m.bbox.height);+ fused.bbox.x = fused.bbox.x.min(m.bbox.x);+ fused.bbox.y = fused.bbox.y.min(m.bbox.y);+ fused.bbox.width = x1 - fused.bbox.x;+ fused.bbox.height = y1 - fused.bbox.y;+ fused.all_bold &= m.all_bold;+ fused.all_italic &= m.all_italic;+ fused.all_mono &= m.all_mono;+ fused.all_strike &= m.all_strike;+ fused.font_size_is_estimated |= m.font_size_is_estimated;+ }+ fused.spans.sort_by(|a, b| a.x.total_cmp(&b.x));+ fused+}++/// Validation A: the fused zone re-classifies as a table via the standard+/// detectors, covering most of its lines.+fn validate_via_detectors(merged: &[ProjectedLine]) -> Option<Vec<TableRun>> {+ let runs = detect_tables(merged);+ let covered: usize = runs+ .iter()+ .filter(|r| match &r.block {+ Block::Table { header, rows } => {+ let cols = header+ .as_ref()+ .map(|h| h.len())+ .or_else(|| rows.first().map(|row| row.len()))+ .unwrap_or(0);+ if cols < 2 {+ return false;+ }+ let (mut cells, mut long) = (0usize, 0usize);+ for row in rows {+ for cell in row {+ let t = cell.trim();+ if t.is_empty() {+ continue;+ }+ cells += 1;+ if t.split_whitespace().count() >= CR_LONG_CELL_WORDS {+ long += 1;+ }+ }+ }+ cells > 0 && (long as f32) <= CR_VALIDATE_MAX_LONG_CELL_FRAC * cells as f32+ }+ _ => false,+ })+ .map(|r| r.end - r.start)+ .sum();+ if covered as f32 >= CR_VALIDATE_MIN_COVERAGE * merged.len() as f32 {+ Some(runs)+ } else {+ None+ }+}++/// Validation B: direct 2-column construction for label/description shapes+/// the standard detectors reject (e.g. labels with list-marker prefixes —+/// `is_label_like` is intentionally strict in normal flow, but here the V-cut+/// geometry has already established two columns). Rows are anchored on+/// left-side lines; a left line within wrap distance of the previous one+/// continues the label cell, right-side lines append to the current row's+/// description cell.+fn try_two_col_direct(clusters: &[Cluster], merged_len: usize, tol: f32) -> Option<Vec<TableRun>> {+ let mut rows: Vec<(String, String, bool)> = Vec::new(); // (left, right, bold)+ let mut last_label_bottom: Option<f32> = None;++ for cluster in clusters {+ if !cluster.left.is_empty() {+ let text = cluster+ .left+ .iter()+ .map(|l| l.text.trim())+ .collect::<Vec<_>>()+ .join(" ");+ let top = cluster+ .left+ .iter()+ .map(|l| l.bbox.y)+ .fold(f32::INFINITY, f32::min);+ let bottom = cluster+ .left+ .iter()+ .map(|l| l.bbox.y + l.bbox.height)+ .fold(f32::NEG_INFINITY, f32::max);+ let h = (bottom - top).max(1.0);+ let wraps = match (last_label_bottom, rows.last()) {+ (Some(lb), Some(_)) => top - lb <= CR_LABEL_WRAP_GAP_FACTOR * h.min(tol * 2.0),+ _ => false,+ };+ if wraps {+ let row = rows.last_mut().unwrap();+ if !row.0.is_empty() {+ row.0.push(' ');+ }+ row.0.push_str(&text);+ } else {+ let bold = cluster.left.iter().all(|l| l.all_bold);+ rows.push((text, String::new(), bold));+ }+ last_label_bottom = Some(bottom);+ }+ if !cluster.right.is_empty() {+ let text = cluster+ .right+ .iter()+ .map(|l| l.text.trim())+ .collect::<Vec<_>>()+ .join(" ");+ if rows.is_empty() {+ rows.push((String::new(), String::new(), false));+ }+ let row = rows.last_mut().unwrap();+ if !row.1.is_empty() {+ row.1.push(' ');+ }+ row.1.push_str(&text);+ }+ }++ if rows.len() < CR_TWO_COL_MIN_ROWS {+ return None;+ }+ // Labels must stay label-shaped after wrap joins; a left side of long+ // sentences is prose the anti-prose gate missed.+ if rows+ .iter()+ .any(|(l, _, _)| l.chars().count() > CR_LABEL_MAX_CHARS)+ {+ return None;+ }+ let both = rows+ .iter()+ .filter(|(l, r, _)| !l.is_empty() && !r.is_empty())+ .count();+ if both < 2 {+ return None;+ }++ // First row promotes to header when bold or when both cells are short+ // header-ish words ("Area" / "Competence").+ let header = {+ let (l, r, bold) = &rows[0];+ if !l.is_empty()+ && !r.is_empty()+ && (*bold || (l.chars().count() <= 20 && r.chars().count() <= 20))+ {+ Some(vec![l.clone(), r.clone()])+ } else {+ None+ }+ };+ let body: Vec<Vec<String>> = rows+ .iter()+ .skip(if header.is_some() { 1 } else { 0 })+ .map(|(l, r, _)| vec![l.clone(), r.clone()])+ .collect();+ if body.len() < 2 {+ return None;+ }+ Some(vec![TableRun {+ start: 0,+ end: merged_len,+ body_start: if header.is_some() { 1 } else { 0 },+ block: Block::Table { header, rows: body },+ }])+}++/// Find one validated cross-region table merge on this page, or `None`.+/// Callers may re-invoke on the spliced result to catch multiple sliced+/// tables per page.+pub(super) fn find_cross_region_table_merge(lines: &[ProjectedLine]) -> Option<CrossRegionMerge> {+ if *super::flags::DISABLE_CROSS_REGION_TABLES {+ return None;+ }+ let debug = *super::flags::DEBUG_CROSS_REGION;+ let leaves = build_leaves(lines);+ if debug {+ for (k, l) in leaves.iter().enumerate() {+ eprintln!(+ "[cross-region] leaf {k}: lines [{},{}) x=[{:.1},{:.1}] y=[{:.1},{:.1}] path={:?}",+ l.start, l.end, l.x0, l.x1, l.y0, l.y1, lines[l.start].region_path+ );+ }+ }+ if leaves.len() < 2 {+ return None;+ }++ for i in 0..leaves.len() {+ for j in (i + 1)..leaves.len() {+ let (a, b) = (&leaves[i], &leaves[j]);+ let (left, right) = if a.x1 <= b.x0 - CR_MIN_COL_GAP_PT {+ (a, b)+ } else if b.x1 <= a.x0 - CR_MIN_COL_GAP_PT {+ (b, a)+ } else if sibling_v_cut_pair(a, b, lines) {+ // Direct siblings of one region-tree node that overlap in y+ // can only come from a V-cut. Their bbox x-extents may overlap+ // (table rows straddling the cut widen the leaf), so the+ // disjointness test above misses them.+ if a.x0 <= b.x0 { (a, b) } else { (b, a) }+ } else {+ continue;+ };+ let ov = left.y1.min(right.y1) - left.y0.max(right.y0);+ let min_h = (left.y1 - left.y0).min(right.y1 - right.y0).max(1.0);+ if ov < CR_MIN_Y_OVERLAP_FRAC * min_h {+ continue;+ }+ // For disjoint sides this is the gutter midpoint; for overlapping+ // sibling slices it's the midpoint of the contested band.+ let cut_x = (left.x1.min(right.x1) + right.x0.max(left.x0)) * 0.5;+ // Union band: a tall side may pair with the first of several+ // stacked leaves on the other side; the extension below pulls the+ // rest in, so the band must cover both sides fully.+ let band = (left.y0.min(right.y0) - 2.0, left.y1.max(right.y1) + 2.0);++ // Extend the pair to every leaf living inside the overlap band and+ // entirely on one side of the cut (a side may itself be H-cut into+ // multiple leaves — doc 146's Area column is).+ let mut set_leaves: Vec<usize> = Vec::new();+ for (k, leaf) in leaves.iter().enumerate() {+ let leaf_h = (leaf.y1 - leaf.y0).max(1.0);+ let leaf_ov = leaf.y1.min(band.1) - leaf.y0.max(band.0);+ if leaf_ov < 0.5 * leaf_h {+ continue;+ }+ // The leaf must sit predominantly on one side of the cut:+ // its overhang across the cut stays within the straddle+ // allowance (widened slices), measured against its own width.+ let width = (leaf.x1 - leaf.x0).max(1.0);+ let center = (leaf.x0 + leaf.x1) * 0.5;+ let overhang = if center < cut_x {+ (leaf.x1 - cut_x).max(0.0)+ } else {+ (cut_x - leaf.x0).max(0.0)+ };+ if overhang <= CR_SIBLING_MAX_X_OVERLAP_FRAC * width {+ set_leaves.push(k);+ }+ }+ if set_leaves.len() < 2 {+ continue;+ }++ // The set's line ranges must tile a contiguous range — a foreign+ // leaf interleaved in reading order means this isn't a clean+ // sliced zone.+ let lo = set_leaves.iter().map(|&k| leaves[k].start).min().unwrap();+ let hi = set_leaves.iter().map(|&k| leaves[k].end).max().unwrap();+ let covered: usize = set_leaves+ .iter()+ .map(|&k| leaves[k].end - leaves[k].start)+ .sum();+ if covered != hi - lo {+ continue;+ }++ // Split the set's lines into sides of the cut.+ let mut set: Vec<(&ProjectedLine, bool)> = Vec::new();+ for &k in &set_leaves {+ let leaf = &leaves[k];+ let is_left = (leaf.x0 + leaf.x1) * 0.5 < cut_x;+ for l in &lines[leaf.start..leaf.end] {+ set.push((l, is_left));+ }+ }+ let left_lines: Vec<&ProjectedLine> =+ set.iter().filter(|(_, s)| *s).map(|(l, _)| *l).collect();+ let right_lines: Vec<&ProjectedLine> =+ set.iter().filter(|(_, s)| !*s).map(|(l, _)| *l).collect();+ if left_lines.is_empty() || right_lines.is_empty() {+ continue;+ }++ // Anti-prose gate: two side-by-side prose columns must stay split.+ let lx0 = left_lines+ .iter()+ .map(|l| l.bbox.x)+ .fold(f32::INFINITY, f32::min);+ let rx0 = right_lines+ .iter()+ .map(|l| l.bbox.x)+ .fold(f32::INFINITY, f32::min);+ let lx1 = left_lines+ .iter()+ .map(|l| l.bbox.x + l.bbox.width)+ .fold(f32::NEG_INFINITY, f32::max);+ let rx1 = right_lines+ .iter()+ .map(|l| l.bbox.x + l.bbox.width)+ .fold(f32::NEG_INFINITY, f32::max);+ if debug {+ let (lw, lv) = side_prose_metrics(&left_lines, lx0, lx1);+ let (rw, rv) = side_prose_metrics(&right_lines, rx0, rx1);+ eprintln!(+ "[cross-region] candidate cut@{cut_x:.1}: left n={} wfill={lw:.2} vfill={lv:.2} | right n={} wfill={rw:.2} vfill={rv:.2}",+ left_lines.len(),+ right_lines.len()+ );+ }+ // Either side reading as flowing prose means the V-cut was right:+ // a body column next to a sidebar / second column must stay split.+ if side_looks_like_prose(&left_lines, lx0, lx1)+ || side_looks_like_prose(&right_lines, rx0, rx1)+ {+ if debug {+ eprintln!("[cross-region] reject cut@{cut_x:.1}: a side is prose-like");+ }+ continue;+ }++ // Row alignment across the cut.+ let tol = CR_ROW_TOL_FACTOR+ * median(set.iter().map(|(l, _)| l.bbox.height).collect()).max(1.0);+ let clusters = cluster_rows(&set, tol);+ let aligned = clusters+ .iter()+ .filter(|c| !c.left.is_empty() && !c.right.is_empty())+ .count();+ let min_side = left_lines.len().min(right_lines.len());+ if aligned < CR_MIN_ALIGNED_ROWS+ || (aligned as f32) < CR_MIN_ALIGNED_FRAC * min_side as f32+ {+ if debug {+ eprintln!(+ "[cross-region] reject cut@{cut_x:.1}: aligned={aligned} of min_side={min_side}"+ );+ }+ continue;+ }++ // Fuse and validate.+ let all: Vec<&ProjectedLine> = set.iter().map(|(l, _)| *l).collect();+ let path = common_path_prefix(&all);+ let merged: Vec<ProjectedLine> =+ clusters.iter().map(|c| fuse_cluster(c, &path)).collect();++ let runs = validate_via_detectors(&merged)+ .or_else(|| try_two_col_direct(&clusters, merged.len(), tol));+ let Some(runs) = runs else {+ if debug {+ eprintln!(+ "[cross-region] reject cut@{cut_x:.1}: fused zone failed table validation"+ );+ }+ continue;+ };+ if debug {+ eprintln!(+ "[cross-region] MERGE cut@{cut_x:.1}: lines [{lo},{hi}) -> {} fused rows, {} run(s)",+ merged.len(),+ runs.len()+ );+ }+ return Some(CrossRegionMerge {+ start: lo,+ end: hi,+ merged,+ runs,+ });+ }+ }+ None+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::markdown_layout::test_helpers::{line, line_with_spans};++ fn at_region(mut l: ProjectedLine, path: &[u16]) -> ProjectedLine {+ l.region_path = path.to_vec();+ l+ }++ /// A 3-col numeric table sliced by a V-cut: label column in one leaf,+ /// two numeric columns in the other. Validation A (standard detectors on+ /// the fused rows) should accept.+ #[test]+ fn sliced_numeric_table_is_merged() {+ let mut lines = Vec::new();+ // Left leaf: row labels.+ for (i, label) in ["MANILA", "CEBU", "CAGAYAN DE ORO", "SUBIC"]+ .iter()+ .enumerate()+ {+ let y = 100.0 + i as f32 * 20.0;+ lines.push(at_region(+ line_with_spans(&[(label, 50.0)], y, 10.0),+ &[0, 0],+ ));+ }+ // Right leaf: two numeric columns per row.+ for (i, (a, b)) in [+ ("2454", "6125"),+ ("1138", "79500"),+ ("958", "13196"),+ ("313", "136"),+ ]+ .iter()+ .enumerate()+ {+ let y = 100.0 + i as f32 * 20.0;+ lines.push(at_region(+ line_with_spans(&[(a, 200.0), (b, 280.0)], y, 10.0),+ &[0, 1],+ ));+ }+ let m = find_cross_region_table_merge(&lines).expect("should merge");+ assert_eq!((m.start, m.end), (0, 8));+ assert_eq!(m.merged.len(), 4);+ assert!(!m.runs.is_empty());+ // Every fused row carries the label and both numbers.+ assert!(m.merged[0].text.contains("MANILA"));+ assert!(m.merged[0].text.contains("2454"));+ assert!(m.merged[0].text.contains("6125"));+ }++ /// Two side-by-side prose columns with coincidentally aligned baselines+ /// must NOT merge — that's the layout the V-cut exists to protect.+ #[test]+ fn two_column_prose_is_not_merged() {+ let mut lines = Vec::new();+ for i in 0..8 {+ let y = 100.0 + i as f32 * 12.0;+ lines.push(at_region(+ line("the quick brown fox jumps over it", 50.0, y, 10.0, 10.0),+ &[0, 0],+ ));+ }+ for i in 0..8 {+ let y = 100.0 + i as f32 * 12.0;+ lines.push(at_region(+ line("a second column of body text here", 250.0, y, 10.0, 10.0),+ &[0, 1],+ ));+ }+ assert!(find_cross_region_table_merge(&lines).is_none());+ }++ /// Label / description shape with list-marker labels (the doc-146 shape):+ /// standard detectors reject it, the direct 2-col path accepts.+ #[test]+ fn sliced_label_description_table_uses_direct_path() {+ let mut lines = Vec::new();+ lines.push(at_region(line("Area", 50.0, 100.0, 10.0, 10.0), &[0, 0]));+ for (i, label) in [+ "1. Embodying values",+ "2. Embracing complexity",+ "3. Envisioning futures",+ ]+ .iter()+ .enumerate()+ {+ let y = 150.0 + i as f32 * 50.0;+ lines.push(at_region(line(label, 50.0, y, 10.0, 10.0), &[0, 0]));+ }+ lines.push(at_region(+ line("Competence", 250.0, 100.0, 10.0, 10.0),+ &[0, 1],+ ));+ for (i, desc) in [+ "1.1 Valuing sustainability 1.2 Supporting fairness",+ "2.1 Systems thinking 2.2 Critical thinking",+ "3.1 Futures literacy 3.2 Adaptability",+ ]+ .iter()+ .enumerate()+ {+ let y = 150.0 + i as f32 * 50.0;+ lines.push(at_region(line(desc, 250.0, y, 10.0, 10.0), &[0, 1]));+ }+ let m = find_cross_region_table_merge(&lines).expect("should merge via direct path");+ assert_eq!(m.runs.len(), 1);+ match &m.runs[0].block {+ Block::Table { header, rows } => {+ assert_eq!(+ header.as_deref(),+ Some(&["Area".to_string(), "Competence".to_string()][..])+ );+ assert_eq!(rows.len(), 3);+ assert!(rows[0][1].contains("1.1 Valuing sustainability"));+ }+ other => panic!("expected table, got {other:?}"),+ }+ }++ /// A single region (no V-cut) never merges.+ #[test]+ fn single_leaf_is_untouched() {+ let lines: Vec<ProjectedLine> = (0..6)+ .map(|i| {+ at_region(+ line("text", 50.0, 100.0 + i as f32 * 12.0, 10.0, 10.0),+ &[0],+ )+ })+ .collect();+ assert!(find_cross_region_table_merge(&lines).is_none());+ }++ /// Vertically stacked leaves (H-cut siblings) never pair: no y-overlap.+ #[test]+ fn stacked_leaves_do_not_pair() {+ let mut lines = Vec::new();+ for i in 0..4 {+ lines.push(at_region(+ line("alpha beta", 50.0, 100.0 + i as f32 * 12.0, 10.0, 10.0),+ &[0],+ ));+ }+ for i in 0..4 {+ lines.push(at_region(+ line("gamma delta", 50.0, 300.0 + i as f32 * 12.0, 10.0, 10.0),+ &[1],+ ));+ }+ assert!(find_cross_region_table_merge(&lines).is_none());+ }+}diff --git a/crates/liteparse/src/markdown_layout/flags.rs b/crates/liteparse/src/markdown_layout/flags.rsnew file mode 100644index 0000000..7dc61d6--- /dev/null+++ b/crates/liteparse/src/markdown_layout/flags.rs@@ -0,0 +1,30 @@+//! Process-wide debug / kill-switch flags, read once from the environment.+//!+//! Each flag mirrors a `LITEPARSE_*` env var. `env::var` scans the process+//! environment table and allocates on a miss, so per-line hot paths cache the+//! lookup in a `LazyLock` instead of re-reading on every call. Each static is+//! `true` when the variable is **set** (matching the historical+//! `env::var(..).is_ok()`); "disable" call sites negate it.++use std::sync::LazyLock;++macro_rules! env_set_flag {+ ($(#[$m:meta])* $name:ident, $var:literal) => {+ $(#[$m])*+ pub(super) static $name: LazyLock<bool> =+ LazyLock::new(|| std::env::var($var).is_ok());+ };+}++env_set_flag!(DEBUG_MD, "LITEPARSE_DEBUG_MD");+env_set_flag!(DEBUG_TABLE, "LITEPARSE_DEBUG_TABLE");+env_set_flag!(DEBUG_RULED, "LITEPARSE_DEBUG_RULED");+env_set_flag!(DEBUG_CROSS_REGION, "LITEPARSE_DEBUG_CROSS_REGION");+env_set_flag!(DISABLE_GLOBAL_RULED, "LITEPARSE_DISABLE_GLOBAL_RULED");+env_set_flag!(DISABLE_HEADING_GUARDS, "LITEPARSE_DISABLE_HEADING_GUARDS");+env_set_flag!(+ DISABLE_CROSS_REGION_TABLES,+ "LITEPARSE_DISABLE_CROSS_REGION_TABLES"+);+env_set_flag!(DISABLE_CLUSTER_MERGE, "LITEPARSE_DISABLE_CLUSTER_MERGE");+env_set_flag!(DISABLE_STRADDLE_GUARD, "LITEPARSE_DISABLE_STRADDLE_GUARD");diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rsindex 9597f52..e56261a 100644--- a/crates/liteparse/src/markdown_layout/tables.rs+++ b/crates/liteparse/src/markdown_layout/tables.rs@@ -2527,121 +2527,218 @@ fn find_grid_components(hs: &[HSeg], vs: &[VSeg]) -> Vec<(Vec<usize>, Vec<usize>comps}-/// Build a `TableRun` for one ruled-grid component. Returns `None` if the-/// resulting grid is too small (< 2 cols or < 2 rows), covers nearly the-/// whole page (likely the page border), or is mostly empty cells.-fn build_ruled_table(- hs: &[HSeg],- vs: &[VSeg],- h_indices: &[usize],- v_indices: &[usize],- lines: &[ProjectedLine],- page_width: f32,- page_height: f32,- global: bool,-) -> Option<(TableRun, Vec<usize>)> {- let dbg = *super::flags::DEBUG_RULED;- let mut xs: Vec<f32> = v_indices.iter().map(|&i| vs[i].x).collect();- xs.sort_by(|a, b| a.total_cmp(b));- // Coarser, mean-centered clustering for column boundaries: cell-border- // rects contribute paired edges 4-6pt apart that would otherwise become- // phantom 5pt "columns" the span splitter then shreds text into.- cluster_boundaries(&mut xs, TABLE_COL_BOUNDARY_CLUSTER_PT);+/// Which ruled-table detection pass is running. The global pass reassembles a+/// grid from page-level graphics (lines arrive in xy_cut leaf order and grid+/// rules may include short decorative horizontals), so it applies extra+/// filtering the per-region pass does not need.+#[derive(Clone, Copy, PartialEq, Eq)]+enum RuledPass {+ PerRegion,+ Global,+}- // Distinct row y-coords (cluster again — multiple H lines may share a y).- // In the global pass, first drop horizontal rules that span only a small- // fraction of the column extent (see `RULED_HLINE_MIN_COVERAGE`).- let raw_ys = |idxs: &[usize]| {- let mut v: Vec<f32> = idxs.iter().map(|&i| hs[i].y).collect();- v.sort_by(|a, b| a.total_cmp(b));- dedup_close(&mut v, TABLE_GRID_CLUSTER_PT);- v- };- let ys: Vec<f32> = if global && xs.len() >= 2 {- let col_lo = xs[0];- let col_hi = xs[xs.len() - 1];- let extent = (col_hi - col_lo).max(1.0);- let kept: Vec<usize> = h_indices- .iter()- .copied()- .filter(|&i| {- let h = &hs[i];- let ov = (h.x_max.min(col_hi) - h.x_min.max(col_lo)).max(0.0);- ov / extent >= RULED_HLINE_MIN_COVERAGE- })- .collect();- let filtered = raw_ys(&kept);- if filtered.len() >= 3 {- filtered- } else {- raw_ys(h_indices)+/// Filter a vector down to the elements whose `keep` flag is set. `keep` must+/// be at least as long as `items`.+fn filter_by<T>(items: Vec<T>, keep: &[bool]) -> Vec<T> {+ items+ .into_iter()+ .zip(keep.iter())+ .filter(|(_, k)| **k)+ .map(|(v, _)| v)+ .collect()+}++/// The per-cell grids `build_ruled_table` fills, collapses, and reads as one+/// unit. Bundling them lets the row/column-collapse passes filter every layer+/// through a single `retain_rows`/`retain_cols` call instead of five parallel+/// `filter().map().collect()` chains — removing the "forgot to filter one+/// array" bug class.+struct CellGrid {+ /// Rendered cell text (each multi-column span whitespace-split at the+ /// crossed column boundaries).+ text: Vec<Vec<String>>,+ /// Per-cell bold flag — starts `true`, cleared when any non-bold span+ /// lands in the cell.+ is_bold: Vec<Vec<bool>>,+ /// Per-cell "carries text" flag.+ has_text: Vec<Vec<bool>>,+ /// Colspan-semantics grid: a span crossing ≥2 column boundaries replicates+ /// its FULL text into every covered cell instead of being split. Consulted+ /// only for header-band flattening — a group header like "North America"+ /// centered over its sub-columns must replicate, while the same geometry in+ /// a data row is a merged run that should split.+ repl: Vec<Vec<String>>,+ /// Per-row flag: the row contains an alpha-dominant multi-column span (a+ /// group-header label, as opposed to a mostly-digit merged data run).+ row_alpha_spanner: Vec<bool>,+}++impl CellGrid {+ fn new(n_rows: usize, n_cols: usize) -> Self {+ CellGrid {+ text: vec![vec![String::new(); n_cols]; n_rows],+ is_bold: vec![vec![true; n_cols]; n_rows],+ has_text: vec![vec![false; n_cols]; n_rows],+ repl: vec![vec![String::new(); n_cols]; n_rows],+ row_alpha_spanner: vec![false; n_rows],}- } else {- raw_ys(h_indices)- };- if dbg {- eprintln!(- "[ruled] component: ys={:?} xs={:?} ({} lines in scope)",- ys,- xs,- lines.len()- );}- // Need ≥2 row boundaries (1 row) and ≥2 column boundaries (1 col); but- // a 1×1 grid is just a callout box, so also require ≥1 inner divider- // (i.e. ys.len() ≥ 3 for ≥2 rows). Single-column tables (`xs.len() == 2`)- // are accepted when row evidence is strong enough — extra guards apply- // below after the empty-row collapse.- if ys.len() < 3 || xs.len() < 2 {- if dbg {- eprintln!(- "[ruled] REJECT grid-too-small ys={} xs={}",- ys.len(),- xs.len()- );- }- return None;+ fn n_rows(&self) -> usize {+ self.text.len()}- let n_rows = ys.len() - 1;- let n_cols = xs.len() - 1;- let bbox = crate::types::Rect {- x: xs[0],- y: ys[0],- width: xs[n_cols] - xs[0],- height: ys[n_rows] - ys[0],- };+ fn n_cols(&self) -> usize {+ self.text.first().map(|r| r.len()).unwrap_or(0)+ }- // Reject page-border-as-table.- if page_width > 0.0 && page_height > 0.0 {- let coverage = (bbox.width / page_width) * (bbox.height / page_height);- if coverage > TABLE_MAX_PAGE_COVERAGE {- if dbg {- eprintln!("[ruled] REJECT page-coverage {coverage:.2}");- }- return None;+ /// Append trimmed text to a cell, marking it as text-bearing. No-op for+ /// blank input.+ fn push_text(&mut self, row: usize, col: usize, txt: &str) {+ let txt = txt.trim();+ if txt.is_empty() {+ return;}+ if !self.text[row][col].is_empty() {+ self.text[row][col].push(' ');+ }+ self.text[row][col].push_str(txt);+ self.has_text[row][col] = true;+ }++ /// Append trimmed text to the colspan-semantics (`repl`) cell. No-op for+ /// blank input.+ fn push_repl(&mut self, row: usize, col: usize, txt: &str) {+ let txt = txt.trim();+ if txt.is_empty() {+ return;+ }+ let dst = &mut self.repl[row][col];+ if !dst.is_empty() {+ dst.push(' ');+ }+ dst.push_str(txt);+ }++ /// Keep only rows `r` where `keep[r]`; every layer filters identically.+ fn retain_rows(&mut self, keep: &[bool]) {+ self.text = filter_by(std::mem::take(&mut self.text), keep);+ self.is_bold = filter_by(std::mem::take(&mut self.is_bold), keep);+ self.has_text = filter_by(std::mem::take(&mut self.has_text), keep);+ self.repl = filter_by(std::mem::take(&mut self.repl), keep);+ self.row_alpha_spanner = filter_by(std::mem::take(&mut self.row_alpha_spanner), keep);+ }++ /// Keep only columns `c` where `keep[c]`; the per-cell layers filter, the+ /// per-row `row_alpha_spanner` is unaffected.+ fn retain_cols(&mut self, keep: &[bool]) {+ self.text = std::mem::take(&mut self.text)+ .into_iter()+ .map(|row| filter_by(row, keep))+ .collect();+ self.is_bold = std::mem::take(&mut self.is_bold)+ .into_iter()+ .map(|row| filter_by(row, keep))+ .collect();+ self.has_text = std::mem::take(&mut self.has_text)+ .into_iter()+ .map(|row| filter_by(row, keep))+ .collect();+ self.repl = std::mem::take(&mut self.repl)+ .into_iter()+ .map(|row| filter_by(row, keep))+ .collect();}- // Assign text to cells. A projected line frequently spans several ruled- // columns (one baseline = one line), so binning whole lines by centroid- // lumps an entire row into one cell and leaves the rest empty — the- // empty-cell filter then rejects the real table. Instead bin each line's- // raw spans by span center; spans whose x-extent crosses one or more- // interior column boundaries are split at the whitespace nearest each- // crossed boundary (same interpolation as the inferred-track path).- let mut cells: Vec<Vec<String>> = vec![vec![String::new(); n_cols]; n_rows];- let mut cell_is_bold: Vec<Vec<bool>> = vec![vec![true; n_cols]; n_rows];- let mut cell_has_text: Vec<Vec<bool>> = vec![vec![false; n_cols]; n_rows];- // Parallel grid where a span crossing ≥2 column boundaries contributes- // its FULL text to every covered cell (colspan semantics) instead of the- // whitespace split. Only consulted for header-band flattening — a group- // header like "North America" centered over its Revenue/Units sub-columns- // must replicate, while the same geometry in a data row is a merged run- // that should split.- let mut cells_repl: Vec<Vec<String>> = vec![vec![String::new(); n_cols]; n_rows];- let mut row_alpha_spanner: Vec<bool> = vec![false; n_rows];+ /// Drop "phantom rows" produced by stacked thin border-strip rects (doc 149+ /// rules each visual row as top-strip ~1pt / body ~22pt / bottom-strip ~5pt,+ /// each surviving the 2pt clustering as its own grid row). A row is dropped+ /// iff it has no text in any cell AND its height is < 80% of the median+ /// non-empty row height — the height gate preserves real fill-in-the-blank+ /// forms whose empty body rows are full height. `ys` are the row boundaries+ /// (length `n_rows + 1`). Returns the kept rows' heights.+ fn collapse_phantom_rows(&mut self, ys: &[f32]) -> Vec<f32> {+ let n_rows = self.n_rows();+ let row_heights: Vec<f32> = (0..n_rows).map(|r| ys[r + 1] - ys[r]).collect();+ let nonempty_heights: Vec<f32> = (0..n_rows)+ .filter(|r| self.has_text[*r].iter().any(|t| *t))+ .map(|r| row_heights[r])+ .collect();+ let median_h = if !nonempty_heights.is_empty() {+ let mut s = nonempty_heights.clone();+ s.sort_by(|a, b| a.total_cmp(b));+ s[s.len() / 2]+ } else {+ let mut s = row_heights.clone();+ s.sort_by(|a, b| a.total_cmp(b));+ s[s.len() / 2]+ };+ let keep: Vec<bool> = (0..n_rows)+ .map(|r| {+ let has_text = self.has_text[r].iter().any(|t| *t);+ has_text || row_heights[r] >= median_h * 0.8+ })+ .collect();+ let kept_row_heights: Vec<f32> = (0..n_rows)+ .filter(|r| keep[*r])+ .map(|r| row_heights[r])+ .collect();+ self.retain_rows(&keep);+ kept_row_heights+ }++ /// Mirror `collapse_phantom_rows` on columns: some ruled tables (doc 149)+ /// draw left/right borders as thin strip rects ~5pt wide, which become+ /// phantom text-less columns. Drop a column iff it is both empty AND+ /// narrower than 30% of the median text-bearing column. `xs` are the column+ /// boundaries (length `n_cols + 1`).+ fn collapse_phantom_cols(&mut self, xs: &[f32]) {+ let n_rows = self.n_rows();+ let n_cols = self.n_cols();+ let col_widths: Vec<f32> = (0..n_cols).map(|c| xs[c + 1] - xs[c]).collect();+ let nonempty_col_widths: Vec<f32> = (0..n_cols)+ .filter(|c| (0..n_rows).any(|r| self.has_text[r][*c]))+ .map(|c| col_widths[c])+ .collect();+ let median_w = if !nonempty_col_widths.is_empty() {+ let mut s = nonempty_col_widths.clone();+ s.sort_by(|a, b| a.total_cmp(b));+ s[s.len() / 2]+ } else {+ let mut s = col_widths.clone();+ s.sort_by(|a, b| a.total_cmp(b));+ s[s.len() / 2]+ };+ let keep_col: Vec<bool> = (0..n_cols)+ .map(|c| {+ let has_text = (0..n_rows).any(|r| self.has_text[r][c]);+ has_text || col_widths[c] >= median_w * 0.3+ })+ .collect();+ self.retain_cols(&keep_col);+ }+}++/// Bin each line's text into the grid cells defined by column boundaries `xs`+/// and row boundaries `ys`. A projected line frequently spans several ruled+/// columns (one baseline = one line), so binning whole lines by centroid would+/// lump an entire row into one cell and leave the rest empty — the empty-cell+/// filter would then reject the real table. Instead bin each line's raw spans+/// by span center; spans whose x-extent crosses one or more interior column+/// boundaries are split at the whitespace nearest each crossed boundary (same+/// interpolation as the inferred-track path). Returns the filled grid and the+/// consumed line indices, or `None` if no line landed inside the grid or too+/// many spans straddle interior boundaries (a decorative box over prose, not a+/// table).+fn assign_cells(+ lines: &[ProjectedLine],+ xs: &[f32],+ ys: &[f32],+ dbg: bool,+) -> Option<(CellGrid, Vec<usize>)> {+ let n_rows = ys.len() - 1;+ let n_cols = xs.len() - 1;+ let mut grid = CellGrid::new(n_rows, n_cols);let mut consumed_indices: Vec<usize> = Vec::new();const GRID_X_SLACK_PT: f32 = 6.0;// Straddle census: spans that cross an interior column boundary by a@@ -2652,22 +2749,6 @@ fn build_ruled_table(let mut span_total = 0usize;let mut span_straddle = 0usize;- let push_cell = |cells: &mut Vec<Vec<String>>,- cell_has_text: &mut Vec<Vec<bool>>,- row: usize,- col: usize,- txt: &str| {- let txt = txt.trim();- if txt.is_empty() {- return;- }- if !cells[row][col].is_empty() {- cells[row][col].push(' ');- }- cells[row][col].push_str(txt);- cell_has_text[row][col] = true;- };-// Iterate lines top-to-bottom. The global pass receives lines in xy_cut// leaf order (column-by-column), which scrambles a multi-line cell's text// when concatenated in array order; y-sorted iteration restores reading@@ -2685,7 +2766,7 @@ fn build_ruled_table(// default when a span's own y doesn't resolve. Spans bin by their own// baseline below — a projected line can merge text from several ruled// rows (e.g. a multi-baseline wrapped header emitted as one line).- let row = match find_bucket(&ys, cy) {+ let row = match find_bucket(ys, cy) {Some(r) => r,None => continue,};@@ -2716,26 +2797,15 @@ fn build_ruled_table(.filter(|s| !s.text.trim().is_empty()).collect();text_spans.sort_by(|a, b| a.x.total_cmp(&b.x));- let push_repl = |cells_repl: &mut Vec<Vec<String>>, row: usize, col: usize, txt: &str| {- let txt = txt.trim();- if txt.is_empty() {- return;- }- let dst = &mut cells_repl[row][col];- if !dst.is_empty() {- dst.push(' ');- }- dst.push_str(txt);- };if text_spans.is_empty() {// No raw text spans (synthetic/OCR lines): old whole-line centroid path.let cx = line.bbox.x + line.bbox.width * 0.5;- if let Some(col) = find_bucket(&xs, cx.clamp(xs[0], xs[n_cols])) {- push_cell(&mut cells, &mut cell_has_text, row, col, &line.text);- push_repl(&mut cells_repl, row, col, &line.text);+ if let Some(col) = find_bucket(xs, cx.clamp(xs[0], xs[n_cols])) {+ grid.push_text(row, col, &line.text);+ grid.push_repl(row, col, &line.text);if !line.all_bold {- cell_is_bold[row][col] = false;+ grid.is_bold[row][col] = false;}consumed_indices.push(idx);}@@ -2746,24 +2816,24 @@ fn build_ruled_table(// Per-span row: a span carries its own baseline y, which may sit// in a different ruled row than the line centroid.let span_cy = span.y + span.height * 0.5;- let row = find_bucket(&ys, span_cy.clamp(ys[0], ys[n_rows])).unwrap_or(row);+ let row = find_bucket(ys, span_cy.clamp(ys[0], ys[n_rows])).unwrap_or(row);let sx0 = (span.x).clamp(xs[0], xs[n_cols]);let sx1 = (span.x + span.width).clamp(xs[0], xs[n_cols]);- let c_lo = find_bucket(&xs, sx0).unwrap_or(0);- let c_hi = find_bucket(&xs, sx1).unwrap_or(n_cols - 1);+ let c_lo = find_bucket(xs, sx0).unwrap_or(0);+ let c_hi = find_bucket(xs, sx1).unwrap_or(n_cols - 1);span_total += 1;{let m0 = (span.x + STRADDLE_MARGIN_PT).clamp(xs[0], xs[n_cols]);let m1 = (span.x + span.width - STRADDLE_MARGIN_PT).clamp(xs[0], xs[n_cols]);- if m1 > m0 && find_bucket(&xs, m0) != find_bucket(&xs, m1) {+ if m1 > m0 && find_bucket(xs, m0) != find_bucket(xs, m1) {span_straddle += 1;}}if c_lo == c_hi {- push_cell(&mut cells, &mut cell_has_text, row, c_lo, &span.text);- push_repl(&mut cells_repl, row, c_lo, &span.text);+ grid.push_text(row, c_lo, &span.text);+ grid.push_repl(row, c_lo, &span.text);if !line.all_bold {- cell_is_bold[row][c_lo] = false;+ grid.is_bold[row][c_lo] = false;}continue;}@@ -2772,29 +2842,29 @@ fn build_ruled_table(// label is alpha-dominant (group headers are words; merged data// runs are mostly digits).for col in c_lo..=c_hi {- push_repl(&mut cells_repl, row, col, &span.text);+ grid.push_repl(row, col, &span.text);}if is_alpha_dominant(&span.text) {- row_alpha_spanner[row] = true;+ grid.row_alpha_spanner[row] = true;}// Span crosses interior boundaries: split at the whitespace// nearest each crossed boundary x (xs[k] is column k's left// boundary, which is exactly the split target).let covered: Vec<usize> = (c_lo..=c_hi).collect();- if let Some(pieces) = split_span_at_anchors(span, &covered, &xs) {+ if let Some(pieces) = split_span_at_anchors(span, &covered, xs) {for (k, piece) in pieces.iter().enumerate() {- push_cell(&mut cells, &mut cell_has_text, row, c_lo + k, piece);+ grid.push_text(row, c_lo + k, piece);if !line.all_bold {- cell_is_bold[row][c_lo + k] = false;+ grid.is_bold[row][c_lo + k] = false;}}} else {// No whitespace to split on — assign whole span by center.let cx = (span.x + span.width * 0.5).clamp(xs[0], xs[n_cols]);- if let Some(col) = find_bucket(&xs, cx) {- push_cell(&mut cells, &mut cell_has_text, row, col, &span.text);+ if let Some(col) = find_bucket(xs, cx) {+ grid.push_text(row, col, &span.text);if !line.all_bold {- cell_is_bold[row][col] = false;+ grid.is_bold[row][col] = false;}}}@@ -2824,273 +2894,176 @@ fn build_ruled_table(return None;}- // Collapse "phantom rows" produced by stacked thin border-strip rects- // (doc 149 draws each visual table row as: top border strip ~1pt, body- // rect ~22pt, bottom border strip ~5pt — each contributes y-coords that- // survive the 2pt clustering as separate grid rows). Rule: drop a row- // iff (a) it has no text in any cell AND (b) its height is < 50% of the- // median non-empty row height. The height gate preserves real- // fill-in-the-blank forms where empty body rows are full-height.- let row_heights: Vec<f32> = (0..n_rows).map(|r| ys[r + 1] - ys[r]).collect();- let nonempty_heights: Vec<f32> = (0..n_rows)- .filter(|r| cell_has_text[*r].iter().any(|t| *t))- .map(|r| row_heights[r])- .collect();- let median_h = if !nonempty_heights.is_empty() {- let mut s = nonempty_heights.clone();- s.sort_by(|a, b| a.total_cmp(b));- s[s.len() / 2]- } else {- let mut s = row_heights.clone();- s.sort_by(|a, b| a.total_cmp(b));- s[s.len() / 2]- };- let keep: Vec<bool> = (0..n_rows)- .map(|r| {- let has_text = cell_has_text[r].iter().any(|t| *t);- has_text || row_heights[r] >= median_h * 0.8+ Some((grid, consumed_indices))+}++/// Colspan header-band flattening (Mode 5 / TRM): a sparse top row whose alpha+/// spanning label covers several columns ("North America" over its+/// Revenue/Units sub-columns) followed by a dense label row is a stacked+/// header. Flatten rows `0..=b` from the colspan-semantics (`repl`) grid into+/// one header row (per-column top-to-bottom join), so each column carries its+/// full layer chain ("North America Revenue") — that chain is what header-keyed+/// evals (and readers) need. Returns `(header_row, body_start)` or `None`.+fn flatten_header_band(+ cells: &[Vec<String>],+ cell_has_text: &[Vec<bool>],+ cells_repl: &[Vec<String>],+ row_alpha_spanner: &[bool],+ n_rows: usize,+ n_cols: usize,+ dbg: bool,+) -> Option<(Vec<String>, usize)> {+ let row_fill =+ |r: usize| cell_has_text[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;+ (0..n_rows)+ .find(|r| row_fill(*r) >= TABLE_ROW_MIN_FILL)+ .and_then(|b| {+ let nonempty = cell_has_text[b].iter().filter(|t| **t).count();+ let alpha_cells = (0..n_cols)+ .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))+ .count();+ // The bottom header layer may carry digit labels (years like+ // "2024") but never measurement values. A decimal / % / $ /+ // dash-placeholder / comma-grouped number in the anchor row means+ // it's the first DATA row of a table whose real header just+ // missed the 0.9-fill anchor (colspan header covering <90% of+ // columns) — folding it would eat a data row (DS5795A_page4).+ let has_value_cell =+ (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));+ let qualifies = (1..=3).contains(&b)+ && b + 1 < n_rows+ && (0..b).any(|r| row_alpha_spanner[r])+ && alpha_cells * 2 >= nonempty+ && !has_value_cell;+ if !qualifies {+ return None;+ }+ let header: Vec<String> = (0..n_cols)+ .map(|c| {+ let mut parts: Vec<&str> = Vec::new();+ for row in cells_repl.iter().take(b + 1) {+ let s = row[c].as_str();+ if s.is_empty() || parts.last() == Some(&s) {+ continue;+ }+ parts.push(s);+ }+ parts.join(" ")+ })+ .collect();+ if header.iter().all(|h| h.is_empty()) {+ return None;+ }+ if dbg {+ eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");+ }+ Some((header, b + 1))})- .collect();- let kept_row_heights: Vec<f32> = (0..n_rows)- .filter(|r| keep[*r])- .map(|r| row_heights[r])- .collect();- let cells: Vec<Vec<String>> = (0..n_rows)- .filter(|r| keep[*r])- .map(|r| cells[r].clone())- .collect();- let cell_has_text: Vec<Vec<bool>> = (0..n_rows)- .filter(|r| keep[*r])- .map(|r| cell_has_text[r].clone())- .collect();- let cell_is_bold: Vec<Vec<bool>> = (0..n_rows)- .filter(|r| keep[*r])- .map(|r| cell_is_bold[r].clone())- .collect();- let cells_repl: Vec<Vec<String>> = (0..n_rows)- .filter(|r| keep[*r])- .map(|r| cells_repl[r].clone())- .collect();- let row_alpha_spanner: Vec<bool> = (0..n_rows)- .filter(|r| keep[*r])- .map(|r| row_alpha_spanner[r])- .collect();- let n_rows = cells.len();- if n_rows < 2 {- if dbg {- eprintln!("[ruled] REJECT rows-after-collapse {n_rows}");- }- return None;- }+}- // Mirror the row-collapse rule on columns. Some ruled tables (doc 149)- // draw their left/right borders as thin strip rects 5pt wide — those- // become phantom columns with no text. Drop columns that are both empty- // AND noticeably narrower than the median text-bearing column.- let col_widths: Vec<f32> = (0..n_cols).map(|c| xs[c + 1] - xs[c]).collect();- let nonempty_col_widths: Vec<f32> = (0..n_cols)- .filter(|c| (0..n_rows).any(|r| cell_has_text[r][*c]))- .map(|c| col_widths[c])- .collect();- let median_w = if !nonempty_col_widths.is_empty() {- let mut s = nonempty_col_widths.clone();- s.sort_by(|a, b| a.total_cmp(b));- s[s.len() / 2]+/// Stacked-header merge (Mode 5): some generators rule every text baseline of a+/// wrapped header cell, slicing one logical header row into several thin sparse+/// rows ("Rated" / "Voltage" / "(VDC)" each in its own band). When the top+/// `k ≥ 2` rows are individually sparse but their union covers most columns AND+/// a dense data row follows, collapse them into a single header row (per-column+/// top-to-bottom join). `flattened` is whether `flatten_header_band` already+/// fired (the two are mutually exclusive). Returns the possibly-merged grid+/// plus a flag for whether the merge happened.+#[allow(clippy::type_complexity)]+fn merge_stacked_header(+ cells: Vec<Vec<String>>,+ cell_has_text: Vec<Vec<bool>>,+ cell_is_bold: Vec<Vec<bool>>,+ n_rows: usize,+ n_cols: usize,+ kept_row_heights: &[f32],+ flattened: bool,+ dbg: bool,+) -> (+ Vec<Vec<String>>,+ Vec<Vec<bool>>,+ Vec<Vec<bool>>,+ usize,+ bool,+) {+ let row_fill =+ |r: usize, has: &[Vec<bool>]| has[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;+ // Anchor on the first fully-dense row (the first real data row).+ let k = (0..n_rows)+ .find(|r| row_fill(*r, &cell_has_text) >= TABLE_ROW_MIN_FILL)+ .unwrap_or(0);+ let union_cols = (0..n_cols)+ .filter(|c| (0..k).any(|r| cell_has_text[r][*c]))+ .count();+ // Tightness: the band rows must be noticeably shorter than the data+ // rows below — ruled-per-baseline header bands sit at text leading+ // (~7pt) while real rows carry cell padding. Without this, a table+ // whose first body rows are legitimately sparse would get merged.+ let band_tight = if k >= 2 && k < kept_row_heights.len() {+ let mut below: Vec<f32> = kept_row_heights[k..].to_vec();+ below.sort_by(|a, b| a.total_cmp(b));+ let median_below = below[below.len() / 2];+ kept_row_heights[..k]+ .iter()+ .all(|h| *h <= 0.75 * median_below)} else {- let mut s = col_widths.clone();- s.sort_by(|a, b| a.total_cmp(b));- s[s.len() / 2]+ false};- let keep_col: Vec<bool> = (0..n_cols)- .map(|c| {- let has_text = (0..n_rows).any(|r| cell_has_text[r][c]);- has_text || col_widths[c] >= median_w * 0.3- })- .collect();- // Cap how aggressively we drop columns. A real table with one phantom- // border-strip column (doc 149: 5pt-wide left border) drops exactly one.- // Anything more than that is almost certainly a chart whose vertical- // grid-lines got merged with text data (doc 078: a chart's 18 V-lines- // straddle a table to its left/right and collapse to 1 col), so the- // "table" is bogus — bail and let the borderless detector handle it.- // Note: keep_col already only drops columns where both (a) no text AND- // (b) width < 30% of median text-bearing column.- let cells: Vec<Vec<String>> = cells- .into_iter()- .map(|row| {- row.into_iter()- .enumerate()- .filter(|(c, _)| keep_col[*c])- .map(|(_, v)| v)- .collect()- })- .collect();- let cell_has_text: Vec<Vec<bool>> = cell_has_text- .into_iter()- .map(|row| {- row.into_iter()- .enumerate()- .filter(|(c, _)| keep_col[*c])- .map(|(_, v)| v)- .collect()- })- .collect();- let cell_is_bold: Vec<Vec<bool>> = cell_is_bold- .into_iter()- .map(|row| {- row.into_iter()- .enumerate()- .filter(|(c, _)| keep_col[*c])- .map(|(_, v)| v)- .collect()- })- .collect();- let cells_repl: Vec<Vec<String>> = cells_repl- .into_iter()- .map(|row| {- row.into_iter()- .enumerate()- .filter(|(c, _)| keep_col[*c])- .map(|(_, v)| v)- .collect()- })- .collect();- let n_cols = cells.first().map(|r| r.len()).unwrap_or(0);- if n_cols == 0 {- return None;- }- // Single-column tables are ambiguous (could be a captioned card) — require- // ≥3 rows of geometric + textual evidence.- if n_cols == 1 && n_rows < 3 {- return None;+ if flattened || k < 2 || k >= n_rows || !band_tight || (union_cols as f32) < 0.7 * n_cols as f32+ {+ return (cells, cell_has_text, cell_is_bold, n_rows, false);}-- // Colspan header-band flattening (Mode 5 / TRM): a sparse top row whose- // alpha spanning label covers several columns ("North America" over its- // Revenue/Units sub-columns) followed by a dense label row is a stacked- // header. Flatten rows 0..=b from the colspan-semantics grid into one- // header row (per-column top-to-bottom join), so each column carries its- // full layer chain ("North America Revenue") — that chain is what- // header-keyed evals (and readers) need.- let flattened_header: Option<(Vec<String>, usize)> = {- let row_fill =- |r: usize| cell_has_text[r].iter().filter(|t| **t).count() as f32 / n_cols as f32;- (0..n_rows)- .find(|r| row_fill(*r) >= TABLE_ROW_MIN_FILL)- .and_then(|b| {- let nonempty = cell_has_text[b].iter().filter(|t| **t).count();- let alpha_cells = (0..n_cols)- .filter(|c| cell_has_text[b][*c] && is_alpha_dominant(&cells[b][*c]))- .count();- // The bottom header layer may carry digit labels (years like- // "2024") but never measurement values. A decimal / % / $ /- // dash-placeholder / comma-grouped number in the anchor row means- // it's the first DATA row of a table whose real header just- // missed the 0.9-fill anchor (colspan header covering <90% of- // columns) — folding it would eat a data row (DS5795A_page4).- let has_value_cell =- (0..n_cols).any(|c| cell_has_text[b][c] && is_value_like(&cells[b][c]));- let qualifies = (1..=3).contains(&b)- && b + 1 < n_rows- && (0..b).any(|r| row_alpha_spanner[r])- && alpha_cells * 2 >= nonempty- && !has_value_cell;- if !qualifies {- return None;- }- let header: Vec<String> = (0..n_cols)- .map(|c| {- let mut parts: Vec<&str> = Vec::new();- for row in cells_repl.iter().take(b + 1) {- let s = row[c].as_str();- if s.is_empty() || parts.last() == Some(&s) {- continue;- }- parts.push(s);- }- parts.join(" ")- })- .collect();- if header.iter().all(|h| h.is_empty()) {- return None;- }- if dbg {- eprintln!("[ruled] colspan header flatten: rows 0..={b} -> {header:?}");+ if dbg {+ eprintln!("[ruled] stacked-header merge: top {k} rows → 1");+ }+ let mut merged_row = vec![String::new(); n_cols];+ let mut merged_has = vec![false; n_cols];+ let mut merged_bold = vec![true; n_cols];+ for r in 0..k {+ for c in 0..n_cols {+ if cell_has_text[r][c] {+ if !merged_row[c].is_empty() {+ merged_row[c].push(' ');}- Some((header, b + 1))- })- };-- // Stacked-header merge (Mode 5): some generators rule every text baseline- // of a wrapped header cell, slicing one logical header row into several- // thin sparse rows ("Rated" / "Voltage" / "(VDC)" each in its own band).- // When the top k≥2 rows are individually sparse but their union covers- // most columns AND a dense data row follows, collapse them into a single- // header row (per-column top-to-bottom join).- let (cells, cell_has_text, cell_is_bold, n_rows, merged_stacked_header) = {- let row_fill = |r: usize, has: &Vec<Vec<bool>>| {- has[r].iter().filter(|t| **t).count() as f32 / n_cols as f32- };- // Anchor on the first fully-dense row (the first real data row).- let k = (0..n_rows)- .find(|r| row_fill(*r, &cell_has_text) >= TABLE_ROW_MIN_FILL)- .unwrap_or(0);- let union_cols = (0..n_cols)- .filter(|c| (0..k).any(|r| cell_has_text[r][*c]))- .count();- // Tightness: the band rows must be noticeably shorter than the data- // rows below — ruled-per-baseline header bands sit at text leading- // (~7pt) while real rows carry cell padding. Without this, a table- // whose first body rows are legitimately sparse would get merged.- let band_tight = if k >= 2 && k < kept_row_heights.len() {- let mut below: Vec<f32> = kept_row_heights[k..].to_vec();- below.sort_by(|a, b| a.total_cmp(b));- let median_below = below[below.len() / 2];- kept_row_heights[..k]- .iter()- .all(|h| *h <= 0.75 * median_below)- } else {- false- };- if flattened_header.is_none()- && k >= 2- && k < n_rows- && band_tight- && union_cols as f32 >= 0.7 * n_cols as f32- {- if dbg {- eprintln!("[ruled] stacked-header merge: top {k} rows → 1");- }- let mut merged_row = vec![String::new(); n_cols];- let mut merged_has = vec![false; n_cols];- let mut merged_bold = vec![true; n_cols];- for r in 0..k {- for c in 0..n_cols {- if cell_has_text[r][c] {- if !merged_row[c].is_empty() {- merged_row[c].push(' ');- }- merged_row[c].push_str(&cells[r][c]);- merged_has[c] = true;- if !cell_is_bold[r][c] {- merged_bold[c] = false;- }- }+ merged_row[c].push_str(&cells[r][c]);+ merged_has[c] = true;+ if !cell_is_bold[r][c] {+ merged_bold[c] = false;}}- let mut new_cells = vec![merged_row];- let mut new_has = vec![merged_has];- let mut new_bold = vec![merged_bold];- new_cells.extend(cells[k..].iter().cloned());- new_has.extend(cell_has_text[k..].iter().cloned());- new_bold.extend(cell_is_bold[k..].iter().cloned());- let nr = new_cells.len();- (new_cells, new_has, new_bold, nr, true)- } else {- (cells, cell_has_text, cell_is_bold, n_rows, false)}- };+ }+ let mut new_cells = vec![merged_row];+ let mut new_has = vec![merged_has];+ let mut new_bold = vec![merged_bold];+ new_cells.extend(cells[k..].iter().cloned());+ new_has.extend(cell_has_text[k..].iter().cloned());+ new_bold.extend(cell_is_bold[k..].iter().cloned());+ let nr = new_cells.len();+ (new_cells, new_has, new_bold, nr, true)+}+/// Density gate: a grid that is mostly empty cells is rejected unless it shows+/// strong table evidence. Three escape hatches keep real tables:+/// - **col0 spine**: a filled, short-text first column (a label column).+/// - **long-prose table**: a large (≥5×3) grid with a bold header band covering+/// ≥3 columns and a dense (≥70%-fill) inner description column — a+/// multi-line legal/reference table whose description wraps over many empty+/// continuation rows (docs 088/089/090).+/// - **flattened header**: a fired colspan header flatten is table evidence on+/// par with a spine.+/// With a spine but above the higher WITH_SPINE ceiling, still reject (unless a+/// long-prose table). `flattened` is whether `flatten_header_band` fired.+/// Returns `true` to keep the table, `false` to reject it.+fn passes_density_gate(+ cells: &[Vec<String>],+ cell_has_text: &[Vec<bool>],+ cell_is_bold: &[Vec<bool>],+ n_rows: usize,+ n_cols: usize,+ flattened: bool,+ dbg: bool,+) -> bool {let total = n_rows * n_cols;let empty_count = cell_has_text.iter()@@ -3098,78 +3071,233 @@ fn build_ruled_table(.filter(|filled| !**filled).count();let empty_frac = (empty_count as f32) / (total as f32);- if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION {- let col0_fill = (0..n_rows).filter(|r| cell_has_text[*r][0]).count() as f32 / n_rows as f32;- let col0_max_chars = (0..n_rows)- .filter(|r| cell_has_text[*r][0])- .map(|r| cells[r][0].len())- .max()- .unwrap_or(0);- let col0_spine =- col0_fill >= TABLE_SPINE_FILL_FRACTION && col0_max_chars <= TABLE_SPINE_MAX_CELL_CHARS;- // Long-prose table bypass: a large grid (≥5×3) with a structurally- // bold row 0 header AND a dense description column (any col with- // ≥70% fill rate) is almost certainly a multi-line legal/reference- // table where the description column wraps over many empty- // continuation rows. Density arbitration in `merge_table_runs`- // prevents a decorative grid that happens to match these criteria- // from beating a real overlapping borderless table.- // Reproducer: docs 088/089/090 multi-page legal report.- // Header may span multiple visual rows (the grid detector slices on- // each text baseline). Treat the first ≤4 rows as the header band- // and require their *union* to cover most columns AND be all-bold.- let header_band = n_rows.min(4);- let mut header_cols_covered = vec![false; n_cols];- let mut header_all_bold = true;- for r in 0..header_band {- for c in 0..n_cols {- if cell_has_text[r][c] {- header_cols_covered[c] = true;- if !cell_is_bold[r][c] {- header_all_bold = false;- }+ if empty_frac <= TABLE_MAX_EMPTY_CELL_FRACTION {+ return true;+ }+ let col0_fill = (0..n_rows).filter(|r| cell_has_text[*r][0]).count() as f32 / n_rows as f32;+ let col0_max_chars = (0..n_rows)+ .filter(|r| cell_has_text[*r][0])+ .map(|r| cells[r][0].len())+ .max()+ .unwrap_or(0);+ let col0_spine =+ col0_fill >= TABLE_SPINE_FILL_FRACTION && col0_max_chars <= TABLE_SPINE_MAX_CELL_CHARS;+ // Header may span multiple visual rows (the grid detector slices on each+ // text baseline). Treat the first ≤4 rows as the header band and require+ // their *union* to cover most columns AND be all-bold.+ let header_band = n_rows.min(4);+ let mut header_cols_covered = vec![false; n_cols];+ let mut header_all_bold = true;+ for r in 0..header_band {+ for c in 0..n_cols {+ if cell_has_text[r][c] {+ header_cols_covered[c] = true;+ if !cell_is_bold[r][c] {+ header_all_bold = false;}}}- let header_coverage = header_cols_covered.iter().filter(|t| **t).count();- let dense_inner_col = (1..n_cols).any(|c| {- let col_fill =- (0..n_rows).filter(|r| cell_has_text[*r][c]).count() as f32 / n_rows as f32;- col_fill >= TABLE_SPINE_FILL_FRACTION- });- // Header coverage doesn't need to span every column — wide-cell- // legal tables often spread the header across many visual baselines- // and only a few columns land in the top-4-rows band. Require ≥3- // columns covered as evidence of a real header, not just a title.- let long_prose_table = n_rows >= 5- && n_cols >= 3- && header_coverage >= 3- && header_all_bold- && dense_inner_col;- // A fired colspan header flatten is table evidence on par with a- // label spine: it required a ≥0.9-fill alpha label row under an- // alpha spanning group label — decorative grids don't have that.- // Still subject to the WITH_SPINE ceiling below.- let flattened_evidence = flattened_header.is_some();- if !col0_spine && !long_prose_table && !flattened_evidence {- if dbg {- let fills: Vec<usize> = (0..n_rows)- .map(|r| cell_has_text[r].iter().filter(|t| **t).count())- .collect();- eprintln!(- "[ruled] REJECT empty-frac {empty_frac:.2} ({n_rows}x{n_cols}, no spine/long-prose) row_fills={fills:?}"- );- }- return None;+ }+ let header_coverage = header_cols_covered.iter().filter(|t| **t).count();+ let dense_inner_col = (1..n_cols).any(|c| {+ let col_fill = (0..n_rows).filter(|r| cell_has_text[*r][c]).count() as f32 / n_rows as f32;+ col_fill >= TABLE_SPINE_FILL_FRACTION+ });+ // Header coverage doesn't need to span every column — wide-cell legal+ // tables often spread the header across many visual baselines and only a+ // few columns land in the top-4-rows band. Require ≥3 columns covered as+ // evidence of a real header, not just a title.+ let long_prose_table =+ n_rows >= 5 && n_cols >= 3 && header_coverage >= 3 && header_all_bold && dense_inner_col;+ if !col0_spine && !long_prose_table && !flattened {+ if dbg {+ let fills: Vec<usize> = (0..n_rows)+ .map(|r| cell_has_text[r].iter().filter(|t| **t).count())+ .collect();+ eprintln!(+ "[ruled] REJECT empty-frac {empty_frac:.2} ({n_rows}x{n_cols}, no spine/long-prose) row_fills={fills:?}"+ );+ }+ return false;+ }+ if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION_WITH_SPINE && !long_prose_table {+ if dbg {+ eprintln!("[ruled] REJECT empty-frac-with-spine {empty_frac:.2}");}- if empty_frac > TABLE_MAX_EMPTY_CELL_FRACTION_WITH_SPINE && !long_prose_table {+ return false;+ }+ true+}++/// Build a `TableRun` for one ruled-grid component. Returns `None` if the+/// resulting grid is too small (< 2 cols or < 2 rows), covers nearly the+/// whole page (likely the page border), or is mostly empty cells.+fn build_ruled_table(+ hs: &[HSeg],+ vs: &[VSeg],+ h_indices: &[usize],+ v_indices: &[usize],+ lines: &[ProjectedLine],+ page_width: f32,+ page_height: f32,+ pass: RuledPass,+) -> Option<(TableRun, Vec<usize>)> {+ let dbg = *super::flags::DEBUG_RULED;+ let mut xs: Vec<f32> = v_indices.iter().map(|&i| vs[i].x).collect();+ xs.sort_by(|a, b| a.total_cmp(b));+ // Coarser, mean-centered clustering for column boundaries: cell-border+ // rects contribute paired edges 4-6pt apart that would otherwise become+ // phantom 5pt "columns" the span splitter then shreds text into.+ cluster_boundaries(&mut xs, TABLE_COL_BOUNDARY_CLUSTER_PT);++ // Distinct row y-coords (cluster again — multiple H lines may share a y).+ // In the global pass, first drop horizontal rules that span only a small+ // fraction of the column extent (see `RULED_HLINE_MIN_COVERAGE`).+ let raw_ys = |idxs: &[usize]| {+ let mut v: Vec<f32> = idxs.iter().map(|&i| hs[i].y).collect();+ v.sort_by(|a, b| a.total_cmp(b));+ dedup_close(&mut v, TABLE_GRID_CLUSTER_PT);+ v+ };+ let ys: Vec<f32> = if pass == RuledPass::Global && xs.len() >= 2 {+ let col_lo = xs[0];+ let col_hi = xs[xs.len() - 1];+ let extent = (col_hi - col_lo).max(1.0);+ let kept: Vec<usize> = h_indices+ .iter()+ .copied()+ .filter(|&i| {+ let h = &hs[i];+ let ov = (h.x_max.min(col_hi) - h.x_min.max(col_lo)).max(0.0);+ ov / extent >= RULED_HLINE_MIN_COVERAGE+ })+ .collect();+ let filtered = raw_ys(&kept);+ if filtered.len() >= 3 {+ filtered+ } else {+ raw_ys(h_indices)+ }+ } else {+ raw_ys(h_indices)+ };+ if dbg {+ eprintln!(+ "[ruled] component: ys={:?} xs={:?} ({} lines in scope)",+ ys,+ xs,+ lines.len()+ );+ }++ // Need ≥2 row boundaries (1 row) and ≥2 column boundaries (1 col); but+ // a 1×1 grid is just a callout box, so also require ≥1 inner divider+ // (i.e. ys.len() ≥ 3 for ≥2 rows). Single-column tables (`xs.len() == 2`)+ // are accepted when row evidence is strong enough — extra guards apply+ // below after the empty-row collapse.+ if ys.len() < 3 || xs.len() < 2 {+ if dbg {+ eprintln!(+ "[ruled] REJECT grid-too-small ys={} xs={}",+ ys.len(),+ xs.len()+ );+ }+ return None;+ }++ let n_rows = ys.len() - 1;+ let n_cols = xs.len() - 1;+ let bbox = crate::types::Rect {+ x: xs[0],+ y: ys[0],+ width: xs[n_cols] - xs[0],+ height: ys[n_rows] - ys[0],+ };++ // Reject page-border-as-table.+ if page_width > 0.0 && page_height > 0.0 {+ let coverage = (bbox.width / page_width) * (bbox.height / page_height);+ if coverage > TABLE_MAX_PAGE_COVERAGE {if dbg {- eprintln!("[ruled] REJECT empty-frac-with-spine {empty_frac:.2}");+ eprintln!("[ruled] REJECT page-coverage {coverage:.2}");}return None;}}+ // Assign text to cells, then reject if no lines landed inside the grid or+ // too many spans straddle interior column boundaries (decorative box, not+ // a table).+ let (mut grid, consumed_indices) = assign_cells(lines, &xs, &ys, dbg)?;++ // Collapse phantom rows (text-less thin border-strip rects) then phantom+ // columns (text-less narrow border strips). A real table with one phantom+ // border-strip column (doc 149) drops exactly one; a chart whose vertical+ // grid-lines merged with text data (doc 078) collapses to 1 col and is+ // rejected below, letting the borderless detector handle it.+ let kept_row_heights = grid.collapse_phantom_rows(&ys);+ let n_rows = grid.n_rows();+ if n_rows < 2 {+ if dbg {+ eprintln!("[ruled] REJECT rows-after-collapse {n_rows}");+ }+ return None;+ }+ grid.collapse_phantom_cols(&xs);+ let n_cols = grid.n_cols();+ if n_cols == 0 {+ return None;+ }+ // Single-column tables are ambiguous (could be a captioned card) — require+ // ≥3 rows of geometric + textual evidence.+ if n_cols == 1 && n_rows < 3 {+ return None;+ }++ // Hand the collapsed grids back to plain locals for the header-detection+ // stages below.+ let CellGrid {+ text: cells,+ is_bold: cell_is_bold,+ has_text: cell_has_text,+ repl: cells_repl,+ row_alpha_spanner,+ } = grid;++ let flattened_header = flatten_header_band(+ &cells,+ &cell_has_text,+ &cells_repl,+ &row_alpha_spanner,+ n_rows,+ n_cols,+ dbg,+ );++ let (cells, cell_has_text, cell_is_bold, n_rows, merged_stacked_header) = merge_stacked_header(+ cells,+ cell_has_text,+ cell_is_bold,+ n_rows,+ n_cols,+ &kept_row_heights,+ flattened_header.is_some(),+ dbg,+ );++ if !passes_density_gate(+ &cells,+ &cell_has_text,+ &cell_is_bold,+ n_rows,+ n_cols,+ flattened_header.is_some(),+ dbg,+ ) {+ return None;+ }+// Header preference order: flattened colspan band (carries the full// per-column layer chain) > merged stacked-header band > bold first row.let header_qualifies = merged_stacked_header@@ -3328,7 +3456,7 @@ fn detect_ruled_tables_impl(graphics: &[GraphicPrimitive],page_width: f32,page_height: f32,- global: bool,+ pass: RuledPass,) -> Vec<(TableRun, Vec<usize>)> {let (hs, vs) = extract_h_v_segments(graphics);let hs = cluster_h_segments(hs);@@ -3347,7 +3475,7 @@ fn detect_ruled_tables_impl(lines,page_width,page_height,- global,+ pass,) {out.push(run);}@@ -3364,10 +3492,16 @@ pub(super) fn detect_ruled_tables(page_width: f32,page_height: f32,) -> Vec<TableRun> {- detect_ruled_tables_impl(lines, graphics, page_width, page_height, false)- .into_iter()- .map(|(r, _)| r)- .collect()+ detect_ruled_tables_impl(+ lines,+ graphics,+ page_width,+ page_height,+ RuledPass::PerRegion,+ )+ .into_iter()+ .map(|(r, _)| r)+ .collect()}/// Page-level ruled-table detection over *all* lines. A table whose rows@@ -3381,7 +3515,7 @@ pub(super) fn detect_ruled_tables_global(page_width: f32,page_height: f32,) -> Vec<(TableRun, Vec<usize>)> {- detect_ruled_tables_impl(lines, graphics, page_width, page_height, true)+ detect_ruled_tables_impl(lines, graphics, page_width, page_height, RuledPass::Global)}/// Count filled (non-empty) cells in a TableRun. GridFallback returns 0 so