Skip to content

A fast, helpful, and open-source document parser

add font resolver

Logan Markewich964f5354dfd0
diff --git a/Cargo.lock b/Cargo.lock
index 2530b72..d15488b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1248,6 +1248,7 @@ checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
name = "liteparse"
version = "2.1.1"
dependencies = [
+ "blake3",
"clap",
"image",
"infer",
@@ -1255,6 +1256,7 @@ dependencies = [
"liteparse-pdfium-sys",
"ordered-float",
"reqwest 0.13.3",
+ "rmp",
"serde",
"serde_json",
"serial_test",
@@ -2029,6 +2031,15 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "rmp"
+version = "0.8.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "rustc-hash"
version = "2.1.2"
diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.toml
index 5576f32..b25e578 100644
--- a/crates/liteparse/Cargo.toml
+++ b/crates/liteparse/Cargo.toml
@@ -19,6 +19,7 @@ default = ["tesseract"]
tesseract = ["dep:tesseract-rs"]
[dependencies]
+blake3 = "1"
clap = { version = "4.5.55", features = ["derive"] }
image = { version = "0.25", default-features = false, features = ["png"] }
infer = "0.19.0"
@@ -36,6 +37,7 @@ url = "2"
web-time = "1.1.0"
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+rmp = "0.8"
tokio = { version = "1.52.3", features = ["rt-multi-thread", "fs", "process"] }
[dev-dependencies]
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index b30adc0..d828337 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -49,7 +49,7 @@ pub(crate) fn extract_pages_from_document(
target_pages: Option<&[u32]>,
max_pages: usize,
) -> Result<Vec<LitePage>, LiteParseError> {
- Ok(extract_pages_and_images(document, target_pages, max_pages, false, false)?.0)
+ Ok(extract_pages_and_images(document, target_pages, max_pages, false, false, None)?.0)
}
/// Same as `extract_pages_from_document` but optionally also renders every
@@ -63,6 +63,7 @@ pub(crate) fn extract_pages_and_images(
max_pages: usize,
render_images: bool,
extract_links: bool,
+ glyph_resolver: Option<&dyn crate::GlyphResolver>,
) -> Result<(Vec<LitePage>, Vec<ExtractedImage>), LiteParseError> {
let page_count = document.page_count();
let mut pages = Vec::new();
@@ -89,7 +90,7 @@ pub(crate) fn extract_pages_and_images(
right: page.width(),
bottom: 0.0,
});
- let mut text_items = extract_page_text_items(&page, &text_page, &view_box)?;
+ let mut text_items = extract_page_text_items(&page, &text_page, &view_box, glyph_resolver)?;
if extract_links {
assign_links(&mut text_items, &page.links(&view_box));
}
@@ -522,6 +523,7 @@ fn extract_page_text_items(
page: &Page,
text_page: &TextPage,
view_box: &RectF,
+ glyph_resolver: Option<&dyn crate::GlyphResolver>,
) -> Result<Vec<TextItem>, LiteParseError> {
let char_count = text_page.char_count();
if char_count <= 0 {
@@ -557,6 +559,7 @@ fn extract_page_text_items(
let mut glyph_decoder = GlyphDecoder::new(
std::env::var("LITEPARSE_DEBUG_GLYPH").is_ok(),
garbage_fonts,
+ glyph_resolver,
);
for i in 0..char_count {
@@ -1015,7 +1018,7 @@ fn adjust_angle_for_rotation(angle_rad: f32, page_rotation: i32) -> f32 {
}
/// Decompose scale factors from a 2D affine matrix.
-/// Computes eigenvalues of M^T * M, matching the platform's Parse_decomposeScale.
+/// Computes eigenvalues of M^T * M.
fn decompose_scale(m: &pdfium::Matrix) -> (f32, f32) {
let (a, b, c, d) = (m.a as f64, m.b as f64, m.c as f64, m.d as f64);
// M^T * M
@@ -1050,7 +1053,6 @@ fn median_f32(values: &[f32]) -> Option<f32> {
}
/// Check if a font is "buggy" based on its name and type.
-/// Mirrors ParseFont_isBuggyFont from the platform.
fn is_buggy_font(font_name: &str, font_type: FontType) -> bool {
// TrueType subset fonts: name starts with "TT" or contains "+TT"
if font_name.starts_with("TT") || font_name.contains("+TT") {
@@ -1067,8 +1069,12 @@ fn is_buggy_font(font_name: &str, font_type: FontType) -> bool {
}
/// Check if a Unicode codepoint indicates buggy encoding.
+/// C0 controls (<=0x1F), DEL + C1 controls (0x7F-0x9F), and the private use area.
+/// None of these are ever legitimate rendered text; C1 controls in particular
+/// are emitted by a common class of subset fonts that mangle ToUnicode into
+/// the 0x80-0x9F range.
fn is_buggy_codepoint(unicode: u32) -> bool {
- unicode <= 0x1F || (unicode > 0xE000 && unicode <= 0xF8FF)
+ unicode <= 0x1F || (0x7F..=0x9F).contains(&unicode) || (unicode > 0xE000 && unicode <= 0xF8FF)
}
fn color_to_argb_hex(c: &pdfium::Color) -> String {
@@ -1083,7 +1089,7 @@ fn color_to_argb_hex(c: &pdfium::Color) -> String {
/// PostScript glyph name the font assigns to the charcode (from /Encoding
/// /Differences or the embedded font program) resolved against the Adobe
/// Glyph List is the authoritative signal in both cases.
-struct GlyphDecoder {
+struct GlyphDecoder<'a> {
fonts: std::collections::HashMap<usize, FontGlyphInfo>,
/// Chars arrive in runs per text object; cache the last object's font key
/// to skip the FPDFTextObj_GetFont FFI call on the common path.
@@ -1092,6 +1098,9 @@ struct GlyphDecoder {
/// Font handles whose /ToUnicode the prescan flagged as garbage (high
/// fraction of control/PUA unicodes across the page).
garbage_fonts: std::collections::HashSet<usize>,
+ /// Optional last-resort recovery hook for untrusted glyphs that the
+ /// built-in glyph-name / reverse-cmap recovery could not decode.
+ resolver: Option<&'a dyn crate::GlyphResolver>,
debug: bool,
}
@@ -1101,6 +1110,13 @@ struct FontGlyphInfo {
/// the ToUnicode as garbage): PDFium's unicode values for this font are
/// untrusted, so every charcode gets a recovery try.
untrusted: bool,
+ /// The font's name matches the legacy buggy-subset heuristic while
+ /// declaring a *standard* base encoding (e.g. MacRomanEncoding) — the
+ /// encoding is a lie, so PDFium derives glyph names from it that are just
+ /// as wrong as the unicode. Skip glyph-name recovery for these and rely on
+ /// the embedded cmap / outline-hash resolver instead (matches the C path,
+ /// which ignores glyph names for `PARSE_TEXT_FONT_BUGGY` fonts).
+ encoding_lies: bool,
/// charcode → resolved replacement text (None = unrecoverable)
cache: std::collections::HashMap<u32, Option<String>>,
/// Lazily-built glyph_index → unicode map from the embedded font
@@ -1108,11 +1124,16 @@ struct FontGlyphInfo {
reverse_cmap: Option<Option<std::collections::HashMap<u32, u32>>>,
}
-impl GlyphDecoder {
- fn new(debug: bool, garbage_fonts: std::collections::HashSet<usize>) -> Self {
+impl<'a> GlyphDecoder<'a> {
+ fn new(
+ debug: bool,
+ garbage_fonts: std::collections::HashSet<usize>,
+ resolver: Option<&'a dyn crate::GlyphResolver>,
+ ) -> Self {
Self {
fonts: std::collections::HashMap::new(),
garbage_fonts,
+ resolver,
last_obj: 0,
last_key: 0,
debug,
@@ -1139,7 +1160,22 @@ impl GlyphDecoder {
self.fonts.entry(key).or_insert_with(|| {
let has_to_unicode = font.has_to_unicode();
let encoding = font.encoding();
+ // Embedded subset fonts whose name matches the legacy "buggy
+ // font" heuristic (TrueType `+TT` / Type1 `......_` subset tags)
+ // routinely lie about their encoding: a standard base encoding
+ // (e.g. MacRomanEncoding) decodes to a shifted alphabet because
+ // the embedded glyph program doesn't follow it. PDFium's unicode
+ // for these looks plausible (printable letters), so the cheap
+ // per-glyph suspicion checks never fire — flag the whole font
+ // untrusted so every glyph goes through recovery. Mirrors the C
+ // path's `PARSE_TEXT_FONT_BUGGY` name flagging (embedded &&
+ // isBuggyFont).
+ let name_buggy = font.is_embedded()
+ && font
+ .base_name()
+ .is_some_and(|name| is_buggy_font(&name, font.font_type()));
let untrusted = garbage
+ || name_buggy
|| (!has_to_unicode
&& !matches!(
encoding.as_deref(),
@@ -1150,17 +1186,19 @@ impl GlyphDecoder {
));
if debug {
eprintln!(
- "[glyph] font={:?} to_unicode={} encoding={:?} garbage={} untrusted={}",
+ "[glyph] font={:?} to_unicode={} encoding={:?} garbage={} name_buggy={} untrusted={}",
font.base_name(),
has_to_unicode,
encoding,
garbage,
+ name_buggy,
untrusted
);
}
FontGlyphInfo {
font,
untrusted,
+ encoding_lies: name_buggy,
cache: std::collections::HashMap::new(),
reverse_cmap: None,
}
@@ -1177,8 +1215,10 @@ impl GlyphDecoder {
return None;
}
let debug = self.debug;
+ let resolver = self.resolver;
let char_code = ch.char_code();
+ let encoding_lies = info.encoding_lies;
let FontGlyphInfo {
font,
cache,
@@ -1189,10 +1229,19 @@ impl GlyphDecoder {
.entry(char_code)
.or_insert_with(|| {
let name = font.char_glyph_name(char_code);
- let resolved = name
- .as_deref()
- .and_then(resolve_glyph_name)
- .filter(|r| r.chars().all(|c| !c.is_control()));
+ // Glyph names of buggy-subset fonts are derived from a lying
+ // base encoding, so they mis-decode exactly like PDFium's
+ // unicode (e.g. charcode 0x53 → name "S" but the glyph draws
+ // 'R'). Skip name recovery for them so the embedded-cmap /
+ // outline-hash resolver below — the only trustworthy signals —
+ // get the chance to correct the glyph.
+ let resolved = if encoding_lies {
+ None
+ } else {
+ name.as_deref()
+ .and_then(resolve_glyph_name)
+ .filter(|r| r.chars().all(|c| !c.is_control()))
+ };
// Fallback: reverse-map the glyph index through the embedded
// font program's own cmap table.
let resolved = resolved.or_else(|| {
@@ -1228,6 +1277,22 @@ impl GlyphDecoder {
None => c.to_string(),
})
});
+ // Last resort: hand the glyph's vector outline to the injected
+ // resolver. Only reached for untrusted glyphs the deterministic
+ // recovery above could not decode.
+ let resolved = resolved.or_else(|| {
+ let resolver = resolver?;
+ let segments =
+ font.glyph_path_segments(char_code, crate::GLYPH_RESOLVER_FONT_SIZE)?;
+ let text = resolver.resolve(&segments)?;
+ if text.is_empty() || text.chars().any(|c| c.is_control()) {
+ return None;
+ }
+ if debug {
+ eprintln!("[glyph] cc=0x{char_code:04X} resolver -> {text:?}");
+ }
+ Some(text)
+ });
if debug {
eprintln!(
"[glyph] cc=0x{char_code:04X} unicode=0x{unicode:04X} name={name:?} -> {resolved:?}"
@@ -1792,6 +1857,11 @@ mod tests {
assert!(is_buggy_codepoint(0xF8FF));
assert!(!is_buggy_codepoint(0xE000));
assert!(!is_buggy_codepoint(0xF900));
+ // DEL + C1 controls (0x7F-0x9F): mangled-ToUnicode signature.
+ assert!(is_buggy_codepoint(0x7F));
+ assert!(is_buggy_codepoint(0x80));
+ assert!(is_buggy_codepoint(0x9F));
+ assert!(!is_buggy_codepoint(0xA0));
}
#[test]
diff --git a/crates/liteparse/src/font_db_resolver.rs b/crates/liteparse/src/font_db_resolver.rs
new file mode 100644
index 0000000..b5d6a44
--- /dev/null
+++ b/crates/liteparse/src/font_db_resolver.rs
@@ -0,0 +1,192 @@
+//! Built-in [`GlyphResolver`] backed by a fragmented glyph-outline → unicode
+//! database (the LlamaParse "font-forge" map).
+//!
+//! The database is a directory of `%02x%02x.msgpack` shards. Each shard is a
+//! stream of concatenated 2-element msgpack arrays `[hash, unicode]`:
+//! * `hash` — a 16-byte `bin` (the first half of the glyph's 32-byte BLAKE3
+//! path hash; the shard filename is its first two bytes).
+//! * `unicode` — a positive integer codepoint.
+//!
+//! To resolve a glyph we hash its outline segments exactly as the producer did
+//! — little-endian `{i32 segment_type, f32 x, f32 y}` per segment, BLAKE3,
+//! truncated to 16 bytes — load the matching shard, and look the key up. This
+//! mirrors the C consumer in `llamaparse/pdfium/parse/src/core/font.c`.
+//!
+//! Not compiled for `wasm32` (no filesystem). Construct directly, or let
+//! [`crate::LiteParse::new`] auto-wire one when `LITEPARSE_FONT_DB_DIR` is set.
+
+use std::collections::HashMap;
+use std::io::{BufReader, Read};
+use std::path::PathBuf;
+use std::sync::{Arc, RwLock};
+
+use crate::glyph_resolver::GlyphResolver;
+
+/// On-disk key width: first 16 bytes of the glyph's BLAKE3 path hash
+/// (`PARSE_FONT_FRAGMENTED_HASH_SIZE` = `BLAKE3_OUT_LEN / 2`).
+const KEY_LEN: usize = 16;
+
+type ShardMap = HashMap<[u8; KEY_LEN], u32>;
+
+/// Resolves buggy-font glyphs against an on-disk outline → unicode database.
+pub struct FontDbResolver {
+ dir: PathBuf,
+ /// Shards loaded on demand and memoised by their 2-byte prefix. `None`
+ /// records a shard that is absent or unreadable so we don't retry it per
+ /// glyph (mirrors the C path treating a missing fragment as "no match").
+ shards: RwLock<HashMap<u16, Option<Arc<ShardMap>>>>,
+ debug: bool,
+}
+
+impl FontDbResolver {
+ /// Create a resolver reading shards from `dir`. No I/O happens here; shards
+ /// load lazily on first lookup, so an empty or missing directory simply
+ /// yields no matches.
+ pub fn new(dir: impl Into<PathBuf>) -> Self {
+ Self {
+ dir: dir.into(),
+ shards: RwLock::new(HashMap::new()),
+ debug: std::env::var("LITEPARSE_DEBUG_GLYPH").is_ok(),
+ }
+ }
+
+ /// First 16 bytes of the BLAKE3 hash of the LE-packed outline segments.
+ fn glyph_key(segments: &[(i32, f32, f32)]) -> [u8; KEY_LEN] {
+ let mut hasher = blake3::Hasher::new();
+ for &(seg_type, x, y) in segments {
+ hasher.update(&seg_type.to_le_bytes());
+ hasher.update(&x.to_le_bytes());
+ hasher.update(&y.to_le_bytes());
+ }
+ let full = hasher.finalize();
+ let mut key = [0u8; KEY_LEN];
+ key.copy_from_slice(&full.as_bytes()[..KEY_LEN]);
+ key
+ }
+
+ /// Return the shard for `prefix`, loading + memoising it on first use.
+ fn shard(&self, prefix: u16) -> Option<Arc<ShardMap>> {
+ if let Some(slot) = self.shards.read().ok()?.get(&prefix) {
+ return slot.clone();
+ }
+ let loaded = self.load_shard(prefix).map(Arc::new);
+ if let Ok(mut w) = self.shards.write() {
+ // Another thread may have inserted while we loaded; keep theirs.
+ return w.entry(prefix).or_insert(loaded).clone();
+ }
+ loaded
+ }
+
+ fn load_shard(&self, prefix: u16) -> Option<ShardMap> {
+ let path = self
+ .dir
+ .join(format!("{:02x}{:02x}.msgpack", prefix >> 8, prefix & 0xff));
+ let file = match std::fs::File::open(&path) {
+ Ok(f) => f,
+ Err(_) => {
+ if self.debug {
+ eprintln!("[glyph] font-db shard not found: {}", path.display());
+ }
+ return None;
+ }
+ };
+ let mut rd = BufReader::new(file);
+ let mut map = ShardMap::new();
+ // Each record is a 2-array `[bin(16+), positive int]`. The stream ends
+ // (or a malformed record appears) and `read_array_len` errors — stop.
+ loop {
+ match rmp::decode::read_array_len(&mut rd) {
+ Ok(2) => {}
+ _ => break,
+ }
+ let bin_len = match rmp::decode::read_bin_len(&mut rd) {
+ Ok(n) => n as usize,
+ Err(_) => break,
+ };
+ if bin_len < KEY_LEN {
+ break;
+ }
+ let mut key = [0u8; KEY_LEN];
+ if rd.read_exact(&mut key).is_err() {
+ break;
+ }
+ // Consume any bytes beyond the 16-byte key (the C reader also keeps
+ // only the first 16 of a `>= 16`-byte bin).
+ if bin_len > KEY_LEN {
+ let mut skip = vec![0u8; bin_len - KEY_LEN];
+ if rd.read_exact(&mut skip).is_err() {
+ break;
+ }
+ }
+ match rmp::decode::read_int::<u32, _>(&mut rd) {
+ Ok(unicode) => {
+ map.insert(key, unicode);
+ }
+ Err(_) => break,
+ }
+ }
+ if self.debug {
+ eprintln!(
+ "[glyph] font-db shard {:02x}{:02x}: {} entries",
+ prefix >> 8,
+ prefix & 0xff,
+ map.len()
+ );
+ }
+ Some(map)
+ }
+}
+
+impl GlyphResolver for FontDbResolver {
+ fn resolve(&self, segments: &[(i32, f32, f32)]) -> Option<String> {
+ let key = Self::glyph_key(segments);
+ let prefix = u16::from(key[0]) << 8 | u16::from(key[1]);
+ let unicode = *self.shard(prefix)?.get(&key)?;
+ let c = char::from_u32(unicode).filter(|c| !c.is_control())?;
+ Some(match crate::glyph_names::presentation_form_expansion(c) {
+ Some(s) => s.to_string(),
+ None => c.to_string(),
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Write `[bin(key), uint(unicode)]` records into the correct shard files
+ /// under `dir`, matching the on-disk format `load_shard` reads.
+ fn write_shards(dir: &std::path::Path, records: &[([u8; KEY_LEN], u32)]) {
+ let mut by_prefix: HashMap<u16, Vec<u8>> = HashMap::new();
+ for (key, unicode) in records {
+ let prefix = u16::from(key[0]) << 8 | u16::from(key[1]);
+ let buf = by_prefix.entry(prefix).or_default();
+ rmp::encode::write_array_len(buf, 2).unwrap();
+ rmp::encode::write_bin(buf, key).unwrap();
+ rmp::encode::write_uint(buf, u64::from(*unicode)).unwrap();
+ }
+ for (prefix, buf) in by_prefix {
+ let path = dir.join(format!("{:02x}{:02x}.msgpack", prefix >> 8, prefix & 0xff));
+ std::fs::write(path, buf).unwrap();
+ }
+ }
+
+ #[test]
+ fn resolves_known_glyph_and_misses_unknown() {
+ let segs = vec![(2i32, 1.0f32, 2.0f32), (0, 3.5, -1.0), (1, 0.0, 0.0)];
+ let key = FontDbResolver::glyph_key(&segs);
+ let dir = tempfile::tempdir().unwrap();
+ write_shards(dir.path(), &[(key, 0x160)]); // U+0160 'Š'
+
+ let resolver = FontDbResolver::new(dir.path());
+ assert_eq!(resolver.resolve(&segs).as_deref(), Some("Š"));
+ // A glyph whose hash isn't in the (loaded) shard → no match.
+ assert_eq!(resolver.resolve(&[(2i32, 9.0f32, 9.0f32)]), None);
+ }
+
+ #[test]
+ fn missing_directory_yields_no_match() {
+ let resolver = FontDbResolver::new("/nonexistent/liteparse/font/db");
+ assert_eq!(resolver.resolve(&[(2i32, 1.0f32, 2.0f32)]), None);
+ }
+}
diff --git a/crates/liteparse/src/glyph_resolver.rs b/crates/liteparse/src/glyph_resolver.rs
new file mode 100644
index 0000000..fe270e7
--- /dev/null
+++ b/crates/liteparse/src/glyph_resolver.rs
@@ -0,0 +1,40 @@
+//! Optional out-of-tree glyph recovery hook.
+//!
+//! liteparse's built-in recovery for fonts with missing/garbage `/ToUnicode`
+//! (PostScript glyph-name → Adobe Glyph List, then the embedded font program's
+//! reverse cmap; see [`crate::extract`]) is deterministic but cannot decode
+//! buggy/obfuscated fonts whose glyph names and cmap are also junk. A
+//! [`GlyphResolver`] lets a caller plug in a richer recovery strategy — e.g. a
+//! glyph-outline → unicode database — without liteparse taking on that
+//! dependency.
+//!
+//! The resolver is supplied the glyph's *vector outline* (path segments), not a
+//! pdfium handle, so the implementation needs no pdfium of its own and stays
+//! decoupled from liteparse's font internals. It is consulted only as a last
+//! resort, after the built-in recovery has failed on a glyph liteparse already
+//! considers untrusted.
+
+/// Font size, in points, at which glyph path segments are sampled before being
+/// handed to a [`GlyphResolver`].
+///
+/// Resolvers that key on a hash of the outline must sample at this exact size
+/// for their keys to line up.
+pub const GLYPH_RESOLVER_FONT_SIZE: f32 = 10.0;
+
+/// Recovers the unicode text for a glyph liteparse's built-in cmap/AGL recovery
+/// could not decode, from the glyph's vector outline.
+///
+/// Implementations live out-of-tree (the published `@llamaindex/liteparse`
+/// ships no resolver). Inject one with [`crate::LiteParse::with_glyph_resolver`].
+pub trait GlyphResolver: Send + Sync {
+ /// Resolve a glyph from its outline.
+ ///
+ /// `segments` is one `(segment_type, x, y)` triple per path segment, as
+ /// produced by `pdfium::Font::glyph_path_segments` at
+ /// [`GLYPH_RESOLVER_FONT_SIZE`] — `segment_type` is the raw pdfium
+ /// `FPDF_SEGMENT_*` value (LINETO=0, BEZIERTO=1, MOVETO=2).
+ ///
+ /// Return the replacement text (one or more chars, e.g. for a ligature),
+ /// or `None` if the glyph is not recognized.
+ fn resolve(&self, segments: &[(i32, f32, f32)]) -> Option<String>;
+}
diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rs
index ecb1913..0efa0ec 100644
--- a/crates/liteparse/src/lib.rs
+++ b/crates/liteparse/src/lib.rs
@@ -7,6 +7,9 @@
// ── Public API re-exports ──────────────────────────────────────────────
pub use config::{LiteParseConfig, OutputFormat};
pub use error::LiteParseError;
+#[cfg(not(target_arch = "wasm32"))]
+pub use font_db_resolver::FontDbResolver;
+pub use glyph_resolver::{GLYPH_RESOLVER_FONT_SIZE, GlyphResolver};
pub use parser::{LiteParse, ParseResult, ScreenshotResult};
pub use search::{SearchOptions, search_items};
pub use types::{ParsedPage, TextItem};
@@ -14,6 +17,7 @@ pub use types::{ParsedPage, TextItem};
// ── Modules with user-facing types (visible in docs) ───────────────────
pub mod config;
pub mod error;
+pub mod glyph_resolver;
pub mod parser;
pub mod search;
pub mod types;
@@ -28,6 +32,9 @@ pub mod extract;
pub mod figure_cluster;
#[doc(hidden)]
pub mod font_cmap;
+#[cfg(not(target_arch = "wasm32"))]
+#[doc(hidden)]
+pub mod font_db_resolver;
#[doc(hidden)]
pub mod glyph_names;
#[doc(hidden)]
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index e345a30..bc25daa 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -41,6 +41,28 @@ pub struct ScreenshotResult {
pub image_bytes: Vec<u8>,
}
+/// Env var pointing at a fragmented glyph-outline → unicode font database
+/// directory (`%02x%02x.msgpack` shards). When set, [`LiteParse::new`]
+/// auto-wires a [`crate::FontDbResolver`] so buggy/obfuscated-font glyphs are
+/// recovered without any extra wiring. Unset (default) leaves the hook dormant.
+#[cfg(not(target_arch = "wasm32"))]
+const FONT_DB_DIR_ENV: &str = "LITEPARSE_FONT_DB_DIR";
+
+/// Build the default glyph resolver from the environment, if configured.
+#[cfg(not(target_arch = "wasm32"))]
+fn default_glyph_resolver() -> Option<std::sync::Arc<dyn crate::GlyphResolver>> {
+ let dir = std::env::var_os(FONT_DB_DIR_ENV)?;
+ if dir.is_empty() {
+ return None;
+ }
+ Some(std::sync::Arc::new(crate::FontDbResolver::new(dir)))
+}
+
+#[cfg(target_arch = "wasm32")]
+fn default_glyph_resolver() -> Option<std::sync::Arc<dyn crate::GlyphResolver>> {
+ None
+}
+
/// Main LiteParse orchestrator.
///
/// ### Thread safety
@@ -63,6 +85,11 @@ pub struct LiteParse {
/// mechanism for plugging an OCR engine in environments without the
/// built-ins (e.g. WASM, where the JS side supplies a callback engine).
ocr_engine_override: Option<std::sync::Arc<dyn OcrEngine>>,
+ /// Optional caller-provided glyph recovery hook. When set, it is consulted
+ /// as a last resort for buggy/obfuscated-font glyphs that liteparse's
+ /// built-in cmap/AGL recovery could not decode. The published package ships
+ /// none; the platform build injects an outline → unicode font-DB resolver.
+ glyph_resolver: Option<std::sync::Arc<dyn crate::GlyphResolver>>,
}
impl LiteParse {
@@ -70,6 +97,7 @@ impl LiteParse {
Self {
config,
ocr_engine_override: None,
+ glyph_resolver: default_glyph_resolver(),
}
}
@@ -80,6 +108,17 @@ impl LiteParse {
self
}
+ /// Inject a glyph recovery hook. When set, glyphs that liteparse considers
+ /// untrusted and cannot decode with its built-in cmap/AGL recovery are
+ /// passed to the resolver as vector-outline segments for a final attempt.
+ pub fn with_glyph_resolver(
+ mut self,
+ resolver: std::sync::Arc<dyn crate::GlyphResolver>,
+ ) -> Self {
+ self.glyph_resolver = Some(resolver);
+ self
+ }
+
/// Parse a document from a file path, returning structured results.
///
/// Non-PDF files are automatically converted to PDF first (requires
@@ -139,6 +178,7 @@ impl LiteParse {
render_images,
self.config.extract_links
&& self.config.output_format == crate::config::OutputFormat::Markdown,
+ self.glyph_resolver.as_deref(),
)?;
let t_extract = web_time::Instant::now();
log(&format!(
diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts
index 3a93b62..cbeae7b 100644
--- a/packages/node/native.d.ts
+++ b/packages/node/native.d.ts
@@ -10,6 +10,11 @@ export interface JsLiteParseConfig {
ocrEnabled?: boolean
/** HTTP OCR server URL. If set, uses HTTP OCR instead of Tesseract. */
ocrServerUrl?: string
+ /**
+ * Extra HTTP headers sent with every request to `ocrServerUrl`
+ * (e.g. `{ Authorization: "Bearer <token>" }`).
+ */
+ ocrServerHeaders?: Record<string, string>
/** Path to tessdata directory for Tesseract. */
tessdataPath?: string
/** Maximum number of pages to parse. */