Skip to content

A fast, helpful, and open-source document parser

feat: raw text items and fallible library init

Logan Markewich5944de212948
diff --git a/crates/liteparse/src/document_metadata.rs b/crates/liteparse/src/document_metadata.rs
index c28ad7b..179dbf1 100644
--- a/crates/liteparse/src/document_metadata.rs
+++ b/crates/liteparse/src/document_metadata.rs
@@ -15,7 +15,7 @@ const XMP_MAX_BYTES: usize = 64 * 1024;
#[cfg(not(target_arch = "wasm32"))]
const XMP_CATALOG_MAX_FILE_BYTES: u64 = 16 * 1024 * 1024;
-pub(crate) fn extract(input: &PdfInput, document: &Document<'_>) -> DocumentMetadata {
+pub fn extract(input: &PdfInput, document: &Document<'_>) -> DocumentMetadata {
let mut metadata = match input {
#[cfg(not(target_arch = "wasm32"))]
PdfInput::Path(path) => std::fs::File::open(path)
@@ -27,8 +27,14 @@ pub(crate) fn extract(input: &PdfInput, document: &Document<'_>) -> DocumentMeta
PdfInput::Path(_) => DocumentMetadata::default(),
};
- metadata.creation_date = document.meta_text("CreationDate");
- metadata.mod_date = document.meta_text("ModDate");
+ // `meta_text` reports an Info key that is present but empty as `Some("")`;
+ // the metadata block has always treated that as absent.
+ metadata.creation_date = document
+ .meta_text("CreationDate")
+ .filter(|value| !value.is_empty());
+ metadata.mod_date = document
+ .meta_text("ModDate")
+ .filter(|value| !value.is_empty());
metadata.file_version = document.file_version();
let security_revision = document.security_handler_revision();
metadata.is_encrypted = Some(security_revision != -1);
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index f8e920e..b314184 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -74,7 +74,7 @@ pub(crate) fn extract_pages_from_document(
}
/// Output of [`extract_pages_and_images`].
-pub(crate) struct ExtractedPages {
+pub struct ExtractedPages {
pub pages: Vec<LitePage>,
pub page_errors: Vec<PageError>,
/// Empty unless `output_options.extract_images` was set.
@@ -96,7 +96,7 @@ pub(crate) struct ExtractedPages {
/// raster image object to bytes (when `output_options.extract_images` is true). Returned
/// `ExtractedImage`s carry the same ids the markdown emitter will reference,
/// so callers can match them up by id.
-pub(crate) fn extract_pages_and_images(
+pub fn extract_pages_and_images(
document: &Document,
target_pages: Option<&[u32]>,
max_pages: usize,
@@ -448,7 +448,7 @@ fn rect_contains_center(rect: &RectF, item: &TextItem) -> bool {
}
#[derive(Debug, Clone, Copy, Default)]
-pub(crate) struct ExtractionOutputOptions {
+pub struct ExtractionOutputOptions {
pub continue_on_page_error: bool,
pub extract_content_bounds: bool,
pub extract_text_metadata: bool,
@@ -2131,7 +2131,7 @@ fn adjust_angle_for_rotation(angle_rad: f32, page_rotation: i32) -> f32 {
/// Decompose scale factors from a 2D affine matrix.
/// Computes eigenvalues of M^T * M.
-fn decompose_scale(m: &pdfium::Matrix) -> (f32, f32) {
+pub(crate) fn decompose_scale(m: &pdfium::Matrix) -> (f32, f32) {
let (a, b, c, d) = (m.a as f64, m.b as f64, m.c as f64, m.d as f64);
// M^T * M
let mt_a = a * a + b * b;
@@ -2165,7 +2165,7 @@ fn median_f32(values: &[f32]) -> Option<f32> {
}
/// Check if a font is "buggy" based on its name and type.
-fn is_buggy_font(font_name: &str, font_type: FontType) -> bool {
+pub(crate) fn is_buggy_font(font_name: &str, font_type: FontType) -> bool {
// TrueType subset fonts: name starts with "TT" or contains "+TT"
if font_name.starts_with("TT") || font_name.contains("+TT") {
return true;
@@ -2185,7 +2185,7 @@ fn is_buggy_font(font_name: &str, font_type: FontType) -> bool {
/// None of these are ever legitimate rendered text; C1 controls in particular
/// are emitted by a common class of subset fonts that mangle ToUnicode into
/// the 0x80-0x9F range.
-fn is_buggy_codepoint(unicode: u32) -> bool {
+pub(crate) fn is_buggy_codepoint(unicode: u32) -> bool {
unicode <= 0x1F || (0x7F..=0x9F).contains(&unicode) || (unicode > 0xE000 && unicode <= 0xF8FF)
}
@@ -2704,7 +2704,7 @@ const CHAR_INFO_CHUNK: usize = 16 * 1024;
/// Chunked reader over the fork's batched char-info API (chromium/8028+).
/// `new` returns None when the loaded pdfium predates the API; callers then
/// fall back to the per-character FFI getters.
-struct CharInfoChunks<'a, 'page, 'lib: 'page> {
+pub(crate) struct CharInfoChunks<'a, 'page, 'lib: 'page> {
text_page: &'a TextPage<'page, 'lib>,
buf: Vec<pdfium::pdfium_sys::FPDF_CHARINFO_LP>,
/// Absolute char index of `buf[0]`.
@@ -2712,7 +2712,7 @@ struct CharInfoChunks<'a, 'page, 'lib: 'page> {
}
impl<'a, 'page, 'lib: 'page> CharInfoChunks<'a, 'page, 'lib> {
- fn new(text_page: &'a TextPage<'page, 'lib>) -> Option<Self> {
+ pub(crate) fn new(text_page: &'a TextPage<'page, 'lib>) -> Option<Self> {
// Empty-buffer call probes symbol support without reading records.
text_page.char_infos_batch(0, &mut [])?;
Some(Self {
@@ -2725,7 +2725,7 @@ impl<'a, 'page, 'lib: 'page> CharInfoChunks<'a, 'page, 'lib> {
/// Record for absolute char index `i` (must be within the page's char
/// count), refilling the chunk buffer on demand. Returns None only if
/// the batch call unexpectedly comes back empty for a valid index.
- fn record(&mut self, i: i32) -> Option<pdfium::pdfium_sys::FPDF_CHARINFO_LP> {
+ pub(crate) fn record(&mut self, i: i32) -> Option<pdfium::pdfium_sys::FPDF_CHARINFO_LP> {
let off = i - self.start;
if off < 0 || off as usize >= self.buf.len() {
self.buf.resize(CHAR_INFO_CHUNK, Default::default());
@@ -2743,55 +2743,55 @@ impl<'a, 'page, 'lib: 'page> CharInfoChunks<'a, 'page, 'lib> {
/// Per-character accessor that reads from a batched record when available
/// and falls back to the per-character FFI getters otherwise. Method
/// semantics mirror the corresponding [`pdfium::TextChar`] methods exactly.
-struct CharView<'a, 'tp> {
- ch: &'a pdfium::TextChar<'tp>,
- rec: Option<pdfium::pdfium_sys::FPDF_CHARINFO_LP>,
+pub(crate) struct CharView<'a, 'tp> {
+ pub(crate) ch: &'a pdfium::TextChar<'tp>,
+ pub(crate) rec: Option<pdfium::pdfium_sys::FPDF_CHARINFO_LP>,
}
impl CharView<'_, '_> {
- fn unicode(&self) -> u32 {
+ pub(crate) fn unicode(&self) -> u32 {
match &self.rec {
Some(rec) => rec.unicode,
None => self.ch.unicode(),
}
}
- fn char_code(&self) -> u32 {
+ pub(crate) fn char_code(&self) -> u32 {
match &self.rec {
Some(rec) => rec.char_code,
None => self.ch.char_code(),
}
}
- fn is_generated(&self) -> bool {
+ pub(crate) fn is_generated(&self) -> bool {
match &self.rec {
Some(rec) => rec.char_type == CHAR_TYPE_GENERATED,
None => self.ch.is_generated(),
}
}
- fn has_unicode_map_error(&self) -> bool {
+ pub(crate) fn has_unicode_map_error(&self) -> bool {
match &self.rec {
Some(rec) => rec.char_type == CHAR_TYPE_NOT_UNICODE,
None => self.ch.has_unicode_map_error(),
}
}
- fn text_render_mode(&self) -> Option<i32> {
+ pub(crate) fn text_render_mode(&self) -> Option<i32> {
match &self.rec {
Some(rec) => (rec.text_render_mode >= 0).then_some(rec.text_render_mode),
None => self.ch.text_render_mode(),
}
}
- fn text_object(&self) -> Option<pdfium::pdfium_sys::FPDF_PAGEOBJECT> {
+ pub(crate) fn text_object(&self) -> Option<pdfium::pdfium_sys::FPDF_PAGEOBJECT> {
match &self.rec {
Some(rec) => (!rec.text_object.is_null()).then_some(rec.text_object),
None => self.ch.text_object(),
}
}
- fn loose_char_box(&self) -> Option<RectF> {
+ pub(crate) fn loose_char_box(&self) -> Option<RectF> {
match &self.rec {
Some(rec) => Some(RectF {
left: rec.loose_box.left,
@@ -2804,7 +2804,7 @@ impl CharView<'_, '_> {
}
/// Strict char box as an f32 rect (page space).
- fn strict_char_box(&self) -> Option<RectF> {
+ pub(crate) fn strict_char_box(&self) -> Option<RectF> {
match &self.rec {
Some(rec) => Some(RectF {
left: rec.left as f32,
diff --git a/crates/liteparse/src/font_db_resolver.rs b/crates/liteparse/src/font_db_resolver.rs
index a5c1184..ee1f78d 100644
--- a/crates/liteparse/src/font_db_resolver.rs
+++ b/crates/liteparse/src/font_db_resolver.rs
@@ -138,15 +138,21 @@ impl FontDbResolver {
impl GlyphResolver for FontDbResolver {
fn resolve(&self, segments: &[(i32, f32, f32)]) -> Option<String> {
- let key = Self::glyph_key(segments);
- let prefix = u16::from(key[0]) << 8 | u16::from(key[1]);
- let unicode = *self.shard(prefix)?.get(&key)?;
+ let unicode = self.resolve_codepoint(segments)?;
let c = char::from_u32(unicode).filter(|c| !c.is_control())?;
Some(match crate::glyph_names::presentation_form_expansion(c) {
Some(s) => s.to_string(),
None => c.to_string(),
})
}
+
+ /// The recorded database value. A value of 0 is a recorded "unknown".
+ fn resolve_codepoint(&self, segments: &[(i32, f32, f32)]) -> Option<u32> {
+ let key = Self::glyph_key(segments);
+ let prefix = u16::from(key[0]) << 8 | u16::from(key[1]);
+ let unicode = *self.shard(prefix)?.get(&key)?;
+ (unicode != 0).then_some(unicode)
+ }
}
#[cfg(test)]
diff --git a/crates/liteparse/src/glyph_names.rs b/crates/liteparse/src/glyph_names.rs
index 1374a51..57c4034 100644
--- a/crates/liteparse/src/glyph_names.rs
+++ b/crates/liteparse/src/glyph_names.rs
@@ -29,6 +29,47 @@ pub fn resolve_glyph_name(name: &str) -> Option<String> {
if out.is_empty() { None } else { Some(out) }
}
+/// Strict single-codepoint glyph-name resolution for the raw item mode
+/// ([`crate::raw_text`]), where every glyph must map to exactly one codepoint.
+/// Differs from [`resolve_glyph_name`] in three ways: no `_` component
+/// splitting, no ligature expansion (the AGL codepoint itself is returned), and
+/// a `uni` name must carry exactly four hex digits. Returns `None` for
+/// anything that does not resolve to exactly one codepoint, and for control /
+/// private-use results, which are never legitimate rendered text.
+pub(crate) fn resolve_glyph_name_codepoint(name: &str) -> Option<u32> {
+ let base = name.split('.').next()?;
+ if base.is_empty() {
+ return None;
+ }
+ let codepoint = algorithmic_codepoint(base).or_else(|| {
+ let idx = AGL_SUBSET.binary_search_by(|(n, _)| (*n).cmp(base)).ok()?;
+ let mut chars = AGL_SUBSET[idx].1.chars();
+ let c = chars.next()?;
+ chars.next().is_none().then_some(c as u32)
+ })?;
+ let control_or_private_use = codepoint <= 0x1F
+ || (0x7F..=0x9F).contains(&codepoint)
+ || (codepoint > 0xE000 && codepoint <= 0xF8FF);
+ (!control_or_private_use).then_some(codepoint)
+}
+
+/// `uniXXXX` (exactly four hex digits) or `uXXXX`..`uXXXXXX`, hex digits in
+/// either case.
+fn algorithmic_codepoint(base: &str) -> Option<u32> {
+ let hex = if let Some(hex) = base.strip_prefix("uni") {
+ (hex.len() == 4).then_some(hex)?
+ } else if let Some(hex) = base.strip_prefix('u') {
+ (4..=6).contains(&hex.len()).then_some(hex)?
+ } else {
+ return None;
+ };
+ if !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
+ return None;
+ }
+ let codepoint = u32::from_str_radix(hex, 16).ok()?;
+ char::from_u32(codepoint).map(|_| codepoint)
+}
+
/// ASCII expansion for ligature presentation forms, matching the existing
/// extraction-time ligature handling.
pub fn presentation_form_expansion(c: char) -> Option<&'static str> {
diff --git a/crates/liteparse/src/glyph_resolver.rs b/crates/liteparse/src/glyph_resolver.rs
index fe270e7..1de247b 100644
--- a/crates/liteparse/src/glyph_resolver.rs
+++ b/crates/liteparse/src/glyph_resolver.rs
@@ -37,4 +37,17 @@ pub trait GlyphResolver: Send + Sync {
/// Return the replacement text (one or more chars, e.g. for a ligature),
/// or `None` if the glyph is not recognized.
fn resolve(&self, segments: &[(i32, f32, f32)]) -> Option<String>;
+
+ /// Resolve a glyph to a single codepoint, without the ligature expansion
+ /// [`resolve`](Self::resolve) may apply. Used by the raw item mode
+ /// ([`crate::raw_text`]), which maps every glyph to exactly one codepoint.
+ /// The default derives it from `resolve` and answers `None` for
+ /// multi-character replacements; a resolver backed by a codepoint database
+ /// should override it to return the recorded value.
+ fn resolve_codepoint(&self, segments: &[(i32, f32, f32)]) -> Option<u32> {
+ let text = self.resolve(segments)?;
+ let mut chars = text.chars();
+ let c = chars.next()?;
+ chars.next().is_none().then_some(c as u32)
+ }
}
diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rs
index ecac613..f875279 100644
--- a/crates/liteparse/src/lib.rs
+++ b/crates/liteparse/src/lib.rs
@@ -11,6 +11,7 @@ pub use error::LiteParseError;
pub use font_db_resolver::FontDbResolver;
pub use glyph_resolver::{GLYPH_RESOLVER_FONT_SIZE, GlyphResolver};
pub use parser::{LiteParse, ParseBatch, ParseResult, ParseSession, ScreenshotResult};
+pub use raw_text::{RawTextItem, extract_raw_text_items};
pub use search::{SearchOptions, search_items};
pub use types::{DocumentMetadata, ParsedPage, TextItem, WordBox};
@@ -20,6 +21,7 @@ pub mod error;
pub mod glyph_resolver;
pub mod layout;
pub mod parser;
+pub mod raw_text;
pub mod search;
pub mod types;
diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs
index 9a2b4f6..77a0c9d 100644
--- a/crates/liteparse/src/parser.rs
+++ b/crates/liteparse/src/parser.rs
@@ -614,8 +614,14 @@ impl LiteParse {
.config
.extract_form_fields
.then(|| document.form_type());
- let creator = document.meta_text("Creator");
- let producer = document.meta_text("Producer");
+ // An Info key that is present but empty is reported as `Some("")`
+ // by `meta_text`; the public metadata keeps treating it as absent.
+ let creator = document
+ .meta_text("Creator")
+ .filter(|value| !value.is_empty());
+ let producer = document
+ .meta_text("Producer")
+ .filter(|value| !value.is_empty());
let doc_meta = want_doc_meta.then(|| {
// AcroForm repair rewrites the file, so provenance has to come
// from the original document; fall back if it no longer loads.
diff --git a/crates/liteparse/src/raw_text.rs b/crates/liteparse/src/raw_text.rs
new file mode 100644
index 0000000..1a4c4fb
--- /dev/null
+++ b/crates/liteparse/src/raw_text.rs
@@ -0,0 +1,432 @@
+//! Raw text items: pdfium's text runs with fixed, heuristic-free segmentation.
+//!
+//! [`extract_raw_text_items`] is the low-level counterpart of the segmenter in
+//! [`crate::extract`]. It applies no layout heuristics — no gap-based line or word
+//! merging, no invisible-text skip, no ligature expansion, no punctuation
+//! normalisation, no dedup — so its output is a stable function of what pdfium
+//! reports and a consumer can build its own segmentation on top. Every glyph pdfium
+//! reports lands in exactly one item, in page order, and a glyph is never dropped on
+//! its own.
+//!
+//! Rules:
+//!
+//! - An item is a run of glyphs from one text object at one angle. A glyph whose
+//! text object differs from the item's first glyph, or whose angle differs from it
+//! by more than 0.01 rad, starts a new item.
+//! - A pdfium-generated `\r` or `\n` closes the open item and is dropped.
+//! - Every other generated glyph, and every ASCII whitespace glyph (`\t`..`\r` and
+//! space; U+00A0, U+3000 and friends do not count), is appended and then closes the
+//! item. A pdfium-synthesised space is therefore always an item's last glyph, which
+//! is what [`RawTextItem::trailing_space_generated`] reports.
+//! - Bounds are the union of the glyphs' loose boxes. Angle, colours, marked-content
+//! id, font and font metrics come from the first glyph. `text_width` sums per-glyph
+//! advance widths.
+//! - A font is "buggy" when it is embedded and its name/type match the subset-font
+//! heuristic, or when any non-generated glyph decodes to a control or private-use
+//! codepoint. A glyph pdfium cannot map to Unicode decodes to 0 and therefore counts,
+//! so every Type3 item is buggy.
+//! - With a [`GlyphResolver`], every non-generated glyph of a buggy item is re-decoded:
+//! Type3 glyph names first, then the resolver on the glyph outline. An unrecognised
+//! glyph becomes a space. Without a resolver the text is left as pdfium decoded it.
+//! - The text is built from the glyph codepoints with C-string semantics: a 0
+//! codepoint (a glyph with no Unicode mapping) terminates the text early, and a
+//! codepoint that is not a Unicode scalar value (a surrogate, or above U+10FFFF)
+//! drops the whole item.
+//!
+//! Coordinates go through [`Page::bounds_to_viewport`] per glyph — the exact
+//! `FPDF_PageToDevice` quantisation — rather than the affine
+//! [`Page::viewport_transform`] approximation, so boxes are reproducible to the last
+//! digit. The viewport is scaled by the page's `/UserUnit`, as everywhere in liteparse.
+
+use pdfium::{Font, FontType, Page, RectF, TextPage};
+
+use crate::GlyphResolver;
+use crate::extract::{
+ CharInfoChunks, CharView, decompose_scale, is_buggy_codepoint, is_buggy_font,
+};
+use crate::glyph_names::resolve_glyph_name_codepoint;
+
+/// Angle difference, in radians, at which a glyph starts a new item.
+const ANGLE_SPLIT_RADIANS: f32 = 0.01;
+
+/// One raw text item, in viewport space (top-left origin, 72 dpi points).
+#[derive(Debug, Clone, PartialEq)]
+pub struct RawTextItem {
+ /// The glyph codepoints as UTF-8. Whitespace is kept, including a trailing
+ /// space; see the module docs for the NUL and invalid-scalar semantics.
+ pub text: String,
+ /// One entry per glyph in page order: the raw char code from the content
+ /// stream, 0 for a pdfium-generated glyph.
+ pub char_codes: Vec<u32>,
+ /// One entry per glyph, parallel to `char_codes`, for Type3 fonts only: the
+ /// PostScript glyph name the font binds to that code, `""` where it binds
+ /// none. `None` for every other font type.
+ pub glyph_names: Option<Vec<String>>,
+ /// Counter-clockwise rotation in radians with the page rotation folded in,
+ /// in `[0, 2π)`. Kept in radians so a consumer can convert in the precision
+ /// it needs.
+ pub angle_radians: f32,
+ /// Sum of the glyphs' advance widths in text space.
+ pub text_width: f32,
+ pub x: f32,
+ pub y: f32,
+ pub width: f32,
+ pub height: f32,
+ /// Marked-content id of the first glyph's text object, when it has one.
+ pub mcid: Option<i32>,
+ /// Base font name with pdfium's subset-tag stripping; `""` when the first
+ /// glyph has no text object or its object has no font.
+ pub font_name: String,
+ pub font_size: f32,
+ pub font_weight: i32,
+ /// `font_size` scaled by the vertical scale of the text matrix; 0 when the
+ /// first glyph has no text object.
+ pub font_height: f32,
+ /// 0 when the font exposes no metrics.
+ pub font_ascent: f32,
+ /// 0 when the font exposes no metrics.
+ pub font_descent: f32,
+ pub font_is_buggy: bool,
+ /// The last glyph is a pdfium-synthesised space, not a space glyph from the
+ /// content stream.
+ pub trailing_space_generated: bool,
+ /// First glyph's fill colour packed as ARGB, when its colour space is
+ /// reportable as RGB.
+ pub fill_color: Option<u32>,
+ /// First glyph's stroke colour packed as ARGB, when reportable.
+ pub stroke_color: Option<u32>,
+}
+
+/// Everything read per glyph before items are formed.
+struct Glyph {
+ index: i32,
+ text_object: Option<pdfium::pdfium_sys::FPDF_PAGEOBJECT>,
+ generated: bool,
+ /// 0 for a generated glyph, which has no source char code.
+ char_code: u32,
+ /// 0 when pdfium reports a unicode-map error for the glyph.
+ unicode: u32,
+ /// Loose glyph box in viewport space.
+ loose: RectF,
+ /// Normalised angle in radians, `[0, 2π)`.
+ angle: f32,
+}
+
+/// Extract the raw text items of `text_page`. `view_box` is the page's crop box
+/// ([`Page::view_box`]); `glyph_resolver` is the optional outline-based recovery
+/// for buggy fonts (see the module docs), `None` to leave buggy text as decoded.
+pub fn extract_raw_text_items(
+ page: &Page,
+ text_page: &TextPage,
+ view_box: &RectF,
+ glyph_resolver: Option<&dyn GlyphResolver>,
+) -> Vec<RawTextItem> {
+ let char_count = text_page.char_count();
+ let page_rotation = page.rotation();
+ let mut chunks = CharInfoChunks::new(text_page);
+ let mut items = Vec::new();
+ let mut run: Vec<Glyph> = Vec::new();
+
+ let flush = |run: &mut Vec<Glyph>, items: &mut Vec<RawTextItem>| {
+ if run.is_empty() {
+ return;
+ }
+ if let Some(item) = build_item(text_page, run, glyph_resolver) {
+ items.push(item);
+ }
+ run.clear();
+ };
+
+ for i in 0..char_count {
+ let ch = text_page.char_at_unchecked(i);
+ let cv = CharView {
+ ch: &ch,
+ rec: chunks.as_mut().and_then(|chunks| chunks.record(i)),
+ };
+ let glyph = load_glyph(page, view_box, page_rotation, &cv, i);
+
+ if let Some(first) = run.first()
+ && (first.text_object != glyph.text_object
+ || (first.angle - glyph.angle).abs() > ANGLE_SPLIT_RADIANS)
+ {
+ flush(&mut run, &mut items);
+ }
+
+ if glyph.generated && matches!(glyph.unicode, 0x0A | 0x0D) {
+ flush(&mut run, &mut items);
+ continue;
+ }
+
+ let closes_item = glyph.generated || is_c_locale_space(glyph.unicode);
+ run.push(glyph);
+ if closes_item {
+ flush(&mut run, &mut items);
+ }
+ }
+ flush(&mut run, &mut items);
+ items
+}
+
+/// The per-glyph reads. A glyph whose loose box cannot be read falls back to its
+/// strict box.
+fn load_glyph(
+ page: &Page,
+ view_box: &RectF,
+ page_rotation: i32,
+ cv: &CharView<'_, '_>,
+ index: i32,
+) -> Glyph {
+ let generated = cv.is_generated();
+ let char_code = if generated { 0 } else { cv.char_code() };
+ let unicode = if cv.has_unicode_map_error() {
+ 0
+ } else {
+ cv.unicode()
+ };
+ let strict = cv.strict_char_box().unwrap_or_default();
+ let loose = cv.loose_char_box().unwrap_or(strict);
+ Glyph {
+ index,
+ text_object: cv.text_object(),
+ generated,
+ char_code,
+ unicode,
+ loose: page.bounds_to_viewport(view_box, &loose),
+ angle: normalize_angle(cv.ch.angle(), page_rotation),
+ }
+}
+
+/// Fold the page's `/Rotate` into pdfium's counter-clockwise glyph angle and
+/// wrap it into `[0, 2π)`. Computed in f64 so the wrap does not lose precision.
+fn normalize_angle(angle_radians: f32, page_rotation: i32) -> f32 {
+ use std::f64::consts::PI;
+ let mut angle = f64::from(angle_radians);
+ match page_rotation {
+ 1 => angle -= 3.0 * PI / 2.0,
+ 2 => angle -= PI,
+ 3 => angle -= PI / 2.0,
+ _ => {}
+ }
+ angle %= 2.0 * PI;
+ if angle < 0.0 {
+ angle += 2.0 * PI;
+ }
+ angle as f32
+}
+
+/// ASCII whitespace only — deliberately not Unicode `White_Space`, so non-breaking
+/// and ideographic spaces stay inside their items.
+fn is_c_locale_space(unicode: u32) -> bool {
+ matches!(unicode, 0x09..=0x0D | 0x20)
+}
+
+fn pack_argb(color: pdfium::Color) -> u32 {
+ u32::from_be_bytes([color.a, color.r, color.g, color.b])
+}
+
+/// Turn a run of glyphs into an item. `None` when a codepoint is not a Unicode
+/// scalar value (see [`c_string_from_utf32`]).
+fn build_item(
+ text_page: &TextPage,
+ glyphs: &[Glyph],
+ glyph_resolver: Option<&dyn GlyphResolver>,
+) -> Option<RawTextItem> {
+ let first = glyphs.first()?;
+ let first_char = text_page.char_at_unchecked(first.index);
+ let font_size = first_char.font_size() as f32;
+ let font = first
+ .text_object
+ .and_then(|obj| unsafe { Font::from_text_object(obj) });
+
+ let mut bounds = first.loose;
+ let mut text_width = 0.0f32;
+ let mut font_name = String::new();
+ let mut font_ascent = 0.0f32;
+ let mut font_descent = 0.0f32;
+ let mut font_is_embedded = false;
+ let mut font_is_buggy = false;
+ if let Some(font) = &font {
+ font_name = font.base_name().unwrap_or_default();
+ font_ascent = font.ascent(font_size).unwrap_or(0.0);
+ font_descent = font.descent(font_size).unwrap_or(0.0);
+ font_is_embedded = font.is_embedded();
+ font_is_buggy = font_is_embedded && is_buggy_font(&font_name, font.font_type());
+ if let Some(width) = font.glyph_width_from_char_code(first.char_code, font_size) {
+ text_width += width;
+ }
+ }
+ let font_height = if first.text_object.is_some() {
+ let scale_y = first_char
+ .matrix()
+ .map(|matrix| decompose_scale(&matrix).1)
+ .unwrap_or(1.0);
+ font_size * scale_y
+ } else {
+ 0.0
+ };
+
+ for glyph in &glyphs[1..] {
+ bounds.left = bounds.left.min(glyph.loose.left);
+ bounds.top = bounds.top.min(glyph.loose.top);
+ bounds.right = bounds.right.max(glyph.loose.right);
+ bounds.bottom = bounds.bottom.max(glyph.loose.bottom);
+ if let Some(font) = &font {
+ let width = if glyph.generated {
+ font.glyph_width(glyph.unicode, font_size)
+ } else {
+ font.glyph_width_from_char_code(glyph.char_code, font_size)
+ };
+ if let Some(width) = width {
+ text_width += width;
+ }
+ }
+ }
+ // A control / private-use decode in an embedded font flags the whole item.
+ // `unicode` is 0 for an unmapped glyph, so those count too.
+ if font_is_embedded && !font_is_buggy {
+ font_is_buggy = glyphs
+ .iter()
+ .any(|glyph| !glyph.generated && is_buggy_codepoint(glyph.unicode));
+ }
+
+ let mut codepoints: Vec<u32> = glyphs.iter().map(|glyph| glyph.unicode).collect();
+ if font_is_buggy && let (Some(font), Some(resolver)) = (&font, glyph_resolver) {
+ let is_type3 = font.font_type() == FontType::Type3;
+ for (glyph, codepoint) in glyphs.iter().zip(codepoints.iter_mut()) {
+ if glyph.generated {
+ continue;
+ }
+ let identified = is_type3
+ .then(|| type3_glyph_codepoint(font, glyph.char_code))
+ .flatten()
+ .or_else(|| identify_glyph(resolver, font, glyph.char_code));
+ *codepoint = identified.unwrap_or(u32::from(' '));
+ }
+ }
+ let text = c_string_from_utf32(&codepoints)?;
+
+ let glyph_names = font
+ .as_ref()
+ .filter(|font| font.font_type() == FontType::Type3)
+ .map(|font| {
+ glyphs
+ .iter()
+ .map(|glyph| font.char_glyph_name(glyph.char_code).unwrap_or_default())
+ .collect()
+ });
+
+ Some(RawTextItem {
+ text,
+ char_codes: glyphs.iter().map(|glyph| glyph.char_code).collect(),
+ glyph_names,
+ angle_radians: first.angle,
+ text_width,
+ x: bounds.left,
+ y: bounds.top,
+ width: bounds.right - bounds.left,
+ height: bounds.bottom - bounds.top,
+ mcid: first_char.marked_content_id(),
+ font_name,
+ font_size,
+ font_weight: first_char.font_weight(),
+ font_height,
+ font_ascent,
+ font_descent,
+ font_is_buggy,
+ trailing_space_generated: glyphs.last().is_some_and(|glyph| glyph.generated),
+ fill_color: first_char.fill_color().map(pack_argb),
+ stroke_color: first_char.stroke_color().map(pack_argb),
+ })
+}
+
+/// The codepoint a Type3 font's own `/Encoding` `/Differences` name resolves to.
+/// A Type3 font has no font program, so this name is the only statement the
+/// document makes about what the glyph means.
+fn type3_glyph_codepoint(font: &Font, char_code: u32) -> Option<u32> {
+ let name = font.char_glyph_name(char_code)?;
+ resolve_glyph_name_codepoint(&name)
+}
+
+/// The resolver's answer for the glyph outline. A glyph with no outline
+/// (whitespace, non-rendered) is unidentified.
+fn identify_glyph(resolver: &dyn GlyphResolver, font: &Font, char_code: u32) -> Option<u32> {
+ let segments = font.glyph_path_segments(char_code, crate::GLYPH_RESOLVER_FONT_SIZE)?;
+ resolver.resolve_codepoint(&segments)
+}
+
+/// Build the item text from raw codepoints with C-string semantics: `None` when
+/// any value is not a Unicode scalar value (the whole item is dropped), otherwise
+/// the text up to the first 0 — a 0 is an unmapped glyph, and it terminates the
+/// string rather than being replaced.
+fn c_string_from_utf32(codepoints: &[u32]) -> Option<String> {
+ let mut chars = Vec::with_capacity(codepoints.len());
+ for &codepoint in codepoints {
+ match char::from_u32(codepoint) {
+ Some(c) => chars.push(c),
+ None => return None,
+ }
+ }
+ Some(chars.into_iter().take_while(|&c| c != '\0').collect())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn c_locale_space_is_ascii_only() {
+ for space in [0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x20] {
+ assert!(is_c_locale_space(space), "{space:#x}");
+ }
+ for not_space in [
+ 0x00, 0x08, 0x0E, 0x1F, 0x21, 0x85, 0xA0, 0x2003, 0x202F, 0x3000,
+ ] {
+ assert!(!is_c_locale_space(not_space), "{not_space:#x}");
+ }
+ }
+
+ #[test]
+ fn utf32_conversion_keeps_c_string_semantics() {
+ assert_eq!(
+ c_string_from_utf32(&[0x48, 0x69, 0x20]).as_deref(),
+ Some("Hi ")
+ );
+ // A 0 terminates the string; what follows is dropped, not the item.
+ assert_eq!(
+ c_string_from_utf32(&[0x48, 0x00, 0x69]).as_deref(),
+ Some("H")
+ );
+ assert_eq!(c_string_from_utf32(&[0x00]).as_deref(), Some(""));
+ assert_eq!(c_string_from_utf32(&[]).as_deref(), Some(""));
+ // A surrogate or out-of-range value drops the item, even after a 0.
+ assert_eq!(c_string_from_utf32(&[0x48, 0xD800]), None);
+ assert_eq!(c_string_from_utf32(&[0x48, 0x00, 0x110000]), None);
+ // Noncharacters are valid scalars and pass through.
+ assert_eq!(c_string_from_utf32(&[0xFFFE]).as_deref(), Some("\u{FFFE}"));
+ }
+
+ #[test]
+ fn angle_folds_page_rotation_and_wraps() {
+ use std::f32::consts::PI;
+ let close = |a: f32, b: f32| (a - b).abs() < 1e-5;
+ assert!(close(normalize_angle(0.0, 0), 0.0));
+ assert!(close(normalize_angle(PI / 2.0, 0), PI / 2.0));
+ // /Rotate 90: pdfium's angle minus 3π/2, wrapped up into [0, 2π).
+ assert!(close(normalize_angle(0.0, 1), PI / 2.0));
+ assert!(close(normalize_angle(0.0, 2), PI));
+ assert!(close(normalize_angle(0.0, 3), 3.0 * PI / 2.0));
+ assert!(close(normalize_angle(-0.5, 0), 2.0 * PI - 0.5));
+ assert!(close(normalize_angle(2.0 * PI + 0.25, 0), 0.25));
+ }
+
+ #[test]
+ fn argb_packing_matches_fpdf_argb() {
+ let color = pdfium::Color {
+ r: 0x12,
+ g: 0x34,
+ b: 0x56,
+ a: 0xFF,
+ };
+ assert_eq!(pack_argb(color), 0xFF12_3456);
+ }
+}
diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs
index a69b348..507ac94 100644
--- a/crates/liteparse/tests/integration_test.rs
+++ b/crates/liteparse/tests/integration_test.rs
@@ -892,3 +892,58 @@ async fn test_ocr_rounds_fill_across_sparse_pages() {
rounds are being cut short by page span"
);
}
+
+/// The raw item mode keeps every glyph: the char codes across all
+/// items add up to the page's char count minus the generated line breaks it
+/// drops, items keep their trailing whitespace, and a generated glyph only
+/// ever closes an item.
+#[test]
+#[serial]
+fn raw_text_items_account_for_every_glyph() {
+ use liteparse::extract_raw_text_items;
+
+ let lib = pdfium::Library::init();
+ let document = lib
+ .load_document("../../integration_tests_data/sample.pdf", None)
+ .expect("sample.pdf loads");
+ let page = document.page(0).expect("page 0 loads");
+ let text_page = page.text().expect("text page loads");
+ let view_box = page.view_box().expect("page has a bounding box");
+
+ let items = extract_raw_text_items(&page, &text_page, &view_box, None);
+ assert!(!items.is_empty(), "sample.pdf page 1 has text");
+
+ let generated_breaks = text_page
+ .chars()
+ .filter(|c| c.is_generated() && matches!(c.unicode(), 0x0A | 0x0D))
+ .count();
+ let glyphs_in_items: usize = items.iter().map(|item| item.char_codes.len()).sum();
+ assert_eq!(
+ glyphs_in_items + generated_breaks,
+ text_page.char_count() as usize,
+ "every glyph lands in exactly one item"
+ );
+
+ for item in &items {
+ assert!(!item.text.is_empty() || item.char_codes.iter().all(|&c| c == 0));
+ assert!(item.width >= 0.0 && item.height >= 0.0);
+ assert!((0.0..std::f32::consts::TAU).contains(&item.angle_radians));
+ // A space glyph is always the last glyph of its item.
+ let spaces: Vec<usize> = item
+ .text
+ .char_indices()
+ .filter(|(_, c)| c.is_ascii_whitespace())
+ .map(|(i, _)| i)
+ .collect();
+ assert!(
+ spaces.iter().all(|&i| i == item.text.len() - 1),
+ "whitespace closes an item: {:?}",
+ item.text
+ );
+ }
+ let text: String = items.iter().map(|item| item.text.as_str()).collect();
+ assert!(
+ text.chars().any(|c| c.is_alphabetic()),
+ "sample text survives: {text:?}"
+ );
+}
diff --git a/crates/pdfium/src/document.rs b/crates/pdfium/src/document.rs
index 826d26a..6a4eaa2 100644
--- a/crates/pdfium/src/document.rs
+++ b/crates/pdfium/src/document.rs
@@ -199,9 +199,10 @@ impl<'lib> Document<'lib> {
self.page(index).map(Some)
}
- /// Read one entry from the document's `/Info` metadata dictionary
- /// (e.g. `"Creator"`, `"Producer"`, `"Title"`). Returns `None` when the
- /// tag is absent or empty.
+ /// Value of `tag` in the document's Info dictionary (`Creator`,
+ /// `Producer`, `CreationDate`, ...). `None` when the document has no Info
+ /// dictionary; `Some("")` when it has one but the key is missing or
+ /// empty — pdfium reports both the same way.
pub fn meta_text(&self, tag: &str) -> Option<String> {
let tag_c = std::ffi::CString::new(tag).ok()?;
let needed = unsafe {
@@ -234,9 +235,10 @@ impl<'lib> Document<'lib> {
} else {
chars
};
- if end == 0 {
- return None;
- }
+ // `end == 0` is an Info dictionary that exists but has no (or an
+ // empty) value for `tag`; pdfium cannot tell those two apart. `None`
+ // is reserved for a document with no Info dictionary at all, which
+ // callers reporting provenance need to distinguish from `""`.
Some(String::from_utf16_lossy(&buf[..end]))
}
diff --git a/crates/pdfium/src/error.rs b/crates/pdfium/src/error.rs
index e8082cf..468dd29 100644
--- a/crates/pdfium/src/error.rs
+++ b/crates/pdfium/src/error.rs
@@ -11,6 +11,8 @@ pub enum PdfiumError {
UnsupportedSecurity,
PageNotFound,
OperationFailed,
+ /// The pdfium shared library could not be located or loaded.
+ LibraryUnavailable,
}
impl PdfiumError {
@@ -37,6 +39,7 @@ impl fmt::Display for PdfiumError {
PdfiumError::UnsupportedSecurity => write!(f, "unsupported security handler"),
PdfiumError::PageNotFound => write!(f, "page not found"),
PdfiumError::OperationFailed => write!(f, "operation failed"),
+ PdfiumError::LibraryUnavailable => write!(f, "pdfium shared library unavailable"),
}
}
}
diff --git a/crates/pdfium/src/library.rs b/crates/pdfium/src/library.rs
index 10ed29f..0ba8b31 100644
--- a/crates/pdfium/src/library.rs
+++ b/crates/pdfium/src/library.rs
@@ -66,9 +66,18 @@ impl Library {
/// Multiple concurrent callers are serialized; only one `Library`
/// instance exists at a time.
pub fn init() -> Library {
+ Self::try_init().expect("failed to load pdfium shared library")
+ }
+
+ /// [`Library::init`] that reports a missing or unloadable pdfium shared
+ /// library as [`PdfiumError::LibraryUnavailable`] instead of panicking.
+ /// Hosts that cannot afford a panic across an FFI boundary (a Node addon,
+ /// where an escaping panic aborts the process) should call this first;
+ /// the search path is described on `pdfium_sys::dynamic::load_default`.
+ pub fn try_init() -> Result<Library, PdfiumError> {
#[cfg(not(target_arch = "wasm32"))]
{
- pdfium_sys::dynamic::load_default().expect("failed to load pdfium shared library");
+ pdfium_sys::dynamic::load_default().map_err(|_| PdfiumError::LibraryUnavailable)?;
// Recover from poisoning: a panic mid-FFI may leave PDFium in
// an odd state, but subsequent calls should still be allowed
// (the worst case is that the next parse also fails cleanly).
@@ -76,12 +85,12 @@ impl Library {
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
INIT.call_once(|| unsafe { ffi!(FPDF_InitLibrary()) });
- Library { _guard: guard }
+ Ok(Library { _guard: guard })
}
#[cfg(target_arch = "wasm32")]
{
INIT.call_once(|| unsafe { ffi!(FPDF_InitLibrary()) });
- Library { _private: () }
+ Ok(Library { _private: () })
}
}