Skip to content

A fast, helpful, and open-source document parser

fix: only map 0x02 to a dash when pdfium flags it as a hyphen

Sacha Bronc8a88b098ca3
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs
index 00cc7ca..34e840c 100644
--- a/crates/liteparse/src/extract.rs
+++ b/crates/liteparse/src/extract.rs
@@ -2751,6 +2751,7 @@ impl ObjMetaCache {
/// char-info records (`FPDF_CHARINFO_LP::char_type`).
const CHAR_TYPE_GENERATED: i32 = 1;
const CHAR_TYPE_NOT_UNICODE: i32 = 2;
+const CHAR_TYPE_HYPHEN: i32 = 3;
/// Records per `FPDFText_GetCharInfoBatch` call (80 bytes each → ~1.3 MB).
const CHAR_INFO_CHUNK: usize = 16 * 1024;
@@ -2831,6 +2832,13 @@ impl CharView<'_, '_> {
}
}
+ pub(crate) fn is_hyphen(&self) -> bool {
+ match &self.rec {
+ Some(rec) => rec.char_type == CHAR_TYPE_HYPHEN,
+ None => self.ch.is_hyphen(),
+ }
+ }
+
pub(crate) fn text_render_mode(&self) -> Option<i32> {
match &self.rec {
Some(rec) => (rec.text_render_mode >= 0).then_some(rec.text_render_mode),
diff --git a/crates/liteparse/src/raw_text.rs b/crates/liteparse/src/raw_text.rs
index 54ee5b4..d6f45d3 100644
--- a/crates/liteparse/src/raw_text.rs
+++ b/crates/liteparse/src/raw_text.rs
@@ -216,7 +216,7 @@ pub fn extract_raw_text_items(
items
}
-/// The code pdfium gives a hyphen at the end of a line (`FPDFText_IsHyphen`).
+/// The code pdfium gives a hyphen it found at the end of a line (`FPDFText_IsHyphen`).
const LINE_END_HYPHEN: u32 = 0x02;
/// The per-glyph reads. A glyph whose loose box cannot be read falls back to its
@@ -234,8 +234,9 @@ fn load_glyph(
true => 0,
// pdfium reports a hyphen drawn at the end of a line as U+0002. Read as
// a control code it would also flag the whole word as a buggy font and
- // re-decode it through the glyph map, so restore the drawn '-'.
- false if !generated && cv.unicode() == LINE_END_HYPHEN => u32::from('-'),
+ // re-decode it through the glyph map, so restore the drawn '-'. A 0x02
+ // pdfium does not flag as a hyphen is a real code and stays as it is.
+ false if cv.unicode() == LINE_END_HYPHEN && cv.is_hyphen() => u32::from('-'),
false => cv.unicode(),
};
let strict = cv.strict_char_box().unwrap_or_default();
diff --git a/crates/liteparse/tests/line_end_hyphen.rs b/crates/liteparse/tests/line_end_hyphen.rs
index db3ef22..bdbb205 100644
--- a/crates/liteparse/tests/line_end_hyphen.rs
+++ b/crates/liteparse/tests/line_end_hyphen.rs
@@ -67,3 +67,51 @@ fn line_end_hyphen_keeps_its_dash() {
"hyphenated word lost its dash: {joined:?}"
);
}
+
+/// A font whose ToUnicode maps `X` to U+0002, written mid-line.
+fn stx_font_pdf() -> Vec<u8> {
+ let cmap = "/CIDInit /ProcSet findresource begin 12 dict begin begincmap /CMapName /T def 1 begincodespacerange <00> <FF> endcodespacerange 1 beginbfchar <58> <0002> endbfchar endcmap CMapName currentdict /CMap defineresource pop end end";
+ let objects = [
+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),
+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>".to_string(),
+ "<< /Type /Font /Subtype /Type1 /BaseFont /Courier /ToUnicode 6 0 R >>".to_string(),
+ stream("", "BT /F1 10 Tf 50 700 Td (abXcd efgh ijkl) Tj ET"),
+ stream("", cmap),
+ ];
+ let mut data = b"%PDF-1.7\n".to_vec();
+ let mut offsets = Vec::new();
+ for (i, object) in objects.iter().enumerate() {
+ offsets.push(data.len());
+ data.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes());
+ }
+ let xref = data.len();
+ data.extend_from_slice(
+ format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes(),
+ );
+ for offset in offsets {
+ data.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
+ }
+ data.extend_from_slice(
+ format!(
+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",
+ objects.len() + 1
+ )
+ .as_bytes(),
+ );
+ data
+}
+
+#[test]
+fn mapped_stx_that_is_not_a_hyphen_is_left_alone() {
+ let items = raw_items(&stx_font_pdf());
+ let joined = items.join("|");
+ assert!(
+ joined.contains("ab\u{2}cd"),
+ "the mapped code changed: {joined:?}"
+ );
+ assert!(
+ !joined.contains("ab-cd"),
+ "a non-hyphen 0x02 became a dash: {joined:?}"
+ );
+}
diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rs
index 7b08234..798b8ff 100644
--- a/crates/pdfium-sys/src/dynamic.rs
+++ b/crates/pdfium-sys/src/dynamic.rs
@@ -287,6 +287,8 @@ pub struct PdfiumBindings {
unsafe extern "C" fn(FPDF_TEXTPAGE, std::os::raw::c_int, *mut FS_MATRIX) -> FPDF_BOOL,
pub FPDFText_IsGenerated:
unsafe extern "C" fn(FPDF_TEXTPAGE, std::os::raw::c_int) -> std::os::raw::c_int,
+ pub FPDFText_IsHyphen:
+ unsafe extern "C" fn(FPDF_TEXTPAGE, std::os::raw::c_int) -> std::os::raw::c_int,
pub FPDFText_HasUnicodeMapError:
unsafe extern "C" fn(FPDF_TEXTPAGE, std::os::raw::c_int) -> std::os::raw::c_int,
pub FPDFText_GetTextObject:
@@ -689,6 +691,7 @@ impl PdfiumBindings {
FPDFText_GetCharOrigin: load_fn!(lib, "FPDFText_GetCharOrigin"),
FPDFText_GetMatrix: load_fn!(lib, "FPDFText_GetMatrix"),
FPDFText_IsGenerated: load_fn!(lib, "FPDFText_IsGenerated"),
+ FPDFText_IsHyphen: load_fn!(lib, "FPDFText_IsHyphen"),
FPDFText_HasUnicodeMapError: load_fn!(lib, "FPDFText_HasUnicodeMapError"),
FPDFText_GetTextObject: load_fn!(lib, "FPDFText_GetTextObject"),
FPDFText_GetStrokeColor: load_fn!(lib, "FPDFText_GetStrokeColor"),
diff --git a/crates/pdfium/src/text_page.rs b/crates/pdfium/src/text_page.rs
index 41d5556..a39251c 100644
--- a/crates/pdfium/src/text_page.rs
+++ b/crates/pdfium/src/text_page.rs
@@ -468,6 +468,11 @@ impl TextChar<'_> {
unsafe { ffi!(FPDFText_IsGenerated(self.text_page.handle, self.index)) == 1 }
}
+ /// A hyphen pdfium found at the end of a line; its unicode reads as 0x02.
+ pub fn is_hyphen(&self) -> bool {
+ unsafe { ffi!(FPDFText_IsHyphen(self.text_page.handle, self.index)) == 1 }
+ }
+
pub fn has_unicode_map_error(&self) -> bool {
unsafe {
ffi!(FPDFText_HasUnicodeMapError(