A fast, helpful, and open-source document parser
fix: only map 0x02 to a dash when pdfium flags it as a hyphen
Sacha Bronc8a88b098ca3
diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rsindex 00cc7ca..34e840c 100644--- a/crates/liteparse/src/extract.rs+++ b/crates/liteparse/src/extract.rs@@ -2751,6 +2751,7 @@ impl ObjMetaCache {/// char-info records (`FPDF_CHARINFO_LP::char_type`).const CHAR_TYPE_GENERATED: i32 = 1;const CHAR_TYPE_NOT_UNICODE: i32 = 2;+const CHAR_TYPE_HYPHEN: i32 = 3;/// Records per `FPDFText_GetCharInfoBatch` call (80 bytes each → ~1.3 MB).const CHAR_INFO_CHUNK: usize = 16 * 1024;@@ -2831,6 +2832,13 @@ impl CharView<'_, '_> {}}+ pub(crate) fn is_hyphen(&self) -> bool {+ match &self.rec {+ Some(rec) => rec.char_type == CHAR_TYPE_HYPHEN,+ None => self.ch.is_hyphen(),+ }+ }+pub(crate) fn text_render_mode(&self) -> Option<i32> {match &self.rec {Some(rec) => (rec.text_render_mode >= 0).then_some(rec.text_render_mode),diff --git a/crates/liteparse/src/raw_text.rs b/crates/liteparse/src/raw_text.rsindex 54ee5b4..d6f45d3 100644--- a/crates/liteparse/src/raw_text.rs+++ b/crates/liteparse/src/raw_text.rs@@ -216,7 +216,7 @@ pub fn extract_raw_text_items(items}-/// The code pdfium gives a hyphen at the end of a line (`FPDFText_IsHyphen`).+/// The code pdfium gives a hyphen it found at the end of a line (`FPDFText_IsHyphen`).const LINE_END_HYPHEN: u32 = 0x02;/// The per-glyph reads. A glyph whose loose box cannot be read falls back to its@@ -234,8 +234,9 @@ fn load_glyph(true => 0,// pdfium reports a hyphen drawn at the end of a line as U+0002. Read as// a control code it would also flag the whole word as a buggy font and- // re-decode it through the glyph map, so restore the drawn '-'.- false if !generated && cv.unicode() == LINE_END_HYPHEN => u32::from('-'),+ // re-decode it through the glyph map, so restore the drawn '-'. A 0x02+ // pdfium does not flag as a hyphen is a real code and stays as it is.+ false if cv.unicode() == LINE_END_HYPHEN && cv.is_hyphen() => u32::from('-'),false => cv.unicode(),};let strict = cv.strict_char_box().unwrap_or_default();diff --git a/crates/liteparse/tests/line_end_hyphen.rs b/crates/liteparse/tests/line_end_hyphen.rsindex db3ef22..bdbb205 100644--- a/crates/liteparse/tests/line_end_hyphen.rs+++ b/crates/liteparse/tests/line_end_hyphen.rs@@ -67,3 +67,51 @@ fn line_end_hyphen_keeps_its_dash() {"hyphenated word lost its dash: {joined:?}");}++/// A font whose ToUnicode maps `X` to U+0002, written mid-line.+fn stx_font_pdf() -> Vec<u8> {+ let cmap = "/CIDInit /ProcSet findresource begin 12 dict begin begincmap /CMapName /T def 1 begincodespacerange <00> <FF> endcodespacerange 1 beginbfchar <58> <0002> endbfchar endcmap CMapName currentdict /CMap defineresource pop end end";+ let objects = [+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>".to_string(),+ "<< /Type /Font /Subtype /Type1 /BaseFont /Courier /ToUnicode 6 0 R >>".to_string(),+ stream("", "BT /F1 10 Tf 50 700 Td (abXcd efgh ijkl) Tj ET"),+ stream("", cmap),+ ];+ let mut data = b"%PDF-1.7\n".to_vec();+ let mut offsets = Vec::new();+ for (i, object) in objects.iter().enumerate() {+ offsets.push(data.len());+ data.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes());+ }+ let xref = data.len();+ data.extend_from_slice(+ format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes(),+ );+ for offset in offsets {+ data.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());+ }+ data.extend_from_slice(+ format!(+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",+ objects.len() + 1+ )+ .as_bytes(),+ );+ data+}++#[test]+fn mapped_stx_that_is_not_a_hyphen_is_left_alone() {+ let items = raw_items(&stx_font_pdf());+ let joined = items.join("|");+ assert!(+ joined.contains("ab\u{2}cd"),+ "the mapped code changed: {joined:?}"+ );+ assert!(+ !joined.contains("ab-cd"),+ "a non-hyphen 0x02 became a dash: {joined:?}"+ );+}diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rsindex 7b08234..798b8ff 100644--- a/crates/pdfium-sys/src/dynamic.rs+++ b/crates/pdfium-sys/src/dynamic.rs@@ -287,6 +287,8 @@ pub struct PdfiumBindings {unsafe extern "C" fn(FPDF_TEXTPAGE, std::os::raw::c_int, *mut FS_MATRIX) -> FPDF_BOOL,pub FPDFText_IsGenerated:unsafe extern "C" fn(FPDF_TEXTPAGE, std::os::raw::c_int) -> std::os::raw::c_int,+ pub FPDFText_IsHyphen:+ unsafe extern "C" fn(FPDF_TEXTPAGE, std::os::raw::c_int) -> std::os::raw::c_int,pub FPDFText_HasUnicodeMapError:unsafe extern "C" fn(FPDF_TEXTPAGE, std::os::raw::c_int) -> std::os::raw::c_int,pub FPDFText_GetTextObject:@@ -689,6 +691,7 @@ impl PdfiumBindings {FPDFText_GetCharOrigin: load_fn!(lib, "FPDFText_GetCharOrigin"),FPDFText_GetMatrix: load_fn!(lib, "FPDFText_GetMatrix"),FPDFText_IsGenerated: load_fn!(lib, "FPDFText_IsGenerated"),+ FPDFText_IsHyphen: load_fn!(lib, "FPDFText_IsHyphen"),FPDFText_HasUnicodeMapError: load_fn!(lib, "FPDFText_HasUnicodeMapError"),FPDFText_GetTextObject: load_fn!(lib, "FPDFText_GetTextObject"),FPDFText_GetStrokeColor: load_fn!(lib, "FPDFText_GetStrokeColor"),diff --git a/crates/pdfium/src/text_page.rs b/crates/pdfium/src/text_page.rsindex 41d5556..a39251c 100644--- a/crates/pdfium/src/text_page.rs+++ b/crates/pdfium/src/text_page.rs@@ -468,6 +468,11 @@ impl TextChar<'_> {unsafe { ffi!(FPDFText_IsGenerated(self.text_page.handle, self.index)) == 1 }}+ /// A hyphen pdfium found at the end of a line; its unicode reads as 0x02.+ pub fn is_hyphen(&self) -> bool {+ unsafe { ffi!(FPDFText_IsHyphen(self.text_page.handle, self.index)) == 1 }+ }+pub fn has_unicode_map_error(&self) -> bool {unsafe {ffi!(FPDFText_HasUnicodeMapError(