A fast, helpful, and open-source document parser
fix: keep the dash of a hyphen at the end of a line
Sacha Bron48bf74ea27d7
diff --git a/crates/liteparse/src/raw_text.rs b/crates/liteparse/src/raw_text.rsindex 29bce08..54ee5b4 100644--- a/crates/liteparse/src/raw_text.rs+++ b/crates/liteparse/src/raw_text.rs@@ -216,6 +216,9 @@ pub fn extract_raw_text_items(items}+/// The code pdfium gives a hyphen at the end of a line (`FPDFText_IsHyphen`).+const LINE_END_HYPHEN: u32 = 0x02;+/// The per-glyph reads. A glyph whose loose box cannot be read falls back to its/// strict box.fn load_glyph(@@ -227,10 +230,13 @@ fn load_glyph() -> Glyph {let generated = cv.is_generated();let char_code = if generated { 0 } else { cv.char_code() };- let unicode = if cv.has_unicode_map_error() {- 0- } else {- cv.unicode()+ let unicode = match cv.has_unicode_map_error() {+ true => 0,+ // pdfium reports a hyphen drawn at the end of a line as U+0002. Read as+ // a control code it would also flag the whole word as a buggy font and+ // re-decode it through the glyph map, so restore the drawn '-'.+ false if !generated && cv.unicode() == LINE_END_HYPHEN => u32::from('-'),+ false => cv.unicode(),};let strict = cv.strict_char_box().unwrap_or_default();let loose = cv.loose_char_box().unwrap_or(strict);diff --git a/crates/liteparse/tests/line_end_hyphen.rs b/crates/liteparse/tests/line_end_hyphen.rsnew file mode 100644index 0000000..db3ef22--- /dev/null+++ b/crates/liteparse/tests/line_end_hyphen.rs@@ -0,0 +1,69 @@+//! pdfium reports a hyphen at the end of a line as U+0002; the extracted text keeps the drawn '-'.+use liteparse::extract_raw_text_items;+use pdfium::Library;++fn pdf(content: &str, forms: &[&str], rotation: i32) -> Vec<u8> {+ let mut objects = vec![+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),+ format!(+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Rotate {rotation} /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>"+ ),+ "<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>".to_string(),+ stream("", content),+ ];+ objects.extend(forms.iter().map(|s| s.to_string()));+ let mut data = b"%PDF-1.7\n".to_vec();+ let mut offsets = Vec::new();+ for (i, object) in objects.iter().enumerate() {+ offsets.push(data.len());+ data.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes());+ }+ let xref = data.len();+ data.extend_from_slice(+ format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes(),+ );+ for offset in offsets {+ data.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());+ }+ data.extend_from_slice(+ format!(+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",+ objects.len() + 1+ )+ .as_bytes(),+ );+ data+}++fn stream(dict: &str, content: &str) -> String {+ format!(+ "<< {dict} /Length {} >>\nstream\n{content}\nendstream",+ content.len()+ )+}++fn raw_items(bytes: &[u8]) -> Vec<String> {+ let lib = Library::init();+ let doc = lib.load_document_from_bytes(bytes, None).unwrap();+ let page = doc.page(0).unwrap();+ let text = page.text().unwrap();+ extract_raw_text_items(&page, &text, &page.view_box().unwrap(), None)+ .into_iter()+ .map(|item| item.text)+ .collect()+}++#[test]+fn line_end_hyphen_keeps_its_dash() {+ let content = "BT /F1 10 Tf 50 700 Td (Used improperly, technolo-) Tj 0 -12 Td (gies can and do result) Tj ET";+ let items = raw_items(&pdf(content, &[], 0));+ let joined = items.join("|");+ assert!(!joined.contains('\u{2}'), "control code leaked: {joined:?}");+ assert!(+ items+ .iter()+ .any(|item| item.trim_end().ends_with("technolo-")),+ "hyphenated word lost its dash: {joined:?}"+ );+}