Skip to content

A fast, helpful, and open-source document parser

fix: keep the dash of a hyphen at the end of a line

Sacha Bron48bf74ea27d7
diff --git a/crates/liteparse/src/raw_text.rs b/crates/liteparse/src/raw_text.rs
index 29bce08..54ee5b4 100644
--- a/crates/liteparse/src/raw_text.rs
+++ b/crates/liteparse/src/raw_text.rs
@@ -216,6 +216,9 @@ pub fn extract_raw_text_items(
items
}
+/// The code pdfium gives a hyphen at the end of a line (`FPDFText_IsHyphen`).
+const LINE_END_HYPHEN: u32 = 0x02;
+
/// The per-glyph reads. A glyph whose loose box cannot be read falls back to its
/// strict box.
fn load_glyph(
@@ -227,10 +230,13 @@ fn load_glyph(
) -> Glyph {
let generated = cv.is_generated();
let char_code = if generated { 0 } else { cv.char_code() };
- let unicode = if cv.has_unicode_map_error() {
- 0
- } else {
- cv.unicode()
+ let unicode = match cv.has_unicode_map_error() {
+ true => 0,
+ // pdfium reports a hyphen drawn at the end of a line as U+0002. Read as
+ // a control code it would also flag the whole word as a buggy font and
+ // re-decode it through the glyph map, so restore the drawn '-'.
+ false if !generated && cv.unicode() == LINE_END_HYPHEN => u32::from('-'),
+ false => cv.unicode(),
};
let strict = cv.strict_char_box().unwrap_or_default();
let loose = cv.loose_char_box().unwrap_or(strict);
diff --git a/crates/liteparse/tests/line_end_hyphen.rs b/crates/liteparse/tests/line_end_hyphen.rs
new file mode 100644
index 0000000..db3ef22
--- /dev/null
+++ b/crates/liteparse/tests/line_end_hyphen.rs
@@ -0,0 +1,69 @@
+//! pdfium reports a hyphen at the end of a line as U+0002; the extracted text keeps the drawn '-'.
+use liteparse::extract_raw_text_items;
+use pdfium::Library;
+
+fn pdf(content: &str, forms: &[&str], rotation: i32) -> Vec<u8> {
+ let mut objects = vec![
+ "<< /Type /Catalog /Pages 2 0 R >>".to_string(),
+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),
+ format!(
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Rotate {rotation} /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>"
+ ),
+ "<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>".to_string(),
+ stream("", content),
+ ];
+ objects.extend(forms.iter().map(|s| s.to_string()));
+ let mut data = b"%PDF-1.7\n".to_vec();
+ let mut offsets = Vec::new();
+ for (i, object) in objects.iter().enumerate() {
+ offsets.push(data.len());
+ data.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes());
+ }
+ let xref = data.len();
+ data.extend_from_slice(
+ format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes(),
+ );
+ for offset in offsets {
+ data.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
+ }
+ data.extend_from_slice(
+ format!(
+ "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",
+ objects.len() + 1
+ )
+ .as_bytes(),
+ );
+ data
+}
+
+fn stream(dict: &str, content: &str) -> String {
+ format!(
+ "<< {dict} /Length {} >>\nstream\n{content}\nendstream",
+ content.len()
+ )
+}
+
+fn raw_items(bytes: &[u8]) -> Vec<String> {
+ let lib = Library::init();
+ let doc = lib.load_document_from_bytes(bytes, None).unwrap();
+ let page = doc.page(0).unwrap();
+ let text = page.text().unwrap();
+ extract_raw_text_items(&page, &text, &page.view_box().unwrap(), None)
+ .into_iter()
+ .map(|item| item.text)
+ .collect()
+}
+
+#[test]
+fn line_end_hyphen_keeps_its_dash() {
+ let content = "BT /F1 10 Tf 50 700 Td (Used improperly, technolo-) Tj 0 -12 Td (gies can and do result) Tj ET";
+ let items = raw_items(&pdf(content, &[], 0));
+ let joined = items.join("|");
+ assert!(!joined.contains('\u{2}'), "control code leaked: {joined:?}");
+ assert!(
+ items
+ .iter()
+ .any(|item| item.trim_end().ends_with("technolo-")),
+ "hyphenated word lost its dash: {joined:?}"
+ );
+}