Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

fix(pdf): decode pdftotext output as UTF-8 (#68)

Shubham Srivastavad8d9f2cf309d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 003c658..31847cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`PyPDF2` is end-of-life and no longer receives security fixes (#54).
### Fixed
+- PDF text extracted via `pdftotext` is now decoded as UTF-8 rather than the
+ process locale encoding, so accented characters and punctuation are no longer
+ mojibake on non-UTF-8 locales (e.g. Windows).
- Text files (`.txt`, `.md`, `.rst`, `.adoc`, `.html`, `.rtf`) saved as UTF-16 or
UTF-32 (e.g. Windows Notepad "Unicode" or PowerShell output) are now decoded by
their byte-order mark instead of being read as `cp1252`/`latin-1` mojibake.
diff --git a/book_to_skill/parsers/pdf.py b/book_to_skill/parsers/pdf.py
index b9432fe..8b9d3c1 100644
--- a/book_to_skill/parsers/pdf.py
+++ b/book_to_skill/parsers/pdf.py
@@ -13,7 +13,8 @@ def extract_with_pdftotext(pdf_path: str) -> str | None:
pdf_path = os.path.abspath(pdf_path)
result = subprocess.run(
["pdftotext", "-layout", pdf_path, "-"],
- capture_output=True, text=True, timeout=120
+ capture_output=True, text=True, timeout=120,
+ encoding="utf-8", errors="replace",
)
if result.returncode == 0 and result.stdout.strip():
return result.stdout
diff --git a/tests/test_book_to_skill.py b/tests/test_book_to_skill.py
index f687997..c3e7ea1 100644
--- a/tests/test_book_to_skill.py
+++ b/tests/test_book_to_skill.py
@@ -34,6 +34,7 @@ from book_to_skill.utils import (
main,
)
from book_to_skill.config import SUPPORTED_EXTENSIONS
+from book_to_skill.parsers import pdf as pdf_parser
from book_to_skill.parsers.text import read_text_file
from book_to_skill.parsers.docx import extract_docx_with_zipfile
from book_to_skill.parsers.rtf import strip_rtf_fallback
@@ -1284,3 +1285,26 @@ class TestTextEncodingDetection:
def test_empty_file_returns_empty_string(self, tmp_path):
# An empty file decodes to "" (not None, which is reserved for read errors).
assert read_text_file(self._write(tmp_path, b"")) == ""
+
+
+class TestPdftotextEncoding:
+ """pdftotext output (UTF-8) is decoded as UTF-8, not the locale encoding."""
+
+ def test_pdftotext_decodes_as_utf8(self, monkeypatch):
+ captured = {}
+
+ class _Result:
+ returncode = 0
+ stdout = "Café — naïve"
+
+ monkeypatch.setattr(pdf_parser.shutil, "which", lambda name: "/usr/bin/pdftotext")
+
+ def fake_run(cmd, **kwargs):
+ captured.update(kwargs)
+ return _Result()
+
+ monkeypatch.setattr(pdf_parser.subprocess, "run", fake_run)
+
+ assert pdf_parser.extract_with_pdftotext("x.pdf") == "Café — naïve"
+ assert captured.get("encoding") == "utf-8"
+ assert captured.get("errors") == "replace"