Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
fix(pdf): decode pdftotext output as UTF-8 (#68)
Shubham Srivastavad8d9f2cf309d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 003c658..31847cc 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -35,6 +35,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0`PyPDF2` is end-of-life and no longer receives security fixes (#54).### Fixed+- PDF text extracted via `pdftotext` is now decoded as UTF-8 rather than the+ process locale encoding, so accented characters and punctuation are no longer+ mojibake on non-UTF-8 locales (e.g. Windows).- Text files (`.txt`, `.md`, `.rst`, `.adoc`, `.html`, `.rtf`) saved as UTF-16 orUTF-32 (e.g. Windows Notepad "Unicode" or PowerShell output) are now decoded bytheir byte-order mark instead of being read as `cp1252`/`latin-1` mojibake.diff --git a/book_to_skill/parsers/pdf.py b/book_to_skill/parsers/pdf.pyindex b9432fe..8b9d3c1 100644--- a/book_to_skill/parsers/pdf.py+++ b/book_to_skill/parsers/pdf.py@@ -13,7 +13,8 @@ def extract_with_pdftotext(pdf_path: str) -> str | None:pdf_path = os.path.abspath(pdf_path)result = subprocess.run(["pdftotext", "-layout", pdf_path, "-"],- capture_output=True, text=True, timeout=120+ capture_output=True, text=True, timeout=120,+ encoding="utf-8", errors="replace",)if result.returncode == 0 and result.stdout.strip():return result.stdoutdiff --git a/tests/test_book_to_skill.py b/tests/test_book_to_skill.pyindex f687997..c3e7ea1 100644--- a/tests/test_book_to_skill.py+++ b/tests/test_book_to_skill.py@@ -34,6 +34,7 @@ from book_to_skill.utils import (main,)from book_to_skill.config import SUPPORTED_EXTENSIONS+from book_to_skill.parsers import pdf as pdf_parserfrom book_to_skill.parsers.text import read_text_filefrom book_to_skill.parsers.docx import extract_docx_with_zipfilefrom book_to_skill.parsers.rtf import strip_rtf_fallback@@ -1284,3 +1285,26 @@ class TestTextEncodingDetection:def test_empty_file_returns_empty_string(self, tmp_path):# An empty file decodes to "" (not None, which is reserved for read errors).assert read_text_file(self._write(tmp_path, b"")) == ""+++class TestPdftotextEncoding:+ """pdftotext output (UTF-8) is decoded as UTF-8, not the locale encoding."""++ def test_pdftotext_decodes_as_utf8(self, monkeypatch):+ captured = {}++ class _Result:+ returncode = 0+ stdout = "Café — naïve"++ monkeypatch.setattr(pdf_parser.shutil, "which", lambda name: "/usr/bin/pdftotext")++ def fake_run(cmd, **kwargs):+ captured.update(kwargs)+ return _Result()++ monkeypatch.setattr(pdf_parser.subprocess, "run", fake_run)++ assert pdf_parser.extract_with_pdftotext("x.pdf") == "Café — naïve"+ assert captured.get("encoding") == "utf-8"+ assert captured.get("errors") == "replace"