Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

fix(extract): write metadata.json as UTF-8 (#105)

Hotragn Pettuganic18b0fb2db8b
diff --git a/book_to_skill/utils.py b/book_to_skill/utils.py
index 7e106b7..ca090a1 100644
--- a/book_to_skill/utils.py
+++ b/book_to_skill/utils.py
@@ -752,7 +752,14 @@ def main():
**consolidated_structure,
}
- OUTPUT_META.write_text(json.dumps(metadata, indent=2, ensure_ascii=False))
+ # encoding="utf-8" is required, not cosmetic: the payload is dumped with
+ # ensure_ascii=False, so any non-ASCII chapter heading, filename or path
+ # reaches the encoder verbatim. Without it, write_text() falls back to the
+ # locale encoding and raises UnicodeEncodeError on a Windows cp1252 host or
+ # under LC_ALL=C — after every source has already been extracted.
+ OUTPUT_META.write_text(
+ json.dumps(metadata, indent=2, ensure_ascii=False), encoding="utf-8"
+ )
page_line = f" Total Pages: {total_pages}"
print("\nExtraction complete:")
diff --git a/tests/test_metadata_encoding.py b/tests/test_metadata_encoding.py
new file mode 100644
index 0000000..f6dd73d
--- /dev/null
+++ b/tests/test_metadata_encoding.py
@@ -0,0 +1,86 @@
+"""metadata.json must be written as UTF-8, not in the host's locale encoding.
+
+`main()` dumps the metadata with ``ensure_ascii=False``, so non-ASCII text is
+passed through verbatim instead of being escaped to ``\\uXXXX``. Any CJK, Thai
+or Korean chapter heading — or an accented filename or path — therefore reaches
+the file encoder as-is. Without an explicit ``encoding=``, ``write_text()`` uses
+the locale encoding and raises ``UnicodeEncodeError`` on a Windows cp1252 host
+or under ``LC_ALL=C``.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+ROOT_DIR = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT_DIR))
+
+from book_to_skill.utils import main
+
+
+class TestMetadataOutputEncoding:
+ # Two "第N章" headings, so chapters_detected is a meaningful assertion too.
+ CJK_SOURCE = "第一章 緒論\n\nBody one.\n\n第二章 架構\n\nBody two.\n"
+
+ def _run_main(self, tmp_path, monkeypatch):
+ source = tmp_path / "cjk.md"
+ source.write_text(self.CJK_SOURCE, encoding="utf-8")
+
+ out_dir = tmp_path / "output"
+ out_meta = out_dir / "metadata.json"
+ monkeypatch.setenv("BOOK_SKILL_WORKDIR", str(out_dir))
+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_DIR", out_dir)
+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_TEXT", out_dir / "full_text.txt")
+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_META", out_meta)
+ monkeypatch.setattr("book_to_skill.utils.prepare_dependencies", lambda *a: None)
+ monkeypatch.setattr(
+ "sys.argv", ["extract.py", str(source), "--install-missing", "no"]
+ )
+
+ main()
+ return out_meta
+
+ def test_metadata_write_declares_utf8(self, tmp_path, monkeypatch):
+ """The metadata write must pass encoding="utf-8" explicitly.
+
+ This is the platform-independent regression guard. On a UTF-8 host the
+ locale default happens to produce the right bytes, so a round-trip
+ assertion alone would pass on unfixed code in CI; only the declared
+ encoding proves the fix. Mirrors TestPdftotextEncoding, which asserts
+ the same way on the pdftotext subprocess call.
+ """
+ captured = {}
+ original_write_text = Path.write_text
+
+ def recording_write_text(self, data, encoding=None, **kwargs):
+ if self.name == "metadata.json":
+ captured["encoding"] = encoding
+ return original_write_text(self, data, encoding=encoding, **kwargs)
+
+ monkeypatch.setattr(Path, "write_text", recording_write_text)
+ self._run_main(tmp_path, monkeypatch)
+
+ assert captured.get("encoding") == "utf-8", (
+ "metadata.json was written with the locale encoding; it must declare "
+ 'encoding="utf-8" because the JSON is dumped with ensure_ascii=False'
+ )
+
+ def test_non_ascii_headings_round_trip_as_utf8(self, tmp_path, monkeypatch):
+ """The CJK headings survive the write and decode back as UTF-8."""
+ out_meta = self._run_main(tmp_path, monkeypatch)
+
+ meta = json.loads(out_meta.read_bytes().decode("utf-8"))
+ assert meta["chapters_detected"] == 2
+ assert "第一章 緒論" in meta["chapter_headings_sample"]
+ assert "第二章 架構" in meta["chapter_headings_sample"]
+
+ def test_metadata_is_valid_utf8_on_disk(self, tmp_path, monkeypatch):
+ """The bytes on disk are UTF-8, whatever the host locale encoding is."""
+ out_meta = self._run_main(tmp_path, monkeypatch)
+
+ raw = out_meta.read_bytes()
+ # Would raise UnicodeDecodeError if the file had been written as cp1252
+ # or another single-byte codec that happened not to fail on write.
+ decoded = raw.decode("utf-8")
+ assert "第一章" in decoded
+ assert "第一章 緒論".encode("utf-8") in raw