Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
fix(rtf): preserve surrogate pairs in fallback (#296)
Stamina9fd332f46697a
diff --git a/book_to_skill/parsers/rtf.py b/book_to_skill/parsers/rtf.pyindex 9df12d8..503a442 100644--- a/book_to_skill/parsers/rtf.py+++ b/book_to_skill/parsers/rtf.py@@ -22,6 +22,34 @@ def _rtf_unicode_repl(match: re.Match) -> str:return chr(cp)+def _decode_rtf_unicode(raw: str) -> str:+ """Combine adjacent UTF-16 surrogate escapes before dropping lone halves."""+ out: list[str] = []+ cursor = 0+ escapes = iter(_RTF_UNICODE.finditer(raw))+ match = next(escapes, None)+ while match is not None:+ out.append(raw[cursor:match.start()])+ high = int(match.group(1)) % 0x10000+ following = next(escapes, None)+ if (+ 0xD800 <= high <= 0xDBFF+ and following is not None+ and following.start() == match.end()+ ):+ low = int(following.group(1)) % 0x10000+ if 0xDC00 <= low <= 0xDFFF:+ out.append(chr(0x10000 + (high - 0xD800) * 0x400 + low - 0xDC00))+ cursor = following.end()+ match = next(escapes, None)+ continue+ out.append(_rtf_unicode_repl(match))+ cursor = match.end()+ match = following+ out.append(raw[cursor:])+ return "".join(out)++def _rtf_ansi_encoding(raw: str) -> str:"""Return the declared ANSI code page, defaulting to Windows-1252."""match = _RTF_ANSI_CODEPAGE.search(raw)@@ -141,7 +169,7 @@ def strip_rtf_fallback(raw: str) -> str:.replace("\\}", "\x03"))ansi_encoding = _rtf_ansi_encoding(raw)- raw = _RTF_UNICODE.sub(_rtf_unicode_repl, raw) # decode \uN escapes first+ raw = _decode_rtf_unicode(raw) # decode \uN escapes first# A hex byte immediately following \uN is its compatibility fallback and# was consumed by _RTF_UNICODE above. Any remaining \'hh escapes are actual# document text. Decode adjacent bytes as a run so multibyte code pages work.diff --git a/tests/test_rtf_surrogate_pairs.py b/tests/test_rtf_surrogate_pairs.pynew file mode 100644index 0000000..2c0f2d4--- /dev/null+++ b/tests/test_rtf_surrogate_pairs.py@@ -0,0 +1,60 @@+"""Supplementary Unicode characters survive the dependency-free RTF fallback."""++import json+import sys+from pathlib import Path++import pytest++ROOT_DIR = Path(__file__).resolve().parent.parent+sys.path.insert(0, str(ROOT_DIR))++from book_to_skill.parsers.rtf import strip_rtf_fallback+from book_to_skill.utils import main+++ "escapes",+ [+ r"\u-10179?\u-8704?",+ r"\u-10179\'3f\u-8704\'3f",+ r"\u-10179 \u-8704 ",+ ],+)+def test_adjacent_surrogate_escapes_form_one_character(escapes):+ assert strip_rtf_fallback(r"{\rtf1 A " + escapes + " B}") == "A 😀 B"+++def test_surrogates_do_not_pair_across_text_or_with_wrong_order():+ raw = r"{\rtf1 A \u-10179?x\u-8704? B \u-8704?\u-10179? C}"+ assert strip_rtf_fallback(raw) == "A x B C"+++def test_multiple_pairs_and_escaped_literal_remain_distinct():+ raw = r"{\rtf1 \u-10179?\u-8704?\u-10179?\u-8704? \\u-10179?}"+ assert strip_rtf_fallback(raw) == r"😀😀 \u-10179?"+++def test_cli_writes_combined_character_with_fallback(tmp_path, monkeypatch):+ source = tmp_path / "supplementary.rtf"+ source.write_text(+ r"{\rtf1 Chapter 1\par A \u-10179?\u-8704? B}", encoding="utf-8"+ )+ output_dir = tmp_path / "output"++ monkeypatch.setitem(sys.modules, "striprtf", None)+ monkeypatch.setitem(sys.modules, "striprtf.striprtf", None)+ monkeypatch.setattr("book_to_skill.utils.prepare_dependencies", lambda *args: None)+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_DIR", output_dir)+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_TEXT", output_dir / "full_text.txt")+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_META", output_dir / "metadata.json")+ monkeypatch.setattr(+ sys, "argv", ["extract.py", str(source), "--install-missing", "no"]+ )++ main()++ corpus = (output_dir / "full_text.txt").read_text(encoding="utf-8")+ metadata = json.loads((output_dir / "metadata.json").read_text(encoding="utf-8"))+ assert corpus.endswith("Chapter 1\n A 😀 B")+ assert metadata["extraction_method"] == "rtf-regex"