Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

fix(rtf): preserve surrogate pairs in fallback (#296)

Stamina9fd332f46697a
diff --git a/book_to_skill/parsers/rtf.py b/book_to_skill/parsers/rtf.py
index 9df12d8..503a442 100644
--- a/book_to_skill/parsers/rtf.py
+++ b/book_to_skill/parsers/rtf.py
@@ -22,6 +22,34 @@ def _rtf_unicode_repl(match: re.Match) -> str:
return chr(cp)
+def _decode_rtf_unicode(raw: str) -> str:
+ """Combine adjacent UTF-16 surrogate escapes before dropping lone halves."""
+ out: list[str] = []
+ cursor = 0
+ escapes = iter(_RTF_UNICODE.finditer(raw))
+ match = next(escapes, None)
+ while match is not None:
+ out.append(raw[cursor:match.start()])
+ high = int(match.group(1)) % 0x10000
+ following = next(escapes, None)
+ if (
+ 0xD800 <= high <= 0xDBFF
+ and following is not None
+ and following.start() == match.end()
+ ):
+ low = int(following.group(1)) % 0x10000
+ if 0xDC00 <= low <= 0xDFFF:
+ out.append(chr(0x10000 + (high - 0xD800) * 0x400 + low - 0xDC00))
+ cursor = following.end()
+ match = next(escapes, None)
+ continue
+ out.append(_rtf_unicode_repl(match))
+ cursor = match.end()
+ match = following
+ out.append(raw[cursor:])
+ return "".join(out)
+
+
def _rtf_ansi_encoding(raw: str) -> str:
"""Return the declared ANSI code page, defaulting to Windows-1252."""
match = _RTF_ANSI_CODEPAGE.search(raw)
@@ -141,7 +169,7 @@ def strip_rtf_fallback(raw: str) -> str:
.replace("\\}", "\x03")
)
ansi_encoding = _rtf_ansi_encoding(raw)
- raw = _RTF_UNICODE.sub(_rtf_unicode_repl, raw) # decode \uN escapes first
+ raw = _decode_rtf_unicode(raw) # decode \uN escapes first
# A hex byte immediately following \uN is its compatibility fallback and
# was consumed by _RTF_UNICODE above. Any remaining \'hh escapes are actual
# document text. Decode adjacent bytes as a run so multibyte code pages work.
diff --git a/tests/test_rtf_surrogate_pairs.py b/tests/test_rtf_surrogate_pairs.py
new file mode 100644
index 0000000..2c0f2d4
--- /dev/null
+++ b/tests/test_rtf_surrogate_pairs.py
@@ -0,0 +1,60 @@
+"""Supplementary Unicode characters survive the dependency-free RTF fallback."""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT_DIR = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT_DIR))
+
+from book_to_skill.parsers.rtf import strip_rtf_fallback
+from book_to_skill.utils import main
+
+
+ "escapes",
+ [
+ r"\u-10179?\u-8704?",
+ r"\u-10179\'3f\u-8704\'3f",
+ r"\u-10179 \u-8704 ",
+ ],
+)
+def test_adjacent_surrogate_escapes_form_one_character(escapes):
+ assert strip_rtf_fallback(r"{\rtf1 A " + escapes + " B}") == "A 😀 B"
+
+
+def test_surrogates_do_not_pair_across_text_or_with_wrong_order():
+ raw = r"{\rtf1 A \u-10179?x\u-8704? B \u-8704?\u-10179? C}"
+ assert strip_rtf_fallback(raw) == "A x B C"
+
+
+def test_multiple_pairs_and_escaped_literal_remain_distinct():
+ raw = r"{\rtf1 \u-10179?\u-8704?\u-10179?\u-8704? \\u-10179?}"
+ assert strip_rtf_fallback(raw) == r"😀😀 \u-10179?"
+
+
+def test_cli_writes_combined_character_with_fallback(tmp_path, monkeypatch):
+ source = tmp_path / "supplementary.rtf"
+ source.write_text(
+ r"{\rtf1 Chapter 1\par A \u-10179?\u-8704? B}", encoding="utf-8"
+ )
+ output_dir = tmp_path / "output"
+
+ monkeypatch.setitem(sys.modules, "striprtf", None)
+ monkeypatch.setitem(sys.modules, "striprtf.striprtf", None)
+ monkeypatch.setattr("book_to_skill.utils.prepare_dependencies", lambda *args: None)
+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_DIR", output_dir)
+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_TEXT", output_dir / "full_text.txt")
+ monkeypatch.setattr("book_to_skill.utils.OUTPUT_META", output_dir / "metadata.json")
+ monkeypatch.setattr(
+ sys, "argv", ["extract.py", str(source), "--install-missing", "no"]
+ )
+
+ main()
+
+ corpus = (output_dir / "full_text.txt").read_text(encoding="utf-8")
+ metadata = json.loads((output_dir / "metadata.json").read_text(encoding="utf-8"))
+ assert corpus.endswith("Chapter 1\n A 😀 B")
+ assert metadata["extraction_method"] == "rtf-regex"