Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
fix(rtf): decode unicode escapes in the dependency-free fallback (#52)
Shubham Srivastava3665d9fe5259
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 915cb03..7238e37 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed++- The dependency-free RTF fallback (used when `striprtf` is not installed) now+ decodes `\uN` unicode escapes — smart quotes, dashes, accented letters — instead+ of dropping them and leaving only the ASCII fallback character.+## [1.2.0] — 2026-06-17### Addeddiff --git a/book_to_skill/parsers/rtf.py b/book_to_skill/parsers/rtf.pyindex e765559..e7f6660 100644--- a/book_to_skill/parsers/rtf.py+++ b/book_to_skill/parsers/rtf.py@@ -5,7 +5,22 @@ from book_to_skill.parsers.text import read_text_filefrom book_to_skill.exceptions import ExtractionError+# RTF unicode escape: \uN (signed decimal) followed by its fallback char(s).+# Decode the code point and drop the standard single fallback — a \'XX hex byte+# or a literal "?". Assumes the default \uc1 (one fallback char); \ucN directives+# and multi-char/group fallbacks are not parsed (best-effort fallback only).+_RTF_UNICODE = re.compile(r"\\u(-?\d+)[ ]?(?:\\'[0-9a-fA-F]{2}|\?)?")+++def _rtf_unicode_repl(match: re.Match) -> str:+ cp = int(match.group(1)) % 0x10000 # RTF uses signed 16-bit; wrap negatives+ if cp == 0 or 0xD800 <= cp <= 0xDFFF: # NUL and lone surrogates: unwanted in text+ return ""+ return chr(cp)++def strip_rtf_fallback(raw: str) -> str:+ raw = _RTF_UNICODE.sub(_rtf_unicode_repl, raw) # decode \uN escapes firstraw = re.sub(r"\\'[0-9a-fA-F]{2}", " ", raw)raw = re.sub(r"\\par[d]?", "\n", raw)raw = re.sub(r"\\tab", "\t", raw)diff --git a/tests/test_book_to_skill.py b/tests/test_book_to_skill.pyindex f8e6e5b..4303b69 100644--- a/tests/test_book_to_skill.py+++ b/tests/test_book_to_skill.py@@ -34,6 +34,7 @@ from book_to_skill.utils import (main,)from book_to_skill.config import SUPPORTED_EXTENSIONS+from book_to_skill.parsers.rtf import strip_rtf_fallback# ═══════════════════════════════════════════════════════════════════════════@@ -989,4 +990,46 @@ class TestParserExceptionLogging:assert result is Nonecaptured = capsys.readouterr()assert "[warn]" in captured.err- assert "failed:" in captured.err\ No newline at end of file+ assert "failed:" in captured.err+++class TestRtfUnicodeFallback:+ """The dependency-free RTF fallback decodes RTF \\uN unicode escapes."""++ _BS = chr(92) # a single backslash, never written as a literal \-escape++ def _esc(self, codepoint, fallback="?"):+ # Build the RTF escape: backslash + "u" + number + one fallback char.+ return self._BS + "u" + str(codepoint) + fallback++ def test_rtf_unicode_right_single_quote(self):+ assert strip_rtf_fallback("It" + self._esc(8217) + "s") == "It’s"++ def test_rtf_unicode_em_dash(self):+ assert strip_rtf_fallback("a " + self._esc(8212) + " b") == "a — b"++ def test_rtf_unicode_accented_letter(self):+ assert strip_rtf_fallback("caf" + self._esc(233)) == "caf\xe9"++ def test_rtf_unicode_hex_fallback_consumed(self):+ # The \uN escape's fallback here is a "\'92" hex byte — it is consumed.+ text = "x" + self._BS + "u8217" + self._BS + "'92y"+ assert strip_rtf_fallback(text) == "x’y"++ def test_rtf_unicode_space_delimited_fallback(self):+ text = "x" + self._BS + "u8217 ?y"+ assert strip_rtf_fallback(text) == "x’y"++ def test_rtf_unicode_negative_codepoint(self):+ # RTF encodes code points > 32767 as negative 16-bit; -3 -> U+FFFD.+ assert strip_rtf_fallback(self._esc(-3)) == "�"++ def test_rtf_fallback_without_unicode_unchanged(self):+ # Regression: control-word-only input is unaffected by the new step.+ assert strip_rtf_fallback(self._BS + "b0 Bold" + self._BS + "b0 off") == "Boldoff"+ assert strip_rtf_fallback("{" + self._BS + "rtf1 hi}") == "hi"++ def test_rtf_unicode_consecutive_escapes_with_hex_fallback(self):+ # Two adjacent \uN escapes, each with a \'XX hex fallback, decode cleanly.+ text = self._BS + "u8220" + self._BS + "'93Hi" + self._BS + "u8221" + self._BS + "'94"+ assert strip_rtf_fallback(text) == "“Hi”"