Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

fix(rtf): decode unicode escapes in the dependency-free fallback (#52)

Shubham Srivastava3665d9fe5259
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 915cb03..7238e37 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+
+- The dependency-free RTF fallback (used when `striprtf` is not installed) now
+ decodes `\uN` unicode escapes — smart quotes, dashes, accented letters — instead
+ of dropping them and leaving only the ASCII fallback character.
+
## [1.2.0] — 2026-06-17
### Added
diff --git a/book_to_skill/parsers/rtf.py b/book_to_skill/parsers/rtf.py
index e765559..e7f6660 100644
--- a/book_to_skill/parsers/rtf.py
+++ b/book_to_skill/parsers/rtf.py
@@ -5,7 +5,22 @@ from book_to_skill.parsers.text import read_text_file
from book_to_skill.exceptions import ExtractionError
+# RTF unicode escape: \uN (signed decimal) followed by its fallback char(s).
+# Decode the code point and drop the standard single fallback — a \'XX hex byte
+# or a literal "?". Assumes the default \uc1 (one fallback char); \ucN directives
+# and multi-char/group fallbacks are not parsed (best-effort fallback only).
+_RTF_UNICODE = re.compile(r"\\u(-?\d+)[ ]?(?:\\'[0-9a-fA-F]{2}|\?)?")
+
+
+def _rtf_unicode_repl(match: re.Match) -> str:
+ cp = int(match.group(1)) % 0x10000 # RTF uses signed 16-bit; wrap negatives
+ if cp == 0 or 0xD800 <= cp <= 0xDFFF: # NUL and lone surrogates: unwanted in text
+ return ""
+ return chr(cp)
+
+
def strip_rtf_fallback(raw: str) -> str:
+ raw = _RTF_UNICODE.sub(_rtf_unicode_repl, raw) # decode \uN escapes first
raw = re.sub(r"\\'[0-9a-fA-F]{2}", " ", raw)
raw = re.sub(r"\\par[d]?", "\n", raw)
raw = re.sub(r"\\tab", "\t", raw)
diff --git a/tests/test_book_to_skill.py b/tests/test_book_to_skill.py
index f8e6e5b..4303b69 100644
--- a/tests/test_book_to_skill.py
+++ b/tests/test_book_to_skill.py
@@ -34,6 +34,7 @@ from book_to_skill.utils import (
main,
)
from book_to_skill.config import SUPPORTED_EXTENSIONS
+from book_to_skill.parsers.rtf import strip_rtf_fallback
# ═══════════════════════════════════════════════════════════════════════════
@@ -989,4 +990,46 @@ class TestParserExceptionLogging:
assert result is None
captured = capsys.readouterr()
assert "[warn]" in captured.err
- assert "failed:" in captured.err
\ No newline at end of file
+ assert "failed:" in captured.err
+
+
+class TestRtfUnicodeFallback:
+ """The dependency-free RTF fallback decodes RTF \\uN unicode escapes."""
+
+ _BS = chr(92) # a single backslash, never written as a literal \-escape
+
+ def _esc(self, codepoint, fallback="?"):
+ # Build the RTF escape: backslash + "u" + number + one fallback char.
+ return self._BS + "u" + str(codepoint) + fallback
+
+ def test_rtf_unicode_right_single_quote(self):
+ assert strip_rtf_fallback("It" + self._esc(8217) + "s") == "It’s"
+
+ def test_rtf_unicode_em_dash(self):
+ assert strip_rtf_fallback("a " + self._esc(8212) + " b") == "a — b"
+
+ def test_rtf_unicode_accented_letter(self):
+ assert strip_rtf_fallback("caf" + self._esc(233)) == "caf\xe9"
+
+ def test_rtf_unicode_hex_fallback_consumed(self):
+ # The \uN escape's fallback here is a "\'92" hex byte — it is consumed.
+ text = "x" + self._BS + "u8217" + self._BS + "'92y"
+ assert strip_rtf_fallback(text) == "x’y"
+
+ def test_rtf_unicode_space_delimited_fallback(self):
+ text = "x" + self._BS + "u8217 ?y"
+ assert strip_rtf_fallback(text) == "x’y"
+
+ def test_rtf_unicode_negative_codepoint(self):
+ # RTF encodes code points > 32767 as negative 16-bit; -3 -> U+FFFD.
+ assert strip_rtf_fallback(self._esc(-3)) == "�"
+
+ def test_rtf_fallback_without_unicode_unchanged(self):
+ # Regression: control-word-only input is unaffected by the new step.
+ assert strip_rtf_fallback(self._BS + "b0 Bold" + self._BS + "b0 off") == "Boldoff"
+ assert strip_rtf_fallback("{" + self._BS + "rtf1 hi}") == "hi"
+
+ def test_rtf_unicode_consecutive_escapes_with_hex_fallback(self):
+ # Two adjacent \uN escapes, each with a \'XX hex fallback, decode cleanly.
+ text = self._BS + "u8220" + self._BS + "'93Hi" + self._BS + "u8221" + self._BS + "'94"
+ assert strip_rtf_fallback(text) == "“Hi”"