Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: decode escaped emoji in ask_user cards as real characters (#1780)

Mohammad Hijjawid16481a77c6f
diff --git a/deeptutor/utils/text_display.py b/deeptutor/utils/text_display.py
index 7663d9803..0365ce017 100644
--- a/deeptutor/utils/text_display.py
+++ b/deeptutor/utils/text_display.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-import codecs
+import json
import re
# Match dense JSON-style ``\\uXXXX`` runs (3+ escapes), mirroring the web
@@ -17,9 +17,17 @@ def decode_escaped_unicode_for_display(text: str) -> str:
def _replace(match: re.Match[str]) -> str:
run = match.group(0)
+ # Decode the run as a JSON string, like the web decoder's JSON.parse:
+ # characters outside the BMP (emoji, math letters) arrive as UTF-16
+ # surrogate pairs that must be joined into one code point.
+ # ``unicode_escape`` keeps the two halves as lone surrogates, which
+ # UTF-8 cannot encode.
try:
- decoded = codecs.decode(run, "unicode_escape")
- except (UnicodeDecodeError, ValueError):
+ decoded = json.loads(f'"{run}"')
+ except ValueError:
+ return run
+ if any(0xD800 <= ord(ch) <= 0xDFFF for ch in decoded):
+ # An unpaired surrogate is not text; keep the escapes visible.
return run
if any(ord(ch) > 0x7F for ch in decoded):
return decoded
diff --git a/tests/tools/test_ask_user.py b/tests/tools/test_ask_user.py
index f69882288..3a752b649 100644
--- a/tests/tools/test_ask_user.py
+++ b/tests/tools/test_ask_user.py
@@ -463,6 +463,18 @@ def test_prompt_decodes_dense_unicode_escapes() -> None:
assert payload.questions[0].prompt == "「数制转换」"
+def test_option_label_decodes_escaped_emoji() -> None:
+ """Escaped emoji arrive as surrogate pairs and must decode to UTF-8 text."""
+ escaped = "\\ud83d\\udc4d\\u662f\\u7684"
+ payload, err = build_ask_user_payload(
+ questions=[{"prompt": "Ready?", "options": [escaped, "No"]}],
+ )
+ assert err is None
+ assert payload is not None
+ label = payload.questions[0].options[0].label
+ assert label == "👍是的"
+
+
# --------------------------- streaming previews ---------------------------
_STREAMED = (
diff --git a/tests/utils/test_text_display.py b/tests/utils/test_text_display.py
index 5a8bee74f..da5508cb5 100644
--- a/tests/utils/test_text_display.py
+++ b/tests/utils/test_text_display.py
@@ -18,3 +18,15 @@ def test_leaves_short_ascii_runs_alone() -> None:
def test_empty_and_plain_text_passthrough() -> None:
assert decode_escaped_unicode_for_display("") == ""
assert decode_escaped_unicode_for_display("hello") == "hello"
+
+
+def test_joins_surrogate_pairs_into_one_character() -> None:
+ # ``json.dumps`` escapes characters outside the BMP as surrogate pairs.
+ escaped = "\\u597d\\u7684\\ud83d\\udc4d"
+ decoded = decode_escaped_unicode_for_display(escaped)
+ assert decoded == "好的👍"
+
+
+def test_leaves_runs_with_unpaired_surrogates_escaped() -> None:
+ text = "\\u4f60\\u597d\\ud83d"
+ assert decode_escaped_unicode_for_display(text) == text