DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix: decode escaped emoji in ask_user cards as real characters (#1780)
Mohammad Hijjawid16481a77c6f
diff --git a/deeptutor/utils/text_display.py b/deeptutor/utils/text_display.pyindex 7663d9803..0365ce017 100644--- a/deeptutor/utils/text_display.py+++ b/deeptutor/utils/text_display.py@@ -2,7 +2,7 @@from __future__ import annotations-import codecs+import jsonimport re# Match dense JSON-style ``\\uXXXX`` runs (3+ escapes), mirroring the web@@ -17,9 +17,17 @@ def decode_escaped_unicode_for_display(text: str) -> str:def _replace(match: re.Match[str]) -> str:run = match.group(0)+ # Decode the run as a JSON string, like the web decoder's JSON.parse:+ # characters outside the BMP (emoji, math letters) arrive as UTF-16+ # surrogate pairs that must be joined into one code point.+ # ``unicode_escape`` keeps the two halves as lone surrogates, which+ # UTF-8 cannot encode.try:- decoded = codecs.decode(run, "unicode_escape")- except (UnicodeDecodeError, ValueError):+ decoded = json.loads(f'"{run}"')+ except ValueError:+ return run+ if any(0xD800 <= ord(ch) <= 0xDFFF for ch in decoded):+ # An unpaired surrogate is not text; keep the escapes visible.return runif any(ord(ch) > 0x7F for ch in decoded):return decodeddiff --git a/tests/tools/test_ask_user.py b/tests/tools/test_ask_user.pyindex f69882288..3a752b649 100644--- a/tests/tools/test_ask_user.py+++ b/tests/tools/test_ask_user.py@@ -463,6 +463,18 @@ def test_prompt_decodes_dense_unicode_escapes() -> None:assert payload.questions[0].prompt == "「数制转换」"+def test_option_label_decodes_escaped_emoji() -> None:+ """Escaped emoji arrive as surrogate pairs and must decode to UTF-8 text."""+ escaped = "\\ud83d\\udc4d\\u662f\\u7684"+ payload, err = build_ask_user_payload(+ questions=[{"prompt": "Ready?", "options": [escaped, "No"]}],+ )+ assert err is None+ assert payload is not None+ label = payload.questions[0].options[0].label+ assert label == "👍是的"++# --------------------------- streaming previews ---------------------------_STREAMED = (diff --git a/tests/utils/test_text_display.py b/tests/utils/test_text_display.pyindex 5a8bee74f..da5508cb5 100644--- a/tests/utils/test_text_display.py+++ b/tests/utils/test_text_display.py@@ -18,3 +18,15 @@ def test_leaves_short_ascii_runs_alone() -> None:def test_empty_and_plain_text_passthrough() -> None:assert decode_escaped_unicode_for_display("") == ""assert decode_escaped_unicode_for_display("hello") == "hello"+++def test_joins_surrogate_pairs_into_one_character() -> None:+ # ``json.dumps`` escapes characters outside the BMP as surrogate pairs.+ escaped = "\\u597d\\u7684\\ud83d\\udc4d"+ decoded = decode_escaped_unicode_for_display(escaped)+ assert decoded == "好的👍"+++def test_leaves_runs_with_unpaired_surrogates_escaped() -> None:+ text = "\\u4f60\\u597d\\ud83d"+ assert decode_escaped_unicode_for_display(text) == text