Skip to content

The agent that grows with you

fix(clarify): show questions and choices with a prose-safe display pass

Teknium7c8dd94f1f1f
diff --git a/agent/redact.py b/agent/redact.py
index 1deb8d1ebf9..62b1052534c 100644
--- a/agent/redact.py
+++ b/agent/redact.py
@@ -699,34 +699,43 @@ _APPROVAL_PAD_RE = re.compile(rf"(?<=\S){_APPROVAL_SPACE}{{21,}}|(?m:^){_APPROVA
# 3+ consecutive blank or whitespace-only lines; the match ends at a newline so the next line keeps its indent.
_APPROVAL_BLANK_RE = re.compile(rf"\n(?:{_APPROVAL_SPACE}*\n){{3,}}")
_APPROVAL_SAME_CHAR_RUN_RE = re.compile(r"(.)\1*", re.DOTALL)
+# Prose mode (questions shown to a human, never executed) surfaces only what can hide or reorder
+# text: controls, bidi embeddings/overrides/isolates, line/paragraph separators and padding. ZWJ,
+# ZWNJ and variation selectors stay, because emoji and scripts such as Persian need them.
+_PROSE_BIDI_CONTROLS = frozenset("\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
-def _escape_approval_run(run: str) -> str:
+def _escape_approval_run(run: str, prose: bool = False) -> str:
"""Escape the invisible code points of a non-ASCII run; 4+ repeats of one show once with their count."""
def one(m: re.Match) -> str:
ch, n = m.group(1), len(m.group())
cat = unicodedata.category(ch)
- if ch not in _APPROVAL_BLANK_GLYPHS and cat[0] != "C" and cat not in ("Zl", "Zp"):
+ if prose:
+ hidden = cat == "Cc" or cat in ("Zl", "Zp") or ch in _PROSE_BIDI_CONTROLS
+ else:
+ hidden = ch in _APPROVAL_BLANK_GLYPHS or cat[0] == "C" or cat in ("Zl", "Zp")
+ if not hidden:
return m.group()
esc = ch.encode("unicode_escape").decode("ascii")
return f"⟨{esc} ×{n}⟩" if n >= 4 else esc * n
return _APPROVAL_SAME_CHAR_RUN_RE.sub(one, run)
-def sanitize_command_for_display(text: str | None) -> str:
+def sanitize_command_for_display(text: str | None, *, prose: bool = False) -> str:
"""Make hidden content in an approval-prompt string visible; clean text comes back unchanged.
- Display-only: the executed command, smart assessment and allowlist keys never see this copy."""
+ Display-only: the executed command, smart assessment and allowlist keys never see this copy.
+ ``prose=True`` is the lighter pass for question text (see ``_PROSE_BIDI_CONTROLS``)."""
if not text:
return ""
- out = _APPROVAL_NON_ASCII_RE.sub(lambda m: _escape_approval_run(m.group()), text)
+ out = _APPROVAL_NON_ASCII_RE.sub(lambda m: _escape_approval_run(m.group(), prose), text)
out = _APPROVAL_BLANK_RE.sub(lambda m: f"\n⟨+{m.group().count(chr(10)) - 1} blank lines⟩\n", out)
return _APPROVAL_PAD_RE.sub(lambda m: f" ⟨+{len(m.group())} whitespace chars⟩ ", out)
-def redact_approval_display(text: str | None, *, force: bool = False) -> str:
+def redact_approval_display(text: str | None, *, force: bool = False, prose: bool = False) -> str:
"""Secret redaction plus :func:`sanitize_command_for_display`, for every approval surface."""
- return sanitize_command_for_display(redact_sensitive_text(str(text or ""), force=force))
+ return sanitize_command_for_display(redact_sensitive_text(str(text or ""), force=force), prose=prose)
def _mask_token(token: str) -> str:
diff --git a/tests/agent/test_approval_display_sanitize.py b/tests/agent/test_approval_display_sanitize.py
index 41c73ed7a45..d1db82e8c44 100644
--- a/tests/agent/test_approval_display_sanitize.py
+++ b/tests/agent/test_approval_display_sanitize.py
@@ -91,9 +91,13 @@ def test_first_render_surfaces_get_the_display_copy_while_values_stay_raw(monkey
# Tool-progress line under the prompt shows the hidden characters instead of collapsing them.
preview = build_tool_preview("terminal", {"command": raw}, max_len=0)
assert "\\u200b" in preview and "\\u2028 \u00d760" in preview
- # Clarify: the surface renders the display copy; the tool result echoes the raw question and choice.
+ # Clarify: prose display copy of question and choices (emoji/ZWJ/ZWNJ/VS16 intact, bidi and padding
+ # surfaced); the tool result echoes the raw question and maps the picked choice back to its raw text.
+ prose = "\u26a0\ufe0f Which branch? \U0001f468\u200d\U0001f4bb \u0645\u06cc\u200c\u062e\u0648\u0627\u0647\u0645"
asked = []
- result = json.loads(clarify_tool([{"question": raw, "choices": ["go\u200b"]}], callback=lambda qs: asked.extend(qs)
- or {"answers": {qs[0]["qid"]: qs[0]["choices_offered"][0]}, "outcome": "submitted"}))
- assert asked[0]["question"] == sanitize_command_for_display(raw)
- assert result["responses"][0]["question"] == raw and result["responses"][0]["user_response"] == "go\u200b"
+ result = json.loads(clarify_tool([{"question": prose + raw, "choices": ["go\u202edab", "ok"]}],
+ callback=lambda qs: asked.extend(qs) or {
+ "answers": {qs[0]["qid"]: qs[0]["choices"][0]}, "outcome": "submitted"}))
+ assert asked[0]["question"] == prose + "rm -rf ./build\u200b-cache\u27e8\\u2028 \u00d760\u27e9; " + _TAIL
+ assert asked[0]["choices"][0] == "go\\u202edab (Recommended)" and asked[0]["choices_offered"][1] == "ok"
+ assert result["responses"][0]["question"] == prose + raw and result["responses"][0]["user_response"] == "go\u202edab"
diff --git a/tools/clarify_tool.py b/tools/clarify_tool.py
index a4a89c358b4..f823265bf84 100644
--- a/tools/clarify_tool.py
+++ b/tools/clarify_tool.py
@@ -92,14 +92,18 @@ def _response_status(qid: str, answers: dict, multi: bool) -> tuple:
return ("skipped" if qid in answers else "unanswered"), None
-def _result(normalized: list[dict], reply: dict) -> str:
+def _result(normalized: list[dict], reply: dict, shown: list[dict] | None = None) -> str:
"""Result JSON from a callback reply ``{"answers": {qid: raw | None}, "outcome", "notice"?}``:
every response carries ``status`` and ``user_response`` (null unless answered); ``outcome``
- says how the wait ended and ``notice`` (surface-supplied) says why."""
+ says how the wait ended and ``notice`` (surface-supplied) says why. A picked display-copy
+ choice (from ``shown``) maps back to the raw choice by position."""
answers = reply.get("answers") or {}
responses = []
- for entry in normalized:
+ for entry, seen in zip(normalized, shown or normalized):
status, value = _response_status(entry["qid"], answers, entry["multi_select"])
+ back = dict(zip(seen["choices_offered"] or [], entry["choices_offered"] or []))
+ if back and value:
+ value = [back.get(v, v) for v in value] if isinstance(value, list) else back.get(value, value)
responses.append({"question": entry["question"], "choices_offered": entry["choices_offered"],
"status": status, "user_response": value})
result: dict[str, object] = {"responses": responses, "outcome": reply["outcome"]}
@@ -118,15 +122,18 @@ def clarify_tool(questions, callback: Optional[Callable] = None) -> str:
return tool_error(_UNAVAILABLE)
from agent.redact import redact_approval_display
from tools.human_input_hooks import human_input_request
- # Every surface renders the display copy of the question; the result echoes the raw text. Choices stay
- # raw: a picked choice is the answer value.
- shown = [{**q, "question": redact_approval_display(q["question"])} for q in normalized]
+
+ def prose(texts):
+ return [redact_approval_display(t, prose=True) for t in texts] if texts else texts
+ # Every surface renders the prose display copy of the question and choices; the result echoes the raw text.
+ shown = [{**q, "question": prose([q["question"]])[0], "choices": prose(q["choices"]),
+ "choices_offered": prose(q["choices_offered"])} for q in normalized]
# Observers see the questions only; the answers stay in the tool result.
with human_input_request("clarify", prompt="\n".join(q["question"] for q in shown)) as human:
try:
reply = callback(shown)
human.outcome = str(reply.get("outcome") or "")
- return _result(normalized, reply)
+ return _result(normalized, reply, shown)
except Exception as exc:
human.outcome = "error"
return tool_error(f"Failed to get user input: {exc}")