Skip to content

Official Codebase of 3DCodeVerse

Python

bench: a claude one-shot reply is every assistant text block, not the last continuation

gaoypeng72fdf61b3c28
diff --git a/eval/bench/_oneshot.py b/eval/bench/_oneshot.py
index 65a0b3e7..85f2a972 100644
--- a/eval/bench/_oneshot.py
+++ b/eval/bench/_oneshot.py
@@ -342,6 +342,25 @@ def _dump(out_dir: Path, prompt: str, argv: list[str], stdout: str, stderr: str)
# ----------------------------------------------------------------------------- claude -p
+def assistant_text(stream: str) -> str:
+ """Concatenate the text blocks of every ``assistant`` event of a ``--output-format stream-json`` run."""
+ parts: list[str] = []
+ for line in stream.splitlines():
+ line = line.strip()
+ if not line.startswith("{"):
+ continue
+ try:
+ ev = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if ev.get("type") != "assistant":
+ continue
+ for block in (ev.get("message") or {}).get("content") or []:
+ if isinstance(block, dict) and block.get("type") == "text" and block.get("text"):
+ parts.append(block["text"])
+ return "".join(parts)
+
+
class ClaudeOneShot:
"""``claude -p`` with every tool disabled and a single turn."""
@@ -359,7 +378,7 @@ class ClaudeOneShot:
self.id = f"oneshot:{self.kind}" + (f":{model}" if model else "")
def argv(self, prompt: str) -> list[str]:
- argv = [self.binary, "-p", prompt, "--output-format", "json", "--tools", "", "--max-turns", "1",
+ argv = [self.binary, "-p", prompt, "--output-format", "stream-json", "--verbose", "--tools", "", "--max-turns", "1",
"--no-session-persistence", "--strict-mcp-config", "--setting-sources", CLAUDE_SETTING_SOURCES]
if self.model:
argv += ["--model", self.model]
@@ -376,7 +395,10 @@ class ClaudeOneShot:
env = parse_claude_json(proc.stdout)
usage = usage_from_envelope(env, self.model or "default") if env else Usage(backend=self.kind, model=self.model)
usage.latency_ms = usage.latency_ms or int(proc.duration_s * 1000)
- text = str((env or {}).get("result") or "")
+ # the answer is EVERY assistant text block in order, not the result envelope's `result`: a reply that
+ # hits the per-message output limit is auto-continued, and `result` holds only the last continuation
+ # (3 sonnet scene answers began mid-statement, p3x_scenes_v2 2026-09-23)
+ text = assistant_text(proc.stdout) or str((env or {}).get("result") or "")
notes = ""
ok = bool(text.strip()) and not proc.timed_out
if proc.timed_out:
diff --git a/eval/tests/test_oneshot.py b/eval/tests/test_oneshot.py
index 41d26845..53efc2af 100644
--- a/eval/tests/test_oneshot.py
+++ b/eval/tests/test_oneshot.py
@@ -140,3 +140,20 @@ def test_claude_one_shot_pins_its_effort_and_never_reads_the_users_settings():
assert a[a.index("--setting-sources") + 1] == "project"
b = ClaudeOneShot("opus").argv("brief")
assert "--effort" in b and b[b.index("--effort") + 1] # the harness's claude_effort, never inherited
+
+
+def test_a_continued_claude_reply_is_read_whole_not_just_its_last_continuation():
+ import json as _json
+
+ from bench._oneshot import assistant_text
+
+ first = "```js\nexport async function createScene({ THREE }) {\n const a = 1;\n"
+ second = " return { scene: null, cameras: [], update() {} };\n}\n```"
+ stream = "\n".join(_json.dumps(e) for e in (
+ {"type": "system", "subtype": "init"},
+ {"type": "assistant", "message": {"content": [{"type": "thinking", "thinking": "..."}, {"type": "text", "text": first}]}},
+ {"type": "user", "message": {"content": [{"type": "text", "text": "Output token limit hit. Resume directly"}]}, "isSynthetic": True},
+ {"type": "assistant", "message": {"content": [{"type": "text", "text": second}]}},
+ {"type": "result", "subtype": "success", "result": second, "is_error": False},
+ ))
+ assert assistant_text(stream) == first + second