Skip to content

Official Codebase of 3DCodeVerse

Python

bench: the claude-code one-shot pins its effort and never reads the user's settings

gaoypengc272e8e8a595
diff --git a/eval/bench/_oneshot.py b/eval/bench/_oneshot.py
index 9b3a5c94..65a0b3e7 100644
--- a/eval/bench/_oneshot.py
+++ b/eval/bench/_oneshot.py
@@ -37,6 +37,7 @@ from pydantic import BaseModel, Field
from bench._infra import is_infra_failure
from codeverse3d.agents.backends import (
+ CLAUDE_SETTING_SOURCES,
effort_overrides,
parse_claude_json,
parse_codex_jsonl,
@@ -347,15 +348,23 @@ class ClaudeOneShot:
kind = "claude-code"
def __init__(self, model: str = "", binary: str | None = None):
- self.model = model
+ # ``claude-code:<model>[@<effort>]`` like codex. The effort is ALWAYS passed and the user's
+ # settings are never read (--setting-sources project, as the harness's own sessions since D83):
+ # an arm that inherited ~/.claude/settings.json's effortLevel xhigh let sonnet think 94k tokens
+ # for 40 min and write no code on every scene brief (p3x_scenes_v2, 2026-09-23).
+ name, _, effort = model.partition("@")
+ self.model = name
+ self.effort = effort.strip().lower() or get_settings().agents.claude_effort
self.binary = binary or get_settings().binaries.claude_cli
self.id = f"oneshot:{self.kind}" + (f":{model}" if model else "")
def argv(self, prompt: str) -> list[str]:
argv = [self.binary, "-p", prompt, "--output-format", "json", "--tools", "", "--max-turns", "1",
- "--no-session-persistence", "--strict-mcp-config"]
+ "--no-session-persistence", "--strict-mcp-config", "--setting-sources", CLAUDE_SETTING_SOURCES]
if self.model:
argv += ["--model", self.model]
+ if self.effort:
+ argv += ["--effort", self.effort]
return argv
def generate(self, prompt: str, *, out_dir: Path, timeout_s: float = DEFAULT_TIMEOUT_S, label: str = "oneshot") -> OneShotResult:
diff --git a/eval/tests/test_oneshot.py b/eval/tests/test_oneshot.py
index 8e30b78c..41d26845 100644
--- a/eval/tests/test_oneshot.py
+++ b/eval/tests/test_oneshot.py
@@ -130,3 +130,13 @@ def test_a_hallucinated_write_tool_of_a_scene_module_is_still_read():
"</parameter></invoke>")
got = extract_files(xml, Language.SCENE_THREEJS)[SCENE_FILE]
assert got.startswith("export async function createScene")
+
+
+def test_claude_one_shot_pins_its_effort_and_never_reads_the_users_settings():
+ from bench._oneshot import ClaudeOneShot
+
+ a = ClaudeOneShot("sonnet@high").argv("brief")
+ assert a[a.index("--model") + 1] == "sonnet" and a[a.index("--effort") + 1] == "high"
+ assert a[a.index("--setting-sources") + 1] == "project"
+ b = ClaudeOneShot("opus").argv("brief")
+ assert "--effort" in b and b[b.index("--effort") + 1] # the harness's claude_effort, never inherited