Skip to content

Official Codebase of 3DCodeVerse

Python

models: a deadline must fit the answer it asked for

gaoypenge46ddc400a81
diff --git a/harness/codeverse/agents/api_agent.py b/harness/codeverse/agents/api_agent.py
index 741a5021..987e3759 100644
--- a/harness/codeverse/agents/api_agent.py
+++ b/harness/codeverse/agents/api_agent.py
@@ -53,6 +53,15 @@ TURN_TOKENS_GROWTH = 1.5
#: floor so a turn that starts near the deadline still gets one real attempt.
TURN_WAIT_MIN_S = 20.0
TURN_WAIT_MAX_S = 120.0
+#: ...but a deadline must also fit the ANSWER the turn asked for, or the tokens are billed
+#: and then discarded on a timeout. Measured 2026-08-27: a 15 996-token turn streamed in
+#: ~90 s (178 tok/s) and planner calls run 145 tok/s p50 / 60 tok/s p10, so with the
+#: envelope at the model's 65 536 ceiling a flat 120 s would cut a long turn off — trading
+#: the token wall for a wait wall. The derived wait is bounded by TURN_WAIT_CEILING_S and
+#: then by the session deadline, which is itself clipped to the run's wall clock.
+TURN_SLOW_TOKENS_PER_S = 145.0
+TURN_WAIT_OVERHEAD_S = 45.0
+TURN_WAIT_CEILING_S = 600.0
DEFAULT_SYSTEM = (
"You are an expert 3D programmer working inside a harness-managed workspace. "
"Use the tools to read, write and verify code under src/ and public/. Write RAW code in the "
@@ -258,9 +267,12 @@ class _Loop:
)
def _turn_wait_s(self) -> float:
- """Retry budget for one model call of this turn: what the session can still
- afford, clipped to ``[TURN_WAIT_MIN_S, TURN_WAIT_MAX_S]``."""
- return max(TURN_WAIT_MIN_S, min(TURN_WAIT_MAX_S, self.t_deadline - time.monotonic()))
+ """Retry budget for one model call of this turn: enough for the answer it asked
+ for, never past what the session can still afford."""
+ want = max(TURN_WAIT_MAX_S,
+ min(TURN_WAIT_CEILING_S,
+ self.turn_tokens / TURN_SLOW_TOKENS_PER_S + TURN_WAIT_OVERHEAD_S))
+ return max(TURN_WAIT_MIN_S, min(want, self.t_deadline - time.monotonic()))
def _generate(self, turn: int) -> ChatResponse | None:
req = ChatRequest(
diff --git a/harness/codeverse/tracks/planner.py b/harness/codeverse/tracks/planner.py
index 2c7a6718..df9501bf 100644
--- a/harness/codeverse/tracks/planner.py
+++ b/harness/codeverse/tracks/planner.py
@@ -66,6 +66,24 @@ TRUNCATION_GROWTH = 1.5
#: storm-day plan stage waited 492 s median per run for 39 s of model time; 300 s is ~4x the
#: worst observed call and replaces the model's 900 s default. The re-asks are on top.
PLAN_MAX_WAIT_S = 300.0
+#: ...but a deadline must fit the ANSWER it asked for, or the tokens are billed and then
+#: thrown away on a timeout. Measured over 395 real planner calls (2026-08-27): output
+#: runs at 145 tok/s p50 and 60 tok/s p10, and the largest answers seen were 32 k tokens
+#: in 152-199 s. At the p10 rate a 32 k answer needs 532 s and a 65 k one 1 089 s, so the
+#: flat 300 s guaranteed a timeout the moment a re-ask grew the budget — which is exactly
+#: how art_med_tool_chest and fancy_bl_windsor_chair died at the planner (2026-08-27).
+PLAN_SLOW_TOKENS_PER_S = 60.0
+PLAN_WAIT_OVERHEAD_S = 60.0
+
+
+def plan_wait_s(tokens: int, guard: object | None = None) -> float:
+ """How long one planner call may take, given the answer size it asked for.
+
+ Never below :data:`PLAN_MAX_WAIT_S`, and clipped to the wall clock the run has left
+ (``BudgetGuard.timeout_s``) so a big plan cannot outlive its own run."""
+ want = max(PLAN_MAX_WAIT_S, tokens / PLAN_SLOW_TOKENS_PER_S + PLAN_WAIT_OVERHEAD_S)
+ fn = getattr(guard, "timeout_s", None)
+ return float(fn(want, floor_s=PLAN_MAX_WAIT_S)) if callable(fn) else want
def plan_tokens(budget: PlanBudget, floor: int) -> int:
@@ -182,7 +200,7 @@ def plan_with_usage[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P]
for attempt in range(2 + MAX_VALIDATION_REASKS + MAX_QUALITY_REASKS):
req = ChatRequest(messages=messages, system=system, response_schema=schema, temperature=temperature,
thinking="medium", max_output_tokens=tokens, label=f"planner{'-retry' if attempt else ''}",
- max_wait_s=PLAN_MAX_WAIT_S)
+ max_wait_s=plan_wait_s(tokens, guard))
try:
resp = model.generate(req)
except Exception as e: # noqa: BLE001 — a truncated plan is retryable; anything else is not
diff --git a/harness/tests/agents/test_api_agent.py b/harness/tests/agents/test_api_agent.py
index 7ae4abd3..2c090fb0 100644
--- a/harness/tests/agents/test_api_agent.py
+++ b/harness/tests/agents/test_api_agent.py
@@ -248,12 +248,14 @@ class _Clock:
def test_each_turn_gets_a_retry_budget_clipped_to_what_the_session_can_afford(tmp_ws, monkeypatch):
"""Audit 2026-08-26 §5.1: 66 give-up spans of the model's 900 s deadline (up to 3 in a row on
one turn: median 923 s, p90 2 743 s) were 30 % of the storm day's waiting, ~1 430 s per run.
- A turn's call may now retry for at most TURN_WAIT_MAX_S = 120 s (a storm-day call succeeds in
- 8.3 s p50 / 31 s p90) and never past the session deadline, floored at 20 s so a late turn
- still gets one real attempt."""
+ A turn's call retries for as long as the answer it asked for plausibly needs (at least
+ TURN_WAIT_MAX_S, at most TURN_WAIT_CEILING_S) and NEVER past the session deadline,
+ floored at 20 s so a late turn still gets one real attempt."""
import codeverse.agents.api_agent as mod
- for timeout_s, want in ((600, 120.0), (60, 60.0), (5, 20.0)):
+ envelope = max(mod.TURN_WAIT_MAX_S, min(mod.TURN_WAIT_CEILING_S,
+ mod.TURN_TOKENS / mod.TURN_SLOW_TOKENS_PER_S + mod.TURN_WAIT_OVERHEAD_S))
+ for timeout_s, want in ((6000, envelope), (60, 60.0), (5, 20.0)):
monkeypatch.setattr(mod, "time", _Clock())
fake = FakeChatModel([resp("", call("list_files")), resp("done")])
res = ApiAgent("fake:fake-1", chat_model=fake).run(_job(tmp_ws, timeout_s=timeout_s))
diff --git a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
index 4d92ac34..98c223da 100644
--- a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
+++ b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
@@ -182,7 +182,13 @@ def test_planner_validates_retries_and_writes(tmp_ws):
assert model.requests[0].response_schema is not None and "PascalCase" in model.requests[0].system
# audit 2026-08-26 §2: the plan stage waited 492 s median per storm-day run for 39 s of
# model time; one planner call may now retry for 300 s (~4x the worst observed call), not 900
- assert all(r.max_wait_s == 300.0 for r in model.requests)
+ # the deadline scales with the answer size it asked for (measured: 60 tok/s p10), never
+ # below the 300 s floor — a flat 300 s guaranteed a timeout once a re-ask grew the budget
+ from codeverse.tracks.planner import plan_wait_s
+
+ # never below the 300 s floor, never more than the answer size justifies; the run's
+ # own wall clock clips it in between (plan_wait_s(tokens, guard))
+ assert all(300.0 <= r.max_wait_s <= plan_wait_s(r.max_output_tokens) for r in model.requests)
# deterministic acceptance items from constraints were appended
texts = " ".join(a.text for a in p.acceptance)
assert "height = 0.820" in texts and "Includes: armrests" in texts