Official Codebase of 3DCodeVerse
Python
models: a deadline must fit the answer it asked for
gaoypenge46ddc400a81
diff --git a/harness/codeverse/agents/api_agent.py b/harness/codeverse/agents/api_agent.pyindex 741a5021..987e3759 100644--- a/harness/codeverse/agents/api_agent.py+++ b/harness/codeverse/agents/api_agent.py@@ -53,6 +53,15 @@ TURN_TOKENS_GROWTH = 1.5#: floor so a turn that starts near the deadline still gets one real attempt.TURN_WAIT_MIN_S = 20.0TURN_WAIT_MAX_S = 120.0+#: ...but a deadline must also fit the ANSWER the turn asked for, or the tokens are billed+#: and then discarded on a timeout. Measured 2026-08-27: a 15 996-token turn streamed in+#: ~90 s (178 tok/s) and planner calls run 145 tok/s p50 / 60 tok/s p10, so with the+#: envelope at the model's 65 536 ceiling a flat 120 s would cut a long turn off — trading+#: the token wall for a wait wall. The derived wait is bounded by TURN_WAIT_CEILING_S and+#: then by the session deadline, which is itself clipped to the run's wall clock.+TURN_SLOW_TOKENS_PER_S = 145.0+TURN_WAIT_OVERHEAD_S = 45.0+TURN_WAIT_CEILING_S = 600.0DEFAULT_SYSTEM = ("You are an expert 3D programmer working inside a harness-managed workspace. ""Use the tools to read, write and verify code under src/ and public/. Write RAW code in the "@@ -258,9 +267,12 @@ class _Loop:)def _turn_wait_s(self) -> float:- """Retry budget for one model call of this turn: what the session can still- afford, clipped to ``[TURN_WAIT_MIN_S, TURN_WAIT_MAX_S]``."""- return max(TURN_WAIT_MIN_S, min(TURN_WAIT_MAX_S, self.t_deadline - time.monotonic()))+ """Retry budget for one model call of this turn: enough for the answer it asked+ for, never past what the session can still afford."""+ want = max(TURN_WAIT_MAX_S,+ min(TURN_WAIT_CEILING_S,+ self.turn_tokens / TURN_SLOW_TOKENS_PER_S + TURN_WAIT_OVERHEAD_S))+ return max(TURN_WAIT_MIN_S, min(want, self.t_deadline - time.monotonic()))def _generate(self, turn: int) -> ChatResponse | None:req = ChatRequest(diff --git a/harness/codeverse/tracks/planner.py b/harness/codeverse/tracks/planner.pyindex 2c7a6718..df9501bf 100644--- a/harness/codeverse/tracks/planner.py+++ b/harness/codeverse/tracks/planner.py@@ -66,6 +66,24 @@ TRUNCATION_GROWTH = 1.5#: storm-day plan stage waited 492 s median per run for 39 s of model time; 300 s is ~4x the#: worst observed call and replaces the model's 900 s default. The re-asks are on top.PLAN_MAX_WAIT_S = 300.0+#: ...but a deadline must fit the ANSWER it asked for, or the tokens are billed and then+#: thrown away on a timeout. Measured over 395 real planner calls (2026-08-27): output+#: runs at 145 tok/s p50 and 60 tok/s p10, and the largest answers seen were 32 k tokens+#: in 152-199 s. At the p10 rate a 32 k answer needs 532 s and a 65 k one 1 089 s, so the+#: flat 300 s guaranteed a timeout the moment a re-ask grew the budget — which is exactly+#: how art_med_tool_chest and fancy_bl_windsor_chair died at the planner (2026-08-27).+PLAN_SLOW_TOKENS_PER_S = 60.0+PLAN_WAIT_OVERHEAD_S = 60.0+++def plan_wait_s(tokens: int, guard: object | None = None) -> float:+ """How long one planner call may take, given the answer size it asked for.++ Never below :data:`PLAN_MAX_WAIT_S`, and clipped to the wall clock the run has left+ (``BudgetGuard.timeout_s``) so a big plan cannot outlive its own run."""+ want = max(PLAN_MAX_WAIT_S, tokens / PLAN_SLOW_TOKENS_PER_S + PLAN_WAIT_OVERHEAD_S)+ fn = getattr(guard, "timeout_s", None)+ return float(fn(want, floor_s=PLAN_MAX_WAIT_S)) if callable(fn) else wantdef plan_tokens(budget: PlanBudget, floor: int) -> int:@@ -182,7 +200,7 @@ def plan_with_usage[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P]for attempt in range(2 + MAX_VALIDATION_REASKS + MAX_QUALITY_REASKS):req = ChatRequest(messages=messages, system=system, response_schema=schema, temperature=temperature,thinking="medium", max_output_tokens=tokens, label=f"planner{'-retry' if attempt else ''}",- max_wait_s=PLAN_MAX_WAIT_S)+ max_wait_s=plan_wait_s(tokens, guard))try:resp = model.generate(req)except Exception as e: # noqa: BLE001 — a truncated plan is retryable; anything else is notdiff --git a/harness/tests/agents/test_api_agent.py b/harness/tests/agents/test_api_agent.pyindex 7ae4abd3..2c090fb0 100644--- a/harness/tests/agents/test_api_agent.py+++ b/harness/tests/agents/test_api_agent.py@@ -248,12 +248,14 @@ class _Clock:def test_each_turn_gets_a_retry_budget_clipped_to_what_the_session_can_afford(tmp_ws, monkeypatch):"""Audit 2026-08-26 §5.1: 66 give-up spans of the model's 900 s deadline (up to 3 in a row onone turn: median 923 s, p90 2 743 s) were 30 % of the storm day's waiting, ~1 430 s per run.- A turn's call may now retry for at most TURN_WAIT_MAX_S = 120 s (a storm-day call succeeds in- 8.3 s p50 / 31 s p90) and never past the session deadline, floored at 20 s so a late turn- still gets one real attempt."""+ A turn's call retries for as long as the answer it asked for plausibly needs (at least+ TURN_WAIT_MAX_S, at most TURN_WAIT_CEILING_S) and NEVER past the session deadline,+ floored at 20 s so a late turn still gets one real attempt."""import codeverse.agents.api_agent as mod- for timeout_s, want in ((600, 120.0), (60, 60.0), (5, 20.0)):+ envelope = max(mod.TURN_WAIT_MAX_S, min(mod.TURN_WAIT_CEILING_S,+ mod.TURN_TOKENS / mod.TURN_SLOW_TOKENS_PER_S + mod.TURN_WAIT_OVERHEAD_S))+ for timeout_s, want in ((6000, envelope), (60, 60.0), (5, 20.0)):monkeypatch.setattr(mod, "time", _Clock())fake = FakeChatModel([resp("", call("list_files")), resp("done")])res = ApiAgent("fake:fake-1", chat_model=fake).run(_job(tmp_ws, timeout_s=timeout_s))diff --git a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py b/harness/tests/orchestrator_tracks/test_generation_planner_repair.pyindex 4d92ac34..98c223da 100644--- a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py+++ b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py@@ -182,7 +182,13 @@ def test_planner_validates_retries_and_writes(tmp_ws):assert model.requests[0].response_schema is not None and "PascalCase" in model.requests[0].system# audit 2026-08-26 §2: the plan stage waited 492 s median per storm-day run for 39 s of# model time; one planner call may now retry for 300 s (~4x the worst observed call), not 900- assert all(r.max_wait_s == 300.0 for r in model.requests)+ # the deadline scales with the answer size it asked for (measured: 60 tok/s p10), never+ # below the 300 s floor — a flat 300 s guaranteed a timeout once a re-ask grew the budget+ from codeverse.tracks.planner import plan_wait_s++ # never below the 300 s floor, never more than the answer size justifies; the run's+ # own wall clock clips it in between (plan_wait_s(tokens, guard))+ assert all(300.0 <= r.max_wait_s <= plan_wait_s(r.max_output_tokens) for r in model.requests)# deterministic acceptance items from constraints were appendedtexts = " ".join(a.text for a in p.acceptance)assert "height = 0.820" in texts and "Includes: armrests" in texts