Official Codebase of 3DCodeVerse
Python
retry: clip one call's retry budget to what the caller can afford (ChatRequest.max_wait_s)
gaoypeng97f19fbf7f18
diff --git a/harness/codeverse/agents/api_agent.py b/harness/codeverse/agents/api_agent.pyindex dfd98abe..74a8ec45 100644--- a/harness/codeverse/agents/api_agent.py+++ b/harness/codeverse/agents/api_agent.py@@ -36,6 +36,14 @@ log = logging.getLogger(__name__)BUILD_TOOL = "build"COMPACT_AT_CHARS = 350_000MODEL_RETRIES = 3+#: retry budget (``ChatRequest.max_wait_s``) for ONE model call of a turn: the session's+#: remaining time, clipped to this window. Audit 2026-08-26 §5.1: with the model's own+#: 900 s deadline x MODEL_RETRIES one turn could wait 2 700 s, and 66 such give-up spans+#: (median 923 s, p90 2 743 s) were 30 % of a storm day's waiting — ~1 430 s per run. A+#: successful storm-day call is 8.3 s p50 / 31 s p90, so 120 s is ~4x p90; 20 s is the+#: floor so a turn that starts near the deadline still gets one real attempt.+TURN_WAIT_MIN_S = 20.0+TURN_WAIT_MAX_S = 120.0DEFAULT_SYSTEM = ("You are an expert 3D programmer working inside a harness-managed workspace. ""Use the tools to read, write and verify code under src/ and public/. Write RAW code in the "@@ -210,6 +218,11 @@ class _Loop:tools=[t.name for t in self.specs],)+ def _turn_wait_s(self) -> float:+ """Retry budget for one model call of this turn: what the session can still+ afford, clipped to ``[TURN_WAIT_MIN_S, TURN_WAIT_MAX_S]``."""+ return max(TURN_WAIT_MIN_S, min(TURN_WAIT_MAX_S, self.t_deadline - time.monotonic()))+def _generate(self, turn: int) -> ChatResponse | None:req = ChatRequest(messages=self.messages,@@ -218,24 +231,33 @@ class _Loop:temperature=self.s.job.api.temperature,thinking=self.s.job.api.thinking,label=f"api-agent:{self.s.label}:t{turn}",+ max_wait_s=self._turn_wait_s(),)delay = 2.0for attempt in range(MODEL_RETRIES):+ if attempt: # a retry gets what is left, not the turn's opening budget again+ req = req.model_copy(update={"max_wait_s": self._turn_wait_s()})try:resp = self.chat.generate(req)self.usage = self.usage + resp.usagereturn respexcept Exception as e: # ModelError or provider glitchretryable = bool(getattr(e, "retryable", False))+ # the session deadline used to be checked only between turns, so a turn that+ # met it inside this loop still slept and re-entered the model's whole+ # retry machine (97 sessions overshot their timeout on 2026-08-26)+ expired = time.monotonic() >= self.t_deadlineself.s.traj.append("model_error",turn=turn,attempt=attempt,error=str(e)[:2000],retryable=retryable,+ deadline_passed=expired,)- if not retryable or attempt == MODEL_RETRIES - 1:- self.errors.append(f"model error on turn {turn}: {type(e).__name__}: {e}")+ if not retryable or expired or attempt == MODEL_RETRIES - 1:+ note = " (session deadline passed; not retried)" if expired and retryable else ""+ self.errors.append(f"model error on turn {turn}: {type(e).__name__}: {e}{note}")return Nonetime.sleep(delay)delay *= 2diff --git a/harness/codeverse/contracts/chat.py b/harness/codeverse/contracts/chat.pyindex 31700bc1..254beef3 100644--- a/harness/codeverse/contracts/chat.py+++ b/harness/codeverse/contracts/chat.py@@ -75,6 +75,13 @@ class ChatRequest(BaseModel):max_output_tokens: int = 16000thinking: Literal["off", "low", "medium", "high"] = "low"label: str = Field(default="", description="for logs/cost ledger: planner / judge / generate ...")+ max_wait_s: float | None = Field(+ default=None,+ gt=0,+ description="the longest this ONE logical call may spend, retries and their waits included; "+ "None = the model's default (models.retry.RETRY_DEADLINE_S = 900 s). A caller clips it to "+ "what it can still afford: an agent turn 20-120 s, a judge sample 240 s, the planner 300 s.",+ )class ChatResponse(BaseModel):diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.pyindex b2ec16a9..d9c495cd 100644--- a/harness/codeverse/judges/vlm_judge.py+++ b/harness/codeverse/judges/vlm_judge.py@@ -7,7 +7,9 @@ so n-sample mean/std measures judge noise. With ``n_samples > 1`` the modelcalls run in parallel (``codeverse.fanout``); results accumulate in sample order. Score/defect penalties/floors/caps/pass are computed in code (``scoring.py``). Images are ≤2×2 montages(``montage.py``); tracks may pass a clay/normals ``geometry_views`` RenderSet. Retries: up to-``max_attempts`` per sample on ``ModelError`` / parse failure; if no sample+``max_attempts`` per sample on ``ModelError`` / parse failure, all of them inside one+``sample_budget_s`` (:data:`SAMPLE_BUDGET_S`) that also clips the model's own retry+deadline (``ChatRequest.max_wait_s``); if no samplesucceeds, a *degraded* Judgment (``passed=False, overall=0, summary'judge_error: …'``, ``raw.status='degraded'``) is returned for the orchestratorto treat as a glitch, never as a score.@@ -41,6 +43,15 @@ from codeverse.models.base import ChatModel, ModelErrorlog = logging.getLogger(__name__)+#: retry budget for ONE judge sample, every attempt included. The verdict call is 42 s+#: p50 / 73 s p90 on a normal day and 50 / 103 s under a storm (audit 2026-08-26 §2), so+#: 240 s is ~2.3x the storm p90. It replaces 3 x (a 300 s read timeout + the model's 900 s+#: retry deadline): two storm-day rounds lost 1 162 s and 927 s that way (three 300 s+#: timeouts each) before a second sample answered in 128 s.+SAMPLE_BUDGET_S = 240.0+#: the floor of one attempt's ``max_wait_s``: a last attempt still gets a real try+SAMPLE_MIN_WAIT_S = 20.0+@dataclassclass JudgeContext:@@ -71,6 +82,7 @@ class VlmJudge:chat_model: ChatModel | None = None,cache_dir: Path | None = None,label: str = "judge",+ sample_budget_s: float | None = None,):self.rubric: Rubric = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)self.model_id = model_id or get_settings().default_judge@@ -78,6 +90,7 @@ class VlmJudge:self.temperature = temperatureself.thinking = thinkingself.max_attempts = max(1, int(max_attempts))+ self.sample_budget_s = SAMPLE_BUDGET_S if sample_budget_s is None else float(sample_budget_s)# payload size: the profile's dial (Settings.judge) unless the caller states onejd = get_settings().judgeself.max_montages = max(1, int(jd.montages if max_montages is None else max_montages))@@ -178,8 +191,17 @@ class VlmJudge:) -> tuple[JudgeOutput | None, Usage, str]:usage = Usage()last = ""+ t_start = time.monotonic()for attempt in range(1, self.max_attempts + 1):- t0 = time.time()+ remaining = self.sample_budget_s - (time.monotonic() - t_start)+ if remaining <= 0:+ last = (f"sample budget of {self.sample_budget_s:.0f}s spent after {attempt - 1} attempt(s)"+ + (f"; last: {last}" if last else ""))+ log.warning("judge %s: %s", req.label, last[:300])+ break+ # the model may retry inside this call, never past what the sample can still afford+ req = req.model_copy(update={"max_wait_s": max(SAMPLE_MIN_WAIT_S, remaining)})+ t0 = time.monotonic()try:resp: ChatResponse = self.model.generate(req)except ModelError as e:@@ -192,7 +214,7 @@ class VlmJudge:payload = resp.parsed if resp.parsed is not None else resp.texttry:out = parse_judge_output(payload, self.rubric, acceptance_ids, measured_scores=measured)- log.debug("judge %s ok in %.1fs", req.label, time.time() - t0)+ log.debug("judge %s ok in %.1fs", req.label, time.monotonic() - t0)return out, usage, ""except JudgeParseError as e:last = f"parse(attempt {attempt}): {e}"diff --git a/harness/codeverse/models/anthropic.py b/harness/codeverse/models/anthropic.pyindex 86354501..7a8b2304 100644--- a/harness/codeverse/models/anthropic.py+++ b/harness/codeverse/models/anthropic.py@@ -120,6 +120,8 @@ class AnthropicModel:delay,)+ # ChatRequest.max_wait_s is not honoured here: with_retries has no deadline, and its+ # 6 attempts x <= 5 s backoff bound one call to ~20 s of waiting plus the round-trips.return with_retries(attempt,is_retryable=lambda e: isinstance(e, ModelError) and e.retryable,diff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.pyindex e53410ae..a5c09227 100644--- a/harness/codeverse/models/gemini.py+++ b/harness/codeverse/models/gemini.py@@ -39,7 +39,7 @@ from codeverse.models.gemini_convert import (from codeverse.models.keypool import MAX_WAIT_S, KeyPool, KeyPoolExhausted, Outcomefrom codeverse.models.parts import Stopwatchfrom codeverse.models.pricing import estimate_cost-from codeverse.models.retry import rotate_with_retries+from codeverse.models.retry import RETRY_DEADLINE_S, rotate_with_retriesfrom codeverse.models.schema_utils import JsonParseError, parse_json_lenientfrom codeverse.models.storm import StormGatefrom codeverse.models.storm import storm_gate as storm_gate_for@@ -245,6 +245,8 @@ class GeminiModel:state["config"] = self._config(request, warnings)return True+ # the caller's budget clips the retry deadline, never extends it+ budget = RETRY_DEADLINE_S if request.max_wait_s is None else min(RETRY_DEADLINE_S, float(request.max_wait_s))return rotate_with_retries(self.pool,lambda key: self._once(key, contents, state["config"], request, warnings),@@ -253,6 +255,7 @@ class GeminiModel:max_attempts=self.max_attempts,base_delay=self.base_delay,max_delay=self.max_delay,+ max_total_s=budget,sleep=self._sleep,on_free_retry=downgrade_thinking,retry_after=_retry_after_s,diff --git a/harness/codeverse/models/openai.py b/harness/codeverse/models/openai.pyindex d95cc349..8d609857 100644--- a/harness/codeverse/models/openai.py+++ b/harness/codeverse/models/openai.py@@ -142,6 +142,8 @@ class OpenAIModel:delay,)+ # ChatRequest.max_wait_s is not honoured here: with_retries has no deadline, and its+ # 6 attempts x <= 5 s backoff bound one call to ~20 s of waiting plus the round-trips.return with_retries(attempt,is_retryable=lambda e: isinstance(e, ModelError) and e.retryable,diff --git a/harness/codeverse/tracks/planner.py b/harness/codeverse/tracks/planner.pyindex 8db97568..aecb1c76 100644--- a/harness/codeverse/tracks/planner.py+++ b/harness/codeverse/tracks/planner.py@@ -61,6 +61,11 @@ PLAN_TOKENS_PER_LEAF = 400PLAN_TOKENS_MAX = 60000#: one more attempt when the model truncates anyway, with half again as much roomTRUNCATION_GROWTH = 1.5+#: retry budget (``ChatRequest.max_wait_s``) for one planner call. Audit 2026-08-26 §2: the+#: call itself is 13.7 s p50 / 32 s p90 / 76 s max (23.4 / 43 / 68 under the storm), yet the+#: storm-day plan stage waited 492 s median per run for 39 s of model time; 300 s is ~4x the+#: worst observed call and replaces the model's 900 s default. The re-asks are on top.+PLAN_MAX_WAIT_S = 300.0def plan_tokens(budget: PlanBudget, floor: int) -> int:@@ -170,7 +175,8 @@ def plan_with_usage[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P]tokens = plan_tokens(budget, max_output_tokens)for attempt in range(2 + MAX_VALIDATION_REASKS + MAX_QUALITY_REASKS):req = ChatRequest(messages=messages, system=system, response_schema=schema, temperature=temperature,- thinking="medium", max_output_tokens=tokens, label=f"planner{'-retry' if attempt else ''}")+ thinking="medium", max_output_tokens=tokens, label=f"planner{'-retry' if attempt else ''}",+ max_wait_s=PLAN_MAX_WAIT_S)try:resp = model.generate(req)except Exception as e: # noqa: BLE001 — a truncated plan is retryable; anything else is notdiff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 6c0213a1..83bd6d00 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1222,6 +1222,25 @@ index in `telemetry/usage.jsonl` so the distribution of calls per key can be reaprobed.++**Follow-up, same day, from the time audit (51 storm-day runs vs 52 baseline; `time_audit/REPORT.md`).**+Three accelerations, all additive and on by default:++*A caller clips the retry budget to what it can afford* (`ChatRequest.max_wait_s`, None = the+model's `RETRY_DEADLINE_S`). Of the 246 297 s a storm-day run spent waiting on the provider,+**30 % (72 921 s) was 66 `model_error` spans** in which one call retried until the 900 s deadline —+median span 923 s, p90 2 743 s, i.e. three consecutive give-ups on ONE agent turn through+`api_agent.MODEL_RETRIES`; 45 sessions were hit, **~1 430 s per run**. `GeminiModel` now passes+`max_total_s = min(900, max_wait_s)` to `rotate_with_retries`. An agent turn asks for+`max(20, min(120, session time left))` (a successful storm-day call is 8.3 s p50 / 31 s p90) and+stops retrying once the session deadline has passed instead of sleeping 2 + 4 s past it (97+sessions overshot their timeout by 182 s median / 1 154 s p90); a judge sample gets+`SAMPLE_BUDGET_S = 240` for all its attempts (the verdict is 42 s p50 / 73 s p90, 50 / 103 s under+the storm; two rounds lost 1 162 s and 927 s to 3 × 300 s timeouts before a second sample answered+in 128 s); the planner 300 s (13.7 s p50 / 32 s p90, max 76 s; the storm-day plan stage waited+492 s median for 39 s of model time). Anthropic / OpenAI go through `with_retries`, which has no+deadline (≈ 20 s of backoff at most), so the field is a no-op there.+## 25. Where the time goes — the 2026-08-26 audit (`docs/TIME_AUDIT_2026-08-26.md`)51 storm-day runs against 52 baseline runs, every stage and every model call, scripts indiff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.mdindex f3958d70..365fc9f5 100644--- a/harness/docs/INTERFACES.md+++ b/harness/docs/INTERFACES.md@@ -49,7 +49,9 @@ append_jsonl_line(path, rec, lock) -> None # one json.dumps(ensu## models/```pythonfrom codeverse.models import get_chat_model # (model_id) -> ChatModel, lru-cached, thread-safe-resp = m.generate(ChatRequest(messages=[...], system=..., response_schema=..., temperature=..., thinking=..., label=...))+resp = m.generate(ChatRequest(messages=[...], system=..., response_schema=..., temperature=..., thinking=..., label=..., max_wait_s=None))+# max_wait_s: the longest this ONE call may spend, retries included (None = models.retry.RETRY_DEADLINE_S = 900 s);+# GeminiModel clips its retry deadline to it; api_agent 20-120 s per turn, VlmJudge 240 s per sample, planner 300 sresp.parsed / resp.text / resp.tool_calls / resp.usage # Usage always has cost_usd (models.pricing)from codeverse.models.keypool import KeyPool, KeyPoolExhaustedKeyPool(keys, *, rpm_per_key=900, tpm_per_key=None, cooldown_s=30, dead_cooldown_s=3600)@@ -265,7 +267,7 @@ python -m codeverse.spatial.mcp_server --workspace <ws> [--track X] [--languagefrom codeverse.judges.rubrics import load_rubric, Rubric # Rubric{…, defects: [DefectItem{id, text, penalty, cap}], caps[{…, when:, kinds}]}from codeverse.judges.vlm_judge import VlmJudgeVlmJudge(rubric="static_object_v1", model_id=None (settings.default_judge = gemini-3.1-pro-preview), n_samples=1,- temperature=0.2, *, thinking="low", max_attempts=3, max_montages=3, detail_crops=2, max_px=1024,+ temperature=0.2, *, thinking="low", max_attempts=3, max_montages=3, detail_crops=2, max_px=1024, sample_budget_s=240,chat_model=None, cache_dir=None, label="judge") # Δ max_images is GONE → montage budgetVlmJudge.judge(inp, *, geometry_views: RenderSet | None = None) -> Judgment # also reads inp.geometry_viewsfrom codeverse.judges.base import JudgeInput # (spec, renders, measurement=None, gates=[], acceptance=[], plan_summary="",diff --git a/harness/tests/agents/test_api_agent.py b/harness/tests/agents/test_api_agent.pyindex 350c58d0..c2ccf82c 100644--- a/harness/tests/agents/test_api_agent.py+++ b/harness/tests/agents/test_api_agent.py@@ -227,3 +227,65 @@ def test_job_images_ride_on_the_first_user_message(tmp_ws, tmp_path):imgs = [pt for pt in first.parts if isinstance(pt, ImagePart)]assert imgs and imgs[0].path == str(png), "the image must be a part of the FIRST user message"assert "judge scored" in (imgs[0].label or "")+++# ------------------------------------------------------ per-turn retry budget (audit 2026-08-26 §5.1)+class _Clock:+ """Fake ``time`` for the agent loop: ``monotonic()`` moves only when a test moves it."""++ def __init__(self, t: float = 1000.0):+ self.t = t+ self.slept: list[float] = []++ def monotonic(self) -> float:+ return self.t++ def sleep(self, s: float) -> None:+ self.slept.append(s)+ self.t += s+++def test_each_turn_gets_a_retry_budget_clipped_to_what_the_session_can_afford(tmp_ws, monkeypatch):+ """Audit 2026-08-26 §5.1: 66 give-up spans of the model's 900 s deadline (up to 3 in a row on+ one turn: median 923 s, p90 2 743 s) were 30 % of the storm day's waiting, ~1 430 s per run.+ A turn's call may now retry for at most TURN_WAIT_MAX_S = 120 s (a storm-day call succeeds in+ 8.3 s p50 / 31 s p90) and never past the session deadline, floored at 20 s so a late turn+ still gets one real attempt."""+ import codeverse.agents.api_agent as mod++ for timeout_s, want in ((600, 120.0), (60, 60.0), (5, 20.0)):+ monkeypatch.setattr(mod, "time", _Clock())+ fake = FakeChatModel([resp("", call("list_files")), resp("done")])+ res = ApiAgent("fake:fake-1", chat_model=fake).run(_job(tmp_ws, timeout_s=timeout_s))+ assert res.ok, res.errors+ assert [r.max_wait_s for r in fake.requests] == [want, want], timeout_s+++def test_a_turn_stops_retrying_once_the_session_deadline_has_passed(tmp_ws, monkeypatch):+ """The deadline used to be checked only at the top of a turn: a turn that met it inside the+ retry loop still slept 2 s + 4 s and re-entered the model's whole retry machine twice more+ (97 sessions overshot their timeout on 2026-08-26 by 182 s median / 1 154 s p90)."""+ import codeverse.agents.api_agent as mod++ clock = _Clock()+ monkeypatch.setattr(mod, "time", clock)++ class Boom(RuntimeError):+ retryable = True++ class StormModel(FakeChatModel):+ def generate(self, request: ChatRequest) -> ChatResponse:+ self.requests.append(request)+ clock.t += request.max_wait_s # the model spent its whole budget retrying+ raise Boom("503 high demand")++ fake = StormModel([])+ res = ApiAgent("fake:fake-1", chat_model=fake).run(_job(tmp_ws, timeout_s=100))+ assert not res.ok and res.exit_reason == "error"+ assert fake.requests[0].max_wait_s == 100.0+ assert len(fake.requests) == 1, "no retry once the deadline has passed"+ assert clock.slept == [], "and no 2 s / 4 s sleep past it"+ assert "deadline passed" in res.errors[0]+ rows = [json.loads(ln) for ln in Path(res.transcript_path).read_text().splitlines()]+ (err_row,) = [r for r in rows if r["kind"] == "model_error"]+ assert err_row["deadline_passed"] is True and err_row["retryable"] is Truediff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.pyindex f5dc6926..40f0415a 100644--- a/harness/tests/judges/test_vlm_judge.py+++ b/harness/tests/judges/test_vlm_judge.py@@ -187,3 +187,69 @@ def test_judge_payload_size_comes_from_the_settings_dial(monkeypatch):s2.apply_profile("economy") # a stated payload survives a profileassert VlmJudge(rubric="static_object_v1", model_id="fake:fake-1").detail_crops == 0get_settings.cache_clear()+++# -------------------------------------------------------- per-sample retry budget (audit 2026-08-26 §5.1)+def _fake_clock(monkeypatch):+ import time as real_time+ from types import SimpleNamespace++ import codeverse.judges.vlm_judge as mod++ clock = {"t": 0.0}+ monkeypatch.setattr(mod, "time", SimpleNamespace(monotonic=lambda: clock["t"], time=real_time.time,+ sleep=real_time.sleep))+ return clock+++def test_each_sample_has_a_retry_budget_and_hands_what_is_left_to_the_model(judge_input, cache_dir, monkeypatch):+ """The verdict call is 42 s p50 / 73 s p90 on a normal day, 50 / 103 s under a storm+ (audit 2026-08-26 §2); two storm-day rounds lost 1 162 s and 927 s to 3 x 300 s timeouts+ before a second sample answered in 128 s. A sample now has SAMPLE_BUDGET_S = 240 s for all+ its attempts, and every attempt tells the model how much of it remains."""+ from codeverse.judges.vlm_judge import SAMPLE_BUDGET_S++ clock = _fake_clock(monkeypatch)++ def slow_503(request):+ clock["t"] += 100.0+ return ModelError("503 high demand", retryable=True, status=503)++ model = FakeChatModel([slow_503, slow_503, good_reply(R, IDS, 0.8)])+ j = _judge(model, cache_dir=cache_dir).judge(judge_input)+ assert j.passed and len(model.requests) == 3+ assert SAMPLE_BUDGET_S == 240.0+ assert [r.max_wait_s for r in model.requests] == [240.0, 140.0, 40.0]+++def test_a_sample_stops_when_its_budget_is_spent(judge_input, cache_dir, monkeypatch):+ clock = _fake_clock(monkeypatch)++ def spent(request):+ clock["t"] += 250.0 # one attempt that retried inside the model for the whole budget+ return ModelError("503 high demand", retryable=True, status=503)++ model = FakeChatModel(default=spent)+ j = _judge(model, cache_dir=cache_dir).judge(judge_input)+ assert is_degraded(j) and len(model.requests) == 1, "max_attempts=3, but the budget is gone"+ assert "sample budget" in j.summary+ # the budget is a dial: a caller that can afford more gets the attempts back+ model = FakeChatModel(default=spent)+ j = _judge(model, cache_dir=cache_dir, sample_budget_s=600).judge(judge_input)+ assert len(model.requests) == 3+ assert [r.max_wait_s for r in model.requests] == [600.0, 350.0, 100.0]+++def test_the_last_attempt_still_gets_a_real_try(judge_input, cache_dir, monkeypatch):+ """A few seconds of budget would be a deadline the model cannot use; the floor is 20 s."""+ from codeverse.judges.vlm_judge import SAMPLE_MIN_WAIT_S++ clock = _fake_clock(monkeypatch)++ def nearly_spent(request):+ clock["t"] += 235.0+ return ModelError("503", retryable=True, status=503)++ model = FakeChatModel([nearly_spent, good_reply(R, IDS, 0.8)])+ j = _judge(model, cache_dir=cache_dir).judge(judge_input)+ assert j.passed and [r.max_wait_s for r in model.requests] == [240.0, SAMPLE_MIN_WAIT_S]diff --git a/harness/tests/models/test_gemini_scheduling.py b/harness/tests/models/test_gemini_scheduling.pyindex 02954bb0..1071dbff 100644--- a/harness/tests/models/test_gemini_scheduling.py+++ b/harness/tests/models/test_gemini_scheduling.py@@ -7,6 +7,7 @@ from __future__ import annotationsimport contextlib+import pytestfrom google.genai import errors as genai_errorsfrom codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart@@ -148,3 +149,36 @@ def test_png_bytes_are_not_counted_as_text():b64 = base64.b64encode(PNG_1PX * 500).decode()req = ChatRequest(messages=[ChatMessage.user("hi", images=[ImagePart(data_b64=b64)])])assert request_tokens(req) < 2000+++# ------------------------------------------------ retry budget + hedge (audit 2026-08-26 §5.1 / §5.2)+def test_max_wait_s_clips_the_retry_deadline_and_never_extends_it(monkeypatch):+ """Audit 2026-08-26 §5.1: 66 give-up spans of the model's 900 s deadline (x3 outer retries)+ were 30 % of a storm day's waiting. ``ChatRequest.max_wait_s`` is the caller's budget for+ the whole call; the model clips its deadline to it and never goes above its own ceiling."""+ import codeverse.models.gemini as gm+ from codeverse.models.retry import RETRY_DEADLINE_S++ seen: list[float] = []+ real = gm.rotate_with_retries++ def spy(pool, call, **kw):+ seen.append(kw["max_total_s"])+ return real(pool, call, **kw)++ monkeypatch.setattr(gm, "rotate_with_retries", spy)+ m, _log, _ = make_model([text_response("a"), text_response("b"), text_response("c")])+ m.generate(ChatRequest(messages=[ChatMessage.user("x")]))+ m.generate(ChatRequest(messages=[ChatMessage.user("x")], max_wait_s=30))+ m.generate(ChatRequest(messages=[ChatMessage.user("x")], max_wait_s=5000))+ assert seen == [RETRY_DEADLINE_S, 30.0, RETRY_DEADLINE_S]+++def test_max_wait_s_must_be_positive():+ """0 would mean "no deadline" to rotate_with_retries — the opposite of what a caller+ that is out of time wants — so the contract refuses it."""+ import pydantic++ with pytest.raises(pydantic.ValidationError):+ ChatRequest(messages=[ChatMessage.user("x")], max_wait_s=0)+diff --git a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py b/harness/tests/orchestrator_tracks/test_generation_planner_repair.pyindex 06c339a8..45d6657d 100644--- a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py+++ b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py@@ -180,6 +180,9 @@ def test_planner_validates_retries_and_writes(tmp_ws):assert isinstance(p, StaticPlan) and p.object_name == "DiningChair" and tmp_ws.plan_path.is_file()assert len(model.requests) == 2 and "failed validation" in model.requests[1].messages[-1].textassert model.requests[0].response_schema is not None and "PascalCase" in model.requests[0].system+ # audit 2026-08-26 §2: the plan stage waited 492 s median per storm-day run for 39 s of+ # model time; one planner call may now retry for 300 s (~4x the worst observed call), not 900+ assert all(r.max_wait_s == 300.0 for r in model.requests)# deterministic acceptance items from constraints were appendedtexts = " ".join(a.text for a in p.acceptance)assert "height = 0.820" in texts and "Includes: armrests" in texts