Skip to content

Official Codebase of 3DCodeVerse

Python

models: generous deadlines everywhere, so nothing dies of a timeout

gaoypengd1746047c30f
diff --git a/harness/codeverse/agents/api_agent.py b/harness/codeverse/agents/api_agent.py
index 987e3759..a544534f 100644
--- a/harness/codeverse/agents/api_agent.py
+++ b/harness/codeverse/agents/api_agent.py
@@ -61,7 +61,7 @@ TURN_WAIT_MAX_S = 120.0
#: then by the session deadline, which is itself clipped to the run's wall clock.
TURN_SLOW_TOKENS_PER_S = 145.0
TURN_WAIT_OVERHEAD_S = 45.0
-TURN_WAIT_CEILING_S = 600.0
+TURN_WAIT_CEILING_S = 900.0
DEFAULT_SYSTEM = (
"You are an expert 3D programmer working inside a harness-managed workspace. "
"Use the tools to read, write and verify code under src/ and public/. Write RAW code in the "
diff --git a/harness/codeverse/config.py b/harness/codeverse/config.py
index 65cb333f..4793ba85 100644
--- a/harness/codeverse/config.py
+++ b/harness/codeverse/config.py
@@ -271,10 +271,13 @@ class Settings(BaseSettings):
# Cost dial: one name that sets model-per-role, judge samples, rounds, candidates,
# turn cap, montage size and the texture pass together (codeverse/cost/profiles.py).
model_timeout_s: float = Field(
- default=300.0,
- description="HTTP read timeout for ONE API model call (per attempt; retries multiply it). "
- "Raise it when the provider is degraded and big structured calls (planner, judge) time out "
- "before they answer: CV3D_MODEL_TIMEOUT_S=900.")
+ default=1200.0,
+ description="HTTP read timeout for ONE API model call (per attempt). It is a CEILING: the "
+ "call ends when the answer does, and gemini.py clips each attempt to the retry budget "
+ "actually left. 300 s was the binding wall once max_output_tokens went to the model's "
+ "65 536 ceiling — measured 2026-08-27, output streams at 145 tok/s p50 and 60 tok/s p10, so "
+ "a 55 800-token plan needs ~930 s and died at the socket, its tokens billed and discarded. "
+ "Owner's rule 2026-08-27: time may be generous, errors may not.")
profile: str = Field(default="balanced", description="economy | balanced | quality")
cost_ledger: bool = Field(default=True, description="append one priced row per model call "
"to the run's telemetry/cost.jsonl (or a per-process log)")
diff --git a/harness/codeverse/flywheel/captions.py b/harness/codeverse/flywheel/captions.py
index 44bc6e84..2f8b8734 100644
--- a/harness/codeverse/flywheel/captions.py
+++ b/harness/codeverse/flywheel/captions.py
@@ -185,7 +185,7 @@ def caption_sample(
problems: list[str] = []
for _attempt in range(2):
resp = model.generate(ChatRequest(messages=messages, system=system, response_schema=schema,
- temperature=0.3, thinking="low", max_wait_s=300.0, label="captioner")) # type: ignore[attr-defined]
+ temperature=0.3, thinking="low", max_wait_s=900.0, label="captioner")) # type: ignore[attr-defined]
cost += resp.usage.cost_usd
data = resp.parsed if isinstance(resp.parsed, dict) else _parse_json(resp.text)
try:
diff --git a/harness/codeverse/judges/pairwise.py b/harness/codeverse/judges/pairwise.py
index ea4cf11e..4ee50688 100644
--- a/harness/codeverse/judges/pairwise.py
+++ b/harness/codeverse/judges/pairwise.py
@@ -232,7 +232,7 @@ class PairwiseJudge:
return ChatRequest(
messages=[ChatMessage(role="user", parts=parts)], system=_SYSTEM,
response_schema=PairwiseReply.model_json_schema(), temperature=self.temperature,
- max_wait_s=240.0, label=label,
+ max_wait_s=900.0, label=label,
)
def _side_images(self, tag: str, rs: RenderSet) -> list[tuple[str, ImagePart]]:
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index b2c6f233..afc5c4c0 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -48,7 +48,7 @@ log = logging.getLogger(__name__)
#: 240 s is ~2.3x the storm p90. It replaces 3 x (a 300 s read timeout + the model's 900 s
#: retry deadline): two storm-day rounds lost 1 162 s and 927 s that way (three 300 s
#: timeouts each) before a second sample answered in 128 s.
-SAMPLE_BUDGET_S = 240.0
+SAMPLE_BUDGET_S = 900.0 # owner 2026-08-27: generous time, zero timeouts (was 240)
#: the floor of one attempt's ``max_wait_s``: a last attempt still gets a real try
SAMPLE_MIN_WAIT_S = 20.0
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.py
index a43528a2..62d28e10 100644
--- a/harness/codeverse/models/retry.py
+++ b/harness/codeverse/models/retry.py
@@ -33,7 +33,7 @@ from codeverse.models.keypool import ACQUIRE_TIMEOUT_S, MAX_WAIT_S, KeyPoolExhau
#: This is the CEILING; a caller that cannot afford it passes a smaller ``max_total_s``
#: (``ChatRequest.max_wait_s``) — audit 2026-08-26 §5.1: 66 give-up spans of 900 s, up to
#: three in a row on one agent turn, were 30 % of a storm day's waiting.
-RETRY_DEADLINE_S = 900.0
+RETRY_DEADLINE_S = 1800.0 # owner 2026-08-27: generous time, zero timeouts (was 900)
#: how many keys a retry is spread over once a call has met its first 503/529.
#: Measured 2026-08-26 §5.2: a failed 503 costs the 21-50 s round-trip the provider
diff --git a/harness/codeverse/reference/gate.py b/harness/codeverse/reference/gate.py
index 03d4869e..99cdea45 100644
--- a/harness/codeverse/reference/gate.py
+++ b/harness/codeverse/reference/gate.py
@@ -88,7 +88,7 @@ def check_plausible(
temperature=temperature,
thinking="low",
max_output_tokens=65_536,
- max_wait_s=240.0,
+ max_wait_s=900.0,
label="reference_gate",
)
try:
diff --git a/harness/codeverse/reference/mismatch.py b/harness/codeverse/reference/mismatch.py
index 45cc615a..77bc3428 100644
--- a/harness/codeverse/reference/mismatch.py
+++ b/harness/codeverse/reference/mismatch.py
@@ -85,7 +85,7 @@ def compare(
images += [ImagePart(path=str(p), label=f"RENDER {i + 1} ({p.stem})") for i, p in enumerate(rens)]
req = ChatRequest(messages=[ChatMessage.user(text, images=images)], system=DIFF_SYSTEM,
response_schema=DiffAnswer.model_json_schema(), temperature=temperature,
- thinking="low", max_output_tokens=65_536, max_wait_s=240.0, label="reference_diff")
+ thinking="low", max_output_tokens=65_536, max_wait_s=900.0, label="reference_diff")
try:
resp = model.generate(req)
except ModelError as e:
diff --git a/harness/codeverse/reference/synth.py b/harness/codeverse/reference/synth.py
index 9d3264f8..c97e0f54 100644
--- a/harness/codeverse/reference/synth.py
+++ b/harness/codeverse/reference/synth.py
@@ -93,7 +93,7 @@ def image_prompt_plan(spec: Spec, view_names: list[str], *, model: Any, temperat
text = IMAGE_PROMPT_USER.format(brief=brief_text(spec), n_views=len(view_names), views=", ".join(view_names))
req = ChatRequest(messages=[ChatMessage.user(text)], system=IMAGE_PROMPT_SYSTEM,
response_schema=ImagePromptPlan.model_json_schema(), temperature=temperature,
- thinking="low", max_output_tokens=65_536, max_wait_s=300.0, label="reference_prompt")
+ thinking="low", max_output_tokens=65_536, max_wait_s=900.0, label="reference_prompt")
try:
resp = model.generate(req)
except ModelError as e:
diff --git a/harness/codeverse/texturing/plan.py b/harness/codeverse/texturing/plan.py
index 514d22b7..aa772b04 100644
--- a/harness/codeverse/texturing/plan.py
+++ b/harness/codeverse/texturing/plan.py
@@ -319,7 +319,7 @@ def material_plan(
messages=[ChatMessage.user(text, images=[ImagePart(path=str(p), label=p.stem) for p in images])],
system="You plan PBR textures for 3D assets. Answer with JSON only.",
response_schema=PlannerOutput.model_json_schema(), temperature=temperature, thinking="low",
- max_output_tokens=65_536, max_wait_s=300.0, label="texture_plan",
+ max_output_tokens=65_536, max_wait_s=900.0, label="texture_plan",
)
resp = model.generate(req)
payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
diff --git a/harness/codeverse/texturing/scene_pack.py b/harness/codeverse/texturing/scene_pack.py
index 217c003d..3f881075 100644
--- a/harness/codeverse/texturing/scene_pack.py
+++ b/harness/codeverse/texturing/scene_pack.py
@@ -148,7 +148,7 @@ def scene_pack_plan(
environment=plan.environment, zones=zones, assets=assets)
req = ChatRequest(messages=[ChatMessage.user(text)], system="You plan texture packs for 3D scenes. JSON only.",
response_schema=PackOutput.model_json_schema(), temperature=temperature, thinking="low",
- max_output_tokens=65_536, max_wait_s=300.0, label="scene_texture_pack")
+ max_output_tokens=65_536, max_wait_s=900.0, label="scene_texture_pack")
resp = model.generate(req)
payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
try:
diff --git a/harness/codeverse/tracks/brief.py b/harness/codeverse/tracks/brief.py
index 9a81b32f..c81ab892 100644
--- a/harness/codeverse/tracks/brief.py
+++ b/harness/codeverse/tracks/brief.py
@@ -133,7 +133,7 @@ def expand_brief(spec: Spec, model_id: str, *, model: Any | None = None, events:
resp = model.generate(ChatRequest(
messages=[ChatMessage.user(user, images=images or None)], system=system,
response_schema=EngineeringBrief.model_json_schema(), temperature=0.3, thinking="low",
- max_output_tokens=BRIEF_MAX_TOKENS, max_wait_s=300.0, label="planner-brief"))
+ max_output_tokens=BRIEF_MAX_TOKENS, max_wait_s=900.0, label="planner-brief"))
usage = usage + resp.usage
# parse_json_lenient, not json.loads: it tolerates fences/prose AND strips the
# C0 controls a model can emit, which json.loads would carry into the brief.
diff --git a/harness/codeverse/tracks/planner.py b/harness/codeverse/tracks/planner.py
index df9501bf..a28c1246 100644
--- a/harness/codeverse/tracks/planner.py
+++ b/harness/codeverse/tracks/planner.py
@@ -65,7 +65,7 @@ TRUNCATION_GROWTH = 1.5
#: call itself is 13.7 s p50 / 32 s p90 / 76 s max (23.4 / 43 / 68 under the storm), yet the
#: storm-day plan stage waited 492 s median per run for 39 s of model time; 300 s is ~4x the
#: worst observed call and replaces the model's 900 s default. The re-asks are on top.
-PLAN_MAX_WAIT_S = 300.0
+PLAN_MAX_WAIT_S = 900.0 # owner's floor 2026-08-27: 15 minutes before a plan may be cut off
#: ...but a deadline must fit the ANSWER it asked for, or the tokens are billed and then
#: thrown away on a timeout. Measured over 395 real planner calls (2026-08-27): output
#: runs at 145 tok/s p50 and 60 tok/s p10, and the largest answers seen were 32 k tokens
diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.py
index 666536f5..b1e7a677 100644
--- a/harness/tests/judges/test_vlm_judge.py
+++ b/harness/tests/judges/test_vlm_judge.py
@@ -252,15 +252,17 @@ def test_each_sample_has_a_retry_budget_and_hands_what_is_left_to_the_model(judg
model = FakeChatModel([slow_503, slow_503, good_reply(R, IDS, 0.8)])
j = _judge(model, cache_dir=cache_dir).judge(judge_input)
assert j.passed and len(model.requests) == 3
- assert SAMPLE_BUDGET_S == 240.0
- assert [r.max_wait_s for r in model.requests] == [240.0, 140.0, 40.0]
+ # each retry gets what is LEFT of the sample budget (the fake clock burns 100 s a call)
+ assert [r.max_wait_s for r in model.requests] == [SAMPLE_BUDGET_S, SAMPLE_BUDGET_S - 100.0, SAMPLE_BUDGET_S - 200.0]
def test_a_sample_stops_when_its_budget_is_spent(judge_input, cache_dir, monkeypatch):
+ from codeverse.judges.vlm_judge import SAMPLE_BUDGET_S
+
clock = _fake_clock(monkeypatch)
def spent(request):
- clock["t"] += 250.0 # one attempt that retried inside the model for the whole budget
+ clock["t"] += SAMPLE_BUDGET_S + 10.0 # one attempt that retried for the whole budget
return ModelError("503 high demand", retryable=True, status=503)
model = FakeChatModel(default=spent)
@@ -269,21 +271,23 @@ def test_a_sample_stops_when_its_budget_is_spent(judge_input, cache_dir, monkeyp
assert "sample budget" in j.summary
# the budget is a dial: a caller that can afford more gets the attempts back
model = FakeChatModel(default=spent)
- j = _judge(model, cache_dir=cache_dir, sample_budget_s=600).judge(judge_input)
+ roomy = 3 * (SAMPLE_BUDGET_S + 10.0)
+ j = _judge(model, cache_dir=cache_dir, sample_budget_s=roomy).judge(judge_input)
assert len(model.requests) == 3
- assert [r.max_wait_s for r in model.requests] == [600.0, 350.0, 100.0]
+ burn = SAMPLE_BUDGET_S + 10.0
+ assert [r.max_wait_s for r in model.requests] == [roomy, roomy - burn, roomy - 2 * burn]
def test_the_last_attempt_still_gets_a_real_try(judge_input, cache_dir, monkeypatch):
"""A few seconds of budget would be a deadline the model cannot use; the floor is 20 s."""
- from codeverse.judges.vlm_judge import SAMPLE_MIN_WAIT_S
+ from codeverse.judges.vlm_judge import SAMPLE_BUDGET_S, SAMPLE_MIN_WAIT_S
clock = _fake_clock(monkeypatch)
def nearly_spent(request):
- clock["t"] += 235.0
+ clock["t"] += SAMPLE_BUDGET_S - 5.0
return ModelError("503", retryable=True, status=503)
model = FakeChatModel([nearly_spent, good_reply(R, IDS, 0.8)])
j = _judge(model, cache_dir=cache_dir).judge(judge_input)
- assert j.passed and [r.max_wait_s for r in model.requests] == [240.0, SAMPLE_MIN_WAIT_S]
+ assert j.passed and [r.max_wait_s for r in model.requests] == [SAMPLE_BUDGET_S, SAMPLE_MIN_WAIT_S]
diff --git a/harness/tests/models/test_gemini.py b/harness/tests/models/test_gemini.py
index e1907795..d9e8f011 100644
--- a/harness/tests/models/test_gemini.py
+++ b/harness/tests/models/test_gemini.py
@@ -101,12 +101,12 @@ def make_model(script: list[Any], keys=("k1", "k2", "k3"), pool: KeyPool | None
def test_read_timeout_comes_from_settings(monkeypatch):
"""CV3D_MODEL_TIMEOUT_S raises the per-attempt HTTP read timeout (degraded-provider escape hatch)."""
- from codeverse.config import get_settings
+ from codeverse.config import Settings, get_settings
from codeverse.models.gemini import GeminiModel
monkeypatch.setenv("GEMINI_API_KEYS", "k1")
get_settings.cache_clear()
- assert GeminiModel("gemini-3.7-flash").timeout_s == 300.0
+ assert GeminiModel("gemini-3.7-flash").timeout_s == Settings().model_timeout_s
monkeypatch.setenv("CV3D_MODEL_TIMEOUT_S", "900")
get_settings.cache_clear()
assert GeminiModel("gemini-3.7-flash").timeout_s == 900.0