Skip to content

Official Codebase of 3DCodeVerse

Python

models: no single retry wait exceeds 3s (was 5s)

gaoypeng1253c757a8ed
diff --git a/harness/codeverse/models/keypool.py b/harness/codeverse/models/keypool.py
index 96d5e23d..69952b55 100644
--- a/harness/codeverse/models/keypool.py
+++ b/harness/codeverse/models/keypool.py
@@ -39,12 +39,14 @@ class KeyPoolExhausted(RuntimeError):
"""Every key is cooling down / throttled and the wait budget ran out."""
-MAX_WAIT_S = 5.0
+MAX_WAIT_S = 3.0
"""The longest any single retry / cooldown wait may be (seconds).
-House rule (owner, 2026-08-24): with 22 keys there is always another key to try,
-so the harness rotates rather than sitting out a long backoff. Patience comes
-from the NUMBER of attempts, never from the length of one sleep.
+House rule (owner, 2026-08-24, tightened 5 s -> 3 s on 2026-08-27): with 22 keys
+there is always another key to try, so the harness rotates rather than sitting
+out a long backoff. Patience comes from the NUMBER of attempts, never from the
+length of one sleep. This is the single source for every wait — generic
+backoff, storm sleeps, 429 cooldowns and the storm gate's park all clip to it.
"""
ACQUIRE_TIMEOUT_S = 120.0
@@ -283,7 +285,7 @@ class KeyPool:
f"all {len(self._states)} keys throttled or dead; the earliest becomes "
f"available in {soonest - now:.0f}s (> {timeout_s}s wait budget)"
)
- wait = max(0.01, min(soonest - now, 5.0))
+ wait = max(0.01, min(soonest - now, MAX_WAIT_S))
if deadline is not None and self._clock() + wait > deadline:
raise KeyPoolExhausted(
f"all {len(self._states)} keys throttled; waited {timeout_s}s"
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.py
index f111c568..a43528a2 100644
--- a/harness/codeverse/models/retry.py
+++ b/harness/codeverse/models/retry.py
@@ -37,7 +37,7 @@ RETRY_DEADLINE_S = 900.0
#: how many keys a retry is spread over once a call has met its first 503/529.
#: Measured 2026-08-26 §5.2: a failed 503 costs the 21-50 s round-trip the provider
-#: holds before rejecting (not the <= 5 s sleep), storm streaks average 4.7 attempts,
+#: holds before rejecting (not the <= MAX_WAIT_S sleep), storm streaks average 4.7 attempts,
#: and a 503 bills nothing — so racing the next attempt on two fresh keys is free.
DEFAULT_HEDGE = 2
@@ -160,7 +160,7 @@ def rotate_with_retries[T](
call is it a
* **capacity storm** (the whole pool is out of capacity, the backstop): that gets
its own patience budget — up to ``storm_attempts`` waits with
- backoff capped at ``storm_max_delay`` = 5 s per wait (the house rule), so
+ backoff capped at ``storm_max_delay`` = ``MAX_WAIT_S`` per wait (the house rule), so
patience comes from the NUMBER of waits (60 x <=5 s ~ 5 min) rather than
from long sleeps that do NOT consume ``max_attempts``. Observed 2026-08-23: a
multi-minute gemini-3.7-flash "high demand" outage killed 8 bench runs
@@ -175,7 +175,7 @@ def rotate_with_retries[T](
FIRST 503/529 of a call onwards, every further attempt is issued on up to ``hedge``
distinct keys at once and the first success wins. Why: on the 2026-08-26 storm
day the cost of a failed 503 was the **21-50 s round-trip** the provider held
- before rejecting, not the <= 5 s sleep (logged sleep was 13 % of the wait), and
+ before rejecting, not the <= MAX_WAIT_S sleep (logged sleep was 13 % of the wait), and
storm streaks averaged **4.7 attempts** — racing two keys cuts the expected number
of rounds to ~1.7. A 503 bills nothing, so while it is storming the hedge is free;
the price is paid only when BOTH keys answer: the second success is discarded and
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index c7a97354..d974fea5 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -97,7 +97,7 @@ codeverse/
(request/response shapes per provider), parts.py; gemini.py (dead-key + free 429 rotation), gemini_image.py (ImageModel),
anthropic.py openai.py, keypool.py ('dead' outcome + TPM reservation/reconcile),
tokens.py (prompt-token estimate for the TPM bucket), pricing.py (version-suffix-only
- fallback), retry.py (MAX_WAIT_S: no single wait > 5 s), storm.py (shared 503 gate —
+ fallback), retry.py (MAX_WAIT_S: no single wait > 3 s), storm.py (shared 503 gate —
measured, ships OFF, see docs/COST.md §21), health.py (preflight probe: is the model
serving? no retries, no backoff), schema_utils.py (strict schema), registry.py
agents/ CodingAgent; gemini_cli.py claude_code.py codex.py antigravity.py api_agent.py
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 521b887a..4e5f0882 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -994,7 +994,7 @@ per-cell latency, so a long battery run in that state wants a *bigger time budge
(`--max-minutes 180`), not a smaller `--parallel`.
What *did* make storms cheap is smaller and already in the retry loop: no single wait
-may exceed `MAX_WAIT_S` = 5 s (patience comes from the number of attempts, 60 of them),
+may exceed `MAX_WAIT_S` = 3 s (5 s until 2026-08-27; patience comes from the number of attempts, 60 of them),
so a storm no longer blocks a worker for minutes — and with `--parallel 8` and
`max_in_flight 64` the other seven cells keep working through it. Deferring a
storm-hit item to the back of the queue was **not** built: at the gate-off settings a
diff --git a/harness/tests/models/test_keypool.py b/harness/tests/models/test_keypool.py
index 7f16068c..9ed1ae38 100644
--- a/harness/tests/models/test_keypool.py
+++ b/harness/tests/models/test_keypool.py
@@ -7,7 +7,7 @@ import time
import pytest
-from codeverse.models.keypool import KeyPool, KeyPoolExhausted, TokenBucket
+from codeverse.models.keypool import MAX_WAIT_S, KeyPool, KeyPoolExhausted, TokenBucket
class Clock:
@@ -71,12 +71,14 @@ def test_rpm_bucket_blocks_then_refills():
def test_all_cooling_blocks_until_cooldown_ends():
+ # a cooldown longer than the house cap is clipped to it: no single wait may
+ # exceed MAX_WAIT_S, patience comes from the number of attempts
pool, clock = make(keys=("a", "b"), cooldown_s=5)
for k in ("a", "b"):
pool.report(k, "429")
t0 = clock.t
assert pool.acquire(timeout_s=60) in ("a", "b")
- assert clock.t - t0 >= 5 - 1e-6
+ assert MAX_WAIT_S - 1e-6 <= clock.t - t0 <= 5 + 1e-6
with pytest.raises(KeyPoolExhausted):
pool.report("a", "429")
pool.report("b", "429")
diff --git a/harness/tests/models/test_storm.py b/harness/tests/models/test_storm.py
index 55e41671..b305b602 100644
--- a/harness/tests/models/test_storm.py
+++ b/harness/tests/models/test_storm.py
@@ -38,11 +38,11 @@ def test_open_gate_never_waits():
def test_a_hit_closes_the_gate_and_the_next_worker_parks():
gate, clock = make(base_delay=4.0)
closed = gate.hit()
- assert closed == 4.0
+ assert closed == min(4.0, MAX_WAIT_S) # the house cap clips the gate's own delay
assert gate.storming
waited = gate.enter()
- assert waited >= 4.0
- assert gate.parked_s >= 4.0
+ assert waited >= closed
+ assert gate.parked_s >= closed
class Parked(Exception):