Skip to content

Official Codebase of 3DCodeVerse

Python

models: a 503 rotates through every untried key before it is a storm

gaoypengbedd25bfdb14
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.py
index b8a5e519..ea3c46d6 100644
--- a/harness/codeverse/models/retry.py
+++ b/harness/codeverse/models/retry.py
@@ -112,7 +112,6 @@ def rotate_with_retries[T](
max_delay: float = MAX_WAIT_S,
storm_attempts: int = 60,
storm_max_delay: float = MAX_WAIT_S,
- storm_quorum: int = 6,
max_total_s: float = RETRY_DEADLINE_S,
hedge: int = DEFAULT_HEDGE,
sleep: Callable[[float], None] = time.sleep,
@@ -140,14 +139,18 @@ def rotate_with_retries[T](
* ``"429"`` → cool the key down (honouring ``retry_after``); rotation is
FREE while an untried key remains (0.5 s courtesy pause), otherwise the
429 counts against ``max_attempts`` with exponential backoff.
- * a 503/529 is tried on a FRESH key first, for free (no sleep, no budget):
- measured 2026-08-26 with one tiny request per key in parallel, three rounds
- 20 s apart, during the day's "high demand" storm — gemini-3.7-flash answered
- on 15/22, 18/22 and 21/22 keys while 5, 4 and 1 keys said 503 at the same
- instant. A 503 is per key at any moment, so the next key is the cure and a
- wait is the last resort. Only once ``storm_quorum`` distinct keys (capped
- at the pool size) have 503'd within this call is it a
- * **capacity storm**: that gets its own patience budget — up to ``storm_attempts`` waits with
+ * ``"503"``/529 → EXACTLY like the 429 path: the key is added to this call's
+ failed set and rotation to an untried key is FREE (no sleep, no budget, no
+ ``max_attempts``) for as long as ANY untried key remains — a 20-key pool
+ rotates 19 times before it ever waits. Measured 2026-08-26 with one tiny
+ request per key in parallel, three rounds 20 s apart, during the day's
+ "high demand" storm — gemini-3.7-flash answered on 15/22, 18/22 and 21/22
+ keys while only 5, 4 and 1 keys said 503 at the same instant. A 503 is per
+ key at any moment, so the next key is the cure and a wait is the last
+ resort. Only once EVERY key in the pool has 503'd within this one logical
+ call is it a
+ * **capacity storm** (the whole pool is out of capacity, the backstop): that gets
+ its own patience budget — up to ``storm_attempts`` waits with
backoff capped at ``storm_max_delay`` = 5 s per wait (the house rule), so
patience comes from the NUMBER of waits (60 x <=5 s ~ 5 min) rather than
from long sleeps that do NOT consume ``max_attempts``. Observed 2026-08-23: a
@@ -318,15 +321,18 @@ def rotate_with_retries[T](
raise err from exc # every key failed the same way: not the keys' fault
if is_storm(err) and storm < storm_attempts and not out_of_time():
failed_keys.update(t.key for t in tries)
- if len(failed_keys) < min(max(1, storm_quorum), len(pool)):
+ if len(failed_keys) < len(pool):
# a 503 is per key at any instant (see the docstring): the next
- # key is the cure — free rotation, no sleep, no storm yet
+ # UNTRIED key is the cure — free rotation, no sleep, no budget,
+ # no storm yet. Same rule as the 429 path: while an untried key
+ # remains, rotation is free; it is only a storm once the WHOLE
+ # pool has 503'd inside this one call.
attempt -= 1
log.warning("%s key …%s 503 (%s); rotating to a fresh key (%d/%d tried%s)",
label, key[-4:], err, len(failed_keys), len(pool),
f", hedging x{hedge}" if hedging else "")
continue
- # capacity storm: enough distinct keys said 503 in this call, so
+ # capacity storm: EVERY key in the pool said 503 in this call, so
# waiting is the only cure — paid from its own budget, not max_attempts
storm += 1
counts["storm"] = storm
diff --git a/harness/tests/models/test_gemini_scheduling.py b/harness/tests/models/test_gemini_scheduling.py
index 0b12320b..22887b42 100644
--- a/harness/tests/models/test_gemini_scheduling.py
+++ b/harness/tests/models/test_gemini_scheduling.py
@@ -106,7 +106,8 @@ def test_a_503_closes_the_shared_gate_and_the_call_still_succeeds():
gate = StormGate("test", base_delay=0.0, max_wait_s=0.0)
pool = KeyPool(["k1", "k2"], cooldown_s=0.0)
# a 503 is per key at any instant (retry.py docstring, measured 2026-08-26): the first
- # one rotates to k2 for free; only k2's 503 completes the 2-key quorum and closes the gate
+ # one rotates to k2 for free; only k2's 503 — no untried key left in this 2-key pool —
+ # makes it a storm and closes the gate
m, _log, _ = make_model([api_error(503, "high demand"), api_error(503, "high demand"), text_response("ok")],
pool=pool, max_attempts=3)
m.storm_gate = gate
diff --git a/harness/tests/models/test_retry.py b/harness/tests/models/test_retry.py
index 72fc29fc..9b2636c6 100644
--- a/harness/tests/models/test_retry.py
+++ b/harness/tests/models/test_retry.py
@@ -248,7 +248,8 @@ def test_503_storm_has_its_own_patience_budget(hedge, want_calls, want_naps):
sleep=naps.append, hedge=hedge,
)
assert out == "ok" and calls["n"] in want_calls
- # the first 503 rotates to k2 for free (no nap); k2's 503 completes the 2-key quorum
+ # the first 503 rotates to k2 for free (no nap); k2's 503 leaves no untried key in the
+ # 2-key pool, so that one is the storm
assert len(naps) == want_naps and all(d <= 0.05 for d in naps) # the cap is the cap
@@ -377,40 +378,50 @@ def test_the_deadline_does_not_cut_a_call_that_is_making_progress():
assert out == "ok" and calls["n"] == 3
-def test_503_rotates_to_a_fresh_key_before_it_is_a_storm():
- """Measured 2026-08-26: one tiny request per key in parallel during the day's storm —
- flash answered on 15/22, 18/22, 21/22 keys while 5/4/1 keys said 503 at the same instant.
- A 503 is per key at any moment; the next key is the cure, the storm wait the last resort."""
+def test_503_rotates_through_every_untried_key_before_it_is_a_storm():
+ """Owner's rule (2026-08-27): a 503 on one key is simply replaced by the next key —
+ exactly like the 429 path, free while an untried key remains. Measured 2026-08-26:
+ one tiny request per key in parallel during the day's storm — flash answered on
+ 15/22, 18/22, 21/22 keys while only 5/4/1 keys said 503 at the same instant. So with
+ a 10-key pool, NINE consecutive 503s on distinct keys still rotate: no sleep, no
+ storm, no ``max_attempts`` spent (``max_attempts=3`` here proves the budget is free)."""
from codeverse.models.base import ModelError
from codeverse.models.keypool import KeyPool
from codeverse.models.retry import rotate_with_retries
from codeverse.models.storm import StormGate
- pool = KeyPool(["k1", "k2", "k3", "k4"], rpm_per_key=10_000)
+ keys = [f"k{i}" for i in range(1, 11)]
+ pool = KeyPool(keys, rpm_per_key=10_000)
gate = StormGate()
+ lock = threading.Lock()
calls: list[str] = []
naps: list[float] = []
def call(key):
- calls.append(key)
- if key in ("k1", "k2"):
+ with lock: # hedged siblings run in threads
+ calls.append(key)
+ nth = len(set(calls))
+ if nth < len(keys): # only the LAST untried key answers
raise RuntimeError("503 high demand")
return "ok"
out = rotate_with_retries(pool, call, classify=lambda e: ModelError(str(e), retryable=True, status=503),
outcome_of=lambda e: "5xx", max_attempts=3, storm_attempts=10, storm_max_delay=0.05,
sleep=naps.append, storm_gate=gate)
- # k1 alone; then, with the default hedge, k2 and k3 raced at once (either may log first)
- assert out == "ok" and calls[0] == "k1" and sorted(calls[1:]) == ["k2", "k3"]
+ # one key per round-trip, never twice: 9 free rotations, then the 10th key lands
+ assert out == "ok" and len(calls) == len(keys) and set(calls) == set(keys)
assert naps == [], "rotation to a fresh key costs no sleep"
- assert gate.n_storms == 0 and not gate.storming, "two keys' 503s are not a storm for the whole process"
+ assert gate.n_storms == 0 and not gate.storming, "an untried key remains: not a storm"
@pytest.mark.parametrize(("hedge", "want_calls", "want_hits"), [(1, {6}, 3), (2, {6, 7}, 2)])
def test_503_on_every_key_is_still_a_storm(hedge, want_calls, want_hits):
- """hedge=1: k1, k2, k3 fail one by one (quorum), then two storm waits, then ok.
- hedge=2: k1 fails alone, k2+k3 fail together (quorum: storm 1), two more fail together
- (storm 2), then the raced pair lands — one storm wait per hedged attempt, not per key."""
+ """The backstop: once EVERY key in the pool has 503'd inside this one call there is
+ no untried key left to rotate to, so the bounded storm wait engages.
+ hedge=1: k1, k2, k3 fail one by one (the whole pool), then two storm waits, then ok.
+ hedge=2: k1 fails alone, k2+k3 fail together (pool exhausted: storm 1), two more fail
+ together (storm 2), then the raced pair lands — one storm wait per hedged attempt,
+ not per key."""
from codeverse.models.base import ModelError
from codeverse.models.keypool import KeyPool
from codeverse.models.retry import rotate_with_retries
@@ -434,7 +445,7 @@ def test_503_on_every_key_is_still_a_storm(hedge, want_calls, want_hits):
outcome_of=lambda e: "5xx", max_attempts=3, storm_attempts=10, storm_max_delay=0.05,
sleep=naps.append, storm_gate=gate, hedge=hedge)
assert out == "ok" and n["c"] in want_calls
- assert gate.n_storms == 1 and gate.n_hits == want_hits, "the storm is declared after all three keys failed, then waited out"
+ assert gate.n_storms == 1 and gate.n_hits == want_hits, "the storm is declared only after all three keys failed, then waited out"
# --------------------------------------------------------------------------- hedging (audit 2026-08-26 §5.2)
@@ -530,7 +541,7 @@ def test_a_hedged_attempt_that_fails_on_every_key_is_one_storm_attempt():
storm_attempts=10, storm_max_delay=0.05, sleep=lambda d: None, storm_gate=gate,
stats=stats)
assert out == "ok"
- # k1 alone; k2+k3 (quorum of 3 → storm 1); two more (storm 2); the raced pair lands
+ # k1 alone; k2+k3 (all 3 keys 503'd → storm 1); two more (storm 2); the raced pair lands
assert stats == {"attempts": 7, "hedged": 3, "storm": 2}
assert gate.n_hits == 2, "two hedged failures = two storm waits, not four"