Official Codebase of 3DCodeVerse
Python
models: a 503 rotates through every untried key before it is a storm
gaoypengbedd25bfdb14
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.pyindex b8a5e519..ea3c46d6 100644--- a/harness/codeverse/models/retry.py+++ b/harness/codeverse/models/retry.py@@ -112,7 +112,6 @@ def rotate_with_retries[T](max_delay: float = MAX_WAIT_S,storm_attempts: int = 60,storm_max_delay: float = MAX_WAIT_S,- storm_quorum: int = 6,max_total_s: float = RETRY_DEADLINE_S,hedge: int = DEFAULT_HEDGE,sleep: Callable[[float], None] = time.sleep,@@ -140,14 +139,18 @@ def rotate_with_retries[T](* ``"429"`` → cool the key down (honouring ``retry_after``); rotation isFREE while an untried key remains (0.5 s courtesy pause), otherwise the429 counts against ``max_attempts`` with exponential backoff.- * a 503/529 is tried on a FRESH key first, for free (no sleep, no budget):- measured 2026-08-26 with one tiny request per key in parallel, three rounds- 20 s apart, during the day's "high demand" storm — gemini-3.7-flash answered- on 15/22, 18/22 and 21/22 keys while 5, 4 and 1 keys said 503 at the same- instant. A 503 is per key at any moment, so the next key is the cure and a- wait is the last resort. Only once ``storm_quorum`` distinct keys (capped- at the pool size) have 503'd within this call is it a- * **capacity storm**: that gets its own patience budget — up to ``storm_attempts`` waits with+ * ``"503"``/529 → EXACTLY like the 429 path: the key is added to this call's+ failed set and rotation to an untried key is FREE (no sleep, no budget, no+ ``max_attempts``) for as long as ANY untried key remains — a 20-key pool+ rotates 19 times before it ever waits. Measured 2026-08-26 with one tiny+ request per key in parallel, three rounds 20 s apart, during the day's+ "high demand" storm — gemini-3.7-flash answered on 15/22, 18/22 and 21/22+ keys while only 5, 4 and 1 keys said 503 at the same instant. A 503 is per+ key at any moment, so the next key is the cure and a wait is the last+ resort. Only once EVERY key in the pool has 503'd within this one logical+ call is it a+ * **capacity storm** (the whole pool is out of capacity, the backstop): that gets+ its own patience budget — up to ``storm_attempts`` waits withbackoff capped at ``storm_max_delay`` = 5 s per wait (the house rule), sopatience comes from the NUMBER of waits (60 x <=5 s ~ 5 min) rather thanfrom long sleeps that do NOT consume ``max_attempts``. Observed 2026-08-23: a@@ -318,15 +321,18 @@ def rotate_with_retries[T](raise err from exc # every key failed the same way: not the keys' faultif is_storm(err) and storm < storm_attempts and not out_of_time():failed_keys.update(t.key for t in tries)- if len(failed_keys) < min(max(1, storm_quorum), len(pool)):+ if len(failed_keys) < len(pool):# a 503 is per key at any instant (see the docstring): the next- # key is the cure — free rotation, no sleep, no storm yet+ # UNTRIED key is the cure — free rotation, no sleep, no budget,+ # no storm yet. Same rule as the 429 path: while an untried key+ # remains, rotation is free; it is only a storm once the WHOLE+ # pool has 503'd inside this one call.attempt -= 1log.warning("%s key …%s 503 (%s); rotating to a fresh key (%d/%d tried%s)",label, key[-4:], err, len(failed_keys), len(pool),f", hedging x{hedge}" if hedging else "")continue- # capacity storm: enough distinct keys said 503 in this call, so+ # capacity storm: EVERY key in the pool said 503 in this call, so# waiting is the only cure — paid from its own budget, not max_attemptsstorm += 1counts["storm"] = stormdiff --git a/harness/tests/models/test_gemini_scheduling.py b/harness/tests/models/test_gemini_scheduling.pyindex 0b12320b..22887b42 100644--- a/harness/tests/models/test_gemini_scheduling.py+++ b/harness/tests/models/test_gemini_scheduling.py@@ -106,7 +106,8 @@ def test_a_503_closes_the_shared_gate_and_the_call_still_succeeds():gate = StormGate("test", base_delay=0.0, max_wait_s=0.0)pool = KeyPool(["k1", "k2"], cooldown_s=0.0)# a 503 is per key at any instant (retry.py docstring, measured 2026-08-26): the first- # one rotates to k2 for free; only k2's 503 completes the 2-key quorum and closes the gate+ # one rotates to k2 for free; only k2's 503 — no untried key left in this 2-key pool —+ # makes it a storm and closes the gatem, _log, _ = make_model([api_error(503, "high demand"), api_error(503, "high demand"), text_response("ok")],pool=pool, max_attempts=3)m.storm_gate = gatediff --git a/harness/tests/models/test_retry.py b/harness/tests/models/test_retry.pyindex 72fc29fc..9b2636c6 100644--- a/harness/tests/models/test_retry.py+++ b/harness/tests/models/test_retry.py@@ -248,7 +248,8 @@ def test_503_storm_has_its_own_patience_budget(hedge, want_calls, want_naps):sleep=naps.append, hedge=hedge,)assert out == "ok" and calls["n"] in want_calls- # the first 503 rotates to k2 for free (no nap); k2's 503 completes the 2-key quorum+ # the first 503 rotates to k2 for free (no nap); k2's 503 leaves no untried key in the+ # 2-key pool, so that one is the stormassert len(naps) == want_naps and all(d <= 0.05 for d in naps) # the cap is the cap@@ -377,40 +378,50 @@ def test_the_deadline_does_not_cut_a_call_that_is_making_progress():assert out == "ok" and calls["n"] == 3-def test_503_rotates_to_a_fresh_key_before_it_is_a_storm():- """Measured 2026-08-26: one tiny request per key in parallel during the day's storm —- flash answered on 15/22, 18/22, 21/22 keys while 5/4/1 keys said 503 at the same instant.- A 503 is per key at any moment; the next key is the cure, the storm wait the last resort."""+def test_503_rotates_through_every_untried_key_before_it_is_a_storm():+ """Owner's rule (2026-08-27): a 503 on one key is simply replaced by the next key —+ exactly like the 429 path, free while an untried key remains. Measured 2026-08-26:+ one tiny request per key in parallel during the day's storm — flash answered on+ 15/22, 18/22, 21/22 keys while only 5/4/1 keys said 503 at the same instant. So with+ a 10-key pool, NINE consecutive 503s on distinct keys still rotate: no sleep, no+ storm, no ``max_attempts`` spent (``max_attempts=3`` here proves the budget is free)."""from codeverse.models.base import ModelErrorfrom codeverse.models.keypool import KeyPoolfrom codeverse.models.retry import rotate_with_retriesfrom codeverse.models.storm import StormGate- pool = KeyPool(["k1", "k2", "k3", "k4"], rpm_per_key=10_000)+ keys = [f"k{i}" for i in range(1, 11)]+ pool = KeyPool(keys, rpm_per_key=10_000)gate = StormGate()+ lock = threading.Lock()calls: list[str] = []naps: list[float] = []def call(key):- calls.append(key)- if key in ("k1", "k2"):+ with lock: # hedged siblings run in threads+ calls.append(key)+ nth = len(set(calls))+ if nth < len(keys): # only the LAST untried key answersraise RuntimeError("503 high demand")return "ok"out = rotate_with_retries(pool, call, classify=lambda e: ModelError(str(e), retryable=True, status=503),outcome_of=lambda e: "5xx", max_attempts=3, storm_attempts=10, storm_max_delay=0.05,sleep=naps.append, storm_gate=gate)- # k1 alone; then, with the default hedge, k2 and k3 raced at once (either may log first)- assert out == "ok" and calls[0] == "k1" and sorted(calls[1:]) == ["k2", "k3"]+ # one key per round-trip, never twice: 9 free rotations, then the 10th key lands+ assert out == "ok" and len(calls) == len(keys) and set(calls) == set(keys)assert naps == [], "rotation to a fresh key costs no sleep"- assert gate.n_storms == 0 and not gate.storming, "two keys' 503s are not a storm for the whole process"+ assert gate.n_storms == 0 and not gate.storming, "an untried key remains: not a storm"@pytest.mark.parametrize(("hedge", "want_calls", "want_hits"), [(1, {6}, 3), (2, {6, 7}, 2)])def test_503_on_every_key_is_still_a_storm(hedge, want_calls, want_hits):- """hedge=1: k1, k2, k3 fail one by one (quorum), then two storm waits, then ok.- hedge=2: k1 fails alone, k2+k3 fail together (quorum: storm 1), two more fail together- (storm 2), then the raced pair lands — one storm wait per hedged attempt, not per key."""+ """The backstop: once EVERY key in the pool has 503'd inside this one call there is+ no untried key left to rotate to, so the bounded storm wait engages.+ hedge=1: k1, k2, k3 fail one by one (the whole pool), then two storm waits, then ok.+ hedge=2: k1 fails alone, k2+k3 fail together (pool exhausted: storm 1), two more fail+ together (storm 2), then the raced pair lands — one storm wait per hedged attempt,+ not per key."""from codeverse.models.base import ModelErrorfrom codeverse.models.keypool import KeyPoolfrom codeverse.models.retry import rotate_with_retries@@ -434,7 +445,7 @@ def test_503_on_every_key_is_still_a_storm(hedge, want_calls, want_hits):outcome_of=lambda e: "5xx", max_attempts=3, storm_attempts=10, storm_max_delay=0.05,sleep=naps.append, storm_gate=gate, hedge=hedge)assert out == "ok" and n["c"] in want_calls- assert gate.n_storms == 1 and gate.n_hits == want_hits, "the storm is declared after all three keys failed, then waited out"+ assert gate.n_storms == 1 and gate.n_hits == want_hits, "the storm is declared only after all three keys failed, then waited out"# --------------------------------------------------------------------------- hedging (audit 2026-08-26 §5.2)@@ -530,7 +541,7 @@ def test_a_hedged_attempt_that_fails_on_every_key_is_one_storm_attempt():storm_attempts=10, storm_max_delay=0.05, sleep=lambda d: None, storm_gate=gate,stats=stats)assert out == "ok"- # k1 alone; k2+k3 (quorum of 3 → storm 1); two more (storm 2); the raced pair lands+ # k1 alone; k2+k3 (all 3 keys 503'd → storm 1); two more (storm 2); the raced pair landsassert stats == {"attempts": 7, "hedged": 3, "storm": 2}assert gate.n_hits == 2, "two hedged failures = two storm waits, not four"