Official Codebase of 3DCodeVerse
Python
retry: hedge the retry of a 503 across two keys (rotate_with_retries hedge=2, Settings.rate.hedge)
gaoypeng6401bb4adea7
diff --git a/harness/codeverse/config.py b/harness/codeverse/config.pyindex 1aeb2983..a69c92f3 100644--- a/harness/codeverse/config.py+++ b/harness/codeverse/config.py@@ -132,6 +132,13 @@ class Rate(BaseModel):"intermittent rather than a clean outage, so parking every worker starves the unlucky call ""(docs/COST.md §21). The mechanism and its counters are kept so the experiment is ""reproducible: CV3D_RATE__STORM_GATE=1, or bench/concurrency_probe.py --storm-gate.")+ hedge: int = Field(+ default=2, ge=1,+ description="keys a retry is raced on once a call has met its first 503 (1 = off). Measured "+ "2026-08-26: a failed 503 costs the 21-50 s round-trip the provider holds before rejecting, "+ "storm streaks average 4.7 attempts, and a 503 bills nothing — so the hedge is free while it "+ "storms and wastes one call only when both keys answer (docs/COST.md §24). "+ "CV3D_RATE__HEDGE=1 for the A/B.")class Judge(BaseModel):diff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.pyindex a5c09227..7fb37595 100644--- a/harness/codeverse/models/gemini.py+++ b/harness/codeverse/models/gemini.py@@ -183,6 +183,7 @@ class GeminiModel:tpm_per_key: int | None = None,storm_gate: StormGate | None = None,storm_attempts: int | None = None,+ hedge: int | None = None,sleep: Callable[[float], None] = time.sleep,client_factory: Callable[[str], Any] | None = None,) -> None:@@ -191,6 +192,9 @@ class GeminiModel:#: whose whole point is a fast verdict — the storm branch does NOT consume#: max_attempts, so max_attempts=1 alone still retried a 503 for up to 15 min.self.storm_attempts = storm_attempts+ #: keys a retry is raced on after the call's first 503 (``Settings.rate.hedge``,+ #: 2; 1 = off) — see ``rotate_with_retries`` and docs/COST.md §24+ self.hedge = max(1, int(_rate().hedge if hedge is None else hedge))keys = list(keys) if keys is not None else _default_keys()if pool is None and not keys:raise ModelError(@@ -256,6 +260,7 @@ class GeminiModel:base_delay=self.base_delay,max_delay=self.max_delay,max_total_s=budget,+ hedge=self.hedge,sleep=self._sleep,on_free_retry=downgrade_thinking,retry_after=_retry_after_s,diff --git a/harness/codeverse/models/gemini_image.py b/harness/codeverse/models/gemini_image.pyindex fd34d15c..5593df7d 100644--- a/harness/codeverse/models/gemini_image.py+++ b/harness/codeverse/models/gemini_image.py@@ -149,11 +149,15 @@ class GeminiImageModel:max_attempts: int = 4,base_delay: float = 1.0,max_delay: float = MAX_WAIT_S,+ hedge: int = 1,sleep: Callable[[float], None] = time.sleep,client_factory: Callable[[str], Any] | None = None,) -> None:self.model = modelself.fallback = fallback if fallback and fallback != model else None+ #: no 503 hedge by default: an image is billed per image, so a hedge that+ #: lands twice pays for two of them (the chat models default to 2)+ self.hedge = max(1, int(hedge))keys = list(keys) if keys is not None else _default_keys()if pool is None and not keys:raise ModelError("no Gemini API keys configured (GEMINI_API_KEYS / ~/.config/astra3d/gemini_keys.env)")@@ -260,6 +264,7 @@ class GeminiImageModel:max_attempts=self.max_attempts,base_delay=self.base_delay,max_delay=self.max_delay,+ hedge=self.hedge,sleep=self._sleep,retry_after=_retry_after_s,tokens_of=lambda r: r[1].input_tokens,diff --git a/harness/codeverse/models/keypool.py b/harness/codeverse/models/keypool.pyindex 202de9f9..00f62058 100644--- a/harness/codeverse/models/keypool.py+++ b/harness/codeverse/models/keypool.py@@ -9,7 +9,8 @@ Used by ``GeminiModel`` (22 keys on the owner's box) but provider-neutral.200 k-token judge verdict and a 2 k-token caption are scheduled differently(``docs/COST.md`` Part III); ``max_in_flight`` additionally caps how many callsmay be out at once, and :meth:`KeyPool.release` (a ``finally`` in- ``rotate_with_retries``) hands the slot back.+ ``rotate_with_retries``) hands the slot back. ``try_acquire()`` is the+ never-waiting variant a hedged retry uses for its extra keys.* ``report(key, outcome)`` feeds back ``ok | 429 | 5xx | error | dead`` so thepool can cool a key down and adjust its health score. ``dead`` is forkey-scoped auth/permission failures (revoked / suspended / invalid key): the@@ -203,6 +204,32 @@ class KeyPool:self._slots.release()raise+ def try_acquire(+ self,+ *,+ tokens_hint: int = 0,+ exclude: set[str] | frozenset[str] | None = None,+ ) -> str | None:+ """:meth:`acquire` that never waits: a key usable *right now* (and a free+ ``max_in_flight`` slot), else ``None`` — nothing is reserved or counted then.++ For the extra keys of a hedged retry (``rotate_with_retries(hedge=...)``): the+ primary request is already in flight, so a partner that is not free at once+ is not worth waiting for. A returned key holds a slot like any other and+ must be :meth:`release`-d."""+ if self._slots is not None and not self._slots.acquire(blocking=False):+ return None+ try:+ return self._acquire_key(tokens_hint, exclude, 0.0)+ except KeyPoolExhausted:+ if self._slots is not None:+ self._slots.release()+ return None+ except BaseException:+ if self._slots is not None:+ self._slots.release()+ raise+def _acquire_key(self,tokens_hint: int,diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.pyindex d35bbcff..b8a5e519 100644--- a/harness/codeverse/models/retry.py+++ b/harness/codeverse/models/retry.py@@ -2,8 +2,8 @@key-rotation state machine used by the Gemini backends.* :func:`with_retries` — plain bounded retry; no provider knowledge.-* :func:`rotate_with_retries` — the dead-key / 429-rotation / backoff machine- over a :class:`~codeverse.models.keypool.KeyPool`. Provider specifics+* :func:`rotate_with_retries` — the dead-key / 429-rotation / 503-hedge / backoff+ machine over a :class:`~codeverse.models.keypool.KeyPool`. Provider specifics(exception classification, outcome mapping, retry-after extraction) areinjected as hooks, so this module stays provider-neutral.@@ -15,8 +15,10 @@ from __future__ import annotationsimport loggingimport randomimport time-from collections.abc import Callable-from typing import TYPE_CHECKING, TypeVar+from collections.abc import Callable, Sequence+from concurrent.futures import Future, ThreadPoolExecutor, as_completed+from dataclasses import dataclass+from typing import TYPE_CHECKING, Any, TypeVarfrom codeverse.models.keypool import MAX_WAIT_S, KeyPoolExhausted@@ -27,8 +29,17 @@ from codeverse.models.keypool import MAX_WAIT_S, KeyPoolExhausted#: when a call is BILLED (a stalled call bills nothing), let a run with --max-minutes 90#: reach 200 minutes during a flash outage. Past this deadline the call gives up and the#: cell is recorded infra_failed, which is re-runnable (`--redo-status infra_failed`).+#: This is the CEILING; a caller that cannot afford it passes a smaller ``max_total_s``+#: (``ChatRequest.max_wait_s``) — audit 2026-08-26 §5.1: 66 give-up spans of 900 s, up to+#: three in a row on one agent turn, were 30 % of a storm day's waiting.RETRY_DEADLINE_S = 900.0+#: how many keys a retry is spread over once a call has met its first 503/529.+#: Measured 2026-08-26 §5.2: a failed 503 costs the 21-50 s round-trip the provider+#: holds before rejecting (not the <= 5 s sleep), storm streaks average 4.7 attempts,+#: and a 503 bills nothing — so racing the next attempt on two fresh keys is free.+DEFAULT_HEDGE = 2+if TYPE_CHECKING: # pragma: no coverfrom codeverse.models.base import ModelErrorfrom codeverse.models.keypool import KeyPool, Outcome@@ -103,6 +114,7 @@ def rotate_with_retries[T](storm_max_delay: float = MAX_WAIT_S,storm_quorum: int = 6,max_total_s: float = RETRY_DEADLINE_S,+ hedge: int = DEFAULT_HEDGE,sleep: Callable[[float], None] = time.sleep,monotonic: Callable[[], float] = time.monotonic,on_free_retry: Callable[[ModelError], bool] | None = None,@@ -111,6 +123,7 @@ def rotate_with_retries[T](tokens_hint: int = 0,storm_gate: StormGate | None = None,label: str = "model",+ stats: dict[str, Any] | None = None,) -> T:"""Run ``call(key)`` against a rotating :class:`KeyPool` until it succeeds.@@ -120,7 +133,7 @@ def rotate_with_retries[T](looked dead during this call are benched (another key proved the requestitself is fine) and the result is returned.* ``on_free_retry(err)`` returning ``True`` → retry at once, for free- (no report, no sleep) — e.g. a thinking-config downgrade.+ (no sleep, no budget) — e.g. a thinking-config downgrade.* ``outcome_of(err) == "dead"`` (key-scoped auth failure) → rotate at oncefor free; the key is only benched once a sibling key proves the requestis fine. Every key failing the same way raises without benching.@@ -146,14 +159,43 @@ def rotate_with_retries[T](* other retryable errors → exponential backoff + jitter until``max_attempts``; non-retryable errors raise immediately.+ **Hedging** (``hedge``, default :data:`DEFAULT_HEDGE`; ``1`` disables). From the+ FIRST 503/529 of a call onwards, every further attempt is issued on up to ``hedge``+ distinct keys at once and the first success wins. Why: on the 2026-08-26 storm+ day the cost of a failed 503 was the **21-50 s round-trip** the provider held+ before rejecting, not the <= 5 s sleep (logged sleep was 13 % of the wait), and+ storm streaks averaged **4.7 attempts** — racing two keys cuts the expected number+ of rounds to ~1.7. A 503 bills nothing, so while it is storming the hedge is free;+ the price is paid only when BOTH keys answer: the second success is discarded and+ that call's tokens are wasted (a few cents for a chat turn — which is why+ ``GeminiImageModel`` passes ``hedge=1``: an image is billed per image).+ Mechanics: the attempt's first key is acquired as usual (it may wait for a slot or+ a cooldown); the extra keys come from :meth:`KeyPool.try_acquire` and are simply+ skipped when no fresh key or ``max_in_flight`` slot is free *right now*, so a hedge+ never waits for a partner and every hedged request holds its own slot. The calls+ run in a per-attempt ``ThreadPoolExecutor`` that is shut down without waiting: a+ loser keeps running until its own round-trip ends, then reports its outcome to+ the pool (a late success still reports its tokens) and releases its slot; the+ winner is returned the moment it lands. When every hedged key fails, the attempt+ is ONE attempt for the storm / backoff accounting and the worst error decides the+ branch (a non-retryable error first, then a plain retryable one, a 429, a dead+ key, and a 503 last).+``tokens_hint`` is the estimated **prompt** tokens of the pending call(:func:`codeverse.models.tokens.request_tokens`); the pool reserves them inthe per-key TPM bucket at ``acquire`` and the reservation is reconciledagainst the provider's real count on ``report``, so a 200 k-token judge calland a 2 k-token caption are scheduled differently.++ ``stats`` (optional, caller-owned dict) receives the call's telemetry on the way+ out — success or failure: ``attempts`` = round-trips issued (hedged siblings+ included; 1 = clean), ``hedged`` = hedged attempts, ``storm`` = storm waits.+ ``GeminiModel`` copies ``attempts`` into ``ChatResponse.raw`` and onto the raised+ ``ModelError`` so the cost ledger can say how hard each call was."""- failed_keys: set[str] = set() # keys that 429'd or looked dead during this call+ failed_keys: set[str] = set() # keys that 429'd / 503'd or looked dead during this calldead_keys: set[str] = set()+ counts = {"attempts": 0, "hedged": 0, "storm": 0}def bench() -> None:# mark keys that failed with key-scoped errors dead — called once the call@@ -162,69 +204,132 @@ def rotate_with_retries[T](pool.report(k, "dead")dead_keys.clear()+ def run_one(key: str) -> _Try:+ """One round-trip on ``key``: report its outcome to the pool and hand the slot+ back. Runs inline for a single key and in a worker thread when hedged, so it+ touches none of this call's state and never raises a ``ModelError`` itself."""+ try:+ try:+ result = call(key)+ except Exception as exc: # noqa: BLE001 - classification is delegated+ err = classify(exc)+ outcome = outcome_of(err)+ # a key that looks dead is reported "error" now and benched only once a+ # sibling proves the request itself is fine (bench())+ pool.report(+ key,+ "error" if outcome == "dead" else outcome,+ reserved=tokens_hint,+ retry_after_s=retry_after(exc) if (outcome == "429" and retry_after) else None,+ )+ return _Try(key, None, err, exc, outcome)+ pool.report(key, "ok", tokens=tokens_of(result) if tokens_of is not None else 0,+ reserved=tokens_hint)+ return _Try(key, result, None, None, "ok")+ finally:+ pool.release()++ def run_attempt(keys: Sequence[str]) -> list[_Try]:+ """Issue ``call`` on every key; return the tries that completed up to and+ including the first success, in completion order. Losers still in flight+ finish on their own (``run_one`` reports and releases for them)."""+ counts["attempts"] += len(keys)+ if len(keys) == 1:+ return [run_one(keys[0])]+ counts["hedged"] += 1+ executor = ThreadPoolExecutor(max_workers=len(keys), thread_name_prefix=f"hedge {label}")+ futures = [executor.submit(run_one, k) for k in keys]+ executor.shutdown(wait=False) # no new work; the threads end with their round-trips+ done: list[_Try] = []+ for fut in as_completed(futures):+ t = fut.result()+ done.append(t)+ if t.err is None:+ for other in futures:+ if not other.done():+ other.add_done_callback(_discard_loser)+ break+ return done+last_err: ModelError | None = Noneattempt = 0storm = 0+ hedging = False # flips on the first 503/529; every later attempt is hedgeddeadline = monotonic() + max_total_s if max_total_s > 0 else float("inf")def out_of_time() -> bool:return monotonic() >= deadline- while attempt < max_attempts:- attempt += 1- # never go back to a key that looked dead this call; throttled keys are- # excluded while an untried one remains, else acquire() waits for a cooldown- exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())- if storm_gate is not None:- storm_gate.enter()- try:- key = pool.acquire(exclude=exclude, tokens_hint=tokens_hint)- except KeyPoolExhausted as exc:- raise classify(exc) from exc- try:- result = call(key)- pool.report(key, "ok", tokens=tokens_of(result) if tokens_of is not None else 0,- reserved=tokens_hint)+ def is_storm(err: ModelError) -> bool:+ return bool(err.retryable and err.status in (503, 529))++ try:+ while attempt < max_attempts:+ attempt += 1+ # never go back to a key that looked dead this call; throttled keys are+ # excluded while an untried one remains, else acquire() waits for a cooldown+ exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())if storm_gate is not None:- storm_gate.ok()- bench()- return result- except Exception as exc: # noqa: BLE001 - classification is delegated- err = classify(exc)+ storm_gate.enter()+ try:+ keys = [pool.acquire(exclude=exclude, tokens_hint=tokens_hint)]+ except KeyPoolExhausted as exc:+ raise classify(exc) from exc+ while hedging and len(keys) < max(1, hedge):+ extra = pool.try_acquire(exclude=exclude | set(keys), tokens_hint=tokens_hint)+ if extra is None:+ break # no fresh key / slot right now: never wait for a hedge partner+ keys.append(extra)+ tries = run_attempt(keys)+ # bookkeeping for every failed try of this attempt: a hedged loser's dead+ # key or 503 counts exactly as a lone one would+ free = False+ for t in tries:+ if t.err is None:+ continue+ if on_free_retry is not None and on_free_retry(t.err):+ free = True+ elif t.outcome == "dead":+ dead_keys.add(t.key)+ failed_keys.add(t.key)+ if is_storm(t.err) and hedge > 1:+ hedging = True+ won = next((t for t in tries if t.err is None), None)+ if won is not None:+ if storm_gate is not None:+ storm_gate.ok()+ bench()+ return won.result+ # every key of this attempt failed: the worst error decides the branch+ worst = min(tries, key=_severity)+ err, exc, key = worst.err, worst.exc, worst.key+ assert err is not Nonelast_err = err- if on_free_retry is not None and on_free_retry(err):+ if free:attempt -= 1continue- outcome = outcome_of(err)- if outcome == "dead":+ if worst.outcome == "dead":# key-scoped: move on at once (no budget, no sleep); the key is only# benched once another key proves the request itself is fine- dead_keys.add(key)- failed_keys.add(key)- pool.report(key, "error", reserved=tokens_hint)if len(dead_keys) < len(pool):log.warning("%s key …%s looks dead (%s); rotating", label, key[-4:], err)attempt -= 1continueraise err from exc # every key failed the same way: not the keys' fault- pool.report(- key,- outcome,- reserved=tokens_hint,- retry_after_s=retry_after(exc) if (outcome == "429" and retry_after) else None,- )- if err.retryable and err.status in (503, 529) and storm < storm_attempts and not out_of_time():- failed_keys.add(key)+ if is_storm(err) and storm < storm_attempts and not out_of_time():+ failed_keys.update(t.key for t in tries)if len(failed_keys) < min(max(1, storm_quorum), len(pool)):# a 503 is per key at any instant (see the docstring): the next# key is the cure — free rotation, no sleep, no storm yetattempt -= 1- log.warning("%s key …%s 503 (%s); rotating to a fresh key (%d/%d tried)",- label, key[-4:], err, len(failed_keys), len(pool))+ log.warning("%s key …%s 503 (%s); rotating to a fresh key (%d/%d tried%s)",+ label, key[-4:], err, len(failed_keys), len(pool),+ f", hedging x{hedge}" if hedging else "")continue# capacity storm: enough distinct keys said 503 in this call, so# waiting is the only cure — paid from its own budget, not max_attemptsstorm += 1+ counts["storm"] = storm# jitter FIRST, cap LAST: capping before a >1 jitter factor let a# "<=5 s" wait land at 6.25 s in the wild (observed 2026-08-24).raw = base_delay * (2 ** min(storm, 8)) * (0.75 + 0.5 * random.random())@@ -232,7 +337,7 @@ def rotate_with_retries[T](if storm_gate is not None:# tell every other worker as well: the next one to arrive parks at# the gate instead of spending its own round-trip to find the storm- delay = max(delay, storm_gate.hit(retry_after(exc) if retry_after else None))+ delay = max(delay, storm_gate.hit(retry_after(exc) if (retry_after and exc) else None))log.warning("%s capacity storm %d/%d (%s); gate closed %.0fs",label, storm, storm_attempts, err, delay)storm_gate.enter()@@ -242,8 +347,8 @@ def rotate_with_retries[T](sleep(delay)attempt -= 1continue- if outcome == "429":- failed_keys.add(key)+ if worst.outcome == "429":+ failed_keys.update(t.key for t in tries if t.outcome == "429")if len(failed_keys) < len(pool):# an untried key remains: rotation is free, only a courtesy pauseattempt -= 1@@ -264,8 +369,45 @@ def rotate_with_retries[T](label, attempt, max_attempts, err, delay,)sleep(delay)- finally:- pool.release()- assert last_err is not None- bench()- raise last_err+ assert last_err is not None+ bench()+ raise last_err+ finally:+ if stats is not None:+ stats.update(counts)+++@dataclass(frozen=True)+class _Try:+ """The outcome of one round-trip on one key (``rotate_with_retries.run_one``)."""++ key: str+ result: Any+ err: ModelError | None+ exc: BaseException | None+ outcome: str+++def _severity(t: _Try) -> int:+ """Rank a failed try so the worst one decides a hedged attempt's branch:+ non-retryable (0) < plain retryable (1) < 429 (2) < dead key (3) < 503/529 (4)."""+ err = t.err+ assert err is not None+ if t.outcome == "dead":+ return 3+ if err.retryable and err.status in (503, 529):+ return 4+ if not err.retryable:+ return 0+ return 2 if t.outcome == "429" else 1+++def _discard_loser(fut: Future[_Try]) -> None:+ """A hedged sibling that finished after the winner: its pool report and slot+ release already happened in ``run_one``; here it is only logged."""+ try:+ t = fut.result()+ except BaseException as exc: # noqa: BLE001 - a crashed loser must not crash the caller+ log.debug("hedge loser crashed: %s", exc)+ return+ log.debug("hedge loser …%s finished (%s); discarded", t.key[-4:], t.outcome)diff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 83bd6d00..8457a132 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1241,6 +1241,21 @@ in 128 s); the planner 300 s (13.7 s p50 / 32 s p90, max 76 s; the storm-day pla492 s median for 39 s of model time). Anthropic / OpenAI go through `with_retries`, which has nodeadline (≈ 20 s of backoff at most), so the field is a no-op there.++*The retry of a 503 is hedged across keys* (`rotate_with_retries(hedge=2)`; `Settings.rate.hedge`,+`CV3D_RATE__HEDGE=1` for the A/B). Logged sleep was only 645 s per cell median — **13 % of the+wait**; `(wait − sleep) / storm lines` = **21.5 s per failed attempt** (p90 28.5, ~50 s late in a+storm): the cost of a 503 is the round-trip the provider holds before rejecting, not the ≤ 5 s+backoff. Storm streaks average **4.7 attempts** (1 101 episodes / 5 143 lines). From a call's+first 503 on, every further attempt is issued on two distinct fresh keys at once — the extra key+from `KeyPool.try_acquire`, never waited for, each request holding its own `max_in_flight` slot —+and the first success is returned; a loser finishes its own round-trip, reports its outcome to+the pool (a late success still reports its tokens) and releases its slot; when both fail it is+ONE storm attempt. Expected rounds per streak drop from ~4.7 to ~1.7, i.e. ~60 % of the retry+wait: **≈ 930 s/run blender, 1 400 s threejs, 600 s cadquery, 450 s graphics**. A 503 bills+nothing, so the hedge is free while it storms; only a success-then-success wastes one call+(cents for a chat turn — `GeminiImageModel` keeps `hedge=1` because an image is billed per image).+## 25. Where the time goes — the 2026-08-26 audit (`docs/TIME_AUDIT_2026-08-26.md`)51 storm-day runs against 52 baseline runs, every stage and every model call, scripts indiff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.mdindex 365fc9f5..2b74d5bd 100644--- a/harness/docs/INTERFACES.md+++ b/harness/docs/INTERFACES.md@@ -56,6 +56,7 @@ resp.parsed / resp.text / resp.tool_calls / resp.usage # Usage always has costfrom codeverse.models.keypool import KeyPool, KeyPoolExhaustedKeyPool(keys, *, rpm_per_key=900, tpm_per_key=None, cooldown_s=30, dead_cooldown_s=3600)pool.acquire(*, tokens_hint=0, exclude=None, timeout_s=120) -> key # raises immediately when every key is dead/cooling past the deadline+pool.try_acquire(*, tokens_hint=0, exclude=None) -> key | None # never waits (a hedged retry's extra key); holds a slot like acquirepool.report(key, "ok"|"429"|"5xx"|"error"|"dead", *, tokens=0, retry_after_s=None) # Δ "dead": health 0, benched dead_cooldown_s, re-probed afterfrom codeverse.models.pricing import estimate_cost # (provider, model, usage) -> usd (unknown model -> 0.0 + one warning)from codeverse.models.schema_utils import to_gemini_schema, to_openai_strict_schema, to_anthropic_schema, parse_json_lenientdiff --git a/harness/tests/models/test_gemini_scheduling.py b/harness/tests/models/test_gemini_scheduling.pyindex 1071dbff..b98ec65d 100644--- a/harness/tests/models/test_gemini_scheduling.py+++ b/harness/tests/models/test_gemini_scheduling.py@@ -182,3 +182,12 @@ def test_max_wait_s_must_be_positive():with pytest.raises(pydantic.ValidationError):ChatRequest(messages=[ChatMessage.user("x")], max_wait_s=0)++def test_the_hedge_is_a_settings_knob_and_a_constructor_argument(monkeypatch):+ from codeverse.config import Rate, get_settings++ assert make_model([])[0].hedge == 2, "Settings.rate.hedge default"+ assert make_model([], hedge=1)[0].hedge == 1+ monkeypatch.setattr(get_settings(), "rate", Rate(hedge=1))+ assert make_model([])[0].hedge == 1, "CV3D_RATE__HEDGE=1 is the A/B switch"+diff --git a/harness/tests/models/test_keypool.py b/harness/tests/models/test_keypool.pyindex d15a940c..8e697127 100644--- a/harness/tests/models/test_keypool.py+++ b/harness/tests/models/test_keypool.py@@ -156,3 +156,21 @@ def test_all_keys_dead_raises_immediately_instead_of_waiting_out_the_timeout():with pytest.raises(KeyPoolExhausted):pool.acquire(timeout_s=120)assert clock.t == t0 # no pointless 120 s wait: availability only moves later+++def test_try_acquire_never_waits_for_a_key_or_a_slot():+ """The extra keys of a hedged retry (``rotate_with_retries(hedge=...)``) come from+ ``try_acquire``: a key that is usable right now, else None — never a wait."""+ never = lambda s: (_ for _ in ()).throw(AssertionError(f"try_acquire slept {s}s")) # noqa: E731+ pool = KeyPool(["k1", "k2"], max_in_flight=2, cooldown_s=30, sleep=never)+ a = pool.try_acquire()+ assert a == "k1" and pool.stats()["in_flight"] == 1+ assert pool.try_acquire(exclude={a}) == "k2"+ assert pool.try_acquire() is None, "both max_in_flight slots are held"+ pool.release()+ pool.release()+ assert pool.stats()["in_flight"] == 0+ pool.report("k1", "429")+ pool.report("k2", "429")+ assert pool.try_acquire() is None, "every key is cooling down: no wait, no key"+ assert pool.stats()["in_flight"] == 0, "a refused try_acquire holds nothing"diff --git a/harness/tests/models/test_retry.py b/harness/tests/models/test_retry.pyindex 893b1d9c..72fc29fc 100644--- a/harness/tests/models/test_retry.py+++ b/harness/tests/models/test_retry.py@@ -3,6 +3,8 @@from __future__ import annotationsimport random+import threading+import timeimport pytest@@ -67,6 +69,7 @@ def test_exhausted_raises_last():from codeverse.models.base import ModelError # noqa: E402from codeverse.models.keypool import KeyPool, KeyPoolExhausted, Outcome # noqa: E402from codeverse.models.retry import rotate_with_retries # noqa: E402+from codeverse.models.storm import StormGate # noqa: E402def _classify(exc: BaseException) -> ModelError:@@ -211,19 +214,28 @@ def test_rotate_retry_after_hint_reaches_the_pool():assert 0 < cooling["…k1"] <= 7.0-def test_503_storm_has_its_own_patience_budget():- """A model-wide 503 storm must not burn the normal attempt budget."""[email protected](("hedge", "want_calls", "want_naps"), [(1, {9}, 7), (2, {9, 10}, 4)])+def test_503_storm_has_its_own_patience_budget(hedge, want_calls, want_naps):+ """A model-wide 503 storm must not burn the normal attempt budget.++ hedge=1: eight sequential 503s then ok — seven storm waits, more than max_attempts=3+ could survive. hedge=2 (the default, audit 2026-08-26 §5.2): from the second 503 on+ every attempt races both keys, so the same eight failures are waited out in FOUR storm+ waits, and the winning attempt's sibling may land too (a tenth call, discarded)."""from codeverse.models.base import ModelErrorfrom codeverse.models.keypool import KeyPoolfrom codeverse.models.retry import rotate_with_retriespool = KeyPool(["k1", "k2"], rpm_per_key=10_000)+ lock = threading.Lock()calls = {"n": 0}naps: list[float] = []def call(key):- calls["n"] += 1- if calls["n"] <= 8: # longer than max_attempts=3 could survive+ with lock:+ calls["n"] += 1+ n = calls["n"]+ if n <= 8: # longer than max_attempts=3 could surviveraise RuntimeError("503 storm")return "ok"@@ -233,11 +245,11 @@ def test_503_storm_has_its_own_patience_budget():out = rotate_with_retries(pool, call, classify=classify, outcome_of=lambda e: "5xx",max_attempts=3, base_delay=0.01, storm_attempts=10, storm_max_delay=0.05,- sleep=naps.append,+ sleep=naps.append, hedge=hedge,)- assert out == "ok" and calls["n"] == 9+ assert out == "ok" and calls["n"] in want_calls# the first 503 rotates to k2 for free (no nap); k2's 503 completes the 2-key quorum- assert len(naps) == 7 and all(d <= 0.05 for d in naps) # the cap is the cap+ assert len(naps) == want_naps and all(d <= 0.05 for d in naps) # the cap is the capdef test_storm_waits_never_exceed_the_house_limit_at_production_defaults():@@ -330,6 +342,7 @@ def test_one_call_cannot_retry_for_hours():classify=lambda e: ModelError(str(e), retryable=True, status=503),outcome_of=lambda e: "5xx", max_attempts=6, storm_attempts=60,sleep=sleep, monotonic=lambda: clock["t"],+ hedge=1, # the fake clock is serial; two hedged 300 s calls would add 600 s, not 300)# the whole point: bounded by the clock, not by 60 x (timeout + wait)assert clock["t"] <= RETRY_DEADLINE_S + 305.0, (@@ -359,6 +372,7 @@ def test_the_deadline_does_not_cut_a_call_that_is_making_progress():outcome_of=lambda e: "5xx", max_attempts=6, storm_attempts=60,sleep=lambda d: clock.__setitem__("t", clock["t"] + d),monotonic=lambda: clock["t"],+ hedge=1, # serial fake clock and call counter (see the hedge tests for the raced form))assert out == "ok" and calls["n"] == 3@@ -386,30 +400,199 @@ def test_503_rotates_to_a_fresh_key_before_it_is_a_storm():out = rotate_with_retries(pool, call, classify=lambda e: ModelError(str(e), retryable=True, status=503),outcome_of=lambda e: "5xx", max_attempts=3, storm_attempts=10, storm_max_delay=0.05,sleep=naps.append, storm_gate=gate)- assert out == "ok" and calls == ["k1", "k2", "k3"]+ # k1 alone; then, with the default hedge, k2 and k3 raced at once (either may log first)+ assert out == "ok" and calls[0] == "k1" and sorted(calls[1:]) == ["k2", "k3"]assert naps == [], "rotation to a fresh key costs no sleep"assert gate.n_storms == 0 and not gate.storming, "two keys' 503s are not a storm for the whole process"-def test_503_on_every_key_is_still_a_storm():[email protected](("hedge", "want_calls", "want_hits"), [(1, {6}, 3), (2, {6, 7}, 2)])+def test_503_on_every_key_is_still_a_storm(hedge, want_calls, want_hits):+ """hedge=1: k1, k2, k3 fail one by one (quorum), then two storm waits, then ok.+ hedge=2: k1 fails alone, k2+k3 fail together (quorum: storm 1), two more fail together+ (storm 2), then the raced pair lands — one storm wait per hedged attempt, not per key."""from codeverse.models.base import ModelErrorfrom codeverse.models.keypool import KeyPoolfrom codeverse.models.retry import rotate_with_retriesfrom codeverse.models.storm import StormGatepool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000)- gate = StormGate()+ gate = StormGate("t", base_delay=0.0, max_wait_s=0.0, probe_lease_s=0.0) # no real waits+ lock = threading.Lock()n = {"c": 0}naps: list[float] = []def call(key):- n["c"] += 1- if n["c"] <= 5:+ with lock:+ n["c"] += 1+ i = n["c"]+ if i <= 5:raise RuntimeError("503 high demand")return "ok"out = rotate_with_retries(pool, call, classify=lambda e: ModelError(str(e), retryable=True, status=503),outcome_of=lambda e: "5xx", max_attempts=3, storm_attempts=10, storm_max_delay=0.05,- sleep=naps.append, storm_gate=gate)- assert out == "ok" and n["c"] == 6- assert gate.n_storms == 1 and gate.n_hits == 3, "the storm is declared after all three keys failed, then waited out"+ sleep=naps.append, storm_gate=gate, hedge=hedge)+ assert out == "ok" and n["c"] in want_calls+ assert gate.n_storms == 1 and gate.n_hits == want_hits, "the storm is declared after all three keys failed, then waited out"+++# --------------------------------------------------------------------------- hedging (audit 2026-08-26 §5.2)+def _wait_idle(pool, timeout: float = 5.0) -> None:+ """Block until every hedged loser has finished and released its slot."""+ t_end = time.monotonic() + timeout+ while pool.stats()["in_flight"] and time.monotonic() < t_end:+ time.sleep(0.005)+ assert pool.stats()["in_flight"] == 0+++def _503(e):+ return ModelError(str(e), retryable=True, status=503)+++def test_after_the_first_503_the_next_attempt_is_hedged_and_the_first_success_wins():+ """Measured 2026-08-26: a failed 503 costs the 21-50 s round-trip the provider holds+ before rejecting, not the <= 5 s sleep (logged sleep was 13 % of the wait), and storm+ streaks average 4.7 attempts. So from the first 503 on, the next attempt goes out on+ two fresh keys at once and the first success is returned while the other is still in+ flight; the loser reports its own outcome and releases its own slot when it lands."""+ pool = KeyPool(["k1", "k2", "k3", "k4"], rpm_per_key=10_000, max_in_flight=8)+ release_k2 = threading.Event()+ lock = threading.Lock()+ calls: list[str] = []+ naps: list[float] = []+ stats: dict = {}++ def call(key):+ with lock:+ calls.append(key)+ if key == "k1":+ raise RuntimeError("503 high demand")+ if key == "k2": # the slow loser: still held by the provider when k3 answers+ assert release_k2.wait(5.0)+ raise RuntimeError("503 high demand")+ return f"ok:{key}"++ out = rotate_with_retries(pool, call, classify=_503, outcome_of=lambda e: "5xx", max_attempts=3,+ storm_attempts=10, sleep=naps.append, tokens_of=lambda r: 7, stats=stats)+ assert out == "ok:k3"+ assert naps == [], "a hedged retry sleeps for nothing"+ assert calls[0] == "k1" and sorted(calls[1:]) == ["k2", "k3"], "two distinct fresh keys, k1 excluded"+ assert stats == {"attempts": 3, "hedged": 1, "storm": 0}+ st = pool.stats()+ assert st["in_flight"] == 1 and st["peak_in_flight"] == 2, "the loser still holds its max_in_flight slot"+ assert st["ok"] == 1 and st["5xx"] == 1+ release_k2.set()+ _wait_idle(pool)+ st = pool.stats()+ assert st["5xx"] == 2 and st["in_flight"] == 0, "the loser's 503 is reported when it lands"+++def test_a_loser_that_succeeds_later_is_discarded_but_its_tokens_are_reported():+ """A success-then-success is the one case the hedge is not free: the second answer is+ thrown away, its tokens still counted against the key (a 503 bills nothing, so while it+ storms the hedge costs nothing)."""+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000, tpm_per_key=1_000_000)+ release_k2 = threading.Event()++ def call(key):+ if key == "k1":+ raise ModelError("503 high demand", retryable=True, status=503)+ if key == "k2":+ assert release_k2.wait(5.0)+ return "late:k2"+ return "ok:k3"++ out = _rotate(pool, call, max_attempts=3, storm_attempts=10, tokens_of=lambda r: 900 if r.endswith("k2") else 100)+ assert out == "ok:k3"+ release_k2.set()+ _wait_idle(pool)+ assert pool.stats()["ok"] == 2+ assert pool._by_key["k2"].tokens_used == 900 and pool._by_key["k3"].tokens_used == 100 # noqa: SLF001+++def test_a_hedged_attempt_that_fails_on_every_key_is_one_storm_attempt():+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000)+ gate = StormGate("t", base_delay=0.0, max_wait_s=0.0, probe_lease_s=0.0) # no real waits+ lock = threading.Lock()+ n = {"c": 0}+ stats: dict = {}++ def call(key):+ with lock:+ n["c"] += 1+ i = n["c"]+ if i <= 5:+ raise RuntimeError("503 high demand")+ return "ok"++ out = rotate_with_retries(pool, call, classify=_503, outcome_of=lambda e: "5xx", max_attempts=3,+ storm_attempts=10, storm_max_delay=0.05, sleep=lambda d: None, storm_gate=gate,+ stats=stats)+ assert out == "ok"+ # k1 alone; k2+k3 (quorum of 3 → storm 1); two more (storm 2); the raced pair lands+ assert stats == {"attempts": 7, "hedged": 3, "storm": 2}+ assert gate.n_hits == 2, "two hedged failures = two storm waits, not four"+++def test_hedge_1_disables_and_a_full_pool_of_slots_degrades_to_one_key():+ lock = threading.Lock()++ def make_call():+ n = {"c": 0}++ def call(key):+ with lock:+ n["c"] += 1+ i = n["c"]+ if i <= 3:+ raise ModelError("503 high demand", retryable=True, status=503)+ return "ok"+ return call++ stats: dict = {}+ _rotate(KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000), make_call(), max_attempts=3, storm_attempts=10,+ storm_max_delay=0.0, hedge=1, stats=stats)+ assert stats["hedged"] == 0 and stats["attempts"] == 4+ # max_in_flight=1: the primary holds the only slot, so a partner is never waited for+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000, max_in_flight=1)+ stats = {}+ _rotate(pool, make_call(), max_attempts=3, storm_attempts=10, storm_max_delay=0.0, stats=stats)+ assert stats["hedged"] == 0 and stats["attempts"] == 4 and pool.stats()["peak_in_flight"] == 1+ # max_in_flight=2: a hedged attempt holds both slots+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000, max_in_flight=2)+ stats = {}+ _rotate(pool, make_call(), max_attempts=3, storm_attempts=10, storm_max_delay=0.0, stats=stats)+ _wait_idle(pool)+ assert stats["hedged"] >= 1 and pool.stats()["peak_in_flight"] == 2+++def test_the_worst_error_decides_a_hedged_attempt():+ """One key says 503 and the other says the request itself is bad: raise the 400 at once+ instead of storming on a request that can never succeed."""+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000)+ stats: dict = {}++ def call(key):+ if key == "k3":+ raise ModelError("bad request", retryable=False, status=400)+ raise ModelError("503 high demand", retryable=True, status=503)++ with pytest.raises(ModelError) as ei:+ _rotate(pool, call, max_attempts=6, storm_attempts=10, stats=stats)+ assert ei.value.status == 400 and stats == {"attempts": 3, "hedged": 1, "storm": 0}+++def test_a_dead_key_among_the_hedge_is_benched_once_the_sibling_wins():+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000)++ def call(key):+ if key == "k1":+ raise ModelError("503 high demand", retryable=True, status=503)+ if key == "k2":+ raise ModelError("suspended", retryable=False, status=403)+ return "ok:k3"++ assert _rotate(pool, call, max_attempts=3, storm_attempts=10) == "ok:k3"+ _wait_idle(pool)+ assert pool.stats()["n_dead"] == 1, "k3 proved the request fine, so k2's 403 was the key's fault"