Skip to content

Official Codebase of 3DCodeVerse

Python

retry: hedge the retry of a 503 across two keys (rotate_with_retries hedge=2, Settings.rate.hedge)

gaoypeng6401bb4adea7
diff --git a/harness/codeverse/config.py b/harness/codeverse/config.py
index 1aeb2983..a69c92f3 100644
--- a/harness/codeverse/config.py
+++ b/harness/codeverse/config.py
@@ -132,6 +132,13 @@ class Rate(BaseModel):
"intermittent rather than a clean outage, so parking every worker starves the unlucky call "
"(docs/COST.md §21). The mechanism and its counters are kept so the experiment is "
"reproducible: CV3D_RATE__STORM_GATE=1, or bench/concurrency_probe.py --storm-gate.")
+ hedge: int = Field(
+ default=2, ge=1,
+ description="keys a retry is raced on once a call has met its first 503 (1 = off). Measured "
+ "2026-08-26: a failed 503 costs the 21-50 s round-trip the provider holds before rejecting, "
+ "storm streaks average 4.7 attempts, and a 503 bills nothing — so the hedge is free while it "
+ "storms and wastes one call only when both keys answer (docs/COST.md §24). "
+ "CV3D_RATE__HEDGE=1 for the A/B.")
class Judge(BaseModel):
diff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.py
index a5c09227..7fb37595 100644
--- a/harness/codeverse/models/gemini.py
+++ b/harness/codeverse/models/gemini.py
@@ -183,6 +183,7 @@ class GeminiModel:
tpm_per_key: int | None = None,
storm_gate: StormGate | None = None,
storm_attempts: int | None = None,
+ hedge: int | None = None,
sleep: Callable[[float], None] = time.sleep,
client_factory: Callable[[str], Any] | None = None,
) -> None:
@@ -191,6 +192,9 @@ class GeminiModel:
#: whose whole point is a fast verdict — the storm branch does NOT consume
#: max_attempts, so max_attempts=1 alone still retried a 503 for up to 15 min.
self.storm_attempts = storm_attempts
+ #: keys a retry is raced on after the call's first 503 (``Settings.rate.hedge``,
+ #: 2; 1 = off) — see ``rotate_with_retries`` and docs/COST.md §24
+ self.hedge = max(1, int(_rate().hedge if hedge is None else hedge))
keys = list(keys) if keys is not None else _default_keys()
if pool is None and not keys:
raise ModelError(
@@ -256,6 +260,7 @@ class GeminiModel:
base_delay=self.base_delay,
max_delay=self.max_delay,
max_total_s=budget,
+ hedge=self.hedge,
sleep=self._sleep,
on_free_retry=downgrade_thinking,
retry_after=_retry_after_s,
diff --git a/harness/codeverse/models/gemini_image.py b/harness/codeverse/models/gemini_image.py
index fd34d15c..5593df7d 100644
--- a/harness/codeverse/models/gemini_image.py
+++ b/harness/codeverse/models/gemini_image.py
@@ -149,11 +149,15 @@ class GeminiImageModel:
max_attempts: int = 4,
base_delay: float = 1.0,
max_delay: float = MAX_WAIT_S,
+ hedge: int = 1,
sleep: Callable[[float], None] = time.sleep,
client_factory: Callable[[str], Any] | None = None,
) -> None:
self.model = model
self.fallback = fallback if fallback and fallback != model else None
+ #: no 503 hedge by default: an image is billed per image, so a hedge that
+ #: lands twice pays for two of them (the chat models default to 2)
+ self.hedge = max(1, int(hedge))
keys = list(keys) if keys is not None else _default_keys()
if pool is None and not keys:
raise ModelError("no Gemini API keys configured (GEMINI_API_KEYS / ~/.config/astra3d/gemini_keys.env)")
@@ -260,6 +264,7 @@ class GeminiImageModel:
max_attempts=self.max_attempts,
base_delay=self.base_delay,
max_delay=self.max_delay,
+ hedge=self.hedge,
sleep=self._sleep,
retry_after=_retry_after_s,
tokens_of=lambda r: r[1].input_tokens,
diff --git a/harness/codeverse/models/keypool.py b/harness/codeverse/models/keypool.py
index 202de9f9..00f62058 100644
--- a/harness/codeverse/models/keypool.py
+++ b/harness/codeverse/models/keypool.py
@@ -9,7 +9,8 @@ Used by ``GeminiModel`` (22 keys on the owner's box) but provider-neutral.
200 k-token judge verdict and a 2 k-token caption are scheduled differently
(``docs/COST.md`` Part III); ``max_in_flight`` additionally caps how many calls
may be out at once, and :meth:`KeyPool.release` (a ``finally`` in
- ``rotate_with_retries``) hands the slot back.
+ ``rotate_with_retries``) hands the slot back. ``try_acquire()`` is the
+ never-waiting variant a hedged retry uses for its extra keys.
* ``report(key, outcome)`` feeds back ``ok | 429 | 5xx | error | dead`` so the
pool can cool a key down and adjust its health score. ``dead`` is for
key-scoped auth/permission failures (revoked / suspended / invalid key): the
@@ -203,6 +204,32 @@ class KeyPool:
self._slots.release()
raise
+ def try_acquire(
+ self,
+ *,
+ tokens_hint: int = 0,
+ exclude: set[str] | frozenset[str] | None = None,
+ ) -> str | None:
+ """:meth:`acquire` that never waits: a key usable *right now* (and a free
+ ``max_in_flight`` slot), else ``None`` — nothing is reserved or counted then.
+
+ For the extra keys of a hedged retry (``rotate_with_retries(hedge=...)``): the
+ primary request is already in flight, so a partner that is not free at once
+ is not worth waiting for. A returned key holds a slot like any other and
+ must be :meth:`release`-d."""
+ if self._slots is not None and not self._slots.acquire(blocking=False):
+ return None
+ try:
+ return self._acquire_key(tokens_hint, exclude, 0.0)
+ except KeyPoolExhausted:
+ if self._slots is not None:
+ self._slots.release()
+ return None
+ except BaseException:
+ if self._slots is not None:
+ self._slots.release()
+ raise
+
def _acquire_key(
self,
tokens_hint: int,
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.py
index d35bbcff..b8a5e519 100644
--- a/harness/codeverse/models/retry.py
+++ b/harness/codeverse/models/retry.py
@@ -2,8 +2,8 @@
key-rotation state machine used by the Gemini backends.
* :func:`with_retries` — plain bounded retry; no provider knowledge.
-* :func:`rotate_with_retries` — the dead-key / 429-rotation / backoff machine
- over a :class:`~codeverse.models.keypool.KeyPool`. Provider specifics
+* :func:`rotate_with_retries` — the dead-key / 429-rotation / 503-hedge / backoff
+ machine over a :class:`~codeverse.models.keypool.KeyPool`. Provider specifics
(exception classification, outcome mapping, retry-after extraction) are
injected as hooks, so this module stays provider-neutral.
@@ -15,8 +15,10 @@ from __future__ import annotations
import logging
import random
import time
-from collections.abc import Callable
-from typing import TYPE_CHECKING, TypeVar
+from collections.abc import Callable, Sequence
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypeVar
from codeverse.models.keypool import MAX_WAIT_S, KeyPoolExhausted
@@ -27,8 +29,17 @@ from codeverse.models.keypool import MAX_WAIT_S, KeyPoolExhausted
#: when a call is BILLED (a stalled call bills nothing), let a run with --max-minutes 90
#: reach 200 minutes during a flash outage. Past this deadline the call gives up and the
#: cell is recorded infra_failed, which is re-runnable (`--redo-status infra_failed`).
+#: This is the CEILING; a caller that cannot afford it passes a smaller ``max_total_s``
+#: (``ChatRequest.max_wait_s``) — audit 2026-08-26 §5.1: 66 give-up spans of 900 s, up to
+#: three in a row on one agent turn, were 30 % of a storm day's waiting.
RETRY_DEADLINE_S = 900.0
+#: how many keys a retry is spread over once a call has met its first 503/529.
+#: Measured 2026-08-26 §5.2: a failed 503 costs the 21-50 s round-trip the provider
+#: holds before rejecting (not the <= 5 s sleep), storm streaks average 4.7 attempts,
+#: and a 503 bills nothing — so racing the next attempt on two fresh keys is free.
+DEFAULT_HEDGE = 2
+
if TYPE_CHECKING: # pragma: no cover
from codeverse.models.base import ModelError
from codeverse.models.keypool import KeyPool, Outcome
@@ -103,6 +114,7 @@ def rotate_with_retries[T](
storm_max_delay: float = MAX_WAIT_S,
storm_quorum: int = 6,
max_total_s: float = RETRY_DEADLINE_S,
+ hedge: int = DEFAULT_HEDGE,
sleep: Callable[[float], None] = time.sleep,
monotonic: Callable[[], float] = time.monotonic,
on_free_retry: Callable[[ModelError], bool] | None = None,
@@ -111,6 +123,7 @@ def rotate_with_retries[T](
tokens_hint: int = 0,
storm_gate: StormGate | None = None,
label: str = "model",
+ stats: dict[str, Any] | None = None,
) -> T:
"""Run ``call(key)`` against a rotating :class:`KeyPool` until it succeeds.
@@ -120,7 +133,7 @@ def rotate_with_retries[T](
looked dead during this call are benched (another key proved the request
itself is fine) and the result is returned.
* ``on_free_retry(err)`` returning ``True`` → retry at once, for free
- (no report, no sleep) — e.g. a thinking-config downgrade.
+ (no sleep, no budget) — e.g. a thinking-config downgrade.
* ``outcome_of(err) == "dead"`` (key-scoped auth failure) → rotate at once
for free; the key is only benched once a sibling key proves the request
is fine. Every key failing the same way raises without benching.
@@ -146,14 +159,43 @@ def rotate_with_retries[T](
* other retryable errors → exponential backoff + jitter until
``max_attempts``; non-retryable errors raise immediately.
+ **Hedging** (``hedge``, default :data:`DEFAULT_HEDGE`; ``1`` disables). From the
+ FIRST 503/529 of a call onwards, every further attempt is issued on up to ``hedge``
+ distinct keys at once and the first success wins. Why: on the 2026-08-26 storm
+ day the cost of a failed 503 was the **21-50 s round-trip** the provider held
+ before rejecting, not the <= 5 s sleep (logged sleep was 13 % of the wait), and
+ storm streaks averaged **4.7 attempts** — racing two keys cuts the expected number
+ of rounds to ~1.7. A 503 bills nothing, so while it is storming the hedge is free;
+ the price is paid only when BOTH keys answer: the second success is discarded and
+ that call's tokens are wasted (a few cents for a chat turn — which is why
+ ``GeminiImageModel`` passes ``hedge=1``: an image is billed per image).
+ Mechanics: the attempt's first key is acquired as usual (it may wait for a slot or
+ a cooldown); the extra keys come from :meth:`KeyPool.try_acquire` and are simply
+ skipped when no fresh key or ``max_in_flight`` slot is free *right now*, so a hedge
+ never waits for a partner and every hedged request holds its own slot. The calls
+ run in a per-attempt ``ThreadPoolExecutor`` that is shut down without waiting: a
+ loser keeps running until its own round-trip ends, then reports its outcome to
+ the pool (a late success still reports its tokens) and releases its slot; the
+ winner is returned the moment it lands. When every hedged key fails, the attempt
+ is ONE attempt for the storm / backoff accounting and the worst error decides the
+ branch (a non-retryable error first, then a plain retryable one, a 429, a dead
+ key, and a 503 last).
+
``tokens_hint`` is the estimated **prompt** tokens of the pending call
(:func:`codeverse.models.tokens.request_tokens`); the pool reserves them in
the per-key TPM bucket at ``acquire`` and the reservation is reconciled
against the provider's real count on ``report``, so a 200 k-token judge call
and a 2 k-token caption are scheduled differently.
+
+ ``stats`` (optional, caller-owned dict) receives the call's telemetry on the way
+ out — success or failure: ``attempts`` = round-trips issued (hedged siblings
+ included; 1 = clean), ``hedged`` = hedged attempts, ``storm`` = storm waits.
+ ``GeminiModel`` copies ``attempts`` into ``ChatResponse.raw`` and onto the raised
+ ``ModelError`` so the cost ledger can say how hard each call was.
"""
- failed_keys: set[str] = set() # keys that 429'd or looked dead during this call
+ failed_keys: set[str] = set() # keys that 429'd / 503'd or looked dead during this call
dead_keys: set[str] = set()
+ counts = {"attempts": 0, "hedged": 0, "storm": 0}
def bench() -> None:
# mark keys that failed with key-scoped errors dead — called once the call
@@ -162,69 +204,132 @@ def rotate_with_retries[T](
pool.report(k, "dead")
dead_keys.clear()
+ def run_one(key: str) -> _Try:
+ """One round-trip on ``key``: report its outcome to the pool and hand the slot
+ back. Runs inline for a single key and in a worker thread when hedged, so it
+ touches none of this call's state and never raises a ``ModelError`` itself."""
+ try:
+ try:
+ result = call(key)
+ except Exception as exc: # noqa: BLE001 - classification is delegated
+ err = classify(exc)
+ outcome = outcome_of(err)
+ # a key that looks dead is reported "error" now and benched only once a
+ # sibling proves the request itself is fine (bench())
+ pool.report(
+ key,
+ "error" if outcome == "dead" else outcome,
+ reserved=tokens_hint,
+ retry_after_s=retry_after(exc) if (outcome == "429" and retry_after) else None,
+ )
+ return _Try(key, None, err, exc, outcome)
+ pool.report(key, "ok", tokens=tokens_of(result) if tokens_of is not None else 0,
+ reserved=tokens_hint)
+ return _Try(key, result, None, None, "ok")
+ finally:
+ pool.release()
+
+ def run_attempt(keys: Sequence[str]) -> list[_Try]:
+ """Issue ``call`` on every key; return the tries that completed up to and
+ including the first success, in completion order. Losers still in flight
+ finish on their own (``run_one`` reports and releases for them)."""
+ counts["attempts"] += len(keys)
+ if len(keys) == 1:
+ return [run_one(keys[0])]
+ counts["hedged"] += 1
+ executor = ThreadPoolExecutor(max_workers=len(keys), thread_name_prefix=f"hedge {label}")
+ futures = [executor.submit(run_one, k) for k in keys]
+ executor.shutdown(wait=False) # no new work; the threads end with their round-trips
+ done: list[_Try] = []
+ for fut in as_completed(futures):
+ t = fut.result()
+ done.append(t)
+ if t.err is None:
+ for other in futures:
+ if not other.done():
+ other.add_done_callback(_discard_loser)
+ break
+ return done
+
last_err: ModelError | None = None
attempt = 0
storm = 0
+ hedging = False # flips on the first 503/529; every later attempt is hedged
deadline = monotonic() + max_total_s if max_total_s > 0 else float("inf")
def out_of_time() -> bool:
return monotonic() >= deadline
- while attempt < max_attempts:
- attempt += 1
- # never go back to a key that looked dead this call; throttled keys are
- # excluded while an untried one remains, else acquire() waits for a cooldown
- exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())
- if storm_gate is not None:
- storm_gate.enter()
- try:
- key = pool.acquire(exclude=exclude, tokens_hint=tokens_hint)
- except KeyPoolExhausted as exc:
- raise classify(exc) from exc
- try:
- result = call(key)
- pool.report(key, "ok", tokens=tokens_of(result) if tokens_of is not None else 0,
- reserved=tokens_hint)
+ def is_storm(err: ModelError) -> bool:
+ return bool(err.retryable and err.status in (503, 529))
+
+ try:
+ while attempt < max_attempts:
+ attempt += 1
+ # never go back to a key that looked dead this call; throttled keys are
+ # excluded while an untried one remains, else acquire() waits for a cooldown
+ exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())
if storm_gate is not None:
- storm_gate.ok()
- bench()
- return result
- except Exception as exc: # noqa: BLE001 - classification is delegated
- err = classify(exc)
+ storm_gate.enter()
+ try:
+ keys = [pool.acquire(exclude=exclude, tokens_hint=tokens_hint)]
+ except KeyPoolExhausted as exc:
+ raise classify(exc) from exc
+ while hedging and len(keys) < max(1, hedge):
+ extra = pool.try_acquire(exclude=exclude | set(keys), tokens_hint=tokens_hint)
+ if extra is None:
+ break # no fresh key / slot right now: never wait for a hedge partner
+ keys.append(extra)
+ tries = run_attempt(keys)
+ # bookkeeping for every failed try of this attempt: a hedged loser's dead
+ # key or 503 counts exactly as a lone one would
+ free = False
+ for t in tries:
+ if t.err is None:
+ continue
+ if on_free_retry is not None and on_free_retry(t.err):
+ free = True
+ elif t.outcome == "dead":
+ dead_keys.add(t.key)
+ failed_keys.add(t.key)
+ if is_storm(t.err) and hedge > 1:
+ hedging = True
+ won = next((t for t in tries if t.err is None), None)
+ if won is not None:
+ if storm_gate is not None:
+ storm_gate.ok()
+ bench()
+ return won.result
+ # every key of this attempt failed: the worst error decides the branch
+ worst = min(tries, key=_severity)
+ err, exc, key = worst.err, worst.exc, worst.key
+ assert err is not None
last_err = err
- if on_free_retry is not None and on_free_retry(err):
+ if free:
attempt -= 1
continue
- outcome = outcome_of(err)
- if outcome == "dead":
+ if worst.outcome == "dead":
# key-scoped: move on at once (no budget, no sleep); the key is only
# benched once another key proves the request itself is fine
- dead_keys.add(key)
- failed_keys.add(key)
- pool.report(key, "error", reserved=tokens_hint)
if len(dead_keys) < len(pool):
log.warning("%s key …%s looks dead (%s); rotating", label, key[-4:], err)
attempt -= 1
continue
raise err from exc # every key failed the same way: not the keys' fault
- pool.report(
- key,
- outcome,
- reserved=tokens_hint,
- retry_after_s=retry_after(exc) if (outcome == "429" and retry_after) else None,
- )
- if err.retryable and err.status in (503, 529) and storm < storm_attempts and not out_of_time():
- failed_keys.add(key)
+ if is_storm(err) and storm < storm_attempts and not out_of_time():
+ failed_keys.update(t.key for t in tries)
if len(failed_keys) < min(max(1, storm_quorum), len(pool)):
# a 503 is per key at any instant (see the docstring): the next
# key is the cure — free rotation, no sleep, no storm yet
attempt -= 1
- log.warning("%s key …%s 503 (%s); rotating to a fresh key (%d/%d tried)",
- label, key[-4:], err, len(failed_keys), len(pool))
+ log.warning("%s key …%s 503 (%s); rotating to a fresh key (%d/%d tried%s)",
+ label, key[-4:], err, len(failed_keys), len(pool),
+ f", hedging x{hedge}" if hedging else "")
continue
# capacity storm: enough distinct keys said 503 in this call, so
# waiting is the only cure — paid from its own budget, not max_attempts
storm += 1
+ counts["storm"] = storm
# jitter FIRST, cap LAST: capping before a >1 jitter factor let a
# "<=5 s" wait land at 6.25 s in the wild (observed 2026-08-24).
raw = base_delay * (2 ** min(storm, 8)) * (0.75 + 0.5 * random.random())
@@ -232,7 +337,7 @@ def rotate_with_retries[T](
if storm_gate is not None:
# tell every other worker as well: the next one to arrive parks at
# the gate instead of spending its own round-trip to find the storm
- delay = max(delay, storm_gate.hit(retry_after(exc) if retry_after else None))
+ delay = max(delay, storm_gate.hit(retry_after(exc) if (retry_after and exc) else None))
log.warning("%s capacity storm %d/%d (%s); gate closed %.0fs",
label, storm, storm_attempts, err, delay)
storm_gate.enter()
@@ -242,8 +347,8 @@ def rotate_with_retries[T](
sleep(delay)
attempt -= 1
continue
- if outcome == "429":
- failed_keys.add(key)
+ if worst.outcome == "429":
+ failed_keys.update(t.key for t in tries if t.outcome == "429")
if len(failed_keys) < len(pool):
# an untried key remains: rotation is free, only a courtesy pause
attempt -= 1
@@ -264,8 +369,45 @@ def rotate_with_retries[T](
label, attempt, max_attempts, err, delay,
)
sleep(delay)
- finally:
- pool.release()
- assert last_err is not None
- bench()
- raise last_err
+ assert last_err is not None
+ bench()
+ raise last_err
+ finally:
+ if stats is not None:
+ stats.update(counts)
+
+
+@dataclass(frozen=True)
+class _Try:
+ """The outcome of one round-trip on one key (``rotate_with_retries.run_one``)."""
+
+ key: str
+ result: Any
+ err: ModelError | None
+ exc: BaseException | None
+ outcome: str
+
+
+def _severity(t: _Try) -> int:
+ """Rank a failed try so the worst one decides a hedged attempt's branch:
+ non-retryable (0) < plain retryable (1) < 429 (2) < dead key (3) < 503/529 (4)."""
+ err = t.err
+ assert err is not None
+ if t.outcome == "dead":
+ return 3
+ if err.retryable and err.status in (503, 529):
+ return 4
+ if not err.retryable:
+ return 0
+ return 2 if t.outcome == "429" else 1
+
+
+def _discard_loser(fut: Future[_Try]) -> None:
+ """A hedged sibling that finished after the winner: its pool report and slot
+ release already happened in ``run_one``; here it is only logged."""
+ try:
+ t = fut.result()
+ except BaseException as exc: # noqa: BLE001 - a crashed loser must not crash the caller
+ log.debug("hedge loser crashed: %s", exc)
+ return
+ log.debug("hedge loser …%s finished (%s); discarded", t.key[-4:], t.outcome)
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 83bd6d00..8457a132 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -1241,6 +1241,21 @@ in 128 s); the planner 300 s (13.7 s p50 / 32 s p90, max 76 s; the storm-day pla
492 s median for 39 s of model time). Anthropic / OpenAI go through `with_retries`, which has no
deadline (≈ 20 s of backoff at most), so the field is a no-op there.
+
+*The retry of a 503 is hedged across keys* (`rotate_with_retries(hedge=2)`; `Settings.rate.hedge`,
+`CV3D_RATE__HEDGE=1` for the A/B). Logged sleep was only 645 s per cell median — **13 % of the
+wait**; `(wait − sleep) / storm lines` = **21.5 s per failed attempt** (p90 28.5, ~50 s late in a
+storm): the cost of a 503 is the round-trip the provider holds before rejecting, not the ≤ 5 s
+backoff. Storm streaks average **4.7 attempts** (1 101 episodes / 5 143 lines). From a call's
+first 503 on, every further attempt is issued on two distinct fresh keys at once — the extra key
+from `KeyPool.try_acquire`, never waited for, each request holding its own `max_in_flight` slot —
+and the first success is returned; a loser finishes its own round-trip, reports its outcome to
+the pool (a late success still reports its tokens) and releases its slot; when both fail it is
+ONE storm attempt. Expected rounds per streak drop from ~4.7 to ~1.7, i.e. ~60 % of the retry
+wait: **≈ 930 s/run blender, 1 400 s threejs, 600 s cadquery, 450 s graphics**. A 503 bills
+nothing, so the hedge is free while it storms; only a success-then-success wastes one call
+(cents for a chat turn — `GeminiImageModel` keeps `hedge=1` because an image is billed per image).
+
## 25. Where the time goes — the 2026-08-26 audit (`docs/TIME_AUDIT_2026-08-26.md`)
51 storm-day runs against 52 baseline runs, every stage and every model call, scripts in
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index 365fc9f5..2b74d5bd 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -56,6 +56,7 @@ resp.parsed / resp.text / resp.tool_calls / resp.usage # Usage always has cost
from codeverse.models.keypool import KeyPool, KeyPoolExhausted
KeyPool(keys, *, rpm_per_key=900, tpm_per_key=None, cooldown_s=30, dead_cooldown_s=3600)
pool.acquire(*, tokens_hint=0, exclude=None, timeout_s=120) -> key # raises immediately when every key is dead/cooling past the deadline
+pool.try_acquire(*, tokens_hint=0, exclude=None) -> key | None # never waits (a hedged retry's extra key); holds a slot like acquire
pool.report(key, "ok"|"429"|"5xx"|"error"|"dead", *, tokens=0, retry_after_s=None) # Δ "dead": health 0, benched dead_cooldown_s, re-probed after
from codeverse.models.pricing import estimate_cost # (provider, model, usage) -> usd (unknown model -> 0.0 + one warning)
from codeverse.models.schema_utils import to_gemini_schema, to_openai_strict_schema, to_anthropic_schema, parse_json_lenient
diff --git a/harness/tests/models/test_gemini_scheduling.py b/harness/tests/models/test_gemini_scheduling.py
index 1071dbff..b98ec65d 100644
--- a/harness/tests/models/test_gemini_scheduling.py
+++ b/harness/tests/models/test_gemini_scheduling.py
@@ -182,3 +182,12 @@ def test_max_wait_s_must_be_positive():
with pytest.raises(pydantic.ValidationError):
ChatRequest(messages=[ChatMessage.user("x")], max_wait_s=0)
+
+def test_the_hedge_is_a_settings_knob_and_a_constructor_argument(monkeypatch):
+ from codeverse.config import Rate, get_settings
+
+ assert make_model([])[0].hedge == 2, "Settings.rate.hedge default"
+ assert make_model([], hedge=1)[0].hedge == 1
+ monkeypatch.setattr(get_settings(), "rate", Rate(hedge=1))
+ assert make_model([])[0].hedge == 1, "CV3D_RATE__HEDGE=1 is the A/B switch"
+
diff --git a/harness/tests/models/test_keypool.py b/harness/tests/models/test_keypool.py
index d15a940c..8e697127 100644
--- a/harness/tests/models/test_keypool.py
+++ b/harness/tests/models/test_keypool.py
@@ -156,3 +156,21 @@ def test_all_keys_dead_raises_immediately_instead_of_waiting_out_the_timeout():
with pytest.raises(KeyPoolExhausted):
pool.acquire(timeout_s=120)
assert clock.t == t0 # no pointless 120 s wait: availability only moves later
+
+
+def test_try_acquire_never_waits_for_a_key_or_a_slot():
+ """The extra keys of a hedged retry (``rotate_with_retries(hedge=...)``) come from
+ ``try_acquire``: a key that is usable right now, else None — never a wait."""
+ never = lambda s: (_ for _ in ()).throw(AssertionError(f"try_acquire slept {s}s")) # noqa: E731
+ pool = KeyPool(["k1", "k2"], max_in_flight=2, cooldown_s=30, sleep=never)
+ a = pool.try_acquire()
+ assert a == "k1" and pool.stats()["in_flight"] == 1
+ assert pool.try_acquire(exclude={a}) == "k2"
+ assert pool.try_acquire() is None, "both max_in_flight slots are held"
+ pool.release()
+ pool.release()
+ assert pool.stats()["in_flight"] == 0
+ pool.report("k1", "429")
+ pool.report("k2", "429")
+ assert pool.try_acquire() is None, "every key is cooling down: no wait, no key"
+ assert pool.stats()["in_flight"] == 0, "a refused try_acquire holds nothing"
diff --git a/harness/tests/models/test_retry.py b/harness/tests/models/test_retry.py
index 893b1d9c..72fc29fc 100644
--- a/harness/tests/models/test_retry.py
+++ b/harness/tests/models/test_retry.py
@@ -3,6 +3,8 @@
from __future__ import annotations
import random
+import threading
+import time
import pytest
@@ -67,6 +69,7 @@ def test_exhausted_raises_last():
from codeverse.models.base import ModelError # noqa: E402
from codeverse.models.keypool import KeyPool, KeyPoolExhausted, Outcome # noqa: E402
from codeverse.models.retry import rotate_with_retries # noqa: E402
+from codeverse.models.storm import StormGate # noqa: E402
def _classify(exc: BaseException) -> ModelError:
@@ -211,19 +214,28 @@ def test_rotate_retry_after_hint_reaches_the_pool():
assert 0 < cooling["…k1"] <= 7.0
-def test_503_storm_has_its_own_patience_budget():
- """A model-wide 503 storm must not burn the normal attempt budget."""
[email protected](("hedge", "want_calls", "want_naps"), [(1, {9}, 7), (2, {9, 10}, 4)])
+def test_503_storm_has_its_own_patience_budget(hedge, want_calls, want_naps):
+ """A model-wide 503 storm must not burn the normal attempt budget.
+
+ hedge=1: eight sequential 503s then ok — seven storm waits, more than max_attempts=3
+ could survive. hedge=2 (the default, audit 2026-08-26 §5.2): from the second 503 on
+ every attempt races both keys, so the same eight failures are waited out in FOUR storm
+ waits, and the winning attempt's sibling may land too (a tenth call, discarded)."""
from codeverse.models.base import ModelError
from codeverse.models.keypool import KeyPool
from codeverse.models.retry import rotate_with_retries
pool = KeyPool(["k1", "k2"], rpm_per_key=10_000)
+ lock = threading.Lock()
calls = {"n": 0}
naps: list[float] = []
def call(key):
- calls["n"] += 1
- if calls["n"] <= 8: # longer than max_attempts=3 could survive
+ with lock:
+ calls["n"] += 1
+ n = calls["n"]
+ if n <= 8: # longer than max_attempts=3 could survive
raise RuntimeError("503 storm")
return "ok"
@@ -233,11 +245,11 @@ def test_503_storm_has_its_own_patience_budget():
out = rotate_with_retries(
pool, call, classify=classify, outcome_of=lambda e: "5xx",
max_attempts=3, base_delay=0.01, storm_attempts=10, storm_max_delay=0.05,
- sleep=naps.append,
+ sleep=naps.append, hedge=hedge,
)
- assert out == "ok" and calls["n"] == 9
+ assert out == "ok" and calls["n"] in want_calls
# the first 503 rotates to k2 for free (no nap); k2's 503 completes the 2-key quorum
- assert len(naps) == 7 and all(d <= 0.05 for d in naps) # the cap is the cap
+ assert len(naps) == want_naps and all(d <= 0.05 for d in naps) # the cap is the cap
def test_storm_waits_never_exceed_the_house_limit_at_production_defaults():
@@ -330,6 +342,7 @@ def test_one_call_cannot_retry_for_hours():
classify=lambda e: ModelError(str(e), retryable=True, status=503),
outcome_of=lambda e: "5xx", max_attempts=6, storm_attempts=60,
sleep=sleep, monotonic=lambda: clock["t"],
+ hedge=1, # the fake clock is serial; two hedged 300 s calls would add 600 s, not 300
)
# the whole point: bounded by the clock, not by 60 x (timeout + wait)
assert clock["t"] <= RETRY_DEADLINE_S + 305.0, (
@@ -359,6 +372,7 @@ def test_the_deadline_does_not_cut_a_call_that_is_making_progress():
outcome_of=lambda e: "5xx", max_attempts=6, storm_attempts=60,
sleep=lambda d: clock.__setitem__("t", clock["t"] + d),
monotonic=lambda: clock["t"],
+ hedge=1, # serial fake clock and call counter (see the hedge tests for the raced form)
)
assert out == "ok" and calls["n"] == 3
@@ -386,30 +400,199 @@ def test_503_rotates_to_a_fresh_key_before_it_is_a_storm():
out = rotate_with_retries(pool, call, classify=lambda e: ModelError(str(e), retryable=True, status=503),
outcome_of=lambda e: "5xx", max_attempts=3, storm_attempts=10, storm_max_delay=0.05,
sleep=naps.append, storm_gate=gate)
- assert out == "ok" and calls == ["k1", "k2", "k3"]
+ # k1 alone; then, with the default hedge, k2 and k3 raced at once (either may log first)
+ assert out == "ok" and calls[0] == "k1" and sorted(calls[1:]) == ["k2", "k3"]
assert naps == [], "rotation to a fresh key costs no sleep"
assert gate.n_storms == 0 and not gate.storming, "two keys' 503s are not a storm for the whole process"
-def test_503_on_every_key_is_still_a_storm():
[email protected](("hedge", "want_calls", "want_hits"), [(1, {6}, 3), (2, {6, 7}, 2)])
+def test_503_on_every_key_is_still_a_storm(hedge, want_calls, want_hits):
+ """hedge=1: k1, k2, k3 fail one by one (quorum), then two storm waits, then ok.
+ hedge=2: k1 fails alone, k2+k3 fail together (quorum: storm 1), two more fail together
+ (storm 2), then the raced pair lands — one storm wait per hedged attempt, not per key."""
from codeverse.models.base import ModelError
from codeverse.models.keypool import KeyPool
from codeverse.models.retry import rotate_with_retries
from codeverse.models.storm import StormGate
pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000)
- gate = StormGate()
+ gate = StormGate("t", base_delay=0.0, max_wait_s=0.0, probe_lease_s=0.0) # no real waits
+ lock = threading.Lock()
n = {"c": 0}
naps: list[float] = []
def call(key):
- n["c"] += 1
- if n["c"] <= 5:
+ with lock:
+ n["c"] += 1
+ i = n["c"]
+ if i <= 5:
raise RuntimeError("503 high demand")
return "ok"
out = rotate_with_retries(pool, call, classify=lambda e: ModelError(str(e), retryable=True, status=503),
outcome_of=lambda e: "5xx", max_attempts=3, storm_attempts=10, storm_max_delay=0.05,
- sleep=naps.append, storm_gate=gate)
- assert out == "ok" and n["c"] == 6
- assert gate.n_storms == 1 and gate.n_hits == 3, "the storm is declared after all three keys failed, then waited out"
+ sleep=naps.append, storm_gate=gate, hedge=hedge)
+ assert out == "ok" and n["c"] in want_calls
+ assert gate.n_storms == 1 and gate.n_hits == want_hits, "the storm is declared after all three keys failed, then waited out"
+
+
+# --------------------------------------------------------------------------- hedging (audit 2026-08-26 §5.2)
+def _wait_idle(pool, timeout: float = 5.0) -> None:
+ """Block until every hedged loser has finished and released its slot."""
+ t_end = time.monotonic() + timeout
+ while pool.stats()["in_flight"] and time.monotonic() < t_end:
+ time.sleep(0.005)
+ assert pool.stats()["in_flight"] == 0
+
+
+def _503(e):
+ return ModelError(str(e), retryable=True, status=503)
+
+
+def test_after_the_first_503_the_next_attempt_is_hedged_and_the_first_success_wins():
+ """Measured 2026-08-26: a failed 503 costs the 21-50 s round-trip the provider holds
+ before rejecting, not the <= 5 s sleep (logged sleep was 13 % of the wait), and storm
+ streaks average 4.7 attempts. So from the first 503 on, the next attempt goes out on
+ two fresh keys at once and the first success is returned while the other is still in
+ flight; the loser reports its own outcome and releases its own slot when it lands."""
+ pool = KeyPool(["k1", "k2", "k3", "k4"], rpm_per_key=10_000, max_in_flight=8)
+ release_k2 = threading.Event()
+ lock = threading.Lock()
+ calls: list[str] = []
+ naps: list[float] = []
+ stats: dict = {}
+
+ def call(key):
+ with lock:
+ calls.append(key)
+ if key == "k1":
+ raise RuntimeError("503 high demand")
+ if key == "k2": # the slow loser: still held by the provider when k3 answers
+ assert release_k2.wait(5.0)
+ raise RuntimeError("503 high demand")
+ return f"ok:{key}"
+
+ out = rotate_with_retries(pool, call, classify=_503, outcome_of=lambda e: "5xx", max_attempts=3,
+ storm_attempts=10, sleep=naps.append, tokens_of=lambda r: 7, stats=stats)
+ assert out == "ok:k3"
+ assert naps == [], "a hedged retry sleeps for nothing"
+ assert calls[0] == "k1" and sorted(calls[1:]) == ["k2", "k3"], "two distinct fresh keys, k1 excluded"
+ assert stats == {"attempts": 3, "hedged": 1, "storm": 0}
+ st = pool.stats()
+ assert st["in_flight"] == 1 and st["peak_in_flight"] == 2, "the loser still holds its max_in_flight slot"
+ assert st["ok"] == 1 and st["5xx"] == 1
+ release_k2.set()
+ _wait_idle(pool)
+ st = pool.stats()
+ assert st["5xx"] == 2 and st["in_flight"] == 0, "the loser's 503 is reported when it lands"
+
+
+def test_a_loser_that_succeeds_later_is_discarded_but_its_tokens_are_reported():
+ """A success-then-success is the one case the hedge is not free: the second answer is
+ thrown away, its tokens still counted against the key (a 503 bills nothing, so while it
+ storms the hedge costs nothing)."""
+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000, tpm_per_key=1_000_000)
+ release_k2 = threading.Event()
+
+ def call(key):
+ if key == "k1":
+ raise ModelError("503 high demand", retryable=True, status=503)
+ if key == "k2":
+ assert release_k2.wait(5.0)
+ return "late:k2"
+ return "ok:k3"
+
+ out = _rotate(pool, call, max_attempts=3, storm_attempts=10, tokens_of=lambda r: 900 if r.endswith("k2") else 100)
+ assert out == "ok:k3"
+ release_k2.set()
+ _wait_idle(pool)
+ assert pool.stats()["ok"] == 2
+ assert pool._by_key["k2"].tokens_used == 900 and pool._by_key["k3"].tokens_used == 100 # noqa: SLF001
+
+
+def test_a_hedged_attempt_that_fails_on_every_key_is_one_storm_attempt():
+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000)
+ gate = StormGate("t", base_delay=0.0, max_wait_s=0.0, probe_lease_s=0.0) # no real waits
+ lock = threading.Lock()
+ n = {"c": 0}
+ stats: dict = {}
+
+ def call(key):
+ with lock:
+ n["c"] += 1
+ i = n["c"]
+ if i <= 5:
+ raise RuntimeError("503 high demand")
+ return "ok"
+
+ out = rotate_with_retries(pool, call, classify=_503, outcome_of=lambda e: "5xx", max_attempts=3,
+ storm_attempts=10, storm_max_delay=0.05, sleep=lambda d: None, storm_gate=gate,
+ stats=stats)
+ assert out == "ok"
+ # k1 alone; k2+k3 (quorum of 3 → storm 1); two more (storm 2); the raced pair lands
+ assert stats == {"attempts": 7, "hedged": 3, "storm": 2}
+ assert gate.n_hits == 2, "two hedged failures = two storm waits, not four"
+
+
+def test_hedge_1_disables_and_a_full_pool_of_slots_degrades_to_one_key():
+ lock = threading.Lock()
+
+ def make_call():
+ n = {"c": 0}
+
+ def call(key):
+ with lock:
+ n["c"] += 1
+ i = n["c"]
+ if i <= 3:
+ raise ModelError("503 high demand", retryable=True, status=503)
+ return "ok"
+ return call
+
+ stats: dict = {}
+ _rotate(KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000), make_call(), max_attempts=3, storm_attempts=10,
+ storm_max_delay=0.0, hedge=1, stats=stats)
+ assert stats["hedged"] == 0 and stats["attempts"] == 4
+ # max_in_flight=1: the primary holds the only slot, so a partner is never waited for
+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000, max_in_flight=1)
+ stats = {}
+ _rotate(pool, make_call(), max_attempts=3, storm_attempts=10, storm_max_delay=0.0, stats=stats)
+ assert stats["hedged"] == 0 and stats["attempts"] == 4 and pool.stats()["peak_in_flight"] == 1
+ # max_in_flight=2: a hedged attempt holds both slots
+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000, max_in_flight=2)
+ stats = {}
+ _rotate(pool, make_call(), max_attempts=3, storm_attempts=10, storm_max_delay=0.0, stats=stats)
+ _wait_idle(pool)
+ assert stats["hedged"] >= 1 and pool.stats()["peak_in_flight"] == 2
+
+
+def test_the_worst_error_decides_a_hedged_attempt():
+ """One key says 503 and the other says the request itself is bad: raise the 400 at once
+ instead of storming on a request that can never succeed."""
+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000)
+ stats: dict = {}
+
+ def call(key):
+ if key == "k3":
+ raise ModelError("bad request", retryable=False, status=400)
+ raise ModelError("503 high demand", retryable=True, status=503)
+
+ with pytest.raises(ModelError) as ei:
+ _rotate(pool, call, max_attempts=6, storm_attempts=10, stats=stats)
+ assert ei.value.status == 400 and stats == {"attempts": 3, "hedged": 1, "storm": 0}
+
+
+def test_a_dead_key_among_the_hedge_is_benched_once_the_sibling_wins():
+ pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000)
+
+ def call(key):
+ if key == "k1":
+ raise ModelError("503 high demand", retryable=True, status=503)
+ if key == "k2":
+ raise ModelError("suspended", retryable=False, status=403)
+ return "ok:k3"
+
+ assert _rotate(pool, call, max_attempts=3, storm_attempts=10) == "ok:k3"
+ _wait_idle(pool)
+ assert pool.stats()["n_dead"] == 1, "k3 proved the request fine, so k2's 403 was the key's fault"