Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): a subscription usage cap cools as quota exhaustion until its reset (#2528)

Call White4ec62c54b942
diff --git a/core/handlers/model_hub/adapter.py b/core/handlers/model_hub/adapter.py
index c08d2b474..35e57a833 100644
--- a/core/handlers/model_hub/adapter.py
+++ b/core/handlers/model_hub/adapter.py
@@ -159,7 +159,9 @@ class RawCallOutcome:
# tokens billed for them whether or not the call ended well.
usage: ProtocolUsageReport | None = None
# Only failed HTTP responses supply retry advice: ASCII, at most 128
- # characters, unparsed. L2 owns validation and the resulting retry policy.
+ # characters, unparsed. The Retry-After header, else a usage cap's own reset
+ # from the error envelope in that header's syntax. L2 owns validation and
+ # the resulting retry policy.
retry_after: str | None = None
# Aware UTC receipt time of those headers, before reading the error body.
response_received_at: datetime | None = None
diff --git a/core/handlers/model_hub/classification.py b/core/handlers/model_hub/classification.py
index ce3176b35..e0ce5de7d 100644
--- a/core/handlers/model_hub/classification.py
+++ b/core/handlers/model_hub/classification.py
@@ -125,6 +125,9 @@ _MACHINE_ERROR_TAXONOMY: Final[Mapping[str, _MachineErrorRule]] = {
"rate_limit_exceeded": _MachineErrorRule("transient", True, 80, "fallback", "rate_limited", None, 60, None),
"quota_exceeded": _MachineErrorRule("transient", True, 80, "fallback", "quota_exhausted", None, 300, None),
"insufficient_quota": _MachineErrorRule("transient", True, 80, "fallback", "quota_exhausted", None, 300, None),
+ # A ChatGPT subscription past its usage cap: a 429 whose envelope also names
+ # the reset, which the engine client carries as retry advice.
+ "usage_limit_reached": _MachineErrorRule("transient", True, 80, "fallback", "quota_exhausted", None, 300, None),
"billing_error": _MachineErrorRule("transient", True, 80, "fallback", "balance_exhausted", None, 0, None),
"overloaded": _MachineErrorRule("transient", True, 80, "fallback", "server_error", None, 30, None),
"overloaded_error": _MachineErrorRule("transient", True, 80, "fallback", "server_error", None, 30, None),
diff --git a/docs/plans/model-hub-contracts/adapter-interface.py b/docs/plans/model-hub-contracts/adapter-interface.py
index c08d2b474..35e57a833 100644
--- a/docs/plans/model-hub-contracts/adapter-interface.py
+++ b/docs/plans/model-hub-contracts/adapter-interface.py
@@ -159,7 +159,9 @@ class RawCallOutcome:
# tokens billed for them whether or not the call ended well.
usage: ProtocolUsageReport | None = None
# Only failed HTTP responses supply retry advice: ASCII, at most 128
- # characters, unparsed. L2 owns validation and the resulting retry policy.
+ # characters, unparsed. The Retry-After header, else a usage cap's own reset
+ # from the error envelope in that header's syntax. L2 owns validation and
+ # the resulting retry policy.
retry_after: str | None = None
# Aware UTC receipt time of those headers, before reading the error body.
response_received_at: datetime | None = None
diff --git a/docs/plans/model-hub-contracts/turn-provenance.schema.json b/docs/plans/model-hub-contracts/turn-provenance.schema.json
index edeade036..43522fd7c 100644
--- a/docs/plans/model-hub-contracts/turn-provenance.schema.json
+++ b/docs/plans/model-hub-contracts/turn-provenance.schema.json
@@ -205,6 +205,7 @@
"rate_limit_exceeded",
"quota_exceeded",
"insufficient_quota",
+ "usage_limit_reached",
"billing_error",
"overloaded",
"overloaded_error",
diff --git a/docs/plans/model-hub.md b/docs/plans/model-hub.md
index d09152e49..ecf0ac796 100644
--- a/docs/plans/model-hub.md
+++ b/docs/plans/model-hub.md
@@ -854,10 +854,16 @@ Each family stays at its final value. Positive jitter adds at most 20%, capped
at that family's local maximum, and never makes a local delay shorter. A valid
`Retry-After` integer or HTTP date uses the response-header receipt instant,
not body completion; admission uses the later of upstream advice and local
-backoff. Malformed, past, oversized, and non-ASCII advice is ignored with a
-redacted diagnostic. Scheduling and window duration are monotonic; aware UTC
-timestamps are read projections. Network state is never persisted. Existing
-shaped cooldown persistence remains compatible with earlier releases.
+backoff. Without that header, a usage cap's own reset is the same advice: in a
+failed response whose error envelope has type `usage_limit_reached`, its
+`resets_at` (Unix seconds) as an HTTP date, else its `resets_in_seconds` as an
+integer. A ChatGPT subscription past its cap answers 429 with both, so it cools
+as quota exhaustion until its reset instead of being re-probed. The same field
+names in any other error envelope are not advice. Malformed, past, oversized, and
+non-ASCII advice is ignored with a redacted diagnostic. Scheduling and window
+duration are monotonic; aware UTC timestamps are read projections. Network
+state is never persisted. Existing shaped cooldown persistence remains
+compatible with earlier releases.
Direct `native_cli` failures do not enter this HTTP coordinator: no Hub handle
can prove their later success. They retain fixed native cooldowns (server 30,
rate 60, quota 300 seconds), existing deadline-based launch eligibility and
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index 2ca4f922a..3fba0f4c0 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -327,6 +327,12 @@ scenarios:
kind: contract
layer: unit
test: tests/test_model_hub_retry_advice.py::test_retry_advice_survives_error_body_delays_and_failures
+ - id: MH-RETRY-ADVICE-002
+ name: A subscription usage cap cools its Source as quota exhaustion until the upstream's reset
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_lifecycle.py::test_a_usage_cap_cools_its_source_as_quota_until_the_upstream_reset
- id: MH-RETRY-EVIDENCE-001
name: Unknown buffered success passes through without proving recovery
status: covered
diff --git a/tests/test_model_hub_retry_advice.py b/tests/test_model_hub_retry_advice.py
index a48aeb703..4bbe6a1a6 100644
--- a/tests/test_model_hub_retry_advice.py
+++ b/tests/test_model_hub_retry_advice.py
@@ -167,6 +167,21 @@ async def test_failed_http_response_carries_only_bounded_ascii_advice(
assert requests == [(WIRE[protocol][0], {"model": "retry-advice/model-a", "stream": stream})]
[email protected](("status", "error_type"), [(429, "rate_limit_error"), (503, "server_error")])
+async def test_only_a_usage_cap_turns_its_body_reset_into_advice(status: int, error_type: str) -> None:
+ """Reset fields in any other error body are not retry advice and cannot set a cooldown."""
+ body = {"error": {"type": error_type, "resets_at": 4102444800, "resets_in_seconds": 600}}
+
+ async def respond(_request: web.Request) -> web.Response:
+ return web.json_response(body, status=status)
+
+ async with _engine(respond) as (client, _requests):
+ outcome, _body = await _invoke(client, "openai_responses", False)
+
+ assert (outcome.http_status, outcome.error_type) == (status, error_type)
+ assert outcome.retry_after is None
+
+
@pytest.mark.parametrize("protocol", WIRE)
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streaming"])
@pytest.mark.parametrize("body_mode", ["slow", "timeout", "disconnect", "malformed"])
diff --git a/tests/test_model_hub_retry_lifecycle.py b/tests/test_model_hub_retry_lifecycle.py
index 71374d078..8d699ff2f 100644
--- a/tests/test_model_hub_retry_lifecycle.py
+++ b/tests/test_model_hub_retry_lifecycle.py
@@ -5,9 +5,10 @@ from __future__ import annotations
import asyncio
import copy
import json
+import time
from contextlib import asynccontextmanager
from dataclasses import replace
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
import aiohttp
@@ -405,6 +406,38 @@ def test_failure_persistence_error_keeps_temporary_admission_and_original_termin
asyncio.run(run())
+def test_a_usage_cap_cools_its_source_as_quota_until_the_upstream_reset(tmp_path):
+ """MH-RETRY-ADVICE-002: a subscription's usage cap waits for its own reset, not a 60 s rate limit."""
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ # A ChatGPT subscription past its weekly cap, as the engine relays it:
+ # the upstream body unchanged and no Retry-After header.
+ resets_in = 475224
+ resets_at = int(time.time()) + resets_in
+ body = json.dumps({"error": {
+ "type": "usage_limit_reached", "message": "The usage limit has been reached",
+ "plan_type": "prolite", "resets_at": resets_at, "eligible_promo": None,
+ "limit_window_minutes": 10080, "resets_in_seconds": resets_in,
+ }}, separators=(",", ":"))
+ calls = []
+
+ async def respond(request):
+ calls.append(await request.json())
+ return web.Response(status=429, text=body, content_type="application/json")
+
+ async with loopback_engine(tmp_path, service, respond):
+ async with gateway_client(service, "codex", models["codex"]) as (_gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
+ ending = (await response.json())["error"]
+ assert (ending["code"], ending["cause"]) == (RECOVERY_EXHAUSTED_CODE, "quota_exhausted")
+ state = service.store.load().sources[0].state
+ assert (state.status, state.detail_key) == ("cooldown", "models.source.cooldown.quota_exhausted")
+ assert state.retry_at == datetime.fromtimestamp(resets_at, timezone.utc).isoformat()
+ # The cap is not probed again inside the turn's recovery window.
+ assert len(calls) == 1 and clock.delays == []
+ asyncio.run(run())
+
+
@pytest.mark.parametrize("persisted", [False, True])
@pytest.mark.parametrize("reason,delay", [
("server_error", 30), ("rate_limited", 60), ("quota_exhausted", 300), ("network", 1),
diff --git a/tests/test_model_hub_structure_guards.py b/tests/test_model_hub_structure_guards.py
index ef52db41b..1fb4fb75c 100644
--- a/tests/test_model_hub_structure_guards.py
+++ b/tests/test_model_hub_structure_guards.py
@@ -307,6 +307,7 @@ MACHINE_ERROR_FAMILY_FIXTURES = {
"rate_limit_exceeded",
"quota_exceeded",
"insufficient_quota",
+ "usage_limit_reached",
"billing_error",
"overloaded",
"overloaded_error",
diff --git a/ui/src/components/settings/models/sourceStatePresentation.test.ts b/ui/src/components/settings/models/sourceStatePresentation.test.ts
index dddccdd69..cd7d11514 100644
--- a/ui/src/components/settings/models/sourceStatePresentation.test.ts
+++ b/ui/src/components/settings/models/sourceStatePresentation.test.ts
@@ -92,6 +92,22 @@ describe('sourceStatePresentation', () => {
});
});
+ it('names a used-up quota cycle instead of counting down to its reset', () => {
+ const now = Date.parse('2026-10-12T17:32:58Z');
+ const cooling = (detailKey: SourceState['detail_key'], retryAt: string) => sourceStatePresentation(
+ state('cooldown', { retry_at: retryAt, detail_key: detailKey }), 'card', 'en', now,
+ );
+ expect(cooling('models.source.cooldown.quota_exhausted', '2026-10-18T05:33:22Z')).toEqual({
+ key: 'models.source.cooldown.quota_exhausted',
+ textClass: 'model-hub-ink-gold',
+ dotClass: 'bg-gold',
+ });
+ expect(cooling('models.source.cooldown.quota_exhausted', '2026-10-12T17:30:00Z').key)
+ .toBe('settings.models.upstream.state.unavailableDue');
+ expect(cooling('models.source.cooldown.rate_limited', '2026-10-12T17:33:58Z').key)
+ .toBe('settings.models.upstream.state.unavailableRetry');
+ });
+
it('does not imply another request ran after a cooldown deadline passed', () => {
expect(sourceStatePresentation(
state('cooldown', { retry_at: '2026-08-11T09:59:00Z' }),
diff --git a/ui/src/components/settings/models/sourceStatePresentation.ts b/ui/src/components/settings/models/sourceStatePresentation.ts
index b0eab3239..c047567db 100644
--- a/ui/src/components/settings/models/sourceStatePresentation.ts
+++ b/ui/src/components/settings/models/sourceStatePresentation.ts
@@ -1,4 +1,5 @@
import type { TranslationKey } from '@/i18n/types';
+import { QUOTA_EXHAUSTED } from './supply';
import type { AgentBackend, AdoptedBy, NeedsActionDetailKey, SourceState, SourceStatus } from './types';
export type SourceStateSurface = 'card' | 'detail';
@@ -70,6 +71,15 @@ const STATUS_RULES: Readonly<Record<SourceStatus, Rule>> = {
dotClass: 'bg-gold',
};
}
+ // A used-up cycle returns at the provider's reset, often days away: the
+ // cause is the reading, not a countdown in minutes.
+ if (state.detail_key === QUOTA_EXHAUSTED) {
+ return {
+ key: QUOTA_EXHAUSTED,
+ textClass: 'model-hub-ink-gold',
+ dotClass: 'bg-gold',
+ };
+ }
return {
key: 'settings.models.upstream.state.unavailableRetry',
values: {
diff --git a/ui/src/components/settings/models/supply.ts b/ui/src/components/settings/models/supply.ts
index 1bce48f40..0e60af251 100644
--- a/ui/src/components/settings/models/supply.ts
+++ b/ui/src/components/settings/models/supply.ts
@@ -27,7 +27,7 @@ export const isUnhealthy = (state: SourceState): boolean =>
state.status !== 'active' && state.status !== 'standby';
/** The one cooldown reason that is about money rather than weather. */
-const QUOTA_EXHAUSTED = 'models.source.cooldown.quota_exhausted';
+export const QUOTA_EXHAUSTED = 'models.source.cooldown.quota_exhausted';
/**
* Worth reading, not just worth reporting — what earns the gold sub-line.
diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.py
index dc8873099..e4e35c6d5 100644
--- a/vibe/model_hub_runtime/client.py
+++ b/vibe/model_hub_runtime/client.py
@@ -15,6 +15,7 @@ from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone
from decimal import Decimal
+from email.utils import format_datetime
from typing import Any, AsyncIterator, BinaryIO, Literal, Mapping, TypeVar, cast
import aiohttp
@@ -564,6 +565,10 @@ class EngineClient:
observed_payload = ProtocolObservation()
finally:
error_body.close()
+ if retry_after is None:
+ retry_after = _upstream_reset_advice(
+ payload, observed_payload.error_envelope_paths or (("error",),),
+ )
if _is_local_model_registration_failure(
response.status,
payload,
@@ -1920,6 +1925,52 @@ def _upstream_error_detail(
return None
+def _upstream_reset_advice(
+ payload: bytes,
+ envelope_paths: tuple[ErrorEnvelopePath, ...],
+) -> str | None:
+ """Project a usage cap's own reset as Retry-After advice, in that header's syntax.
+
+ A ChatGPT subscription past its cap answers 429 ``usage_limit_reached`` with
+ ``resets_at`` (Unix seconds) and ``resets_in_seconds`` in the error envelope
+ and no header. The first becomes an HTTP-date and the second delay-seconds;
+ L2 validates either exactly like the header. Only that envelope's reset is
+ advice: the same field names in any other error say nothing about when the
+ Source can serve again.
+ """
+
+ fields = ("type", "resets_at", "resets_in_seconds")
+ reset_paths = tuple((*path, field) for path in envelope_paths for field in fields)
+ values: dict[JSONPath, object] = {}
+
+ def visit(
+ path: JSONPath,
+ event: JSONEvent,
+ value: object | None,
+ _scope: JSONScope,
+ ) -> None:
+ if event == "replace":
+ values.pop(path, None)
+ elif event == "scalar":
+ values[path] = value
+
+ if not payload or not project_json_reader(io.BytesIO(payload), reset_paths, visit):
+ return None
+ for path in envelope_paths:
+ if values.get((*path, "type")) != "usage_limit_reached":
+ continue
+ resets_at = values.get((*path, "resets_at"))
+ if type(resets_at) is int and resets_at > 0:
+ try:
+ return format_datetime(datetime.fromtimestamp(resets_at, timezone.utc), usegmt=True)
+ except (OverflowError, OSError, ValueError):
+ pass
+ resets_in_seconds = values.get((*path, "resets_in_seconds"))
+ if type(resets_in_seconds) is int and resets_in_seconds >= 0:
+ return str(resets_in_seconds)
+ return None
+
+
def _bounded_upstream_detail(message: str) -> str | None:
# Control characters (terminal escapes, BEL) would act on whatever renders
# the text, a log viewer included; they carry no message content.