Skip to content

The agent that grows with you

fix(usage-window): retry a plan sync the refresh did not fix

Teknium9c033102cb21
diff --git a/agent/plan_snapshot_sync.py b/agent/plan_snapshot_sync.py
index 672a94c8858..0273fe5875d 100644
--- a/agent/plan_snapshot_sync.py
+++ b/agent/plan_snapshot_sync.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import logging
import threading
+import time
from typing import Any, Optional
logger = logging.getLogger(__name__)
@@ -18,9 +19,11 @@ logger = logging.getLogger(__name__)
_lock = threading.Lock()
# Both keyed by Hermes home: one process can serve several profiles, each with its own account.
_inflight: set[str] = set()
-# What the headers said the last time a refresh ran. The portal can trail the gateway by a few minutes
-# (usage is ingested in batches), so the same mismatch is refreshed once, not on every reply meanwhile.
-_last_attempt: dict[str, tuple[bool, frozenset[str]]] = {}
+# What the headers said the last time a refresh ran, and when. The portal can trail the gateway by a few
+# minutes (usage is ingested in batches), so the same mismatch is refreshed once per retry interval, not on
+# every reply meanwhile, and not never again: a refresh that read the portal before it caught up retries.
+_last_attempt: dict[str, tuple[tuple[bool, frozenset[str]], float]] = {}
+_RETRY_AFTER_S = 120.0
def _profile_key() -> str:
@@ -56,7 +59,8 @@ def plan_snapshot_is_stale(state: Any, snapshot: Any) -> bool:
def _refresh(plan_changed: bool, key: str) -> None:
try:
from hermes_cli.nous_account import get_nous_portal_account_info
- get_nous_portal_account_info(force_fresh=True)
+ if getattr(get_nous_portal_account_info(force_fresh=True), "source", None) == "error":
+ _last_attempt.pop(key, None) # a failed fetch (reported, not raised) is no attempt either
if plan_changed:
# The per-model levels (Large / Free tags) come with the catalog; refetch so they move with the plan.
from hermes_cli.models_pricing import get_pricing_for_provider
@@ -91,11 +95,13 @@ def sync_plan_snapshot(state: Any) -> bool:
on_window = bool(getattr(state, "is_usage_window", False))
seen = (on_window, frozenset(g.id for g in getattr(state, "gauges", ()) if g.full))
plan_changed = on_window != (_snapshot_full_ids(snapshot) is not None)
+ now = time.monotonic()
with _lock:
- if key in _inflight or _last_attempt.get(key) == seen:
+ last = _last_attempt.get(key)
+ if key in _inflight or (last is not None and last[0] == seen and now - last[1] < _RETRY_AFTER_S):
return False
_inflight.add(key)
- _last_attempt[key] = seen
+ _last_attempt[key] = (seen, now)
# Under this profile's contextvars (its Hermes home and auth store), as the credits seed thread does.
from agent.memory_provider import spawn_context_thread
spawn_context_thread(_refresh, args=(plan_changed, key), name="nous-plan-sync").start()
diff --git a/tests/agent/test_plan_snapshot_sync.py b/tests/agent/test_plan_snapshot_sync.py
index 013ed88f1bd..51a04d29716 100644
--- a/tests/agent/test_plan_snapshot_sync.py
+++ b/tests/agent/test_plan_snapshot_sync.py
@@ -77,6 +77,27 @@ def test_a_limit_change_refreshes_only_the_account(wired):
assert calls == ["account"]
+def test_a_mismatch_the_refresh_did_not_fix_retries(wired, monkeypatch):
+ """A refresh that read a lagging portal (same stale gauges back) or a failed fetch reported as
+ ``source="error"`` must not end re-syncing for that mismatch: an error rearms at once, a lag after the
+ retry interval."""
+ import hermes_cli.nous_account as na
+ calls, box = wired
+ box["snap"] = ON_PLAN
+ monkeypatch.setattr(na, "get_nous_portal_account_info",
+ lambda force_fresh=False: calls.append("account") or _snap(None, "error"))
+ assert sync.sync_plan_snapshot(_state(True, ["day"])) is True
+ assert sync.sync_plan_snapshot(_state(True, ["day"])) is True # the fetch failed: try again
+ clock = {"now": 1_000.0}
+ monkeypatch.setattr(sync.time, "monotonic", lambda: clock["now"])
+ monkeypatch.setattr(na, "get_nous_portal_account_info", lambda force_fresh=False: calls.append("account") or ON_PLAN)
+ assert sync.sync_plan_snapshot(_state(True, ["day"])) is True
+ assert sync.sync_plan_snapshot(_state(True, ["day"])) is False # the portal lags: not on every reply
+ clock["now"] += sync._RETRY_AFTER_S
+ assert sync.sync_plan_snapshot(_state(True, ["day"])) is True
+ assert calls == ["account"] * 4
+
+
def test_back_in_step_rearms(wired):
_calls, box = wired
assert sync.sync_plan_snapshot(_state(True)) is True