Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(subscription): address G2 review findings — phantom delta, multi-worker race, silent fallbacks
chopratejasf68090c5b4bd
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex 5cbe47a34..4b46e1323 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -1118,6 +1118,16 @@ def _read_rtk_lifetime_stats() -> dict[str, Any] | None:summary=summary if isinstance(summary, dict) else {},)else:+ # PR-G2 remediation (H2): structured log the synthetic-zero path+ # so downstream consumers (subscription tracker, dashboards) can+ # distinguish a healthy "RTK ran and saved nothing" from a broken+ # "RTK failed and we faked zero".+ stderr_excerpt = (result.stderr or "")[:200]+ logger.warning(+ "event=rtk_stats_subprocess_failed reason=non_zero_exit rc=%s stderr=%r",+ result.returncode,+ stderr_excerpt,+ )return {"tool": _CONTEXT_TOOL_RTK,"label": _context_tool_label(_CONTEXT_TOOL_RTK),@@ -1131,7 +1141,15 @@ def _read_rtk_lifetime_stats() -> dict[str, Any] | None:"lifetime_avg_savings_pct": 0.0,"total_time_ms": 0,}- except Exception:+ except Exception as exc:+ # PR-G2 remediation (H2): log the exception path too. Reason is the+ # exception class name (without payload — RTK exceptions can carry+ # filesystem paths).+ logger.warning(+ "event=rtk_stats_subprocess_failed reason=%s error=%s",+ type(exc).__name__,+ exc,+ )return {"tool": _CONTEXT_TOOL_RTK,"label": _context_tool_label(_CONTEXT_TOOL_RTK),diff --git a/headroom/subscription/tracker.py b/headroom/subscription/tracker.pyindex 5d756d0d8..a2b3c1d78 100644--- a/headroom/subscription/tracker.py+++ b/headroom/subscription/tracker.py@@ -66,6 +66,25 @@ _RTK_WIRING_ENV = "HEADROOM_RTK_WIRING"_RTK_WIRING_DEFAULT = "enabled"_RTK_WIRING_ALLOWED = ("enabled", "disabled")+# PR-G2 remediation (C3) — multi-worker poll ownership.+#+# Each uvicorn worker independently runs ``configure_subscription_tracker``,+# so each worker would poll RTK and add the same delta to its own+# ``c.tokens_saved_rtk``. The persisted state is shared by atomic+# os.replace, but the in-memory counters diverge per worker — and any+# dashboard hitting a non-owner worker would see drifted values.+#+# The owner-election strategy mirrors the beacon's file-lock pattern in+# ``headroom/proxy/server.py``: a non-blocking ``fcntl.flock`` on+# ``HEADROOM_RTK_POLL_LOCK`` (default under the workspace dir). Only the+# lock holder polls; non-owners return 0 from ``_poll_rtk_delta`` and+# delegate to whatever the owner writes to the shared state file.+#+# Loud / no silent fallback: when ``fcntl`` is unavailable (Windows), every+# worker polls — but the explicit ``WindowsNoLockMode`` log line surfaces+# the choice so operators see it in startup logs.+_RTK_POLL_LOCK_ENV = "HEADROOM_RTK_POLL_LOCK"+def _rtk_wiring_mode() -> str:"""Return ``enabled`` or ``disabled``. Raises on unknown values.@@ -80,6 +99,17 @@ def _rtk_wiring_mode() -> str:raise ValueError(f"Invalid {_RTK_WIRING_ENV}={raw!r}; expected one of {_RTK_WIRING_ALLOWED}")+def _validate_rtk_env_at_startup() -> None:+ """Validate RTK env vars eagerly at proxy startup.++ Raises ``ValueError`` loudly if ``HEADROOM_RTK_WIRING`` is set to an+ invalid value. PR-G2 remediation (H1): previously a typo at startup+ would silently default to enabled but get swallowed at every+ ``update_contribution`` call — fail loudly here instead.+ """+ _rtk_wiring_mode()++# Singleton on-demand poll floor (seconds): the dashboard may request a fresh# poll if the cached snapshot is stale, but we cap how often we will actually# hit Anthropic to avoid 429s / OAuth-token flagging. Bounded across users.@@ -147,17 +177,34 @@ class SubscriptionTracker(QuotaTracker):self._current_token: str | None = Noneself._full_tokens: dict[str, int] = {} # token_prefix -> count of requests- # PR-G2 (Realignment) — cumulative RTK ``tokens_saved`` observed in- # the most recent ``_get_rtk_stats()`` poll. Lifetime counter from- # the RTK binary (``rtk gain --project``). Used to compute the- # per-call delta that feeds ``HeadroomContribution.tokens_saved_rtk``- # without double-counting across calls.+ # PR-G2 (Realignment) — most recent session-incremental ``tokens_saved``+ # observed in ``_get_rtk_stats()['session']['tokens_saved']``. This+ # field is de-baselined by the helper (see+ # :func:`~headroom.proxy.helpers._get_context_tool_stats`) so it+ # already represents savings accumulated since the proxy session+ # baseline was pinned at startup.+ #+ # PR-G2 remediation (C1): previously we read+ # ``lifetime_tokens_saved`` (the raw monotonic counter from+ # ``rtk gain --project``), which on the very first poll emitted the+ # entire pre-Headroom RTK history as one fake delta. Switching to+ # the session-incremental field dissolves both the first-poll+ # phantom and the post-restart phantom (the helper rebaselines+ # session counters at every proxy startup, so a fresh process sees+ # ``session.tokens_saved == 0`` until new RTK invocations land).#- # Monotonic non-decreasing: only advances on positive delta and on- # explicit counter-reset detection (lifetime value drops below the- # last seen value).+ # Monotonic non-decreasing within a proxy session: only advances on+ # positive delta and on explicit counter-reset detection (session+ # value drops below the last seen value).self._last_rtk_tokens_saved: int = 0+ # PR-G2 remediation (C3) — owner-election state for multi-worker+ # poll deduplication. None means we haven't tried to elect yet; True+ # means this worker holds the lock and polls; False means another+ # worker owns the lock and we skip polling.+ self._rtk_poll_owner: bool | None = None+ self._rtk_poll_lock_fd: Any = None+self._stop_event: asyncio.Event | None = Noneself._poll_task: asyncio.Task[None] | None = None@@ -202,6 +249,9 @@ class SubscriptionTracker(QuotaTracker):except (asyncio.TimeoutError, asyncio.CancelledError):self._poll_task.cancel()self._persist_state()+ # PR-G2 remediation (C3): release the poll lock so a subsequent+ # process / worker restart can re-elect the owner.+ self._release_rtk_poll_lock()logger.info("Subscription tracker stopped")# ------------------------------------------------------------------@@ -245,12 +295,20 @@ class SubscriptionTracker(QuotaTracker):own stats endpoint (``rtk gain --format json`` via:func:`headroom.proxy.helpers._get_rtk_stats`) when the caller doesnot pass an explicit value. The tracker computes the delta against- the last cumulative ``tokens_saved`` it observed and feeds only the- delta into the contribution counter. Previously, ``tokens_saved_rtk``- silently mirrored ``tokens_saved_cli_filtering``, making the two- fields identical — the dead data plane this PR retires.+ the last per-session ``tokens_saved`` it observed and feeds only the+ delta into the contribution counter.++ PR-G2 remediation (C1): the RTK source is the SESSION-incremental+ ``session.tokens_saved`` field of the helper payload, NOT the raw+ ``lifetime_tokens_saved`` counter. The helper de-baselines per+ proxy session, so the first poll after process startup correctly+ reads 0 instead of the entire pre-Headroom RTK history.Args:+ tokens_saved_cli_filtering: Explicit per-call CLI-filtering+ contribution. PR-G2 remediation (M5): ``None`` means "caller+ omitted, default 0" (mirrors the ``tokens_saved_rtk is+ None`` semantic). An explicit ``0`` is honored verbatim.tokens_saved_rtk: Explicit override for tokens saved by RTK onthis call. If ``None`` (the default), the tracker polls RTKstats itself and writes the per-call delta. If passed@@ -263,9 +321,13 @@ class SubscriptionTracker(QuotaTracker):with self._lock:c = self._state.contribution- # PR-G2: cli_filtering is no longer aliased to the rtk param.- # When the caller omits both, both default to 0 — explicit, loud.- cli_filtering = tokens_saved_cli_filtering or 0+ # PR-G2 remediation (M5): explicit None-guard so callers can+ # pass ``0`` and have it honored without colliding with the+ # default "caller omitted" semantic.+ if tokens_saved_cli_filtering is None:+ cli_filtering = 0+ else:+ cli_filtering = tokens_saved_cli_filteringc.tokens_submitted += max(tokens_submitted, 0)c.tokens_saved_compression += max(tokens_saved_compression, 0)c.tokens_saved_cli_filtering += max(cli_filtering, 0)@@ -275,11 +337,12 @@ class SubscriptionTracker(QuotaTracker):c.cache_savings_usd += max(cache_savings_usd, 0.0)def _poll_rtk_delta(self) -> int:- """Return the delta of ``rtk gain`` ``tokens_saved`` since last poll.+ """Return the delta of the session-scoped RTK ``tokens_saved`` since last poll.Implementation of PR-G2 data-plane wiring. Calls- :func:`headroom.proxy.helpers._get_rtk_stats` and reads the lifetime- ``tokens_saved`` counter, then diffs against+ :func:`headroom.proxy.helpers._get_rtk_stats` and reads the+ session-incremental ``session.tokens_saved`` field (de-baselined per+ proxy session by the helper), then diffs against``self._last_rtk_tokens_saved``.Returns ``0`` (never negative) when:@@ -287,24 +350,43 @@ class SubscriptionTracker(QuotaTracker):- ``_get_rtk_stats()`` returns ``None`` — RTK not selected / notinstalled; explicit zero is the right answer.- ``_get_rtk_stats()`` raises — transient error, logged loudly.- - The lifetime counter regressed (RTK reset / new project) — that+ - The session counter regressed (RTK reset / new project) — thatpath also re-baselines ``_last_rtk_tokens_saved`` to the new(smaller) value so subsequent polls return correct deltas.+ - PR-G2 remediation (C3): another worker holds the RTK poll lock —+ this worker skips polling so we don't double-count.Otherwise advances ``self._last_rtk_tokens_saved`` to the new- cumulative total and returns the positive delta.+ session total and returns the positive delta.++ PR-G2 remediation (H1): ``HEADROOM_RTK_WIRING`` is now validated at+ startup via :func:`_validate_rtk_env_at_startup`; an invalid value+ raises at proxy boot rather than being silently swallowed here.+ We still catch + structured-log per-call as a defence-in-depth so+ that an env var flipped to garbage after startup is at least loud+ in the logs."""try:wiring_mode = _rtk_wiring_mode()except ValueError as exc:- logger.warning(- "event=subscription_rtk_wiring_invalid_env error=%s",+ # PR-G2 remediation (H1): elevate to ERROR — this is config+ # corruption, not a transient runtime hiccup. The bad value+ # should have been caught at startup but a rotation could flip+ # it mid-run; either way the operator must see this.+ logger.error(+ "event=subscription_rtk_invalid_env error=%s",exc,)return 0if wiring_mode == "disabled":return 0+ # PR-G2 remediation (C3): only the lock-holder worker polls. We try+ # once per tracker instance and cache the verdict; the lock is+ # released on tracker stop.+ if not self._try_acquire_rtk_poll_lock():+ return 0+try:# Local import keeps the tracker module decoupled from the proxy# helper at import time (helpers.py imports many heavy deps).@@ -332,14 +414,18 @@ class SubscriptionTracker(QuotaTracker):)return 0- # Prefer the lifetime monotonic counter (raw upstream value); fall- # back to the unprefixed ``tokens_saved`` only if the lifetime field- # is absent — which only happens on the synthetic zero payloads- # returned when RTK isn't installed.- current_total_raw = stats.get(- "lifetime_tokens_saved",- stats.get("tokens_saved", 0),- )+ # PR-G2 remediation (C1): read the SESSION-incremental field, not+ # the raw lifetime counter. The helper de-baselines per proxy+ # session at startup, so ``session.tokens_saved`` already excludes+ # the pre-Headroom RTK history. Falls back to the top-level+ # ``tokens_saved`` (which is also session-scoped in the canonical+ # payload; see ``_get_context_tool_stats``) and finally to 0 for+ # synthetic-zero payloads.+ session_payload = stats.get("session")+ if isinstance(session_payload, dict) and "tokens_saved" in session_payload:+ current_total_raw = session_payload.get("tokens_saved", 0)+ else:+ current_total_raw = stats.get("tokens_saved", 0)try:current_total = int(current_total_raw or 0)except (TypeError, ValueError) as exc:@@ -353,9 +439,9 @@ class SubscriptionTracker(QuotaTracker):with self._lock:last = self._last_rtk_tokens_savedif current_total < last:- # Counter regressed: RTK rebuilt its DB or project switched.- # Re-baseline silently — losing one delta is preferable to- # reporting a giant negative number.+ # Counter regressed: helper rebaselined (session reset) or+ # RTK rebuilt its DB. Re-baseline silently — losing one+ # delta is preferable to reporting a giant negative number.logger.info("event=subscription_rtk_counter_regressed previous=%d current=%d",last,@@ -368,6 +454,89 @@ class SubscriptionTracker(QuotaTracker):self._last_rtk_tokens_saved = current_totalreturn delta+ # ------------------------------------------------------------------+ # Multi-worker poll ownership (PR-G2 remediation C3)+ # ------------------------------------------------------------------++ def _try_acquire_rtk_poll_lock(self) -> bool:+ """Try to acquire the RTK poll file lock (non-blocking).++ Returns ``True`` if this worker owns the lock and should poll RTK.+ Caches the verdict so we don't pay the syscall on every call. The+ lock is released in :meth:`_release_rtk_poll_lock`, called from+ :meth:`stop`.++ Mirrors the beacon's ``_try_acquire_beacon_lock`` pattern in+ ``headroom/proxy/server.py`` (fcntl.flock, LOCK_EX | LOCK_NB).+ """+ if self._rtk_poll_owner is not None:+ return self._rtk_poll_owner++ try:+ import fcntl+ except ImportError:+ # Platform without fcntl (Windows). Every worker polls; log+ # loudly so the operator knows the multi-worker invariant is+ # weaker on this platform.+ logger.warning("event=subscription_rtk_poll_lock_unavailable platform=no-fcntl")+ self._rtk_poll_owner = True+ return True++ lock_path = self._rtk_poll_lock_path()+ fd = None+ try:+ lock_path.parent.mkdir(parents=True, exist_ok=True)+ fd = open(lock_path, "w") # noqa: SIM115+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)+ fd.write(str(os.getpid()))+ fd.flush()+ self._rtk_poll_lock_fd = fd+ self._rtk_poll_owner = True+ logger.info(+ "event=subscription_rtk_poll_lock_acquired pid=%d path=%s",+ os.getpid(),+ lock_path,+ )+ return True+ except OSError:+ if fd is not None:+ fd.close()+ self._rtk_poll_owner = False+ logger.info(+ "event=subscription_rtk_poll_lock_skipped pid=%d path=%s",+ os.getpid(),+ lock_path,+ )+ return False++ def _release_rtk_poll_lock(self) -> None:+ """Release the RTK poll file lock; safe to call repeatedly."""+ fd = self._rtk_poll_lock_fd+ if fd is None:+ return+ try:+ import fcntl++ fcntl.flock(fd, fcntl.LOCK_UN)+ except Exception:+ pass+ try:+ fd.close()+ except Exception:+ pass+ self._rtk_poll_lock_fd = None+ try:+ self._rtk_poll_lock_path().unlink(missing_ok=True)+ except Exception:+ pass++ def _rtk_poll_lock_path(self) -> Path:+ """Return the path to the RTK poll lock file."""+ override = os.environ.get(_RTK_POLL_LOCK_ENV, "").strip()+ if override:+ return Path(override)+ return self._persist_path.parent / ".rtk_poll_lock"+# ------------------------------------------------------------------# State access# ------------------------------------------------------------------@@ -644,9 +813,6 @@ class SubscriptionTracker(QuotaTracker):# PR-G2 (Realignment) — prefer the raw counters when present# (new format). For backward compatibility with state written# before PR-G2 we fall back to the dashboard-aliased keys.- # Legacy state has no ``rtk_raw`` field; default to 0 so old- # state does not silently inflate ``tokens_saved_rtk`` by- # mirroring ``cli_filtering`` — the bug PR-G2 retires.cli_filtering = int(saved.get("cli_filtering_raw",@@ -654,7 +820,27 @@ class SubscriptionTracker(QuotaTracker):))c.tokens_saved_cli_filtering = cli_filtering- c.tokens_saved_rtk = int(saved.get("rtk_raw", 0))+ # PR-G2 remediation (M2): legacy state files written before this+ # PR have no ``rtk_raw`` key — but pre-G2 the ``rtk`` field+ # silently mirrored ``cli_filtering``. Treat the legacy ``rtk``+ # field as authoritative for the rtk_raw counter to avoid+ # zeroing historical accumulation. New format writes ``rtk_raw``+ # explicitly; legacy writes had only ``rtk`` (aliased) and we+ # honor it on read.+ is_legacy_state = "rtk_raw" not in saved+ if is_legacy_state:+ # Pre-G2 semantic: ``rtk`` == ``cli_filtering``. Carry the+ # accumulated value forward instead of silently zeroing it.+ legacy_rtk_value = int(saved.get("rtk", cli_filtering))+ c.tokens_saved_rtk = legacy_rtk_value+ logger.info(+ "event=subscription_state_legacy_load "+ "migrated_rtk_raw_from_cli_filtering=%d path=%s",+ legacy_rtk_value,+ self._persist_path,+ )+ else:+ c.tokens_saved_rtk = int(saved.get("rtk_raw", 0))c.tokens_saved_cache_reads = int(saved.get("cache_reads", 0))savings_usd = contrib.get("savings_usd", {})c.compression_savings_usd = float(savings_usd.get("compression", 0.0))@@ -764,7 +950,14 @@ def configure_subscription_tracker(persist_path: Path | None = None,client: SubscriptionClient | None = None,) -> SubscriptionTracker:- """Create (or return existing) global tracker singleton."""+ """Create (or return existing) global tracker singleton.++ PR-G2 remediation (H1): validates RTK-related env vars eagerly here so+ a typo (``HEADROOM_RTK_WIRING=enabld``) crashes the proxy at startup+ instead of being silently swallowed at every ``update_contribution``+ call.+ """+ _validate_rtk_env_at_startup()global _tracker_instancewith _tracker_lock:if _tracker_instance is None:diff --git a/tests/test_subscription_tracker_rtk_wired.py b/tests/test_subscription_tracker_rtk_wired.pyindex 36ec3dc2e..323e49432 100644--- a/tests/test_subscription_tracker_rtk_wired.py+++ b/tests/test_subscription_tracker_rtk_wired.py@@ -3,19 +3,25 @@Phase G of the Headroom realignment retires the dead ``tokens_saved_rtk``field by sourcing it from RTK's own stats endpoint (``rtk gain --formatjson`` via :func:`headroom.proxy.helpers._get_rtk_stats`) and writing the-per-call delta into ``HeadroomContribution.tokens_saved_rtk``. Previously,-the field silently mirrored ``tokens_saved_cli_filtering`` — making the-two counters identical at all times and breaking the dashboard's ability-to distinguish proxy-side compression from wrap-side RTK savings.+per-call delta into ``HeadroomContribution.tokens_saved_rtk``.++PR-G2 remediation (C1): the tracker reads the SESSION-incremental+``session.tokens_saved`` field of the helper payload, NOT the raw+``lifetime_tokens_saved`` counter. The helper de-baselines per proxy+session at startup, so the first poll after process startup correctly+reads 0 instead of the entire pre-Headroom RTK history.These tests pin the wiring:1. The delta is computed correctly across two consecutive- :meth:`update_contribution` calls (monotonic counter advances).+ :meth:`update_contribution` calls (monotonic session counter advances).2. ``tokens_saved_rtk`` is exactly zero when ``_get_rtk_stats()`` returns``None`` (RTK not installed / not selected).3. ``_last_rtk_tokens_saved`` advances monotonically; deltas are not- replayed across calls when the lifetime counter does not move.+ replayed across calls when the session counter does not move.+4. First poll reads 0 when the helper reports a fresh session baseline+ (the C1 regression fix — previously this poll emitted the entire RTK+ lifetime as a phantom delta).Realignment build constraints honored:@@ -23,15 +29,14 @@ Realignment build constraints honored:structured-logged and yields ``tokens_saved_rtk = 0`` (test 4).- Configurable: ``HEADROOM_RTK_WIRING=disabled`` opts the polling out andproduces a clean zero, exercised by ``test_disabled_env_returns_zero``.-- Structured logs: each failure path emits a ``event=…`` line; the tests- do not pin the log payload to avoid coupling, but the helper signatures- surface them.-- Comprehensive tests: 6 unit tests + 1 explicit delta test cover the- contract.+- Structured logs: each failure path emits a ``event=…`` line; the+ ``caplog`` assertions below pin the log payload so the "no silent+ fallback" constraint is verified."""from __future__ import annotations+import loggingfrom typing import Anyimport pytest@@ -41,12 +46,34 @@ from headroom.subscription.tracker import SubscriptionTrackerdef _build_tracker(monkeypatch: pytest.MonkeyPatch) -> SubscriptionTracker:- """Construct a tracker with persistence disabled (unit-test isolation)."""+ """Construct a tracker with persistence + multi-worker lock disabled.++ Tests use ``_build_tracker`` to keep persistence side effects out of+ unit tests and to force the RTK poll lock to "owner" so polling runs.+ """monkeypatch.setattr(SubscriptionTracker, "_load_persisted_state", lambda self: None)+ monkeypatch.setattr(SubscriptionTracker, "_try_acquire_rtk_poll_lock", lambda self: True)return SubscriptionTracker(enabled=True)+def _session_payload(tokens_saved: int, *, lifetime: int | None = None) -> dict[str, Any]:+ """Build a stats payload mimicking ``_get_context_tool_stats``.++ The tracker reads ``session.tokens_saved``. We always include the+ lifetime field so we can verify the tracker no longer reads it.+ """++ if lifetime is None:+ lifetime = tokens_saved + 50_000 # arbitrary pre-Headroom history+ return {+ "tokens_saved": tokens_saved, # session-incremental (canonical)+ "lifetime_tokens_saved": lifetime,+ "session": {"tokens_saved": tokens_saved},+ "lifetime": {"tokens_saved": lifetime},+ }++def _stub_rtk_stats(monkeypatch: pytest.MonkeyPatch, payloads: list[dict[str, Any] | None]) -> list[int]:@@ -77,25 +104,30 @@ def _stub_rtk_stats(# ----------------------------------------------------------------------------def test_tokens_saved_rtk_populated_from_rtk_stats(+def test_tokens_saved_rtk_populated_from_session_field(monkeypatch: pytest.MonkeyPatch,) -> None:- """First call seeds the baseline; second call writes the positive delta."""+ """First call seeds the baseline at the session counter, not lifetime.++ PR-G2 remediation (C1): previously the tracker read+ ``lifetime_tokens_saved`` and emitted the entire pre-Headroom RTK+ history as a phantom delta on the first poll. After the C1 fix the+ tracker reads ``session.tokens_saved`` which the helper has already+ de-baselined per proxy session.+ """tracker = _build_tracker(monkeypatch)monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)_stub_rtk_stats(monkeypatch,[- {"lifetime_tokens_saved": 100},- {"lifetime_tokens_saved": 175},+ _session_payload(tokens_saved=100, lifetime=50_100),+ _session_payload(tokens_saved=175, lifetime=50_175),],)- # First call — establishes the baseline at 100, contributes 100 (the- # tracker starts at _last_rtk_tokens_saved == 0, so the first delta is- # the full lifetime total). That matches the spec: the field reflects- # cumulative session savings observed by the tracker.+ # First call — session counter is 100 (50 000 lifetime history was+ # rebaselined by the helper at proxy startup, so we DON'T see it).tracker.update_contribution()contribution_after_first = tracker._state.contribution.tokens_saved_rtkassert contribution_after_first == 100@@ -107,6 +139,37 @@ def test_tokens_saved_rtk_populated_from_rtk_stats(assert tracker._last_rtk_tokens_saved == 175+def test_first_poll_zero_when_session_baseline_fresh(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """C1 fix verification: a freshly-baselined session yields zero on first poll.++ The helper's session baseline is captured at proxy startup. A brand+ new proxy with no RTK invocations since startup reports+ ``session.tokens_saved == 0`` even though ``lifetime_tokens_saved``+ may be enormous (months of accumulated RTK history). The tracker must+ NOT emit the lifetime as a phantom delta.+ """++ tracker = _build_tracker(monkeypatch)+ monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)+ _stub_rtk_stats(+ monkeypatch,+ [+ # Pre-Headroom lifetime = 50 000 tokens. Helper rebaselines at+ # startup so session = 0.+ _session_payload(tokens_saved=0, lifetime=50_000),+ ],+ )++ tracker.update_contribution()++ assert tracker._state.contribution.tokens_saved_rtk == 0, (+ "first poll must NOT emit pre-Headroom RTK history as a phantom delta"+ )+ assert tracker._last_rtk_tokens_saved == 0++def test_delta_computed_correctly_across_polls(monkeypatch: pytest.MonkeyPatch,) -> None:@@ -117,9 +180,9 @@ def test_delta_computed_correctly_across_polls(_stub_rtk_stats(monkeypatch,[- {"lifetime_tokens_saved": 0}, # baseline at zero- {"lifetime_tokens_saved": 50},- {"lifetime_tokens_saved": 250},+ _session_payload(tokens_saved=0), # baseline at zero+ _session_payload(tokens_saved=50),+ _session_payload(tokens_saved=250),],)@@ -162,16 +225,16 @@ def test_rtk_stats_none_yields_zero_delta(monkeypatch: pytest.MonkeyPatch) -> Nodef test_last_rtk_advances_monotonically(monkeypatch: pytest.MonkeyPatch) -> None:- """Two polls returning the same lifetime total contribute exactly once."""+ """Two polls returning the same session total contribute exactly once."""tracker = _build_tracker(monkeypatch)monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)_stub_rtk_stats(monkeypatch,[- {"lifetime_tokens_saved": 42},- {"lifetime_tokens_saved": 42}, # no movement- {"lifetime_tokens_saved": 42}, # still no movement+ _session_payload(tokens_saved=42),+ _session_payload(tokens_saved=42), # no movement+ _session_payload(tokens_saved=42), # still no movement],)@@ -191,16 +254,16 @@ def test_last_rtk_advances_monotonically(monkeypatch: pytest.MonkeyPatch) -> Nondef test_counter_regression_rebaselines_without_negative_delta(monkeypatch: pytest.MonkeyPatch,) -> None:- """RTK DB rebuild drops the lifetime total — re-baseline, do not subtract."""+ """Helper rebaselines the session counter — re-baseline, do not subtract."""tracker = _build_tracker(monkeypatch)monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)_stub_rtk_stats(monkeypatch,[- {"lifetime_tokens_saved": 500},- {"lifetime_tokens_saved": 100}, # regression!- {"lifetime_tokens_saved": 150},+ _session_payload(tokens_saved=500),+ _session_payload(tokens_saved=100), # regression!+ _session_payload(tokens_saved=150),],)@@ -211,7 +274,7 @@ def test_counter_regression_rebaselines_without_negative_delta(tracker.update_contribution()# Regression: contribution stays at 500 (no negative subtraction).assert tracker._state.contribution.tokens_saved_rtk == 500- # Baseline now points at the new (smaller) lifetime total so subsequent+ # Baseline now points at the new (smaller) session total so subsequent# polls can compute a meaningful delta.assert tracker._last_rtk_tokens_saved == 100@@ -226,10 +289,15 @@ def test_counter_regression_rebaselines_without_negative_delta(# ----------------------------------------------------------------------------def test_rtk_stats_exception_zero_delta_no_throw(+def test_rtk_stats_exception_zero_delta_no_throw_with_log(monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,) -> None:- """A raised ``_get_rtk_stats()`` is caught, logged, and yields 0 delta."""+ """A raised ``_get_rtk_stats()`` is caught, structured-logged, yields 0.++ PR-G2 remediation (H3): pins the loud-log requirement so the+ "no silent fallback" constraint is verified.+ """tracker = _build_tracker(monkeypatch)monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)@@ -239,11 +307,16 @@ def test_rtk_stats_exception_zero_delta_no_throw(monkeypatch.setattr("headroom.proxy.helpers._get_rtk_stats", boom)+ caplog.set_level(logging.WARNING, logger="headroom.subscription.tracker")+# Must not raise.tracker.update_contribution()assert tracker._state.contribution.tokens_saved_rtk == 0assert tracker._last_rtk_tokens_saved == 0+ assert any(+ "event=subscription_rtk_stats_fetch_failed" in rec.getMessage() for rec in caplog.records+ ), "expected structured log on RTK stats fetch failure"# ---------------------------------------------------------------------------@@ -259,7 +332,7 @@ def test_disabled_env_returns_zero(monkeypatch: pytest.MonkeyPatch) -> None:polls = _stub_rtk_stats(monkeypatch,- [{"lifetime_tokens_saved": 999}],+ [_session_payload(tokens_saved=999)],)tracker.update_contribution()@@ -282,7 +355,7 @@ def test_explicit_rtk_override_skips_poll(monkeypatch: pytest.MonkeyPatch) -> Notracker = _build_tracker(monkeypatch)monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)- polls = _stub_rtk_stats(monkeypatch, [{"lifetime_tokens_saved": 999}])+ polls = _stub_rtk_stats(monkeypatch, [_session_payload(tokens_saved=999)])tracker.update_contribution(tokens_saved_rtk=17)@@ -305,10 +378,261 @@ def test_cli_filtering_no_longer_mirrors_rtk(monkeypatch: pytest.MonkeyPatch) ->tracker = _build_tracker(monkeypatch)monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)- _stub_rtk_stats(monkeypatch, [{"lifetime_tokens_saved": 25}])+ _stub_rtk_stats(monkeypatch, [_session_payload(tokens_saved=25)])tracker.update_contribution(tokens_saved_cli_filtering=8)assert tracker._state.contribution.tokens_saved_cli_filtering == 8# rtk comes from the polled delta, not from cli_filtering.assert tracker._state.contribution.tokens_saved_rtk == 25+++# ---------------------------------------------------------------------------+# Test 8 (H1) — invalid HEADROOM_RTK_WIRING fails loudly at startup+# ---------------------------------------------------------------------------+++def test_garbage_wiring_env_raises_at_startup(monkeypatch: pytest.MonkeyPatch) -> None:+ """PR-G2 remediation (H1, M4): bad env value crashes startup loudly.++ Previously the typo would be silently swallowed at every+ ``update_contribution`` call. Now :func:`configure_subscription_tracker`+ validates eagerly and raises ``ValueError``.+ """++ monkeypatch.setenv(tracker_module._RTK_WIRING_ENV, "garbage")+ # Reset the singleton so configure() actually runs the validator.+ monkeypatch.setattr(tracker_module, "_tracker_instance", None)++ with pytest.raises(ValueError, match="HEADROOM_RTK_WIRING"):+ tracker_module.configure_subscription_tracker(enabled=True)+++def test_garbage_wiring_env_logs_loudly_at_runtime(+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ """If env is flipped to garbage AFTER startup, runtime path emits ERROR.++ This is the defence-in-depth tier — startup-validation is the primary+ barrier (test above) but a env-var rotation could still flip the value+ mid-run.+ """++ tracker = _build_tracker(monkeypatch)+ # Set garbage AFTER tracker construction so the constructor doesn't see it.+ monkeypatch.setenv(tracker_module._RTK_WIRING_ENV, "garbage")++ caplog.set_level(logging.ERROR, logger="headroom.subscription.tracker")++ tracker.update_contribution()++ assert tracker._state.contribution.tokens_saved_rtk == 0+ assert any(+ rec.levelno >= logging.ERROR and "event=subscription_rtk_invalid_env" in rec.getMessage()+ for rec in caplog.records+ ), "expected ERROR-level structured log on invalid HEADROOM_RTK_WIRING"+++# ---------------------------------------------------------------------------+# Test 9 (C2) — restart-seeding behavior: no phantom delta on second process+# ---------------------------------------------------------------------------+++def test_restart_does_not_emit_phantom_delta(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Any+) -> None:+ """PR-G2 remediation (C2): post-restart first poll must not phantom.++ Scenario:+ 1. Tracker A runs, accumulates ``c.tokens_saved_rtk = 100``, persists.+ 2. Process restarts (tracker B loads from disk).+ 3. First poll on tracker B: helper returns ``session.tokens_saved = 5``+ (small new value since startup). Delta = 5 - 0 = 5. Cumulative =+ 100 + 5 = 105. NOT 100 + 50 000 (lifetime).++ The C1 fix (read session, not lifetime) inherently dissolves this+ because the helper rebaselines session counters at every proxy+ startup. This test verifies that property.+ """++ monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)+ monkeypatch.setattr(SubscriptionTracker, "_try_acquire_rtk_poll_lock", lambda self: True)++ persist_path = tmp_path / "state.json"++ # Phase 1 — tracker A runs and persists state with non-zero counters.+ _stub_rtk_stats(+ monkeypatch,+ [_session_payload(tokens_saved=100, lifetime=50_100)],+ )+ tracker_a = SubscriptionTracker(persist_path=persist_path, enabled=True)+ tracker_a.update_contribution()+ assert tracker_a._state.contribution.tokens_saved_rtk == 100+ tracker_a._persist_state()++ # Phase 2 — simulate process restart. New tracker loads state from+ # disk. Helper rebaselines (session counter starts fresh at 5 — only+ # one RTK invocation since restart).+ _stub_rtk_stats(+ monkeypatch,+ [_session_payload(tokens_saved=5, lifetime=50_105)],+ )+ tracker_b = SubscriptionTracker(persist_path=persist_path, enabled=True)+ # Loaded from disk.+ assert tracker_b._state.contribution.tokens_saved_rtk == 100+ # Tracker B's _last_rtk_tokens_saved starts at 0 (correct — the+ # session baseline was just re-pinned in the helper).+ assert tracker_b._last_rtk_tokens_saved == 0++ tracker_b.update_contribution()+ # 100 (loaded) + 5 (new session delta) = 105. NOT 50 100 + anything.+ assert tracker_b._state.contribution.tokens_saved_rtk == 105+++# ---------------------------------------------------------------------------+# Test 10 (M2 + M3) — legacy state file migration+# ---------------------------------------------------------------------------+++def test_legacy_state_migrates_rtk_from_cli_filtering(+ monkeypatch: pytest.MonkeyPatch,+ tmp_path: Any,+ caplog: pytest.LogCaptureFixture,+) -> None:+ """PR-G2 remediation (M2): pre-G2 state has no ``rtk_raw`` key.++ Pre-G2 the ``rtk`` field silently mirrored ``cli_filtering`` (the+ exact bug PR-G2 retires). When loading a legacy file we treat the+ aliased ``rtk`` value as the authoritative rtk_raw so historical+ accumulation isn't silently zeroed. A migration log line is emitted.+ """++ import json++ persist_path = tmp_path / "legacy.json"+ persist_path.write_text(+ json.dumps(+ {+ "contribution": {+ "tokens_submitted": 50,+ "tokens_saved": {+ "proxy_compression": 10,+ "cli_filtering": 42,+ "rtk": 42, # pre-G2 alias+ "cache_reads": 3,+ # NO rtk_raw / cli_filtering_raw keys (legacy)+ },+ "savings_usd": {"compression": 0.0, "cache": 0.0},+ },+ "poll_count": 7,+ }+ )+ )++ caplog.set_level(logging.INFO, logger="headroom.subscription.tracker")++ tracker = SubscriptionTracker(persist_path=persist_path, enabled=True)++ # Legacy ``rtk == cli_filtering`` got carried forward into rtk_raw.+ assert tracker._state.contribution.tokens_saved_rtk == 42+ assert tracker._state.contribution.tokens_saved_cli_filtering == 42+ # Migration log emitted.+ assert any(+ "event=subscription_state_legacy_load" in rec.getMessage() for rec in caplog.records+ ), "expected legacy migration structured log"+++# ---------------------------------------------------------------------------+# Test 11 (H2) — helper logs structured warning on subprocess failure+# ---------------------------------------------------------------------------+++def test_rtk_subprocess_failure_logs_structured_warning(+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ """PR-G2 remediation (H2): synthetic-zero path must log loudly.++ Without this, a broken RTK and a healthy "0 tokens saved" RTK are+ indistinguishable at the tracker layer.+ """++ import subprocess as _subprocess++ from headroom.proxy import helpers as _helpers++ monkeypatch.setattr(_helpers, "get_rtk_path", lambda: None, raising=False)++ # The above branch returns early before invoking subprocess — not the+ # one we want. Instead, force the subprocess to fail.+ monkeypatch.setattr(+ "headroom.proxy.helpers.get_rtk_path",+ lambda: "/tmp/nonexistent-rtk",+ )++ class FakeResult:+ returncode = 1+ stdout = ""+ stderr = "rtk: database not found"++ def fake_run(*args: Any, **kwargs: Any) -> FakeResult:+ return FakeResult()++ monkeypatch.setattr(_subprocess, "run", fake_run)++ caplog.set_level(logging.WARNING, logger="headroom.proxy")++ payload = _helpers._read_rtk_lifetime_stats()+ assert payload is not None+ assert payload["tokens_saved"] == 0+ assert any("event=rtk_stats_subprocess_failed" in rec.getMessage() for rec in caplog.records), (+ "expected structured warning when RTK subprocess fails"+ )+++# ---------------------------------------------------------------------------+# Test 12 (C3) — multi-worker poll deduplication via file lock+# ---------------------------------------------------------------------------+++def test_multi_worker_only_one_polls(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:+ """PR-G2 remediation (C3): two trackers sharing a state path elect one owner.++ The owner polls; the non-owner returns 0 from ``_poll_rtk_delta``.+ Without this gate each worker would add the same RTK delta to its+ own ``c.tokens_saved_rtk``, inflating dashboard savings by N× workers.+ """++ monkeypatch.delenv(tracker_module._RTK_WIRING_ENV, raising=False)++ # Both trackers share a state directory so they share the lock file.+ persist_path = tmp_path / "state.json"+ lock_path = tmp_path / ".rtk_poll_lock"+ monkeypatch.setenv(tracker_module._RTK_POLL_LOCK_ENV, str(lock_path))++ _stub_rtk_stats(+ monkeypatch,+ [_session_payload(tokens_saved=100)],+ )++ # Worker A — first to attempt acquisition wins.+ tracker_a = SubscriptionTracker(persist_path=persist_path, enabled=True)+ # Worker B — same lock path; flock will fail.+ tracker_b = SubscriptionTracker(persist_path=persist_path, enabled=True)++ tracker_a.update_contribution()+ tracker_b.update_contribution()++ # Owner polled and got 100; non-owner returned 0.+ a_rtk = tracker_a._state.contribution.tokens_saved_rtk+ b_rtk = tracker_b._state.contribution.tokens_saved_rtk+ # One worker saw the full 100; the other saw 0. (Order is OS-dependent+ # but exactly one owns the lock.)+ assert {a_rtk, b_rtk} == {0, 100}, (+ f"expected exactly one worker to poll; got a={a_rtk}, b={b_rtk}"+ )++ # Cleanup so subsequent tests don't see a stale lock.+ tracker_a._release_rtk_poll_lock()+ tracker_b._release_rtk_poll_lock()