Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(grok): attribute api.x.ai chat outcomes as provider xai, recognize grok-shell UA (#2626)
gglucassb1fb162b6223
diff --git a/headroom/cache_economics.py b/headroom/cache_economics.pynew file mode 100644index 000000000..76be1e40d--- /dev/null+++ b/headroom/cache_economics.py@@ -0,0 +1,32 @@+"""Lightweight provider cache fallbacks shared by all pricing surfaces.++Published per-model catalog rates always take precedence. These ratios are+estimates for priced models whose catalog row has no cache rates. xAI uses+a 0.16 read ratio and no write premium; known model rates override it.+"""++from typing import Any++CACHE_ECONOMICS: dict[str, dict[str, Any]] = {+ "anthropic": {+ "read_multiplier": 0.1,+ "write_multiplier": 1.25,+ "label": "Explicit breakpoints, 5-min TTL",+ },+ "openai": {+ "read_multiplier": 0.5,+ "write_multiplier": 1.0,+ "label": "Automatic, no TTL control",+ },+ "gemini": {+ "read_multiplier": 0.1,+ "write_multiplier": 1.0,+ "label": "Explicit cachedContent, configurable TTL",+ },+ "bedrock": {+ "read_multiplier": 0.1,+ "write_multiplier": 1.25,+ "label": "Same as Anthropic (Bedrock)",+ },+ "xai": {"read_multiplier": 0.16, "write_multiplier": 1.0, "label": "Automatic, no TTL control"},+}diff --git a/headroom/pricing/counterfactual.py b/headroom/pricing/counterfactual.pyindex c32168690..b14b9e3c0 100644--- a/headroom/pricing/counterfactual.py+++ b/headroom/pricing/counterfactual.py@@ -61,6 +61,7 @@ from enum import Enumfrom functools import lru_cachefrom typing import Any+from headroom.cache_economics import CACHE_ECONOMICSfrom headroom.pricing.cache_ttl import CACHE_WRITE_MULTIPLIERSlogger = logging.getLogger(__name__)@@ -72,30 +73,8 @@ LONG_CONTEXT_THRESHOLD_TOKENS = 200_000#: Provider-level cache discount ratios, as a fraction of the base input price.#: FALLBACK ONLY — the per-model LiteLLM catalog is always preferred, because-#: these go stale per model and per context tier. Kept here (rather than in-#: ``proxy/cost.py``, where they used to live) so every consumer shares one copy.-CACHE_ECONOMICS: dict[str, dict[str, Any]] = {- "anthropic": {- "read_multiplier": 0.1,- "write_multiplier": 1.25,- "label": "Explicit breakpoints, 5-min TTL",- },- "openai": {- "read_multiplier": 0.5,- "write_multiplier": 1.0,- "label": "Automatic, no TTL control",- },- "gemini": {- "read_multiplier": 0.1,- "write_multiplier": 1.0,- "label": "Explicit cachedContent, configurable TTL",- },- "bedrock": {- "read_multiplier": 0.1,- "write_multiplier": 1.25,- "label": "Same as Anthropic (Bedrock)",- },-}+#: these go stale per model and per context tier. The lightweight shared table+#: is also used by the dashboard, without importing the pricing package there.class Region(str, Enum):@@ -514,7 +493,15 @@ def resolve_rates():# Catalog priced the model but published no cache rates. Fall back to# the provider ratio table if we can identify the provider.- econ = CACHE_ECONOMICS.get((provider or "").split(":")[-1].strip().lower())+ # Model-only cost consumers do not have a request provider to pass.+ # Prefer explicit routing context, otherwise use the catalog identity.+ resolved_provider = provider or info.get("litellm_provider") or ""+ provider_key = (+ resolved_provider.split(":")[-1].strip().lower()+ if isinstance(resolved_provider, str)+ else ""+ )+ econ = CACHE_ECONOMICS.get(provider_key)if econ:return CacheRates(read=base * float(econ["read_multiplier"]),diff --git a/headroom/proxy/auth_policy.py b/headroom/proxy/auth_policy.pyindex 2bc1be668..fc7f43941 100644--- a/headroom/proxy/auth_policy.py+++ b/headroom/proxy/auth_policy.py@@ -19,7 +19,6 @@ SUBSCRIPTION_UA_PREFIXES: tuple[str, ...] = ("claude-code/","codex-cli/","cursor/",- "grok/","claude-vscode/","github-copilot/","anthropic-cli/",@@ -34,6 +33,11 @@ CLIENT_UA_MAP: tuple[tuple[str, str], ...] = (("anthropic-cli/", "anthropic-cli"),("codex-cli/", "codex"),("cursor/", "cursor"),+ # Current Grok Build releases send "grok-shell/<version>" (verified against+ # grok 0.2.112); older builds sent "grok/". Deliberately NOT added to+ # SUBSCRIPTION_UA_PREFIXES: API-key grok traffic is pay-per-token and+ # should keep the aggressive compression policy.+ ("grok-shell/", "grok_build"),("grok/", "grok_build"),("zed/", "zed"),("aider/", "aider"),diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex af2e7a246..a5b27261f 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -15,6 +15,7 @@ from collections import dequefrom datetime import datetime, timedeltafrom typing import TYPE_CHECKING, Any, NamedTuple+from headroom.cache_economics import CACHE_ECONOMICS as _CACHE_ECONOMICSfrom headroom.proxy.budget_basis_policy import (BUDGET_BASIS_BLOCK,BUDGET_BASIS_IGNORE,@@ -109,28 +110,6 @@ class CostEntry(NamedTuple):# source for cache economics, and these ratios stand in when a model publishes# no cache pricing. Hardcoded ratios go stale per model and per context tier# (Anthropic's >200k rates differ), so they are never preferred over the catalog.-_CACHE_ECONOMICS = {- "anthropic": {- "read_multiplier": 0.1,- "write_multiplier": 1.25,- "label": "Explicit breakpoints, 5-min TTL",- },- "openai": {- "read_multiplier": 0.5,- "write_multiplier": 1.0,- "label": "Automatic, no TTL control",- },- "gemini": {- "read_multiplier": 0.1,- "write_multiplier": 1.0,- "label": "Explicit cachedContent, configurable TTL",- },- "bedrock": {- "read_multiplier": 0.1,- "write_multiplier": 1.25,- "label": "Same as Anthropic (Bedrock)",- },-}#: Context size at which the major catalogs publish a second, higher price@@ -285,6 +264,7 @@ def build_prefix_cache_stats((provider in ("anthropic", "vertex:anthropic") and "claude" in model_name)or (provider == "openai" and any(p in model_name for p in _openai_prefixes))or (provider == "gemini" and "gemini" in model_name)+ or (provider == "xai" and "grok" in model_name)or (provider == "bedrock" and "claude" in model_name))if is_match and tokens_sent > best_tokens:diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex b486cd592..beed80845 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -3801,11 +3801,16 @@ class OpenAIHandlerMixin:handler_path,custom_upstream_base_url or "",)- # Fixed taxonomy from the shared helper (zen, zai, meta, openai); any- # other custom base is the shared "custom" bucket. Never derive the+ # Fixed taxonomy from the shared helper (zen, zai, meta, openai, xai);+ # any other custom base is the shared "custom" bucket. Never derive the# label from the request-controlled hostname — see the review on #3759.+ # Grok CLI routed to xAI without a base-url header is still xai.openai_chat_outcome_provider = custom_chat_provider or (- CUSTOM_BASE_PROVIDER if custom_upstream_base_url else "openai"+ CUSTOM_BASE_PROVIDER+ if custom_upstream_base_url+ else "xai"+ if _is_xai_upstream(upstream_base_url)+ else "openai")# Memory: Get user ID when memory is enabled. Reads `request.headers`diff --git a/headroom/proxy/passthrough.py b/headroom/proxy/passthrough.pyindex 49a4fff70..30cbc2cfd 100644--- a/headroom/proxy/passthrough.py+++ b/headroom/proxy/passthrough.py@@ -14,6 +14,8 @@ CUSTOM_BASE_CHAT_PROVIDERS = {"api.openai.com": "openai","api.z.ai": "zai","api.meta.ai": "meta",+ # Grok Build: kept apart from OpenAI so savings rollups get their own row.+ "api.x.ai": "xai",}# Fixed label for chat traffic on any other custom base.@@ -45,7 +47,8 @@ def custom_base_passthrough_telemetry(method: str, path: str, base_url: str) ->if method.upper() != "POST":return "", ""try:- host = (urlparse(base_url.strip()).hostname or "").lower()+ # A fully qualified name (trailing dot) is the same host.+ host = (urlparse(base_url.strip()).hostname or "").lower().rstrip(".")except ValueError:return "", ""normalized_path = path[1:] if path.startswith("/") else pathdiff --git a/tests/test_auth_policy.py b/tests/test_auth_policy.pyindex 25417fabe..1066d3f93 100644--- a/tests/test_auth_policy.py+++ b/tests/test_auth_policy.py@@ -43,11 +43,22 @@ def test_client_explicit_override_wins_over_user_agent() -> None:assert classify_client_signals(signals) == "aider"-def test_grok_build_user_agent_is_subscription_client() -> None:+def test_grok_build_user_agent_keeps_credential_driven_policy() -> None:signals = AuthSignals(user_agent="grok/1.2.3")- assert classify_auth_signals(signals) is AuthMode.SUBSCRIPTION+ assert classify_auth_signals(signals) is AuthMode.PAYG+ assert classify_client_signals(signals) == "grok_build"+++def test_grok_shell_user_agent_maps_to_grok_build_but_not_subscription() -> None:+ # Current Grok Build releases send grok-shell/<version> (grok 0.2.112).+ # Client classification must recognize it; auth mode deliberately stays+ # key-driven (API-key grok is pay-per-token and should keep the+ # aggressive policy), so grok-shell/ is NOT a subscription UA prefix.+ signals = AuthSignals(user_agent="grok-shell/0.2.112 (macos; aarch64)")+assert classify_client_signals(signals) == "grok_build"+ assert classify_auth_signals(signals) is not AuthMode.SUBSCRIPTIONdef test_codex_stamp_only_for_unidentified_responses_callers() -> None:diff --git a/tests/test_proxy_cache_ttl_metrics.py b/tests/test_proxy_cache_ttl_metrics.pyindex 59eab50ee..8b6c2dcb6 100644--- a/tests/test_proxy_cache_ttl_metrics.py+++ b/tests/test_proxy_cache_ttl_metrics.py@@ -498,3 +498,23 @@ def test_reset_runtime_clears_miss_attribution() -> None:asyncio.run(metrics.record_cache_miss_attribution("anthropic", "ttl_expiry"))asyncio.run(metrics.reset_runtime())assert dict(metrics.cache_miss_attribution_by_provider) == {}+++def test_prefix_cache_stats_prices_xai_cache_reads_with_grok_models(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Grok traffic books under ``xai``; its cache reads must be priced with a+ Grok model, not dropped to $0 because no model matched the provider."""+ metrics = PrometheusMetrics()+ metrics.cache_by_provider["xai"].update(+ {"requests": 1, "hit_requests": 1, "cache_read_tokens": 1_000_000}+ )+ tracker = CostTracker()+ tracker._tokens_sent_by_model.update({"grok-4": 1})+ monkeypatch.setattr(CostTracker, "_get_list_price", lambda _self, _model: 3.0)+ monkeypatch.setattr(CostTracker, "_get_cache_prices", lambda _self, _model: None)++ stats = build_prefix_cache_stats(metrics, tracker)++ # No catalog cache rates: xAI's own fallback ratio, not Anthropic's 0.1.+ assert stats["by_provider"]["xai"]["savings_usd"] == round(1_000_000 * 3e-6 * (1 - 0.16), 4)diff --git a/tests/test_proxy_openai_cache_stability.py b/tests/test_proxy_openai_cache_stability.pyindex c3ba1a172..5abf41019 100644--- a/tests/test_proxy_openai_cache_stability.py+++ b/tests/test_proxy_openai_cache_stability.py@@ -1161,11 +1161,23 @@ def test_openai_chat_outcome_provider_uses_the_fixed_taxonomy(_chat("https://llm.example.internal/v1")# No header: the plain OpenAI path keeps its own label (control)._chat(None)+ _chat("https://api.x.ai")+ _chat("https://api.x.ai.")++ # Grok CLI on the default OpenAI target is routed to api.x.ai without+ # a base-url header (#2693); it is still xAI traffic, not OpenAI's.+ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key", "user-agent": "grok-shell/0.2.117"},+ json={"model": "grok-4", "messages": [{"role": "user", "content": "hi"}]},+ )+ assert response.status_code == 200by_provider = proxy.metrics.requests_by_providerassert by_provider["meta"] == 2assert by_provider["custom"] == 1assert by_provider["openai"] == 1+ assert by_provider["xai"] == 3def test_openai_chat_custom_base_flood_cannot_grow_the_provider_set(diff --git a/tests/test_proxy_passthrough.py b/tests/test_proxy_passthrough.pyindex 08111e1ce..4923bf61f 100644--- a/tests/test_proxy_passthrough.py+++ b/tests/test_proxy_passthrough.py@@ -118,3 +118,30 @@ def test_custom_base_passthrough_telemetry_requires_a_chat_path_segment() -> Non"chat/completions","zai",)+++def test_custom_base_passthrough_telemetry_recognizes_xai_chat() -> None:+ # Grok Build routes plain OpenAI chat completions through custom-base+ # routing with base_url https://api.x.ai; attribute the provider as xai+ # so savings rollups separate grok traffic from OpenAI's.+ assert custom_base_passthrough_telemetry(+ "POST",+ "/v1/chat/completions",+ "https://api.x.ai",+ ) == ("chat/completions", "xai")+ assert custom_base_passthrough_telemetry(+ "POST",+ "/v1/chat/completions",+ "https://api.x.ai.",+ ) == ("chat/completions", "xai")+ # Other xai paths and methods stay unlabelled.+ assert custom_base_passthrough_telemetry(+ "GET",+ "/v1/chat/completions",+ "https://api.x.ai",+ ) == ("", "")+ assert custom_base_passthrough_telemetry(+ "POST",+ "/v1/models",+ "https://api.x.ai",+ ) == ("", "")diff --git a/tests/test_xai_cache_economics.py b/tests/test_xai_cache_economics.pynew file mode 100644index 000000000..1177855d3--- /dev/null+++ b/tests/test_xai_cache_economics.py@@ -0,0 +1,89 @@+"""xAI cached input must have the same price in savings and cost totals."""++import pytest++[email protected]("inferred, expected", [(True, 0.0058), (False, 0.0108)])[email protected]("ttl_split", [False, True])+def test_xai_inferred_writes_are_not_billed_twice(monkeypatch, inferred, expected, ttl_split):+ import litellm++ from headroom.proxy.cost import CostTracker++ model = "xai/grok-review-inferred"+ monkeypatch.setitem(+ litellm.model_cost,+ model,+ {+ "input_cost_per_token": 1e-6,+ "output_cost_per_token": 2e-6,+ "litellm_provider": "xai",+ "mode": "chat",+ },+ )+ tracker = CostTracker()+ tracker.record_tokens(+ model,+ tokens_saved=0,+ tokens_sent=10_000,+ cache_read_tokens=5_000,+ cache_write_tokens=5_000,+ cache_write_5m_tokens=2_000 if ttl_split else 0,+ cache_write_1h_tokens=3_000 if ttl_split else 0,+ uncached_tokens=5_000,+ cache_inferred=inferred,+ )+ assert tracker.totals()[1] == pytest.approx(expected)+ assert tracker.stats()["total_input_cost_usd"] == pytest.approx(expected)++[email protected]("catalog_read", [None, 0.05e-6])+def test_xai_cache_read_pricing_matches_cost_totals(monkeypatch, catalog_read):+ import litellm++ from headroom.pricing.counterfactual import CacheMix, Region, price_savings, resolve_rates+ from headroom.proxy.cost import CostTracker++ model = "xai/grok-review-fallback" if catalog_read is None else "xai/grok-review-catalog"+ row = {+ "input_cost_per_token": 1e-6,+ "output_cost_per_token": 2e-6,+ "litellm_provider": "xai",+ "mode": "chat",+ }+ if catalog_read is not None:+ row["cache_read_input_token_cost"] = catalog_read+ monkeypatch.setitem(litellm.model_cost, model, row)+ resolve_rates.cache_clear()+ try:+ expected_read = catalog_read if catalog_read is not None else 0.16e-6+ mix = CacheMix.from_usage(cache_read_tokens=10_000)+ saving = price_savings(10_000, model=model, mix=mix, provider="xai", region=Region.PREFIX)+ assert saving.usd == pytest.approx(10_000 * expected_read)+ tracker = CostTracker()+ tracker.record_tokens(model, tokens_saved=0, tokens_sent=10_000, cache_read_tokens=10_000)+ assert tracker.stats()["total_input_cost_usd"] == pytest.approx(10_000 * expected_read)+ finally:+ resolve_rates.cache_clear()++[email protected]("user_agent", ["grok/1.2.3", "grok-shell/0.2.112"])+def test_grok_api_keys_keep_payg_policy_across_client_versions(user_agent):+ from headroom.proxy.auth_policy import (+ AuthMode,+ AuthSignals,+ classify_auth_signals,+ classify_client_signals,+ )++ signals = AuthSignals(user_agent=user_agent, authorization="Bearer xai-test-key")+ assert classify_client_signals(signals) == "grok_build"+ assert classify_auth_signals(signals) is AuthMode.PAYG++[email protected]("user_agent", ["grok/1.2.3", "grok-shell/0.2.112"])+def test_grok_oauth_tokens_keep_oauth_policy_across_client_versions(user_agent):+ from headroom.proxy.auth_policy import AuthMode, AuthSignals, classify_auth_signals++ signals = AuthSignals(user_agent=user_agent, authorization="Bearer header.payload.signature")+ assert classify_auth_signals(signals) is AuthMode.OAUTH