Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): make --stateless write nothing to the workspace (#3846)
Tejas Chopra3c9ed0101a83
diff --git a/headroom/cache/compression_store.py b/headroom/cache/compression_store.pyindex 1712fa007..2611c5346 100644--- a/headroom/cache/compression_store.py+++ b/headroom/cache/compression_store.py@@ -1114,11 +1114,24 @@ def _create_default_ccr_backend() -> CompressionStoreBackend | None:"memory" opts back into the in-process dict. Other values loadadapters via setuptools entry point 'headroom.ccr_backend'.Returns None to use InMemoryBackend.++ Stateless mode (``--stateless`` / ``HEADROOM_STATELESS``) never opens the+ SQLite file: the entries it would hold are the verbatim tool outputs that+ stateless deployments run stateless to keep off disk. The default and an+ explicit ``sqlite`` both fall back to the in-process store (retrieval then+ does not survive a restart or cross workers, which is the documented+ stateless trade-off). A named entry-point backend is still honoured — the+ operator chose it explicitly, and it is how a stateless multi-worker+ deployment gets a non-file (e.g. redis) store."""+ from ..paths import persistence_allowed+backend_type = (os.environ.get("HEADROOM_CCR_BACKEND") or "").strip().lower()if backend_type == "memory":return Noneif not backend_type or backend_type == "sqlite":+ if not persistence_allowed("CCR retrieval store (ccr_store.db)"):+ return Nonetry:from .backends.sqlite import SQLiteBackend@@ -1197,6 +1210,22 @@ def get_compression_store(return _compression_store+def detach_compression_store() -> None:+ """Drop the global compression store without clearing it.++ The next :func:`get_compression_store` builds a fresh store from the+ current settings. Unlike :func:`reset_compression_store`, the old store's+ entries are left as they are: nothing is deleted from its backend, so a+ SQLite file is neither written nor emptied. Used when the proxy switches to+ stateless mode at runtime. The old backend is not closed, because code+ that already holds the old store may still be using it.+ """+ global _compression_store++ with _store_lock:+ _compression_store = None++def reset_compression_store() -> None:"""Reset the global compression store. Mainly for testing."""global _compression_storediff --git a/headroom/ccr/mcp_server.py b/headroom/ccr/mcp_server.pyindex f013e092e..53fe4f977 100644--- a/headroom/ccr/mcp_server.py+++ b/headroom/ccr/mcp_server.py@@ -209,7 +209,13 @@ SESSION_WINDOW_SECONDS = 7200 # 2 hours — events older than this are pruneddef _append_shared_event(event: dict[str, Any]) -> None:- """Append an event to the shared stats file (cross-process, file-locked)."""+ """Append an event to the shared stats file (cross-process, file-locked).++ No-op in stateless mode (``HEADROOM_STATELESS``): the aggregate then covers+ only this process, which is the documented stateless trade-off.+ """+ if not _paths.persistence_allowed("MCP session stats"):+ returntry:SHARED_STATS_DIR.mkdir(parents=True, exist_ok=True)event["pid"] = os.getpid()@@ -249,8 +255,9 @@ def _read_shared_events(window_seconds: int = SESSION_WINDOW_SECONDS) -> list[dikeep_lines.append(line + "\n")except json.JSONDecodeError:continue- # Prune old entries (only if we dropped some)- if len(keep_lines) < len(lines):+ # Prune old entries (only if we dropped some). Never rewrite the file in+ # stateless mode — reading is fine, writing is not.+ if len(keep_lines) < len(lines) and _paths.persistence_allowed("MCP session stats"):try:with open(SHARED_STATS_FILE, "w") as f:if _HAS_FCNTL:diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 30132ac0f..57dd8b390 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -1300,9 +1300,12 @@ def proxy(_paths.codex_wire_debug_dir())- # Stateless mode: suppress TOIN filesystem persistence+ # Stateless mode: suppress TOIN filesystem persistence, and export the flag+ # so code that runs before the proxy records it (the update check) and+ # child processes see the same answer as paths.process_is_stateless().if is_stateless:os.environ["HEADROOM_TOIN_BACKEND"] = "none"+ os.environ["HEADROOM_STATELESS"] = "1"# Licence token (HEADROOM_LICENSE; HEADROOM_LICENSE_KEY is a deprecated# alias). Having one set never enables outbound usage reporting: thatdiff --git a/headroom/memory/sync.py b/headroom/memory/sync.pyindex 0b3c80fd8..6117d3fcc 100644--- a/headroom/memory/sync.py+++ b/headroom/memory/sync.py@@ -132,7 +132,9 @@ def _load_sync_state(state_path: Path) -> dict[str, Any]:def _save_sync_state(state_path: Path, state: dict[str, Any]) -> None:- """Save sync state to disk."""+ """Save sync state to disk (skipped in stateless mode)."""+ if not _paths.persistence_allowed("memory sync state"):+ returnstate_path.parent.mkdir(parents=True, exist_ok=True)state_path.write_text(json.dumps(state, indent=2), encoding="utf-8", newline="\n")diff --git a/headroom/paths.py b/headroom/paths.pyindex 0e8fb55fa..76502adb7 100644--- a/headroom/paths.py+++ b/headroom/paths.py@@ -125,6 +125,46 @@ def process_is_stateless() -> bool:return _env("HEADROOM_STATELESS").lower() in ("1", "true", "yes", "on")+# Purposes for which a "skipped because stateless" notice has already been+# logged, so a busy proxy says it once per persister, not once per turn.+_PERSISTENCE_NOTICED: set[str] = set()+++def persistence_allowed(purpose: str) -> bool:+ """Whether a runtime persister may write *purpose* to the workspace now.++ This is the one predicate every on-disk store consults before it creates or+ writes a file under the workspace: the CCR retrieval store, the licence+ cache, MCP session stats, the savings ledger, subscription state, memory+ sync state, the update-check cache. Returns ``False`` in stateless mode+ (``--stateless`` or ``HEADROOM_STATELESS``), in which case the caller keeps+ its in-memory state and skips the write. The first refusal for each+ *purpose* is logged at INFO so an operator can see what stateless mode+ turned off; later refusals are silent.++ Use this rather than checking :func:`process_is_stateless` inline so the+ stateless guarantee ("writes nothing to the workspace") is enforced in one+ place and its coverage can be read off the call sites.+ """++ if not process_is_stateless():+ return True+ if purpose not in _PERSISTENCE_NOTICED:+ _PERSISTENCE_NOTICED.add(purpose)+ import logging++ logging.getLogger(__name__).info(+ "Stateless mode: not persisting %s to disk (kept in memory only).", purpose+ )+ return False+++def _reset_persistence_notices() -> None:+ """Forget which stateless notices were logged. For tests."""++ _PERSISTENCE_NOTICED.clear()++def _resolve(explicit: str | os.PathLike[str] | None, env_var: str, derived: Path) -> Path:"""Apply the standard precedence: explicit > env > derived.@@ -441,6 +481,7 @@ __all__ = ["HEADROOM_SETTINGS_PATH_ENV","set_process_stateless","process_is_stateless",+ "persistence_allowed","config_dir","workspace_dir","ensure_config_dir",diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 37a090d57..f8ea28528 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -715,7 +715,10 @@ def _apply_stateless_persistence(config: ProxyConfig) -> None:files are written to the workspace.Covers TOIN (the always-on serving writer): it keeps learning patterns- in-memory but never reads or writes ``toin.json``. An empty ``storage_path``+ in-memory but never reads or writes ``toin.json``. Also detaches the CCR+ retrieval store so it is rebuilt in-memory (see below). Every other+ workspace writer consults ``paths.persistence_allowed()`` at its write+ site, so it needs no special handling here. An empty ``storage_path``makes the backend ``None``, which no-ops load/save/auto-save. The savingssubsystem is handled separately via ``PrometheusMetrics(stateless=...)``.@@ -735,12 +738,21 @@ def _apply_stateless_persistence(config: ProxyConfig) -> None:"""if not getattr(config, "stateless", False):return+ from headroom.cache.compression_store import detach_compression_storefrom headroom.telemetry.toin import TOINConfig, get_toin, reset_toin# Reset first so this wins regardless of whether the singleton was already# created with a filesystem backend earlier in the process.reset_toin()get_toin(TOINConfig(storage_path=""))+ # Same for the CCR retrieval store: drop any singleton built before the+ # stateless flag was recorded so the next get_compression_store() goes+ # through _create_default_ccr_backend(), which refuses the SQLite file in+ # stateless mode. The store holds verbatim tool output — the one thing a+ # stateless deployment most needs kept off disk. Detach, not reset: reset+ # clears the old store, which deletes every row from an existing SQLite+ # file — a destructive write on the way into stateless mode.+ detach_compression_store()def _provider_httpx_client_options(diff --git a/headroom/savings_ledger.py b/headroom/savings_ledger.pyindex 44cd336a3..823efa445 100644--- a/headroom/savings_ledger.py+++ b/headroom/savings_ledger.py@@ -395,6 +395,8 @@ def record_savings_event(if cost_usd is None:event["cost_usd"] = round(list_usd, 6)+ if not _paths.persistence_allowed("savings ledger"):+ return Falsetarget = _resolve_path(path)try:target.parent.mkdir(parents=True, exist_ok=True)diff --git a/headroom/subscription/tracker.py b/headroom/subscription/tracker.pyindex 2b1d0e50d..dc802d22b 100644--- a/headroom/subscription/tracker.py+++ b/headroom/subscription/tracker.py@@ -459,6 +459,8 @@ class SubscriptionTracker(QuotaTracker):# ------------------------------------------------------------------def _persist_state(self) -> None:+ if not _paths.persistence_allowed("subscription quota state"):+ returntry:self._persist_path.parent.mkdir(parents=True, exist_ok=True)with self._lock:diff --git a/headroom/telemetry/reporter.py b/headroom/telemetry/reporter.pyindex 15217d334..b7f39118c 100644--- a/headroom/telemetry/reporter.py+++ b/headroom/telemetry/reporter.py@@ -358,9 +358,11 @@ class UsageReporter:self._last_report_time = datetime.now(timezone.utc)def _save_cache(self) -> None:- """Save license info to local cache file."""+ """Save license info to local cache file (skipped in stateless mode)."""if self._license_info is None:return+ if not _paths.persistence_allowed("licence validation cache"):+ returntry:self._cache_path.parent.mkdir(parents=True, exist_ok=True)self._cache_path.write_text(diff --git a/headroom/update_check.py b/headroom/update_check.pyindex 7ab5dcbec..d11cf8dcf 100644--- a/headroom/update_check.py+++ b/headroom/update_check.py@@ -38,7 +38,6 @@ _CACHE_FILE = "update_check.json"_CHECK_TTL_SECONDS = 86_400_OFF_VALUES = frozenset(("off", "false", "0", "no", "disable", "disabled"))-_TRUE_VALUES = frozenset(("on", "true", "1", "yes", "enable", "enabled"))def _env_off(name: str, default: str = "on") -> bool:@@ -46,11 +45,6 @@ def _env_off(name: str, default: str = "on") -> bool:return os.environ.get(name, default).strip().lower() in _OFF_VALUES-def _env_on(name: str) -> bool:- """Return True when env var ``name`` is set to a truthy/on value."""- return os.environ.get(name, "").strip().lower() in _TRUE_VALUES--def is_update_check_enabled() -> bool:"""Whether the update check / banner should run at all.@@ -65,7 +59,9 @@ def is_update_check_enabled() -> bool:return Falseif _env_off("HEADROOM_UPDATE_CHECK"):return False- if _env_on("HEADROOM_STATELESS"):+ from headroom.paths import process_is_stateless++ if process_is_stateless():return Falseif os.environ.get("CI", "").strip():return False@@ -133,8 +129,10 @@ def read_cache() -> dict[str, Any] | None:def write_cache(latest_version: str, *, now: float | None = None) -> None:"""Persist the latest-known version + check timestamp. Never raises."""try:- from headroom.paths import ensure_workspace_dir+ from headroom.paths import ensure_workspace_dir, persistence_allowed+ if not persistence_allowed("update-check cache"):+ returnensure_workspace_dir()payload = {"last_check": now if now is not None else time.time(),diff --git a/tests/test_stateless_no_disk.py b/tests/test_stateless_no_disk.pynew file mode 100644index 000000000..7b773a197--- /dev/null+++ b/tests/test_stateless_no_disk.py@@ -0,0 +1,332 @@+"""Stateless mode must not write to the workspace — end to end and per writer.++``--stateless`` is the flag regulated and ephemeral deployments pick so that no+request content lands on disk. Before this fix the CCR retrieval store (whose+default backend became SQLite) wrote verbatim tool output to ``ccr_store.db``+under a stateless proxy, and several smaller persisters (licence cache, MCP+session stats, savings ledger, subscription state, memory sync state, the+update-check cache) had no stateless check at all.++The end-to-end test snapshots a private workspace, drives a tool-heavy turn+through ``/v1/compress`` (which stores CCR originals exactly as the data plane+does) and asserts nothing appeared. The control test runs the same turn without+``--stateless`` and asserts the store file *does* appear, so the assertion is+known to be reachable. The remaining tests pin each writer to the shared+``paths.persistence_allowed()`` predicate.+"""++from __future__ import annotations++import json+import os+from pathlib import Path++import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient++from headroom import paths+from headroom.cache import compression_store as cs+from headroom.cache.backends import InMemoryBackend++# The runtime log is the one file stateless mode is documented to keep writing+# (it is created owner-only and carries no request content by default).+_ALLOWED_TOP_LEVEL = {"logs"}++[email protected](autouse=True)+def _isolate_process_state(monkeypatch):+ """Process-global flags and singletons must never leak between tests."""+ monkeypatch.delenv("HEADROOM_STATELESS", raising=False)+ monkeypatch.delenv("HEADROOM_CCR_BACKEND", raising=False)+ monkeypatch.delenv("HEADROOM_CCR_SQLITE_PATH", raising=False)+ paths.set_process_stateless(False)+ paths._reset_persistence_notices()+ cs.reset_compression_store()+ yield+ paths.set_process_stateless(False)+ paths._reset_persistence_notices()+ cs.reset_compression_store()+ try:+ from headroom.telemetry.toin import reset_toin++ reset_toin()+ except Exception:+ pass+++def workspace(tmp_path, monkeypatch) -> Path:+ """A private HOME and workspace so the test can assert on the whole tree."""+ home = tmp_path / "home"+ home.mkdir()+ ws = home / ".headroom"+ monkeypatch.setenv("HOME", str(home))+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(ws))+ monkeypatch.chdir(tmp_path)+ return ws+++def _snapshot(root: Path) -> set[str]:+ if not root.exists():+ return set()+ return {+ str(p.relative_to(root))+ for p in root.rglob("*")+ if p.is_file() and p.relative_to(root).parts[0] not in _ALLOWED_TOP_LEVEL+ }+++def _tool_heavy_messages() -> list[dict]:+ large = json.dumps(+ [+ {+ "id": i,+ "name": f"Item {i}",+ "description": (+ f"Detailed description for item {i}: status active, created on "+ f"2024-01-{(i % 28) + 1:02d}, category=electronics, price={i * 10.99:.2f}."+ ),+ "tags": ["electronics", "sale", "featured"],+ }+ for i in range(200)+ ]+ )+ return [+ {"role": "user", "content": "What items are available?"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_1",+ "type": "function",+ "function": {"name": "list", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "call_1", "content": large},+ {"role": "user", "content": "Summarize the first 5 items."},+ ]+++def _run_turn(stateless: bool) -> None:+ from headroom.proxy.server import ProxyConfig, create_app++ config = ProxyConfig(+ stateless=stateless,+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:+ resp = client.post(+ "/v1/compress", json={"messages": _tool_heavy_messages(), "model": "gpt-4"}+ )+ assert resp.status_code == 200, resp.text+ assert resp.json()["tokens_saved"] > 0, "turn must actually exercise the CCR store"+++# ---- end to end -----------------------------------------------------------+++def test_stateless_turn_writes_nothing_to_the_workspace(workspace):+ before = _snapshot(workspace)+ _run_turn(stateless=True)+ after = _snapshot(workspace)+ assert after - before == set(), f"stateless proxy wrote: {sorted(after - before)}"+ assert not (workspace / "ccr_store.db").exists()+ assert isinstance(cs.get_compression_store()._backend, InMemoryBackend)+++def test_stateful_turn_does_write_the_ccr_store(workspace):+ """Control: the same turn without --stateless persists, so the assertion above is live."""+ _run_turn(stateless=False)+ assert (workspace / "ccr_store.db").exists()+++def _ccr_db_state(workspace: Path) -> tuple[int, dict[str, bytes]]:+ """Row count plus the exact bytes of ccr_store.db and its WAL/SHM files."""+ import sqlite3++ db = workspace / "ccr_store.db"+ conn = sqlite3.connect(f"file:{db}?mode=ro", uri=True)+ try:+ rows = conn.execute("SELECT COUNT(*) FROM ccr_entries").fetchone()[0]+ finally:+ conn.close()+ files = {p.name: p.read_bytes() for p in sorted(workspace.glob("ccr_store.db*"))}+ return rows, files+++def test_stateless_switch_leaves_an_existing_sqlite_store_untouched(workspace):+ """A SQLite singleton built before the flag is swapped out, not cleared.++ Entering stateless mode must not write to (or empty) a ccr_store.db that+ already holds entries; it only stops using it.+ """+ from headroom.proxy.server import ProxyConfig, _apply_stateless_persistence++ stateful = cs.get_compression_store() # default backend: SQLite+ assert not isinstance(stateful._backend, InMemoryBackend)+ kept = stateful.store('{"rows": [1, 2, 3]}', '{"rows": [1]}')+ rows_before, bytes_before = _ccr_db_state(workspace)+ assert rows_before == 1++ paths.set_process_stateless(True)+ _apply_stateless_persistence(ProxyConfig(stateless=True))+ stateless = cs.get_compression_store()+ assert isinstance(stateless._backend, InMemoryBackend)+ fresh = stateless.store('{"rows": [4, 5, 6]}', '{"rows": [4]}')+ assert stateless.retrieve(fresh) is not None++ rows_after, bytes_after = _ccr_db_state(workspace)+ assert rows_after == rows_before, "stateless switch deleted persisted CCR entries"+ assert bytes_after == bytes_before, "stateless switch wrote to ccr_store.db"+ assert stateful.retrieve(kept) is not None+++# ---- the shared predicate ---------------------------------------------------+++def test_persistence_allowed_follows_the_stateless_flag(caplog):+ assert paths.persistence_allowed("x") is True+ paths.set_process_stateless(True)+ import logging++ with caplog.at_level(logging.INFO, logger="headroom.paths"):+ assert paths.persistence_allowed("x") is False+ assert paths.persistence_allowed("x") is False+ assert paths.persistence_allowed("y") is False+ notices = [r for r in caplog.records if "not persisting" in r.getMessage()]+ assert [r.getMessage().split("persisting ")[1].split(" to")[0] for r in notices] == ["x", "y"]+++def test_persistence_allowed_honours_env(monkeypatch):+ monkeypatch.setenv("HEADROOM_STATELESS", "true")+ assert paths.persistence_allowed("x") is False+++# ---- CCR backend selection --------------------------------------------------++[email protected]("backend_env", [None, "sqlite"])+def test_default_ccr_backend_is_memory_under_stateless(workspace, monkeypatch, backend_env):+ if backend_env is not None:+ monkeypatch.setenv("HEADROOM_CCR_BACKEND", backend_env)+ paths.set_process_stateless(True)+ assert cs._create_default_ccr_backend() is None+ assert not (workspace / "ccr_store.db").exists()+++def test_default_ccr_backend_is_sqlite_when_stateful(workspace):+ backend = cs._create_default_ccr_backend()+ assert backend is not None and not isinstance(backend, InMemoryBackend)+ assert (workspace / "ccr_store.db").exists()+++# ---- individual writers -----------------------------------------------------+++def test_license_cache_not_written_under_stateless(workspace, tmp_path):+ from headroom.telemetry.reporter import LicenseInfo, UsageReporter++ cache = tmp_path / "license.json"+ reporter = UsageReporter("hlk_test", cache_path=cache)+ reporter._license_info = LicenseInfo(status="active")+ paths.set_process_stateless(True)+ reporter._save_cache()+ assert not cache.exists()+ paths.set_process_stateless(False)+ reporter._save_cache()+ assert cache.exists()+++def test_mcp_session_stats_not_written_under_stateless(workspace, monkeypatch):+ from headroom.ccr import mcp_server++ stats_file = workspace / "session_stats.jsonl"+ monkeypatch.setattr(mcp_server, "SHARED_STATS_DIR", workspace)+ monkeypatch.setattr(mcp_server, "SHARED_STATS_FILE", stats_file)+ paths.set_process_stateless(True)+ mcp_server._append_shared_event({"timestamp": 1.0, "type": "compression"})+ assert not stats_file.exists()+ paths.set_process_stateless(False)+ mcp_server._append_shared_event({"timestamp": 1.0, "type": "compression"})+ assert stats_file.exists()+++def test_savings_ledger_not_written_under_stateless(workspace, tmp_path):+ from headroom import savings_ledger++ target = tmp_path / "events.jsonl"+ paths.set_process_stateless(True)+ assert (+ savings_ledger.record_savings_event(tokens_before=100, tokens_after=10, path=target)+ is False+ )+ assert not target.exists()+ paths.set_process_stateless(False)+ assert (+ savings_ledger.record_savings_event(tokens_before=100, tokens_after=10, path=target) is True+ )+ assert target.exists()+++def test_subscription_state_not_persisted_under_stateless(workspace, tmp_path):+ from headroom.subscription.tracker import SubscriptionTracker++ persist = tmp_path / "sub.json"+ tracker = SubscriptionTracker(persist_path=persist)+ paths.set_process_stateless(True)+ tracker._persist_state()+ assert not persist.exists()+ paths.set_process_stateless(False)+ tracker._persist_state()+ assert persist.exists()+++def test_memory_sync_state_not_written_under_stateless(tmp_path):+ from headroom.memory.sync import _save_sync_state++ state = tmp_path / "sync.json"+ paths.set_process_stateless(True)+ _save_sync_state(state, {"a": 1})+ assert not state.exists()+ paths.set_process_stateless(False)+ _save_sync_state(state, {"a": 1})+ assert state.exists()+++def test_update_check_cache_not_written_under_stateless(workspace):+ from headroom import update_check++ paths.set_process_stateless(True)+ assert update_check.is_update_check_enabled() is False+ update_check.write_cache("9.9.9", now=1.0)+ assert not update_check._cache_path().exists()+++def test_cli_stateless_flag_exports_env_for_children(monkeypatch):+ """`headroom proxy --stateless` must make HEADROOM_STATELESS visible to child processes."""+ from click.testing import CliRunner++ from headroom.cli import main++ monkeypatch.delenv("HEADROOM_STATELESS", raising=False)+ captured: dict[str, str] = {}++ def fake_run_server(*_args, **_kwargs):+ captured["HEADROOM_STATELESS"] = os.environ.get("HEADROOM_STATELESS", "")++ # The CLI imports run_server lazily from headroom.proxy.server, so patch it there.+ import headroom.proxy.server as proxy_server++ monkeypatch.setattr(proxy_server, "run_server", fake_run_server)+ result = CliRunner().invoke(main, ["proxy", "--stateless", "--port", "18999"])+ assert result.exit_code == 0, result.output+ assert captured.get("HEADROOM_STATELESS") == "1"