Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Add --stateless mode and graceful degradation for enterprise deployments
chopratejasacb8ff39ef84
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 84c66db01..a3863595e 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -179,6 +179,13 @@ from .main import mainis_flag=True,help="Disable anonymous usage telemetry (env: HEADROOM_TELEMETRY=off)",)+ "--stateless",+ is_flag=True,+ help="Disable all filesystem writes — run purely in-memory. "+ "For containerized / read-only / load-balanced deployments. "+ "(env: HEADROOM_STATELESS=true)",+)@click.pass_contextdef proxy(ctx: click.Context,@@ -213,6 +220,7 @@ def proxy(bedrock_region: str | None,bedrock_profile: str | None,no_telemetry: bool,+ stateless: bool,) -> None:"""Start the optimization proxy server.@@ -251,10 +259,22 @@ def proxy(mode or os.environ.get("HEADROOM_MODE") or PROXY_MODE_TOKEN)+ # Stateless mode: CLI flag or env var+ is_stateless = stateless or os.environ.get("HEADROOM_STATELESS", "").lower() in (+ "true",+ "1",+ "yes",+ "on",+ )+# Telemetry opt-out: --no-telemetry flag sets the env varif no_telemetry:os.environ["HEADROOM_TELEMETRY"] = "off"+ # Stateless mode: suppress TOIN filesystem persistence+ if is_stateless:+ os.environ["HEADROOM_TOIN_BACKEND"] = "none"+# License key for managed/enterprise deployments (optional)license_key = os.environ.get("HEADROOM_LICENSE_KEY")@@ -272,7 +292,7 @@ def proxy(connect_timeout_seconds=connect_timeout_secondsif connect_timeout_seconds is not Noneelse 10,- log_file=log_file,+ log_file=None if is_stateless else log_file,budget_limit_usd=budget,# Code graph: live file watcher for incremental reindexingcode_graph_watcher=code_graph,@@ -284,13 +304,15 @@ def proxy(intelligent_context_compress_first=not no_compress_first,# Memory System (Multi-Provider with auto-detection)# --learn implies --memory (need backend for storing patterns)- memory_enabled=memory or (learn and not no_learn),+ # Stateless mode disables memory (requires SQLite on disk)+ memory_enabled=False if is_stateless else (memory or (learn and not no_learn)),memory_db_path=memory_db_path,memory_inject_tools=not no_memory_tools,memory_inject_context=not no_memory_context,memory_top_k=memory_top_k,# Traffic Learning: only with --learn, never with --no-learn- traffic_learning_enabled=learn and not no_learn,+ # Stateless mode disables learning (requires filesystem)+ traffic_learning_enabled=False if is_stateless else (learn and not no_learn),traffic_learning_agent_type=os.environ.get("HEADROOM_AGENT_TYPE", "unknown"),# Backend (Anthropic direct, Bedrock, LiteLLM, or any-llm)backend=backend,@@ -299,6 +321,8 @@ def proxy(anyllm_provider=effective_anyllm_provider,# License / Usage Reporting (managed/enterprise)license_key=license_key,+ # Stateless mode: disable all filesystem writes+ stateless=is_stateless,)memory_status = "DISABLED"@@ -355,6 +379,13 @@ Memory (Multi-Provider):- Database: {config.memory_db_path}"""+ # Stateless mode warning+ stateless_line = ""+ if is_stateless:+ stateless_line = (+ " Stateless: YES (no filesystem writes — memory, logs, TOIN disabled)\n"+ )+from headroom.telemetry.beacon import is_telemetry_enabled# Build telemetry section for the startup banner@@ -381,7 +412,7 @@ Starting proxy server...Rate Limit: {"ENABLED" if config.rate_limit_enabled else "DISABLED"}Memory: {memory_status}License: {license_status}-{telemetry_line}+{stateless_line}{telemetry_line}{backend_section}Routing:/v1/messages → {anthropic_url}diff --git a/headroom/install/state.py b/headroom/install/state.pyindex e3882e288..06b3bf023 100644--- a/headroom/install/state.py+++ b/headroom/install/state.py@@ -3,21 +3,30 @@from __future__ import annotationsimport json+import loggingimport shutilfrom dataclasses import asdictfrom .models import ArtifactRecord, DeploymentManifest, ManagedMutation, iso_utc_nowfrom .paths import deploy_root, manifest_path, profile_root+logger = logging.getLogger(__name__)-def save_manifest(manifest: DeploymentManifest) -> None:- """Persist a deployment manifest to disk."""- root = profile_root(manifest.profile)- root.mkdir(parents=True, exist_ok=True)- manifest.updated_at = iso_utc_now()- path = manifest_path(manifest.profile)- path.write_text(json.dumps(asdict(manifest), indent=2) + "\n")+def save_manifest(manifest: DeploymentManifest) -> None:+ """Persist a deployment manifest to disk.++ Gracefully handles read-only filesystems by logging a warning+ instead of crashing.+ """+ try:+ root = profile_root(manifest.profile)+ root.mkdir(parents=True, exist_ok=True)+ manifest.updated_at = iso_utc_now()+ path = manifest_path(manifest.profile)+ path.write_text(json.dumps(asdict(manifest), indent=2) + "\n")+ except OSError as e:+ logger.warning("Cannot save deployment manifest: %s — continuing without persistence", e)def load_manifest(profile: str = "default") -> DeploymentManifest | None:diff --git a/headroom/proxy/models.py b/headroom/proxy/models.pyindex 33d33592e..446bae552 100644--- a/headroom/proxy/models.py+++ b/headroom/proxy/models.py@@ -207,3 +207,6 @@ class ProxyConfig:subscription_tracking_enabled: bool = Truesubscription_poll_interval_s: int = 10subscription_active_window_s: int = 60++ # Stateless mode — disable all filesystem writes for read-only / container deployments+ stateless: bool = Falsediff --git a/headroom/proxy/request_logger.py b/headroom/proxy/request_logger.pyindex 458c28e6d..c4c04a73f 100644--- a/headroom/proxy/request_logger.py+++ b/headroom/proxy/request_logger.py@@ -8,6 +8,7 @@ Extracted from server.py for maintainability.from __future__ import annotationsimport json+import loggingimport sysfrom collections import dequefrom dataclasses import asdict@@ -19,11 +20,15 @@ if TYPE_CHECKING:from headroom.proxy.models import RequestLog+logger = logging.getLogger(__name__)+class RequestLogger:"""Log requests to JSONL file.Uses a deque with max 10,000 entries to prevent unbounded memory growth.+ Gracefully degrades to in-memory-only if the log file cannot be written+ (read-only filesystem, permissions error, etc.)."""MAX_LOG_ENTRIES = 10_000@@ -35,19 +40,30 @@ class RequestLogger:self._logs: deque[RequestLog] = deque(maxlen=self.MAX_LOG_ENTRIES)if self.log_file:- self.log_file.parent.mkdir(parents=True, exist_ok=True)+ try:+ self.log_file.parent.mkdir(parents=True, exist_ok=True)+ except OSError as e:+ logger.warning(+ "Cannot create log directory %s: %s — logging to memory only",+ self.log_file.parent,+ e,+ )+ self.log_file = Nonedef log(self, entry: RequestLog):"""Log a request. Oldest entries are automatically removed when limit reached."""self._logs.append(entry)if self.log_file:- with open(self.log_file, "a") as f:- log_dict = asdict(entry)- if not self.log_full_messages:- log_dict.pop("request_messages", None)- log_dict.pop("response_content", None)- f.write(json.dumps(log_dict) + "\n")+ try:+ with open(self.log_file, "a") as f:+ log_dict = asdict(entry)+ if not self.log_full_messages:+ log_dict.pop("request_messages", None)+ log_dict.pop("response_content", None)+ f.write(json.dumps(log_dict) + "\n")+ except OSError:+ pass # Graceful degradation: memory-only logging continuesdef get_recent(self, n: int = 100) -> list[dict]:"""Get recent log entries."""diff --git a/headroom/telemetry/toin.py b/headroom/telemetry/toin.pyindex 4b7eb7061..191b48efb 100644--- a/headroom/telemetry/toin.py+++ b/headroom/telemetry/toin.py@@ -1174,11 +1174,11 @@ class ToolIntelligenceNetwork:canonical = "null"elif isinstance(value, bool):canonical = "true" if value else "false"- elif isinstance(value, (int, float)):+ elif isinstance(value, int | float):canonical = str(value)elif isinstance(value, str):canonical = value- elif isinstance(value, (list, dict)):+ elif isinstance(value, list | dict):# For complex types, use JSON serializationtry:canonical = json.dumps(value, sort_keys=True, default=str)@@ -1573,6 +1573,8 @@ def _create_default_toin_backend() -> Any:backend_type = (os.environ.get(TOIN_BACKEND_ENV_VAR) or "").strip().lower()if not backend_type or backend_type == "filesystem":return None+ if backend_type == "none":+ return None # Explicit in-memory-only (e.g. --stateless mode)try:from importlib.metadata import entry_points