Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Add --stateless mode and graceful degradation for enterprise deployments

chopratejasacb8ff39ef84
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index 84c66db01..a3863595e 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -179,6 +179,13 @@ from .main import main
is_flag=True,
help="Disable anonymous usage telemetry (env: HEADROOM_TELEMETRY=off)",
)
+ "--stateless",
+ is_flag=True,
+ help="Disable all filesystem writes — run purely in-memory. "
+ "For containerized / read-only / load-balanced deployments. "
+ "(env: HEADROOM_STATELESS=true)",
+)
@click.pass_context
def proxy(
ctx: click.Context,
@@ -213,6 +220,7 @@ def proxy(
bedrock_region: str | None,
bedrock_profile: str | None,
no_telemetry: bool,
+ stateless: bool,
) -> None:
"""Start the optimization proxy server.
@@ -251,10 +259,22 @@ def proxy(
mode or os.environ.get("HEADROOM_MODE") or PROXY_MODE_TOKEN
)
+ # Stateless mode: CLI flag or env var
+ is_stateless = stateless or os.environ.get("HEADROOM_STATELESS", "").lower() in (
+ "true",
+ "1",
+ "yes",
+ "on",
+ )
+
# Telemetry opt-out: --no-telemetry flag sets the env var
if no_telemetry:
os.environ["HEADROOM_TELEMETRY"] = "off"
+ # Stateless mode: suppress TOIN filesystem persistence
+ if is_stateless:
+ os.environ["HEADROOM_TOIN_BACKEND"] = "none"
+
# License key for managed/enterprise deployments (optional)
license_key = os.environ.get("HEADROOM_LICENSE_KEY")
@@ -272,7 +292,7 @@ def proxy(
connect_timeout_seconds=connect_timeout_seconds
if connect_timeout_seconds is not None
else 10,
- log_file=log_file,
+ log_file=None if is_stateless else log_file,
budget_limit_usd=budget,
# Code graph: live file watcher for incremental reindexing
code_graph_watcher=code_graph,
@@ -284,13 +304,15 @@ def proxy(
intelligent_context_compress_first=not no_compress_first,
# Memory System (Multi-Provider with auto-detection)
# --learn implies --memory (need backend for storing patterns)
- memory_enabled=memory or (learn and not no_learn),
+ # Stateless mode disables memory (requires SQLite on disk)
+ memory_enabled=False if is_stateless else (memory or (learn and not no_learn)),
memory_db_path=memory_db_path,
memory_inject_tools=not no_memory_tools,
memory_inject_context=not no_memory_context,
memory_top_k=memory_top_k,
# Traffic Learning: only with --learn, never with --no-learn
- traffic_learning_enabled=learn and not no_learn,
+ # Stateless mode disables learning (requires filesystem)
+ traffic_learning_enabled=False if is_stateless else (learn and not no_learn),
traffic_learning_agent_type=os.environ.get("HEADROOM_AGENT_TYPE", "unknown"),
# Backend (Anthropic direct, Bedrock, LiteLLM, or any-llm)
backend=backend,
@@ -299,6 +321,8 @@ def proxy(
anyllm_provider=effective_anyllm_provider,
# License / Usage Reporting (managed/enterprise)
license_key=license_key,
+ # Stateless mode: disable all filesystem writes
+ stateless=is_stateless,
)
memory_status = "DISABLED"
@@ -355,6 +379,13 @@ Memory (Multi-Provider):
- Database: {config.memory_db_path}
"""
+ # Stateless mode warning
+ stateless_line = ""
+ if is_stateless:
+ stateless_line = (
+ " Stateless: YES (no filesystem writes — memory, logs, TOIN disabled)\n"
+ )
+
from headroom.telemetry.beacon import is_telemetry_enabled
# Build telemetry section for the startup banner
@@ -381,7 +412,7 @@ Starting proxy server...
Rate Limit: {"ENABLED" if config.rate_limit_enabled else "DISABLED"}
Memory: {memory_status}
License: {license_status}
-{telemetry_line}
+{stateless_line}{telemetry_line}
{backend_section}
Routing:
/v1/messages → {anthropic_url}
diff --git a/headroom/install/state.py b/headroom/install/state.py
index e3882e288..06b3bf023 100644
--- a/headroom/install/state.py
+++ b/headroom/install/state.py
@@ -3,21 +3,30 @@
from __future__ import annotations
import json
+import logging
import shutil
from dataclasses import asdict
from .models import ArtifactRecord, DeploymentManifest, ManagedMutation, iso_utc_now
from .paths import deploy_root, manifest_path, profile_root
+logger = logging.getLogger(__name__)
-def save_manifest(manifest: DeploymentManifest) -> None:
- """Persist a deployment manifest to disk."""
- root = profile_root(manifest.profile)
- root.mkdir(parents=True, exist_ok=True)
- manifest.updated_at = iso_utc_now()
- path = manifest_path(manifest.profile)
- path.write_text(json.dumps(asdict(manifest), indent=2) + "\n")
+def save_manifest(manifest: DeploymentManifest) -> None:
+ """Persist a deployment manifest to disk.
+
+ Gracefully handles read-only filesystems by logging a warning
+ instead of crashing.
+ """
+ try:
+ root = profile_root(manifest.profile)
+ root.mkdir(parents=True, exist_ok=True)
+ manifest.updated_at = iso_utc_now()
+ path = manifest_path(manifest.profile)
+ path.write_text(json.dumps(asdict(manifest), indent=2) + "\n")
+ except OSError as e:
+ logger.warning("Cannot save deployment manifest: %s — continuing without persistence", e)
def load_manifest(profile: str = "default") -> DeploymentManifest | None:
diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py
index 33d33592e..446bae552 100644
--- a/headroom/proxy/models.py
+++ b/headroom/proxy/models.py
@@ -207,3 +207,6 @@ class ProxyConfig:
subscription_tracking_enabled: bool = True
subscription_poll_interval_s: int = 10
subscription_active_window_s: int = 60
+
+ # Stateless mode — disable all filesystem writes for read-only / container deployments
+ stateless: bool = False
diff --git a/headroom/proxy/request_logger.py b/headroom/proxy/request_logger.py
index 458c28e6d..c4c04a73f 100644
--- a/headroom/proxy/request_logger.py
+++ b/headroom/proxy/request_logger.py
@@ -8,6 +8,7 @@ Extracted from server.py for maintainability.
from __future__ import annotations
import json
+import logging
import sys
from collections import deque
from dataclasses import asdict
@@ -19,11 +20,15 @@ if TYPE_CHECKING:
from headroom.proxy.models import RequestLog
+logger = logging.getLogger(__name__)
+
class RequestLogger:
"""Log requests to JSONL file.
Uses a deque with max 10,000 entries to prevent unbounded memory growth.
+ Gracefully degrades to in-memory-only if the log file cannot be written
+ (read-only filesystem, permissions error, etc.).
"""
MAX_LOG_ENTRIES = 10_000
@@ -35,19 +40,30 @@ class RequestLogger:
self._logs: deque[RequestLog] = deque(maxlen=self.MAX_LOG_ENTRIES)
if self.log_file:
- self.log_file.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ self.log_file.parent.mkdir(parents=True, exist_ok=True)
+ except OSError as e:
+ logger.warning(
+ "Cannot create log directory %s: %s — logging to memory only",
+ self.log_file.parent,
+ e,
+ )
+ self.log_file = None
def log(self, entry: RequestLog):
"""Log a request. Oldest entries are automatically removed when limit reached."""
self._logs.append(entry)
if self.log_file:
- with open(self.log_file, "a") as f:
- log_dict = asdict(entry)
- if not self.log_full_messages:
- log_dict.pop("request_messages", None)
- log_dict.pop("response_content", None)
- f.write(json.dumps(log_dict) + "\n")
+ try:
+ with open(self.log_file, "a") as f:
+ log_dict = asdict(entry)
+ if not self.log_full_messages:
+ log_dict.pop("request_messages", None)
+ log_dict.pop("response_content", None)
+ f.write(json.dumps(log_dict) + "\n")
+ except OSError:
+ pass # Graceful degradation: memory-only logging continues
def get_recent(self, n: int = 100) -> list[dict]:
"""Get recent log entries."""
diff --git a/headroom/telemetry/toin.py b/headroom/telemetry/toin.py
index 4b7eb7061..191b48efb 100644
--- a/headroom/telemetry/toin.py
+++ b/headroom/telemetry/toin.py
@@ -1174,11 +1174,11 @@ class ToolIntelligenceNetwork:
canonical = "null"
elif isinstance(value, bool):
canonical = "true" if value else "false"
- elif isinstance(value, (int, float)):
+ elif isinstance(value, int | float):
canonical = str(value)
elif isinstance(value, str):
canonical = value
- elif isinstance(value, (list, dict)):
+ elif isinstance(value, list | dict):
# For complex types, use JSON serialization
try:
canonical = json.dumps(value, sort_keys=True, default=str)
@@ -1573,6 +1573,8 @@ def _create_default_toin_backend() -> Any:
backend_type = (os.environ.get(TOIN_BACKEND_ENV_VAR) or "").strip().lower()
if not backend_type or backend_type == "filesystem":
return None
+ if backend_type == "none":
+ return None # Explicit in-memory-only (e.g. --stateless mode)
try:
from importlib.metadata import entry_points