Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Fix remaining review issues: null safety, compressed count, batch tokens
chopratejas7f849658ce56
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex 6b55a5d7f..a1d341a09 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -240,7 +240,7 @@ def build_session_summary(if entry.model and "count_tokens" in entry.model:uncompressed_reasons["passthrough"] += 1continue- if entry.tokens_saved > 0 and entry.savings_percent > 0:+ if entry.tokens_saved > 0:compressed_requests.append({"savings_pct": round(entry.savings_percent, 1),diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.pyindex 021b3f5db..14076ec63 100644--- a/headroom/proxy/handlers/batch.py+++ b/headroom/proxy/handlers/batch.py@@ -134,9 +134,15 @@ class BatchHandlerMixin:continue# Apply optimization+ original_tokens = 0 # Set before try so error handler can use it+ optimized_tokens = 0try:- # Default context limit for most models- context_limit = 128000+ # Look up model context limit, fall back to 128K+ context_limit = (+ self.openai_provider.get_context_limit(model)+ if hasattr(self, "openai_provider")+ else 128000+ )# Use OpenAI pipeline (similar message format after conversion)result = self.openai_pipeline.apply(@@ -219,8 +225,7 @@ class BatchHandlerMixin:)# Pass through unchanged on failure — count original as optimizedcompressed_requests.append(batch_req)- # original_tokens may be unbound if pipeline failed before producing result- # Just skip the token accounting for this failed request+ total_optimized_tokens += original_tokens # 0 if pipeline never ran# Update body with compressed requestsbody["batch"]["input_config"]["requests"]["requests"] = compressed_requestsdiff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex ad9f7ed0a..ea9065915 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -80,7 +80,7 @@ class StreamingMixin:usage["input_tokens"] = chunk_usage.get("prompt_tokens", 0)usage["output_tokens"] = chunk_usage.get("completion_tokens", 0)# OpenAI has cached tokens in prompt_tokens_details- details = chunk_usage.get("prompt_tokens_details", {})+ details = chunk_usage.get("prompt_tokens_details") or {}usage["cache_read_input_tokens"] = details.get("cached_tokens", 0)elif provider == "gemini":@@ -164,7 +164,7 @@ class StreamingMixin:if chunk_usage:usage_found["input_tokens"] = chunk_usage.get("prompt_tokens", 0)usage_found["output_tokens"] = chunk_usage.get("completion_tokens", 0)- details = chunk_usage.get("prompt_tokens_details", {})+ details = chunk_usage.get("prompt_tokens_details") or {}usage_found["cache_read_input_tokens"] = details.get("cached_tokens", 0)elif provider == "gemini":