Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Fix remaining review issues: null safety, compressed count, batch tokens

chopratejas7f849658ce56
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py
index 6b55a5d7f..a1d341a09 100644
--- a/headroom/proxy/cost.py
+++ b/headroom/proxy/cost.py
@@ -240,7 +240,7 @@ def build_session_summary(
if entry.model and "count_tokens" in entry.model:
uncompressed_reasons["passthrough"] += 1
continue
- if entry.tokens_saved > 0 and entry.savings_percent > 0:
+ if entry.tokens_saved > 0:
compressed_requests.append(
{
"savings_pct": round(entry.savings_percent, 1),
diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.py
index 021b3f5db..14076ec63 100644
--- a/headroom/proxy/handlers/batch.py
+++ b/headroom/proxy/handlers/batch.py
@@ -134,9 +134,15 @@ class BatchHandlerMixin:
continue
# Apply optimization
+ original_tokens = 0 # Set before try so error handler can use it
+ optimized_tokens = 0
try:
- # Default context limit for most models
- context_limit = 128000
+ # Look up model context limit, fall back to 128K
+ context_limit = (
+ self.openai_provider.get_context_limit(model)
+ if hasattr(self, "openai_provider")
+ else 128000
+ )
# Use OpenAI pipeline (similar message format after conversion)
result = self.openai_pipeline.apply(
@@ -219,8 +225,7 @@ class BatchHandlerMixin:
)
# Pass through unchanged on failure — count original as optimized
compressed_requests.append(batch_req)
- # original_tokens may be unbound if pipeline failed before producing result
- # Just skip the token accounting for this failed request
+ total_optimized_tokens += original_tokens # 0 if pipeline never ran
# Update body with compressed requests
body["batch"]["input_config"]["requests"]["requests"] = compressed_requests
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index ad9f7ed0a..ea9065915 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -80,7 +80,7 @@ class StreamingMixin:
usage["input_tokens"] = chunk_usage.get("prompt_tokens", 0)
usage["output_tokens"] = chunk_usage.get("completion_tokens", 0)
# OpenAI has cached tokens in prompt_tokens_details
- details = chunk_usage.get("prompt_tokens_details", {})
+ details = chunk_usage.get("prompt_tokens_details") or {}
usage["cache_read_input_tokens"] = details.get("cached_tokens", 0)
elif provider == "gemini":
@@ -164,7 +164,7 @@ class StreamingMixin:
if chunk_usage:
usage_found["input_tokens"] = chunk_usage.get("prompt_tokens", 0)
usage_found["output_tokens"] = chunk_usage.get("completion_tokens", 0)
- details = chunk_usage.get("prompt_tokens_details", {})
+ details = chunk_usage.get("prompt_tokens_details") or {}
usage_found["cache_read_input_tokens"] = details.get("cached_tokens", 0)
elif provider == "gemini":