Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Fix code review issues: batch NameError, null safety, token types
chopratejasaf418a255608
diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.pyindex 27276c351..021b3f5db 100644--- a/headroom/proxy/handlers/batch.py+++ b/headroom/proxy/handlers/batch.py@@ -217,9 +217,10 @@ class BatchHandlerMixin:logger.warning(f"[{request_id}] Optimization failed for Google batch request {idx}: {e}")- # Pass through unchanged on failure+ # Pass through unchanged on failure — count original as optimizedcompressed_requests.append(batch_req)- total_optimized_tokens += original_tokens+ # original_tokens may be unbound if pipeline failed before producing result+ # Just skip the token accounting for this failed request# Update body with compressed requestsbody["batch"]["input_config"]["requests"]["requests"] = compressed_requestsdiff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 4bdcc1ec1..d7585a50c 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -497,7 +497,7 @@ class OpenAIHandlerMixin:output_tokens = usage.get("completion_tokens", 0)# OpenAI returns cached_tokens in prompt_tokens_details# These are charged at 50% of the input price- prompt_details = usage.get("prompt_tokens_details", {})+ prompt_details = usage.get("prompt_tokens_details") or {}cache_read_tokens = prompt_details.get("cached_tokens", 0)except (KeyError, TypeError, AttributeError) as e:logger.debug(diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex f417cb197..fea2d8273 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -191,5 +191,7 @@ async def _read_request_json(request: Request) -> dict[str, Any]:except UnicodeDecodeError as exc:raise ValueError(f"Request body is not valid UTF-8 (possibly compressed?): {exc}") from exc- result: dict[str, Any] = json.loads(text)+ result = json.loads(text)+ if not isinstance(result, dict):+ raise ValueError("Request body must be a JSON object, not " + type(result).__name__)return resultdiff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 83d5a41c8..deceef9a7 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -132,15 +132,9 @@ logging.basicConfig(logger = logging.getLogger("headroom.proxy")# Always-on file logging to ~/.headroom/logs/ for `headroom perf` analysis-_HEADROOM_LOG_DIR = Path.home() / ".headroom" / "logs"--_setup_file_logging()-# Maximum rate limiter buckets (prevents DoS via spoofed API keys)-MAX_RATE_LIMITER_BUCKETS = 1000-# Compression pipeline timeout in secondsdiff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.pyindex a0b782044..2d9a7c7b2 100644--- a/headroom/tokenizers/base.py+++ b/headroom/tokenizers/base.py@@ -132,7 +132,7 @@ class BaseTokenizer(ABC):if part_type == "text":total += self.count_text(part.get("text", ""))- elif part_type in ("image_url", "image"):+ elif part_type in ("image_url", "image", "input_image"):# Images are NOT tokenized as text — they have a pixel-based cost.# Anthropic: tokens = (width * height) / 750, max ~1600 after resize.# OpenAI: similar tile-based calculation, ~765 tokens for high-detail.@@ -140,6 +140,9 @@ class BaseTokenizer(ABC):# This prevents the base64 blob from being json.dumps'd and counted# as text tokens (1MB image = ~330K fake tokens without this).total += 1600+ elif part_type in ("input_audio", "audio"):+ # Audio has fixed token cost, not tokenized as text+ total += 200elif part_type == "tool_result":content = part.get("content", "")if isinstance(content, str):