Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Fix token savings metrics mismatch between pipeline and server
chopratejas5e3aa3880f20
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 03392b44d..e6eb5a0e6 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -1564,7 +1564,7 @@ class HeadroomProxy:# Count original tokenstokenizer = get_tokenizer(model)- original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)+ original_tokens = tokenizer.count_messages(messages)# Apply optimizationtransforms_applied = []@@ -1583,9 +1583,9 @@ class HeadroomProxy:if result.messages != messages:optimized_messages = result.messagestransforms_applied = result.transforms_applied- optimized_tokens = sum(- tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages- )+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_afterexcept Exception as e:logger.warning(f"Optimization failed: {e}")@@ -2219,11 +2219,6 @@ class HeadroomProxy:compressed_requests.append(batch_req)continue- # Count original tokens- tokenizer = get_tokenizer(model)- original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)- total_original_tokens += original_tokens-# Apply optimizationtry:context_limit = self.anthropic_provider.get_context_limit(model)@@ -2234,9 +2229,10 @@ class HeadroomProxy:)optimized_messages = result.messages- optimized_tokens = sum(- tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages- )+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after+ total_original_tokens += original_tokenstotal_optimized_tokens += optimized_tokenstokens_saved = original_tokens - optimized_tokenstotal_tokens_saved += tokens_saved@@ -2667,11 +2663,6 @@ class HeadroomProxy:compressed_requests.append(batch_req)continue- # Count original tokens- tokenizer = get_tokenizer(model)- original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)- total_original_tokens += original_tokens-# Apply optimizationtry:# Default context limit for most models@@ -2685,9 +2676,10 @@ class HeadroomProxy:)optimized_messages = result.messages- optimized_tokens = sum(- tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages- )+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after+ total_original_tokens += original_tokenstotal_optimized_tokens += optimized_tokenstokens_saved = original_tokens - optimized_tokenstotal_tokens_saved += tokens_saved@@ -3971,7 +3963,7 @@ class HeadroomProxy:# Token countingtokenizer = get_tokenizer(model)- original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)+ original_tokens = tokenizer.count_messages(messages)# Optimizationtransforms_applied = []@@ -3989,9 +3981,9 @@ class HeadroomProxy:if result.messages != messages:optimized_messages = result.messagestransforms_applied = result.transforms_applied- optimized_tokens = sum(- tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages- )+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_afterexcept Exception as e:logger.warning(f"Optimization failed: {e}")@@ -4476,12 +4468,6 @@ class HeadroomProxy:total_requests += 1continue- # Count original tokens- original_tokens = sum(- tokenizer.count_text(str(m.get("content", ""))) for m in messages- )- total_original_tokens += original_tokens-# Compress messages using the OpenAI pipelineif self.config.optimize:try:@@ -4492,16 +4478,20 @@ class HeadroomProxy:model_limit=context_limit,)compressed_messages = result.messages+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ compressed_tokens = result.tokens_afterexcept Exception as e:logger.warning(f"[{request_id}] Compression failed for line {i}: {e}")compressed_messages = messages+ original_tokens = tokenizer.count_messages(messages)+ compressed_tokens = original_tokenselse:compressed_messages = messages+ original_tokens = tokenizer.count_messages(messages)+ compressed_tokens = original_tokens- # Count compressed tokens- compressed_tokens = sum(- tokenizer.count_text(str(m.get("content", ""))) for m in compressed_messages- )+ total_original_tokens += original_tokenstotal_compressed_tokens += compressed_tokenstokens_saved = original_tokens - compressed_tokens@@ -4758,7 +4748,7 @@ class HeadroomProxy:# Token counting on converted messagestokenizer = get_tokenizer(model)- original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)+ original_tokens = tokenizer.count_messages(messages)# Note: We pass through to OpenAI without optimization for now# The Responses API has different semantics that may not work well with compression@@ -4974,7 +4964,7 @@ class HeadroomProxy:# Token countingtokenizer = get_tokenizer(model)- original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)+ original_tokens = tokenizer.count_messages(messages)# Optimizationtransforms_applied: list[str] = []@@ -4993,9 +4983,9 @@ class HeadroomProxy:if result.messages != messages:optimized_messages = result.messagestransforms_applied = result.transforms_applied- optimized_tokens = sum(- tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages- )+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_afterexcept Exception as e:logger.warning(f"[{request_id}] Gemini optimization failed: {e}")@@ -5268,7 +5258,7 @@ class HeadroomProxy:# Token counting (original)tokenizer = get_tokenizer(model)- original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)+ original_tokens = tokenizer.count_messages(messages)# Apply compression using the same pipeline as generateContenttransforms_applied: list[str] = []