Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Fix CI lint/format failures after proxy mode hardening

JerrettDavis83b730f2b9cc
diff --git a/benchmarks/proxy_mode_benchmark.py b/benchmarks/proxy_mode_benchmark.py
index ce1961bb7..f723bae38 100644
--- a/benchmarks/proxy_mode_benchmark.py
+++ b/benchmarks/proxy_mode_benchmark.py
@@ -28,7 +28,6 @@ from headroom.proxy.server import HeadroomProxy
from headroom.tokenizers import get_tokenizer
from headroom.utils import extract_user_query
-
MODEL = "claude-sonnet-4-6"
@@ -50,7 +49,11 @@ class ModeBenchmarkResult:
@property
def cache_hit_pct(self) -> float:
- total = self.total_cache_read_tokens + self.total_cache_write_tokens + self.total_uncached_tokens
+ total = (
+ self.total_cache_read_tokens
+ + self.total_cache_write_tokens
+ + self.total_uncached_tokens
+ )
if total <= 0:
return 0.0
return self.total_cache_read_tokens / total * 100.0
@@ -76,7 +79,10 @@ def _build_conversation(turn: int) -> list[dict[str, Any]]:
for t in range(1, turn):
messages.extend(
[
- {"role": "user", "content": f"Analyze tool output turn {t} and summarize anomalies."},
+ {
+ "role": "user",
+ "content": f"Analyze tool output turn {t} and summarize anomalies.",
+ },
{
"role": "user",
"content": [
@@ -93,7 +99,10 @@ def _build_conversation(turn: int) -> list[dict[str, Any]]:
# Current turn: user request + fresh tool output, no assistant response yet.
messages.extend(
[
- {"role": "user", "content": f"Analyze tool output turn {turn} and summarize anomalies."},
+ {
+ "role": "user",
+ "content": f"Analyze tool output turn {turn} and summarize anomalies.",
+ },
{
"role": "user",
"content": [
diff --git a/headroom/ccr/batch_processor.py b/headroom/ccr/batch_processor.py
index fd51861ac..bfe73b317 100644
--- a/headroom/ccr/batch_processor.py
+++ b/headroom/ccr/batch_processor.py
@@ -343,6 +343,7 @@ class BatchResultProcessor:
"max_tokens": request_context.extras.get("max_tokens", 4096),
}
if tools:
+
def _tool_sort_key(tool: dict[str, Any]) -> tuple[str, str]:
name = (
str(tool.get("name", ""))
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index cd733b437..6b94ae04f 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -172,8 +172,8 @@ class AnthropicHandlerMixin:
_get_image_compressor,
_read_request_json,
)
- from headroom.proxy.modes import is_cache_mode, is_token_mode
from headroom.proxy.models import RequestLog
+ from headroom.proxy.modes import is_cache_mode, is_token_mode
from headroom.tokenizers import get_tokenizer
from headroom.utils import extract_user_query
@@ -594,10 +594,12 @@ class AnthropicHandlerMixin:
)
if memory_context:
if frozen_message_count > 0:
- optimized_messages = self._append_context_to_latest_non_frozen_user_turn(
- optimized_messages,
- memory_context,
- frozen_message_count=frozen_message_count,
+ optimized_messages = (
+ self._append_context_to_latest_non_frozen_user_turn(
+ optimized_messages,
+ memory_context,
+ frozen_message_count=frozen_message_count,
+ )
)
logger.info(
f"[{request_id}] Memory: Appended {len(memory_context)} chars "
@@ -1192,8 +1194,8 @@ class AnthropicHandlerMixin:
from fastapi.responses import JSONResponse, Response
from headroom.ccr import CCRToolInjector
- from headroom.proxy.modes import is_cache_mode
from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_json
+ from headroom.proxy.modes import is_cache_mode
from headroom.utils import extract_user_query
start_time = time.time()
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 070d12a0e..f6b3e9cb7 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -6,8 +6,8 @@ Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.
from __future__ import annotations
import asyncio
-import copy
import contextlib
+import copy
import json
import logging
import os
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 87215aa61..4a01a3d59 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -102,16 +102,16 @@ from headroom.proxy.helpers import (
_read_request_json, # noqa: F401
_setup_file_logging, # noqa: F401
)
+from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
+
+# Data models (extracted to headroom/proxy/models.py for maintainability)
+from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401
from headroom.proxy.modes import (
PROXY_MODE_CACHE,
PROXY_MODE_TOKEN,
is_token_mode,
normalize_proxy_mode,
)
-from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
-
-# Data models (extracted to headroom/proxy/models.py for maintainability)
-from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401
from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401
from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
from headroom.proxy.request_logger import RequestLogger # noqa: F401
diff --git a/tests/test_proxy_anthropic_cache_stability.py b/tests/test_proxy_anthropic_cache_stability.py
index 6481dc4f5..6b02eef7c 100644
--- a/tests/test_proxy_anthropic_cache_stability.py
+++ b/tests/test_proxy_anthropic_cache_stability.py
@@ -152,7 +152,11 @@ def test_image_compression_does_not_touch_previous_turns_if_last_message_not_use
"content": [
{
"type": "image",
- "source": {"type": "base64", "media_type": "image/png", "data": "OLD_IMAGE_BYTES"},
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": "OLD_IMAGE_BYTES",
+ },
}
],
},
@@ -192,7 +196,12 @@ def test_anthropic_batch_tools_sorted_deterministically_before_forward() -> None
"id": "msgbatch_1",
"type": "message_batch",
"processing_status": "in_progress",
- "request_counts": {"processing": 1, "succeeded": 0, "errored": 0, "canceled": 0},
+ "request_counts": {
+ "processing": 1,
+ "succeeded": 0,
+ "errored": 0,
+ "canceled": 0,
+ },
},
)
@@ -210,9 +219,21 @@ def test_anthropic_batch_tools_sorted_deterministically_before_forward() -> None
"max_tokens": 128,
"messages": [{"role": "user", "content": "hello"}],
"tools": [
- {"name": "zeta", "description": "z", "input_schema": {"type": "object"}},
- {"name": "alpha", "description": "a", "input_schema": {"type": "object"}},
- {"name": "mu", "description": "m", "input_schema": {"type": "object"}},
+ {
+ "name": "zeta",
+ "description": "z",
+ "input_schema": {"type": "object"},
+ },
+ {
+ "name": "alpha",
+ "description": "a",
+ "input_schema": {"type": "object"},
+ },
+ {
+ "name": "mu",
+ "description": "m",
+ "input_schema": {"type": "object"},
+ },
],
},
}
@@ -262,7 +283,9 @@ def test_token_mode_freeze_is_capped_by_prefix_tracker() -> None:
proxy.config.image_optimize = False
fake_tracker = _FakePrefixTracker(frozen_count=1)
- proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "stable-session"
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stable-session"
+ )
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
class _FakeCompressionCache:
@@ -332,7 +355,9 @@ def test_memory_context_avoids_system_mutation_when_prefix_frozen() -> None:
proxy.config.ccr_proactive_expansion = False
fake_tracker = _FakePrefixTracker(frozen_count=1)
- proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "stable-session"
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stable-session"
+ )
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
proxy.memory_handler = SimpleNamespace(
@@ -396,7 +421,9 @@ def test_ccr_system_instruction_injection_disabled_when_prefix_frozen(monkeypatc
proxy.config.ccr_inject_system_instructions = True
fake_tracker = _FakePrefixTracker(frozen_count=1)
- proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "stable-session"
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stable-session"
+ )
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
class _FakeInjector:
@@ -457,7 +484,9 @@ def test_previous_turns_always_frozen_only_final_turn_mutable() -> None:
proxy.config.image_optimize = False
fake_tracker = _FakePrefixTracker(frozen_count=0)
- proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "stable-session"
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stable-session"
+ )
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
def _fake_apply(**kwargs):
@@ -538,7 +567,12 @@ def test_batch_optimization_freezes_previous_turns_only() -> None:
"id": "msgbatch_2",
"type": "message_batch",
"processing_status": "in_progress",
- "request_counts": {"processing": 1, "succeeded": 0, "errored": 0, "canceled": 0},
+ "request_counts": {
+ "processing": 1,
+ "succeeded": 0,
+ "errored": 0,
+ "canceled": 0,
+ },
},
)
@@ -578,7 +612,9 @@ def test_token_mode_does_not_force_freeze_all_previous_turns() -> None:
proxy.config.image_optimize = False
fake_tracker = _FakePrefixTracker(frozen_count=0)
- proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "stable-session"
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stable-session"
+ )
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
class _FakeCompressionCache:
@@ -652,7 +688,9 @@ def test_cache_mode_restores_frozen_prefix_if_transform_mutates_history() -> Non
proxy.config.image_optimize = False
fake_tracker = _FakePrefixTracker(frozen_count=0)
- proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "stable-session"
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stable-session"
+ )
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
original_messages = [
diff --git a/tests/test_proxy_openai_cache_stability.py b/tests/test_proxy_openai_cache_stability.py
index a016fb9c0..41e4d1bf1 100644
--- a/tests/test_proxy_openai_cache_stability.py
+++ b/tests/test_proxy_openai_cache_stability.py
@@ -49,7 +49,9 @@ def test_openai_cache_mode_freezes_previous_turns() -> None:
proxy.config.mode = "cache"
fake_tracker = _FakePrefixTracker(frozen_count=0)
- proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "stable-session"
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stable-session"
+ )
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
def _fake_apply(**kwargs):
@@ -71,7 +73,11 @@ def test_openai_cache_mode_freezes_previous_turns() -> None:
json={
"id": "chatcmpl_1",
"choices": [
- {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
],
"usage": {"prompt_tokens": 60, "completion_tokens": 3, "total_tokens": 63},
},
@@ -104,7 +110,9 @@ def test_openai_cache_mode_restores_mutated_frozen_prefix() -> None:
proxy.config.mode = "cache"
fake_tracker = _FakePrefixTracker(frozen_count=0)
- proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "stable-session"
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stable-session"
+ )
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
original_messages = [
@@ -134,7 +142,11 @@ def test_openai_cache_mode_restores_mutated_frozen_prefix() -> None:
json={
"id": "chatcmpl_2",
"choices": [
- {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
],
"usage": {"prompt_tokens": 65, "completion_tokens": 3, "total_tokens": 68},
},