Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(batch): preserve native Gemini continuation tools (#4211)
Mansurshakh Japarov875745889ff0
diff --git a/docs/content/docs/ccr.mdx b/docs/content/docs/ccr.mdxindex a0db4efea..6ba202dbe 100644--- a/docs/content/docs/ccr.mdx+++ b/docs/content/docs/ccr.mdx@@ -82,6 +82,10 @@ The client never sees CCR tool calls on the Anthropic and OpenAI proxy paths; Healso return `finish_reason=MALFORMED_FUNCTION_CALL` on large function-responsecontinuations after CCR retrieval; that separate limitation remains tracked in[issue #2041](https://github.com/headroomlabs-ai/headroom/issues/2041).+ Native Google batch contexts retain the effective forwarded Tool objects,+ including injected retrieval declarations and sibling Google Search or code+ execution tools. Continuations use those declarations alongside the original+ conversation rather than rebuilding a function-declaration wrapper.</Callout>## Phase 4: Context Trackerdiff --git a/headroom/ccr/batch_processor.py b/headroom/ccr/batch_processor.pyindex 49abf4230..2b1fc26e3 100644--- a/headroom/ccr/batch_processor.py+++ b/headroom/ccr/batch_processor.py@@ -403,7 +403,7 @@ class BatchResultProcessor:"""Make Google/Gemini continuation call.Note: Google format uses 'contents' not 'messages',- and 'parts' format for messages.+ and 'parts' format for messages. Tools are already native Google Tool objects."""model = request_context.modelurl = f"{self.api_urls['google']}/v1beta/models/{model}:generateContent"@@ -424,7 +424,7 @@ class BatchResultProcessor:# Add toolsif tools:- body["tools"] = [{"functionDeclarations": tools}]+ body["tools"] = toolsresponse = await self.http_client.post(url,diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.pyindex 0e88313c2..f028bdfcb 100644--- a/headroom/proxy/handlers/batch.py+++ b/headroom/proxy/handlers/batch.py@@ -374,6 +374,7 @@ class BatchHandlerMixin:await self._store_google_batch_context(batch_name,requests_list,+ compressed_requests,model,api_key,)@@ -619,6 +620,7 @@ class BatchHandlerMixin:self,batch_name: str,requests_list: list[dict[str, Any]],+ forwarded_requests: list[dict[str, Any]],model: str,api_key: str | None,) -> None:@@ -627,6 +629,7 @@ class BatchHandlerMixin:Args:batch_name: The batch name from the API response.requests_list: The original batch requests.+ forwarded_requests: Effective requests with any injected native tools.model: The model used for the batch.api_key: The API key for continuation calls."""@@ -640,7 +643,7 @@ class BatchHandlerMixin:api_base_url=self.GEMINI_API_URL,)- for batch_req in requests_list:+ for batch_req, forwarded_req in zip(requests_list, forwarded_requests, strict=True):metadata = batch_req.get("metadata", {})custom_id = metadata.get("key", "")req_content = batch_req.get("request", {})@@ -661,7 +664,7 @@ class BatchHandlerMixin:BatchRequestContext(custom_id=custom_id,messages=messages,- tools=req_content.get("tools"),+ tools=forwarded_req.get("request", {}).get("tools"),model=model,system_instruction=sys_text,)diff --git a/tests/test_ccr_batch_processor.py b/tests/test_ccr_batch_processor.pyindex fc6273f25..bf5377aa1 100644--- a/tests/test_ccr_batch_processor.py+++ b/tests/test_ccr_batch_processor.py@@ -1,22 +1,25 @@"""Tests for CCR batch result processor.These tests verify that:-1. BatchResultProcessor class initialization works correctly+1. Native tool schemas permit real CCR batch continuations2. Result parsing for Anthropic, OpenAI, and Google batch formats3. CCR tool call detection in batch results4. Continuation call handling works for all providers5. Error cases and edge cases are handled gracefully"""+import json+import refrom unittest.mock import AsyncMock, MagicMock, patchimport httpximport pytest+from headroom.cache import compression_store as compression_store_module+from headroom.cache.compression_store import CompressionStore, set_request_compression_store+from headroom.ccr import batch_store as batch_store_modulefrom headroom.ccr.batch_processor import (BatchResultProcessor,- BatchResultProcessorConfig,- ProcessedBatchResult,process_batch_results,)from headroom.ccr.batch_store import (@@ -28,107 +31,6 @@ from headroom.ccr.batch_store import (from headroom.ccr.tool_injection import CCR_TOOL_NAME-class TestBatchResultProcessorConfig:- """Test BatchResultProcessorConfig dataclass."""-- def test_default_config(self):- """Default config values."""- config = BatchResultProcessorConfig()-- assert config.enabled is True- assert config.continuation_timeout == 120- assert config.max_continuation_rounds == 3-- def test_custom_config(self):- """Custom config values."""- config = BatchResultProcessorConfig(- enabled=False,- continuation_timeout=60,- max_continuation_rounds=5,- )-- assert config.enabled is False- assert config.continuation_timeout == 60- assert config.max_continuation_rounds == 5---class TestProcessedBatchResult:- """Test ProcessedBatchResult dataclass."""-- def test_default_values(self):- """Default values for ProcessedBatchResult."""- result = ProcessedBatchResult(- custom_id="req_123",- result={"content": "test"},- )-- assert result.custom_id == "req_123"- assert result.result == {"content": "test"}- assert result.was_processed is False- assert result.continuation_rounds == 0- assert result.error is None-- def test_processed_result(self):- """ProcessedBatchResult with CCR processing."""- result = ProcessedBatchResult(- custom_id="req_456",- result={"content": "processed"},- was_processed=True,- continuation_rounds=2,- )-- assert result.was_processed is True- assert result.continuation_rounds == 2-- def test_error_result(self):- """ProcessedBatchResult with error."""- result = ProcessedBatchResult(- custom_id="req_789",- result={"content": "partial"},- error="Retrieval failed",- )-- assert result.error == "Retrieval failed"---class TestBatchResultProcessorInit:- """Test BatchResultProcessor initialization."""-- def test_default_initialization(self):- """Initialize with default config."""- http_client = MagicMock(spec=httpx.AsyncClient)- processor = BatchResultProcessor(http_client)-- assert processor.http_client == http_client- assert processor.config.enabled is True- assert processor.config.continuation_timeout == 120- assert processor.ccr_handler is not None-- def test_custom_config_initialization(self):- """Initialize with custom config."""- http_client = MagicMock(spec=httpx.AsyncClient)- config = BatchResultProcessorConfig(- enabled=False,- continuation_timeout=60,- )- processor = BatchResultProcessor(http_client, config)-- assert processor.config.enabled is False- assert processor.config.continuation_timeout == 60-- def test_api_urls_set(self):- """API URLs are set for all providers."""- http_client = MagicMock(spec=httpx.AsyncClient)- processor = BatchResultProcessor(http_client)-- assert "anthropic" in processor.api_urls- assert "openai" in processor.api_urls- assert "google" in processor.api_urls- assert processor.api_urls["anthropic"] == "https://api.anthropic.com"- assert processor.api_urls["openai"] == "https://api.openai.com"- assert processor.api_urls["google"] == "https://generativelanguage.googleapis.com"--class TestCustomIdExtraction:"""Test _get_custom_id method for different providers."""@@ -684,47 +586,196 @@ class TestContinuationCalls:assert call_args.kwargs["json"]["model"] == "gpt-4"@pytest.mark.asyncio- async def test_google_continuation_call(self):- """Test Google continuation call format."""- mock_response = MagicMock()- mock_response.json.return_value = {- "candidates": [{"content": {"parts": [{"text": "Final answer"}]}}]- }- mock_response.raise_for_status = MagicMock()-- http_client = AsyncMock(spec=httpx.AsyncClient)- http_client.post.return_value = mock_response-- processor = BatchResultProcessor(http_client)+ @pytest.mark.parametrize("client_tools", ["functions", "siblings", "none"])+ async def test_google_batch_create_to_results_retains_injected_native_tools(+ self, monkeypatch, tmp_path, client_tools+ ):+ from starlette.requests import Request++ from headroom.providers.openai import OpenAIProvider+ from headroom.proxy.server import HeadroomProxy, ProxyConfig+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig+ from headroom.transforms.pipeline import TransformPipeline++ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))+ original_records = [+ f"Record {i}: amount={i * 13}, owner=account-{i}, unique source information "+ "with a detailed transaction description."+ for i in range(200)+ ]+ original = json.dumps(original_records, indent=2)+ store = CompressionStore(enable_feedback=False)+ previous_store = compression_store_module._request_ccr_store.get()+ set_request_compression_store(store)+ monkeypatch.setattr(batch_store_module, "_batch_context_store", BatchContextStore())+ tools = [] if client_tools == "none" else [{"googleSearch": {}}, {"codeExecution": {}}]+ if client_tools == "functions":+ tools.insert(+ 0,+ {+ "functionDeclarations": [+ {+ "name": "lookup",+ "description": "Look up one record.",+ "parameters": {"type": "OBJECT"},+ }+ ]+ },+ )+ forwarded = None++ def admit_wire_request(request):+ nonlocal forwarded+ body = json.loads(request.content)+ if request.url.path.endswith(":batchGenerateContent"):+ forwarded = body["batch"]["input_config"]["requests"]["requests"][0]["request"]+ return httpx.Response(200, json={"name": "batches/native-tools"})+ recovered = [+ part["functionResponse"]["response"].get("original_content")+ for content in body["contents"]+ for part in content["parts"]+ if "functionResponse" in part+ ]+ declarations = [+ declaration+ for tool in body.get("tools", [])+ for declaration in tool.get("functionDeclarations", [])+ ]+ expected_names = (+ {CCR_TOOL_NAME, "lookup"} if client_tools == "functions" else {CCR_TOOL_NAME}+ )+ admitted = (+ {declaration.get("name") for declaration in declarations} == expected_names+ and body.get("tools") == forwarded.get("tools")+ and all(+ all(declaration in declarations for declaration in tool["functionDeclarations"])+ if "functionDeclarations" in tool+ else tool in body["tools"]+ for tool in tools+ )+ and any(+ isinstance(value, str) and json.loads(value) == original_records+ for value in recovered+ )+ )+ return httpx.Response(+ 200 if admitted else 400,+ json={+ "candidates": [+ {"content": {"role": "model", "parts": [{"text": "All records recovered"}]}}+ ]+ }+ if admitted+ else {"error": {"message": "Missing effective native tools or original records"}},+ )- request_context = BatchRequestContext(- custom_id="req_1",- messages=[{"role": "user", "content": "Hello"}],- model="gemini-pro",- system_instruction="Be helpful.",+ proxy = HeadroomProxy(+ ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ retry_enabled=False,+ image_optimize=False,+ ))- batch_context = BatchContext(- batch_id="batch_123",- provider="google",- api_key="google_api_key",+ proxy.openai_pipeline = TransformPipeline(+ provider=OpenAIProvider(),+ transforms=[+ ContentRouter(+ ContentRouterConfig(+ skip_user_messages=False,+ protect_analysis_context=False,+ enable_kompress=False,+ smart_crusher_max_items_after_crush=12,+ smart_crusher_with_compaction=False,+ )+ )+ ],)-- await processor._google_continuation(- [{"role": "user", "content": "Hello"}],- [{"name": "test_tool", "parameters": {}}],- request_context,- batch_context,+ payload = json.dumps(+ {+ "batch": {+ "input_config": {+ "requests": {+ "requests": [+ {+ "metadata": {"key": "request"},+ "request": {+ "contents": [+ {"role": "user", "parts": [{"text": original}]}+ ],+ "tools": tools,+ },+ }+ ]+ }+ }+ }+ }+ ).encode()+ chunks = iter(+ [+ {"type": "http.request", "body": payload[:100], "more_body": True},+ {"type": "http.request", "body": payload[100:], "more_body": False},+ ])- # Verify API call- http_client.post.assert_called_once()- call_args = http_client.post.call_args+ async def receive():+ return next(chunks, {"type": "http.disconnect"})- assert "generativelanguage.googleapis.com" in call_args.args[0]- assert "gemini-pro" in call_args.args[0]- assert "key=google_api_key" in call_args.args[0]- assert "contents" in call_args.kwargs["json"]- assert "systemInstruction" in call_args.kwargs["json"]+ request = Request(+ {+ "type": "http",+ "method": "POST",+ "path": "/v1beta/models/gemini-2.5-flash:batchGenerateContent",+ "query_string": b"",+ "headers": [(b"content-type", b"application/json")],+ "scheme": "http",+ "server": ("127.0.0.1", 80),+ },+ receive,+ )+ try:+ async with httpx.AsyncClient(+ transport=httpx.MockTransport(admit_wire_request)+ ) as client:+ proxy.http_client = client+ created = await proxy.handle_google_batch_create(request, "gemini-2.5-flash")+ assert created.status_code == 200+ hash_key = re.search(+ r"(?:<<ccr:|Retrieve (?:more|original): hash=)([a-f0-9]{12,24})",+ json.dumps(forwarded),+ ).group(1)+ response = {+ "candidates": [+ {+ "content": {+ "role": "model",+ "parts": [+ {+ "functionCall": {+ "name": CCR_TOOL_NAME,+ "args": {"hash": hash_key},+ }+ }+ ],+ }+ }+ ]+ }+ processed = await BatchResultProcessor(client).process_results(+ "batches/native-tools",+ [{"metadata": {"key": "request"}, "response": response}],+ "google",+ )+ assert processed[0].result["response"]["candidates"][0]["content"]["parts"] == [+ {"text": "All records recovered"}+ ]+ finally:+ set_request_compression_store(previous_store)+ proxy._compression_executor.shutdown(wait=True)@pytest.mark.asyncioasync def test_continuation_with_tools(self):diff --git a/tests/test_proxy_handlers_batch.py b/tests/test_proxy_handlers_batch.pyindex 1658a78bb..fb33b291d 100644--- a/tests/test_proxy_handlers_batch.py+++ b/tests/test_proxy_handlers_batch.py@@ -1004,62 +1004,6 @@ async def test_download_and_upload_openai_file_helpers() -> None:assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None-async def test_store_google_batch_context_persists_transformed_requests(- monkeypatch: pytest.MonkeyPatch,-) -> None:- stored_contexts: list[object] = []-- class FakeBatchContext:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs- self.requests: list[object] = []-- def add_request(self, request) -> None: # noqa: ANN001- self.requests.append(request)-- class FakeBatchRequestContext:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs-- class FakeStore:- async def store(self, context) -> None: # noqa: ANN001- stored_contexts.append(context)-- monkeypatch.setitem(- sys.modules,- "headroom.ccr",- SimpleNamespace(- BatchContext=FakeBatchContext,- BatchRequestContext=FakeBatchRequestContext,- get_batch_context_store=lambda: FakeStore(),- ),- )-- handler = DummyBatchHandler()- await handler._store_google_batch_context(- "batches/123",- [- {- "metadata": {"key": "req-1"},- "request": {- "contents": [{"parts": [{"text": "hello"}]}],- "systemInstruction": {"parts": [{"text": "system"}]},- "tools": [{"name": "tool"}],- },- }- ],- "gemini-2.0",- "api-key",- )-- context = stored_contexts[0]- assert context.kwargs["batch_id"] == "batches/123"- assert context.requests[0].kwargs["custom_id"] == "req-1"- assert context.requests[0].kwargs["messages"] == [{"role": "user", "content": "hello"}]- assert context.requests[0].kwargs["system_instruction"] == "system"--@pytest.mark.asyncioasync def test_handle_google_batch_results_passes_through_early_exit_cases(monkeypatch: pytest.MonkeyPatch,@@ -1299,11 +1243,6 @@ async def test_handle_google_batch_create_success_and_failure_paths(monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))- stored: list[tuple[str, list[dict[str, object]], str, str | None]] = []-- async def fake_store(batch_name, requests_list, model, api_key): # noqa: ANN001- stored.append((batch_name, requests_list, model, api_key))-async def fake_retry(method, url, headers, body, **kwargs): # noqa: ANN001return FakeResponse(status_code=200,@@ -1333,7 +1272,6 @@ async def test_handle_google_batch_create_success_and_failure_paths(monkeypatch.setattr("headroom.proxy.helpers._read_request_json", good_payload)monkeypatch.setattr(handler, "_retry_request", fake_retry)- monkeypatch.setattr(handler, "_store_google_batch_context", fake_store)response = await handler.handle_google_batch_create(FakeRequest("{}", headers={"x-goog-api-key": "secret"}),@@ -1343,9 +1281,6 @@ async def test_handle_google_batch_create_success_and_failure_paths(assert dict(response.headers)["x-upstream"] == "1"assert handler.metrics.record_calls[-1]["provider"] == "google"assert handler.metrics.record_calls[-1]["tokens_saved"] == 60- assert stored[0][0] == "batches/123"- assert stored[0][2:] == ("gemini-pro", "secret")- assert stored[0][1][0]["metadata"] == {"key": "req-1"}async def broken_retry(method, url, headers, body, **kwargs): # noqa: ANN001raise RuntimeError("forward failed")@@ -1673,59 +1608,6 @@ async def test_batch_helper_methods_and_openai_file_error_branches() -> None:assert await handler._upload_openai_file("{}", "missing-id.jsonl", {}) is None-async def test_store_google_batch_context_without_system_text(- monkeypatch: pytest.MonkeyPatch,-) -> None:- stored_contexts: list[object] = []-- class FakeBatchContext:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs- self.requests: list[object] = []-- def add_request(self, request) -> None: # noqa: ANN001- self.requests.append(request)-- class FakeBatchRequestContext:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs-- class FakeStore:- async def store(self, context) -> None: # noqa: ANN001- stored_contexts.append(context)-- handler = DummyBatchHandler()- monkeypatch.setitem(- sys.modules,- "headroom.ccr",- SimpleNamespace(- BatchContext=FakeBatchContext,- BatchRequestContext=FakeBatchRequestContext,- get_batch_context_store=lambda: FakeStore(),- ),- )-- await handler._store_google_batch_context(- "batches/456",- [- {- "request": {- "contents": [{"parts": [{"text": "hello"}]}],- "systemInstruction": {"parts": ["bad"]},- }- }- ],- "gemini-2.0",- None,- )-- context = stored_contexts[0]- assert context.kwargs["api_key"] is None- assert context.requests[0].kwargs["custom_id"] == ""- assert context.requests[0].kwargs["system_instruction"] is None--@pytest.mark.asyncioasync def test_compress_batch_jsonl_skips_blank_lines_and_preserves_tools_when_not_injected(monkeypatch: pytest.MonkeyPatch,