Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(batch): preserve native Gemini continuation tools (#4211)

Mansurshakh Japarov875745889ff0
diff --git a/docs/content/docs/ccr.mdx b/docs/content/docs/ccr.mdx
index a0db4efea..6ba202dbe 100644
--- a/docs/content/docs/ccr.mdx
+++ b/docs/content/docs/ccr.mdx
@@ -82,6 +82,10 @@ The client never sees CCR tool calls on the Anthropic and OpenAI proxy paths; He
also return `finish_reason=MALFORMED_FUNCTION_CALL` on large function-response
continuations after CCR retrieval; that separate limitation remains tracked in
[issue #2041](https://github.com/headroomlabs-ai/headroom/issues/2041).
+ Native Google batch contexts retain the effective forwarded Tool objects,
+ including injected retrieval declarations and sibling Google Search or code
+ execution tools. Continuations use those declarations alongside the original
+ conversation rather than rebuilding a function-declaration wrapper.
</Callout>
## Phase 4: Context Tracker
diff --git a/headroom/ccr/batch_processor.py b/headroom/ccr/batch_processor.py
index 49abf4230..2b1fc26e3 100644
--- a/headroom/ccr/batch_processor.py
+++ b/headroom/ccr/batch_processor.py
@@ -403,7 +403,7 @@ class BatchResultProcessor:
"""Make Google/Gemini continuation call.
Note: Google format uses 'contents' not 'messages',
- and 'parts' format for messages.
+ and 'parts' format for messages. Tools are already native Google Tool objects.
"""
model = request_context.model
url = f"{self.api_urls['google']}/v1beta/models/{model}:generateContent"
@@ -424,7 +424,7 @@ class BatchResultProcessor:
# Add tools
if tools:
- body["tools"] = [{"functionDeclarations": tools}]
+ body["tools"] = tools
response = await self.http_client.post(
url,
diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.py
index 0e88313c2..f028bdfcb 100644
--- a/headroom/proxy/handlers/batch.py
+++ b/headroom/proxy/handlers/batch.py
@@ -374,6 +374,7 @@ class BatchHandlerMixin:
await self._store_google_batch_context(
batch_name,
requests_list,
+ compressed_requests,
model,
api_key,
)
@@ -619,6 +620,7 @@ class BatchHandlerMixin:
self,
batch_name: str,
requests_list: list[dict[str, Any]],
+ forwarded_requests: list[dict[str, Any]],
model: str,
api_key: str | None,
) -> None:
@@ -627,6 +629,7 @@ class BatchHandlerMixin:
Args:
batch_name: The batch name from the API response.
requests_list: The original batch requests.
+ forwarded_requests: Effective requests with any injected native tools.
model: The model used for the batch.
api_key: The API key for continuation calls.
"""
@@ -640,7 +643,7 @@ class BatchHandlerMixin:
api_base_url=self.GEMINI_API_URL,
)
- for batch_req in requests_list:
+ for batch_req, forwarded_req in zip(requests_list, forwarded_requests, strict=True):
metadata = batch_req.get("metadata", {})
custom_id = metadata.get("key", "")
req_content = batch_req.get("request", {})
@@ -661,7 +664,7 @@ class BatchHandlerMixin:
BatchRequestContext(
custom_id=custom_id,
messages=messages,
- tools=req_content.get("tools"),
+ tools=forwarded_req.get("request", {}).get("tools"),
model=model,
system_instruction=sys_text,
)
diff --git a/tests/test_ccr_batch_processor.py b/tests/test_ccr_batch_processor.py
index fc6273f25..bf5377aa1 100644
--- a/tests/test_ccr_batch_processor.py
+++ b/tests/test_ccr_batch_processor.py
@@ -1,22 +1,25 @@
"""Tests for CCR batch result processor.
These tests verify that:
-1. BatchResultProcessor class initialization works correctly
+1. Native tool schemas permit real CCR batch continuations
2. Result parsing for Anthropic, OpenAI, and Google batch formats
3. CCR tool call detection in batch results
4. Continuation call handling works for all providers
5. Error cases and edge cases are handled gracefully
"""
+import json
+import re
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
+from headroom.cache import compression_store as compression_store_module
+from headroom.cache.compression_store import CompressionStore, set_request_compression_store
+from headroom.ccr import batch_store as batch_store_module
from headroom.ccr.batch_processor import (
BatchResultProcessor,
- BatchResultProcessorConfig,
- ProcessedBatchResult,
process_batch_results,
)
from headroom.ccr.batch_store import (
@@ -28,107 +31,6 @@ from headroom.ccr.batch_store import (
from headroom.ccr.tool_injection import CCR_TOOL_NAME
-class TestBatchResultProcessorConfig:
- """Test BatchResultProcessorConfig dataclass."""
-
- def test_default_config(self):
- """Default config values."""
- config = BatchResultProcessorConfig()
-
- assert config.enabled is True
- assert config.continuation_timeout == 120
- assert config.max_continuation_rounds == 3
-
- def test_custom_config(self):
- """Custom config values."""
- config = BatchResultProcessorConfig(
- enabled=False,
- continuation_timeout=60,
- max_continuation_rounds=5,
- )
-
- assert config.enabled is False
- assert config.continuation_timeout == 60
- assert config.max_continuation_rounds == 5
-
-
-class TestProcessedBatchResult:
- """Test ProcessedBatchResult dataclass."""
-
- def test_default_values(self):
- """Default values for ProcessedBatchResult."""
- result = ProcessedBatchResult(
- custom_id="req_123",
- result={"content": "test"},
- )
-
- assert result.custom_id == "req_123"
- assert result.result == {"content": "test"}
- assert result.was_processed is False
- assert result.continuation_rounds == 0
- assert result.error is None
-
- def test_processed_result(self):
- """ProcessedBatchResult with CCR processing."""
- result = ProcessedBatchResult(
- custom_id="req_456",
- result={"content": "processed"},
- was_processed=True,
- continuation_rounds=2,
- )
-
- assert result.was_processed is True
- assert result.continuation_rounds == 2
-
- def test_error_result(self):
- """ProcessedBatchResult with error."""
- result = ProcessedBatchResult(
- custom_id="req_789",
- result={"content": "partial"},
- error="Retrieval failed",
- )
-
- assert result.error == "Retrieval failed"
-
-
-class TestBatchResultProcessorInit:
- """Test BatchResultProcessor initialization."""
-
- def test_default_initialization(self):
- """Initialize with default config."""
- http_client = MagicMock(spec=httpx.AsyncClient)
- processor = BatchResultProcessor(http_client)
-
- assert processor.http_client == http_client
- assert processor.config.enabled is True
- assert processor.config.continuation_timeout == 120
- assert processor.ccr_handler is not None
-
- def test_custom_config_initialization(self):
- """Initialize with custom config."""
- http_client = MagicMock(spec=httpx.AsyncClient)
- config = BatchResultProcessorConfig(
- enabled=False,
- continuation_timeout=60,
- )
- processor = BatchResultProcessor(http_client, config)
-
- assert processor.config.enabled is False
- assert processor.config.continuation_timeout == 60
-
- def test_api_urls_set(self):
- """API URLs are set for all providers."""
- http_client = MagicMock(spec=httpx.AsyncClient)
- processor = BatchResultProcessor(http_client)
-
- assert "anthropic" in processor.api_urls
- assert "openai" in processor.api_urls
- assert "google" in processor.api_urls
- assert processor.api_urls["anthropic"] == "https://api.anthropic.com"
- assert processor.api_urls["openai"] == "https://api.openai.com"
- assert processor.api_urls["google"] == "https://generativelanguage.googleapis.com"
-
-
class TestCustomIdExtraction:
"""Test _get_custom_id method for different providers."""
@@ -684,47 +586,196 @@ class TestContinuationCalls:
assert call_args.kwargs["json"]["model"] == "gpt-4"
@pytest.mark.asyncio
- async def test_google_continuation_call(self):
- """Test Google continuation call format."""
- mock_response = MagicMock()
- mock_response.json.return_value = {
- "candidates": [{"content": {"parts": [{"text": "Final answer"}]}}]
- }
- mock_response.raise_for_status = MagicMock()
-
- http_client = AsyncMock(spec=httpx.AsyncClient)
- http_client.post.return_value = mock_response
-
- processor = BatchResultProcessor(http_client)
+ @pytest.mark.parametrize("client_tools", ["functions", "siblings", "none"])
+ async def test_google_batch_create_to_results_retains_injected_native_tools(
+ self, monkeypatch, tmp_path, client_tools
+ ):
+ from starlette.requests import Request
+
+ from headroom.providers.openai import OpenAIProvider
+ from headroom.proxy.server import HeadroomProxy, ProxyConfig
+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
+ from headroom.transforms.pipeline import TransformPipeline
+
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
+ original_records = [
+ f"Record {i}: amount={i * 13}, owner=account-{i}, unique source information "
+ "with a detailed transaction description."
+ for i in range(200)
+ ]
+ original = json.dumps(original_records, indent=2)
+ store = CompressionStore(enable_feedback=False)
+ previous_store = compression_store_module._request_ccr_store.get()
+ set_request_compression_store(store)
+ monkeypatch.setattr(batch_store_module, "_batch_context_store", BatchContextStore())
+ tools = [] if client_tools == "none" else [{"googleSearch": {}}, {"codeExecution": {}}]
+ if client_tools == "functions":
+ tools.insert(
+ 0,
+ {
+ "functionDeclarations": [
+ {
+ "name": "lookup",
+ "description": "Look up one record.",
+ "parameters": {"type": "OBJECT"},
+ }
+ ]
+ },
+ )
+ forwarded = None
+
+ def admit_wire_request(request):
+ nonlocal forwarded
+ body = json.loads(request.content)
+ if request.url.path.endswith(":batchGenerateContent"):
+ forwarded = body["batch"]["input_config"]["requests"]["requests"][0]["request"]
+ return httpx.Response(200, json={"name": "batches/native-tools"})
+ recovered = [
+ part["functionResponse"]["response"].get("original_content")
+ for content in body["contents"]
+ for part in content["parts"]
+ if "functionResponse" in part
+ ]
+ declarations = [
+ declaration
+ for tool in body.get("tools", [])
+ for declaration in tool.get("functionDeclarations", [])
+ ]
+ expected_names = (
+ {CCR_TOOL_NAME, "lookup"} if client_tools == "functions" else {CCR_TOOL_NAME}
+ )
+ admitted = (
+ {declaration.get("name") for declaration in declarations} == expected_names
+ and body.get("tools") == forwarded.get("tools")
+ and all(
+ all(declaration in declarations for declaration in tool["functionDeclarations"])
+ if "functionDeclarations" in tool
+ else tool in body["tools"]
+ for tool in tools
+ )
+ and any(
+ isinstance(value, str) and json.loads(value) == original_records
+ for value in recovered
+ )
+ )
+ return httpx.Response(
+ 200 if admitted else 400,
+ json={
+ "candidates": [
+ {"content": {"role": "model", "parts": [{"text": "All records recovered"}]}}
+ ]
+ }
+ if admitted
+ else {"error": {"message": "Missing effective native tools or original records"}},
+ )
- request_context = BatchRequestContext(
- custom_id="req_1",
- messages=[{"role": "user", "content": "Hello"}],
- model="gemini-pro",
- system_instruction="Be helpful.",
+ proxy = HeadroomProxy(
+ ProxyConfig(
+ optimize=True,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ retry_enabled=False,
+ image_optimize=False,
+ )
)
- batch_context = BatchContext(
- batch_id="batch_123",
- provider="google",
- api_key="google_api_key",
+ proxy.openai_pipeline = TransformPipeline(
+ provider=OpenAIProvider(),
+ transforms=[
+ ContentRouter(
+ ContentRouterConfig(
+ skip_user_messages=False,
+ protect_analysis_context=False,
+ enable_kompress=False,
+ smart_crusher_max_items_after_crush=12,
+ smart_crusher_with_compaction=False,
+ )
+ )
+ ],
)
-
- await processor._google_continuation(
- [{"role": "user", "content": "Hello"}],
- [{"name": "test_tool", "parameters": {}}],
- request_context,
- batch_context,
+ payload = json.dumps(
+ {
+ "batch": {
+ "input_config": {
+ "requests": {
+ "requests": [
+ {
+ "metadata": {"key": "request"},
+ "request": {
+ "contents": [
+ {"role": "user", "parts": [{"text": original}]}
+ ],
+ "tools": tools,
+ },
+ }
+ ]
+ }
+ }
+ }
+ }
+ ).encode()
+ chunks = iter(
+ [
+ {"type": "http.request", "body": payload[:100], "more_body": True},
+ {"type": "http.request", "body": payload[100:], "more_body": False},
+ ]
)
- # Verify API call
- http_client.post.assert_called_once()
- call_args = http_client.post.call_args
+ async def receive():
+ return next(chunks, {"type": "http.disconnect"})
- assert "generativelanguage.googleapis.com" in call_args.args[0]
- assert "gemini-pro" in call_args.args[0]
- assert "key=google_api_key" in call_args.args[0]
- assert "contents" in call_args.kwargs["json"]
- assert "systemInstruction" in call_args.kwargs["json"]
+ request = Request(
+ {
+ "type": "http",
+ "method": "POST",
+ "path": "/v1beta/models/gemini-2.5-flash:batchGenerateContent",
+ "query_string": b"",
+ "headers": [(b"content-type", b"application/json")],
+ "scheme": "http",
+ "server": ("127.0.0.1", 80),
+ },
+ receive,
+ )
+ try:
+ async with httpx.AsyncClient(
+ transport=httpx.MockTransport(admit_wire_request)
+ ) as client:
+ proxy.http_client = client
+ created = await proxy.handle_google_batch_create(request, "gemini-2.5-flash")
+ assert created.status_code == 200
+ hash_key = re.search(
+ r"(?:<<ccr:|Retrieve (?:more|original): hash=)([a-f0-9]{12,24})",
+ json.dumps(forwarded),
+ ).group(1)
+ response = {
+ "candidates": [
+ {
+ "content": {
+ "role": "model",
+ "parts": [
+ {
+ "functionCall": {
+ "name": CCR_TOOL_NAME,
+ "args": {"hash": hash_key},
+ }
+ }
+ ],
+ }
+ }
+ ]
+ }
+ processed = await BatchResultProcessor(client).process_results(
+ "batches/native-tools",
+ [{"metadata": {"key": "request"}, "response": response}],
+ "google",
+ )
+ assert processed[0].result["response"]["candidates"][0]["content"]["parts"] == [
+ {"text": "All records recovered"}
+ ]
+ finally:
+ set_request_compression_store(previous_store)
+ proxy._compression_executor.shutdown(wait=True)
@pytest.mark.asyncio
async def test_continuation_with_tools(self):
diff --git a/tests/test_proxy_handlers_batch.py b/tests/test_proxy_handlers_batch.py
index 1658a78bb..fb33b291d 100644
--- a/tests/test_proxy_handlers_batch.py
+++ b/tests/test_proxy_handlers_batch.py
@@ -1004,62 +1004,6 @@ async def test_download_and_upload_openai_file_helpers() -> None:
assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None
-async def test_store_google_batch_context_persists_transformed_requests(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- stored_contexts: list[object] = []
-
- class FakeBatchContext:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
- self.requests: list[object] = []
-
- def add_request(self, request) -> None: # noqa: ANN001
- self.requests.append(request)
-
- class FakeBatchRequestContext:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
-
- class FakeStore:
- async def store(self, context) -> None: # noqa: ANN001
- stored_contexts.append(context)
-
- monkeypatch.setitem(
- sys.modules,
- "headroom.ccr",
- SimpleNamespace(
- BatchContext=FakeBatchContext,
- BatchRequestContext=FakeBatchRequestContext,
- get_batch_context_store=lambda: FakeStore(),
- ),
- )
-
- handler = DummyBatchHandler()
- await handler._store_google_batch_context(
- "batches/123",
- [
- {
- "metadata": {"key": "req-1"},
- "request": {
- "contents": [{"parts": [{"text": "hello"}]}],
- "systemInstruction": {"parts": [{"text": "system"}]},
- "tools": [{"name": "tool"}],
- },
- }
- ],
- "gemini-2.0",
- "api-key",
- )
-
- context = stored_contexts[0]
- assert context.kwargs["batch_id"] == "batches/123"
- assert context.requests[0].kwargs["custom_id"] == "req-1"
- assert context.requests[0].kwargs["messages"] == [{"role": "user", "content": "hello"}]
- assert context.requests[0].kwargs["system_instruction"] == "system"
-
-
@pytest.mark.asyncio
async def test_handle_google_batch_results_passes_through_early_exit_cases(
monkeypatch: pytest.MonkeyPatch,
@@ -1299,11 +1243,6 @@ async def test_handle_google_batch_create_success_and_failure_paths(
monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))
- stored: list[tuple[str, list[dict[str, object]], str, str | None]] = []
-
- async def fake_store(batch_name, requests_list, model, api_key): # noqa: ANN001
- stored.append((batch_name, requests_list, model, api_key))
-
async def fake_retry(method, url, headers, body, **kwargs): # noqa: ANN001
return FakeResponse(
status_code=200,
@@ -1333,7 +1272,6 @@ async def test_handle_google_batch_create_success_and_failure_paths(
monkeypatch.setattr("headroom.proxy.helpers._read_request_json", good_payload)
monkeypatch.setattr(handler, "_retry_request", fake_retry)
- monkeypatch.setattr(handler, "_store_google_batch_context", fake_store)
response = await handler.handle_google_batch_create(
FakeRequest("{}", headers={"x-goog-api-key": "secret"}),
@@ -1343,9 +1281,6 @@ async def test_handle_google_batch_create_success_and_failure_paths(
assert dict(response.headers)["x-upstream"] == "1"
assert handler.metrics.record_calls[-1]["provider"] == "google"
assert handler.metrics.record_calls[-1]["tokens_saved"] == 60
- assert stored[0][0] == "batches/123"
- assert stored[0][2:] == ("gemini-pro", "secret")
- assert stored[0][1][0]["metadata"] == {"key": "req-1"}
async def broken_retry(method, url, headers, body, **kwargs): # noqa: ANN001
raise RuntimeError("forward failed")
@@ -1673,59 +1608,6 @@ async def test_batch_helper_methods_and_openai_file_error_branches() -> None:
assert await handler._upload_openai_file("{}", "missing-id.jsonl", {}) is None
-async def test_store_google_batch_context_without_system_text(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- stored_contexts: list[object] = []
-
- class FakeBatchContext:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
- self.requests: list[object] = []
-
- def add_request(self, request) -> None: # noqa: ANN001
- self.requests.append(request)
-
- class FakeBatchRequestContext:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
-
- class FakeStore:
- async def store(self, context) -> None: # noqa: ANN001
- stored_contexts.append(context)
-
- handler = DummyBatchHandler()
- monkeypatch.setitem(
- sys.modules,
- "headroom.ccr",
- SimpleNamespace(
- BatchContext=FakeBatchContext,
- BatchRequestContext=FakeBatchRequestContext,
- get_batch_context_store=lambda: FakeStore(),
- ),
- )
-
- await handler._store_google_batch_context(
- "batches/456",
- [
- {
- "request": {
- "contents": [{"parts": [{"text": "hello"}]}],
- "systemInstruction": {"parts": ["bad"]},
- }
- }
- ],
- "gemini-2.0",
- None,
- )
-
- context = stored_contexts[0]
- assert context.kwargs["api_key"] is None
- assert context.requests[0].kwargs["custom_id"] == ""
- assert context.requests[0].kwargs["system_instruction"] is None
-
-
@pytest.mark.asyncio
async def test_compress_batch_jsonl_skips_blank_lines_and_preserves_tools_when_not_injected(
monkeypatch: pytest.MonkeyPatch,