Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

test: restore CCR module cleanup in batch tests

JerrettDavis65481d243a8b
diff --git a/tests/test_proxy_handlers_batch.py b/tests/test_proxy_handlers_batch.py
index 0fa2453cc..dd87376ae 100644
--- a/tests/test_proxy_handlers_batch.py
+++ b/tests/test_proxy_handlers_batch.py
@@ -1,1047 +1,1053 @@
-from __future__ import annotations
-
-import json
-import sys
-from types import SimpleNamespace
-
-import pytest
-
-from headroom.proxy.handlers import batch as batch_module
-
-
-class FakeResponse:
- def __init__(
- self,
- *,
- status_code: int = 200,
- content: bytes = b"{}",
- headers: dict[str, str] | None = None,
- text: str | None = None,
- json_data=None, # noqa: ANN001
- ) -> None:
- self.status_code = status_code
- self.content = content
- self.headers = headers or {}
- self.text = text if text is not None else content.decode("utf-8", errors="ignore")
- self._json_data = json_data
-
- def json(self): # noqa: ANN201
- if self._json_data is not None:
- return self._json_data
- return json.loads(self.text)
-
-
-class FakeHttpClient:
- def __init__(self) -> None:
- self.posts: list[dict[str, object]] = []
- self.gets: list[dict[str, object]] = []
- self.requests: list[dict[str, object]] = []
- self.post_response = FakeResponse()
- self.get_response = FakeResponse()
- self.raise_post: Exception | None = None
- self.raise_get: Exception | None = None
-
- async def post(self, url: str, **kwargs): # noqa: ANN003, ANN201
- self.posts.append({"url": url, **kwargs})
- if self.raise_post is not None:
- raise self.raise_post
- return self.post_response
-
- async def get(self, url: str, **kwargs): # noqa: ANN003, ANN201
- self.gets.append({"url": url, **kwargs})
- if self.raise_get is not None:
- raise self.raise_get
- return self.get_response
-
- async def request(self, method: str, url: str, **kwargs): # noqa: ANN003, ANN201
- self.requests.append({"method": method, "url": url, **kwargs})
- if self.raise_get is not None:
- raise self.raise_get
- return self.get_response
-
-
-class FakeMetrics:
- def __init__(self) -> None:
- self.record_calls: list[dict[str, object]] = []
- self.failed_calls: list[dict[str, object]] = []
-
- async def record_request(self, **kwargs) -> None: # noqa: ANN003
- self.record_calls.append(kwargs)
-
- async def record_failed(self, **kwargs) -> None: # noqa: ANN003
- self.failed_calls.append(kwargs)
-
-
-class DummyBatchHandler(batch_module.BatchHandlerMixin):
- OPENAI_API_URL = "https://openai.example"
- GEMINI_API_URL = "https://gemini.example"
-
- def __init__(self) -> None:
- self.http_client = FakeHttpClient()
- self.metrics = FakeMetrics()
- self.config = SimpleNamespace(
- optimize=False,
- ccr_inject_tool=False,
- ccr_inject_system_instructions=False,
- )
- self.openai_provider = SimpleNamespace(get_context_limit=lambda model: 8192)
- self.openai_pipeline = SimpleNamespace(apply=lambda **kwargs: None)
- self._request_counter = 0
- self._retry_response = FakeResponse()
-
- async def _next_request_id(self) -> str:
- self._request_counter += 1
- return f"req-{self._request_counter}"
-
- async def handle_passthrough(self, request, base_url): # noqa: ANN001, ANN201
- return {"request": request, "base_url": base_url}
-
- async def _retry_request(self, method, url, headers, body): # noqa: ANN001, ANN201
- return self._retry_response
-
- def _gemini_contents_to_messages(self, contents, system_instruction): # noqa: ANN001, ANN201
- messages = [{"role": "user", "content": part["parts"][0]["text"]} for part in contents]
- return messages, []
-
- def _messages_to_gemini_contents(self, messages): # noqa: ANN001, ANN201
- return ([{"parts": [{"text": message["content"]}]} for message in messages], None)
-
-
-class FakeRequest:
- def __init__(
- self,
- body: bytes | str,
- *,
- headers: dict[str, str] | None = None,
- method: str = "POST",
- path: str = "/v1/batches",
- query: str = "",
- ) -> None:
- self._body = body.encode("utf-8") if isinstance(body, str) else body
- self.headers = headers or {}
- self.method = method
- self.url = SimpleNamespace(path=path, query=query)
-
- async def body(self) -> bytes:
- return self._body
-
-
-def install_batch_support_modules(
- monkeypatch: pytest.MonkeyPatch,
- *,
- injector_result=None, # noqa: ANN001
- tokenizer_count: int = 10,
-) -> None:
- class FakeInjector:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
-
- def process_request(self, messages, tools): # noqa: ANN001, ANN201
- if injector_result is not None:
- return injector_result
- return messages, tools, False
-
- class FakeTokenizer:
- def count_messages(self, messages) -> int: # noqa: ANN001
- return tokenizer_count
-
- monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))
- monkeypatch.setitem(
- sys.modules,
- "headroom.tokenizers",
- SimpleNamespace(get_tokenizer=lambda model: FakeTokenizer()),
- )
- monkeypatch.setitem(
- sys.modules,
- "headroom.utils",
- SimpleNamespace(extract_user_query=lambda messages: "query"),
- )
-
-
-async def test_compress_batch_jsonl_without_optimization_handles_invalid_lines(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- install_batch_support_modules(monkeypatch, tokenizer_count=12)
- handler = DummyBatchHandler()
- content = "\n".join(
- [
- json.dumps(
- {"body": {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}}
- ),
- json.dumps({"body": {"model": "gpt-4o", "messages": []}}),
- "not-json",
- ]
- )
-
- lines, stats = await handler._compress_batch_jsonl(content, "req-1")
-
- assert len(lines) == 3
- assert json.loads(lines[0])["body"]["messages"][0]["content"] == "hi"
- assert lines[2] == "not-json"
- assert stats == {
- "total_requests": 3,
- "total_original_tokens": 12,
- "total_compressed_tokens": 12,
- "total_tokens_saved": 0,
- "savings_percent": 0.0,
- "errors": 1,
- }
-
-
-async def test_compress_batch_jsonl_uses_pipeline_and_ccr_injection(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- install_batch_support_modules(
- monkeypatch,
- injector_result=(
- [{"role": "system", "content": "compressed"}],
- [{"name": "retrieval"}],
- True,
- ),
- )
- handler = DummyBatchHandler()
- handler.config.optimize = True
- handler.config.ccr_inject_tool = True
- handler.openai_pipeline = SimpleNamespace(
- apply=lambda **kwargs: SimpleNamespace(
- messages=[{"role": "assistant", "content": "short"}],
- tokens_before=100,
- tokens_after=40,
- )
- )
-
- lines, stats = await handler._compress_batch_jsonl(
- json.dumps(
- {
- "body": {
- "model": "gpt-4o-mini",
- "messages": [{"role": "user", "content": "hello"}],
- "tools": [{"name": "existing"}],
- }
- }
- ),
- "req-2",
- )
-
- body = json.loads(lines[0])["body"]
- assert body["messages"] == [{"role": "system", "content": "compressed"}]
- assert body["tools"] == [{"name": "retrieval"}]
- assert stats["total_tokens_saved"] == 60
- assert stats["savings_percent"] == 60.0
-
-
-async def test_compress_batch_jsonl_falls_back_when_pipeline_raises(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- install_batch_support_modules(monkeypatch, tokenizer_count=33)
- handler = DummyBatchHandler()
- handler.config.optimize = True
- handler.openai_pipeline = SimpleNamespace(
- apply=lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))
- )
-
- lines, stats = await handler._compress_batch_jsonl(
- json.dumps({"body": {"messages": [{"role": "user", "content": "hello"}]}}),
- "req-3",
- )
-
- assert json.loads(lines[0])["body"]["messages"][0]["content"] == "hello"
- assert stats["total_original_tokens"] == 33
- assert stats["total_compressed_tokens"] == 33
-
-
-async def test_batch_passthrough_forwards_request_and_strips_response_headers() -> None:
- handler = DummyBatchHandler()
- handler.http_client.post_response = FakeResponse(
- content=b'{"ok":true}',
- headers={"content-encoding": "gzip", "content-length": "20", "x-kept": "1"},
- )
-
- response = await handler._batch_passthrough(
- FakeRequest(
- '{"input_file_id":"file-1"}', headers={"host": "example", "content-length": "10"}
- ),
- {"input_file_id": "file-1"},
- )
-
- assert response.status_code == 200
- assert dict(response.headers)["x-kept"] == "1"
- assert "content-encoding" not in dict(response.headers)
- assert handler.http_client.posts[0]["url"] == "https://openai.example/v1/batches"
-
-
-async def test_handle_batch_create_validates_json_and_required_fields(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- handler = DummyBatchHandler()
-
- async def raise_bad_json(request): # noqa: ANN001
- raise ValueError("bad json")
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", raise_bad_json)
-
- bad = await handler.handle_batch_create(FakeRequest("{}"))
- assert bad.status_code == 400
- assert bad.body.decode().find("invalid_json") > 0
-
- async def missing_file_payload(request): # noqa: ANN001
- return {"endpoint": "/v1/chat/completions"}
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", missing_file_payload)
- missing_file = await handler.handle_batch_create(FakeRequest("{}"))
- assert missing_file.status_code == 400
- assert missing_file.body.decode().find("input_file_id is required") > 0
-
- async def missing_endpoint_payload(request): # noqa: ANN001
- return {"input_file_id": "file-1"}
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", missing_endpoint_payload)
- missing_endpoint = await handler.handle_batch_create(FakeRequest("{}"))
- assert missing_endpoint.status_code == 400
- assert missing_endpoint.body.decode().find("endpoint is required") > 0
-
-
-async def test_handle_batch_create_passthrough_and_download_failure(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- handler = DummyBatchHandler()
- passthrough_response = SimpleNamespace(marker="passthrough")
-
- async def fake_passthrough(request, body): # noqa: ANN001
- return passthrough_response
-
- monkeypatch.setattr(handler, "_batch_passthrough", fake_passthrough)
-
- async def passthrough_payload(request): # noqa: ANN001
- return {"input_file_id": "file-1", "endpoint": "/v1/responses"}
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", passthrough_payload)
- assert await handler.handle_batch_create(FakeRequest("{}")) is passthrough_response
-
- async def download_missing_payload(request): # noqa: ANN001
- return {"input_file_id": "file-1", "endpoint": "/v1/chat/completions"}
-
- async def missing_download(file_id, headers): # noqa: ANN001
- return None
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", download_missing_payload)
- monkeypatch.setattr(handler, "_download_openai_file", missing_download)
- missing = await handler.handle_batch_create(FakeRequest("{}"))
- assert missing.status_code == 404
- assert missing.body.decode().find("file_not_found") > 0
-
-
-async def test_handle_batch_create_handles_empty_upload_failure_and_success(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- handler = DummyBatchHandler()
-
- async def request_payload(request): # noqa: ANN001
- return {
- "input_file_id": "file-1",
- "endpoint": "/v1/chat/completions",
- "completion_window": "12h",
- "metadata": {"source": "test"},
- }
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", request_payload)
-
- async def fake_download(file_id, headers): # noqa: ANN001
- return "downloaded"
-
- monkeypatch.setattr(handler, "_download_openai_file", fake_download)
-
- async def empty_compress(content, request_id): # noqa: ANN001
- return [], {
- "total_requests": 0,
- "total_original_tokens": 0,
- "total_compressed_tokens": 0,
- "total_tokens_saved": 0,
- "savings_percent": 0.0,
- "errors": 0,
- }
-
- monkeypatch.setattr(handler, "_compress_batch_jsonl", empty_compress)
- empty = await handler.handle_batch_create(FakeRequest("{}"))
- assert empty.status_code == 400
- assert empty.body.decode().find("empty_file") > 0
-
- async def compressed(content, request_id): # noqa: ANN001
- return ['{"body":{}}'], {
- "total_requests": 1,
- "total_original_tokens": 20,
- "total_compressed_tokens": 10,
- "total_tokens_saved": 10,
- "savings_percent": 50.0,
- "errors": 0,
- }
-
- monkeypatch.setattr(handler, "_compress_batch_jsonl", compressed)
-
- async def upload_failed_file(content, filename, headers): # noqa: ANN001
- return None
-
- monkeypatch.setattr(handler, "_upload_openai_file", upload_failed_file)
- upload_failed = await handler.handle_batch_create(FakeRequest("{}"))
- assert upload_failed.status_code == 500
- assert upload_failed.body.decode().find("upload_failed") > 0
-
- handler.http_client.post_response = FakeResponse(
- content=b'{"id":"batch_123","object":"batch"}',
- headers={"content-encoding": "gzip", "content-length": "12", "x-openai": "1"},
- )
-
- async def upload_success(content, filename, headers): # noqa: ANN001
- return "file-compressed"
-
- monkeypatch.setattr(handler, "_upload_openai_file", upload_success)
- success = await handler.handle_batch_create(
- FakeRequest(
- "{}", headers={"host": "proxy", "content-length": "4", "authorization": "Bearer test"}
- )
- )
-
- assert success.status_code == 200
- success_headers = dict(success.headers)
- assert success_headers["x-headroom-tokens-saved"] == "10"
- assert success_headers["x-headroom-savings-percent"] == "50.0"
- assert success_headers["x-openai"] == "1"
- sent_body = handler.http_client.posts[-1]["json"]
- assert sent_body["metadata"]["headroom_compressed"] == "true"
- assert sent_body["metadata"]["headroom_original_file_id"] == "file-1"
- assert handler.metrics.record_calls[-1]["provider"] == "openai"
-
-
-async def test_handle_batch_create_records_failure_on_exception(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- handler = DummyBatchHandler()
-
- async def request_payload(request): # noqa: ANN001
- return {"input_file_id": "file-1", "endpoint": "/v1/chat/completions"}
-
- async def boom(file_id, headers): # noqa: ANN001
- raise RuntimeError("boom")
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", request_payload)
- monkeypatch.setattr(handler, "_download_openai_file", boom)
-
- response = await handler.handle_batch_create(FakeRequest("{}"))
-
- assert response.status_code == 500
- assert handler.metrics.failed_calls == [{"provider": "batch"}]
-
-
-async def test_download_and_upload_openai_file_helpers() -> None:
- handler = DummyBatchHandler()
- handler.http_client.get_response = FakeResponse(status_code=200, text="jsonl-content")
- downloaded = await handler._download_openai_file("file-1", {"authorization": "Bearer token"})
- assert downloaded == "jsonl-content"
- assert handler.http_client.gets[0]["url"] == "https://openai.example/v1/files/file-1/content"
-
- handler.http_client.get_response = FakeResponse(status_code=404, text="missing")
- assert await handler._download_openai_file("file-2", {}) is None
-
- handler.http_client.post_response = FakeResponse(
- status_code=200,
- json_data={"id": "file-uploaded"},
- headers={"content-type": "application/json"},
- )
- file_id = await handler._upload_openai_file(
- '{"body":{}}',
- "compressed.jsonl",
- {"authorization": "Bearer token", "content-type": "application/json"},
- )
- assert file_id == "file-uploaded"
- post_call = handler.http_client.posts[-1]
- assert post_call["headers"] == {"authorization": "Bearer token"}
- assert post_call["files"]["file"][0] == "compressed.jsonl"
-
- handler.http_client.post_response = FakeResponse(status_code=500, text="fail")
- assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None
- handler.http_client.raise_post = RuntimeError("network")
- assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None
-
-
-async def test_store_google_batch_context_persists_transformed_requests(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- stored_contexts: list[object] = []
-
- class FakeBatchContext:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
- self.requests: list[object] = []
-
- def add_request(self, request) -> None: # noqa: ANN001
- self.requests.append(request)
-
- class FakeBatchRequestContext:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
-
- class FakeStore:
- async def store(self, context) -> None: # noqa: ANN001
- stored_contexts.append(context)
-
- monkeypatch.setitem(
- sys.modules,
- "headroom.ccr",
- SimpleNamespace(
- BatchContext=FakeBatchContext,
- BatchRequestContext=FakeBatchRequestContext,
- get_batch_context_store=lambda: FakeStore(),
- ),
- )
-
- handler = DummyBatchHandler()
- await handler._store_google_batch_context(
- "batches/123",
- [
- {
- "metadata": {"key": "req-1"},
- "request": {
- "contents": [{"parts": [{"text": "hello"}]}],
- "systemInstruction": {"parts": [{"text": "system"}]},
- "tools": [{"name": "tool"}],
- },
- }
- ],
- "gemini-2.0",
- "api-key",
- )
-
- context = stored_contexts[0]
- assert context.kwargs["batch_id"] == "batches/123"
- assert context.requests[0].kwargs["custom_id"] == "req-1"
- assert context.requests[0].kwargs["messages"] == [{"role": "user", "content": "hello"}]
- assert context.requests[0].kwargs["system_instruction"] == "system"
-
-
-async def test_handle_google_batch_results_passes_through_early_exit_cases(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- class FakeStore:
- async def get(self, batch_name): # noqa: ANN001
- return None
-
- monkeypatch.setitem(
- sys.modules,
- "headroom.ccr",
- SimpleNamespace(
- BatchResultProcessor=lambda http_client: None,
- get_batch_context_store=lambda: FakeStore(),
- ),
- )
-
- handler = DummyBatchHandler()
- request = FakeRequest(
- "{}", headers={"x-goog-api-key": "secret"}, method="GET", path="/v1beta/batches/b1"
- )
-
- handler.http_client.get_response = FakeResponse(
- status_code=500, content=b"bad", headers={"x-upstream": "1"}
- )
- error_response = await handler.handle_google_batch_results(request, "batches/b1")
- assert error_response.status_code == 500
- assert dict(error_response.headers)["x-upstream"] == "1"
-
- class BadJsonResponse(FakeResponse):
- def json(self): # noqa: ANN201
- raise json.JSONDecodeError("bad", "x", 0)
-
- handler.http_client.get_response = BadJsonResponse(
- status_code=200, content=b"plain", headers={"x-upstream": "2"}
- )
- non_json = await handler.handle_google_batch_results(request, "batches/b1")
- assert non_json.status_code == 200
- assert dict(non_json.headers)["x-upstream"] == "2"
-
- handler.http_client.get_response = FakeResponse(
- status_code=200,
- content=b"{}",
- json_data={"metadata": {"state": "RUNNING"}},
- )
- running = await handler.handle_google_batch_results(request, "batches/b1")
- assert running.status_code == 200
-
- handler.http_client.get_response = FakeResponse(
- status_code=200,
- content=b"{}",
- json_data={"metadata": {"state": "SUCCEEDED"}, "response": {"responses": []}},
- )
- no_results = await handler.handle_google_batch_results(request, "batches/b1")
- assert no_results.status_code == 200
-
- handler.http_client.get_response = FakeResponse(
- status_code=200,
- content=b"{}",
- json_data={"metadata": {"state": "SUCCEEDED"}, "response": {"responses": [{"id": 1}]}},
- )
- handler.config.ccr_inject_tool = False
- no_ccr = await handler.handle_google_batch_results(request, "batches/b1")
- assert no_ccr.status_code == 200
- assert "key=secret" in handler.http_client.gets[-1]["url"]
-
-
-async def test_handle_google_batch_results_processes_completed_results(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- processed_calls: list[tuple[str, list[object], str]] = []
-
- class FakeProcessed:
- def __init__(
- self, result, custom_id: str, was_processed: bool, continuation_rounds: int
- ) -> None: # noqa: ANN001
- self.result = result
- self.custom_id = custom_id
- self.was_processed = was_processed
- self.continuation_rounds = continuation_rounds
-
- class FakeProcessor:
- def __init__(self, http_client) -> None: # noqa: ANN001
- self.http_client = http_client
-
- async def process_results(self, batch_name, results, provider): # noqa: ANN001
- processed_calls.append((batch_name, results, provider))
- return [
- FakeProcessed({"id": "processed"}, "req-1", True, 2),
- FakeProcessed({"id": "unchanged"}, "req-2", False, 0),
- ]
-
- class FakeStore:
- async def get(self, batch_name): # noqa: ANN001
- return SimpleNamespace(batch_name=batch_name)
-
- monkeypatch.setitem(
- sys.modules,
- "headroom.ccr",
- SimpleNamespace(
- BatchResultProcessor=FakeProcessor,
- get_batch_context_store=lambda: FakeStore(),
- ),
- )
-
- handler = DummyBatchHandler()
- handler.config.ccr_inject_tool = True
- handler.http_client.get_response = FakeResponse(
- status_code=200,
- content=b"{}",
- json_data={
- "metadata": {"state": "SUCCEEDED"},
- "response": {"responses": [{"id": "raw-1"}, {"id": "raw-2"}]},
- },
- )
-
- response = await handler.handle_google_batch_results(
- FakeRequest("{}", method="GET", path="/v1beta/batches/b1"),
- "batches/b1",
- )
-
- payload = json.loads(response.body)
- assert payload["response"]["responses"] == [{"id": "processed"}, {"id": "unchanged"}]
- assert processed_calls == [("batches/b1", [{"id": "raw-1"}, {"id": "raw-2"}], "google")]
- assert handler.metrics.record_calls[-1]["model"] == "batch:ccr-processed"
-
-
-async def test_google_batch_passthrough_helpers_forward_and_track_metrics() -> None:
- handler = DummyBatchHandler()
- handler.http_client.post_response = FakeResponse(
- content=b'{"ok":true}',
- headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "1"},
- )
- handler.http_client.post_response = FakeResponse(
- content=b'{"ok":true}',
- headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "1"},
- )
-
- passthrough = await handler._google_batch_passthrough(
- FakeRequest(
- "body", headers={"host": "proxy", "content-length": "4", "x-goog-api-key": "secret"}
- ),
- "gemini-pro",
- {"batch": {}},
- )
- assert passthrough.status_code == 200
- assert dict(passthrough.headers)["x-kept"] == "1"
- assert "key=secret" in handler.http_client.posts[-1]["url"]
- assert handler.metrics.record_calls[-1]["model"] == "passthrough:batch:gemini-pro"
-
- handler.http_client.get_response = FakeResponse(
- content=b'{"state":"ok"}',
- headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "2"},
- )
- response = await handler.handle_google_batch_passthrough(
- FakeRequest(
- "ping",
- headers={"host": "proxy", "x-goog-api-key": "secret"},
- method="DELETE",
- path="/v1beta/batches/b1",
- query="alt=json",
- ),
- "b1",
- )
- assert response.status_code == 200
- assert dict(response.headers)["x-kept"] == "2"
- get_call = handler.http_client.requests[-1]
- assert get_call["url"] == "https://gemini.example/v1beta/batches/b1?alt=json&key=secret"
- assert handler.metrics.record_calls[-1]["model"] == "passthrough:batches"
-
-
-async def test_handle_google_batch_create_validates_and_passthroughs(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- install_batch_support_modules(monkeypatch)
- handler = DummyBatchHandler()
-
- too_large = await handler.handle_google_batch_create(
- FakeRequest("{}", headers={"content-length": str(200 * 1024 * 1024)}),
- "gemini-pro",
- )
- assert too_large.status_code == 413
-
- async def bad_json(request): # noqa: ANN001
- raise ValueError("bad json")
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", bad_json)
- invalid = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")
- assert invalid.status_code == 400
-
- passthrough_response = SimpleNamespace(kind="passthrough")
-
- async def fake_google_passthrough(request, model, body=None): # noqa: ANN001
- return passthrough_response
-
- async def no_inline(request): # noqa: ANN001
- return {"batch": {"input_config": {"requests": {"requests": []}}}}
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", no_inline)
- monkeypatch.setattr(handler, "_google_batch_passthrough", fake_google_passthrough)
- assert (
- await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")
- is passthrough_response
- )
-
-
-async def test_handle_google_batch_create_success_and_failure_paths(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- install_batch_support_modules(monkeypatch)
- handler = DummyBatchHandler()
- handler.config.optimize = True
- handler.config.ccr_inject_tool = True
- handler.openai_pipeline = SimpleNamespace(
- apply=lambda **kwargs: SimpleNamespace(
- messages=[{"role": "user", "content": "compressed"}],
- timing={"compress": 1.2},
- tokens_before=100,
- tokens_after=40,
- )
- )
-
- class FakeInjector:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- pass
-
- def process_request(self, messages, tools): # noqa: ANN001, ANN201
- return (
- messages + [{"role": "system", "content": "retrieval"}],
- [{"name": "retrieval"}],
- True,
- )
-
- monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))
-
- stored: list[tuple[str, list[dict[str, object]], str, str | None]] = []
-
- async def fake_store(batch_name, requests_list, model, api_key): # noqa: ANN001
- stored.append((batch_name, requests_list, model, api_key))
-
- async def fake_retry(method, url, headers, body): # noqa: ANN001
- return FakeResponse(
- status_code=200,
- content=b'{"name":"batches/123"}',
- headers={"content-encoding": "gzip", "content-length": "10", "x-upstream": "1"},
- json_data={"name": "batches/123"},
- )
-
- async def good_payload(request): # noqa: ANN001
- return {
- "batch": {
- "input_config": {
- "requests": {
- "requests": [
- {
- "request": {
- "contents": [{"parts": [{"text": "hello"}]}],
- "tools": [{"functionDeclarations": [{"name": "existing"}]}],
- },
- "metadata": {"key": "req-1"},
- }
- ]
- }
- }
- }
- }
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", good_payload)
- monkeypatch.setattr(handler, "_retry_request", fake_retry)
- monkeypatch.setattr(handler, "_store_google_batch_context", fake_store)
-
- response = await handler.handle_google_batch_create(
- FakeRequest("{}", headers={"x-goog-api-key": "secret"}),
- "gemini-pro",
- )
- assert response.status_code == 200
- assert dict(response.headers)["x-upstream"] == "1"
- assert handler.metrics.record_calls[-1]["provider"] == "google"
- assert handler.metrics.record_calls[-1]["tokens_saved"] == 60
- assert stored[0][0] == "batches/123"
- assert stored[0][2:] == ("gemini-pro", "secret")
- assert stored[0][1][0]["metadata"] == {"key": "req-1"}
-
- async def broken_retry(method, url, headers, body): # noqa: ANN001
- raise RuntimeError("forward failed")
-
- monkeypatch.setattr(handler, "_retry_request", broken_retry)
- failed = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")
- assert failed.status_code == 500
-
-
-async def test_handle_google_batch_create_covers_passthrough_revert_and_store_failures(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- install_batch_support_modules(
- monkeypatch, injector_result=([{"role": "user", "content": "kept"}], None, False)
- )
- handler = DummyBatchHandler()
- handler.config.optimize = True
- handler.config.ccr_inject_tool = True
-
- pipeline_calls: list[dict[str, object]] = []
- handler.openai_pipeline = SimpleNamespace(
- apply=lambda **kwargs: pipeline_calls.append(kwargs)
- or SimpleNamespace(
- messages=[{"role": "user", "content": "inflated"}],
- timing={},
- tokens_before=40,
- tokens_after=80,
- )
- )
-
- def fake_to_messages(contents, system_instruction): # noqa: ANN001, ANN201
- if contents and "inlineData" in contents[0]["parts"][0]:
- return ([{"role": "user", "content": "binary"}], [0])
- return ([{"role": "user", "content": "compress"}], [])
-
- def fake_to_gemini(messages): # noqa: ANN001, ANN201
- return ([{"parts": [{"text": "new"}]}], {"parts": [{"text": "sys"}]})
-
- async def payload(request): # noqa: ANN001
- return {
- "batch": {
- "input_config": {
- "requests": {
- "requests": [
- {"request": {"contents": []}, "metadata": {"key": "empty"}},
- {
- "request": {"contents": [{"parts": [{"inlineData": "x"}]}]},
- "metadata": {"key": "preserved"},
- },
- {
- "request": {
- "contents": [{"parts": [{"text": "hello"}]}],
- "tools": [
- {"other": True},
- {"functionDeclarations": [{"name": "existing"}]},
- ],
- },
- "metadata": {"key": "optimized"},
- },
- ]
- }
- }
- }
- }
-
- seen_bodies: list[dict[str, object]] = []
-
- async def retry(method, url, headers, body): # noqa: ANN001
- seen_bodies.append(body)
- return FakeResponse(status_code=200, content=b"{}", json_data={"name": "batches/123"})
-
- async def broken_store(batch_name, requests_list, model, api_key): # noqa: ANN001
- raise RuntimeError("store failed")
-
- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", payload)
- monkeypatch.setattr(handler, "_gemini_contents_to_messages", fake_to_messages)
- monkeypatch.setattr(handler, "_messages_to_gemini_contents", fake_to_gemini)
- monkeypatch.setattr(handler, "_retry_request", retry)
- monkeypatch.setattr(handler, "_store_google_batch_context", broken_store)
-
- response = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")
- assert response.status_code == 200
- assert len(pipeline_calls) == 1
- assert handler.metrics.record_calls[-1]["tokens_saved"] == 0
- assert (
- seen_bodies[0]["batch"]["input_config"]["requests"]["requests"][0]["metadata"]["key"]
- == "empty"
- )
- optimized = seen_bodies[0]["batch"]["input_config"]["requests"]["requests"][2]["request"]
- assert optimized["contents"][0] == {"parts": [{"text": "new"}]}
- assert optimized["systemInstruction"] == {"parts": [{"text": "sys"}]}
-
-
-async def test_google_batch_passthrough_without_body_and_query_variants() -> None:
- handler = DummyBatchHandler()
- handler.http_client.post_response = FakeResponse(content=b"ok", headers={"x-upstream": "1"})
-
- response = await handler._google_batch_passthrough(
- FakeRequest("raw-body", headers={"host": "proxy"}, method="POST"),
- "gemini-pro",
- )
- assert response.status_code == 200
- assert handler.http_client.posts[-1]["content"] == b"raw-body"
-
- handler.http_client.get_response = FakeResponse(content=b"{}", headers={"x-upstream": "2"})
- passthrough = await handler.handle_google_batch_passthrough(
- FakeRequest(
- "{}",
- headers={"host": "proxy", "x-goog-api-key": "secret"},
- method="GET",
- path="/v1beta/batches/b1",
- ),
- "b1",
- )
- assert passthrough.status_code == 200
- assert (
- handler.http_client.requests[-1]["url"]
- == "https://gemini.example/v1beta/batches/b1?key=secret"
- )
-
-
-async def test_batch_helper_methods_and_openai_file_error_branches() -> None:
- handler = DummyBatchHandler()
- marker = object()
-
- async def fake_passthrough(request, base_url): # noqa: ANN001
- return marker
-
- handler.handle_passthrough = fake_passthrough
- request = FakeRequest("{}")
- assert await handler.handle_batch_list(request) is marker
- assert await handler.handle_batch_get(request, "b1") is marker
- assert await handler.handle_batch_cancel(request, "b1") is marker
-
- handler.http_client.raise_get = RuntimeError("download boom")
- assert await handler._download_openai_file("file-1", {}) is None
-
- handler.http_client.raise_get = None
- handler.http_client.post_response = FakeResponse(status_code=200, json_data={})
- assert await handler._upload_openai_file("{}", "missing-id.jsonl", {}) is None
-
-
-async def test_store_google_batch_context_without_system_text() -> None:
- stored_contexts: list[object] = []
-
- class FakeBatchContext:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
- self.requests: list[object] = []
-
- def add_request(self, request) -> None: # noqa: ANN001
- self.requests.append(request)
-
- class FakeBatchRequestContext:
- def __init__(self, **kwargs) -> None: # noqa: ANN003
- self.kwargs = kwargs
-
- class FakeStore:
- async def store(self, context) -> None: # noqa: ANN001
- stored_contexts.append(context)
-
- handler = DummyBatchHandler()
- sys.modules["headroom.ccr"] = SimpleNamespace(
- BatchContext=FakeBatchContext,
- BatchRequestContext=FakeBatchRequestContext,
- get_batch_context_store=lambda: FakeStore(),
- )
-
- await handler._store_google_batch_context(
- "batches/456",
- [
- {
- "request": {
- "contents": [{"parts": [{"text": "hello"}]}],
- "systemInstruction": {"parts": ["bad"]},
- }
- }
- ],
- "gemini-2.0",
- None,
- )
-
- context = stored_contexts[0]
- assert context.kwargs["api_key"] is None
- assert context.requests[0].kwargs["custom_id"] == ""
- assert context.requests[0].kwargs["system_instruction"] is None
-
-
-async def test_compress_batch_jsonl_skips_blank_lines_and_preserves_tools_when_not_injected(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- install_batch_support_modules(
- monkeypatch,
- injector_result=([{"role": "assistant", "content": "short"}], [{"name": "orig"}], False),
- )
- handler = DummyBatchHandler()
- handler.config.optimize = True
- handler.config.ccr_inject_tool = True
- handler.openai_pipeline = SimpleNamespace(
- apply=lambda **kwargs: SimpleNamespace(
- messages=[{"role": "assistant", "content": "short"}],
- tokens_before=50,
- tokens_after=10,
- )
- )
-
- lines, stats = await handler._compress_batch_jsonl(
- "\n"
- + json.dumps(
- {
- "body": {
- "model": "gpt-4o",
- "messages": [{"role": "user", "content": "hello"}],
- "tools": [{"name": "orig"}],
- }
- }
- )
- + "\n",
- "req-extra",
- )
-
- assert len(lines) == 1
- body = json.loads(lines[0])["body"]
- assert body["tools"] == [{"name": "orig"}]
- assert stats["total_requests"] == 1
- assert stats["errors"] == 0
+from __future__ import annotations
+
+import json
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+from headroom.proxy.handlers import batch as batch_module
+
+
+class FakeResponse:
+ def __init__(
+ self,
+ *,
+ status_code: int = 200,
+ content: bytes = b"{}",
+ headers: dict[str, str] | None = None,
+ text: str | None = None,
+ json_data=None, # noqa: ANN001
+ ) -> None:
+ self.status_code = status_code
+ self.content = content
+ self.headers = headers or {}
+ self.text = text if text is not None else content.decode("utf-8", errors="ignore")
+ self._json_data = json_data
+
+ def json(self): # noqa: ANN201
+ if self._json_data is not None:
+ return self._json_data
+ return json.loads(self.text)
+
+
+class FakeHttpClient:
+ def __init__(self) -> None:
+ self.posts: list[dict[str, object]] = []
+ self.gets: list[dict[str, object]] = []
+ self.requests: list[dict[str, object]] = []
+ self.post_response = FakeResponse()
+ self.get_response = FakeResponse()
+ self.raise_post: Exception | None = None
+ self.raise_get: Exception | None = None
+
+ async def post(self, url: str, **kwargs): # noqa: ANN003, ANN201
+ self.posts.append({"url": url, **kwargs})
+ if self.raise_post is not None:
+ raise self.raise_post
+ return self.post_response
+
+ async def get(self, url: str, **kwargs): # noqa: ANN003, ANN201
+ self.gets.append({"url": url, **kwargs})
+ if self.raise_get is not None:
+ raise self.raise_get
+ return self.get_response
+
+ async def request(self, method: str, url: str, **kwargs): # noqa: ANN003, ANN201
+ self.requests.append({"method": method, "url": url, **kwargs})
+ if self.raise_get is not None:
+ raise self.raise_get
+ return self.get_response
+
+
+class FakeMetrics:
+ def __init__(self) -> None:
+ self.record_calls: list[dict[str, object]] = []
+ self.failed_calls: list[dict[str, object]] = []
+
+ async def record_request(self, **kwargs) -> None: # noqa: ANN003
+ self.record_calls.append(kwargs)
+
+ async def record_failed(self, **kwargs) -> None: # noqa: ANN003
+ self.failed_calls.append(kwargs)
+
+
+class DummyBatchHandler(batch_module.BatchHandlerMixin):
+ OPENAI_API_URL = "https://openai.example"
+ GEMINI_API_URL = "https://gemini.example"
+
+ def __init__(self) -> None:
+ self.http_client = FakeHttpClient()
+ self.metrics = FakeMetrics()
+ self.config = SimpleNamespace(
+ optimize=False,
+ ccr_inject_tool=False,
+ ccr_inject_system_instructions=False,
+ )
+ self.openai_provider = SimpleNamespace(get_context_limit=lambda model: 8192)
+ self.openai_pipeline = SimpleNamespace(apply=lambda **kwargs: None)
+ self._request_counter = 0
+ self._retry_response = FakeResponse()
+
+ async def _next_request_id(self) -> str:
+ self._request_counter += 1
+ return f"req-{self._request_counter}"
+
+ async def handle_passthrough(self, request, base_url): # noqa: ANN001, ANN201
+ return {"request": request, "base_url": base_url}
+
+ async def _retry_request(self, method, url, headers, body): # noqa: ANN001, ANN201
+ return self._retry_response
+
+ def _gemini_contents_to_messages(self, contents, system_instruction): # noqa: ANN001, ANN201
+ messages = [{"role": "user", "content": part["parts"][0]["text"]} for part in contents]
+ return messages, []
+
+ def _messages_to_gemini_contents(self, messages): # noqa: ANN001, ANN201
+ return ([{"parts": [{"text": message["content"]}]} for message in messages], None)
+
+
+class FakeRequest:
+ def __init__(
+ self,
+ body: bytes | str,
+ *,
+ headers: dict[str, str] | None = None,
+ method: str = "POST",
+ path: str = "/v1/batches",
+ query: str = "",
+ ) -> None:
+ self._body = body.encode("utf-8") if isinstance(body, str) else body
+ self.headers = headers or {}
+ self.method = method
+ self.url = SimpleNamespace(path=path, query=query)
+
+ async def body(self) -> bytes:
+ return self._body
+
+
+def install_batch_support_modules(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ injector_result=None, # noqa: ANN001
+ tokenizer_count: int = 10,
+) -> None:
+ class FakeInjector:
+ def __init__(self, **kwargs) -> None: # noqa: ANN003
+ self.kwargs = kwargs
+
+ def process_request(self, messages, tools): # noqa: ANN001, ANN201
+ if injector_result is not None:
+ return injector_result
+ return messages, tools, False
+
+ class FakeTokenizer:
+ def count_messages(self, messages) -> int: # noqa: ANN001
+ return tokenizer_count
+
+ monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))
+ monkeypatch.setitem(
+ sys.modules,
+ "headroom.tokenizers",
+ SimpleNamespace(get_tokenizer=lambda model: FakeTokenizer()),
+ )
+ monkeypatch.setitem(
+ sys.modules,
+ "headroom.utils",
+ SimpleNamespace(extract_user_query=lambda messages: "query"),
+ )
+
+
+async def test_compress_batch_jsonl_without_optimization_handles_invalid_lines(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_batch_support_modules(monkeypatch, tokenizer_count=12)
+ handler = DummyBatchHandler()
+ content = "\n".join(
+ [
+ json.dumps(
+ {"body": {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}}
+ ),
+ json.dumps({"body": {"model": "gpt-4o", "messages": []}}),
+ "not-json",
+ ]
+ )
+
+ lines, stats = await handler._compress_batch_jsonl(content, "req-1")
+
+ assert len(lines) == 3
+ assert json.loads(lines[0])["body"]["messages"][0]["content"] == "hi"
+ assert lines[2] == "not-json"
+ assert stats == {
+ "total_requests": 3,
+ "total_original_tokens": 12,
+ "total_compressed_tokens": 12,
+ "total_tokens_saved": 0,
+ "savings_percent": 0.0,
+ "errors": 1,
+ }
+
+
+async def test_compress_batch_jsonl_uses_pipeline_and_ccr_injection(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_batch_support_modules(
+ monkeypatch,
+ injector_result=(
+ [{"role": "system", "content": "compressed"}],
+ [{"name": "retrieval"}],
+ True,
+ ),
+ )
+ handler = DummyBatchHandler()
+ handler.config.optimize = True
+ handler.config.ccr_inject_tool = True
+ handler.openai_pipeline = SimpleNamespace(
+ apply=lambda **kwargs: SimpleNamespace(
+ messages=[{"role": "assistant", "content": "short"}],
+ tokens_before=100,
+ tokens_after=40,
+ )
+ )
+
+ lines, stats = await handler._compress_batch_jsonl(
+ json.dumps(
+ {
+ "body": {
+ "model": "gpt-4o-mini",
+ "messages": [{"role": "user", "content": "hello"}],
+ "tools": [{"name": "existing"}],
+ }
+ }
+ ),
+ "req-2",
+ )
+
+ body = json.loads(lines[0])["body"]
+ assert body["messages"] == [{"role": "system", "content": "compressed"}]
+ assert body["tools"] == [{"name": "retrieval"}]
+ assert stats["total_tokens_saved"] == 60
+ assert stats["savings_percent"] == 60.0
+
+
+async def test_compress_batch_jsonl_falls_back_when_pipeline_raises(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_batch_support_modules(monkeypatch, tokenizer_count=33)
+ handler = DummyBatchHandler()
+ handler.config.optimize = True
+ handler.openai_pipeline = SimpleNamespace(
+ apply=lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))
+ )
+
+ lines, stats = await handler._compress_batch_jsonl(
+ json.dumps({"body": {"messages": [{"role": "user", "content": "hello"}]}}),
+ "req-3",
+ )
+
+ assert json.loads(lines[0])["body"]["messages"][0]["content"] == "hello"
+ assert stats["total_original_tokens"] == 33
+ assert stats["total_compressed_tokens"] == 33
+
+
+async def test_batch_passthrough_forwards_request_and_strips_response_headers() -> None:
+ handler = DummyBatchHandler()
+ handler.http_client.post_response = FakeResponse(
+ content=b'{"ok":true}',
+ headers={"content-encoding": "gzip", "content-length": "20", "x-kept": "1"},
+ )
+
+ response = await handler._batch_passthrough(
+ FakeRequest(
+ '{"input_file_id":"file-1"}', headers={"host": "example", "content-length": "10"}
+ ),
+ {"input_file_id": "file-1"},
+ )
+
+ assert response.status_code == 200
+ assert dict(response.headers)["x-kept"] == "1"
+ assert "content-encoding" not in dict(response.headers)
+ assert handler.http_client.posts[0]["url"] == "https://openai.example/v1/batches"
+
+
+async def test_handle_batch_create_validates_json_and_required_fields(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ handler = DummyBatchHandler()
+
+ async def raise_bad_json(request): # noqa: ANN001
+ raise ValueError("bad json")
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", raise_bad_json)
+
+ bad = await handler.handle_batch_create(FakeRequest("{}"))
+ assert bad.status_code == 400
+ assert bad.body.decode().find("invalid_json") > 0
+
+ async def missing_file_payload(request): # noqa: ANN001
+ return {"endpoint": "/v1/chat/completions"}
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", missing_file_payload)
+ missing_file = await handler.handle_batch_create(FakeRequest("{}"))
+ assert missing_file.status_code == 400
+ assert missing_file.body.decode().find("input_file_id is required") > 0
+
+ async def missing_endpoint_payload(request): # noqa: ANN001
+ return {"input_file_id": "file-1"}
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", missing_endpoint_payload)
+ missing_endpoint = await handler.handle_batch_create(FakeRequest("{}"))
+ assert missing_endpoint.status_code == 400
+ assert missing_endpoint.body.decode().find("endpoint is required") > 0
+
+
+async def test_handle_batch_create_passthrough_and_download_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ handler = DummyBatchHandler()
+ passthrough_response = SimpleNamespace(marker="passthrough")
+
+ async def fake_passthrough(request, body): # noqa: ANN001
+ return passthrough_response
+
+ monkeypatch.setattr(handler, "_batch_passthrough", fake_passthrough)
+
+ async def passthrough_payload(request): # noqa: ANN001
+ return {"input_file_id": "file-1", "endpoint": "/v1/responses"}
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", passthrough_payload)
+ assert await handler.handle_batch_create(FakeRequest("{}")) is passthrough_response
+
+ async def download_missing_payload(request): # noqa: ANN001
+ return {"input_file_id": "file-1", "endpoint": "/v1/chat/completions"}
+
+ async def missing_download(file_id, headers): # noqa: ANN001
+ return None
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", download_missing_payload)
+ monkeypatch.setattr(handler, "_download_openai_file", missing_download)
+ missing = await handler.handle_batch_create(FakeRequest("{}"))
+ assert missing.status_code == 404
+ assert missing.body.decode().find("file_not_found") > 0
+
+
+async def test_handle_batch_create_handles_empty_upload_failure_and_success(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ handler = DummyBatchHandler()
+
+ async def request_payload(request): # noqa: ANN001
+ return {
+ "input_file_id": "file-1",
+ "endpoint": "/v1/chat/completions",
+ "completion_window": "12h",
+ "metadata": {"source": "test"},
+ }
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", request_payload)
+
+ async def fake_download(file_id, headers): # noqa: ANN001
+ return "downloaded"
+
+ monkeypatch.setattr(handler, "_download_openai_file", fake_download)
+
+ async def empty_compress(content, request_id): # noqa: ANN001
+ return [], {
+ "total_requests": 0,
+ "total_original_tokens": 0,
+ "total_compressed_tokens": 0,
+ "total_tokens_saved": 0,
+ "savings_percent": 0.0,
+ "errors": 0,
+ }
+
+ monkeypatch.setattr(handler, "_compress_batch_jsonl", empty_compress)
+ empty = await handler.handle_batch_create(FakeRequest("{}"))
+ assert empty.status_code == 400
+ assert empty.body.decode().find("empty_file") > 0
+
+ async def compressed(content, request_id): # noqa: ANN001
+ return ['{"body":{}}'], {
+ "total_requests": 1,
+ "total_original_tokens": 20,
+ "total_compressed_tokens": 10,
+ "total_tokens_saved": 10,
+ "savings_percent": 50.0,
+ "errors": 0,
+ }
+
+ monkeypatch.setattr(handler, "_compress_batch_jsonl", compressed)
+
+ async def upload_failed_file(content, filename, headers): # noqa: ANN001
+ return None
+
+ monkeypatch.setattr(handler, "_upload_openai_file", upload_failed_file)
+ upload_failed = await handler.handle_batch_create(FakeRequest("{}"))
+ assert upload_failed.status_code == 500
+ assert upload_failed.body.decode().find("upload_failed") > 0
+
+ handler.http_client.post_response = FakeResponse(
+ content=b'{"id":"batch_123","object":"batch"}',
+ headers={"content-encoding": "gzip", "content-length": "12", "x-openai": "1"},
+ )
+
+ async def upload_success(content, filename, headers): # noqa: ANN001
+ return "file-compressed"
+
+ monkeypatch.setattr(handler, "_upload_openai_file", upload_success)
+ success = await handler.handle_batch_create(
+ FakeRequest(
+ "{}", headers={"host": "proxy", "content-length": "4", "authorization": "Bearer test"}
+ )
+ )
+
+ assert success.status_code == 200
+ success_headers = dict(success.headers)
+ assert success_headers["x-headroom-tokens-saved"] == "10"
+ assert success_headers["x-headroom-savings-percent"] == "50.0"
+ assert success_headers["x-openai"] == "1"
+ sent_body = handler.http_client.posts[-1]["json"]
+ assert sent_body["metadata"]["headroom_compressed"] == "true"
+ assert sent_body["metadata"]["headroom_original_file_id"] == "file-1"
+ assert handler.metrics.record_calls[-1]["provider"] == "openai"
+
+
+async def test_handle_batch_create_records_failure_on_exception(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ handler = DummyBatchHandler()
+
+ async def request_payload(request): # noqa: ANN001
+ return {"input_file_id": "file-1", "endpoint": "/v1/chat/completions"}
+
+ async def boom(file_id, headers): # noqa: ANN001
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", request_payload)
+ monkeypatch.setattr(handler, "_download_openai_file", boom)
+
+ response = await handler.handle_batch_create(FakeRequest("{}"))
+
+ assert response.status_code == 500
+ assert handler.metrics.failed_calls == [{"provider": "batch"}]
+
+
+async def test_download_and_upload_openai_file_helpers() -> None:
+ handler = DummyBatchHandler()
+ handler.http_client.get_response = FakeResponse(status_code=200, text="jsonl-content")
+ downloaded = await handler._download_openai_file("file-1", {"authorization": "Bearer token"})
+ assert downloaded == "jsonl-content"
+ assert handler.http_client.gets[0]["url"] == "https://openai.example/v1/files/file-1/content"
+
+ handler.http_client.get_response = FakeResponse(status_code=404, text="missing")
+ assert await handler._download_openai_file("file-2", {}) is None
+
+ handler.http_client.post_response = FakeResponse(
+ status_code=200,
+ json_data={"id": "file-uploaded"},
+ headers={"content-type": "application/json"},
+ )
+ file_id = await handler._upload_openai_file(
+ '{"body":{}}',
+ "compressed.jsonl",
+ {"authorization": "Bearer token", "content-type": "application/json"},
+ )
+ assert file_id == "file-uploaded"
+ post_call = handler.http_client.posts[-1]
+ assert post_call["headers"] == {"authorization": "Bearer token"}
+ assert post_call["files"]["file"][0] == "compressed.jsonl"
+
+ handler.http_client.post_response = FakeResponse(status_code=500, text="fail")
+ assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None
+ handler.http_client.raise_post = RuntimeError("network")
+ assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None
+
+
+async def test_store_google_batch_context_persists_transformed_requests(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ stored_contexts: list[object] = []
+
+ class FakeBatchContext:
+ def __init__(self, **kwargs) -> None: # noqa: ANN003
+ self.kwargs = kwargs
+ self.requests: list[object] = []
+
+ def add_request(self, request) -> None: # noqa: ANN001
+ self.requests.append(request)
+
+ class FakeBatchRequestContext:
+ def __init__(self, **kwargs) -> None: # noqa: ANN003
+ self.kwargs = kwargs
+
+ class FakeStore:
+ async def store(self, context) -> None: # noqa: ANN001
+ stored_contexts.append(context)
+
+ monkeypatch.setitem(
+ sys.modules,
+ "headroom.ccr",
+ SimpleNamespace(
+ BatchContext=FakeBatchContext,
+ BatchRequestContext=FakeBatchRequestContext,
+ get_batch_context_store=lambda: FakeStore(),
+ ),
+ )
+
+ handler = DummyBatchHandler()
+ await handler._store_google_batch_context(
+ "batches/123",
+ [
+ {
+ "metadata": {"key": "req-1"},
+ "request": {
+ "contents": [{"parts": [{"text": "hello"}]}],
+ "systemInstruction": {"parts": [{"text": "system"}]},
+ "tools": [{"name": "tool"}],
+ },
+ }
+ ],
+ "gemini-2.0",
+ "api-key",
+ )
+
+ context = stored_contexts[0]
+ assert context.kwargs["batch_id"] == "batches/123"
+ assert context.requests[0].kwargs["custom_id"] == "req-1"
+ assert context.requests[0].kwargs["messages"] == [{"role": "user", "content": "hello"}]
+ assert context.requests[0].kwargs["system_instruction"] == "system"
+
+
+async def test_handle_google_batch_results_passes_through_early_exit_cases(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class FakeStore:
+ async def get(self, batch_name): # noqa: ANN001
+ return None
+
+ monkeypatch.setitem(
+ sys.modules,
+ "headroom.ccr",
+ SimpleNamespace(
+ BatchResultProcessor=lambda http_client: None,
+ get_batch_context_store=lambda: FakeStore(),
+ ),
+ )
+
+ handler = DummyBatchHandler()
+ request = FakeRequest(
+ "{}", headers={"x-goog-api-key": "secret"}, method="GET", path="/v1beta/batches/b1"
+ )
+
+ handler.http_client.get_response = FakeResponse(
+ status_code=500, content=b"bad", headers={"x-upstream": "1"}
+ )
+ error_response = await handler.handle_google_batch_results(request, "batches/b1")
+ assert error_response.status_code == 500
+ assert dict(error_response.headers)["x-upstream"] == "1"
+
+ class BadJsonResponse(FakeResponse):
+ def json(self): # noqa: ANN201
+ raise json.JSONDecodeError("bad", "x", 0)
+
+ handler.http_client.get_response = BadJsonResponse(
+ status_code=200, content=b"plain", headers={"x-upstream": "2"}
+ )
+ non_json = await handler.handle_google_batch_results(request, "batches/b1")
+ assert non_json.status_code == 200
+ assert dict(non_json.headers)["x-upstream"] == "2"
+
+ handler.http_client.get_response = FakeResponse(
+ status_code=200,
+ content=b"{}",
+ json_data={"metadata": {"state": "RUNNING"}},
+ )
+ running = await handler.handle_google_batch_results(request, "batches/b1")
+ assert running.status_code == 200
+
+ handler.http_client.get_response = FakeResponse(
+ status_code=200,
+ content=b"{}",
+ json_data={"metadata": {"state": "SUCCEEDED"}, "response": {"responses": []}},
+ )
+ no_results = await handler.handle_google_batch_results(request, "batches/b1")
+ assert no_results.status_code == 200
+
+ handler.http_client.get_response = FakeResponse(
+ status_code=200,
+ content=b"{}",
+ json_data={"metadata": {"state": "SUCCEEDED"}, "response": {"responses": [{"id": 1}]}},
+ )
+ handler.config.ccr_inject_tool = False
+ no_ccr = await handler.handle_google_batch_results(request, "batches/b1")
+ assert no_ccr.status_code == 200
+ assert "key=secret" in handler.http_client.gets[-1]["url"]
+
+
+async def test_handle_google_batch_results_processes_completed_results(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ processed_calls: list[tuple[str, list[object], str]] = []
+
+ class FakeProcessed:
+ def __init__(
+ self, result, custom_id: str, was_processed: bool, continuation_rounds: int
+ ) -> None: # noqa: ANN001
+ self.result = result
+ self.custom_id = custom_id
+ self.was_processed = was_processed
+ self.continuation_rounds = continuation_rounds
+
+ class FakeProcessor:
+ def __init__(self, http_client) -> None: # noqa: ANN001
+ self.http_client = http_client
+
+ async def process_results(self, batch_name, results, provider): # noqa: ANN001
+ processed_calls.append((batch_name, results, provider))
+ return [
+ FakeProcessed({"id": "processed"}, "req-1", True, 2),
+ FakeProcessed({"id": "unchanged"}, "req-2", False, 0),
+ ]
+
+ class FakeStore:
+ async def get(self, batch_name): # noqa: ANN001
+ return SimpleNamespace(batch_name=batch_name)
+
+ monkeypatch.setitem(
+ sys.modules,
+ "headroom.ccr",
+ SimpleNamespace(
+ BatchResultProcessor=FakeProcessor,
+ get_batch_context_store=lambda: FakeStore(),
+ ),
+ )
+
+ handler = DummyBatchHandler()
+ handler.config.ccr_inject_tool = True
+ handler.http_client.get_response = FakeResponse(
+ status_code=200,
+ content=b"{}",
+ json_data={
+ "metadata": {"state": "SUCCEEDED"},
+ "response": {"responses": [{"id": "raw-1"}, {"id": "raw-2"}]},
+ },
+ )
+
+ response = await handler.handle_google_batch_results(
+ FakeRequest("{}", method="GET", path="/v1beta/batches/b1"),
+ "batches/b1",
+ )
+
+ payload = json.loads(response.body)
+ assert payload["response"]["responses"] == [{"id": "processed"}, {"id": "unchanged"}]
+ assert processed_calls == [("batches/b1", [{"id": "raw-1"}, {"id": "raw-2"}], "google")]
+ assert handler.metrics.record_calls[-1]["model"] == "batch:ccr-processed"
+
+
+async def test_google_batch_passthrough_helpers_forward_and_track_metrics() -> None:
+ handler = DummyBatchHandler()
+ handler.http_client.post_response = FakeResponse(
+ content=b'{"ok":true}',
+ headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "1"},
+ )
+ handler.http_client.post_response = FakeResponse(
+ content=b'{"ok":true}',
+ headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "1"},
+ )
+
+ passthrough = await handler._google_batch_passthrough(
+ FakeRequest(
+ "body", headers={"host": "proxy", "content-length": "4", "x-goog-api-key": "secret"}
+ ),
+ "gemini-pro",
+ {"batch": {}},
+ )
+ assert passthrough.status_code == 200
+ assert dict(passthrough.headers)["x-kept"] == "1"
+ assert "key=secret" in handler.http_client.posts[-1]["url"]
+ assert handler.metrics.record_calls[-1]["model"] == "passthrough:batch:gemini-pro"
+
+ handler.http_client.get_response = FakeResponse(
+ content=b'{"state":"ok"}',
+ headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "2"},
+ )
+ response = await handler.handle_google_batch_passthrough(
+ FakeRequest(
+ "ping",
+ headers={"host": "proxy", "x-goog-api-key": "secret"},
+ method="DELETE",
+ path="/v1beta/batches/b1",
+ query="alt=json",
+ ),
+ "b1",
+ )
+ assert response.status_code == 200
+ assert dict(response.headers)["x-kept"] == "2"
+ get_call = handler.http_client.requests[-1]
+ assert get_call["url"] == "https://gemini.example/v1beta/batches/b1?alt=json&key=secret"
+ assert handler.metrics.record_calls[-1]["model"] == "passthrough:batches"
+
+
+async def test_handle_google_batch_create_validates_and_passthroughs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_batch_support_modules(monkeypatch)
+ handler = DummyBatchHandler()
+
+ too_large = await handler.handle_google_batch_create(
+ FakeRequest("{}", headers={"content-length": str(200 * 1024 * 1024)}),
+ "gemini-pro",
+ )
+ assert too_large.status_code == 413
+
+ async def bad_json(request): # noqa: ANN001
+ raise ValueError("bad json")
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", bad_json)
+ invalid = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")
+ assert invalid.status_code == 400
+
+ passthrough_response = SimpleNamespace(kind="passthrough")
+
+ async def fake_google_passthrough(request, model, body=None): # noqa: ANN001
+ return passthrough_response
+
+ async def no_inline(request): # noqa: ANN001
+ return {"batch": {"input_config": {"requests": {"requests": []}}}}
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", no_inline)
+ monkeypatch.setattr(handler, "_google_batch_passthrough", fake_google_passthrough)
+ assert (
+ await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")
+ is passthrough_response
+ )
+
+
+async def test_handle_google_batch_create_success_and_failure_paths(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_batch_support_modules(monkeypatch)
+ handler = DummyBatchHandler()
+ handler.config.optimize = True
+ handler.config.ccr_inject_tool = True
+ handler.openai_pipeline = SimpleNamespace(
+ apply=lambda **kwargs: SimpleNamespace(
+ messages=[{"role": "user", "content": "compressed"}],
+ timing={"compress": 1.2},
+ tokens_before=100,
+ tokens_after=40,
+ )
+ )
+
+ class FakeInjector:
+ def __init__(self, **kwargs) -> None: # noqa: ANN003
+ pass
+
+ def process_request(self, messages, tools): # noqa: ANN001, ANN201
+ return (
+ messages + [{"role": "system", "content": "retrieval"}],
+ [{"name": "retrieval"}],
+ True,
+ )
+
+ monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))
+
+ stored: list[tuple[str, list[dict[str, object]], str, str | None]] = []
+
+ async def fake_store(batch_name, requests_list, model, api_key): # noqa: ANN001
+ stored.append((batch_name, requests_list, model, api_key))
+
+ async def fake_retry(method, url, headers, body): # noqa: ANN001
+ return FakeResponse(
+ status_code=200,
+ content=b'{"name":"batches/123"}',
+ headers={"content-encoding": "gzip", "content-length": "10", "x-upstream": "1"},
+ json_data={"name": "batches/123"},
+ )
+
+ async def good_payload(request): # noqa: ANN001
+ return {
+ "batch": {
+ "input_config": {
+ "requests": {
+ "requests": [
+ {
+ "request": {
+ "contents": [{"parts": [{"text": "hello"}]}],
+ "tools": [{"functionDeclarations": [{"name": "existing"}]}],
+ },
+ "metadata": {"key": "req-1"},
+ }
+ ]
+ }
+ }
+ }
+ }
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", good_payload)
+ monkeypatch.setattr(handler, "_retry_request", fake_retry)
+ monkeypatch.setattr(handler, "_store_google_batch_context", fake_store)
+
+ response = await handler.handle_google_batch_create(
+ FakeRequest("{}", headers={"x-goog-api-key": "secret"}),
+ "gemini-pro",
+ )
+ assert response.status_code == 200
+ assert dict(response.headers)["x-upstream"] == "1"
+ assert handler.metrics.record_calls[-1]["provider"] == "google"
+ assert handler.metrics.record_calls[-1]["tokens_saved"] == 60
+ assert stored[0][0] == "batches/123"
+ assert stored[0][2:] == ("gemini-pro", "secret")
+ assert stored[0][1][0]["metadata"] == {"key": "req-1"}
+
+ async def broken_retry(method, url, headers, body): # noqa: ANN001
+ raise RuntimeError("forward failed")
+
+ monkeypatch.setattr(handler, "_retry_request", broken_retry)
+ failed = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")
+ assert failed.status_code == 500
+
+
+async def test_handle_google_batch_create_covers_passthrough_revert_and_store_failures(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_batch_support_modules(
+ monkeypatch, injector_result=([{"role": "user", "content": "kept"}], None, False)
+ )
+ handler = DummyBatchHandler()
+ handler.config.optimize = True
+ handler.config.ccr_inject_tool = True
+
+ pipeline_calls: list[dict[str, object]] = []
+ handler.openai_pipeline = SimpleNamespace(
+ apply=lambda **kwargs: pipeline_calls.append(kwargs)
+ or SimpleNamespace(
+ messages=[{"role": "user", "content": "inflated"}],
+ timing={},
+ tokens_before=40,
+ tokens_after=80,
+ )
+ )
+
+ def fake_to_messages(contents, system_instruction): # noqa: ANN001, ANN201
+ if contents and "inlineData" in contents[0]["parts"][0]:
+ return ([{"role": "user", "content": "binary"}], [0])
+ return ([{"role": "user", "content": "compress"}], [])
+
+ def fake_to_gemini(messages): # noqa: ANN001, ANN201
+ return ([{"parts": [{"text": "new"}]}], {"parts": [{"text": "sys"}]})
+
+ async def payload(request): # noqa: ANN001
+ return {
+ "batch": {
+ "input_config": {
+ "requests": {
+ "requests": [
+ {"request": {"contents": []}, "metadata": {"key": "empty"}},
+ {
+ "request": {"contents": [{"parts": [{"inlineData": "x"}]}]},
+ "metadata": {"key": "preserved"},
+ },
+ {
+ "request": {
+ "contents": [{"parts": [{"text": "hello"}]}],
+ "tools": [
+ {"other": True},
+ {"functionDeclarations": [{"name": "existing"}]},
+ ],
+ },
+ "metadata": {"key": "optimized"},
+ },
+ ]
+ }
+ }
+ }
+ }
+
+ seen_bodies: list[dict[str, object]] = []
+
+ async def retry(method, url, headers, body): # noqa: ANN001
+ seen_bodies.append(body)
+ return FakeResponse(status_code=200, content=b"{}", json_data={"name": "batches/123"})
+
+ async def broken_store(batch_name, requests_list, model, api_key): # noqa: ANN001
+ raise RuntimeError("store failed")
+
+ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", payload)
+ monkeypatch.setattr(handler, "_gemini_contents_to_messages", fake_to_messages)
+ monkeypatch.setattr(handler, "_messages_to_gemini_contents", fake_to_gemini)
+ monkeypatch.setattr(handler, "_retry_request", retry)
+ monkeypatch.setattr(handler, "_store_google_batch_context", broken_store)
+
+ response = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")
+ assert response.status_code == 200
+ assert len(pipeline_calls) == 1
+ assert handler.metrics.record_calls[-1]["tokens_saved"] == 0
+ assert (
+ seen_bodies[0]["batch"]["input_config"]["requests"]["requests"][0]["metadata"]["key"]
+ == "empty"
+ )
+ optimized = seen_bodies[0]["batch"]["input_config"]["requests"]["requests"][2]["request"]
+ assert optimized["contents"][0] == {"parts": [{"text": "new"}]}
+ assert optimized["systemInstruction"] == {"parts": [{"text": "sys"}]}
+
+
+async def test_google_batch_passthrough_without_body_and_query_variants() -> None:
+ handler = DummyBatchHandler()
+ handler.http_client.post_response = FakeResponse(content=b"ok", headers={"x-upstream": "1"})
+
+ response = await handler._google_batch_passthrough(
+ FakeRequest("raw-body", headers={"host": "proxy"}, method="POST"),
+ "gemini-pro",
+ )
+ assert response.status_code == 200
+ assert handler.http_client.posts[-1]["content"] == b"raw-body"
+
+ handler.http_client.get_response = FakeResponse(content=b"{}", headers={"x-upstream": "2"})
+ passthrough = await handler.handle_google_batch_passthrough(
+ FakeRequest(
+ "{}",
+ headers={"host": "proxy", "x-goog-api-key": "secret"},
+ method="GET",
+ path="/v1beta/batches/b1",
+ ),
+ "b1",
+ )
+ assert passthrough.status_code == 200
+ assert (
+ handler.http_client.requests[-1]["url"]
+ == "https://gemini.example/v1beta/batches/b1?key=secret"
+ )
+
+
+async def test_batch_helper_methods_and_openai_file_error_branches() -> None:
+ handler = DummyBatchHandler()
+ marker = object()
+
+ async def fake_passthrough(request, base_url): # noqa: ANN001
+ return marker
+
+ handler.handle_passthrough = fake_passthrough
+ request = FakeRequest("{}")
+ assert await handler.handle_batch_list(request) is marker
+ assert await handler.handle_batch_get(request, "b1") is marker
+ assert await handler.handle_batch_cancel(request, "b1") is marker
+
+ handler.http_client.raise_get = RuntimeError("download boom")
+ assert await handler._download_openai_file("file-1", {}) is None
+
+ handler.http_client.raise_get = None
+ handler.http_client.post_response = FakeResponse(status_code=200, json_data={})
+ assert await handler._upload_openai_file("{}", "missing-id.jsonl", {}) is None
+
+
+async def test_store_google_batch_context_without_system_text(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ stored_contexts: list[object] = []
+
+ class FakeBatchContext:
+ def __init__(self, **kwargs) -> None: # noqa: ANN003
+ self.kwargs = kwargs
+ self.requests: list[object] = []
+
+ def add_request(self, request) -> None: # noqa: ANN001
+ self.requests.append(request)
+
+ class FakeBatchRequestContext:
+ def __init__(self, **kwargs) -> None: # noqa: ANN003
+ self.kwargs = kwargs
+
+ class FakeStore:
+ async def store(self, context) -> None: # noqa: ANN001
+ stored_contexts.append(context)
+
+ handler = DummyBatchHandler()
+ monkeypatch.setitem(
+ sys.modules,
+ "headroom.ccr",
+ SimpleNamespace(
+ BatchContext=FakeBatchContext,
+ BatchRequestContext=FakeBatchRequestContext,
+ get_batch_context_store=lambda: FakeStore(),
+ ),
+ )
+
+ await handler._store_google_batch_context(
+ "batches/456",
+ [
+ {
+ "request": {
+ "contents": [{"parts": [{"text": "hello"}]}],
+ "systemInstruction": {"parts": ["bad"]},
+ }
+ }
+ ],
+ "gemini-2.0",
+ None,
+ )
+
+ context = stored_contexts[0]
+ assert context.kwargs["api_key"] is None
+ assert context.requests[0].kwargs["custom_id"] == ""
+ assert context.requests[0].kwargs["system_instruction"] is None
+
+
+async def test_compress_batch_jsonl_skips_blank_lines_and_preserves_tools_when_not_injected(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_batch_support_modules(
+ monkeypatch,
+ injector_result=([{"role": "assistant", "content": "short"}], [{"name": "orig"}], False),
+ )
+ handler = DummyBatchHandler()
+ handler.config.optimize = True
+ handler.config.ccr_inject_tool = True
+ handler.openai_pipeline = SimpleNamespace(
+ apply=lambda **kwargs: SimpleNamespace(
+ messages=[{"role": "assistant", "content": "short"}],
+ tokens_before=50,
+ tokens_after=10,
+ )
+ )
+
+ lines, stats = await handler._compress_batch_jsonl(
+ "\n"
+ + json.dumps(
+ {
+ "body": {
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hello"}],
+ "tools": [{"name": "orig"}],
+ }
+ }
+ )
+ + "\n",
+ "req-extra",
+ )
+
+ assert len(lines) == 1
+ body = json.loads(lines[0])["body"]
+ assert body["tools"] == [{"name": "orig"}]
+ assert stats["total_requests"] == 1
+ assert stats["errors"] == 0