Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(proxy): route Gemini plugin traffic through native transforms (#2697)
Rod Boev0ad996a82eef
diff --git a/headroom/providers/proxy_routes.py b/headroom/providers/proxy_routes.pyindex d780b9420..643aee69b 100644--- a/headroom/providers/proxy_routes.py+++ b/headroom/providers/proxy_routes.py@@ -134,14 +134,21 @@ def _register_provider_handler_route(app: FastAPI, proxy: Any, spec: ProviderHanmodel: str = "",):handler = getattr(proxy, spec.handler_name)+ handler_kwargs: dict[str, str] = {}+ custom_base = request.headers.get("x-headroom-base-url", "").strip()+ if custom_base and spec.supports_custom_base_url:+ if not await is_safe_upstream_url_async(custom_base):+ logger.warning("rejecting unsafe x-headroom-base-url: %r", custom_base)+ raise HTTPException(status_code=400, detail="Rejected unsafe upstream base URL")+ handler_kwargs["upstream_base_url"] = custom_base.rstrip("/")if spec.path_param is None:- return await handler(request)+ return await handler(request, **handler_kwargs)path_args = {"batch_id": batch_id,"batch_name": batch_name,"model": model,}- return await handler(request, path_args[spec.path_param])+ return await handler(request, path_args[spec.path_param], **handler_kwargs)provider_handler.__name__ = (spec.handler_name.replace("handle_", "")diff --git a/headroom/providers/route_specs.py b/headroom/providers/route_specs.pyindex 28d9bf7d7..2b03c33ba 100644--- a/headroom/providers/route_specs.py+++ b/headroom/providers/route_specs.py@@ -23,6 +23,7 @@ class ProviderHandlerRoute:path: strhandler_name: strpath_param: str | None = None+ supports_custom_base_url: bool = FalseANTHROPIC_PASSTHROUGH_ROUTES: tuple[ProviderPassthroughRoute, ...] = (@@ -141,18 +142,21 @@ GEMINI_HANDLER_ROUTES: tuple[ProviderHandlerRoute, ...] = ("/v1beta/models/{model}:generateContent","handle_gemini_generate_content","model",+ True,),ProviderHandlerRoute("POST","/v1beta/models/{model}:streamGenerateContent","handle_gemini_stream_generate_content","model",+ True,),ProviderHandlerRoute("POST","/v1beta/models/{model}:countTokens","handle_gemini_count_tokens","model",+ True,),)diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pyindex e5ae2b668..b8778fffa 100644--- a/headroom/proxy/handlers/gemini.py+++ b/headroom/proxy/handlers/gemini.py@@ -10,6 +10,7 @@ import jsonimport loggingimport timefrom typing import TYPE_CHECKING, Any+from urllib.parse import unquote_plusif TYPE_CHECKING:from fastapi import Request@@ -152,6 +153,8 @@ class GeminiHandlerMixin:- functionResponse: Responses to function calls- executableCode / codeExecutionResult: Gemini code-execution parts,echoed back in contents[] on later turns+ - thought / thoughtSignature: thinking parts, which must round-trip intact+ - non-string text, which the text round trip cannot rebuildArgs:content: A single Gemini content entry with 'parts' list.@@ -160,7 +163,7 @@ class GeminiHandlerMixin:True if any part contains non-text data."""for part in self._dict_parts(content):- if any(+ if not isinstance(part.get("text", ""), str) or any(key in partfor key in ("inlineData",@@ -169,6 +172,8 @@ class GeminiHandlerMixin:"functionResponse","executableCode","codeExecutionResult",+ "thought",+ "thoughtSignature",)):return True@@ -195,7 +200,7 @@ class GeminiHandlerMixin:opt_iter = iter(optimized_contents)result: list[dict] = []for idx, content in enumerate(original_contents):- had_text = any("text" in p for p in self._dict_parts(content))+ had_text = any(isinstance(p.get("text"), str) for p in self._dict_parts(content))if idx in preserved_indices:result.append(preserved_contents[idx])if had_text:@@ -241,7 +246,7 @@ class GeminiHandlerMixin:# Add system instruction as system messageif system_instruction:sys_parts = self._dict_parts(system_instruction)- text_parts = [p.get("text", "") for p in sys_parts if "text" in p]+ text_parts = [p["text"] for p in sys_parts if isinstance(p.get("text"), str)]if text_parts:messages.append({"role": "system", "content": "\n".join(text_parts)})@@ -257,7 +262,7 @@ class GeminiHandlerMixin:role = "assistant"parts = self._dict_parts(content)- text_parts = [p.get("text", "") for p in parts if "text" in p]+ text_parts = [p["text"] for p in parts if isinstance(p.get("text"), str)]if text_parts:messages.append({"role": role, "content": "\n".join(text_parts)})@@ -329,6 +334,15 @@ class GeminiHandlerMixin:from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_jsonfrom headroom.utils import extract_user_query+ def _stream_url(base_url: str) -> str:+ query_parts = []+ for part in request.url.query.split("&") if request.url.query else []:+ key = unquote_plus(part.split("=", 1)[0])+ if key != "alt":+ query_parts.append(part)+ query_parts.append("alt=sse")+ return f"{base_url}?{'&'.join(query_parts)}"+start_time = time.time()request_id = await self._next_request_id()@@ -360,6 +374,9 @@ class GeminiHandlerMixin:)contents = body.get("contents", [])+ if not isinstance(contents, list):+ # Malformed contents: nothing to compress, so forward the body as-is+ contents = []headers = dict(request.headers.items())headers.pop("host", None)@@ -494,16 +511,11 @@ class GeminiHandlerMixin:if is_streaming:if upstream_base_url:- stream_url = url- separator = "&" if "?" in stream_url else "?"- if "alt=" not in request.url.query:- stream_url = f"{stream_url}{separator}alt=sse"+ stream_url = _stream_url(url.split("?", 1)[0])else:- stream_url = (- f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"+ stream_url = _stream_url(+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent")- if "key" in query_params and not upstream_base_url:- stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"return await self._stream_response(stream_url,headers,@@ -828,16 +840,11 @@ class GeminiHandlerMixin:if is_streaming:# For streaming, use streamGenerateContent endpointif upstream_base_url:- stream_url = url- separator = "&" if "?" in stream_url else "?"- if "alt=" not in request.url.query:- stream_url = f"{stream_url}{separator}alt=sse"+ stream_url = _stream_url(url.split("?", 1)[0])else:- stream_url = (- f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"+ stream_url = _stream_url(+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent")- if "key" in query_params and not upstream_base_url:- stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"return await self._stream_response(stream_url,@@ -1314,95 +1321,13 @@ class GeminiHandlerMixin:self,request: Request,model: str,- ) -> StreamingResponse | JSONResponse:+ upstream_base_url: str | None = None,+ ) -> Response | StreamingResponse | JSONResponse:"""Handle Gemini streaming endpoint /v1beta/models/{model}:streamGenerateContent."""- from fastapi import HTTPException- from fastapi.responses import JSONResponse-- from headroom.proxy.helpers import _read_request_json-- start_time = time.time()- request_id = await self._next_request_id()-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "code": 400,- }- },- )-- contents = body.get("contents", [])-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)- tags = extract_tags(headers)- from headroom.proxy.savings_attribution import bind_scope-- bind_scope(tags, request.scope)- # Streaming variant — delegates to _stream_response which- # classifies the client itself from headers.- # PR-A5 (P5-49): strip internal x-headroom-* before forwarding upstream.- from headroom.proxy.helpers import _strip_internal_headers, log_outbound_headers-- _pre_strip_count_gem_stream = sum(1 for k in headers if k.lower().startswith("x-headroom-"))- headers = _strip_internal_headers(headers)- log_outbound_headers(- forwarder="gemini_stream_generate_content",- stripped_count=_pre_strip_count_gem_stream,- request_id=request_id,- )-- # Budget check- cost_tracker = self.cost_tracker- if cost_tracker:- allowed, remaining = cost_tracker.check_budget()- if not allowed:- raise HTTPException(- status_code=429,- detail=cost_tracker.budget_denial_detail(),- )-- # Token counting (offloaded off the event loop — GH #1701). Reuse the- # shared _dict_parts coercion and keep only str text values: count_text- # raises on a non-str part value and the fail-open path re-runs the same- # input, so a malformed part would otherwise 500 the streaming request.- text_parts = [- part["text"]- for content in (contents if isinstance(contents, list) else [])- for part in self._dict_parts(content)- if isinstance(part.get("text"), str)- ]- _, original_tokens = await self._count_texts_offloaded(model, text_parts)-- optimization_latency = (time.time() - start_time) * 1000-- # Build URL with SSE param- query_params = dict(request.query_params)- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"- if "key" in query_params:- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"-- return await self._stream_response(- url,- headers,- body,- "gemini",+ return await self.handle_gemini_generate_content(+ request,model,- request_id,- original_tokens,- original_tokens,- 0, # tokens_saved- [], # transforms_applied- tags,- optimization_latency,+ upstream_base_url=upstream_base_url,)async def handle_gemini_count_tokens(diff --git a/tests/test_provider_route_specs.py b/tests/test_provider_route_specs.pyindex fdb4ae029..bf860c5fb 100644--- a/tests/test_provider_route_specs.py+++ b/tests/test_provider_route_specs.py@@ -93,18 +93,21 @@ def test_direct_handler_routes_model_endpoint_intent() -> None:"/v1beta/models/{model}:generateContent","handle_gemini_generate_content","model",+ True,),ProviderHandlerRoute("POST","/v1beta/models/{model}:streamGenerateContent","handle_gemini_stream_generate_content","model",+ True,),ProviderHandlerRoute("POST","/v1beta/models/{model}:countTokens","handle_gemini_count_tokens","model",+ True,),)assert CLOUDCODE_HANDLER_ROUTES == (@@ -121,6 +124,31 @@ def test_direct_handler_routes_model_endpoint_intent() -> None:)+def test_gemini_model_routes_all_follow_the_tagged_upstream() -> None:+ """All three Gemini model endpoints must agree on which upstream they reach.++ Counting tokens against one host and generating against another gives the+ client counts for a model the serving host may not have, and sends the+ tagged gateway's credential to Google.+ """+ assert {+ (spec.path, spec.handler_name)+ for spec in GEMINI_HANDLER_ROUTES+ if spec.supports_custom_base_url+ } == {+ ("/v1beta/models/{model}:generateContent", "handle_gemini_generate_content"),+ ("/v1beta/models/{model}:streamGenerateContent", "handle_gemini_stream_generate_content"),+ ("/v1beta/models/{model}:countTokens", "handle_gemini_count_tokens"),+ }+++def test_non_gemini_handler_routes_do_not_opt_into_the_custom_base() -> None:+ for spec in PROVIDER_HANDLER_ROUTES:+ if spec in GEMINI_HANDLER_ROUTES:+ continue+ assert not spec.supports_custom_base_url, spec.path++def test_batch_handler_routes_model_endpoint_intent() -> None:assert ANTHROPIC_BATCH_ROUTES[0] == ProviderHandlerRoute("POST",diff --git a/tests/test_proxy/test_gemini_plugin_routed_streaming.py b/tests/test_proxy/test_gemini_plugin_routed_streaming.pynew file mode 100644index 000000000..950c569e0--- /dev/null+++ b/tests/test_proxy/test_gemini_plugin_routed_streaming.py@@ -0,0 +1,577 @@+"""Production-route regressions for native Gemini model requests."""++from __future__ import annotations++from unittest.mock import AsyncMock++import httpx+import pytest++pytest.importorskip("fastapi")++from fastapi import Request # noqa: E402+from fastapi.responses import JSONResponse, StreamingResponse # noqa: E402+from fastapi.testclient import TestClient # noqa: E402++from headroom.compress import CompressResult # noqa: E402+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++# The OpenCode transport sets x-headroom-base-url to the upstream *origin* and+# forwards the upstream pathname verbatim, so a custom Google endpoint arrives+# as origin + /v1beta/models/{model}:... on the proxy.+PLUGIN_BASE_URL = "https://gateway.example"+DIRECT_STREAM_URL = (+ "https://generativelanguage.googleapis.com/v1beta/models/"+ "gemini-2.5-flash:streamGenerateContent?alt=sse"+)+PLUGIN_STREAM_URL = (+ "https://gateway.example/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse"+)+DIRECT_KEY_STREAM_URL = (+ "https://generativelanguage.googleapis.com/v1beta/models/"+ "gemini-2.5-flash:streamGenerateContent?key=secret&alt=sse"+)+PLUGIN_QUERY_STREAM_URL = "https://gateway.example/v1beta/models/gemini-2.5-flash:streamGenerateContent?trace=1&foo=bar&alt=sse"+DIRECT_ENCODED_QUERY_STREAM_URL = (+ "https://generativelanguage.googleapis.com/v1beta/models/"+ "gemini-2.5-flash:streamGenerateContent?trace=hello%20world&encoded=%7E&alt=sse"+)++[email protected](autouse=True)+def _allow_fake_plugin_gateway(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_ALLOWED_BASE_URLS", "gateway.example")+++def _config() -> ProxyConfig:+ return ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )+++def _streaming_optimize_config() -> ProxyConfig:+ return ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )+++def test_plugin_generate_content_uses_native_gemini_route_and_preserves_body() -> None:+ body = {+ "contents": [+ {+ "role": "user",+ "parts": [+ {"text": "Keep this prompt."},+ {"inlineData": {"mimeType": "image/png", "data": "aW1hZ2U="}},+ {"functionCall": {"name": "lookup", "args": {"city": "Paris"}}},+ ],+ }+ ],+ "generationConfig": {"temperature": 0.2},+ }+ upstream = {+ "candidates": [{"content": {"parts": [{"text": "answer"}]}}],+ "usageMetadata": {"promptTokenCount": 42, "candidatesTokenCount": 7},+ }+ captured: dict[str, object] = {}++ async def fake_retry(+ method: str,+ url: str,+ headers: dict[str, str],+ request_body: dict,+ **_: object,+ ) -> httpx.Response:+ captured.update(method=method, url=url, headers=headers, body=request_body)+ return httpx.Response(200, json=upstream)++ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy._retry_request = fake_retry+ response = client.post(+ "/v1beta/models/gemini-2.5-flash:generateContent",+ headers={+ "x-headroom-base-url": PLUGIN_BASE_URL,+ },+ json=body,+ )++ assert response.status_code == 200, response.text+ assert response.json() == upstream+ assert captured["method"] == "POST"+ assert captured["url"] == (+ "https://gateway.example/v1beta/models/gemini-2.5-flash:generateContent"+ )+ assert captured["body"] == body+ outbound_headers = captured["headers"]+ assert isinstance(outbound_headers, dict)+ assert not any(key.lower().startswith("x-headroom-") for key in outbound_headers)+++ ("headers", "expected_url"),+ [+ ({}, DIRECT_STREAM_URL),+ ({"x-headroom-base-url": PLUGIN_BASE_URL}, PLUGIN_STREAM_URL),+ ],+ ids=["direct", "plugin"],+)+def test_stream_generate_content_preserves_sse_and_native_parts(+ headers: dict[str, str], expected_url: str+) -> None:+ body = {+ "contents": [+ {"role": "user", "parts": [{"text": "Keep this prompt."}]},+ {+ "role": "model",+ "parts": [+ {"functionCall": {"name": "lookup", "args": {"city": "Paris"}}},+ {"inlineData": {"mimeType": "image/png", "data": "aW1hZ2U="}},+ ],+ },+ ]+ }+ sse_bytes = (+ b'data: {"candidates":[{"content":{"parts":[{"text":"answer"}]}}],'+ b'"usageMetadata":{"promptTokenCount":42,"candidatesTokenCount":7}}\n\n'+ )+ captured: dict[str, object] = {}++ async def fake_stream_response(+ url: str,+ headers: dict[str, str],+ request_body: dict,+ *_: object,+ **__: object,+ ) -> StreamingResponse:+ captured.update(url=url, headers=headers, body=request_body)+ return StreamingResponse(iter([sse_bytes]), media_type="text/event-stream")++ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy._stream_response = fake_stream_response+ response = client.post(+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",+ headers=headers,+ json=body,+ )++ assert response.status_code == 200, response.text+ assert response.content == sse_bytes+ assert captured["url"] == expected_url+ assert captured["body"] == body+ outbound_headers = captured["headers"]+ assert isinstance(outbound_headers, dict)+ assert not any(key.lower().startswith("x-headroom-") for key in outbound_headers)+++def test_adjacent_custom_base_path_stays_passthrough(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_ALLOWED_BASE_URLS", PLUGIN_BASE_URL)+ captured: dict[str, object] = {}++ async def fake_passthrough(+ request: Request,+ base_url: str,+ sub_path: str = "",+ provider_name: str = "",+ ) -> dict[str, object]:+ captured.update(+ path=request.url.path,+ base_url=base_url,+ sub_path=sub_path,+ provider_name=provider_name,+ )+ return {"handler": "passthrough"}++ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy.handle_passthrough = fake_passthrough+ response = client.post(+ "/v1/adjacent-provider-path",+ headers={"x-headroom-base-url": PLUGIN_BASE_URL},+ content=b"adjacent",+ )++ assert response.status_code == 200, response.text+ assert captured == {+ "path": "/v1/adjacent-provider-path",+ "base_url": PLUGIN_BASE_URL,+ "sub_path": "",+ "provider_name": "",+ }+++ ("headers", "expected_url"),+ [+ ({}, DIRECT_STREAM_URL),+ ({"x-headroom-base-url": PLUGIN_BASE_URL}, PLUGIN_STREAM_URL),+ ],+ ids=["direct", "plugin"],+)+def test_native_streaming_reaches_the_transform_pipeline(+ headers: dict[str, str], expected_url: str+) -> None:+ body = {+ "contents": [+ {"role": "user", "parts": [{"text": "compress me " * 200}]},+ ]+ }+ captured: dict[str, object] = {}++ def fake_apply(**kwargs: object) -> CompressResult:+ messages = kwargs["messages"]+ assert isinstance(messages, list)+ captured["pipeline_messages"] = messages+ return CompressResult(+ messages=[{"role": "user", "content": "compressed"}],+ tokens_before=1000,+ tokens_after=10,+ tokens_saved=990,+ compression_ratio=0.99,+ transforms_applied=["smart_crusher"],+ )++ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:+ captured["url"] = args[0]+ captured["body"] = args[2]+ captured["transforms_applied"] = args[9]+ return StreamingResponse(iter([b"data: {}\n\n"]), media_type="text/event-stream")++ app = create_app(_streaming_optimize_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy.openai_pipeline.apply = fake_apply+ proxy._stream_response = fake_stream_response+ response = client.post(+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",+ headers=headers,+ json=body,+ )++ assert response.status_code == 200, response.text+ assert captured["transforms_applied"] == ["smart_crusher"]+ assert captured["url"] == expected_url+ outbound = captured["body"]+ assert isinstance(outbound, dict)+ assert outbound["contents"] == [{"role": "user", "parts": [{"text": "compressed"}]}]+ assert captured["pipeline_messages"]+++ ("headers", "expected_url"),+ [+ ({}, DIRECT_STREAM_URL),+ ({"x-headroom-base-url": PLUGIN_BASE_URL}, PLUGIN_STREAM_URL),+ ],+ ids=["direct", "plugin"],+)+def test_native_streaming_transforms_text_and_preserves_native_parts(+ headers: dict[str, str], expected_url: str+) -> None:+ body = {+ "contents": [+ {+ "role": "user",+ "parts": [{"text": "compress me " * 200}],+ },+ {+ "role": "model",+ "parts": [+ {"functionCall": {"name": "lookup", "args": {"city": "Paris"}}},+ {"inlineData": {"mimeType": "image/png", "data": "aW1hZ2U="}},+ ],+ },+ ]+ }+ captured: dict[str, object] = {}++ def fake_apply(**kwargs: object) -> CompressResult:+ messages = kwargs["messages"]+ assert isinstance(messages, list)+ captured["pipeline_messages"] = messages+ return CompressResult(+ messages=[{"role": "user", "content": "compressed"}],+ tokens_before=1000,+ tokens_after=10,+ tokens_saved=990,+ compression_ratio=0.99,+ transforms_applied=["smart_crusher"],+ )++ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:+ captured["url"] = args[0]+ captured["body"] = args[2]+ captured["transforms_applied"] = args[9]+ return StreamingResponse(iter([b"data: mixed\n\n"]), media_type="text/event-stream")++ app = create_app(_streaming_optimize_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy.openai_pipeline.apply = fake_apply+ proxy._stream_response = fake_stream_response+ response = client.post(+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",+ headers=headers,+ json=body,+ )++ assert response.status_code == 200, response.text+ assert response.content == b"data: mixed\n\n"+ assert captured["url"] == expected_url+ assert captured["transforms_applied"] == ["smart_crusher"]+ outbound = captured["body"]+ assert isinstance(outbound, dict)+ assert outbound["contents"] == [+ {"role": "user", "parts": [{"text": "compressed"}]},+ {+ "role": "model",+ "parts": [+ {"functionCall": {"name": "lookup", "args": {"city": "Paris"}}},+ {"inlineData": {"mimeType": "image/png", "data": "aW1hZ2U="}},+ ],+ },+ ]+++ ("headers", "query", "all_non_text", "expected_url"),+ [+ ({}, "key=secret&alt=sse", False, DIRECT_KEY_STREAM_URL),+ (+ {"x-headroom-base-url": PLUGIN_BASE_URL},+ "trace=1&foo=bar",+ False,+ PLUGIN_QUERY_STREAM_URL,+ ),+ ({}, "trace=hello%20world&encoded=%7E&alt=raw", False, DIRECT_ENCODED_QUERY_STREAM_URL),+ ({}, "key=secret&alt=sse", True, DIRECT_KEY_STREAM_URL),+ (+ {"x-headroom-base-url": PLUGIN_BASE_URL},+ "trace=1&foo=bar",+ True,+ PLUGIN_QUERY_STREAM_URL,+ ),+ ],+ ids=[+ "direct-normal",+ "plugin-normal",+ "direct-encoded-query",+ "direct-all-native",+ "plugin-all-native",+ ],+)+def test_native_streaming_preserves_query_and_adds_sse_once(+ headers: dict[str, str], query: str, all_non_text: bool, expected_url: str+) -> None:+ parts = (+ [{"functionCall": {"name": "lookup", "args": {"city": "Paris"}}}]+ if all_non_text+ else [{"text": "keep this"}]+ )+ body = {"contents": [{"role": "user", "parts": parts}]}+ captured: dict[str, object] = {}++ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:+ captured["url"] = args[0]+ return StreamingResponse(iter([b"data: query\n\n"]), media_type="text/event-stream")++ app = create_app(_streaming_optimize_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy._stream_response = fake_stream_response+ response = client.post(+ f"/v1beta/models/gemini-2.5-flash:streamGenerateContent?{query}",+ headers=headers,+ json=body,+ )++ assert response.status_code == 200, response.text+ assert response.content == b"data: query\n\n"+ assert captured["url"] == expected_url+++ ("path", "handler_name"),+ [+ ("/v1beta/models/gemini-2.5-flash:generateContent", "handle_gemini_generate_content"),+ (+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",+ "handle_gemini_stream_generate_content",+ ),+ ("/v1beta/models/gemini-2.5-flash:countTokens", "handle_gemini_count_tokens"),+ ],+ ids=["generate", "stream", "count"],+)+def test_native_gemini_routes_reject_loopback_custom_base_before_handler(+ monkeypatch: pytest.MonkeyPatch, path: str, handler_name: str+) -> None:+ monkeypatch.delenv("HEADROOM_ALLOWED_BASE_URLS", raising=False)+ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ handler = AsyncMock(return_value=JSONResponse({"ok": True}))+ setattr(proxy, handler_name, handler)+ response = client.post(+ path,+ headers={"x-headroom-base-url": "http://127.0.0.1:12345"},+ json={"contents": []},+ )++ assert response.status_code == 400+ handler.assert_not_awaited()+++def test_gemini_count_tokens_follows_the_same_tagged_upstream() -> None:+ """Counting and generating must agree on the upstream.++ Otherwise the client counts tokens at Google for a model the tagged gateway+ serves, and the gateway's key travels to Google.+ """+ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy.handle_gemini_count_tokens = AsyncMock(return_value=JSONResponse({"ok": True}))+ response = client.post(+ "/v1beta/models/gemini-2.5-flash:countTokens",+ headers={"x-headroom-base-url": PLUGIN_BASE_URL},+ json={"contents": []},+ )++ assert response.status_code == 200, response.text+ proxy.handle_gemini_count_tokens.assert_awaited_once()+ call = proxy.handle_gemini_count_tokens.await_args+ assert call is not None+ assert call.args[1] == "gemini-2.5-flash"+ assert call.kwargs == {"upstream_base_url": PLUGIN_BASE_URL}+++def test_gemini_count_tokens_without_the_header_stays_on_the_default_upstream() -> None:+ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy.handle_gemini_count_tokens = AsyncMock(return_value=JSONResponse({"ok": True}))+ response = client.post(+ "/v1beta/models/gemini-2.5-flash:countTokens",+ json={"contents": []},+ )++ assert response.status_code == 200, response.text+ call = proxy.handle_gemini_count_tokens.await_args+ assert call is not None+ assert call.kwargs == {}+++THINKING_PART = {+ "text": "internal reasoning",+ "thought": True,+ "thoughtSignature": "signed-reasoning-token",+}+++ ("path", "headers"),+ [+ ("/v1beta/models/gemini-2.5-flash:streamGenerateContent", {}),+ (+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",+ {"x-headroom-base-url": PLUGIN_BASE_URL},+ ),+ ("/v1beta/models/gemini-2.5-flash:generateContent", {}),+ (+ "/v1beta/models/gemini-2.5-flash:generateContent",+ {"x-headroom-base-url": PLUGIN_BASE_URL},+ ),+ ],+ ids=["stream-direct", "stream-plugin", "generate-direct", "generate-plugin"],+)+def test_thinking_parts_survive_compression_elsewhere(path: str, headers: dict[str, str]) -> None:+ body = {+ "contents": [+ {"role": "user", "parts": [{"text": "compress me " * 200}]},+ {"role": "model", "parts": [dict(THINKING_PART), {"text": "visible answer"}]},+ ]+ }+ captured: dict[str, object] = {}++ def fake_apply(**kwargs: object) -> CompressResult:+ messages = kwargs["messages"]+ assert isinstance(messages, list)+ return CompressResult(+ messages=[{"role": "user", "content": "compressed"}, *messages[1:]],+ tokens_before=1000,+ tokens_after=10,+ tokens_saved=990,+ compression_ratio=0.99,+ transforms_applied=["smart_crusher"],+ )++ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:+ captured["body"] = args[2]+ return StreamingResponse(iter([b"data: {}\n\n"]), media_type="text/event-stream")++ async def fake_retry(+ method: str, url: str, headers: dict[str, str], request_body: dict, **_: object+ ) -> httpx.Response:+ captured["body"] = request_body+ return httpx.Response(200, json={"candidates": []})++ app = create_app(_streaming_optimize_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy.openai_pipeline.apply = fake_apply+ proxy._stream_response = fake_stream_response+ proxy._retry_request = fake_retry+ response = client.post(path, headers=headers, json=body)++ assert response.status_code == 200, response.text+ outbound = captured["body"]+ assert isinstance(outbound, dict)+ assert outbound["contents"] == [+ {"role": "user", "parts": [{"text": "compressed"}]},+ {"role": "model", "parts": [THINKING_PART, {"text": "visible answer"}]},+ ]+++ "body",+ [+ {"contents": None},+ {"contents": [{"role": "user", "parts": [{"text": 42}]}]},+ ],+ ids=["null-contents", "numeric-text"],+)+def test_malformed_stream_body_is_forwarded(body: dict) -> None:+ captured: dict[str, object] = {}++ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:+ captured["body"] = args[2]+ return StreamingResponse(iter([b"data: {}\n\n"]), media_type="text/event-stream")++ app = create_app(_streaming_optimize_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy._stream_response = fake_stream_response+ response = client.post("/v1beta/models/gemini-2.5-flash:streamGenerateContent", json=body)++ assert response.status_code == 200, response.text+ assert captured["body"] == bodydiff --git a/tests/test_tokenizer_count_offload.py b/tests/test_tokenizer_count_offload.pyindex 77db5a4d8..56b8fdcc4 100644--- a/tests/test_tokenizer_count_offload.py+++ b/tests/test_tokenizer_count_offload.py@@ -72,13 +72,10 @@ def test_handlers_offload_token_counting_and_batch_apply() -> None:fn = GeminiHandlerMixin.handle_gemini_stream_generate_contentassert inspect.iscoroutinefunction(fn)src = inspect.getsource(fn)- assert "_count_texts_offloaded(" in src, "streaming Gemini text counting not offloaded"- assert "tokenizer = get_tokenizer(" not in src, "tokenizer resolved inline on the loop"- assert "count_text(" not in src, "streaming Gemini count_text still runs on the loop"- assert "_dict_parts(" in src, "streaming Gemini must reuse the shared _dict_parts coercion"- assert 'isinstance(part.get("text"), str)' in src, (- "streaming Gemini must skip non-str text so count_text can't 500"- )+ assert "return await self.handle_gemini_generate_content(" in src+ assert "upstream_base_url=upstream_base_url" in src+ assert "_count_texts_offloaded(" not in src+ assert "_dict_parts(" not in srcfor mixin, method in ((AnthropicHandlerMixin, "handle_anthropic_batch_create"),