Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(proxy): route Gemini plugin traffic through native transforms (#2697)

Rod Boev0ad996a82eef
diff --git a/headroom/providers/proxy_routes.py b/headroom/providers/proxy_routes.py
index d780b9420..643aee69b 100644
--- a/headroom/providers/proxy_routes.py
+++ b/headroom/providers/proxy_routes.py
@@ -134,14 +134,21 @@ def _register_provider_handler_route(app: FastAPI, proxy: Any, spec: ProviderHan
model: str = "",
):
handler = getattr(proxy, spec.handler_name)
+ handler_kwargs: dict[str, str] = {}
+ custom_base = request.headers.get("x-headroom-base-url", "").strip()
+ if custom_base and spec.supports_custom_base_url:
+ if not await is_safe_upstream_url_async(custom_base):
+ logger.warning("rejecting unsafe x-headroom-base-url: %r", custom_base)
+ raise HTTPException(status_code=400, detail="Rejected unsafe upstream base URL")
+ handler_kwargs["upstream_base_url"] = custom_base.rstrip("/")
if spec.path_param is None:
- return await handler(request)
+ return await handler(request, **handler_kwargs)
path_args = {
"batch_id": batch_id,
"batch_name": batch_name,
"model": model,
}
- return await handler(request, path_args[spec.path_param])
+ return await handler(request, path_args[spec.path_param], **handler_kwargs)
provider_handler.__name__ = (
spec.handler_name.replace("handle_", "")
diff --git a/headroom/providers/route_specs.py b/headroom/providers/route_specs.py
index 28d9bf7d7..2b03c33ba 100644
--- a/headroom/providers/route_specs.py
+++ b/headroom/providers/route_specs.py
@@ -23,6 +23,7 @@ class ProviderHandlerRoute:
path: str
handler_name: str
path_param: str | None = None
+ supports_custom_base_url: bool = False
ANTHROPIC_PASSTHROUGH_ROUTES: tuple[ProviderPassthroughRoute, ...] = (
@@ -141,18 +142,21 @@ GEMINI_HANDLER_ROUTES: tuple[ProviderHandlerRoute, ...] = (
"/v1beta/models/{model}:generateContent",
"handle_gemini_generate_content",
"model",
+ True,
),
ProviderHandlerRoute(
"POST",
"/v1beta/models/{model}:streamGenerateContent",
"handle_gemini_stream_generate_content",
"model",
+ True,
),
ProviderHandlerRoute(
"POST",
"/v1beta/models/{model}:countTokens",
"handle_gemini_count_tokens",
"model",
+ True,
),
)
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.py
index e5ae2b668..b8778fffa 100644
--- a/headroom/proxy/handlers/gemini.py
+++ b/headroom/proxy/handlers/gemini.py
@@ -10,6 +10,7 @@ import json
import logging
import time
from typing import TYPE_CHECKING, Any
+from urllib.parse import unquote_plus
if TYPE_CHECKING:
from fastapi import Request
@@ -152,6 +153,8 @@ class GeminiHandlerMixin:
- functionResponse: Responses to function calls
- executableCode / codeExecutionResult: Gemini code-execution parts,
echoed back in contents[] on later turns
+ - thought / thoughtSignature: thinking parts, which must round-trip intact
+ - non-string text, which the text round trip cannot rebuild
Args:
content: A single Gemini content entry with 'parts' list.
@@ -160,7 +163,7 @@ class GeminiHandlerMixin:
True if any part contains non-text data.
"""
for part in self._dict_parts(content):
- if any(
+ if not isinstance(part.get("text", ""), str) or any(
key in part
for key in (
"inlineData",
@@ -169,6 +172,8 @@ class GeminiHandlerMixin:
"functionResponse",
"executableCode",
"codeExecutionResult",
+ "thought",
+ "thoughtSignature",
)
):
return True
@@ -195,7 +200,7 @@ class GeminiHandlerMixin:
opt_iter = iter(optimized_contents)
result: list[dict] = []
for idx, content in enumerate(original_contents):
- had_text = any("text" in p for p in self._dict_parts(content))
+ had_text = any(isinstance(p.get("text"), str) for p in self._dict_parts(content))
if idx in preserved_indices:
result.append(preserved_contents[idx])
if had_text:
@@ -241,7 +246,7 @@ class GeminiHandlerMixin:
# Add system instruction as system message
if system_instruction:
sys_parts = self._dict_parts(system_instruction)
- text_parts = [p.get("text", "") for p in sys_parts if "text" in p]
+ text_parts = [p["text"] for p in sys_parts if isinstance(p.get("text"), str)]
if text_parts:
messages.append({"role": "system", "content": "\n".join(text_parts)})
@@ -257,7 +262,7 @@ class GeminiHandlerMixin:
role = "assistant"
parts = self._dict_parts(content)
- text_parts = [p.get("text", "") for p in parts if "text" in p]
+ text_parts = [p["text"] for p in parts if isinstance(p.get("text"), str)]
if text_parts:
messages.append({"role": role, "content": "\n".join(text_parts)})
@@ -329,6 +334,15 @@ class GeminiHandlerMixin:
from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_json
from headroom.utils import extract_user_query
+ def _stream_url(base_url: str) -> str:
+ query_parts = []
+ for part in request.url.query.split("&") if request.url.query else []:
+ key = unquote_plus(part.split("=", 1)[0])
+ if key != "alt":
+ query_parts.append(part)
+ query_parts.append("alt=sse")
+ return f"{base_url}?{'&'.join(query_parts)}"
+
start_time = time.time()
request_id = await self._next_request_id()
@@ -360,6 +374,9 @@ class GeminiHandlerMixin:
)
contents = body.get("contents", [])
+ if not isinstance(contents, list):
+ # Malformed contents: nothing to compress, so forward the body as-is
+ contents = []
headers = dict(request.headers.items())
headers.pop("host", None)
@@ -494,16 +511,11 @@ class GeminiHandlerMixin:
if is_streaming:
if upstream_base_url:
- stream_url = url
- separator = "&" if "?" in stream_url else "?"
- if "alt=" not in request.url.query:
- stream_url = f"{stream_url}{separator}alt=sse"
+ stream_url = _stream_url(url.split("?", 1)[0])
else:
- stream_url = (
- f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
+ stream_url = _stream_url(
+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent"
)
- if "key" in query_params and not upstream_base_url:
- stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
return await self._stream_response(
stream_url,
headers,
@@ -828,16 +840,11 @@ class GeminiHandlerMixin:
if is_streaming:
# For streaming, use streamGenerateContent endpoint
if upstream_base_url:
- stream_url = url
- separator = "&" if "?" in stream_url else "?"
- if "alt=" not in request.url.query:
- stream_url = f"{stream_url}{separator}alt=sse"
+ stream_url = _stream_url(url.split("?", 1)[0])
else:
- stream_url = (
- f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
+ stream_url = _stream_url(
+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent"
)
- if "key" in query_params and not upstream_base_url:
- stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
return await self._stream_response(
stream_url,
@@ -1314,95 +1321,13 @@ class GeminiHandlerMixin:
self,
request: Request,
model: str,
- ) -> StreamingResponse | JSONResponse:
+ upstream_base_url: str | None = None,
+ ) -> Response | StreamingResponse | JSONResponse:
"""Handle Gemini streaming endpoint /v1beta/models/{model}:streamGenerateContent."""
- from fastapi import HTTPException
- from fastapi.responses import JSONResponse
-
- from headroom.proxy.helpers import _read_request_json
-
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "code": 400,
- }
- },
- )
-
- contents = body.get("contents", [])
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- tags = extract_tags(headers)
- from headroom.proxy.savings_attribution import bind_scope
-
- bind_scope(tags, request.scope)
- # Streaming variant — delegates to _stream_response which
- # classifies the client itself from headers.
- # PR-A5 (P5-49): strip internal x-headroom-* before forwarding upstream.
- from headroom.proxy.helpers import _strip_internal_headers, log_outbound_headers
-
- _pre_strip_count_gem_stream = sum(1 for k in headers if k.lower().startswith("x-headroom-"))
- headers = _strip_internal_headers(headers)
- log_outbound_headers(
- forwarder="gemini_stream_generate_content",
- stripped_count=_pre_strip_count_gem_stream,
- request_id=request_id,
- )
-
- # Budget check
- cost_tracker = self.cost_tracker
- if cost_tracker:
- allowed, remaining = cost_tracker.check_budget()
- if not allowed:
- raise HTTPException(
- status_code=429,
- detail=cost_tracker.budget_denial_detail(),
- )
-
- # Token counting (offloaded off the event loop — GH #1701). Reuse the
- # shared _dict_parts coercion and keep only str text values: count_text
- # raises on a non-str part value and the fail-open path re-runs the same
- # input, so a malformed part would otherwise 500 the streaming request.
- text_parts = [
- part["text"]
- for content in (contents if isinstance(contents, list) else [])
- for part in self._dict_parts(content)
- if isinstance(part.get("text"), str)
- ]
- _, original_tokens = await self._count_texts_offloaded(model, text_parts)
-
- optimization_latency = (time.time() - start_time) * 1000
-
- # Build URL with SSE param
- query_params = dict(request.query_params)
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
- if "key" in query_params:
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
-
- return await self._stream_response(
- url,
- headers,
- body,
- "gemini",
+ return await self.handle_gemini_generate_content(
+ request,
model,
- request_id,
- original_tokens,
- original_tokens,
- 0, # tokens_saved
- [], # transforms_applied
- tags,
- optimization_latency,
+ upstream_base_url=upstream_base_url,
)
async def handle_gemini_count_tokens(
diff --git a/tests/test_provider_route_specs.py b/tests/test_provider_route_specs.py
index fdb4ae029..bf860c5fb 100644
--- a/tests/test_provider_route_specs.py
+++ b/tests/test_provider_route_specs.py
@@ -93,18 +93,21 @@ def test_direct_handler_routes_model_endpoint_intent() -> None:
"/v1beta/models/{model}:generateContent",
"handle_gemini_generate_content",
"model",
+ True,
),
ProviderHandlerRoute(
"POST",
"/v1beta/models/{model}:streamGenerateContent",
"handle_gemini_stream_generate_content",
"model",
+ True,
),
ProviderHandlerRoute(
"POST",
"/v1beta/models/{model}:countTokens",
"handle_gemini_count_tokens",
"model",
+ True,
),
)
assert CLOUDCODE_HANDLER_ROUTES == (
@@ -121,6 +124,31 @@ def test_direct_handler_routes_model_endpoint_intent() -> None:
)
+def test_gemini_model_routes_all_follow_the_tagged_upstream() -> None:
+ """All three Gemini model endpoints must agree on which upstream they reach.
+
+ Counting tokens against one host and generating against another gives the
+ client counts for a model the serving host may not have, and sends the
+ tagged gateway's credential to Google.
+ """
+ assert {
+ (spec.path, spec.handler_name)
+ for spec in GEMINI_HANDLER_ROUTES
+ if spec.supports_custom_base_url
+ } == {
+ ("/v1beta/models/{model}:generateContent", "handle_gemini_generate_content"),
+ ("/v1beta/models/{model}:streamGenerateContent", "handle_gemini_stream_generate_content"),
+ ("/v1beta/models/{model}:countTokens", "handle_gemini_count_tokens"),
+ }
+
+
+def test_non_gemini_handler_routes_do_not_opt_into_the_custom_base() -> None:
+ for spec in PROVIDER_HANDLER_ROUTES:
+ if spec in GEMINI_HANDLER_ROUTES:
+ continue
+ assert not spec.supports_custom_base_url, spec.path
+
+
def test_batch_handler_routes_model_endpoint_intent() -> None:
assert ANTHROPIC_BATCH_ROUTES[0] == ProviderHandlerRoute(
"POST",
diff --git a/tests/test_proxy/test_gemini_plugin_routed_streaming.py b/tests/test_proxy/test_gemini_plugin_routed_streaming.py
new file mode 100644
index 000000000..950c569e0
--- /dev/null
+++ b/tests/test_proxy/test_gemini_plugin_routed_streaming.py
@@ -0,0 +1,577 @@
+"""Production-route regressions for native Gemini model requests."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi import Request # noqa: E402
+from fastapi.responses import JSONResponse, StreamingResponse # noqa: E402
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.compress import CompressResult # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+# The OpenCode transport sets x-headroom-base-url to the upstream *origin* and
+# forwards the upstream pathname verbatim, so a custom Google endpoint arrives
+# as origin + /v1beta/models/{model}:... on the proxy.
+PLUGIN_BASE_URL = "https://gateway.example"
+DIRECT_STREAM_URL = (
+ "https://generativelanguage.googleapis.com/v1beta/models/"
+ "gemini-2.5-flash:streamGenerateContent?alt=sse"
+)
+PLUGIN_STREAM_URL = (
+ "https://gateway.example/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse"
+)
+DIRECT_KEY_STREAM_URL = (
+ "https://generativelanguage.googleapis.com/v1beta/models/"
+ "gemini-2.5-flash:streamGenerateContent?key=secret&alt=sse"
+)
+PLUGIN_QUERY_STREAM_URL = "https://gateway.example/v1beta/models/gemini-2.5-flash:streamGenerateContent?trace=1&foo=bar&alt=sse"
+DIRECT_ENCODED_QUERY_STREAM_URL = (
+ "https://generativelanguage.googleapis.com/v1beta/models/"
+ "gemini-2.5-flash:streamGenerateContent?trace=hello%20world&encoded=%7E&alt=sse"
+)
+
+
[email protected](autouse=True)
+def _allow_fake_plugin_gateway(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_ALLOWED_BASE_URLS", "gateway.example")
+
+
+def _config() -> ProxyConfig:
+ return ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+
+
+def _streaming_optimize_config() -> ProxyConfig:
+ return ProxyConfig(
+ optimize=True,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+
+
+def test_plugin_generate_content_uses_native_gemini_route_and_preserves_body() -> None:
+ body = {
+ "contents": [
+ {
+ "role": "user",
+ "parts": [
+ {"text": "Keep this prompt."},
+ {"inlineData": {"mimeType": "image/png", "data": "aW1hZ2U="}},
+ {"functionCall": {"name": "lookup", "args": {"city": "Paris"}}},
+ ],
+ }
+ ],
+ "generationConfig": {"temperature": 0.2},
+ }
+ upstream = {
+ "candidates": [{"content": {"parts": [{"text": "answer"}]}}],
+ "usageMetadata": {"promptTokenCount": 42, "candidatesTokenCount": 7},
+ }
+ captured: dict[str, object] = {}
+
+ async def fake_retry(
+ method: str,
+ url: str,
+ headers: dict[str, str],
+ request_body: dict,
+ **_: object,
+ ) -> httpx.Response:
+ captured.update(method=method, url=url, headers=headers, body=request_body)
+ return httpx.Response(200, json=upstream)
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy._retry_request = fake_retry
+ response = client.post(
+ "/v1beta/models/gemini-2.5-flash:generateContent",
+ headers={
+ "x-headroom-base-url": PLUGIN_BASE_URL,
+ },
+ json=body,
+ )
+
+ assert response.status_code == 200, response.text
+ assert response.json() == upstream
+ assert captured["method"] == "POST"
+ assert captured["url"] == (
+ "https://gateway.example/v1beta/models/gemini-2.5-flash:generateContent"
+ )
+ assert captured["body"] == body
+ outbound_headers = captured["headers"]
+ assert isinstance(outbound_headers, dict)
+ assert not any(key.lower().startswith("x-headroom-") for key in outbound_headers)
+
+
+ ("headers", "expected_url"),
+ [
+ ({}, DIRECT_STREAM_URL),
+ ({"x-headroom-base-url": PLUGIN_BASE_URL}, PLUGIN_STREAM_URL),
+ ],
+ ids=["direct", "plugin"],
+)
+def test_stream_generate_content_preserves_sse_and_native_parts(
+ headers: dict[str, str], expected_url: str
+) -> None:
+ body = {
+ "contents": [
+ {"role": "user", "parts": [{"text": "Keep this prompt."}]},
+ {
+ "role": "model",
+ "parts": [
+ {"functionCall": {"name": "lookup", "args": {"city": "Paris"}}},
+ {"inlineData": {"mimeType": "image/png", "data": "aW1hZ2U="}},
+ ],
+ },
+ ]
+ }
+ sse_bytes = (
+ b'data: {"candidates":[{"content":{"parts":[{"text":"answer"}]}}],'
+ b'"usageMetadata":{"promptTokenCount":42,"candidatesTokenCount":7}}\n\n'
+ )
+ captured: dict[str, object] = {}
+
+ async def fake_stream_response(
+ url: str,
+ headers: dict[str, str],
+ request_body: dict,
+ *_: object,
+ **__: object,
+ ) -> StreamingResponse:
+ captured.update(url=url, headers=headers, body=request_body)
+ return StreamingResponse(iter([sse_bytes]), media_type="text/event-stream")
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy._stream_response = fake_stream_response
+ response = client.post(
+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",
+ headers=headers,
+ json=body,
+ )
+
+ assert response.status_code == 200, response.text
+ assert response.content == sse_bytes
+ assert captured["url"] == expected_url
+ assert captured["body"] == body
+ outbound_headers = captured["headers"]
+ assert isinstance(outbound_headers, dict)
+ assert not any(key.lower().startswith("x-headroom-") for key in outbound_headers)
+
+
+def test_adjacent_custom_base_path_stays_passthrough(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_ALLOWED_BASE_URLS", PLUGIN_BASE_URL)
+ captured: dict[str, object] = {}
+
+ async def fake_passthrough(
+ request: Request,
+ base_url: str,
+ sub_path: str = "",
+ provider_name: str = "",
+ ) -> dict[str, object]:
+ captured.update(
+ path=request.url.path,
+ base_url=base_url,
+ sub_path=sub_path,
+ provider_name=provider_name,
+ )
+ return {"handler": "passthrough"}
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy.handle_passthrough = fake_passthrough
+ response = client.post(
+ "/v1/adjacent-provider-path",
+ headers={"x-headroom-base-url": PLUGIN_BASE_URL},
+ content=b"adjacent",
+ )
+
+ assert response.status_code == 200, response.text
+ assert captured == {
+ "path": "/v1/adjacent-provider-path",
+ "base_url": PLUGIN_BASE_URL,
+ "sub_path": "",
+ "provider_name": "",
+ }
+
+
+ ("headers", "expected_url"),
+ [
+ ({}, DIRECT_STREAM_URL),
+ ({"x-headroom-base-url": PLUGIN_BASE_URL}, PLUGIN_STREAM_URL),
+ ],
+ ids=["direct", "plugin"],
+)
+def test_native_streaming_reaches_the_transform_pipeline(
+ headers: dict[str, str], expected_url: str
+) -> None:
+ body = {
+ "contents": [
+ {"role": "user", "parts": [{"text": "compress me " * 200}]},
+ ]
+ }
+ captured: dict[str, object] = {}
+
+ def fake_apply(**kwargs: object) -> CompressResult:
+ messages = kwargs["messages"]
+ assert isinstance(messages, list)
+ captured["pipeline_messages"] = messages
+ return CompressResult(
+ messages=[{"role": "user", "content": "compressed"}],
+ tokens_before=1000,
+ tokens_after=10,
+ tokens_saved=990,
+ compression_ratio=0.99,
+ transforms_applied=["smart_crusher"],
+ )
+
+ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:
+ captured["url"] = args[0]
+ captured["body"] = args[2]
+ captured["transforms_applied"] = args[9]
+ return StreamingResponse(iter([b"data: {}\n\n"]), media_type="text/event-stream")
+
+ app = create_app(_streaming_optimize_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy.openai_pipeline.apply = fake_apply
+ proxy._stream_response = fake_stream_response
+ response = client.post(
+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",
+ headers=headers,
+ json=body,
+ )
+
+ assert response.status_code == 200, response.text
+ assert captured["transforms_applied"] == ["smart_crusher"]
+ assert captured["url"] == expected_url
+ outbound = captured["body"]
+ assert isinstance(outbound, dict)
+ assert outbound["contents"] == [{"role": "user", "parts": [{"text": "compressed"}]}]
+ assert captured["pipeline_messages"]
+
+
+ ("headers", "expected_url"),
+ [
+ ({}, DIRECT_STREAM_URL),
+ ({"x-headroom-base-url": PLUGIN_BASE_URL}, PLUGIN_STREAM_URL),
+ ],
+ ids=["direct", "plugin"],
+)
+def test_native_streaming_transforms_text_and_preserves_native_parts(
+ headers: dict[str, str], expected_url: str
+) -> None:
+ body = {
+ "contents": [
+ {
+ "role": "user",
+ "parts": [{"text": "compress me " * 200}],
+ },
+ {
+ "role": "model",
+ "parts": [
+ {"functionCall": {"name": "lookup", "args": {"city": "Paris"}}},
+ {"inlineData": {"mimeType": "image/png", "data": "aW1hZ2U="}},
+ ],
+ },
+ ]
+ }
+ captured: dict[str, object] = {}
+
+ def fake_apply(**kwargs: object) -> CompressResult:
+ messages = kwargs["messages"]
+ assert isinstance(messages, list)
+ captured["pipeline_messages"] = messages
+ return CompressResult(
+ messages=[{"role": "user", "content": "compressed"}],
+ tokens_before=1000,
+ tokens_after=10,
+ tokens_saved=990,
+ compression_ratio=0.99,
+ transforms_applied=["smart_crusher"],
+ )
+
+ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:
+ captured["url"] = args[0]
+ captured["body"] = args[2]
+ captured["transforms_applied"] = args[9]
+ return StreamingResponse(iter([b"data: mixed\n\n"]), media_type="text/event-stream")
+
+ app = create_app(_streaming_optimize_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy.openai_pipeline.apply = fake_apply
+ proxy._stream_response = fake_stream_response
+ response = client.post(
+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",
+ headers=headers,
+ json=body,
+ )
+
+ assert response.status_code == 200, response.text
+ assert response.content == b"data: mixed\n\n"
+ assert captured["url"] == expected_url
+ assert captured["transforms_applied"] == ["smart_crusher"]
+ outbound = captured["body"]
+ assert isinstance(outbound, dict)
+ assert outbound["contents"] == [
+ {"role": "user", "parts": [{"text": "compressed"}]},
+ {
+ "role": "model",
+ "parts": [
+ {"functionCall": {"name": "lookup", "args": {"city": "Paris"}}},
+ {"inlineData": {"mimeType": "image/png", "data": "aW1hZ2U="}},
+ ],
+ },
+ ]
+
+
+ ("headers", "query", "all_non_text", "expected_url"),
+ [
+ ({}, "key=secret&alt=sse", False, DIRECT_KEY_STREAM_URL),
+ (
+ {"x-headroom-base-url": PLUGIN_BASE_URL},
+ "trace=1&foo=bar",
+ False,
+ PLUGIN_QUERY_STREAM_URL,
+ ),
+ ({}, "trace=hello%20world&encoded=%7E&alt=raw", False, DIRECT_ENCODED_QUERY_STREAM_URL),
+ ({}, "key=secret&alt=sse", True, DIRECT_KEY_STREAM_URL),
+ (
+ {"x-headroom-base-url": PLUGIN_BASE_URL},
+ "trace=1&foo=bar",
+ True,
+ PLUGIN_QUERY_STREAM_URL,
+ ),
+ ],
+ ids=[
+ "direct-normal",
+ "plugin-normal",
+ "direct-encoded-query",
+ "direct-all-native",
+ "plugin-all-native",
+ ],
+)
+def test_native_streaming_preserves_query_and_adds_sse_once(
+ headers: dict[str, str], query: str, all_non_text: bool, expected_url: str
+) -> None:
+ parts = (
+ [{"functionCall": {"name": "lookup", "args": {"city": "Paris"}}}]
+ if all_non_text
+ else [{"text": "keep this"}]
+ )
+ body = {"contents": [{"role": "user", "parts": parts}]}
+ captured: dict[str, object] = {}
+
+ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:
+ captured["url"] = args[0]
+ return StreamingResponse(iter([b"data: query\n\n"]), media_type="text/event-stream")
+
+ app = create_app(_streaming_optimize_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy._stream_response = fake_stream_response
+ response = client.post(
+ f"/v1beta/models/gemini-2.5-flash:streamGenerateContent?{query}",
+ headers=headers,
+ json=body,
+ )
+
+ assert response.status_code == 200, response.text
+ assert response.content == b"data: query\n\n"
+ assert captured["url"] == expected_url
+
+
+ ("path", "handler_name"),
+ [
+ ("/v1beta/models/gemini-2.5-flash:generateContent", "handle_gemini_generate_content"),
+ (
+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",
+ "handle_gemini_stream_generate_content",
+ ),
+ ("/v1beta/models/gemini-2.5-flash:countTokens", "handle_gemini_count_tokens"),
+ ],
+ ids=["generate", "stream", "count"],
+)
+def test_native_gemini_routes_reject_loopback_custom_base_before_handler(
+ monkeypatch: pytest.MonkeyPatch, path: str, handler_name: str
+) -> None:
+ monkeypatch.delenv("HEADROOM_ALLOWED_BASE_URLS", raising=False)
+ app = create_app(_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ handler = AsyncMock(return_value=JSONResponse({"ok": True}))
+ setattr(proxy, handler_name, handler)
+ response = client.post(
+ path,
+ headers={"x-headroom-base-url": "http://127.0.0.1:12345"},
+ json={"contents": []},
+ )
+
+ assert response.status_code == 400
+ handler.assert_not_awaited()
+
+
+def test_gemini_count_tokens_follows_the_same_tagged_upstream() -> None:
+ """Counting and generating must agree on the upstream.
+
+ Otherwise the client counts tokens at Google for a model the tagged gateway
+ serves, and the gateway's key travels to Google.
+ """
+ app = create_app(_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy.handle_gemini_count_tokens = AsyncMock(return_value=JSONResponse({"ok": True}))
+ response = client.post(
+ "/v1beta/models/gemini-2.5-flash:countTokens",
+ headers={"x-headroom-base-url": PLUGIN_BASE_URL},
+ json={"contents": []},
+ )
+
+ assert response.status_code == 200, response.text
+ proxy.handle_gemini_count_tokens.assert_awaited_once()
+ call = proxy.handle_gemini_count_tokens.await_args
+ assert call is not None
+ assert call.args[1] == "gemini-2.5-flash"
+ assert call.kwargs == {"upstream_base_url": PLUGIN_BASE_URL}
+
+
+def test_gemini_count_tokens_without_the_header_stays_on_the_default_upstream() -> None:
+ app = create_app(_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy.handle_gemini_count_tokens = AsyncMock(return_value=JSONResponse({"ok": True}))
+ response = client.post(
+ "/v1beta/models/gemini-2.5-flash:countTokens",
+ json={"contents": []},
+ )
+
+ assert response.status_code == 200, response.text
+ call = proxy.handle_gemini_count_tokens.await_args
+ assert call is not None
+ assert call.kwargs == {}
+
+
+THINKING_PART = {
+ "text": "internal reasoning",
+ "thought": True,
+ "thoughtSignature": "signed-reasoning-token",
+}
+
+
+ ("path", "headers"),
+ [
+ ("/v1beta/models/gemini-2.5-flash:streamGenerateContent", {}),
+ (
+ "/v1beta/models/gemini-2.5-flash:streamGenerateContent",
+ {"x-headroom-base-url": PLUGIN_BASE_URL},
+ ),
+ ("/v1beta/models/gemini-2.5-flash:generateContent", {}),
+ (
+ "/v1beta/models/gemini-2.5-flash:generateContent",
+ {"x-headroom-base-url": PLUGIN_BASE_URL},
+ ),
+ ],
+ ids=["stream-direct", "stream-plugin", "generate-direct", "generate-plugin"],
+)
+def test_thinking_parts_survive_compression_elsewhere(path: str, headers: dict[str, str]) -> None:
+ body = {
+ "contents": [
+ {"role": "user", "parts": [{"text": "compress me " * 200}]},
+ {"role": "model", "parts": [dict(THINKING_PART), {"text": "visible answer"}]},
+ ]
+ }
+ captured: dict[str, object] = {}
+
+ def fake_apply(**kwargs: object) -> CompressResult:
+ messages = kwargs["messages"]
+ assert isinstance(messages, list)
+ return CompressResult(
+ messages=[{"role": "user", "content": "compressed"}, *messages[1:]],
+ tokens_before=1000,
+ tokens_after=10,
+ tokens_saved=990,
+ compression_ratio=0.99,
+ transforms_applied=["smart_crusher"],
+ )
+
+ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:
+ captured["body"] = args[2]
+ return StreamingResponse(iter([b"data: {}\n\n"]), media_type="text/event-stream")
+
+ async def fake_retry(
+ method: str, url: str, headers: dict[str, str], request_body: dict, **_: object
+ ) -> httpx.Response:
+ captured["body"] = request_body
+ return httpx.Response(200, json={"candidates": []})
+
+ app = create_app(_streaming_optimize_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy.openai_pipeline.apply = fake_apply
+ proxy._stream_response = fake_stream_response
+ proxy._retry_request = fake_retry
+ response = client.post(path, headers=headers, json=body)
+
+ assert response.status_code == 200, response.text
+ outbound = captured["body"]
+ assert isinstance(outbound, dict)
+ assert outbound["contents"] == [
+ {"role": "user", "parts": [{"text": "compressed"}]},
+ {"role": "model", "parts": [THINKING_PART, {"text": "visible answer"}]},
+ ]
+
+
+ "body",
+ [
+ {"contents": None},
+ {"contents": [{"role": "user", "parts": [{"text": 42}]}]},
+ ],
+ ids=["null-contents", "numeric-text"],
+)
+def test_malformed_stream_body_is_forwarded(body: dict) -> None:
+ captured: dict[str, object] = {}
+
+ async def fake_stream_response(*args: object, **__: object) -> StreamingResponse:
+ captured["body"] = args[2]
+ return StreamingResponse(iter([b"data: {}\n\n"]), media_type="text/event-stream")
+
+ app = create_app(_streaming_optimize_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy._stream_response = fake_stream_response
+ response = client.post("/v1beta/models/gemini-2.5-flash:streamGenerateContent", json=body)
+
+ assert response.status_code == 200, response.text
+ assert captured["body"] == body
diff --git a/tests/test_tokenizer_count_offload.py b/tests/test_tokenizer_count_offload.py
index 77db5a4d8..56b8fdcc4 100644
--- a/tests/test_tokenizer_count_offload.py
+++ b/tests/test_tokenizer_count_offload.py
@@ -72,13 +72,10 @@ def test_handlers_offload_token_counting_and_batch_apply() -> None:
fn = GeminiHandlerMixin.handle_gemini_stream_generate_content
assert inspect.iscoroutinefunction(fn)
src = inspect.getsource(fn)
- assert "_count_texts_offloaded(" in src, "streaming Gemini text counting not offloaded"
- assert "tokenizer = get_tokenizer(" not in src, "tokenizer resolved inline on the loop"
- assert "count_text(" not in src, "streaming Gemini count_text still runs on the loop"
- assert "_dict_parts(" in src, "streaming Gemini must reuse the shared _dict_parts coercion"
- assert 'isinstance(part.get("text"), str)' in src, (
- "streaming Gemini must skip non-str text so count_text can't 500"
- )
+ assert "return await self.handle_gemini_generate_content(" in src
+ assert "upstream_base_url=upstream_base_url" in src
+ assert "_count_texts_offloaded(" not in src
+ assert "_dict_parts(" not in src
for mixin, method in (
(AnthropicHandlerMixin, "handle_anthropic_batch_create"),