Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Merge branch 'main' into fix/codex-wrap-proxy-mcp-sync

Tejas Choprac7af8307d1aa
diff --cc tests/test_proxy_streaming_request_logger.py
index aad6d50c0,c1f769fc2..035d9570c
--- a/tests/test_proxy_streaming_request_logger.py
+++ b/tests/test_proxy_streaming_request_logger.py
@@@ -183,51 -153,57 +183,102 @@@ async def test_finalize_stream_response
assert entries[0]["savings_percent"] == 0
+async def test_finalize_openai_responses_stream_uses_provider_usage_for_dashboard():
+ proxy = _build_proxy_with_real_logger(log_full_messages=False)
+ state = _stream_state(output_tokens=6_635)
+ state["input_tokens"] = 844_000
+ state["cache_read_input_tokens"] = 657_400
+
+ await proxy._finalize_stream_response(
+ body={"model": "gpt-5.5", "input": [{"type": "message", "role": "user"}]},
+ provider="openai",
+ model="gpt-5.5",
+ request_id="req-openai-responses-stream",
+ original_tokens=0,
+ optimized_tokens=0,
+ tokens_saved=663_000,
+ transforms_applied=["openai_responses_live_zone"],
+ optimization_latency=26.0,
+ stream_state=state,
+ start_time=0.0,
+ )
+
+ entries = proxy.logger.get_recent(10)
+ assert len(entries) == 1
+ entry = entries[0]
+ assert entry["input_tokens_optimized"] == 844_000
+ assert entry["input_tokens_original"] == 1_507_000
+ assert entry["tokens_saved"] == 663_000
+ assert entry["savings_percent"] == pytest.approx(663_000 / 1_507_000 * 100)
+ assert entry["output_tokens"] == 6_635
+
+ proxy.metrics.record_request.assert_awaited_once()
+ metrics_kwargs = proxy.metrics.record_request.await_args.kwargs
+ assert metrics_kwargs["input_tokens"] == 844_000
+ assert metrics_kwargs["output_tokens"] == 6_635
+ assert metrics_kwargs["tokens_saved"] == 663_000
+ assert metrics_kwargs["cache_read_tokens"] == 657_400
+ assert metrics_kwargs["uncached_input_tokens"] == 186_600
+
+ proxy.cost_tracker.record_tokens.assert_called_once()
+ cost_args, cost_kwargs = proxy.cost_tracker.record_tokens.call_args
+ assert cost_args[:3] == ("gpt-5.5", 663_000, 844_000)
+ assert cost_kwargs["cache_read_tokens"] == 657_400
+ assert cost_kwargs["uncached_tokens"] == 186_600
+
+
+ @pytest.mark.asyncio
+ async def test_finalize_stream_response_recovers_usage_from_truncated_buffer() -> None:
+ """When upstream truncates mid-event (no trailing \\n\\n), the per-chunk
+ parser leaves the message_start usage event sitting in sse_buffer and
+ PERF logs cache_read=cache_write=0 — which then poisons the freeze
+ heuristic on the next request. The finalizer must flush the residual
+ buffer so the real cache_read / cache_creation tokens still land in
+ the log even on aborted streams.
+ """
+ proxy = _build_proxy_with_real_logger(log_full_messages=False)
+
+ partial_message_start = (
+ b"event: message_start\n"
+ b'data: {"type":"message_start","message":{"id":"msg_x",'
+ b'"type":"message","role":"assistant","model":"claude-sonnet-4-6",'
+ b'"content":[],"stop_reason":null,"usage":{'
+ b'"input_tokens":1234,"cache_read_input_tokens":50000,'
+ b'"cache_creation_input_tokens":2500,"output_tokens":1}}}'
+ )
+
+ state = {
+ "output_tokens": None,
+ "total_bytes": len(partial_message_start),
+ "ttfb_ms": 35.0,
+ "input_tokens": None,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ "cache_creation_ephemeral_5m_input_tokens": 0,
+ "cache_creation_ephemeral_1h_input_tokens": 0,
+ "sse_buffer": bytearray(partial_message_start),
+ }
+
+ await proxy._finalize_stream_response(
+ body={"messages": [{"role": "user", "content": "hi"}]},
+ provider="anthropic",
+ model="claude-sonnet-4-6",
+ request_id="req-stream-truncated",
+ original_tokens=2000,
+ optimized_tokens=1800,
+ tokens_saved=200,
+ transforms_applied=[],
+ optimization_latency=5.0,
+ stream_state=state,
+ start_time=0.0,
+ )
+
+ assert state["input_tokens"] == 1234
+ assert state["cache_read_input_tokens"] == 50000
+ assert state["cache_creation_input_tokens"] == 2500
+
+
@pytest.mark.asyncio
async def test_finalize_stream_response_no_op_when_logger_disabled():
proxy = _build_proxy_with_real_logger(log_full_messages=False)