Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Merge branch 'main' into fix/codex-wrap-proxy-mcp-sync
Tejas Choprac7af8307d1aa
diff --cc tests/test_proxy_streaming_request_logger.pyindex aad6d50c0,c1f769fc2..035d9570c--- a/tests/test_proxy_streaming_request_logger.py+++ b/tests/test_proxy_streaming_request_logger.py@@@ -183,51 -153,57 +183,102 @@@ async def test_finalize_stream_responseassert entries[0]["savings_percent"] == 0+async def test_finalize_openai_responses_stream_uses_provider_usage_for_dashboard():+ proxy = _build_proxy_with_real_logger(log_full_messages=False)+ state = _stream_state(output_tokens=6_635)+ state["input_tokens"] = 844_000+ state["cache_read_input_tokens"] = 657_400++ await proxy._finalize_stream_response(+ body={"model": "gpt-5.5", "input": [{"type": "message", "role": "user"}]},+ provider="openai",+ model="gpt-5.5",+ request_id="req-openai-responses-stream",+ original_tokens=0,+ optimized_tokens=0,+ tokens_saved=663_000,+ transforms_applied=["openai_responses_live_zone"],+ optimization_latency=26.0,+ stream_state=state,+ start_time=0.0,+ )++ entries = proxy.logger.get_recent(10)+ assert len(entries) == 1+ entry = entries[0]+ assert entry["input_tokens_optimized"] == 844_000+ assert entry["input_tokens_original"] == 1_507_000+ assert entry["tokens_saved"] == 663_000+ assert entry["savings_percent"] == pytest.approx(663_000 / 1_507_000 * 100)+ assert entry["output_tokens"] == 6_635++ proxy.metrics.record_request.assert_awaited_once()+ metrics_kwargs = proxy.metrics.record_request.await_args.kwargs+ assert metrics_kwargs["input_tokens"] == 844_000+ assert metrics_kwargs["output_tokens"] == 6_635+ assert metrics_kwargs["tokens_saved"] == 663_000+ assert metrics_kwargs["cache_read_tokens"] == 657_400+ assert metrics_kwargs["uncached_input_tokens"] == 186_600++ proxy.cost_tracker.record_tokens.assert_called_once()+ cost_args, cost_kwargs = proxy.cost_tracker.record_tokens.call_args+ assert cost_args[:3] == ("gpt-5.5", 663_000, 844_000)+ assert cost_kwargs["cache_read_tokens"] == 657_400+ assert cost_kwargs["uncached_tokens"] == 186_600+++ @pytest.mark.asyncio+ async def test_finalize_stream_response_recovers_usage_from_truncated_buffer() -> None:+ """When upstream truncates mid-event (no trailing \\n\\n), the per-chunk+ parser leaves the message_start usage event sitting in sse_buffer and+ PERF logs cache_read=cache_write=0 — which then poisons the freeze+ heuristic on the next request. The finalizer must flush the residual+ buffer so the real cache_read / cache_creation tokens still land in+ the log even on aborted streams.+ """+ proxy = _build_proxy_with_real_logger(log_full_messages=False)++ partial_message_start = (+ b"event: message_start\n"+ b'data: {"type":"message_start","message":{"id":"msg_x",'+ b'"type":"message","role":"assistant","model":"claude-sonnet-4-6",'+ b'"content":[],"stop_reason":null,"usage":{'+ b'"input_tokens":1234,"cache_read_input_tokens":50000,'+ b'"cache_creation_input_tokens":2500,"output_tokens":1}}}'+ )++ state = {+ "output_tokens": None,+ "total_bytes": len(partial_message_start),+ "ttfb_ms": 35.0,+ "input_tokens": None,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ "cache_creation_ephemeral_5m_input_tokens": 0,+ "cache_creation_ephemeral_1h_input_tokens": 0,+ "sse_buffer": bytearray(partial_message_start),+ }++ await proxy._finalize_stream_response(+ body={"messages": [{"role": "user", "content": "hi"}]},+ provider="anthropic",+ model="claude-sonnet-4-6",+ request_id="req-stream-truncated",+ original_tokens=2000,+ optimized_tokens=1800,+ tokens_saved=200,+ transforms_applied=[],+ optimization_latency=5.0,+ stream_state=state,+ start_time=0.0,+ )++ assert state["input_tokens"] == 1234+ assert state["cache_read_input_tokens"] == 50000+ assert state["cache_creation_input_tokens"] == 2500++@pytest.mark.asyncioasync def test_finalize_stream_response_no_op_when_logger_disabled():proxy = _build_proxy_with_real_logger(log_full_messages=False)