Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Add tests for streaming resilience and concurrent session safety
chopratejas636abd22f3ec
diff --git a/tests/test_proxy_streaming_resilience.py b/tests/test_proxy_streaming_resilience.pynew file mode 100644index 000000000..3bfd3fce9--- /dev/null+++ b/tests/test_proxy_streaming_resilience.py@@ -0,0 +1,621 @@+"""Tests for proxy streaming resilience and concurrent session handling.++These tests verify:+1. CostTracker model resolution caching (prevents event loop blocking)+2. Streaming generate() error handling (prevents ASGI crashes)+3. Concurrent session safety (multiple sessions don't interfere)+"""++import asyncio+import json+import time+from unittest.mock import MagicMock, patch++import httpx+import pytest++# ---------------------------------------------------------------------------+# CostTracker model resolution caching+# ---------------------------------------------------------------------------+++class TestModelResolutionCaching:+ """Test that _resolve_litellm_model caches results to avoid repeated sync calls."""++ def setup_method(self):+ """Clear the cache before each test."""+ from headroom.proxy.server import CostTracker++ CostTracker._resolved_model_cache.clear()++ def test_cache_returns_same_result_on_second_call(self):+ """First call resolves, second call returns cached value without calling litellm."""+ from headroom.proxy.server import CostTracker++ with patch.object(+ CostTracker, "_resolve_litellm_model_uncached", return_value="anthropic/claude-opus-4-6"+ ) as mock_uncached:+ # First call — should invoke uncached resolution+ result1 = CostTracker._resolve_litellm_model("claude-opus-4-6")+ assert result1 == "anthropic/claude-opus-4-6"+ assert mock_uncached.call_count == 1++ # Second call — should use cache, NOT call uncached again+ result2 = CostTracker._resolve_litellm_model("claude-opus-4-6")+ assert result2 == "anthropic/claude-opus-4-6"+ assert mock_uncached.call_count == 1 # Still 1, not 2++ def test_cache_is_per_model_name(self):+ """Different model names get separate cache entries."""+ from headroom.proxy.server import CostTracker++ with patch.object(+ CostTracker,+ "_resolve_litellm_model_uncached",+ side_effect=lambda m: f"resolved/{m}",+ ) as mock_uncached:+ result1 = CostTracker._resolve_litellm_model("gpt-4o")+ result2 = CostTracker._resolve_litellm_model("claude-opus-4-6")+ result3 = CostTracker._resolve_litellm_model("gpt-4o") # cached++ assert result1 == "resolved/gpt-4o"+ assert result2 == "resolved/claude-opus-4-6"+ assert result3 == "resolved/gpt-4o"+ assert mock_uncached.call_count == 2 # Only 2, not 3++ def test_cached_call_is_fast(self):+ """Cached resolution should be sub-millisecond (dict lookup)."""+ from headroom.proxy.server import CostTracker++ # Pre-populate cache+ CostTracker._resolved_model_cache["test-model"] = "resolved/test-model"++ start = time.perf_counter()+ for _ in range(10_000):+ CostTracker._resolve_litellm_model("test-model")+ elapsed_ms = (time.perf_counter() - start) * 1000++ # 10k lookups should take < 50ms (dict lookup is ~0.001ms each)+ assert elapsed_ms < 50, f"10k cached lookups took {elapsed_ms:.1f}ms — too slow"++ def test_uncached_adds_provider_prefix_for_claude(self):+ """_resolve_litellm_model_uncached tries provider prefix for claude- models."""+ from headroom.proxy.server import CostTracker++ with (+ patch("headroom.proxy.server.LITELLM_AVAILABLE", True),+ patch("headroom.proxy.server.litellm") as mock_litellm,+ ):+ # First call (bare name) fails, second call (prefixed) succeeds+ mock_litellm.cost_per_token.side_effect = [+ Exception("Unknown model"), # bare "claude-opus-4-6"+ (0.001, 0.002), # "anthropic/claude-opus-4-6"+ ]++ result = CostTracker._resolve_litellm_model_uncached("claude-opus-4-6")+ assert result == "anthropic/claude-opus-4-6"++ def test_uncached_adds_provider_prefix_for_gpt(self):+ """_resolve_litellm_model_uncached tries provider prefix for gpt- models."""+ from headroom.proxy.server import CostTracker++ with (+ patch("headroom.proxy.server.LITELLM_AVAILABLE", True),+ patch("headroom.proxy.server.litellm") as mock_litellm,+ ):+ mock_litellm.cost_per_token.side_effect = [+ Exception("Unknown model"),+ (0.001, 0.002),+ ]++ result = CostTracker._resolve_litellm_model_uncached("gpt-4o")+ assert result == "openai/gpt-4o"++ def test_uncached_adds_provider_prefix_for_gemini(self):+ """_resolve_litellm_model_uncached tries provider prefix for gemini- models."""+ from headroom.proxy.server import CostTracker++ with (+ patch("headroom.proxy.server.LITELLM_AVAILABLE", True),+ patch("headroom.proxy.server.litellm") as mock_litellm,+ ):+ mock_litellm.cost_per_token.side_effect = [+ Exception("Unknown model"),+ (0.001, 0.002),+ ]++ result = CostTracker._resolve_litellm_model_uncached("gemini-1.5-pro")+ assert result == "google/gemini-1.5-pro"++ def test_uncached_returns_original_when_both_fail(self):+ """If both bare and prefixed lookups fail, return original model name."""+ from headroom.proxy.server import CostTracker++ with (+ patch("headroom.proxy.server.LITELLM_AVAILABLE", True),+ patch("headroom.proxy.server.litellm") as mock_litellm,+ ):+ mock_litellm.cost_per_token.side_effect = Exception("Unknown model")++ result = CostTracker._resolve_litellm_model_uncached("totally-unknown-model-xyz")+ assert result == "totally-unknown-model-xyz"++ def test_uncached_returns_original_when_litellm_unavailable(self):+ """When litellm is not available, return model as-is."""+ from headroom.proxy.server import CostTracker++ with patch("headroom.proxy.server.LITELLM_AVAILABLE", False):+ result = CostTracker._resolve_litellm_model_uncached("claude-opus-4-6")+ assert result == "claude-opus-4-6"++ def test_uncached_returns_bare_when_it_works(self):+ """If bare model name works, don't add prefix."""+ from headroom.proxy.server import CostTracker++ with (+ patch("headroom.proxy.server.LITELLM_AVAILABLE", True),+ patch("headroom.proxy.server.litellm") as mock_litellm,+ ):+ mock_litellm.cost_per_token.return_value = (0.001, 0.002)++ result = CostTracker._resolve_litellm_model_uncached("claude-3-5-sonnet-20241022")+ assert result == "claude-3-5-sonnet-20241022"++ def test_cache_is_class_level_shared_across_instances(self):+ """Cache is shared across CostTracker instances (class variable)."""+ from headroom.proxy.server import CostTracker++ tracker1 = CostTracker()+ tracker2 = CostTracker()++ with patch.object(+ CostTracker, "_resolve_litellm_model_uncached", return_value="resolved/model-a"+ ) as mock_uncached:+ # Resolve via instance 1+ result1 = tracker1._resolve_litellm_model("model-a")+ assert mock_uncached.call_count == 1++ # Instance 2 should get cached result+ result2 = tracker2._resolve_litellm_model("model-a")+ assert mock_uncached.call_count == 1 # Not called again+ assert result1 == result2+++# ---------------------------------------------------------------------------+# Streaming generate() error handling+# ---------------------------------------------------------------------------+++class TestStreamingErrorHandling:+ """Test that streaming errors are caught and returned as SSE error events."""++ @pytest.mark.asyncio+ async def test_connect_error_yields_sse_error(self):+ """httpx.ConnectError should yield an SSE error event, not crash."""+ proxy = self._create_mock_proxy()++ # Make http_client.stream raise ConnectError+ connect_error = httpx.ConnectError("Connection refused")+ proxy.http_client.stream = MagicMock(side_effect=connect_error)++ chunks = []+ async for chunk in self._call_generate(proxy):+ chunks.append(chunk)++ # Should have yielded an error event, not crashed+ assert len(chunks) >= 1+ error_data = self._parse_sse_error(chunks[-1])+ assert error_data["error"]["type"] == "connection_error"+ assert "Connection refused" in error_data["error"]["message"]++ @pytest.mark.asyncio+ async def test_connect_timeout_yields_sse_error(self):+ """httpx.ConnectTimeout should yield an SSE error event."""+ proxy = self._create_mock_proxy()++ timeout_error = httpx.ConnectTimeout("Timed out connecting")+ proxy.http_client.stream = MagicMock(side_effect=timeout_error)++ chunks = []+ async for chunk in self._call_generate(proxy):+ chunks.append(chunk)++ assert len(chunks) >= 1+ error_data = self._parse_sse_error(chunks[-1])+ assert error_data["error"]["type"] == "connection_error"++ @pytest.mark.asyncio+ async def test_pool_timeout_yields_sse_error(self):+ """httpx.PoolTimeout should yield an SSE error event."""+ proxy = self._create_mock_proxy()++ pool_error = httpx.PoolTimeout("Pool timeout: all connections busy")+ proxy.http_client.stream = MagicMock(side_effect=pool_error)++ chunks = []+ async for chunk in self._call_generate(proxy):+ chunks.append(chunk)++ assert len(chunks) >= 1+ error_data = self._parse_sse_error(chunks[-1])+ assert error_data["error"]["type"] == "connection_error"+ assert "Pool timeout" in error_data["error"]["message"]++ @pytest.mark.asyncio+ async def test_http_status_error_forwards_upstream_response(self):+ """httpx.HTTPStatusError should forward the upstream error body."""+ proxy = self._create_mock_proxy()++ # Create a realistic HTTP 429 error+ mock_response = MagicMock()+ upstream_error_body = json.dumps(+ {"error": {"type": "rate_limit_error", "message": "Too many requests"}}+ ).encode()+ mock_response.content = upstream_error_body+ mock_response.status_code = 429++ mock_request = MagicMock()+ http_error = httpx.HTTPStatusError(+ "429 Too Many Requests", request=mock_request, response=mock_response+ )+ proxy.http_client.stream = MagicMock(side_effect=http_error)++ chunks = []+ async for chunk in self._call_generate(proxy):+ chunks.append(chunk)++ # Should forward the upstream error response body+ assert len(chunks) >= 1+ assert upstream_error_body in chunks++ @pytest.mark.asyncio+ async def test_unexpected_error_yields_sse_error(self):+ """Unexpected exceptions should yield an SSE error event, not crash."""+ proxy = self._create_mock_proxy()++ proxy.http_client.stream = MagicMock(+ side_effect=RuntimeError("Something unexpected went wrong")+ )++ chunks = []+ async for chunk in self._call_generate(proxy):+ chunks.append(chunk)++ assert len(chunks) >= 1+ error_data = self._parse_sse_error(chunks[-1])+ assert error_data["error"]["type"] == "api_error"+ assert "Something unexpected" in error_data["error"]["message"]++ @pytest.mark.asyncio+ async def test_finally_block_runs_after_error(self):+ """The finally block (metrics recording) should still run after errors."""+ proxy = self._create_mock_proxy()++ proxy.http_client.stream = MagicMock(side_effect=httpx.ConnectError("fail"))++ # Track that generate completes fully (including finally)+ chunks = []+ async for chunk in self._call_generate(proxy):+ chunks.append(chunk)++ # If we got here without exception, the finally block didn't re-raise+ assert len(chunks) >= 1++ @pytest.mark.asyncio+ async def test_error_event_is_valid_sse_format(self):+ """Error events should be valid SSE format (event: error\\ndata: {...}\\n\\n)."""+ proxy = self._create_mock_proxy()+ proxy.http_client.stream = MagicMock(side_effect=httpx.ConnectError("refused"))++ chunks = []+ async for chunk in self._call_generate(proxy):+ chunks.append(chunk)++ raw = chunks[-1].decode("utf-8")+ assert raw.startswith("event: error\n")+ assert "data: " in raw+ assert raw.endswith("\n\n")++ # Data portion should be valid JSON+ data_line = [line for line in raw.split("\n") if line.startswith("data: ")][0]+ json_str = data_line[len("data: ") :]+ parsed = json.loads(json_str)+ assert "type" in parsed+ assert "error" in parsed++ # --- Helpers ---++ def _create_mock_proxy(self):+ """Create a HeadroomProxy-like object with mocked internals for testing generate()."""+ from headroom.proxy.server import HeadroomProxy++ proxy = object.__new__(HeadroomProxy)+ proxy.http_client = MagicMock(spec=httpx.AsyncClient)+ proxy.cost_tracker = MagicMock()+ proxy.cost_tracker.estimate_cost.return_value = 0.001+ proxy.cost_tracker.record_request.return_value = None+ proxy.stats = {+ "requests_total": 0,+ "requests_optimized": 0,+ "tokens": {"original": 0, "optimized": 0, "saved": 0},+ "cost": {"total_usd": 0, "savings_usd": 0},+ "errors": 0,+ "active_requests": 0,+ "requests_per_model": {},+ }+ proxy.memory_manager = None+ proxy._config = MagicMock()+ proxy._config.memory_enabled = False+ proxy._parse_sse_usage_from_buffer = MagicMock(return_value=None)+ return proxy++ async def _call_generate(self, proxy):+ """Call the streaming generate pattern matching server.py's generate() function.++ Since generate() is a nested closure inside _handle_openai_streaming,+ we test the error handling pattern directly — same try/except/finally+ structure as the real code.+ """+ url = "https://api.openai.com/v1/chat/completions"+ body = {"model": "gpt-4o", "messages": [{"role": "user", "content": "Hi"}], "stream": True}+ headers = {"Authorization": "Bearer sk-test"}++ try:+ async with proxy.http_client.stream("POST", url, json=body, headers=headers) as resp:+ async for chunk in resp.aiter_bytes():+ yield chunk+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:+ error_event = {+ "type": "error",+ "error": {+ "type": "connection_error",+ "message": f"Failed to connect to upstream API: {e}",+ },+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()+ except httpx.HTTPStatusError as e:+ yield e.response.content+ except Exception as e:+ error_event = {+ "type": "error",+ "error": {"type": "api_error", "message": str(e)},+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()+ finally:+ # Mirrors the finally block in server.py — should not raise+ pass++ def _parse_sse_error(self, chunk: bytes) -> dict:+ """Parse an SSE error event chunk into a dict."""+ raw = chunk.decode("utf-8")+ for line in raw.split("\n"):+ if line.startswith("data: "):+ return json.loads(line[len("data: ") :])+ raise ValueError(f"No data: line found in SSE chunk: {raw}")+++# ---------------------------------------------------------------------------+# Concurrent session safety+# ---------------------------------------------------------------------------+++class TestConcurrentSessionSafety:+ """Test that multiple concurrent sessions don't interfere with each other."""++ def setup_method(self):+ from headroom.proxy.server import CostTracker++ CostTracker._resolved_model_cache.clear()++ @pytest.mark.asyncio+ async def test_concurrent_model_resolution_is_safe(self):+ """Multiple concurrent tasks resolving the same model should all get correct result."""+ from headroom.proxy.server import CostTracker++ call_count = 0++ def slow_uncached(model: str) -> str:+ nonlocal call_count+ call_count += 1+ # Simulate the slow litellm lookup+ return f"resolved/{model}"++ with patch.object(+ CostTracker, "_resolve_litellm_model_uncached", side_effect=slow_uncached+ ):+ # Launch 50 concurrent resolution tasks for the same model+ tasks = [+ asyncio.to_thread(CostTracker._resolve_litellm_model, "claude-opus-4-6")+ for _ in range(50)+ ]+ results = await asyncio.gather(*tasks)++ # All should get the same result+ assert all(r == "resolved/claude-opus-4-6" for r in results)+ # Uncached should be called very few times (ideally 1, but a few races are OK)+ assert call_count <= 5, f"Uncached called {call_count} times — expected ~1"++ @pytest.mark.asyncio+ async def test_concurrent_resolution_different_models(self):+ """Concurrent resolution of different models should each resolve independently."""+ from headroom.proxy.server import CostTracker++ models = ["gpt-4o", "claude-opus-4-6", "gemini-1.5-pro", "gpt-4o-mini"]++ with patch.object(+ CostTracker,+ "_resolve_litellm_model_uncached",+ side_effect=lambda m: f"resolved/{m}",+ ):+ tasks = [+ asyncio.to_thread(CostTracker._resolve_litellm_model, model)+ for model in models * 10 # 40 tasks total+ ]+ results = await asyncio.gather(*tasks)++ # Verify each model resolved correctly+ for i, model in enumerate(models * 10):+ assert results[i] == f"resolved/{model}"++ # Cache should have exactly 4 entries+ assert len(CostTracker._resolved_model_cache) == 4++ @pytest.mark.asyncio+ async def test_concurrent_streaming_errors_are_independent(self):+ """Each session's streaming error should be independent — one failure shouldn't affect others."""++ async def simulate_session(session_id: int, should_fail: bool):+ """Simulate a streaming session that either succeeds or fails."""+ chunks = []++ try:+ if should_fail:+ raise httpx.ConnectError(f"Session {session_id} connection refused")+ else:+ # Successful session+ for i in range(3):+ chunks.append(f"data: chunk-{session_id}-{i}\n\n".encode())+ await asyncio.sleep(0.001)+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:+ error_event = {+ "type": "error",+ "error": {+ "type": "connection_error",+ "message": str(e),+ },+ }+ chunks.append(f"event: error\ndata: {json.dumps(error_event)}\n\n".encode())++ return session_id, chunks, should_fail++ # Run 10 sessions: odd ones fail, even ones succeed+ tasks = [simulate_session(i, should_fail=(i % 2 == 1)) for i in range(10)]+ results = await asyncio.gather(*tasks)++ for session_id, chunks, should_fail in results:+ if should_fail:+ # Failed sessions should have an error chunk+ assert len(chunks) == 1+ error_data = json.loads(chunks[0].decode("utf-8").split("data: ")[1].strip())+ assert error_data["error"]["type"] == "connection_error"+ assert f"Session {session_id}" in error_data["error"]["message"]+ else:+ # Successful sessions should have their data chunks+ assert len(chunks) == 3+ for i, chunk in enumerate(chunks):+ assert f"chunk-{session_id}-{i}".encode() in chunk++ @pytest.mark.asyncio+ async def test_estimate_cost_concurrent_with_caching(self):+ """Multiple concurrent estimate_cost calls should not block each other."""+ from headroom.proxy.server import CostTracker++ tracker = CostTracker()++ # Pre-populate cache to simulate steady-state+ CostTracker._resolved_model_cache["gpt-4o"] = "openai/gpt-4o"++ with (+ patch("headroom.proxy.server.LITELLM_AVAILABLE", True),+ patch("headroom.proxy.server.litellm") as mock_litellm,+ ):+ mock_litellm.cost_per_token.return_value = (0.001, 0.002)+ mock_litellm.get_model_info.return_value = {}++ start = time.perf_counter()+ tasks = [+ asyncio.to_thread(tracker.estimate_cost, "gpt-4o", 1000, 500) for _ in range(100)+ ]+ results = await asyncio.gather(*tasks)+ elapsed_ms = (time.perf_counter() - start) * 1000++ # All should return a valid cost+ assert all(r is not None and r > 0 for r in results)+ # 100 concurrent calls should complete quickly (no blocking)+ assert elapsed_ms < 5000, f"100 concurrent estimate_cost took {elapsed_ms:.0f}ms"+++# ---------------------------------------------------------------------------+# Cost tracking — no double-counting of cache tokens+# ---------------------------------------------------------------------------+++class TestCostTrackingAccuracy:+ """Test that cost calculations don't double-count cache tokens."""++ def setup_method(self):+ from headroom.proxy.server import CostTracker++ CostTracker._resolved_model_cache.clear()++ def test_estimate_cost_separates_input_and_cache(self):+ """Input tokens and cache tokens should be billed separately, not double-counted."""+ from headroom.proxy.server import CostTracker++ tracker = CostTracker()++ with (+ patch("headroom.proxy.server.LITELLM_AVAILABLE", True),+ patch("headroom.proxy.server.litellm") as mock_litellm,+ ):+ # Setup: $10/M input, $30/M output+ def mock_cost(model, prompt_tokens, completion_tokens):+ input_cost = prompt_tokens * 0.00001+ output_cost = completion_tokens * 0.00003+ return (input_cost, output_cost)++ mock_litellm.cost_per_token.side_effect = mock_cost+ mock_litellm.get_model_info.return_value = {+ "cache_read_input_token_cost": 0.000001, # 10% of input+ "cache_creation_input_token_cost": 0.0000125, # 125% of input+ }++ # 1000 input + 500 cache_read + 200 cache_write + 100 output+ cost = tracker.estimate_cost(+ model="gpt-4o",+ input_tokens=1000,+ output_tokens=100,+ cache_read_tokens=500,+ cache_write_tokens=200,+ )++ assert cost is not None+ # input_cost = 1000 * 0.00001 = 0.01+ # output_cost = 100 * 0.00003 = 0.003+ # cache_read = 500 * 0.000001 = 0.0005+ # cache_write = 200 * 0.0000125 = 0.0025+ expected = 0.01 + 0.003 + 0.0005 + 0.0025+ assert abs(cost - expected) < 0.0001, f"Expected {expected}, got {cost}"++ def test_estimate_cost_without_cache_tokens(self):+ """Cost without cache tokens should just be input + output."""+ from headroom.proxy.server import CostTracker++ tracker = CostTracker()++ with (+ patch("headroom.proxy.server.LITELLM_AVAILABLE", True),+ patch("headroom.proxy.server.litellm") as mock_litellm,+ ):+ mock_litellm.cost_per_token.side_effect = (+ lambda model, prompt_tokens, completion_tokens: (+ prompt_tokens * 0.00001,+ completion_tokens * 0.00003,+ )+ )+ mock_litellm.get_model_info.return_value = {}++ cost = tracker.estimate_cost("gpt-4o", input_tokens=1000, output_tokens=100)++ expected = 1000 * 0.00001 + 100 * 0.00003+ assert abs(cost - expected) < 0.0001++ def test_estimate_cost_returns_none_without_litellm(self):+ """When litellm is unavailable, estimate_cost should return None."""+ from headroom.proxy.server import CostTracker++ tracker = CostTracker()++ with patch("headroom.proxy.server.LITELLM_AVAILABLE", False):+ cost = tracker.estimate_cost("gpt-4o", input_tokens=1000, output_tokens=100)+ assert cost is None