Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Fix streaming crashes under concurrent sessions

chopratejas4cdaa49153ef
diff --git a/headroom/__init__.py b/headroom/__init__.py
index 6527ce6fd..b44b96882 100644
--- a/headroom/__init__.py
+++ b/headroom/__init__.py
@@ -142,7 +142,7 @@ from .transforms import (
TransformPipeline,
)
-__version__ = "0.3.2"
+__version__ = "0.3.3"
__all__ = [
# Main client
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index bfe83c39c..7bc2a78b6 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -539,9 +539,28 @@ class CostTracker:
self._total_savings_usd: float = 0
self._last_prune_time: datetime = datetime.now()
+ # Cache resolved model names to avoid repeated litellm lookups.
+ # This is critical: litellm.cost_per_token() is synchronous and can block
+ # the async event loop if it triggers I/O (lazy model info download).
+ _resolved_model_cache: dict[str, str] = {}
+
+ @classmethod
+ def _resolve_litellm_model(cls, model: str) -> str:
+ """Resolve model name to one LiteLLM recognizes, adding provider prefix if needed.
+
+ Results are cached per model name to avoid blocking the event loop
+ with repeated synchronous litellm lookups.
+ """
+ if model in cls._resolved_model_cache:
+ return cls._resolved_model_cache[model]
+
+ resolved = cls._resolve_litellm_model_uncached(model)
+ cls._resolved_model_cache[model] = resolved
+ return resolved
+
@staticmethod
- def _resolve_litellm_model(model: str) -> str:
- """Resolve model name to one LiteLLM recognizes, adding provider prefix if needed."""
+ def _resolve_litellm_model_uncached(model: str) -> str:
+ """Uncached resolution — called once per unique model name."""
if not LITELLM_AVAILABLE:
return model
@@ -3686,6 +3705,27 @@ class HeadroomProxy:
# No memory tool calls, yield original buffered chunks
for chunk in buffered_chunks:
yield chunk
+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
+ logger.error(f"[{request_id}] Connection error to upstream API: {e}")
+ error_event = {
+ "type": "error",
+ "error": {
+ "type": "connection_error",
+ "message": f"Failed to connect to upstream API: {e}",
+ },
+ }
+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
+ except httpx.HTTPStatusError as e:
+ logger.error(f"[{request_id}] HTTP error from upstream API: {e}")
+ # Forward the upstream error response
+ yield e.response.content
+ except Exception as e:
+ logger.error(f"[{request_id}] Unexpected streaming error: {e}")
+ error_event = {
+ "type": "error",
+ "error": {"type": "api_error", "message": str(e)},
+ }
+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
finally:
# Record metrics after stream completes
total_latency = (time.time() - start_time) * 1000
diff --git a/pyproject.toml b/pyproject.toml
index 4668d85c6..be3290130 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "headroom-ai"
-version = "0.3.2"
+version = "0.3.3"
description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
readme = "README.md"
license = "Apache-2.0"