Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Fix streaming crashes under concurrent sessions
chopratejas4cdaa49153ef
diff --git a/headroom/__init__.py b/headroom/__init__.pyindex 6527ce6fd..b44b96882 100644--- a/headroom/__init__.py+++ b/headroom/__init__.py@@ -142,7 +142,7 @@ from .transforms import (TransformPipeline,)-__version__ = "0.3.2"+__version__ = "0.3.3"__all__ = [# Main clientdiff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex bfe83c39c..7bc2a78b6 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -539,9 +539,28 @@ class CostTracker:self._total_savings_usd: float = 0self._last_prune_time: datetime = datetime.now()+ # Cache resolved model names to avoid repeated litellm lookups.+ # This is critical: litellm.cost_per_token() is synchronous and can block+ # the async event loop if it triggers I/O (lazy model info download).+ _resolved_model_cache: dict[str, str] = {}++ @classmethod+ def _resolve_litellm_model(cls, model: str) -> str:+ """Resolve model name to one LiteLLM recognizes, adding provider prefix if needed.++ Results are cached per model name to avoid blocking the event loop+ with repeated synchronous litellm lookups.+ """+ if model in cls._resolved_model_cache:+ return cls._resolved_model_cache[model]++ resolved = cls._resolve_litellm_model_uncached(model)+ cls._resolved_model_cache[model] = resolved+ return resolved+@staticmethod- def _resolve_litellm_model(model: str) -> str:- """Resolve model name to one LiteLLM recognizes, adding provider prefix if needed."""+ def _resolve_litellm_model_uncached(model: str) -> str:+ """Uncached resolution — called once per unique model name."""if not LITELLM_AVAILABLE:return model@@ -3686,6 +3705,27 @@ class HeadroomProxy:# No memory tool calls, yield original buffered chunksfor chunk in buffered_chunks:yield chunk+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:+ logger.error(f"[{request_id}] Connection error to upstream API: {e}")+ error_event = {+ "type": "error",+ "error": {+ "type": "connection_error",+ "message": f"Failed to connect to upstream API: {e}",+ },+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()+ except httpx.HTTPStatusError as e:+ logger.error(f"[{request_id}] HTTP error from upstream API: {e}")+ # Forward the upstream error response+ yield e.response.content+ except Exception as e:+ logger.error(f"[{request_id}] Unexpected streaming error: {e}")+ error_event = {+ "type": "error",+ "error": {"type": "api_error", "message": str(e)},+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()finally:# Record metrics after stream completestotal_latency = (time.time() - start_time) * 1000diff --git a/pyproject.toml b/pyproject.tomlindex 4668d85c6..be3290130 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "hatchling.build"[project]name = "headroom-ai"-version = "0.3.2"+version = "0.3.3"description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"readme = "README.md"license = "Apache-2.0"