Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Adding any-llm as a backed provider

angptfeeb72c0fc24
diff --git a/headroom/backends/__init__.py b/headroom/backends/__init__.py
index c77203e12..861e3c026 100644
--- a/headroom/backends/__init__.py
+++ b/headroom/backends/__init__.py
@@ -3,17 +3,20 @@
Backends handle the translation between the proxy's canonical format
(Anthropic Messages API) and provider-specific APIs.
-Uses LiteLLM for broad provider support:
-- bedrock: AWS Bedrock (Claude, Cohere, Mistral, etc.)
-- vertex_ai: Google Vertex AI (Claude, Gemini, etc.)
-- azure: Azure OpenAI (GPT-4, etc.)
-- And 100+ more providers...
+Supported backend libraries:
+- LiteLLM: 100+ providers (bedrock, vertex_ai, azure, openrouter, etc.)
+- any-llm: 38+ providers (openai, anthropic, mistral, groq, ollama, etc.)
Usage:
+ # LiteLLM backend
headroom proxy --backend litellm-bedrock --region us-west-2
+
+ # any-llm backend
+ headroom proxy --backend anyllm --anyllm-provider openai
"""
+from .anyllm import AnyLLMBackend
from .base import Backend, BackendResponse, StreamEvent
from .litellm import LiteLLMBackend
-__all__ = ["Backend", "BackendResponse", "StreamEvent", "LiteLLMBackend"]
+__all__ = ["Backend", "BackendResponse", "StreamEvent", "LiteLLMBackend", "AnyLLMBackend"]
diff --git a/headroom/backends/anyllm.py b/headroom/backends/anyllm.py
new file mode 100644
index 000000000..c822f3a43
--- /dev/null
+++ b/headroom/backends/anyllm.py
@@ -0,0 +1,476 @@
+"""any-llm backend for Headroom.
+
+Talk to 38+ LLM providers (OpenAI, Mistral, Groq, Ollama, Bedrock, etc.)
+through a single interface. Auth and format translation handled automatically.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from collections.abc import AsyncIterator
+from typing import Any
+
+from .base import Backend, BackendResponse, StreamEvent
+
+logger = logging.getLogger(__name__)
+
+try:
+ from any_llm import acompletion
+
+ ANYLLM_AVAILABLE = True
+except ImportError:
+ ANYLLM_AVAILABLE = False
+ acompletion = None # type: ignore
+
+
+class AnyLLMBackend(Backend):
+ """Backend using any-llm for multi-provider support."""
+
+ def __init__(
+ self,
+ provider: str = "openai",
+ api_key: str | None = None,
+ api_base: str | None = None,
+ ):
+ if not ANYLLM_AVAILABLE:
+ raise ImportError(
+ "any-llm-sdk is required for AnyLLMBackend. "
+ "Install with: pip install 'any-llm-sdk[all]'"
+ )
+
+ self.provider = provider.lower()
+ self.api_key = api_key
+ self.api_base = api_base
+
+ logger.info(f"any-llm backend initialized (provider={provider})")
+
+ @property
+ def name(self) -> str:
+ return f"anyllm-{self.provider}"
+
+ def map_model_id(self, model: str) -> str:
+ """Pass through model name - any-llm handles provider-specific naming."""
+ return model
+
+ def supports_model(self, model: str) -> bool:
+ """any-llm supports any model the provider supports."""
+ return True
+
+ def _convert_messages_for_anyllm(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Convert Anthropic message format to OpenAI/any-llm format."""
+ converted = []
+ for msg in messages:
+ role = msg.get("role", "user")
+ content = msg.get("content", "")
+
+ if isinstance(content, str):
+ converted.append({"role": role, "content": content})
+ continue
+
+ if isinstance(content, list):
+ text_parts = []
+ has_complex_content = False
+
+ for block in content:
+ if isinstance(block, dict):
+ if block.get("type") == "text":
+ text_parts.append(block.get("text", ""))
+ elif block.get("type") in ("tool_use", "tool_result", "image"):
+ has_complex_content = True
+ break
+
+ if not has_complex_content and text_parts:
+ converted.append({"role": role, "content": "\n".join(text_parts)})
+ else:
+ openai_content = self._convert_content_blocks(content)
+ converted.append({"role": role, "content": openai_content})
+
+ return converted
+
+ def _convert_content_blocks(self, blocks: list[dict[str, Any]]) -> list[dict[str, Any]] | str:
+ """Convert Anthropic content blocks to OpenAI format."""
+ openai_blocks = []
+
+ for block in blocks:
+ block_type = block.get("type")
+
+ if block_type == "text":
+ openai_blocks.append({"type": "text", "text": block.get("text", "")})
+
+ elif block_type == "image":
+ source = block.get("source", {})
+ if source.get("type") == "base64":
+ media_type = source.get("media_type", "image/png")
+ data = source.get("data", "")
+ openai_blocks.append(
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:{media_type};base64,{data}"},
+ }
+ )
+ elif source.get("type") == "url":
+ openai_blocks.append(
+ {"type": "image_url", "image_url": {"url": source.get("url", "")}}
+ )
+
+ if len(openai_blocks) == 1 and openai_blocks[0].get("type") == "text":
+ return openai_blocks[0]["text"]
+
+ return openai_blocks if openai_blocks else ""
+
+ def _to_anthropic_response(
+ self,
+ anyllm_response: Any,
+ original_model: str,
+ ) -> dict[str, Any]:
+ """Convert any-llm/OpenAI response to Anthropic format."""
+ msg_id = f"msg_{uuid.uuid4().hex[:24]}"
+
+ choice = anyllm_response.choices[0]
+ message = choice.message
+
+ content = []
+ if message.content:
+ content.append({"type": "text", "text": message.content})
+
+ if hasattr(message, "tool_calls") and message.tool_calls:
+ for tc in message.tool_calls:
+ content.append(
+ {
+ "type": "tool_use",
+ "id": tc.id,
+ "name": tc.function.name,
+ "input": tc.function.arguments,
+ }
+ )
+
+ stop_reason_map = {
+ "stop": "end_turn",
+ "length": "max_tokens",
+ "tool_calls": "tool_use",
+ "content_filter": "end_turn",
+ }
+ finish_reason = getattr(choice, "finish_reason", "stop") or "stop"
+ stop_reason = stop_reason_map.get(finish_reason, "end_turn")
+
+ usage = {"input_tokens": 0, "output_tokens": 0}
+ if hasattr(anyllm_response, "usage") and anyllm_response.usage:
+ usage = {
+ "input_tokens": getattr(anyllm_response.usage, "prompt_tokens", 0) or 0,
+ "output_tokens": getattr(anyllm_response.usage, "completion_tokens", 0) or 0,
+ }
+
+ return {
+ "id": msg_id,
+ "type": "message",
+ "role": "assistant",
+ "content": content,
+ "model": original_model,
+ "stop_reason": stop_reason,
+ "stop_sequence": None,
+ "usage": usage,
+ }
+
+ async def send_message(
+ self,
+ body: dict[str, Any],
+ headers: dict[str, str],
+ ) -> BackendResponse:
+ """Send message via any-llm."""
+ original_model = body.get("model", "gpt-4o")
+
+ try:
+ messages = self._convert_messages_for_anyllm(body.get("messages", []))
+
+ kwargs: dict[str, Any] = {
+ "model": original_model,
+ "provider": self.provider,
+ "messages": messages,
+ "stream": False,
+ }
+
+ if "max_tokens" in body:
+ kwargs["max_tokens"] = body["max_tokens"]
+ if "temperature" in body:
+ kwargs["temperature"] = body["temperature"]
+ if "top_p" in body:
+ kwargs["top_p"] = body["top_p"]
+ if "stop_sequences" in body:
+ kwargs["stop"] = body["stop_sequences"]
+
+ if "system" in body:
+ system = body["system"]
+ if isinstance(system, str):
+ kwargs["messages"].insert(0, {"role": "system", "content": system})
+ elif isinstance(system, list):
+ system_text = " ".join(
+ s.get("text", "") if isinstance(s, dict) else str(s) for s in system
+ )
+ kwargs["messages"].insert(0, {"role": "system", "content": system_text})
+
+ if self.api_key:
+ kwargs["api_key"] = self.api_key
+ if self.api_base:
+ kwargs["api_base"] = self.api_base
+
+ logger.debug(f"any-llm request: provider={self.provider}, model={original_model}")
+
+ response = await acompletion(**kwargs)
+ anthropic_response = self._to_anthropic_response(response, original_model)
+
+ return BackendResponse(
+ body=anthropic_response,
+ status_code=200,
+ headers={"content-type": "application/json"},
+ )
+
+ except Exception as e:
+ logger.error(f"any-llm error: {e}")
+
+ error_type = "api_error"
+ status_code = 500
+
+ error_str = str(e).lower()
+ if "authentication" in error_str or "api_key" in error_str or "api key" in error_str:
+ error_type = "authentication_error"
+ status_code = 401
+ elif "rate" in error_str or "limit" in error_str:
+ error_type = "rate_limit_error"
+ status_code = 429
+ elif "not found" in error_str or "model" in error_str:
+ error_type = "not_found_error"
+ status_code = 404
+
+ return BackendResponse(
+ body={
+ "type": "error",
+ "error": {"type": error_type, "message": str(e)},
+ },
+ status_code=status_code,
+ error=str(e),
+ )
+
+ async def stream_message(
+ self,
+ body: dict[str, Any],
+ headers: dict[str, str],
+ ) -> AsyncIterator[StreamEvent]:
+ """Stream message via any-llm."""
+ original_model = body.get("model", "gpt-4o")
+
+ try:
+ messages = self._convert_messages_for_anyllm(body.get("messages", []))
+
+ kwargs: dict[str, Any] = {
+ "model": original_model,
+ "provider": self.provider,
+ "messages": messages,
+ "stream": True,
+ }
+
+ if "max_tokens" in body:
+ kwargs["max_tokens"] = body["max_tokens"]
+ if "temperature" in body:
+ kwargs["temperature"] = body["temperature"]
+ if "system" in body:
+ system = body["system"]
+ if isinstance(system, str):
+ kwargs["messages"].insert(0, {"role": "system", "content": system})
+
+ if self.api_key:
+ kwargs["api_key"] = self.api_key
+ if self.api_base:
+ kwargs["api_base"] = self.api_base
+
+ msg_id = f"msg_{uuid.uuid4().hex[:24]}"
+
+ yield StreamEvent(
+ event_type="message_start",
+ data={
+ "type": "message_start",
+ "message": {
+ "id": msg_id,
+ "type": "message",
+ "role": "assistant",
+ "content": [],
+ "model": original_model,
+ "stop_reason": None,
+ "stop_sequence": None,
+ "usage": {"input_tokens": 0, "output_tokens": 0},
+ },
+ },
+ )
+
+ yield StreamEvent(
+ event_type="content_block_start",
+ data={
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {"type": "text", "text": ""},
+ },
+ )
+
+ response = await acompletion(**kwargs)
+ output_tokens = 0
+
+ async for chunk in response:
+ if hasattr(chunk, "choices") and chunk.choices:
+ delta = chunk.choices[0].delta
+ if hasattr(delta, "content") and delta.content:
+ yield StreamEvent(
+ event_type="content_block_delta",
+ data={
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "text_delta", "text": delta.content},
+ },
+ )
+ output_tokens += 1
+
+ yield StreamEvent(
+ event_type="content_block_stop",
+ data={"type": "content_block_stop", "index": 0},
+ )
+
+ yield StreamEvent(
+ event_type="message_delta",
+ data={
+ "type": "message_delta",
+ "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+ "usage": {"output_tokens": output_tokens},
+ },
+ )
+
+ yield StreamEvent(
+ event_type="message_stop",
+ data={"type": "message_stop"},
+ )
+
+ except Exception as e:
+ logger.error(f"any-llm streaming error: {e}")
+ yield StreamEvent(
+ event_type="error",
+ data={
+ "type": "error",
+ "error": {"type": "api_error", "message": str(e)},
+ },
+ )
+
+ async def send_openai_message(
+ self,
+ body: dict[str, Any],
+ headers: dict[str, str],
+ ) -> BackendResponse:
+ """Send OpenAI-format message via any-llm (no conversion)."""
+ original_model = body.get("model", "gpt-4o")
+
+ try:
+ kwargs: dict[str, Any] = {
+ "model": original_model,
+ "provider": self.provider,
+ "messages": body.get("messages", []),
+ "stream": False,
+ }
+
+ for param in [
+ "max_tokens",
+ "temperature",
+ "top_p",
+ "stop",
+ "tools",
+ "tool_choice",
+ "response_format",
+ "seed",
+ "n",
+ ]:
+ if param in body:
+ kwargs[param] = body[param]
+
+ if self.api_key:
+ kwargs["api_key"] = self.api_key
+ if self.api_base:
+ kwargs["api_base"] = self.api_base
+
+ logger.debug(f"any-llm OpenAI request: provider={self.provider}, model={original_model}")
+
+ response = await acompletion(**kwargs)
+
+ response_dict = {
+ "id": response.id,
+ "object": "chat.completion",
+ "created": response.created,
+ "model": original_model,
+ "choices": [
+ {
+ "index": c.index,
+ "message": {
+ "role": c.message.role,
+ "content": c.message.content,
+ **(
+ {
+ "tool_calls": [
+ {
+ "id": tc.id,
+ "type": "function",
+ "function": {
+ "name": tc.function.name,
+ "arguments": tc.function.arguments,
+ },
+ }
+ for tc in c.message.tool_calls
+ ]
+ }
+ if c.message.tool_calls
+ else {}
+ ),
+ },
+ "finish_reason": c.finish_reason,
+ }
+ for c in response.choices
+ ],
+ "usage": {
+ "prompt_tokens": response.usage.prompt_tokens if response.usage else 0,
+ "completion_tokens": response.usage.completion_tokens if response.usage else 0,
+ "total_tokens": response.usage.total_tokens if response.usage else 0,
+ },
+ }
+
+ return BackendResponse(
+ body=response_dict,
+ status_code=200,
+ headers={"content-type": "application/json"},
+ )
+
+ except Exception as e:
+ logger.error(f"any-llm OpenAI error: {e}")
+
+ error_type = "api_error"
+ status_code = 500
+
+ error_str = str(e).lower()
+ if "authentication" in error_str or "api_key" in error_str:
+ error_type = "invalid_api_key"
+ status_code = 401
+ elif "rate" in error_str or "limit" in error_str:
+ error_type = "rate_limit_exceeded"
+ status_code = 429
+ elif "not found" in error_str:
+ error_type = "model_not_found"
+ status_code = 404
+
+ return BackendResponse(
+ body={
+ "error": {
+ "message": str(e),
+ "type": error_type,
+ "code": error_type,
+ }
+ },
+ status_code=status_code,
+ error=str(e),
+ )
+
+ async def close(self) -> None:
+ """Clean up (no-op for any-llm)."""
+ pass
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index 6c094b04e..0fcc98179 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -73,9 +73,14 @@ from .main import main
default="anthropic",
help=(
"API backend: 'anthropic' (direct), 'bedrock' (AWS), 'openrouter' (OpenRouter), "
- "or 'litellm-<provider>' (e.g., litellm-vertex)"
+ "'anyllm' (any-llm), or 'litellm-<provider>' (e.g., litellm-vertex)"
),
)
+ "--anyllm-provider",
+ default="openai",
+ help="Provider for any-llm backend: openai, mistral, groq, ollama, etc. (default: openai)",
+)
@click.option(
"--region",
default="us-west-2",
@@ -114,6 +119,7 @@ def proxy(
no_memory_context: bool,
memory_top_k: int,
backend: str,
+ anyllm_provider: str,
region: str,
bedrock_region: str | None,
bedrock_profile: str | None,
@@ -166,10 +172,11 @@ def proxy(
memory_inject_tools=not no_memory_tools,
memory_inject_context=not no_memory_context,
memory_top_k=memory_top_k,
- # Backend (Anthropic direct, Bedrock, or LiteLLM)
+ # Backend (Anthropic direct, Bedrock, LiteLLM, or any-llm)
backend=backend,
bedrock_region=bedrock_region or region,
bedrock_profile=bedrock_profile,
+ anyllm_provider=anyllm_provider,
)
memory_status = "DISABLED"
@@ -180,8 +187,15 @@ def proxy(
backend_status = "Anthropic (direct API)"
backend_section = ""
- if config.backend != "anthropic":
- # Get provider config from registry
+ if config.backend == "anyllm" or config.backend.startswith("anyllm-"):
+ # any-llm backend
+ backend_status = f"{anyllm_provider.title()} via any-llm"
+ backend_section = """
+ Set credentials for your provider (e.g., OPENAI_API_KEY, MISTRAL_API_KEY)
+ Providers: https://mozilla-ai.github.io/any-llm/providers/
+"""
+ elif config.backend != "anthropic":
+ # LiteLLM backend
from headroom.backends.litellm import get_provider_config
provider = config.backend.replace("litellm-", "")
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 037cd4cea..7bb9aeccb 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -57,7 +57,7 @@ except ImportError:
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from headroom import __version__
-from headroom.backends import LiteLLMBackend
+from headroom.backends import AnyLLMBackend, LiteLLMBackend
from headroom.backends.base import Backend
from headroom.cache.compression_feedback import get_compression_feedback
from headroom.cache.compression_store import get_compression_store
@@ -213,11 +213,13 @@ class ProxyConfig:
openai_api_url: str | None = None # Custom OpenAI API URL override
gemini_api_url: str | None = None # Custom Gemini API URL override
- # Backend: "anthropic" (direct API), "bedrock" (AWS Bedrock), or "litellm-*" (via LiteLLM)
+ # Backend: "anthropic" (direct API), "litellm-*" (via LiteLLM), or "anyllm" (via any-llm)
# LiteLLM backends: "litellm-bedrock", "litellm-vertex", "litellm-azure", etc.
+ # any-llm backends: "anyllm" with --anyllm-provider (openai, mistral, groq, etc.)
backend: str = "anthropic"
bedrock_region: str = "us-west-2" # AWS region for Bedrock/LiteLLM
bedrock_profile: str | None = None # AWS profile (optional)
+ anyllm_provider: str = "openai" # any-llm provider (openai, mistral, groq, etc.)
# Optimization
optimize: bool = True
@@ -1080,28 +1082,41 @@ class HeadroomProxy:
# HTTP client
self.http_client: httpx.AsyncClient | None = None
- # Backend for Anthropic API (direct or via LiteLLM)
- # Supports: "anthropic" (direct), "bedrock", "vertex", or "litellm-<provider>"
+ # Backend for Anthropic API (direct, LiteLLM, or any-llm)
+ # Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"
self.anthropic_backend: Backend | None = None
if config.backend != "anthropic":
- # Normalize backend name: "bedrock" -> "litellm-bedrock"
backend = config.backend
- if not backend.startswith("litellm-"):
- backend = f"litellm-{backend}"
- provider = backend.replace("litellm-", "")
- try:
- self.anthropic_backend = LiteLLMBackend(
- provider=provider,
- region=config.bedrock_region,
- )
- logger.info(
- f"LiteLLM backend enabled (provider={provider}, region={config.bedrock_region})"
- )
- except ImportError as e:
- logger.warning(f"LiteLLM backend not available: {e}")
- except Exception as e:
- logger.error(f"Failed to initialize LiteLLM backend: {e}")
+ # Handle any-llm backend
+ if backend == "anyllm" or backend.startswith("anyllm-"):
+ provider = config.anyllm_provider
+ try:
+ self.anthropic_backend = AnyLLMBackend(provider=provider)
+ logger.info(f"any-llm backend enabled (provider={provider})")
+ except ImportError as e:
+ logger.warning(f"any-llm backend not available: {e}")
+ except Exception as e:
+ logger.error(f"Failed to initialize any-llm backend: {e}")
+ else:
+ # Handle LiteLLM backend
+ # Normalize backend name: "bedrock" -> "litellm-bedrock"
+ if not backend.startswith("litellm-"):
+ backend = f"litellm-{backend}"
+ provider = backend.replace("litellm-", "")
+
+ try:
+ self.anthropic_backend = LiteLLMBackend(
+ provider=provider,
+ region=config.bedrock_region,
+ )
+ logger.info(
+ f"LiteLLM backend enabled (provider={provider}, region={config.bedrock_region})"
+ )
+ except ImportError as e:
+ logger.warning(f"LiteLLM backend not available: {e}")
+ except Exception as e:
+ logger.error(f"Failed to initialize LiteLLM backend: {e}")
# Request counter for IDs
self._request_counter = 0
@@ -6527,6 +6542,8 @@ def run_server(
# Backend status - use provider registry for display info
if config.backend == "anthropic":
backend_status = "ANTHROPIC (direct API)"
+ elif config.backend == "anyllm" or config.backend.startswith("anyllm-"):
+ backend_status = f"{config.anyllm_provider.title()} via any-llm"
else:
from headroom.backends.litellm import get_provider_config
@@ -6637,12 +6654,12 @@ if __name__ == "__main__":
"--openai-api-url", help=f"Custom OpenAI API URL (default: {HeadroomProxy.OPENAI_API_URL})"
)
- # Backend (anthropic direct, bedrock, or openrouter)
+ # Backend (anthropic direct, bedrock, openrouter, or anyllm)
parser.add_argument(
"--backend",
- choices=["anthropic", "bedrock", "openrouter"],
+ choices=["anthropic", "bedrock", "openrouter", "anyllm"],
default="anthropic",
- help="Backend for Anthropic API: 'anthropic' (direct), 'bedrock' (AWS), or 'openrouter' (OpenRouter)",
+ help="Backend for Anthropic API: 'anthropic' (direct), 'bedrock' (AWS), 'openrouter', or 'anyllm' (any-llm)",
)
parser.add_argument(
"--bedrock-region",
@@ -6657,6 +6674,11 @@ if __name__ == "__main__":
"--openrouter-api-key",
help="OpenRouter API key (or set OPENROUTER_API_KEY env var)",
)
+ parser.add_argument(
+ "--anyllm-provider",
+ default="openai",
+ help="any-llm provider: openai, anthropic, mistral, groq, ollama, bedrock, etc. (default: openai)",
+ )
# Connection pool (scalability)
parser.add_argument(
@@ -6791,6 +6813,7 @@ if __name__ == "__main__":
backend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]
bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", args.bedrock_region),
bedrock_profile=args.bedrock_profile or os.environ.get("AWS_PROFILE"),
+ anyllm_provider=_get_env_str("HEADROOM_ANYLLM_PROVIDER", args.anyllm_provider),
optimize=optimize,
min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", args.min_tokens),
max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", args.max_items),
diff --git a/pyproject.toml b/pyproject.toml
index e3fee8b03..51d7cd725 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -86,6 +86,10 @@ llmlingua = [
code = [
"tree-sitter-language-pack>=0.10.0",
]
+# any-llm multi-provider backend (requires Python 3.11+)
+anyllm = [
+ "any-llm-sdk>=1.0.0",
+]
# Agno agent framework integration
agno = [
"agno>=1.0.0",