Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""ChatUnifier — the single SERVICE facade for chat-model dispatch.
Reads the ``ProviderRegistry`` to route ``chat`` and ``fetch_models`` calls
to the right provider implementation, translates each provider's typed
SDK exceptions into ``NodeUserError`` at one catch site, and applies the
``incompatible_models`` JSON filter uniformly. ``services/ai.py`` delegates
to this class — there is no per-provider Python anywhere else.
Wired into the DI container once at startup with the parsed
``llm_defaults.json`` + the ``AuthService`` (for ``{provider}_proxy``
credential lookups).
"""
from __future__ import annotations
import asyncio
import hashlib
import json
from collections import OrderedDict
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, TYPE_CHECKING
import httpx
from core.encryption import fingerprint_credential
from core.logging import get_logger
from services.llm.config import resolve_credential, split_provider_ref, supports_effort
from services.llm.endpoints import base_url_key, redact_url, unconfigured_endpoint_message
from services.llm.protocol import (
LLMError,
LLMErrorCategory,
LLMProvider,
LLMResponse,
Message,
StreamEvent,
StreamSink,
ThinkingConfig,
ToolDef,
)
from services.llm.registry import ProviderSpec, get_provider, has_provider
from services.plugin import NodeUserError
if TYPE_CHECKING:
from services.auth import AuthService
from services.gateway.routing import GatewayRoute, GatewayRouter
logger = get_logger(__name__)
def _provider_failure(provider: str, exc: BaseException) -> LLMError:
"""Normalize typed transport errors that native SDKs can pass through."""
error = LLMError.from_exception(provider, exc)
if isinstance(exc, httpx.TransportError):
error.category = LLMErrorCategory.TIMEOUT if isinstance(exc, httpx.TimeoutException) else LLMErrorCategory.CONNECTION
error.retryable = True
return error
def _gateway_request_extras(route: "GatewayRoute") -> Dict[str, Any]:
"""Per-request headers and body for a call the gateway serves."""
from services.gateway.attribution import replay_cache, request_headers
extras: Dict[str, Any] = {"extra_headers": request_headers()}
cache = replay_cache(route.api_key)
if cache is not None:
extras["extra_body"] = cache
return extras
def _explain_gateway_failure(error: LLMError, exc: BaseException) -> None:
"""Gateway wording for a failed gateway call (services/gateway/errors.py)."""
from services.gateway.errors import explain
explain(error, exc)
@dataclass(eq=False)
class _ClientEntry:
"""One cached SDK client plus its in-flight lease count."""
client: LLMProvider
leases: int = 0
retired: bool = False
class ChatUnifier:
"""Single facade for chat-model execution.
Construction: one instance lives in the DI container, sharing
``llm_defaults.json`` + ``AuthService`` with the rest of the backend.
Public surface: ``chat()``, ``fetch_models()``, ``is_registered()``.
Everything else is private.
``gateway`` (services/gateway/routing.py) sends a chat call that comes
with no key of the install's own to the OpenCompany Gateway when it serves
the provider. ``fetch_models`` never goes there: it validates a key
against its own vendor.
"""
def __init__(
self,
defaults: Dict[str, Any],
auth_service: "AuthService",
*,
client_cache_size: int = 32,
gateway: Optional["GatewayRouter"] = None,
):
self._defaults = defaults
self._auth = auth_service
self._gateway = gateway
self._client_cache_size = max(0, int(client_cache_size))
self._client_cache: OrderedDict[str, _ClientEntry] = OrderedDict()
self._retired_entries: List[_ClientEntry] = []
self._cache_lock = asyncio.Lock()
# ------------------------------------------------------------------
# public API
# ------------------------------------------------------------------
async def chat(
self,
*,
provider: str,
api_key: str,
messages: List[Message],
model: str,
temperature: float = 0.7,
max_tokens: int = 4096,
thinking: Optional[ThinkingConfig] = None,
tools: Optional[List[ToolDef]] = None,
context_management: Optional[Dict[str, Any]] = None,
sdk_max_retries: int = 2,
translate_errors: bool = True,
on_event: Optional[StreamSink] = None,
effort: Optional[str] = None,
) -> LLMResponse:
"""Execute a chat completion against the named provider.
Raises ``NodeUserError`` on typed SDK failures (bad key, context
overflow, missing model, server unreachable, …) — every other
exception flows through unchanged so genuine server bugs keep
their full traceback via ``BaseNode.execute()`` 's generic
``except Exception``.
``provider`` is a provider reference: a registered id, or a named
endpoint ``openai_compatible:<slug>`` (RFC-0003 D13).
``on_event`` receives the response as it is produced. A provider
that declares ``streaming`` in llm_defaults.json streams it; for any
other the events come from the finished response, one per kind, so
a caller handles both the same way. The response is the same either
way.
``effort`` is the owner's chat choice (``low`` / ``high``). It reaches
the provider only for a model listed in its ``effort_models``; any
other model answers as it would without it.
An empty ``api_key`` means the install has no key of its own for the
provider. If the OpenCompany Gateway serves it and is connected, the
call goes there with the gateway key, attribution headers and, inside
a Temporal activity, a replay-cache namespace (services/gateway).
"""
from services.llm.observation import observe_stage
observe_stage("routing")
provider_id = split_provider_ref(provider)[0]
spec = get_provider(provider_id)
streams = on_event is not None and self._streams(provider_id)
route = await self._gateway.route(provider) if not api_key and self._gateway is not None else None
api_key = route.api_key if route is not None else self._resolve_credential(provider, api_key)
entry: Optional[_ClientEntry] = None
try:
observe_stage("client_acquisition")
entry = await self._acquire_client(
spec, api_key, provider_ref=provider, sdk_max_retries=sdk_max_retries, route=route
)
kwargs: Dict[str, Any] = {
"model": model,
"temperature": temperature,
"max_tokens": max_tokens,
"thinking": thinking,
"tools": tools,
"context_management": context_management,
}
if streams:
kwargs["on_event"] = on_event
if effort and supports_effort(provider, model):
kwargs["effort"] = effort
if route is not None:
kwargs.update(_gateway_request_extras(route))
observe_stage("sdk_request")
response = await entry.client.chat(messages, **kwargs)
observe_stage("delivery", finish_reason=response.finish_reason,
input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens)
if on_event is not None and not streams:
await _replay_response(response, on_event)
return response
except LLMError as error:
observe_stage("normalization", provider_request_id=error.request_id)
# Raised by provider logic rather than the SDK, e.g. a 2xx that
# carried an error body instead of a completion. Logged whether
# or not it is translated: agent steps take it untranslated, and
# only this layer knows where the call went (RFC-0003 D10).
if route is not None:
_explain_gateway_failure(error, error)
self._log_failure("LLM provider request failed", error, entry, provider)
if not translate_errors:
raise
raise error.as_node_error() from error
except spec.sdk_exception_types + (httpx.TransportError, httpx.HTTPStatusError, TimeoutError) as e:
# A raw TimeoutError (an SDK's own deadline) is a timeout to
# classify and retry, not a programming error.
error = _provider_failure(provider, e)
observe_stage("normalization", provider_request_id=error.request_id)
if route is not None:
_explain_gateway_failure(error, e)
self._log_failure("LLM provider request failed", error, entry, provider)
if not translate_errors:
raise error from e
raise error.as_node_error() from error
except (ValueError, TypeError, OSError) as e:
# Only normalize generic configuration/transport failures raised
# while constructing a provider client. Once a client exists,
# unexpected exceptions from provider logic remain programming
# errors and keep their traceback.
if entry is not None:
raise
category = (
LLMErrorCategory.CONNECTION
if isinstance(e, OSError)
else LLMErrorCategory.INVALID_REQUEST
)
error = LLMError(
message=str(e),
provider=provider,
category=category,
retryable=category == LLMErrorCategory.CONNECTION,
)
if route is not None:
_explain_gateway_failure(error, e)
logger.warning(
"LLM client construction failed",
provider=error.provider,
category=error.category.value,
retryable=error.retryable,
)
if not translate_errors:
raise error from e
raise error.as_node_error() from error
finally:
if entry is not None:
observe_stage("client_release")
await self._release_client(entry)
async def fetch_models(self, *, provider: str, api_key: str) -> List[str]:
"""List available models from the named provider.
Applies the JSON-driven ``incompatible_models`` filter
(``providers.<name>.incompatible_models`` in ``llm_defaults.json``)
uniformly. An absent key is a no-op — every provider gets the
filter for free without per-provider Python.
"""
spec = get_provider(split_provider_ref(provider)[0])
api_key = self._resolve_credential(provider, api_key)
entry: Optional[_ClientEntry] = None
try:
entry = await self._acquire_client(
spec, api_key, provider_ref=provider, sdk_max_retries=2
)
models = await entry.client.fetch_models(api_key)
except LLMError as error:
self._log_failure("LLM model-list request failed", error, entry, provider)
raise error.as_node_error() from error
except spec.sdk_exception_types + (httpx.TransportError, httpx.HTTPStatusError) as e:
error = _provider_failure(provider, e)
self._log_failure("LLM model-list request failed", error, entry, provider)
raise error.as_node_error() from error
except (ValueError, TypeError, OSError) as e:
if entry is not None:
raise
category = (
LLMErrorCategory.CONNECTION
if isinstance(e, OSError)
else LLMErrorCategory.INVALID_REQUEST
)
error = LLMError(
message=str(e),
provider=provider,
category=category,
retryable=category == LLMErrorCategory.CONNECTION,
)
logger.warning(
"LLM client construction failed",
provider=error.provider,
category=error.category.value,
retryable=error.retryable,
)
raise error.as_node_error() from error
finally:
if entry is not None:
await self._release_client(entry)
blocklist = self._incompatible_models(provider)
if not blocklist:
return models
return [m for m in models if m not in blocklist]
def _streams(self, provider_id: str) -> bool:
"""Whether the provider streams its responses (``streaming`` in
llm_defaults.json). Declared per provider once its stream is known
to report what the finished response does, usage included."""
config = (self._defaults.get("providers") or {}).get(provider_id) or {}
return config.get("streaming") is True
def is_registered(self, provider: str) -> bool:
"""Cheap probe used by callers that want graceful fallback when a
provider is not wired into the registry.
Replaces the legacy ``is_native_provider`` function in
``services/llm/factory.py`` — the unifier IS the routing layer,
so registry membership is the source of truth.
"""
return has_provider(split_provider_ref(provider)[0])
# ------------------------------------------------------------------
# internals
# ------------------------------------------------------------------
async def aclose(self) -> None:
"""Close and evict all cached SDK clients during worker shutdown."""
to_close: List[LLMProvider] = []
async with self._cache_lock:
entries = list(self._client_cache.values())
self._client_cache.clear()
for entry in entries:
entry.retired = True
if entry.leases:
if entry not in self._retired_entries:
self._retired_entries.append(entry)
else:
to_close.append(entry.client)
for client in to_close:
await self._close_client(client)
async def _build_client(
self,
spec: ProviderSpec,
api_key: str,
*,
sdk_max_retries: int,
provider_ref: Optional[str] = None,
) -> LLMProvider:
"""Instantiate the provider implementation.
Pulls the user-configured ``{provider}_proxy`` URL from the
encrypted credentials store and merges it with the provider's
static ``client_kwargs`` (used by OpenAI-compatible providers to
pin their ``base_url``).
"""
entry = await self._get_or_create_entry(
spec,
api_key,
provider_ref=provider_ref,
sdk_max_retries=sdk_max_retries,
acquire=False,
)
return entry.client
async def _acquire_client(
self,
spec: ProviderSpec,
api_key: str,
*,
sdk_max_retries: int,
provider_ref: Optional[str] = None,
route: Optional["GatewayRoute"] = None,
) -> _ClientEntry:
"""Return an entry leased until ``_release_client`` is called."""
return await self._get_or_create_entry(
spec,
api_key,
provider_ref=provider_ref,
sdk_max_retries=sdk_max_retries,
acquire=True,
route=route,
)
async def _get_or_create_entry(
self,
spec: ProviderSpec,
api_key: str,
*,
sdk_max_retries: int,
acquire: bool,
provider_ref: Optional[str] = None,
route: Optional["GatewayRoute"] = None,
) -> _ClientEntry:
# The base-URL row is keyed by the full reference, so each named
# endpoint reads its own URL while sharing one registration.
ref = provider_ref or spec.name
from services.llm.observation import observe_stage
if route is not None:
# A gateway call goes to the gateway, never to a saved base URL.
proxy_url: Optional[str] = route.base_url
else:
proxy_url = await self._auth.resolve_api_key(base_url_key(ref))
unconfigured = unconfigured_endpoint_message(ref)
if unconfigured and not proxy_url:
# A named endpoint exists only as its credential rows. Without
# the URL row the SDK would default to api.openai.com.
raise NodeUserError(unconfigured)
model_prefix = route.model_prefix if route is not None else ""
observe_stage("client_acquisition", route_source="gateway" if route is not None else "saved_proxy" if proxy_url else "provider_default")
factory_kwargs = {
"api_key": api_key,
"proxy_url": proxy_url,
"max_retries": max(0, int(sdk_max_retries)),
**spec.client_kwargs,
}
if model_prefix:
factory_kwargs["model_prefix"] = model_prefix
if not self._client_cache_size:
observe_stage("client_construction", cache_hit=False)
return _ClientEntry(
client=spec.factory(**factory_kwargs),
leases=1 if acquire else 0,
retired=True,
)
key = self._client_cache_key(
spec=spec,
api_key=api_key,
proxy_url=proxy_url,
sdk_max_retries=sdk_max_retries,
model_prefix=model_prefix,
)
evicted: Optional[_ClientEntry] = None
observe_stage("client_lock")
async with self._cache_lock:
cached = self._client_cache.get(key)
if cached is not None:
observe_stage("client_acquisition", cache_hit=True)
self._client_cache.move_to_end(key)
if acquire:
cached.leases += 1
return cached
observe_stage("client_construction", cache_hit=False)
entry = _ClientEntry(
client=spec.factory(**factory_kwargs),
leases=1 if acquire else 0,
)
self._client_cache[key] = entry
if len(self._client_cache) > self._client_cache_size:
_, evicted = self._client_cache.popitem(last=False)
evicted.retired = True
if evicted.leases:
self._retired_entries.append(evicted)
if evicted is not None:
if not evicted.leases:
await self._close_client(evicted.client)
return entry
async def _release_client(self, entry: _ClientEntry) -> None:
should_close = False
async with self._cache_lock:
if entry.leases:
entry.leases -= 1
if entry.retired and not entry.leases:
if entry in self._retired_entries:
self._retired_entries.remove(entry)
should_close = True
if should_close:
await self._close_client(entry.client)
@staticmethod
def _client_cache_key(
*,
spec: ProviderSpec,
api_key: str,
proxy_url: Optional[str],
sdk_max_retries: int,
model_prefix: str = "",
) -> str:
material = json.dumps(
{
"provider": spec.name,
"proxy_url": proxy_url or "",
"client_kwargs": spec.client_kwargs,
"max_retries": max(0, int(sdk_max_retries)),
"model_prefix": model_prefix,
},
sort_keys=True,
separators=(",", ":"),
default=str,
)
# The credential component is appended as a PBKDF2 fingerprint rather
# than folded into the SHA-256 material: credential-derived bytes must
# never enter a fast hash (CodeQL py/weak-sensitive-data-hashing).
return (
hashlib.sha256(material.encode("utf-8")).hexdigest()
+ ":"
+ fingerprint_credential(api_key)
)
@staticmethod
async def _close_client(client: LLMProvider) -> None:
close = getattr(client, "aclose", None)
if close is None:
return
try:
result = close()
if hasattr(result, "__await__"):
await result
except Exception as exc:
logger.warning("Failed to close cached LLM client", error=str(exc))
def _incompatible_models(self, provider: str) -> set[str]:
"""Read ``providers.<name>.incompatible_models`` from llm_defaults.json."""
raw = (
self._defaults.get("providers", {})
.get(split_provider_ref(provider)[0], {})
.get("incompatible_models")
)
return set(raw or ())
@staticmethod
def _resolve_credential(provider: str, api_key: Optional[str]) -> str:
"""The key to send, or a user-facing error before any client exists.
An empty key must never reach an SDK: the OpenAI SDK would read
``OPENAI_API_KEY`` and send it to the configured base URL
(RFC-0003 D7). Keyless local servers get their declared placeholder.
"""
try:
return resolve_credential(provider, api_key)
except ValueError:
raise NodeUserError(
f"No API key is configured for {provider!r}. Add one under Credentials."
) from None
@staticmethod
def _log_failure(event: str, error: LLMError, entry: Optional[_ClientEntry], provider: str) -> None:
"""One WARN line per provider failure, naming where the call went.
``provider`` is the reference the call was made with, so a named
endpoint keeps its slug even when the provider raised under its
generic name. ``url`` is redacted (no userinfo, query or fragment)
and ``url_source`` says which setting produced it, so a routing
mistake is distinguishable from a model failure (RFC-0003 D10).
The key is never logged.
"""
client = entry.client if entry is not None else None
logger.warning(
event,
provider=provider,
category=error.category.value,
retryable=error.retryable,
status_code=error.status_code,
provider_code=error.provider_code,
request_id=error.request_id,
retry_after=error.retry_after,
url=redact_url(getattr(client, "endpoint_url", None)),
url_source=getattr(client, "url_source", None),
**({"diagnostic_code": error.diagnostic_code} if error.diagnostic_code else {}),
)
async def _replay_response(response: LLMResponse, on_event: StreamSink) -> None:
"""Hand a finished response to a sink as if it had streamed: its
reasoning, then its text, each in one event."""
if response.thinking:
await on_event(StreamEvent("reasoning", response.thinking))
if response.content:
await on_event(StreamEvent("text", response.content))