Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Phase A1 contract — every shipped provider self-registers on import.
Locks the plugin shape: importing ``services.llm`` (and transitively
``services.llm.providers``) must populate the global registry with the
full set of native + OpenAI-compat providers. If any provider drops its
``register_provider(...)`` call, this test catches it.
"""
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import services.llm # noqa: F401 — side-effect import populates the registry
from services.llm.protocol import ToolDef
from services.llm.registry import all_providers, get_provider, has_provider
EXPECTED_REGISTERED_PROVIDERS = {
# Dedicated SDKs
"anthropic",
"openai",
"gemini",
"openrouter",
# OpenAI-compatible providers (registered via _compat.py)
"xai",
"deepseek",
"kimi",
"mistral",
"ollama",
"lmstudio",
# Migrated from LangChain fallback in Phase D — both expose
# OpenAI-compatible /v1 endpoints so they share OpenAIProvider.
"groq",
"cerebras",
# Indic-first; OpenAI-compatible chat endpoint but no model-list route,
# which it declares via `supports_model_listing: false`.
"sarvam",
# The one registration behind every user-named endpoint (RFC-0003 D13).
"openai_compatible",
}
COMPAT_PROVIDERS = (
"xai",
"deepseek",
"kimi",
"mistral",
"ollama",
"lmstudio",
"groq",
"cerebras",
"sarvam",
)
def test_all_expected_providers_registered():
"""Every native + compat provider populates the registry on import."""
assert set(all_providers()) >= EXPECTED_REGISTERED_PROVIDERS
def test_all_thirteen_providers_route_to_the_expected_native_class():
from services.llm.providers.anthropic import AnthropicProvider
from services.llm.providers.gemini import GeminiProvider
from services.llm.providers.openai import OpenAIProvider
from services.llm.providers.openrouter import OpenRouterProvider
assert get_provider("anthropic").factory is AnthropicProvider
assert get_provider("gemini").factory is GeminiProvider
assert get_provider("openai").factory is OpenAIProvider
assert get_provider("openrouter").factory is OpenRouterProvider
for name in COMPAT_PROVIDERS:
spec = get_provider(name)
assert spec.factory is OpenAIProvider
assert spec.client_kwargs["provider_name"] == name
def test_dedicated_providers_have_typed_sdk_exceptions():
"""The four dedicated SDKs declare their typed exception classes."""
import anthropic
import openai
from google.genai import errors as google_genai_errors
assert get_provider("anthropic").sdk_exception_types == (anthropic.APIError,)
assert get_provider("openai").sdk_exception_types == (openai.OpenAIError,)
assert get_provider("gemini").sdk_exception_types == (google_genai_errors.APIError,)
assert get_provider("openrouter").sdk_exception_types == (openai.OpenAIError,)
def test_compat_providers_carry_base_url_in_client_kwargs():
"""Every OpenAI-compat provider pins its base_url declaratively."""
expected_base_urls = {
"xai": "https://api.x.ai/v1",
"deepseek": "https://api.deepseek.com",
"kimi": "https://api.moonshot.ai/v1",
"mistral": "https://api.mistral.ai/v1",
"ollama": "http://localhost:11434/v1",
"lmstudio": "http://localhost:1234/v1",
"groq": "https://api.groq.com/openai/v1",
"cerebras": "https://api.cerebras.ai/v1",
"sarvam": "https://api.sarvam.ai/v1",
}
for name, url in expected_base_urls.items():
spec = get_provider(name)
assert spec.client_kwargs.get("base_url") == url, (
f"{name}: expected base_url={url!r}, got {spec.client_kwargs!r}"
)
@pytest.mark.parametrize("provider_name", COMPAT_PROVIDERS)
def test_every_compat_provider_inherits_shared_response_and_tool_contract(
provider_name,
):
"""One matrix gate covers normalization/schema behavior for all 9."""
from services.llm.providers.openai import OpenAIProvider
spec = get_provider(provider_name)
with patch("openai.AsyncOpenAI"):
provider = OpenAIProvider(
"key",
provider_name=provider_name,
base_url=spec.client_kwargs["base_url"],
)
response = SimpleNamespace(
choices=[
SimpleNamespace(
message=SimpleNamespace(
content="answer",
reasoning="reasoning",
reasoning_content=None,
reasoning_details=None,
refusal=None,
tool_calls=[
SimpleNamespace(
id="call-1",
function=SimpleNamespace(
name="lookup",
arguments="{not-json",
),
)
],
),
finish_reason="tool_calls",
)
],
usage=SimpleNamespace(
prompt_tokens=10,
completion_tokens=4,
total_tokens=14,
prompt_tokens_details=SimpleNamespace(cached_tokens=2),
completion_tokens_details=SimpleNamespace(reasoning_tokens=3),
),
)
normalized = provider._normalize(response, "provider-model")
assert normalized.content == "answer"
assert normalized.thinking == "reasoning"
assert normalized.finish_reason == "tool_calls"
assert normalized.usage.total_tokens == 14
assert normalized.usage.cache_read_tokens == 2
assert normalized.usage.reasoning_tokens == 3
assert normalized.tool_calls[0].raw_arguments == "{not-json"
assert normalized.tool_calls[0].parse_error
tool = provider._to_api_tool(
ToolDef(
name="lookup",
description="Lookup",
parameters={
"type": "object",
"properties": {
"mode": {"enum": ["fast", "deep"]},
"query": {
"anyOf": [
{"type": "string"},
{"type": "null"},
]
},
},
"required": ["mode"],
},
)
)
assert tool["function"]["parameters"]["properties"]["mode"]["enum"] == [
"fast",
"deep",
]
assert spec.sdk_exception_refs == ("openai:OpenAIError",)
def test_endpoint_provider_is_registered_once_without_a_base_url():
"""RFC-0003 AG17: named endpoints share one OpenAIProvider registration.
It must not pin a ``base_url``: each endpoint's resolved URL lives on its
own ``openai_compatible:<slug>_proxy`` row, and a pinned default would be
used silently when that row is missing.
"""
from services.llm.providers.openai import OpenAIProvider
spec = get_provider("openai_compatible")
assert spec.factory is OpenAIProvider
assert spec.client_kwargs == {"provider_name": "openai_compatible"}
assert spec.sdk_exception_refs == ("openai:OpenAIError",)
assert list(all_providers()).count("openai_compatible") == 1
def test_unknown_provider_raises_node_user_error():
"""Unknown provider lookups surface as NodeUserError (user-correctable)."""
from services.plugin import NodeUserError
with pytest.raises(NodeUserError, match="Unknown LLM provider"):
get_provider("definitely-not-real")
def test_has_provider_does_not_raise():
"""``has_provider`` is the cheap membership probe used by ai.py."""
assert has_provider("anthropic") is True
assert has_provider("definitely-not-real") is False