Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
perf(llm): resolve provider SDK exception types lazily at except time
trohitgc2b20217f148
diff --git a/server/services/ai.py b/server/services/ai.pyindex 4490db2d..8b53bf0b 100644--- a/server/services/ai.py+++ b/server/services/ai.py@@ -33,13 +33,13 @@ from typing import Awaitable, Dict, Any, List, Optional, Callable, Type, TYPE_CHfrom langchain_core.messages import BaseMessage # eager: type hint for _run_agent_loopimport json-# Eager imports — both are tiny and read on every chat/agent execution-# path. ``openai`` is the canonical SDK whose typed exception hierarchy-# (``BadRequestError`` / ``AuthenticationError`` / ``RateLimitError`` / …)-# is the contract this module dispatches on; ``NodeUserError`` is the-# framework's "user-correctable, no-traceback" sentinel used to surface-# those typed errors cleanly through ``BaseNode.execute()``.-import openai+# ``NodeUserError`` is the framework's "user-correctable, no-traceback"+# sentinel used to surface typed SDK errors cleanly through+# ``BaseNode.execute()``. The ``openai`` SDK itself is imported+# function-locally in ``execute_agent`` / ``execute_chat_agent`` (its+# only consumers here, via ``except openai.OpenAIError``) — a top-level+# import cost seconds of boot time and re-created the eager-LLM-SDK+# anti-pattern (docs-internal/performance.md).from services.plugin import NodeUserErrorif TYPE_CHECKING:@@ -1583,6 +1583,8 @@ class AIService:workflow_id: Optional workflow ID for scoped status broadcastscontext: Optional execution context with nodes, edges for nested agent delegation"""+ import openai # local: bound before the try whose except clause needs it+from langchain_core.messages import HumanMessage, SystemMessagestart_time = time.time()@@ -2131,6 +2133,8 @@ class AIService:workflow_id: Optional workflow ID for scoped status broadcastscontext: Optional execution context with nodes, edges for nested agent delegation"""+ import openai # local: bound before the try whose except clause needs it+from langchain_core.messages import HumanMessage, SystemMessagestart_time = time.time()diff --git a/server/services/llm/providers/_compat.py b/server/services/llm/providers/_compat.pyindex 92871913..55dba8b8 100644--- a/server/services/llm/providers/_compat.py+++ b/server/services/llm/providers/_compat.py@@ -17,8 +17,6 @@ from __future__ import annotationsfrom typing import Tuple-import openai as _openai_sdk-from core.logging import get_loggerfrom services.llm.providers.openai import OpenAIProviderfrom services.llm.registry import ProviderSpec, register_provider@@ -100,7 +98,7 @@ def _register_compat_providers() -> None:ProviderSpec(name=name,factory=OpenAIProvider,- sdk_exception_types=(_openai_sdk.OpenAIError,),+ sdk_exception_refs=("openai:OpenAIError",),client_kwargs={"base_url": base_url},))diff --git a/server/services/llm/providers/anthropic.py b/server/services/llm/providers/anthropic.pyindex ac3e613c..b24b0727 100644--- a/server/services/llm/providers/anthropic.py+++ b/server/services/llm/providers/anthropic.py@@ -185,19 +185,18 @@ class AnthropicProvider:# ---------------------------------------------------------------------------# Plugin self-registration# ----------------------------------------------------------------------------# Module load triggers registration into the global ProviderRegistry. The-# eager top-level ``import anthropic`` is required to resolve the typed-# ``APIError`` class used by the unifier's translation layer; the SDK is-# small enough that the cost is in the same band as the existing eager-# ``import openai`` in ``services/ai.py``.+# Module load triggers registration into the global ProviderRegistry.+# The typed ``APIError`` class is declared as a lazy "module:Class" ref+# (resolved via ``pkgutil.resolve_name`` at except/read time) so that+# registration never imports the SDK — the eager import here used to+# cost seconds of boot time (docs-internal/performance.md).-import anthropic as _anthropic_sdkfrom services.llm.registry import ProviderSpec, register_providerregister_provider(ProviderSpec(name="anthropic",factory=AnthropicProvider,- sdk_exception_types=(_anthropic_sdk.APIError,),+ sdk_exception_refs=("anthropic:APIError",),))diff --git a/server/services/llm/providers/gemini.py b/server/services/llm/providers/gemini.pyindex 1e97fa29..9323c6be 100644--- a/server/services/llm/providers/gemini.py+++ b/server/services/llm/providers/gemini.py@@ -268,14 +268,17 @@ class GeminiProvider:# translates it into ``NodeUserError`` at one catch site so the error# surfaces as a single WARN line through ``BaseNode.execute()`` instead# of bubbling up as a bare ``RuntimeError`` traceback.+#+# Declared as a lazy "module:Class" ref — ``google.genai`` is the single+# most expensive SDK import on the boot path (~4s warm / ~15s cold via+# google.auth + api_core + protobuf); the ref defers it to first use.-from google.genai import errors as _google_genai_errorsfrom services.llm.registry import ProviderSpec, register_providerregister_provider(ProviderSpec(name="gemini",factory=GeminiProvider,- sdk_exception_types=(_google_genai_errors.APIError,),+ sdk_exception_refs=("google.genai.errors:APIError",),))diff --git a/server/services/llm/providers/openai.py b/server/services/llm/providers/openai.pyindex f3dfe146..58994a94 100644--- a/server/services/llm/providers/openai.py+++ b/server/services/llm/providers/openai.py@@ -179,22 +179,21 @@ class OpenAIProvider:# ---------------------------------------------------------------------------# Plugin self-registration# ----------------------------------------------------------------------------# Registers ``openai`` into the global registry. The ``openai`` SDK is-# already eagerly imported by ``services/ai.py`` so the module-level-# import here is free (Python caches the module).+# Registers ``openai`` into the global registry. The typed+# ``OpenAIError`` class is declared as a lazy "module:Class" ref so+# registration never imports the SDK at boot.## OpenAI-compatible providers (xai / deepseek / kimi / mistral / ollama /# lmstudio) register separately in ``_compat.py`` — they reuse# ``OpenAIProvider`` as their factory but pin ``base_url`` via# ``ProviderSpec.client_kwargs`` rather than per-provider Python.-import openai as _openai_sdkfrom services.llm.registry import ProviderSpec, register_providerregister_provider(ProviderSpec(name="openai",factory=OpenAIProvider,- sdk_exception_types=(_openai_sdk.OpenAIError,),+ sdk_exception_refs=("openai:OpenAIError",),))diff --git a/server/services/llm/providers/openrouter.py b/server/services/llm/providers/openrouter.pyindex 172ff1b6..032fd800 100644--- a/server/services/llm/providers/openrouter.py+++ b/server/services/llm/providers/openrouter.py@@ -85,15 +85,14 @@ class OpenRouterProvider(OpenAIProvider):# ---------------------------------------------------------------------------# OpenRouter rides the OpenAI Python SDK with a different ``base_url`` +# extra headers, so its typed exceptions are ``openai.OpenAIError``-# subclasses — same exception tuple as the openai provider above.+# subclasses — same lazy exception ref as the openai provider.-import openai as _openai_sdkfrom services.llm.registry import ProviderSpec, register_providerregister_provider(ProviderSpec(name="openrouter",factory=OpenRouterProvider,- sdk_exception_types=(_openai_sdk.OpenAIError,),+ sdk_exception_refs=("openai:OpenAIError",),))diff --git a/server/services/llm/registry.py b/server/services/llm/registry.pyindex 8c5baa80..8fc47b7f 100644--- a/server/services/llm/registry.py+++ b/server/services/llm/registry.py@@ -9,6 +9,8 @@ inside ``services/ai.py`` after this layer is wired.from __future__ import annotations+import functools+import pkgutilfrom dataclasses import dataclass, fieldfrom typing import Any, Callable, Dict, List, Tuple, Type@@ -19,6 +21,22 @@ from services.plugin import NodeUserErrorlogger = get_logger(__name__)[email protected]_cache(maxsize=None)+def _resolve_exc(ref: str) -> Type[BaseException]:+ """Resolve a ``"module:ClassName"`` ref to the exception class.++ Called lazily from ``ProviderSpec.sdk_exception_types`` — by the time+ anything needs the resolved classes (an ``except`` clause matching a+ raised SDK error, or a test reading the tuple), the provider factory+ has already imported the SDK to build its client, so this is a+ ``sys.modules`` cache hit, never a cold import on the boot path.+ """+ obj = pkgutil.resolve_name(ref)+ if not (isinstance(obj, type) and issubclass(obj, BaseException)):+ raise TypeError(f"{ref!r} did not resolve to an exception class: {obj!r}")+ return obj++@dataclass(frozen=True)class ProviderSpec:"""Declarative spec for a chat-model provider plugin.@@ -29,11 +47,14 @@ class ProviderSpec:``providers.<name>`` block in ``llm_defaults.json``.factory: callable returning an ``LLMProvider`` instance.Invoked as ``factory(api_key=..., proxy_url=..., **client_kwargs)``.- sdk_exception_types: tuple of typed SDK error classes raised by- this provider's underlying SDK. The unifier wraps a single- ``except spec.sdk_exception_types`` around every call site so- user-correctable errors surface as ``NodeUserError`` without- a per-provider branch in ai.py.+ sdk_exception_refs: tuple of ``"module:ClassName"`` refs (the+ packaging entry-points format, resolved via+ ``pkgutil.resolve_name``) naming the typed error classes+ raised by this provider's SDK. Refs — not classes — so that+ registering a provider at boot never imports its SDK+ (docs-internal/performance.md anti-pattern; cost ~7s warm /+ ~45s cold across openai + anthropic + google.genai). The+ resolved classes are exposed by ``sdk_exception_types``.client_kwargs: static keyword arguments passed to ``factory`` onevery instantiation. Used by OpenAI-compatible providers(xai / deepseek / kimi / mistral / ollama / lmstudio / groq /@@ -42,9 +63,19 @@ class ProviderSpec:name: strfactory: Callable[..., LLMProvider]- sdk_exception_types: Tuple[Type[Exception], ...]+ sdk_exception_refs: Tuple[str, ...]client_kwargs: Dict[str, Any] = field(default_factory=dict)+ @property+ def sdk_exception_types(self) -> Tuple[Type[BaseException], ...]:+ """Resolved typed SDK error classes.++ The unifier wraps a single ``except spec.sdk_exception_types``+ around every call site so user-correctable errors surface as+ ``NodeUserError`` without a per-provider branch in ai.py.+ """+ return tuple(_resolve_exc(ref) for ref in self.sdk_exception_refs)+_REGISTRY: Dict[str, ProviderSpec] = {}@@ -60,9 +91,12 @@ def register_provider(spec: ProviderSpec) -> None:reload cycles). Re-registration with a different spec raises soaccidental shadowing is loud."""- if not spec.sdk_exception_types:+ # Check the raw refs, NOT the resolving property — reading the+ # property here would import every SDK at registration time and+ # defeat the lazy-ref design.+ if not spec.sdk_exception_refs:raise ValueError(- f"ProviderSpec({spec.name!r}) declares empty sdk_exception_types. "+ f"ProviderSpec({spec.name!r}) declares empty sdk_exception_refs. ""Every provider must surface its typed SDK error class so the ""unifier can translate it into NodeUserError.")diff --git a/server/tests/llm/test_lazy_sdk_imports.py b/server/tests/llm/test_lazy_sdk_imports.pynew file mode 100644index 00000000..4d907ac5--- /dev/null+++ b/server/tests/llm/test_lazy_sdk_imports.py@@ -0,0 +1,84 @@+"""Boot-path purity: registering LLM providers must not import their SDKs.++The provider registration blocks used to do eager ``import anthropic`` /+``import openai`` / ``from google.genai import errors`` at module bottom+just to populate ``ProviderSpec.sdk_exception_types`` — reintroducing the+eager-LLM-SDK-at-boot anti-pattern (docs-internal/performance.md) at+~7s warm / ~45s cold. The specs now carry lazy ``"module:ClassName"``+refs resolved via ``pkgutil.resolve_name`` at except/read time.++These tests run the import in a clean subprocess (the pytest process+itself has SDKs loaded by other tests) and assert none of the heavy SDK+modules land in ``sys.modules``. They are the guard that keeps the next+provider from quietly reintroducing the eager import.+"""++from __future__ import annotations++import subprocess+import sys+from pathlib import Path++SERVER_DIR = Path(__file__).resolve().parents[2]++_HEAVY_SDKS = ("openai", "anthropic", "google.genai")+++def _run_probe(code: str) -> str:+ """Run ``code`` in a clean interpreter with cwd=server/ and return stdout."""+ result = subprocess.run(+ [sys.executable, "-c", code],+ cwd=SERVER_DIR,+ capture_output=True,+ text=True,+ timeout=120,+ )+ assert result.returncode == 0, (+ f"probe subprocess failed (rc={result.returncode}):\n{result.stderr}"+ )+ return result.stdout+++def test_importing_llm_package_does_not_import_sdks():+ out = _run_probe(+ "import sys\n"+ "import services.llm\n"+ "import services.llm.providers\n"+ f"leaked = [m for m in {_HEAVY_SDKS!r} if m in sys.modules]\n"+ "print('LEAKED=' + ','.join(leaked))\n"+ )+ assert "LEAKED=\n" in out or out.strip().endswith("LEAKED="), (+ f"provider registration imported heavy SDKs at boot: {out!r}"+ )+++def test_importing_services_ai_does_not_import_openai():+ out = _run_probe(+ "import sys\n"+ "import services.ai\n"+ "print('LEAKED=' + ('openai' if 'openai' in sys.modules else ''))\n"+ )+ assert out.strip().endswith("LEAKED="), (+ f"services.ai imported the openai SDK at module level: {out!r}"+ )+++def test_all_registered_refs_resolve_to_exception_classes():+ """Every declared ref must resolve — typo guard, run in-process.++ Complements ``test_plugin_shape.py``'s non-empty/exception-class+ checks: this exercises the actual lazy resolution path end-to-end+ for every registered provider.+ """+ import services.llm.providers # noqa: F401 — triggers registration++ from services.llm.registry import all_providers, get_provider++ for name in all_providers():+ spec = get_provider(name)+ resolved = spec.sdk_exception_types+ assert resolved, f"provider {name!r} resolved to an empty tuple"+ for exc in resolved:+ assert isinstance(exc, type) and issubclass(exc, BaseException), (+ f"provider {name!r} ref resolved to non-exception: {exc!r}"+ )diff --git a/server/tests/llm/test_unifier_incompatible_models_filter.py b/server/tests/llm/test_unifier_incompatible_models_filter.pyindex 0030b47a..cb0fc925 100644--- a/server/tests/llm/test_unifier_incompatible_models_filter.py+++ b/server/tests/llm/test_unifier_incompatible_models_filter.py@@ -36,7 +36,7 @@ def _swap_provider_factory(name: str, models: list[str]):_registry._REGISTRY[name] = ProviderSpec(name=original.name,factory=lambda **kwargs: stub_client,- sdk_exception_types=original.sdk_exception_types,+ sdk_exception_refs=original.sdk_exception_refs,client_kwargs=original.client_kwargs,)return originaldiff --git a/server/tests/llm/test_unifier_typed_errors.py b/server/tests/llm/test_unifier_typed_errors.pyindex c6d0bdc6..96d6a752 100644--- a/server/tests/llm/test_unifier_typed_errors.py+++ b/server/tests/llm/test_unifier_typed_errors.py@@ -45,11 +45,11 @@ def _replace_factory(provider_name: str, exc: Exception):"""Re-register the named provider with a factory returning the failing client."""original = get_provider(provider_name)failing_client = _stub_client_that_raises(exc)- # Build a wrapping spec — same exception tuple, factory swapped for a stub+ # Build a wrapping spec — same exception refs, factory swapped for a stubspec = ProviderSpec(name=original.name,factory=lambda **kwargs: failing_client,- sdk_exception_types=original.sdk_exception_types,+ sdk_exception_refs=original.sdk_exception_refs,client_kwargs=original.client_kwargs,)# Direct registry write (bypasses register_provider's conflict guard