Skip to content

Self-improving AI Employees for everyone

perf(llm): resolve provider SDK exception types lazily at except time

trohitgc2b20217f148
diff --git a/server/services/ai.py b/server/services/ai.py
index 4490db2d..8b53bf0b 100644
--- a/server/services/ai.py
+++ b/server/services/ai.py
@@ -33,13 +33,13 @@ from typing import Awaitable, Dict, Any, List, Optional, Callable, Type, TYPE_CH
from langchain_core.messages import BaseMessage # eager: type hint for _run_agent_loop
import json
-# Eager imports — both are tiny and read on every chat/agent execution
-# path. ``openai`` is the canonical SDK whose typed exception hierarchy
-# (``BadRequestError`` / ``AuthenticationError`` / ``RateLimitError`` / …)
-# is the contract this module dispatches on; ``NodeUserError`` is the
-# framework's "user-correctable, no-traceback" sentinel used to surface
-# those typed errors cleanly through ``BaseNode.execute()``.
-import openai
+# ``NodeUserError`` is the framework's "user-correctable, no-traceback"
+# sentinel used to surface typed SDK errors cleanly through
+# ``BaseNode.execute()``. The ``openai`` SDK itself is imported
+# function-locally in ``execute_agent`` / ``execute_chat_agent`` (its
+# only consumers here, via ``except openai.OpenAIError``) — a top-level
+# import cost seconds of boot time and re-created the eager-LLM-SDK
+# anti-pattern (docs-internal/performance.md).
from services.plugin import NodeUserError
if TYPE_CHECKING:
@@ -1583,6 +1583,8 @@ class AIService:
workflow_id: Optional workflow ID for scoped status broadcasts
context: Optional execution context with nodes, edges for nested agent delegation
"""
+ import openai # local: bound before the try whose except clause needs it
+
from langchain_core.messages import HumanMessage, SystemMessage
start_time = time.time()
@@ -2131,6 +2133,8 @@ class AIService:
workflow_id: Optional workflow ID for scoped status broadcasts
context: Optional execution context with nodes, edges for nested agent delegation
"""
+ import openai # local: bound before the try whose except clause needs it
+
from langchain_core.messages import HumanMessage, SystemMessage
start_time = time.time()
diff --git a/server/services/llm/providers/_compat.py b/server/services/llm/providers/_compat.py
index 92871913..55dba8b8 100644
--- a/server/services/llm/providers/_compat.py
+++ b/server/services/llm/providers/_compat.py
@@ -17,8 +17,6 @@ from __future__ import annotations
from typing import Tuple
-import openai as _openai_sdk
-
from core.logging import get_logger
from services.llm.providers.openai import OpenAIProvider
from services.llm.registry import ProviderSpec, register_provider
@@ -100,7 +98,7 @@ def _register_compat_providers() -> None:
ProviderSpec(
name=name,
factory=OpenAIProvider,
- sdk_exception_types=(_openai_sdk.OpenAIError,),
+ sdk_exception_refs=("openai:OpenAIError",),
client_kwargs={"base_url": base_url},
)
)
diff --git a/server/services/llm/providers/anthropic.py b/server/services/llm/providers/anthropic.py
index ac3e613c..b24b0727 100644
--- a/server/services/llm/providers/anthropic.py
+++ b/server/services/llm/providers/anthropic.py
@@ -185,19 +185,18 @@ class AnthropicProvider:
# ---------------------------------------------------------------------------
# Plugin self-registration
# ---------------------------------------------------------------------------
-# Module load triggers registration into the global ProviderRegistry. The
-# eager top-level ``import anthropic`` is required to resolve the typed
-# ``APIError`` class used by the unifier's translation layer; the SDK is
-# small enough that the cost is in the same band as the existing eager
-# ``import openai`` in ``services/ai.py``.
+# Module load triggers registration into the global ProviderRegistry.
+# The typed ``APIError`` class is declared as a lazy "module:Class" ref
+# (resolved via ``pkgutil.resolve_name`` at except/read time) so that
+# registration never imports the SDK — the eager import here used to
+# cost seconds of boot time (docs-internal/performance.md).
-import anthropic as _anthropic_sdk
from services.llm.registry import ProviderSpec, register_provider
register_provider(
ProviderSpec(
name="anthropic",
factory=AnthropicProvider,
- sdk_exception_types=(_anthropic_sdk.APIError,),
+ sdk_exception_refs=("anthropic:APIError",),
)
)
diff --git a/server/services/llm/providers/gemini.py b/server/services/llm/providers/gemini.py
index 1e97fa29..9323c6be 100644
--- a/server/services/llm/providers/gemini.py
+++ b/server/services/llm/providers/gemini.py
@@ -268,14 +268,17 @@ class GeminiProvider:
# translates it into ``NodeUserError`` at one catch site so the error
# surfaces as a single WARN line through ``BaseNode.execute()`` instead
# of bubbling up as a bare ``RuntimeError`` traceback.
+#
+# Declared as a lazy "module:Class" ref — ``google.genai`` is the single
+# most expensive SDK import on the boot path (~4s warm / ~15s cold via
+# google.auth + api_core + protobuf); the ref defers it to first use.
-from google.genai import errors as _google_genai_errors
from services.llm.registry import ProviderSpec, register_provider
register_provider(
ProviderSpec(
name="gemini",
factory=GeminiProvider,
- sdk_exception_types=(_google_genai_errors.APIError,),
+ sdk_exception_refs=("google.genai.errors:APIError",),
)
)
diff --git a/server/services/llm/providers/openai.py b/server/services/llm/providers/openai.py
index f3dfe146..58994a94 100644
--- a/server/services/llm/providers/openai.py
+++ b/server/services/llm/providers/openai.py
@@ -179,22 +179,21 @@ class OpenAIProvider:
# ---------------------------------------------------------------------------
# Plugin self-registration
# ---------------------------------------------------------------------------
-# Registers ``openai`` into the global registry. The ``openai`` SDK is
-# already eagerly imported by ``services/ai.py`` so the module-level
-# import here is free (Python caches the module).
+# Registers ``openai`` into the global registry. The typed
+# ``OpenAIError`` class is declared as a lazy "module:Class" ref so
+# registration never imports the SDK at boot.
#
# OpenAI-compatible providers (xai / deepseek / kimi / mistral / ollama /
# lmstudio) register separately in ``_compat.py`` — they reuse
# ``OpenAIProvider`` as their factory but pin ``base_url`` via
# ``ProviderSpec.client_kwargs`` rather than per-provider Python.
-import openai as _openai_sdk
from services.llm.registry import ProviderSpec, register_provider
register_provider(
ProviderSpec(
name="openai",
factory=OpenAIProvider,
- sdk_exception_types=(_openai_sdk.OpenAIError,),
+ sdk_exception_refs=("openai:OpenAIError",),
)
)
diff --git a/server/services/llm/providers/openrouter.py b/server/services/llm/providers/openrouter.py
index 172ff1b6..032fd800 100644
--- a/server/services/llm/providers/openrouter.py
+++ b/server/services/llm/providers/openrouter.py
@@ -85,15 +85,14 @@ class OpenRouterProvider(OpenAIProvider):
# ---------------------------------------------------------------------------
# OpenRouter rides the OpenAI Python SDK with a different ``base_url`` +
# extra headers, so its typed exceptions are ``openai.OpenAIError``
-# subclasses — same exception tuple as the openai provider above.
+# subclasses — same lazy exception ref as the openai provider.
-import openai as _openai_sdk
from services.llm.registry import ProviderSpec, register_provider
register_provider(
ProviderSpec(
name="openrouter",
factory=OpenRouterProvider,
- sdk_exception_types=(_openai_sdk.OpenAIError,),
+ sdk_exception_refs=("openai:OpenAIError",),
)
)
diff --git a/server/services/llm/registry.py b/server/services/llm/registry.py
index 8c5baa80..8fc47b7f 100644
--- a/server/services/llm/registry.py
+++ b/server/services/llm/registry.py
@@ -9,6 +9,8 @@ inside ``services/ai.py`` after this layer is wired.
from __future__ import annotations
+import functools
+import pkgutil
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Tuple, Type
@@ -19,6 +21,22 @@ from services.plugin import NodeUserError
logger = get_logger(__name__)
[email protected]_cache(maxsize=None)
+def _resolve_exc(ref: str) -> Type[BaseException]:
+ """Resolve a ``"module:ClassName"`` ref to the exception class.
+
+ Called lazily from ``ProviderSpec.sdk_exception_types`` — by the time
+ anything needs the resolved classes (an ``except`` clause matching a
+ raised SDK error, or a test reading the tuple), the provider factory
+ has already imported the SDK to build its client, so this is a
+ ``sys.modules`` cache hit, never a cold import on the boot path.
+ """
+ obj = pkgutil.resolve_name(ref)
+ if not (isinstance(obj, type) and issubclass(obj, BaseException)):
+ raise TypeError(f"{ref!r} did not resolve to an exception class: {obj!r}")
+ return obj
+
+
@dataclass(frozen=True)
class ProviderSpec:
"""Declarative spec for a chat-model provider plugin.
@@ -29,11 +47,14 @@ class ProviderSpec:
``providers.<name>`` block in ``llm_defaults.json``.
factory: callable returning an ``LLMProvider`` instance.
Invoked as ``factory(api_key=..., proxy_url=..., **client_kwargs)``.
- sdk_exception_types: tuple of typed SDK error classes raised by
- this provider's underlying SDK. The unifier wraps a single
- ``except spec.sdk_exception_types`` around every call site so
- user-correctable errors surface as ``NodeUserError`` without
- a per-provider branch in ai.py.
+ sdk_exception_refs: tuple of ``"module:ClassName"`` refs (the
+ packaging entry-points format, resolved via
+ ``pkgutil.resolve_name``) naming the typed error classes
+ raised by this provider's SDK. Refs — not classes — so that
+ registering a provider at boot never imports its SDK
+ (docs-internal/performance.md anti-pattern; cost ~7s warm /
+ ~45s cold across openai + anthropic + google.genai). The
+ resolved classes are exposed by ``sdk_exception_types``.
client_kwargs: static keyword arguments passed to ``factory`` on
every instantiation. Used by OpenAI-compatible providers
(xai / deepseek / kimi / mistral / ollama / lmstudio / groq /
@@ -42,9 +63,19 @@ class ProviderSpec:
name: str
factory: Callable[..., LLMProvider]
- sdk_exception_types: Tuple[Type[Exception], ...]
+ sdk_exception_refs: Tuple[str, ...]
client_kwargs: Dict[str, Any] = field(default_factory=dict)
+ @property
+ def sdk_exception_types(self) -> Tuple[Type[BaseException], ...]:
+ """Resolved typed SDK error classes.
+
+ The unifier wraps a single ``except spec.sdk_exception_types``
+ around every call site so user-correctable errors surface as
+ ``NodeUserError`` without a per-provider branch in ai.py.
+ """
+ return tuple(_resolve_exc(ref) for ref in self.sdk_exception_refs)
+
_REGISTRY: Dict[str, ProviderSpec] = {}
@@ -60,9 +91,12 @@ def register_provider(spec: ProviderSpec) -> None:
reload cycles). Re-registration with a different spec raises so
accidental shadowing is loud.
"""
- if not spec.sdk_exception_types:
+ # Check the raw refs, NOT the resolving property — reading the
+ # property here would import every SDK at registration time and
+ # defeat the lazy-ref design.
+ if not spec.sdk_exception_refs:
raise ValueError(
- f"ProviderSpec({spec.name!r}) declares empty sdk_exception_types. "
+ f"ProviderSpec({spec.name!r}) declares empty sdk_exception_refs. "
"Every provider must surface its typed SDK error class so the "
"unifier can translate it into NodeUserError."
)
diff --git a/server/tests/llm/test_lazy_sdk_imports.py b/server/tests/llm/test_lazy_sdk_imports.py
new file mode 100644
index 00000000..4d907ac5
--- /dev/null
+++ b/server/tests/llm/test_lazy_sdk_imports.py
@@ -0,0 +1,84 @@
+"""Boot-path purity: registering LLM providers must not import their SDKs.
+
+The provider registration blocks used to do eager ``import anthropic`` /
+``import openai`` / ``from google.genai import errors`` at module bottom
+just to populate ``ProviderSpec.sdk_exception_types`` — reintroducing the
+eager-LLM-SDK-at-boot anti-pattern (docs-internal/performance.md) at
+~7s warm / ~45s cold. The specs now carry lazy ``"module:ClassName"``
+refs resolved via ``pkgutil.resolve_name`` at except/read time.
+
+These tests run the import in a clean subprocess (the pytest process
+itself has SDKs loaded by other tests) and assert none of the heavy SDK
+modules land in ``sys.modules``. They are the guard that keeps the next
+provider from quietly reintroducing the eager import.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+SERVER_DIR = Path(__file__).resolve().parents[2]
+
+_HEAVY_SDKS = ("openai", "anthropic", "google.genai")
+
+
+def _run_probe(code: str) -> str:
+ """Run ``code`` in a clean interpreter with cwd=server/ and return stdout."""
+ result = subprocess.run(
+ [sys.executable, "-c", code],
+ cwd=SERVER_DIR,
+ capture_output=True,
+ text=True,
+ timeout=120,
+ )
+ assert result.returncode == 0, (
+ f"probe subprocess failed (rc={result.returncode}):\n{result.stderr}"
+ )
+ return result.stdout
+
+
+def test_importing_llm_package_does_not_import_sdks():
+ out = _run_probe(
+ "import sys\n"
+ "import services.llm\n"
+ "import services.llm.providers\n"
+ f"leaked = [m for m in {_HEAVY_SDKS!r} if m in sys.modules]\n"
+ "print('LEAKED=' + ','.join(leaked))\n"
+ )
+ assert "LEAKED=\n" in out or out.strip().endswith("LEAKED="), (
+ f"provider registration imported heavy SDKs at boot: {out!r}"
+ )
+
+
+def test_importing_services_ai_does_not_import_openai():
+ out = _run_probe(
+ "import sys\n"
+ "import services.ai\n"
+ "print('LEAKED=' + ('openai' if 'openai' in sys.modules else ''))\n"
+ )
+ assert out.strip().endswith("LEAKED="), (
+ f"services.ai imported the openai SDK at module level: {out!r}"
+ )
+
+
+def test_all_registered_refs_resolve_to_exception_classes():
+ """Every declared ref must resolve — typo guard, run in-process.
+
+ Complements ``test_plugin_shape.py``'s non-empty/exception-class
+ checks: this exercises the actual lazy resolution path end-to-end
+ for every registered provider.
+ """
+ import services.llm.providers # noqa: F401 — triggers registration
+
+ from services.llm.registry import all_providers, get_provider
+
+ for name in all_providers():
+ spec = get_provider(name)
+ resolved = spec.sdk_exception_types
+ assert resolved, f"provider {name!r} resolved to an empty tuple"
+ for exc in resolved:
+ assert isinstance(exc, type) and issubclass(exc, BaseException), (
+ f"provider {name!r} ref resolved to non-exception: {exc!r}"
+ )
diff --git a/server/tests/llm/test_unifier_incompatible_models_filter.py b/server/tests/llm/test_unifier_incompatible_models_filter.py
index 0030b47a..cb0fc925 100644
--- a/server/tests/llm/test_unifier_incompatible_models_filter.py
+++ b/server/tests/llm/test_unifier_incompatible_models_filter.py
@@ -36,7 +36,7 @@ def _swap_provider_factory(name: str, models: list[str]):
_registry._REGISTRY[name] = ProviderSpec(
name=original.name,
factory=lambda **kwargs: stub_client,
- sdk_exception_types=original.sdk_exception_types,
+ sdk_exception_refs=original.sdk_exception_refs,
client_kwargs=original.client_kwargs,
)
return original
diff --git a/server/tests/llm/test_unifier_typed_errors.py b/server/tests/llm/test_unifier_typed_errors.py
index c6d0bdc6..96d6a752 100644
--- a/server/tests/llm/test_unifier_typed_errors.py
+++ b/server/tests/llm/test_unifier_typed_errors.py
@@ -45,11 +45,11 @@ def _replace_factory(provider_name: str, exc: Exception):
"""Re-register the named provider with a factory returning the failing client."""
original = get_provider(provider_name)
failing_client = _stub_client_that_raises(exc)
- # Build a wrapping spec — same exception tuple, factory swapped for a stub
+ # Build a wrapping spec — same exception refs, factory swapped for a stub
spec = ProviderSpec(
name=original.name,
factory=lambda **kwargs: failing_client,
- sdk_exception_types=original.sdk_exception_types,
+ sdk_exception_refs=original.sdk_exception_refs,
client_kwargs=original.client_kwargs,
)
# Direct registry write (bypasses register_provider's conflict guard