Skip to content

Self-improving AI Employees for everyone

fix(local-llm): typed SDK probes + per-model context, not JSON guesses

trohitg94805f4b73a0
diff --git a/client/src/components/ParameterRenderer.tsx b/client/src/components/ParameterRenderer.tsx
index 65ca1c43..7058b59e 100644
--- a/client/src/components/ParameterRenderer.tsx
+++ b/client/src/components/ParameterRenderer.tsx
@@ -18,7 +18,10 @@ import { AI_MODEL_PROVIDER_MAP } from '../lib/aiModelProviders';
import { resolveNodeDescription } from '../lib/nodeSpec';
// Map node types to provider keys for AI model nodes
-// Uses the centralized map from aiModelNodes + legacy aliases
+// Uses the centralized map from aiModelProviders.ts. Local-server
+// providers (ollama, lmstudio) live in that map alongside the cloud
+// providers, so no per-type override is needed here. The remaining
+// aliases below cover renamed-but-still-saved-in-old-workflows types.
const NODE_TYPE_TO_PROVIDER: Record<string, string> = {
...AI_MODEL_PROVIDER_MAP,
// Legacy aliases for backward compatibility
@@ -26,7 +29,6 @@ const NODE_TYPE_TO_PROVIDER: Record<string, string> = {
'googleChatModel': 'gemini',
'azureChatModel': 'azure_openai',
'cohereChatModel': 'cohere',
- 'ollamaChatModel': 'ollama',
};
// Collection Renderer - n8n official style
diff --git a/client/src/components/credentials/sections/ProviderDefaultsSection.tsx b/client/src/components/credentials/sections/ProviderDefaultsSection.tsx
index ec2e2909..de4cf7ff 100644
--- a/client/src/components/credentials/sections/ProviderDefaultsSection.tsx
+++ b/client/src/components/credentials/sections/ProviderDefaultsSection.tsx
@@ -71,9 +71,18 @@ const ProviderDefaultsSection: React.FC<Props> = ({ providerId }) => {
const thinkingEnabled = form.watch('thinking_enabled');
const { isDirty } = form.formState;
- // Load defaults + models on mount.
+ // Load defaults + models on mount AND on providerId change.
+ // Reset models / constraints synchronously when the provider switches
+ // so a stale list from the previous panel can't bleed through. Without
+ // this, opening "OpenAI" then "LM Studio" left the OpenAI model list
+ // visible in the LM Studio dropdown — the dropdown only saw an
+ // explicit `setModels(m)` when the new fetch returned a non-empty
+ // list, so an empty `lmstudio` response (no Fetch clicked yet) was a
+ // no-op and the previous panel's state survived.
useEffect(() => {
if (!isConnected) return;
+ setModels([]);
+ setConstraints(null);
let cancelled = false;
(async () => {
setLoading(true);
@@ -84,7 +93,10 @@ const ProviderDefaultsSection: React.FC<Props> = ({ providerId }) => {
]);
if (!cancelled) {
form.reset((d as Partial<ProviderDefaults>) ?? {});
- if (m?.length) setModels(m);
+ // Unconditional set: clears the dropdown when the fetch comes
+ // back empty (e.g. local-LLM panel before "Fetch" was clicked,
+ // or a freshly-removed key).
+ setModels(m ?? []);
}
} finally {
if (!cancelled) setLoading(false);
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index e941c4d3..fbc0ebcd 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -2093,9 +2093,15 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
provider,
api_key: apiKey
});
+ // Backend returns one of:
+ // { success: true, valid: true, models } — key is good
+ // { success: true, valid: false, message } — clean rejection (401/403/timeout/etc)
+ // { success: false, error } — handler bug (uncaught exception)
+ // Surface the message field first; fall back to error so the toast
+ // shows the actual reason instead of a generic "Validation failed".
const result = {
- valid: response.valid || false,
- message: response.message,
+ valid: response.valid === true,
+ message: response.message ?? response.error,
models: response.models
};
diff --git a/client/src/lib/aiModelProviders.ts b/client/src/lib/aiModelProviders.ts
index a527029a..6ec60dc9 100644
--- a/client/src/lib/aiModelProviders.ts
+++ b/client/src/lib/aiModelProviders.ts
@@ -18,6 +18,14 @@ const AI_MODEL_PROVIDERS = [
'deepseek',
'kimi',
'mistral',
+ // Local-server providers — the chat-model plugins exist
+ // (`ollamaChatModel`, `lmstudioChatModel`), so they MUST live in this
+ // map for `ParameterRenderer` to derive the provider id from the node
+ // type when the schema-implicit `provider` param is absent. Without
+ // them the model dropdown stays empty and the runtime falls back to
+ // the OpenAI cloud.
+ 'ollama',
+ 'lmstudio',
] as const;
/** `<provider>ChatModel` node type → provider id (matches the backend-served
diff --git a/server/constants.py b/server/constants.py
index 8b7dde79..4d42c576 100644
--- a/server/constants.py
+++ b/server/constants.py
@@ -352,31 +352,50 @@ WORKFLOW_TRIGGER_TYPES: FrozenSet[str] = frozenset([
def detect_ai_provider(node_type: str, parameters: dict = None) -> str:
"""Detect AI provider from node type or parameters.
+ Substring match against the node type's lower-case form. Local-server
+ providers (ollama, lmstudio) MUST be checked here — without these
+ branches `lmstudioChatModel` falls through to the final ``return
+ 'openai'`` and ``execute_chat`` ends up calling api.openai.com with
+ the local-server placeholder key, which is the exact symptom users
+ see as "401 from OpenAI when I picked LM Studio".
+
Args:
- node_type: The node type string
- parameters: Optional parameters dict (used for aiAgent/chatAgent)
+ node_type: The node type string (e.g. "ollamaChatModel").
+ parameters: Optional parameters dict (used for aiAgent/chatAgent
+ where the provider lives in a dropdown, not the type).
Returns:
- Provider string: 'openai', 'anthropic', 'gemini', 'openrouter', 'groq', or 'cerebras'
+ Provider string matching a key in ``services.ai.PROVIDER_CONFIGS``.
"""
# AI Agent types get provider from parameters
if node_type in AI_AGENT_TYPES:
return (parameters or {}).get('provider', 'openai')
- elif 'deepseek' in node_type.lower():
+ nt = node_type.lower()
+ # Order matters: more-specific tokens first, so e.g. `lm_studio` /
+ # `lmstudio` / `lm-studio` all classify before any future generic
+ # `openai`-prefixed match could shadow them.
+ if 'deepseek' in nt:
return 'deepseek'
- elif 'kimi' in node_type.lower():
+ if 'kimi' in nt:
return 'kimi'
- elif 'mistral' in node_type.lower():
+ if 'mistral' in nt:
return 'mistral'
- elif 'cerebras' in node_type.lower():
+ if 'cerebras' in nt:
return 'cerebras'
- elif 'groq' in node_type.lower():
+ if 'groq' in nt:
return 'groq'
- elif 'openrouter' in node_type.lower():
+ if 'openrouter' in nt:
return 'openrouter'
- elif 'anthropic' in node_type.lower():
+ if 'anthropic' in nt:
return 'anthropic'
- elif 'gemini' in node_type.lower():
+ if 'gemini' in nt:
return 'gemini'
- else:
- return 'openai'
+ # Local-server providers — match the LMStudioChatModelNode /
+ # OllamaChatModelNode plugin types so the runtime path reads the
+ # correct {provider}_proxy credential and the openai SDK is pointed
+ # at the user's local server, not api.openai.com.
+ if 'lmstudio' in nt or 'lm_studio' in nt or 'lm-studio' in nt:
+ return 'lmstudio'
+ if 'ollama' in nt:
+ return 'ollama'
+ return 'openai'
diff --git a/server/core/credentials_database.py b/server/core/credentials_database.py
index eedf176c..22c143f8 100644
--- a/server/core/credentials_database.py
+++ b/server/core/credentials_database.py
@@ -189,6 +189,7 @@ class CredentialsDatabase:
api_key: str,
models: Optional[List[str]] = None,
session_id: str = "default",
+ model_params: Optional[Dict[str, Dict[str, Any]]] = None,
) -> None:
"""
Save encrypted API key.
@@ -198,10 +199,23 @@ class CredentialsDatabase:
api_key: Plaintext API key to encrypt and store
models: List of available models for this key
session_id: Session identifier (default: "default")
+ model_params: Optional per-model parameters keyed by model id, e.g.
+ ``{"qwen2.5-7b": {"context_length": 8192}}``. Used by local
+ providers (Ollama, LM Studio) where the context window depends
+ on what the user has loaded — the value is fetched from the
+ official SDK during validation. Stored alongside the model
+ list under the same JSON column (``model_params`` subkey) so
+ ``model_registry.get_context_length()`` can read the real
+ ctx instead of a JSON default. Cloud providers leave this
+ empty — their per-model params live in ``model_registry.json``
+ from OpenRouter.
"""
key_id = f"{session_id}_{provider}"
encrypted = self.encryption.encrypt(api_key)
key_hash = hashlib.sha256(api_key.encode()).hexdigest()[:16]
+ models_blob: Dict[str, Any] = {"models": models or []}
+ if model_params:
+ models_blob["model_params"] = model_params
async with self.get_session() as session:
existing = await session.get(EncryptedAPIKey, key_id)
@@ -210,7 +224,7 @@ class CredentialsDatabase:
if existing:
existing.key_encrypted = encrypted
existing.key_hash = key_hash
- existing.models = {"models": models or []}
+ existing.models = models_blob
existing.is_valid = True
existing.last_validated = now
existing.updated_at = now
@@ -222,7 +236,7 @@ class CredentialsDatabase:
session_id=session_id,
key_encrypted=encrypted,
key_hash=key_hash,
- models={"models": models or []},
+ models=models_blob,
is_valid=True,
last_validated=now,
)
@@ -230,6 +244,25 @@ class CredentialsDatabase:
await session.commit()
logger.debug(f"Saved encrypted API key for provider: {provider}")
+ async def get_api_key_model_params(
+ self, provider: str, session_id: str = "default"
+ ) -> Dict[str, Dict[str, Any]]:
+ """Return the per-model param map stored alongside the model list.
+
+ Empty dict if the provider has no entry, no params were stored,
+ or the row predates the ``model_params`` column. The runtime
+ path (``model_registry.get_context_length`` etc.) consults this
+ before falling back to JSON defaults so local-LLM context comes
+ from the actual loaded model, not a guess.
+ """
+ key_id = f"{session_id}_{provider}"
+ async with self.get_session() as session:
+ row = await session.get(EncryptedAPIKey, key_id)
+ if not row or not row.models:
+ return {}
+ params = row.models.get("model_params") or {}
+ return params if isinstance(params, dict) else {}
+
async def get_api_key(
self, provider: str, session_id: str = "default"
) -> Optional[str]:
diff --git a/server/nodes/agent/_inline.py b/server/nodes/agent/_inline.py
index 6732a136..1b5ab431 100644
--- a/server/nodes/agent/_inline.py
+++ b/server/nodes/agent/_inline.py
@@ -152,3 +152,5 @@ async def prepare_agent_call(
"context": context,
"database": database,
}
+
+
diff --git a/server/nodes/agent/_specialized.py b/server/nodes/agent/_specialized.py
index e499fff5..89625cf2 100644
--- a/server/nodes/agent/_specialized.py
+++ b/server/nodes/agent/_specialized.py
@@ -38,6 +38,9 @@ class SpecializedAgentParams(BaseModel):
provider: Literal[
"openai", "anthropic", "gemini", "openrouter",
"groq", "cerebras", "deepseek", "kimi", "mistral",
+ # Local-server providers — see ai_agent.Params for the proxy_url
+ # rationale. Same fix; same reason.
+ "ollama", "lmstudio",
] = "openai"
model: str = Field(
default="", json_schema_extra={"placeholder": "Select a model..."},
diff --git a/server/nodes/agent/ai_agent.py b/server/nodes/agent/ai_agent.py
index c5e3724a..1ffe7dfa 100644
--- a/server/nodes/agent/ai_agent.py
+++ b/server/nodes/agent/ai_agent.py
@@ -41,6 +41,12 @@ class AIAgentParams(BaseModel):
provider: Literal[
"openai", "anthropic", "gemini", "openrouter",
"groq", "cerebras", "deepseek", "kimi", "mistral",
+ # Local-server providers — agent execution reads
+ # ``{provider}_proxy`` to point the LangChain client at the
+ # user's localhost server. Without these entries the dropdown
+ # silently falls back to ``"openai"`` and execute_agent ends
+ # up calling api.openai.com instead.
+ "ollama", "lmstudio",
] = "openai"
model: str = Field(
default="", json_schema_extra={"placeholder": "Select a model..."},
@@ -126,6 +132,10 @@ class AIAgentNode(ActionNode):
context=ctx.raw, database=database,
log_prefix="[AI Agent]",
)
+ # ``execute_agent`` raises ``NodeUserError`` directly for typed
+ # openai SDK failures (context overflow, bad key, server down).
+ # Anything that comes back here as ``success=False`` is a real
+ # bug worth a stacktrace via ``RuntimeError``.
response = await ai_service.execute_agent(ctx.node_id, **kwargs)
if response.get("success"):
return response.get("result") or response
diff --git a/server/nodes/agent/chat_agent.py b/server/nodes/agent/chat_agent.py
index f8295744..a28f9a84 100644
--- a/server/nodes/agent/chat_agent.py
+++ b/server/nodes/agent/chat_agent.py
@@ -28,6 +28,9 @@ class ChatAgentParams(BaseModel):
provider: Literal[
"openai", "anthropic", "gemini", "openrouter",
"groq", "cerebras", "deepseek", "kimi", "mistral",
+ # Local-server providers — see ai_agent.Params for the proxy_url
+ # rationale. Same fix; same reason.
+ "ollama", "lmstudio",
] = "openai"
model: str = Field(
default="", json_schema_extra={"placeholder": "Select a model..."},
diff --git a/server/nodes/model/_base.py b/server/nodes/model/_base.py
index bd4aa099..95e3e332 100644
--- a/server/nodes/model/_base.py
+++ b/server/nodes/model/_base.py
@@ -100,6 +100,11 @@ class ChatModelBase(ActionNode, abstract=True):
from core.container import container
ai_service = container.ai_service()
+ # ``execute_chat`` raises ``NodeUserError`` directly for typed
+ # openai SDK failures (context overflow, bad key, server down) —
+ # BaseNode.execute() then logs at WARN with no traceback.
+ # ``RuntimeError`` here only fires for "success=False" responses
+ # that aren't typed SDK errors (real bugs worth a stacktrace).
response = await ai_service.execute_chat(
ctx.node_id, self.type, params.model_dump(),
)
diff --git a/server/nodes/model/_local_validator.py b/server/nodes/model/_local_validator.py
index 62fe122a..d36b688d 100644
--- a/server/nodes/model/_local_validator.py
+++ b/server/nodes/model/_local_validator.py
@@ -11,23 +11,39 @@ for these providers; the ``api_key`` field carries the user's Base URL,
not a secret. We:
1. Save the URL under ``{provider}_proxy`` — the existing Ollama-style
- auth-delegation key that ``AIService.create_model`` and
- ``AIService.fetch_models`` already read.
-2. Call ``ai_service.fetch_models(provider, "ollama")`` which (after the
- proxy_url passthrough in services/ai.py) probes the user's URL and
- returns the actually-installed models.
-3. Store the placeholder api_key + discovered model list under the
- provider id so the Provider Defaults dropdown shows real entries.
+ auth-delegation key that the runtime path in ``services/ai.py``
+ already reads.
+2. Probe the user's server via the *official* SDK (``ollama`` for
+ Ollama, ``lmstudio`` for LM Studio) — list installed models and
+ their actually-loaded ``context_length``. SDK-driven introspection
+ beats hand-rolled httpx against ``/api/show`` / ``/api/v0/models``
+ because the SDK ships the typed result struct (``ShowResponse``,
+ ``LlmInstanceInfo``) and stays compatible with version drift
+ upstream.
+3. Store the placeholder api_key + discovered model list + per-model
+ ``context_length`` under the provider id. ``model_registry`` reads
+ the per-model context at runtime so a chat call never assumes a
+ bogus 32K when the user has a 4K-loaded model.
4. Return ``valid=True`` only when at least one model was found, so a
misconfigured URL surfaces as a clear "no models" message instead
of a silent success.
+
+Connection failures (server down, wrong port, auth refused, timeout)
+are caught off the SDK's own exceptions and mapped to specific
+user-facing toasts — operators see "is the server running?" for
+connect-refused, "wrong path" for 404, etc. The catalogue ``stored``
+flag flips to False on every failure path via the broadcaster, so a
+failed re-probe clears the previously-green palette dot.
"""
from __future__ import annotations
import time
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional, Tuple
+import httpx
+import lmstudio
+import ollama
from fastapi import WebSocket
from core.container import container
@@ -37,6 +53,192 @@ from services.status_broadcaster import get_status_broadcaster
logger = get_logger(__name__)
+def _classify_http_error(provider: str, base_url: str, exc: BaseException) -> Tuple[str, str]:
+ """Map an httpx / generic exception to (log_summary, user_message).
+
+ Both SDKs (ollama-python, lmstudio) raise httpx errors underneath.
+ Mapping these directly gives clean operator logs ("connect-refused"
+ vs "404 vs "timeout") + actionable user toasts without depending
+ on the openai SDK exception hierarchy.
+ """
+ display = "LM Studio" if provider == "lmstudio" else provider.capitalize()
+
+ if isinstance(exc, httpx.TimeoutException):
+ return ("timeout", f"Request to {base_url} timed out — server may be overloaded or unreachable")
+
+ if isinstance(exc, httpx.ConnectError):
+ return ("connect-refused",
+ f"Could not reach {display} at {base_url}. Is the server running?")
+
+ if isinstance(exc, httpx.HTTPStatusError):
+ status = exc.response.status_code
+ if status in (401, 403):
+ return (f"HTTP {status}", f"{display} rejected the request — server requires auth.")
+ if status == 404:
+ url = base_url.rstrip("/")
+ if url.endswith("/v1"):
+ hint = f"the {display} server is reachable but expected endpoints aren't exposed — check the server version."
+ else:
+ hint = f"the URL likely needs to end with `/v1` (e.g. {url}/v1)."
+ return (f"HTTP {status}", f"{display} returned 404 — {hint}")
+ if status == 429:
+ return (f"HTTP {status}", f"{display} rate-limited the request — try again shortly.")
+ return (f"HTTP {status}", f"{display} returned HTTP {status} — check the server logs.")
+
+ if isinstance(exc, httpx.RequestError):
+ return ("network-error", f"Network error reaching {display} at {base_url}: {exc.__class__.__name__}")
+
+ # Unknown exception type: surface the class name + message so the
+ # operator can tell what they're looking at without a stacktrace.
+ return (type(exc).__name__, f"Could not reach {display}: {exc}")
+
+
+async def _fail(
+ provider: str,
+ message: str,
+ *,
+ has_key: bool = False,
+) -> Dict[str, Any]:
+ """Common rejection path: broadcast invalid status + return envelope.
+
+ `has_key=True` is used after the URL has been persisted so the
+ palette doesn't go straight to the "unconfigured" gray dot — the
+ user's URL is on file, it just can't reach a working server right
+ now. `has_key=False` is used when we abort before persisting (only
+ the "Base URL required" early-exit today).
+ """
+ await get_status_broadcaster().update_api_key_status(
+ provider=provider, valid=False, message=message,
+ has_key=has_key, models=[],
+ )
+ return {
+ "provider": provider, "success": True, "valid": False,
+ "message": message, "models": [], "timestamp": time.time(),
+ }
+
+
+def _strip_v1_path(base_url: str) -> str:
+ """Return ``base_url`` with a trailing ``/v1`` segment stripped.
+
+ The user's stored URL is the OpenAI-compatible base
+ (``http://host:port/v1``). Both Ollama's REST API and LM Studio's
+ SDK want the host:port without the OpenAI-compat suffix.
+ """
+ u = base_url.rstrip("/")
+ if u.endswith("/v1"):
+ return u[: -len("/v1")]
+ return u
+
+
+async def _fetch_ollama_models(base_url: str) -> List[Dict[str, Any]]:
+ """List currently-loaded Ollama models with their actual params.
+
+ Uses ``ollama.AsyncClient.ps()`` — the official "list running models"
+ endpoint. Returns a typed ``ProcessResponse`` whose ``models[]``
+ entries already carry every field we need as proper Pydantic
+ attributes (no dict-key hunting, no Modelfile parameters parsing):
+
+ - ``model`` — canonical name passed to ``/v1/chat/completions``
+ - ``context_length``— live server-side n_ctx (this is the value
+ that produces the 400 overflow when a
+ prompt exceeds it)
+ - ``details`` — typed ``ModelDetails`` (family, parameter_size,
+ quantization_level)
+
+ If the user has models *pulled* but none currently loaded, ``ps()``
+ returns an empty list — same semantics as LM Studio's
+ ``list_loaded()``. The validator surfaces this as the "load a model
+ and click Fetch again" message, which is already accurate.
+ """
+ host = _strip_v1_path(base_url)
+ client = ollama.AsyncClient(host=host, timeout=10.0)
+ try:
+ running = await client.ps()
+ finally:
+ try:
+ await client._client.aclose()
+ except Exception:
+ pass
+
+ out: List[Dict[str, Any]] = []
+ for m in running.models or []:
+ mid = m.model or m.name
+ if not mid:
+ continue
+ entry: Dict[str, Any] = {"id": mid}
+ if m.context_length:
+ entry["context_length"] = int(m.context_length)
+ if m.details:
+ if m.details.family:
+ entry["architecture"] = m.details.family
+ if m.details.parameter_size:
+ entry["param_size"] = m.details.parameter_size
+ if m.details.quantization_level:
+ entry["quantization"] = m.details.quantization_level
+ if m.details.format:
+ entry["format"] = m.details.format
+ out.append(entry)
+ return out
+
+
+async def _fetch_lmstudio_models(base_url: str) -> List[Dict[str, Any]]:
+ """List currently-loaded LM Studio models with their actual params.
+
+ Uses ``lmstudio.AsyncClient.llm.list_loaded()`` — each handle's
+ ``get_info()`` returns a typed ``LlmInstanceInfo``. We read only
+ SDK-typed fields (``context_length``, ``max_context_length``,
+ ``vision``, ``trained_for_tool_use``, ``architecture``,
+ ``params_string``); no string parsing.
+
+ LM Studio's SDK takes ``api_host`` as ``host:port`` (no scheme, no
+ path), so the user's stored ``http://host:port/v1`` is stripped
+ before construction.
+ """
+ host = _strip_v1_path(base_url)
+ api_host = host.split("://", 1)[-1]
+
+ client = lmstudio.AsyncClient(api_host=api_host)
+ out: List[Dict[str, Any]] = []
+ async with client:
+ loaded = await client.llm.list_loaded()
+ for handle in loaded:
+ try:
+ info = await handle.get_info()
+ except Exception as e:
+ logger.info("[lmstudio] get_info skipped for %s: %s",
+ getattr(handle, "identifier", "<unknown>"), type(e).__name__)
+ continue
+ mid = info.identifier or info.model_key
+ if not mid:
+ continue
+ entry: Dict[str, Any] = {"id": mid}
+ if info.context_length:
+ entry["context_length"] = int(info.context_length)
+ if info.max_context_length:
+ entry["max_context_length"] = int(info.max_context_length)
+ if info.vision is not None:
+ entry["vision"] = bool(info.vision)
+ if info.trained_for_tool_use is not None:
+ entry["supports_tools"] = bool(info.trained_for_tool_use)
+ if info.architecture:
+ entry["architecture"] = info.architecture
+ if info.params_string:
+ entry["param_size"] = info.params_string
+ if info.format:
+ entry["format"] = info.format
+ out.append(entry)
+ return out
+
+
+async def _fetch_local_models(provider: str, base_url: str) -> List[Dict[str, Any]]:
+ """Dispatch to the per-provider SDK probe."""
+ if provider == "ollama":
+ return await _fetch_ollama_models(base_url)
+ if provider == "lmstudio":
+ return await _fetch_lmstudio_models(base_url)
+ raise ValueError(f"Unsupported local provider: {provider}")
+
+
async def validate_local_llm(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Validator for ollama / lmstudio. Matches the SPECIAL_PROVIDER_VALIDATORS contract."""
provider = data["provider"].lower()
@@ -47,13 +249,13 @@ async def validate_local_llm(data: Dict[str, Any], websocket: WebSocket) -> Dict
return {"success": False, "valid": False, "error": "Base URL required"}
auth_service = container.auth_service()
- ai_service = container.ai_service()
- broadcaster = get_status_broadcaster()
- # Persist the URL first so the upcoming fetch_models call can read
- # it via the existing {provider}_proxy lookup. Without this step
- # the probe would hit the JSON default base_url, not the user's
- # custom server.
+ # Persist the URL first so the runtime path (services/ai.py) can
+ # read it via the existing {provider}_proxy lookup even before the
+ # probe succeeds. The URL stays persisted on probe failure so the
+ # user can re-click "Fetch" without re-entering it; the failure
+ # branch broadcasts has_key=True so the palette dot reflects "URL
+ # on file but currently unreachable" rather than "unconfigured".
await auth_service.store_api_key(
provider=f"{provider}_proxy",
api_key=base_url,
@@ -62,37 +264,74 @@ async def validate_local_llm(data: Dict[str, Any], websocket: WebSocket) -> Dict
)
try:
- models = await ai_service.fetch_models(provider, "ollama")
+ entries = await _fetch_local_models(provider, base_url)
+ except (httpx.HTTPError, lmstudio.LMStudioError) as e:
+ log_summary, user_msg = _classify_http_error(provider, base_url, e)
+ logger.warning("[%s] model probe failed (%s) at %s", provider, log_summary, base_url)
+ return await _fail(provider, user_msg, has_key=True)
except Exception as e:
- logger.warning("[%s] fetch_models failed: %s", provider, e)
- models = []
-
- if not models:
- message = (
- f"Saved URL but {provider} returned no models — is the server "
- "running and a model loaded?"
- )
- await broadcaster.update_api_key_status(
- provider=provider, valid=False, message=message,
- has_key=True, models=[],
- )
- return {"success": True, "valid": False, "message": message, "models": []}
-
- # Store placeholder api_key + the real model list. The placeholder
- # ("ollama") is the documented value LiteLLM / OpenAI clients accept
- # for unauthenticated local servers; it's never sent over the wire
- # because OpenAIProvider rewrites it when proxy_url is set.
+ log_summary, user_msg = _classify_http_error(provider, base_url, e)
+ logger.warning("[%s] model probe unexpected error (%s) at %s: %s",
+ provider, log_summary, base_url, e)
+ return await _fail(provider, user_msg, has_key=True)
+
+ if not entries:
+ # Server reachable, responded with empty model list. Different
+ # failure mode from the connect-error branches above — keep the
+ # original "load a model" hint here since it's now accurate.
+ display = "LM Studio" if provider == "lmstudio" else provider.capitalize()
+ message = f"Connected to {display} at {base_url}, but no models are loaded. Load a model in {display} and click Fetch again."
+ logger.info("[%s] reachable at %s but returned 0 models", provider, base_url)
+ return await _fail(provider, message, has_key=True)
+
+ # Pivot the SDK-probed entries into the storage shape: a parallel
+ # ``models`` list (for the legacy readers) plus a ``model_params``
+ # dict carrying every typed field the SDK exposed
+ # (``context_length``, ``vision``, ``supports_tools``,
+ # ``architecture``, ``param_size``, ``quantization``,
+ # ``max_context_length``). ``model_registry.register_local_model``
+ # consumes these directly to populate ``ModelInfo``.
+ models = [e["id"] for e in entries]
+ model_params: Dict[str, Dict[str, Any]] = {
+ e["id"]: {k: v for k, v in e.items() if k != "id"}
+ for e in entries
+ if any(k != "id" for k in e)
+ }
+
+ # Store placeholder api_key + the real model list + per-model params.
+ # The placeholder ("ollama") is the documented value the OpenAI-style
+ # auth-delegation path expects when no real key is needed; it never
+ # leaves the process because the runtime SDK rewrites it when
+ # proxy_url is set.
await auth_service.store_api_key(
provider=provider,
api_key="ollama",
models=models,
session_id=session_id,
+ model_params=model_params,
)
- await broadcaster.update_api_key_status(
+
+ # Register each model in the in-memory model registry so the sync
+ # ``get_context_length`` / ``get_max_output_tokens`` lookups pick up
+ # the real loaded n_ctx without re-querying the DB on every chat
+ # call. Also keeps the runtime path branchless — local and cloud
+ # models share the same ``provider/model_id`` registry key.
+ from services.model_registry import get_model_registry
+ registry = get_model_registry()
+ for mid, params in model_params.items():
+ registry.register_local_model(provider, mid, params)
+
+ await get_status_broadcaster().update_api_key_status(
provider=provider, valid=True,
message=f"{len(models)} model(s) discovered at {base_url}",
has_key=True, models=models,
)
+ ctx_summary = ", ".join(
+ f"{mid}={p['context_length']}"
+ for mid, p in model_params.items()
+ ) or "no per-model context info"
+ logger.info("[%s] discovered %d model(s) at %s (%s)",
+ provider, len(models), base_url, ctx_summary)
return {
"provider": provider, "success": True, "valid": True,
"models": models, "message": f"Connected to {provider} at {base_url}",
diff --git a/server/pyproject.toml b/server/pyproject.toml
index c7aaf405..d20e6fe6 100644
--- a/server/pyproject.toml
+++ b/server/pyproject.toml
@@ -92,6 +92,14 @@ dependencies = [
# Markdown formatting (GFM to platform-native)
"markdown-it-py>=3.0.0",
+ # Local LLM SDKs — used by `nodes/model/_local_validator.py` to
+ # introspect the user's actually-loaded models (context_length,
+ # max_output_tokens). Replaces hand-rolled httpx probes against
+ # /api/show and /api/v0/models so each SDK's own typed result
+ # struct is the source of truth.
+ "ollama>=0.6.0",
+ "lmstudio>=1.5.0",
+
# Gmail API
"google-api-python-client>=2.100.0",
"google-auth>=2.27.0",
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index d17a8ebe..b812499a 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -1227,7 +1227,18 @@ async def handle_validate_api_key(data: Dict[str, Any], websocket: WebSocket) ->
``/v1/models`` probe via ``ai_service.fetch_models``. The frontend
calls this one handler for every provider — no per-provider branching
on the TypeScript side.
+
+ Invalid-key responses (401/403) and provider-side errors (4xx/5xx,
+ timeouts, DNS failures) are mapped to a clean
+ ``{success: True, valid: False, message: "..."}`` envelope — same
+ shape the Maps/Apify validators return — so the frontend toast
+ surfaces the actual reason instead of the generic
+ "Validation failed" fallback. These are user-correctable input
+ failures, not handler bugs, so they log at WARNING with a one-liner
+ rather than ERROR with a stacktrace.
"""
+ import httpx
+
provider = data["provider"].lower()
normalized = dict(data, provider=provider)
@@ -1241,7 +1252,32 @@ async def handle_validate_api_key(data: Dict[str, Any], websocket: WebSocket) ->
# Fetch models for AI providers (any provider with a models_endpoint in llm_defaults.json)
from services.ai import PROVIDER_CONFIGS
- models = await ai_service.fetch_models(provider, api_key) if provider in PROVIDER_CONFIGS else []
+ try:
+ models = await ai_service.fetch_models(provider, api_key) if provider in PROVIDER_CONFIGS else []
+ except httpx.HTTPStatusError as e:
+ status = e.response.status_code
+ if status in (401, 403):
+ msg = "Invalid API key"
+ elif status == 429:
+ msg = "Rate limited by provider — try again in a moment"
+ elif 500 <= status < 600:
+ msg = f"Provider error ({status}) — try again later"
+ else:
+ msg = f"Provider rejected the key ({status})"
+ logger.warning("API key validation failed for %s: HTTP %s", provider, status)
+ await broadcaster.update_api_key_status(
+ provider=provider, valid=False, message=msg, has_key=False, models=[]
+ )
+ return {"provider": provider, "valid": False, "message": msg, "timestamp": time.time()}
+ except (httpx.TimeoutException, httpx.RequestError) as e:
+ # DNS, connect, read timeouts — the provider is unreachable.
+ logger.warning("API key validation could not reach %s: %s", provider, e)
+ msg = "Could not reach provider — check your network and try again"
+ await broadcaster.update_api_key_status(
+ provider=provider, valid=False, message=msg, has_key=False, models=[]
+ )
+ return {"provider": provider, "valid": False, "message": msg, "timestamp": time.time()}
+
await auth_service.store_api_key(provider=provider, api_key=api_key, models=models,
session_id=data.get("session_id", "default"))
# Broadcast with hasKey and models so frontend can update reactively
diff --git a/server/services/ai.py b/server/services/ai.py
index 1728d920..81651783 100644
--- a/server/services/ai.py
+++ b/server/services/ai.py
@@ -41,6 +41,15 @@ import operator
from langchain_core.messages import BaseMessage # eager: needed for AgentState type resolution
import json
+# Eager imports — both are tiny and read on every chat/agent execution
+# path. ``openai`` is the canonical SDK whose typed exception hierarchy
+# (``BadRequestError`` / ``AuthenticationError`` / ``RateLimitError`` / …)
+# is the contract this module dispatches on; ``NodeUserError`` is the
+# framework's "user-correctable, no-traceback" sentinel used to surface
+# those typed errors cleanly through ``BaseNode.execute()``.
+import openai
+from services.plugin import NodeUserError
+
if TYPE_CHECKING:
from langchain_openai import ChatOpenAI # noqa: F401
from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, ToolMessage # noqa: F401
@@ -456,9 +465,24 @@ def detect_provider_from_model(model: str) -> str:
def is_model_valid_for_provider(model: str, provider: str) -> bool:
- """Check if model name matches the provider's patterns."""
- # OpenRouter is a proxy supporting all models (provider/model format) - always valid
- if provider == 'openrouter':
+ """Check if model name matches the provider's patterns.
+
+ Pattern-matching is meaningful for cloud providers — `gpt-*` is OpenAI,
+ `claude-*` is Anthropic, etc. — and the check guards against picking
+ a model from one provider's dropdown after switching to another.
+
+ For "open-world" providers (OpenRouter proxy, local Ollama / LM Studio
+ servers) the model namespace is whatever the user has installed:
+ `llama-3.2`, `qwen2.5-coder`, `phi-3-mini`, custom GGUF files, etc.
+ None of those contain the literal substrings `ollama` or `lmstudio`,
+ so applying the cloud-style filter produces a false negative on every
+ valid local model — the call site then "uses default" which for a
+ local provider is the SAME model name, emitting a confusing
+ "invalid ... using default: <same name>" log line. Treat all three
+ as always-valid; the local-server SDK will reject genuinely missing
+ models at request time with a clear 404.
+ """
+ if provider in ('openrouter', 'ollama', 'lmstudio'):
return True
config = get_provider_configs().get(provider)
if not config:
@@ -1590,6 +1614,16 @@ class AIService:
"execution_time": time.time() - start_time
}
+ except openai.OpenAIError as e:
+ # Typed openai SDK exception — context overflow (BadRequestError),
+ # bad key (AuthenticationError), missing model (NotFoundError),
+ # server unreachable (APIConnectionError), etc. All
+ # user-correctable. Propagate as NodeUserError so BaseNode.execute()
+ # logs at WARN with no traceback.
+ log_api_call(logger, provider if 'provider' in locals() else 'unknown',
+ model if 'model' in locals() else 'unknown', "chat", False, error=str(e))
+ raise NodeUserError(str(e)) from e
+
except Exception as e:
logger.error("AI execution failed", node_id=node_id, error=str(e))
log_api_call(logger, provider if 'provider' in locals() else 'unknown',
@@ -2069,6 +2103,13 @@ class AIService:
"execution_time": time.time() - start_time
}
+ except openai.OpenAIError as e:
+ # See execute_chat for the rationale — typed SDK errors are
+ # user-correctable and re-raised as NodeUserError so BaseNode
+ # logs at WARN without a traceback.
+ log_api_call(logger, provider, model, "agent", False, error=str(e))
+ raise NodeUserError(str(e)) from e
+
except Exception as e:
logger.error("[LangGraph] AI agent execution failed", node_id=node_id, error=str(e))
log_api_call(logger, provider, model, "agent", False, error=str(e))
@@ -2525,6 +2566,11 @@ class AIService:
"execution_time": time.time() - start_time
}
+ except openai.OpenAIError as e:
+ # Typed SDK error — see execute_chat for rationale.
+ log_api_call(logger, provider, model, "chat_agent", False, error=str(e))
+ raise NodeUserError(str(e)) from e
+
except Exception as e:
logger.error("[ChatAgent] Execution failed", node_id=node_id, error=str(e))
log_api_call(logger, provider, model, "chat_agent", False, error=str(e))
diff --git a/server/services/auth.py b/server/services/auth.py
index fcdfb515..876c0ac0 100644
--- a/server/services/auth.py
+++ b/server/services/auth.py
@@ -113,7 +113,8 @@ class AuthService:
provider: str,
api_key: str,
models: List[str],
- session_id: str = "default"
+ session_id: str = "default",
+ model_params: Optional[Dict[str, Dict[str, Any]]] = None,
) -> bool:
"""Store API key with models in encrypted credentials database.
@@ -122,6 +123,11 @@ class AuthService:
api_key: The API key to store (will be encrypted)
models: List of available models for this key
session_id: Session identifier for multi-user support
+ model_params: Optional per-model parameters (context_length etc.)
+ — used by local providers (Ollama, LM Studio) where the
+ context window depends on what the user has loaded.
+ Forwarded straight to the credentials DB so
+ ``model_registry`` can read real values at runtime.
Returns:
True if stored successfully, False otherwise
@@ -136,7 +142,8 @@ class AuthService:
provider=provider,
api_key=api_key,
models=models,
- session_id=session_id
+ session_id=session_id,
+ model_params=model_params,
)
# 2. Cache update only after DB write succeeds. One entry,
@@ -160,6 +167,26 @@ class AuthService:
logger.error("Failed to store API key", provider=provider, error=str(e))
return False
+ async def get_model_params(
+ self, provider: str, session_id: str = "default"
+ ) -> Dict[str, Dict[str, Any]]:
+ """Return per-model params (context_length etc.) for the provider.
+
+ Reads straight from the credentials DB — there's no in-memory
+ cache for these because they're consulted at most once per
+ chat-model execution and the DB read is cheap. Empty dict for
+ cloud providers (whose per-model params live in
+ ``model_registry.json``) and for any local provider that hasn't
+ been validated yet.
+ """
+ try:
+ return await self.credentials_db.get_api_key_model_params(
+ provider, session_id
+ )
+ except Exception as e:
+ logger.error("Failed to get model_params", provider=provider, error=str(e))
+ return {}
+
async def get_api_key(self, provider: str, session_id: str = "default") -> Optional[str]:
"""Get decrypted API key.
diff --git a/server/services/llm/config.py b/server/services/llm/config.py
index 8f04dd60..045fddb8 100644
--- a/server/services/llm/config.py
+++ b/server/services/llm/config.py
@@ -109,6 +109,13 @@ def detect_provider_from_model(model: str) -> str:
def is_model_valid_for_provider(model: str, provider: str) -> bool:
+ # Open-world providers — OpenRouter is a multi-vendor proxy, ollama
+ # and lmstudio serve user-installed local models whose names don't
+ # match any "lmstudio"/"ollama" substring. Treat as always-valid;
+ # the upstream API will 404 a genuinely missing model. See the
+ # mirror in services/ai.py for the full rationale.
+ if provider in ('openrouter', 'ollama', 'lmstudio'):
+ return True
cfg = PROVIDER_CONFIGS.get(provider)
if not cfg:
return True
diff --git a/server/services/model_registry.py b/server/services/model_registry.py
index c0a2c1da..1c25aaf5 100644
--- a/server/services/model_registry.py
+++ b/server/services/model_registry.py
@@ -235,6 +235,82 @@ class ModelRegistryService:
return None
+ def register_local_model(
+ self,
+ provider: str,
+ model_id: str,
+ params: Dict[str, Any],
+ ) -> None:
+ """Register an Ollama / LM Studio model with its actual params.
+
+ Called from ``validate_local_llm`` after the official SDK probe
+ (``ollama.AsyncClient.ps()`` / ``lmstudio.AsyncClient.llm.list_loaded()``)
+ returns each currently-loaded model with its typed params:
+ ``context_length`` (live n_ctx the server enforces),
+ ``vision`` / ``supports_tools`` capability flags, and metadata
+ (``architecture``, ``param_size``, ``quantization``). The sync
+ ``get_context_length`` / ``get_max_output_tokens`` lookups find
+ this entry first, so chat / agent execution honour the real
+ n_ctx the server is serving — no JSON guess, no string parsing.
+
+ Stored under the same ``{provider}/{local_id}`` key shape as
+ cloud models so ``get_model_info``'s waterfall matches without
+ any per-provider branching. ``provider`` is canonical
+ (``ollama`` / ``lmstudio``).
+
+ Idempotent: a re-validation overwrites the prior entry.
+
+ Persisted to the same ``model_registry.json`` cache the
+ OpenRouter refresh writes — the entry survives a server
+ restart, so the user doesn't have to re-click "Fetch" every
+ time the process bounces just to keep the n_ctx context
+ registered for their local model.
+ """
+ ctx = int(params.get("context_length") or 0)
+ max_out = int(params.get("max_output_tokens") or 0)
+ # Default max_output to 25% of context (OpenRouter convention for
+ # local models), capped at 4096 — local backends rarely benefit
+ # from larger budgets and overcommitting eats into the prompt.
+ if not max_out and ctx:
+ max_out = min(4096, max(512, ctx // 4))
+
+ # Capability flags + metadata from the SDK probe go into
+ # `supported_parameters` so they survive the JSON roundtrip via
+ # `ModelInfo.to_dict` / `from_dict`. Cloud models populate this
+ # from OpenRouter's listing; for locals we use it as a typed
+ # capability bag so downstream code (tool-binding, vision UI
+ # gating) can read the same fields regardless of provider.
+ supported: List[str] = []
+ if params.get("supports_tools"):
+ supported.append("tools")
+ if params.get("vision"):
+ supported.append("vision")
+
+ # Match the user-friendly temperature default for OpenAI-compat
+ # local servers (0..2 range — same as the JSON default).
+ info = ModelInfo(
+ id=f"{provider}/{model_id}",
+ name=model_id,
+ provider=provider,
+ local_id=model_id,
+ context_length=ctx,
+ max_output_tokens=max_out,
+ temperature_range=(0.0, 2.0),
+ supported_parameters=supported,
+ )
+ self._models[f"{provider}/{model_id}"] = info
+ logger.info(
+ "[%s] registered model %s (ctx=%s, max_out=%s, tools=%s, vision=%s)",
+ provider, model_id, ctx, max_out,
+ params.get("supports_tools", False), params.get("vision", False),
+ )
+ # Persist so the entry survives process restart.
+ try:
+ self._save_cache()
+ except Exception as e: # noqa: BLE001 — best-effort persistence
+ logger.warning("Failed to persist local model entry %s/%s: %s",
+ provider, model_id, e)
+
def get_max_output_tokens(self, model: str, provider: str) -> int:
"""Get max output tokens: registry -> llm_defaults -> 4096."""
info = self.get_model_info(model, provider)