Skip to content

Self-improving AI Employees for everyone

fix(llm): keep an endpoint save inside the request's time budget

trohitg8140a1e91541
diff --git a/client/src/components/credentials/panels/ApiKeyPanel.tsx b/client/src/components/credentials/panels/ApiKeyPanel.tsx
index 23e28ba1..f628f552 100644
--- a/client/src/components/credentials/panels/ApiKeyPanel.tsx
+++ b/client/src/components/credentials/panels/ApiKeyPanel.tsx
@@ -17,6 +17,7 @@ import { useCredentialPanel } from '../useCredentialPanel';
import { ProviderDefaultsSection, LlmUsageSection, ApiUsageSection } from '../sections';
import { NodeIcon } from '../../../assets/icons';
import { theme } from '../../../styles/theme';
+import { CREDENTIAL_PROBE_REQUEST_TIMEOUT } from '@/contexts/WebSocketContext';
import type { ServerEndpointSummary } from '@/hooks/useCatalogueQuery';
import type { ProviderConfig } from '../types';
import EndpointList from './EndpointList';
@@ -47,12 +48,13 @@ const ApiKeyPanel: React.FC<{ config: ProviderConfig; visible: boolean }> = ({ c
// Every field goes out under its catalogue key; the backend Credential
// decides what each means. A rejected save carries its reason in
// `message`, which the generic executor does not surface on its own.
+ // Both probe the server, so they get the probe budget, not the default.
const addEndpoint = async () => {
const res = await panel.actions.sendWs('validate_api_key', {
provider: config.id,
api_key: inputValue.trim(),
...Object.fromEntries(secondaryFields.map((sf) => [sf.key, String(panel.values[sf.key] ?? '').trim()])),
- });
+ }, CREDENTIAL_PROBE_REQUEST_TIMEOUT);
if (res?.valid) {
for (const f of config.fields ?? []) panel.form.setFieldValue(f.key, '');
} else if (res?.message) {
@@ -64,7 +66,7 @@ const ApiKeyPanel: React.FC<{ config: ProviderConfig; visible: boolean }> = ({ c
provider: config.id,
api_key: endpoint.base_url || endpoint.ref,
ref: endpoint.ref,
- });
+ }, CREDENTIAL_PROBE_REQUEST_TIMEOUT);
if (res && !res.valid && res.message) panel.setError(res.message);
};
const removeEndpoint = (endpoint: ServerEndpointSummary) =>
diff --git a/client/src/components/credentials/panels/__tests__/ApiKeyPanel.test.tsx b/client/src/components/credentials/panels/__tests__/ApiKeyPanel.test.tsx
new file mode 100644
index 00000000..5400f1ab
--- /dev/null
+++ b/client/src/components/credentials/panels/__tests__/ApiKeyPanel.test.tsx
@@ -0,0 +1,127 @@
+/**
+ * ApiKeyPanel for a provider that holds several rows (named OpenAI-compatible
+ * endpoints): the form sends every field under its catalogue key, refresh and
+ * remove address one endpoint by its reference, and the two requests that
+ * probe the server get the probe budget instead of the 30 s default.
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { render, screen } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+
+const sendWs = vi.fn();
+const setFieldValue = vi.fn();
+const setError = vi.fn();
+
+vi.mock('../../useCredentialPanel', () => ({
+ useCredentialPanel: () => ({
+ values: {
+ openai_compatible_url: 'http://gpu:8000',
+ openai_compatible_label: 'Lab',
+ openai_compatible_api_key: 'sk-1',
+ },
+ loading: null,
+ error: null,
+ stored: false,
+ setError,
+ form: { setFieldValue },
+ actions: { sendWs, validate: vi.fn(), save: vi.fn(), remove: vi.fn() },
+ }),
+}));
+
+vi.mock('../../../../assets/icons', () => ({ NodeIcon: () => null }));
+
+import ApiKeyPanel from '../ApiKeyPanel';
+import { CREDENTIAL_PROBE_REQUEST_TIMEOUT } from '@/contexts/WebSocketContext';
+import type { ProviderConfig } from '../../types';
+
+const LAB = {
+ ref: 'openai_compatible:lab',
+ label: 'Lab',
+ base_url: 'http://gpu:8000/v1',
+ kind: 'generic',
+ model_count: 1,
+};
+
+const CONFIG: ProviderConfig = {
+ id: 'openai_compatible',
+ name: 'OpenAI-compatible',
+ category: 'ai',
+ categoryLabel: 'AI Models',
+ color: '',
+ kind: 'apiKey',
+ iconRef: 'lucide:Plug',
+ fields: [
+ { key: 'openai_compatible_url', label: 'Base URL' },
+ { key: 'openai_compatible_label', label: 'Label' },
+ { key: 'openai_compatible_api_key', label: 'API key', secret: true },
+ ],
+ stored: true,
+ endpoints: [LAB],
+};
+
+describe('ApiKeyPanel with named endpoints', () => {
+ beforeEach(() => {
+ sendWs.mockReset();
+ setFieldValue.mockReset();
+ setError.mockReset();
+ });
+
+ it('adds an endpoint with every field under its catalogue key', async () => {
+ sendWs.mockResolvedValue({ valid: true });
+ render(<ApiKeyPanel config={CONFIG} visible />);
+
+ await userEvent.click(screen.getByRole('button', { name: 'Add endpoint' }));
+
+ expect(sendWs).toHaveBeenCalledWith(
+ 'validate_api_key',
+ {
+ provider: 'openai_compatible',
+ api_key: 'http://gpu:8000',
+ openai_compatible_label: 'Lab',
+ openai_compatible_api_key: 'sk-1',
+ },
+ CREDENTIAL_PROBE_REQUEST_TIMEOUT,
+ );
+ expect(setFieldValue.mock.calls).toEqual([
+ ['openai_compatible_url', ''],
+ ['openai_compatible_label', ''],
+ ['openai_compatible_api_key', ''],
+ ]);
+ });
+
+ it("shows the server's reason when a save is refused and keeps the form", async () => {
+ sendWs.mockResolvedValue({ valid: false, message: 'No OpenAI-compatible API answered at http://gpu:8000.' });
+ render(<ApiKeyPanel config={CONFIG} visible />);
+
+ await userEvent.click(screen.getByRole('button', { name: 'Add endpoint' }));
+
+ expect(setError).toHaveBeenCalledWith('No OpenAI-compatible API answered at http://gpu:8000.');
+ expect(setFieldValue).not.toHaveBeenCalled();
+ });
+
+ it('refreshes an endpoint by its reference', async () => {
+ sendWs.mockResolvedValue({ valid: true });
+ render(<ApiKeyPanel config={CONFIG} visible />);
+
+ await userEvent.click(screen.getByRole('button', { name: 'Refresh Lab' }));
+
+ expect(sendWs).toHaveBeenCalledWith(
+ 'validate_api_key',
+ { provider: 'openai_compatible', api_key: 'http://gpu:8000/v1', ref: 'openai_compatible:lab' },
+ CREDENTIAL_PROBE_REQUEST_TIMEOUT,
+ );
+ });
+
+ it('removes an endpoint by its reference', async () => {
+ render(<ApiKeyPanel config={CONFIG} visible />);
+
+ await userEvent.click(screen.getByRole('button', { name: 'Remove Lab' }));
+
+ expect(sendWs).toHaveBeenCalledWith('delete_api_key', { provider: 'openai_compatible:lab' });
+ });
+
+ it('gives a probing request longer than the default request timeout', () => {
+ expect(CREDENTIAL_PROBE_REQUEST_TIMEOUT).toBeGreaterThan(30_000);
+ });
+});
diff --git a/client/src/components/credentials/useCredentialPanel.ts b/client/src/components/credentials/useCredentialPanel.ts
index c074e181..f939cb41 100644
--- a/client/src/components/credentials/useCredentialPanel.ts
+++ b/client/src/components/credentials/useCredentialPanel.ts
@@ -213,7 +213,8 @@ export function useCredentialPanel(config: ProviderConfig, visible: boolean) {
return sendRequest(config.ws!.logout, {});
}),
oauthRefresh: () => execute('refresh', () => sendRequest(config.ws!.status, {})),
- sendWs: (type: string, data?: Record<string, any>) => execute(type, () => sendRequest(type, data ?? {})),
+ sendWs: (type: string, data?: Record<string, any>, timeoutMs?: number) =>
+ execute(type, () => sendRequest(type, data ?? {}, timeoutMs)),
};
return {
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index 6b22b6d8..709f80b7 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -73,6 +73,13 @@ const REQUEST_TIMEOUT = 30000;
// while retaining a finite bound so a lost response cannot lock controls forever.
export const WORKFLOW_CONTROL_REQUEST_TIMEOUT = 5 * 60 * 1000;
+// validate_api_key probes the user's server when the credential is a Base
+// URL (Ollama, LM Studio, named OpenAI-compatible endpoints): it roots the
+// URL, detects the server's kind and lists its models. The backend bounds
+// each step (nodes/model/_local_validator.py); this stays above their sum, so
+// a slow save is not cut off here and then completes behind the user's back.
+export const CREDENTIAL_PROBE_REQUEST_TIMEOUT = 60 * 1000;
+
// Maximum queued sends before backpressure kicks in (FIFO eviction of oldest)
const QUEUE_MAX_SIZE = 200;
@@ -472,8 +479,9 @@ interface WebSocketContextValue {
clearChatMessages: () => void;
sendChatMessage: (message: string, nodeId?: string) => Promise<void>;
- // Generic request method
- sendRequest: <T = any>(type: string, data?: Record<string, any>) => Promise<T>;
+ // Generic request method. timeoutMs: omitted = the 30 s default,
+ // negative = no timeout.
+ sendRequest: <T = any>(type: string, data?: Record<string, any>, timeoutMs?: number) => Promise<T>;
// Generic broadcast subscription. Returns an unsubscribe fn.
// Use for ad-hoc backend-pushed events like `workflow_ops_apply`
@@ -3123,7 +3131,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const response = await sendRequest<any>('validate_api_key', {
provider,
api_key: apiKey
- });
+ }, CREDENTIAL_PROBE_REQUEST_TIMEOUT);
// Backend returns one of:
// { success: true, valid: true, models } — key is good
// { success: true, valid: false, message } — clean rejection (401/403/timeout/etc)
diff --git a/server/nodes/model/_local_validator.py b/server/nodes/model/_local_validator.py
index fb176199..cb4ad6d5 100644
--- a/server/nodes/model/_local_validator.py
+++ b/server/nodes/model/_local_validator.py
@@ -30,6 +30,7 @@ a re-Fetch must not take down a configuration that works.
from __future__ import annotations
+import asyncio
import time
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
@@ -47,9 +48,14 @@ logger = get_logger(__name__)
_DISPLAY_NAMES = {"ollama": "Ollama", "lmstudio": "LM Studio"}
-# Native routes are only asked once the host is known to answer, so a
-# short timeout is enough and keeps a save well inside the WS budget.
+# A save runs inside one WS request, so every step after rooting is bounded
+# (the frontend waits CREDENTIAL_PROBE_REQUEST_TIMEOUT). Native routes are
+# only asked once the host is known to answer, so a short timeout is enough;
+# the three are asked at once, so a slow host costs one timeout, not three.
_NATIVE_TIMEOUT_SECONDS = 3.0
+# The Ollama and LM Studio SDK probes. Ollama's client times out on its own;
+# LM Studio's takes no timeout, so the bound is applied around both.
+_SDK_PROBE_TIMEOUT_SECONDS = 10.0
def _classify_http_error(display: str, base_url: str, exc: BaseException) -> Tuple[str, str]:
@@ -59,7 +65,7 @@ def _classify_http_error(display: str, base_url: str, exc: BaseException) -> Tup
Rooting has already succeeded by the time a native probe runs, so a
failure here is about the native API, not the base URL.
"""
- if isinstance(exc, httpx.TimeoutException):
+ if isinstance(exc, (httpx.TimeoutException, asyncio.TimeoutError)):
return ("timeout", f"Request to {base_url} timed out — server may be overloaded or unreachable")
if isinstance(exc, httpx.ConnectError):
@@ -236,22 +242,25 @@ async def _detect_kind(base_url: str, api_key: str) -> Tuple[str, Optional[Dict[
"""Which server answers at ``base_url``, from its native routes.
Each route is accepted only by body shape, never by status: LM Studio
- answers HTTP 200 to routes it does not serve. llama.cpp is asked first
- because its server also mimics some Ollama routes. Returns the kind and,
- for llama.cpp, the ``/props`` body (it carries ``n_ctx``).
+ answers HTTP 200 to routes it does not serve. The three routes are asked
+ at once; llama.cpp wins when several answer, because its server also
+ mimics some Ollama routes. Returns the kind and, for llama.cpp, the
+ ``/props`` body (it carries ``n_ctx``).
"""
host = _strip_v1_path(base_url)
headers = {"Authorization": f"Bearer {api_key}"}
async with httpx.AsyncClient(timeout=_NATIVE_TIMEOUT_SECONDS, headers=headers) as client:
- props = await _get_json(client, f"{host}/props")
- if isinstance(props, dict) and isinstance(props.get("default_generation_settings"), dict):
- return "llamacpp", props
- body = await _get_json(client, f"{host}/api/v1/models")
- if isinstance(body, dict) and isinstance(body.get("models"), list):
- return "lmstudio", None
- body = await _get_json(client, f"{host}/api/version")
- if isinstance(body, dict) and isinstance(body.get("version"), str):
- return "ollama", None
+ props, lmstudio_models, ollama_version = await asyncio.gather(
+ _get_json(client, f"{host}/props"),
+ _get_json(client, f"{host}/api/v1/models"),
+ _get_json(client, f"{host}/api/version"),
+ )
+ if isinstance(props, dict) and isinstance(props.get("default_generation_settings"), dict):
+ return "llamacpp", props
+ if isinstance(lmstudio_models, dict) and isinstance(lmstudio_models.get("models"), list):
+ return "lmstudio", None
+ if isinstance(ollama_version, dict) and isinstance(ollama_version.get("version"), str):
+ return "ollama", None
return "generic", None
@@ -307,7 +316,8 @@ async def _describe_models(
"""
if kind in ("ollama", "lmstudio"):
fetch = _fetch_ollama_models if kind == "ollama" else _fetch_lmstudio_models
- return {e["id"]: {k: v for k, v in e.items() if k != "id"} for e in await fetch(base_url)}
+ loaded = await asyncio.wait_for(fetch(base_url), _SDK_PROBE_TIMEOUT_SECONDS)
+ return {e["id"]: {k: v for k, v in e.items() if k != "id"} for e in loaded}
ids = [entry for entry in entries if isinstance(getattr(entry, "id", None), str) and entry.id]
if kind == "llamacpp":
diff --git a/server/services/model_registry.py b/server/services/model_registry.py
index 46fe00a9..605cdef0 100644
--- a/server/services/model_registry.py
+++ b/server/services/model_registry.py
@@ -16,6 +16,7 @@ temperature constraints, thinking capabilities). Three sources, three files:
Falls back to llm_defaults.json offline.
"""
+import asyncio
import json
import re
from dataclasses import dataclass, asdict, field
@@ -46,6 +47,11 @@ _LITELLM_FIELDS = (
"input_cost_per_token",
"output_cost_per_token",
)
+# An on-demand fetch runs inside an endpoint save, so it is bounded in
+# total, and a failed one is not repeated on every save: offline, each save
+# would otherwise wait out the timeout. The 24 h refresh is not affected.
+_LITELLM_ON_DEMAND_TIMEOUT_SECONDS = 8.0
+_LITELLM_RETRY_AFTER = timedelta(minutes=10)
def _local_models_path() -> Path:
@@ -155,6 +161,7 @@ class ModelRegistryService:
# Kept apart so an OpenRouter refresh cannot replace them.
self._local_models: Dict[str, ModelInfo] = {}
self._litellm: Dict[str, Dict[str, Any]] = {}
+ self._litellm_failed_at: Optional[datetime] = None
self._cache_timestamp: Optional[datetime] = None
self._llm_defaults: Dict[str, Any] = {}
@@ -673,15 +680,24 @@ class ModelRegistryService:
matches = [spec for key, spec in self._litellm.items() if key.partition("/")[2] == model_id]
return matches[0] if len(matches) == 1 else None
- async def ensure_litellm_table(self) -> None:
+ async def ensure_litellm_table(self, timeout: float = _LITELLM_ON_DEMAND_TIMEOUT_SECONDS) -> None:
"""Fetch the LiteLLM table if this process has none yet.
The 24 h refresh keeps it current; this covers a first endpoint
save on a machine whose OpenRouter snapshot was still fresh, so
- the refresh has not run yet.
+ the refresh has not run yet. It runs inside that save, so it gets
+ ``timeout`` seconds in total, and after a failure it waits
+ ``_LITELLM_RETRY_AFTER`` before trying again. Never raises.
"""
- if not self._litellm:
- await self._refresh_litellm()
+ if self._litellm:
+ return
+ if self._litellm_failed_at and datetime.now(timezone.utc) - self._litellm_failed_at < _LITELLM_RETRY_AFTER:
+ return
+ try:
+ await asyncio.wait_for(self._refresh_litellm(), timeout)
+ except asyncio.TimeoutError:
+ self._litellm_failed_at = datetime.now(timezone.utc)
+ logger.warning(f"Model registry: LiteLLM table fetch took longer than {timeout:g}s; not retried for a while")
async def _refresh_litellm(self) -> None:
"""Fetch and trim the LiteLLM table. Never raises."""
@@ -691,14 +707,17 @@ class ModelRegistryService:
response.raise_for_status()
raw = response.json()
except Exception as e: # noqa: BLE001 — a stale copy beats none
+ self._litellm_failed_at = datetime.now(timezone.utc)
logger.warning(f"Model registry: LiteLLM table refresh failed, keeping the cached copy: {e}")
return
table = self._parse_litellm(raw)
if not table:
+ self._litellm_failed_at = datetime.now(timezone.utc)
logger.warning("Model registry: LiteLLM table had no chat models, keeping the cached copy")
return
self._litellm = table
+ self._litellm_failed_at = None
self._save_litellm_cache()
logger.info(f"Model registry: loaded {len(table)} chat models from LiteLLM")
diff --git a/server/tests/nodes/test_local_llm_save.py b/server/tests/nodes/test_local_llm_save.py
index ae0ca5df..06812f1a 100644
--- a/server/tests/nodes/test_local_llm_save.py
+++ b/server/tests/nodes/test_local_llm_save.py
@@ -8,6 +8,7 @@ what matters here is that they are still the ones called.
from __future__ import annotations
+import asyncio
from typing import Any, Dict, List, Optional
from unittest.mock import AsyncMock, MagicMock
@@ -119,6 +120,31 @@ class TestKindDetection:
assert await lv._detect_kind("http://host:8000/v1", "k") == ("generic", None)
+ @respx.mock
+ async def test_llamacpp_wins_when_it_also_answers_like_ollama(self):
+ props = {"default_generation_settings": {"n_ctx": 4096}}
+ respx.get("http://host:8080/props").mock(return_value=httpx.Response(200, json=props))
+ respx.get("http://host:8080/api/version").mock(return_value=httpx.Response(200, json={"version": "0.1"}))
+
+ assert await lv._detect_kind("http://host:8080/v1", "k") == ("llamacpp", props)
+
+ async def test_the_native_routes_are_asked_at_once(self, monkeypatch):
+ # A slow host then costs one probe timeout, not three.
+ in_flight = peak = 0
+
+ async def get_json(client, url):
+ nonlocal in_flight, peak
+ in_flight += 1
+ peak = max(peak, in_flight)
+ await asyncio.sleep(0.01)
+ in_flight -= 1
+ return None
+
+ monkeypatch.setattr(lv, "_get_json", get_json)
+
+ assert await lv._detect_kind("http://host:8000/v1", "k") == ("generic", None)
+ assert peak == 3
+
class TestFailedSaveChangesNothing:
@respx.mock
@@ -168,6 +194,23 @@ class TestFailedSaveChangesNothing:
assert registry.get_model_info("model.gguf", REF) is None
broadcaster.update_api_key_status.assert_not_awaited()
+ @respx.mock
+ async def test_a_hung_sdk_probe_fails_the_save_within_its_bound(self, auth, broadcaster, registry, monkeypatch):
+ # LM Studio's SDK takes no timeout of its own.
+ async def never_answers(base_url):
+ await asyncio.sleep(5)
+
+ respx.get("http://host:1234/models").mock(return_value=httpx.Response(200, json=_models("qwen3")))
+ monkeypatch.setattr(lv, "_fetch_lmstudio_models", never_answers)
+ monkeypatch.setattr(lv, "_SDK_PROBE_TIMEOUT_SECONDS", 0.05)
+
+ result = await lv.validate_local_llm({"provider": "lmstudio", "api_key": "http://host:1234"})
+
+ assert result["valid"] is False
+ assert "timed out" in result["message"]
+ assert auth.stored == []
+ broadcaster.update_api_key_status.assert_not_awaited()
+
@respx.mock
async def test_a_rejected_first_save_leaves_no_url_row(self, auth, broadcaster, registry):
auth.rejects.add(REF)
diff --git a/server/tests/services/test_model_registry_caches.py b/server/tests/services/test_model_registry_caches.py
index 28aa8ed7..f1c70d50 100644
--- a/server/tests/services/test_model_registry_caches.py
+++ b/server/tests/services/test_model_registry_caches.py
@@ -8,6 +8,7 @@ under DATA_DIR, and matched conservatively.
from __future__ import annotations
+import asyncio
import json
import httpx
@@ -155,6 +156,31 @@ class TestLiteLLMTable:
assert registry.lookup_litellm("gpt-4o") is None
+ @respx.mock
+ async def test_an_on_demand_fetch_that_failed_is_not_repeated_on_every_save(self, registry):
+ route = respx.get(mr.LITELLM_MODELS_URL).mock(side_effect=httpx.ConnectError("offline"))
+
+ await registry.ensure_litellm_table()
+ await registry.ensure_litellm_table()
+
+ assert route.call_count == 1
+
+ async def test_an_on_demand_fetch_is_bounded_in_total(self, registry, monkeypatch):
+ calls = 0
+
+ async def slow_refresh():
+ nonlocal calls
+ calls += 1
+ await asyncio.sleep(5)
+
+ monkeypatch.setattr(registry, "_refresh_litellm", slow_refresh)
+
+ await asyncio.wait_for(registry.ensure_litellm_table(timeout=0.05), 1)
+ await registry.ensure_litellm_table(timeout=0.05)
+
+ assert calls == 1
+ assert registry.lookup_litellm("gpt-4o") is None
+
@respx.mock
async def test_the_cache_is_reloaded_on_startup(self, registry):
_mock_feeds()