Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(llm): keep an endpoint save inside the request's time budget
trohitg8140a1e91541
diff --git a/client/src/components/credentials/panels/ApiKeyPanel.tsx b/client/src/components/credentials/panels/ApiKeyPanel.tsxindex 23e28ba1..f628f552 100644--- a/client/src/components/credentials/panels/ApiKeyPanel.tsx+++ b/client/src/components/credentials/panels/ApiKeyPanel.tsx@@ -17,6 +17,7 @@ import { useCredentialPanel } from '../useCredentialPanel';import { ProviderDefaultsSection, LlmUsageSection, ApiUsageSection } from '../sections';import { NodeIcon } from '../../../assets/icons';import { theme } from '../../../styles/theme';+import { CREDENTIAL_PROBE_REQUEST_TIMEOUT } from '@/contexts/WebSocketContext';import type { ServerEndpointSummary } from '@/hooks/useCatalogueQuery';import type { ProviderConfig } from '../types';import EndpointList from './EndpointList';@@ -47,12 +48,13 @@ const ApiKeyPanel: React.FC<{ config: ProviderConfig; visible: boolean }> = ({ c// Every field goes out under its catalogue key; the backend Credential// decides what each means. A rejected save carries its reason in// `message`, which the generic executor does not surface on its own.+ // Both probe the server, so they get the probe budget, not the default.const addEndpoint = async () => {const res = await panel.actions.sendWs('validate_api_key', {provider: config.id,api_key: inputValue.trim(),...Object.fromEntries(secondaryFields.map((sf) => [sf.key, String(panel.values[sf.key] ?? '').trim()])),- });+ }, CREDENTIAL_PROBE_REQUEST_TIMEOUT);if (res?.valid) {for (const f of config.fields ?? []) panel.form.setFieldValue(f.key, '');} else if (res?.message) {@@ -64,7 +66,7 @@ const ApiKeyPanel: React.FC<{ config: ProviderConfig; visible: boolean }> = ({ cprovider: config.id,api_key: endpoint.base_url || endpoint.ref,ref: endpoint.ref,- });+ }, CREDENTIAL_PROBE_REQUEST_TIMEOUT);if (res && !res.valid && res.message) panel.setError(res.message);};const removeEndpoint = (endpoint: ServerEndpointSummary) =>diff --git a/client/src/components/credentials/panels/__tests__/ApiKeyPanel.test.tsx b/client/src/components/credentials/panels/__tests__/ApiKeyPanel.test.tsxnew file mode 100644index 00000000..5400f1ab--- /dev/null+++ b/client/src/components/credentials/panels/__tests__/ApiKeyPanel.test.tsx@@ -0,0 +1,127 @@+/**+ * ApiKeyPanel for a provider that holds several rows (named OpenAI-compatible+ * endpoints): the form sends every field under its catalogue key, refresh and+ * remove address one endpoint by its reference, and the two requests that+ * probe the server get the probe budget instead of the 30 s default.+ */++import { beforeEach, describe, expect, it, vi } from 'vitest';+import { render, screen } from '@testing-library/react';+import userEvent from '@testing-library/user-event';++const sendWs = vi.fn();+const setFieldValue = vi.fn();+const setError = vi.fn();++vi.mock('../../useCredentialPanel', () => ({+ useCredentialPanel: () => ({+ values: {+ openai_compatible_url: 'http://gpu:8000',+ openai_compatible_label: 'Lab',+ openai_compatible_api_key: 'sk-1',+ },+ loading: null,+ error: null,+ stored: false,+ setError,+ form: { setFieldValue },+ actions: { sendWs, validate: vi.fn(), save: vi.fn(), remove: vi.fn() },+ }),+}));++vi.mock('../../../../assets/icons', () => ({ NodeIcon: () => null }));++import ApiKeyPanel from '../ApiKeyPanel';+import { CREDENTIAL_PROBE_REQUEST_TIMEOUT } from '@/contexts/WebSocketContext';+import type { ProviderConfig } from '../../types';++const LAB = {+ ref: 'openai_compatible:lab',+ label: 'Lab',+ base_url: 'http://gpu:8000/v1',+ kind: 'generic',+ model_count: 1,+};++const CONFIG: ProviderConfig = {+ id: 'openai_compatible',+ name: 'OpenAI-compatible',+ category: 'ai',+ categoryLabel: 'AI Models',+ color: '',+ kind: 'apiKey',+ iconRef: 'lucide:Plug',+ fields: [+ { key: 'openai_compatible_url', label: 'Base URL' },+ { key: 'openai_compatible_label', label: 'Label' },+ { key: 'openai_compatible_api_key', label: 'API key', secret: true },+ ],+ stored: true,+ endpoints: [LAB],+};++describe('ApiKeyPanel with named endpoints', () => {+ beforeEach(() => {+ sendWs.mockReset();+ setFieldValue.mockReset();+ setError.mockReset();+ });++ it('adds an endpoint with every field under its catalogue key', async () => {+ sendWs.mockResolvedValue({ valid: true });+ render(<ApiKeyPanel config={CONFIG} visible />);++ await userEvent.click(screen.getByRole('button', { name: 'Add endpoint' }));++ expect(sendWs).toHaveBeenCalledWith(+ 'validate_api_key',+ {+ provider: 'openai_compatible',+ api_key: 'http://gpu:8000',+ openai_compatible_label: 'Lab',+ openai_compatible_api_key: 'sk-1',+ },+ CREDENTIAL_PROBE_REQUEST_TIMEOUT,+ );+ expect(setFieldValue.mock.calls).toEqual([+ ['openai_compatible_url', ''],+ ['openai_compatible_label', ''],+ ['openai_compatible_api_key', ''],+ ]);+ });++ it("shows the server's reason when a save is refused and keeps the form", async () => {+ sendWs.mockResolvedValue({ valid: false, message: 'No OpenAI-compatible API answered at http://gpu:8000.' });+ render(<ApiKeyPanel config={CONFIG} visible />);++ await userEvent.click(screen.getByRole('button', { name: 'Add endpoint' }));++ expect(setError).toHaveBeenCalledWith('No OpenAI-compatible API answered at http://gpu:8000.');+ expect(setFieldValue).not.toHaveBeenCalled();+ });++ it('refreshes an endpoint by its reference', async () => {+ sendWs.mockResolvedValue({ valid: true });+ render(<ApiKeyPanel config={CONFIG} visible />);++ await userEvent.click(screen.getByRole('button', { name: 'Refresh Lab' }));++ expect(sendWs).toHaveBeenCalledWith(+ 'validate_api_key',+ { provider: 'openai_compatible', api_key: 'http://gpu:8000/v1', ref: 'openai_compatible:lab' },+ CREDENTIAL_PROBE_REQUEST_TIMEOUT,+ );+ });++ it('removes an endpoint by its reference', async () => {+ render(<ApiKeyPanel config={CONFIG} visible />);++ await userEvent.click(screen.getByRole('button', { name: 'Remove Lab' }));++ expect(sendWs).toHaveBeenCalledWith('delete_api_key', { provider: 'openai_compatible:lab' });+ });++ it('gives a probing request longer than the default request timeout', () => {+ expect(CREDENTIAL_PROBE_REQUEST_TIMEOUT).toBeGreaterThan(30_000);+ });+});diff --git a/client/src/components/credentials/useCredentialPanel.ts b/client/src/components/credentials/useCredentialPanel.tsindex c074e181..f939cb41 100644--- a/client/src/components/credentials/useCredentialPanel.ts+++ b/client/src/components/credentials/useCredentialPanel.ts@@ -213,7 +213,8 @@ export function useCredentialPanel(config: ProviderConfig, visible: boolean) {return sendRequest(config.ws!.logout, {});}),oauthRefresh: () => execute('refresh', () => sendRequest(config.ws!.status, {})),- sendWs: (type: string, data?: Record<string, any>) => execute(type, () => sendRequest(type, data ?? {})),+ sendWs: (type: string, data?: Record<string, any>, timeoutMs?: number) =>+ execute(type, () => sendRequest(type, data ?? {}, timeoutMs)),};return {diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsxindex 6b22b6d8..709f80b7 100644--- a/client/src/contexts/WebSocketContext.tsx+++ b/client/src/contexts/WebSocketContext.tsx@@ -73,6 +73,13 @@ const REQUEST_TIMEOUT = 30000;// while retaining a finite bound so a lost response cannot lock controls forever.export const WORKFLOW_CONTROL_REQUEST_TIMEOUT = 5 * 60 * 1000;+// validate_api_key probes the user's server when the credential is a Base+// URL (Ollama, LM Studio, named OpenAI-compatible endpoints): it roots the+// URL, detects the server's kind and lists its models. The backend bounds+// each step (nodes/model/_local_validator.py); this stays above their sum, so+// a slow save is not cut off here and then completes behind the user's back.+export const CREDENTIAL_PROBE_REQUEST_TIMEOUT = 60 * 1000;+// Maximum queued sends before backpressure kicks in (FIFO eviction of oldest)const QUEUE_MAX_SIZE = 200;@@ -472,8 +479,9 @@ interface WebSocketContextValue {clearChatMessages: () => void;sendChatMessage: (message: string, nodeId?: string) => Promise<void>;- // Generic request method- sendRequest: <T = any>(type: string, data?: Record<string, any>) => Promise<T>;+ // Generic request method. timeoutMs: omitted = the 30 s default,+ // negative = no timeout.+ sendRequest: <T = any>(type: string, data?: Record<string, any>, timeoutMs?: number) => Promise<T>;// Generic broadcast subscription. Returns an unsubscribe fn.// Use for ad-hoc backend-pushed events like `workflow_ops_apply`@@ -3123,7 +3131,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chiconst response = await sendRequest<any>('validate_api_key', {provider,api_key: apiKey- });+ }, CREDENTIAL_PROBE_REQUEST_TIMEOUT);// Backend returns one of:// { success: true, valid: true, models } — key is good// { success: true, valid: false, message } — clean rejection (401/403/timeout/etc)diff --git a/server/nodes/model/_local_validator.py b/server/nodes/model/_local_validator.pyindex fb176199..cb4ad6d5 100644--- a/server/nodes/model/_local_validator.py+++ b/server/nodes/model/_local_validator.py@@ -30,6 +30,7 @@ a re-Fetch must not take down a configuration that works.from __future__ import annotations+import asyncioimport timefrom datetime import datetime, timezonefrom typing import Any, Dict, List, Optional, Tuple@@ -47,9 +48,14 @@ logger = get_logger(__name__)_DISPLAY_NAMES = {"ollama": "Ollama", "lmstudio": "LM Studio"}-# Native routes are only asked once the host is known to answer, so a-# short timeout is enough and keeps a save well inside the WS budget.+# A save runs inside one WS request, so every step after rooting is bounded+# (the frontend waits CREDENTIAL_PROBE_REQUEST_TIMEOUT). Native routes are+# only asked once the host is known to answer, so a short timeout is enough;+# the three are asked at once, so a slow host costs one timeout, not three._NATIVE_TIMEOUT_SECONDS = 3.0+# The Ollama and LM Studio SDK probes. Ollama's client times out on its own;+# LM Studio's takes no timeout, so the bound is applied around both.+_SDK_PROBE_TIMEOUT_SECONDS = 10.0def _classify_http_error(display: str, base_url: str, exc: BaseException) -> Tuple[str, str]:@@ -59,7 +65,7 @@ def _classify_http_error(display: str, base_url: str, exc: BaseException) -> TupRooting has already succeeded by the time a native probe runs, so afailure here is about the native API, not the base URL."""- if isinstance(exc, httpx.TimeoutException):+ if isinstance(exc, (httpx.TimeoutException, asyncio.TimeoutError)):return ("timeout", f"Request to {base_url} timed out — server may be overloaded or unreachable")if isinstance(exc, httpx.ConnectError):@@ -236,22 +242,25 @@ async def _detect_kind(base_url: str, api_key: str) -> Tuple[str, Optional[Dict["""Which server answers at ``base_url``, from its native routes.Each route is accepted only by body shape, never by status: LM Studio- answers HTTP 200 to routes it does not serve. llama.cpp is asked first- because its server also mimics some Ollama routes. Returns the kind and,- for llama.cpp, the ``/props`` body (it carries ``n_ctx``).+ answers HTTP 200 to routes it does not serve. The three routes are asked+ at once; llama.cpp wins when several answer, because its server also+ mimics some Ollama routes. Returns the kind and, for llama.cpp, the+ ``/props`` body (it carries ``n_ctx``)."""host = _strip_v1_path(base_url)headers = {"Authorization": f"Bearer {api_key}"}async with httpx.AsyncClient(timeout=_NATIVE_TIMEOUT_SECONDS, headers=headers) as client:- props = await _get_json(client, f"{host}/props")- if isinstance(props, dict) and isinstance(props.get("default_generation_settings"), dict):- return "llamacpp", props- body = await _get_json(client, f"{host}/api/v1/models")- if isinstance(body, dict) and isinstance(body.get("models"), list):- return "lmstudio", None- body = await _get_json(client, f"{host}/api/version")- if isinstance(body, dict) and isinstance(body.get("version"), str):- return "ollama", None+ props, lmstudio_models, ollama_version = await asyncio.gather(+ _get_json(client, f"{host}/props"),+ _get_json(client, f"{host}/api/v1/models"),+ _get_json(client, f"{host}/api/version"),+ )+ if isinstance(props, dict) and isinstance(props.get("default_generation_settings"), dict):+ return "llamacpp", props+ if isinstance(lmstudio_models, dict) and isinstance(lmstudio_models.get("models"), list):+ return "lmstudio", None+ if isinstance(ollama_version, dict) and isinstance(ollama_version.get("version"), str):+ return "ollama", Nonereturn "generic", None@@ -307,7 +316,8 @@ async def _describe_models("""if kind in ("ollama", "lmstudio"):fetch = _fetch_ollama_models if kind == "ollama" else _fetch_lmstudio_models- return {e["id"]: {k: v for k, v in e.items() if k != "id"} for e in await fetch(base_url)}+ loaded = await asyncio.wait_for(fetch(base_url), _SDK_PROBE_TIMEOUT_SECONDS)+ return {e["id"]: {k: v for k, v in e.items() if k != "id"} for e in loaded}ids = [entry for entry in entries if isinstance(getattr(entry, "id", None), str) and entry.id]if kind == "llamacpp":diff --git a/server/services/model_registry.py b/server/services/model_registry.pyindex 46fe00a9..605cdef0 100644--- a/server/services/model_registry.py+++ b/server/services/model_registry.py@@ -16,6 +16,7 @@ temperature constraints, thinking capabilities). Three sources, three files:Falls back to llm_defaults.json offline."""+import asyncioimport jsonimport refrom dataclasses import dataclass, asdict, field@@ -46,6 +47,11 @@ _LITELLM_FIELDS = ("input_cost_per_token","output_cost_per_token",)+# An on-demand fetch runs inside an endpoint save, so it is bounded in+# total, and a failed one is not repeated on every save: offline, each save+# would otherwise wait out the timeout. The 24 h refresh is not affected.+_LITELLM_ON_DEMAND_TIMEOUT_SECONDS = 8.0+_LITELLM_RETRY_AFTER = timedelta(minutes=10)def _local_models_path() -> Path:@@ -155,6 +161,7 @@ class ModelRegistryService:# Kept apart so an OpenRouter refresh cannot replace them.self._local_models: Dict[str, ModelInfo] = {}self._litellm: Dict[str, Dict[str, Any]] = {}+ self._litellm_failed_at: Optional[datetime] = Noneself._cache_timestamp: Optional[datetime] = Noneself._llm_defaults: Dict[str, Any] = {}@@ -673,15 +680,24 @@ class ModelRegistryService:matches = [spec for key, spec in self._litellm.items() if key.partition("/")[2] == model_id]return matches[0] if len(matches) == 1 else None- async def ensure_litellm_table(self) -> None:+ async def ensure_litellm_table(self, timeout: float = _LITELLM_ON_DEMAND_TIMEOUT_SECONDS) -> None:"""Fetch the LiteLLM table if this process has none yet.The 24 h refresh keeps it current; this covers a first endpointsave on a machine whose OpenRouter snapshot was still fresh, so- the refresh has not run yet.+ the refresh has not run yet. It runs inside that save, so it gets+ ``timeout`` seconds in total, and after a failure it waits+ ``_LITELLM_RETRY_AFTER`` before trying again. Never raises."""- if not self._litellm:- await self._refresh_litellm()+ if self._litellm:+ return+ if self._litellm_failed_at and datetime.now(timezone.utc) - self._litellm_failed_at < _LITELLM_RETRY_AFTER:+ return+ try:+ await asyncio.wait_for(self._refresh_litellm(), timeout)+ except asyncio.TimeoutError:+ self._litellm_failed_at = datetime.now(timezone.utc)+ logger.warning(f"Model registry: LiteLLM table fetch took longer than {timeout:g}s; not retried for a while")async def _refresh_litellm(self) -> None:"""Fetch and trim the LiteLLM table. Never raises."""@@ -691,14 +707,17 @@ class ModelRegistryService:response.raise_for_status()raw = response.json()except Exception as e: # noqa: BLE001 — a stale copy beats none+ self._litellm_failed_at = datetime.now(timezone.utc)logger.warning(f"Model registry: LiteLLM table refresh failed, keeping the cached copy: {e}")returntable = self._parse_litellm(raw)if not table:+ self._litellm_failed_at = datetime.now(timezone.utc)logger.warning("Model registry: LiteLLM table had no chat models, keeping the cached copy")returnself._litellm = table+ self._litellm_failed_at = Noneself._save_litellm_cache()logger.info(f"Model registry: loaded {len(table)} chat models from LiteLLM")diff --git a/server/tests/nodes/test_local_llm_save.py b/server/tests/nodes/test_local_llm_save.pyindex ae0ca5df..06812f1a 100644--- a/server/tests/nodes/test_local_llm_save.py+++ b/server/tests/nodes/test_local_llm_save.py@@ -8,6 +8,7 @@ what matters here is that they are still the ones called.from __future__ import annotations+import asynciofrom typing import Any, Dict, List, Optionalfrom unittest.mock import AsyncMock, MagicMock@@ -119,6 +120,31 @@ class TestKindDetection:assert await lv._detect_kind("http://host:8000/v1", "k") == ("generic", None)+ @respx.mock+ async def test_llamacpp_wins_when_it_also_answers_like_ollama(self):+ props = {"default_generation_settings": {"n_ctx": 4096}}+ respx.get("http://host:8080/props").mock(return_value=httpx.Response(200, json=props))+ respx.get("http://host:8080/api/version").mock(return_value=httpx.Response(200, json={"version": "0.1"}))++ assert await lv._detect_kind("http://host:8080/v1", "k") == ("llamacpp", props)++ async def test_the_native_routes_are_asked_at_once(self, monkeypatch):+ # A slow host then costs one probe timeout, not three.+ in_flight = peak = 0++ async def get_json(client, url):+ nonlocal in_flight, peak+ in_flight += 1+ peak = max(peak, in_flight)+ await asyncio.sleep(0.01)+ in_flight -= 1+ return None++ monkeypatch.setattr(lv, "_get_json", get_json)++ assert await lv._detect_kind("http://host:8000/v1", "k") == ("generic", None)+ assert peak == 3+class TestFailedSaveChangesNothing:@respx.mock@@ -168,6 +194,23 @@ class TestFailedSaveChangesNothing:assert registry.get_model_info("model.gguf", REF) is Nonebroadcaster.update_api_key_status.assert_not_awaited()+ @respx.mock+ async def test_a_hung_sdk_probe_fails_the_save_within_its_bound(self, auth, broadcaster, registry, monkeypatch):+ # LM Studio's SDK takes no timeout of its own.+ async def never_answers(base_url):+ await asyncio.sleep(5)++ respx.get("http://host:1234/models").mock(return_value=httpx.Response(200, json=_models("qwen3")))+ monkeypatch.setattr(lv, "_fetch_lmstudio_models", never_answers)+ monkeypatch.setattr(lv, "_SDK_PROBE_TIMEOUT_SECONDS", 0.05)++ result = await lv.validate_local_llm({"provider": "lmstudio", "api_key": "http://host:1234"})++ assert result["valid"] is False+ assert "timed out" in result["message"]+ assert auth.stored == []+ broadcaster.update_api_key_status.assert_not_awaited()+@respx.mockasync def test_a_rejected_first_save_leaves_no_url_row(self, auth, broadcaster, registry):auth.rejects.add(REF)diff --git a/server/tests/services/test_model_registry_caches.py b/server/tests/services/test_model_registry_caches.pyindex 28aa8ed7..f1c70d50 100644--- a/server/tests/services/test_model_registry_caches.py+++ b/server/tests/services/test_model_registry_caches.py@@ -8,6 +8,7 @@ under DATA_DIR, and matched conservatively.from __future__ import annotations+import asyncioimport jsonimport httpx@@ -155,6 +156,31 @@ class TestLiteLLMTable:assert registry.lookup_litellm("gpt-4o") is None+ @respx.mock+ async def test_an_on_demand_fetch_that_failed_is_not_repeated_on_every_save(self, registry):+ route = respx.get(mr.LITELLM_MODELS_URL).mock(side_effect=httpx.ConnectError("offline"))++ await registry.ensure_litellm_table()+ await registry.ensure_litellm_table()++ assert route.call_count == 1++ async def test_an_on_demand_fetch_is_bounded_in_total(self, registry, monkeypatch):+ calls = 0++ async def slow_refresh():+ nonlocal calls+ calls += 1+ await asyncio.sleep(5)++ monkeypatch.setattr(registry, "_refresh_litellm", slow_refresh)++ await asyncio.wait_for(registry.ensure_litellm_table(timeout=0.05), 1)+ await registry.ensure_litellm_table(timeout=0.05)++ assert calls == 1+ assert registry.lookup_litellm("gpt-4o") is None+@respx.mockasync def test_the_cache_is_reloaded_on_startup(self, registry):_mock_feeds()