Open-source personal finance manager. Self-hosted, privacy-first.
fix: agents config failures that were silent or unexplained (#577)
Tássio Noronha4c4c7f03cb5e
diff --git a/.env.example b/.env.exampleindex b1026ff5..4c4e9486 100644--- a/.env.example+++ b/.env.example@@ -69,6 +69,9 @@ ENABLE_BANKING_APP_ID=# AI Agents (optional) — to enable, set BOTH:# 1) the feature flag# 2) COMPOSE_PROFILES so `docker compose up` starts the mcp-server container+# (Docker Compose only. On a bare-metal/LXC install set just the flag,+# run `uvicorn mcp_server.main:app --port 8765` next to the API, and set+# AGENTS_BUILTIN_MCP_URL=http://127.0.0.1:8765/mcp)# AGENTS_ENABLED=true# COMPOSE_PROFILES=agents# Public URL external agents use to reach the built-in MCP server, shown in thediff --git a/README.md b/README.mdindex 30566634..7e365278 100644--- a/README.md+++ b/README.md@@ -171,6 +171,22 @@ COMPOSE_PROFILES=agentsThen `docker compose up -d`. Settings → AI Agents to add a provider connection. Off by default; zero cost when off.+### Without Docker++`COMPOSE_PROFILES=agents` only tells Docker Compose to start the extra `mcp-server` container, so on a bare-metal or LXC install set `AGENTS_ENABLED=true` alone. The built-in MCP server is a plain uvicorn app in the same virtualenv; run it next to the API and point the backend at it:++```bash+# alongside the API/worker/beat processes+uvicorn mcp_server.main:app --host 127.0.0.1 --port 8765+```++```+AGENTS_ENABLED=true+AGENTS_BUILTIN_MCP_URL=http://127.0.0.1:8765/mcp+```++Without that server the agents still chat, but they have no tools and cannot read your data. The backend log says which MCP server it failed to reach.+## Tech Stack| Layer | Stack |diff --git a/backend/app/agents/config.py b/backend/app/agents/config.pyindex 9faae635..e84c2147 100644--- a/backend/app/agents/config.py+++ b/backend/app/agents/config.py@@ -1,4 +1,5 @@from functools import lru_cache+from pathlib import Pathfrom pydantic_settings import BaseSettings, SettingsConfigDict@@ -63,7 +64,14 @@ class AgentSettings(BaseSettings):knowledge_storage_path: str = "/app/data/agent_knowledge"knowledge_max_file_size_mb: int = 25- model_config = SettingsConfigDict(env_file=".env", env_prefix="AGENTS_", extra="ignore")+ # Same env_file pair as the main Settings: the CWD-relative ".env" for+ # backward compatibility plus the anchored backend/.env, so the API and the+ # Celery worker/beat read the same file whatever directory they start from.+ model_config = SettingsConfigDict(+ env_file=(".env", Path(__file__).resolve().parents[2] / ".env"),+ env_prefix="AGENTS_",+ extra="ignore",+ )@lru_cachediff --git a/backend/app/agents/mcp/client.py b/backend/app/agents/mcp/client.pyindex 1503ed75..8d4e276b 100644--- a/backend/app/agents/mcp/client.py+++ b/backend/app/agents/mcp/client.py@@ -5,6 +5,7 @@ Per call, mints a short-lived JWT scoped to (user_id, conversation_id)."""from __future__ import annotations+import loggingimport uuidfrom dataclasses import dataclassfrom typing import Any, Optional@@ -15,6 +16,8 @@ from app.agents.config import get_agent_settingsfrom app.agents.mcp.auth import mint_tokenfrom app.agents.providers.base import ToolDefinition+logger = logging.getLogger(__name__)+@dataclassclass ToolHandle:@@ -146,7 +149,18 @@ class MCPRegistry:for client in self._servers.values():try:tools = await client.list_tools(token=token)- except Exception:+ except Exception as exc:+ # A server that can't be reached costs the agent every tool it+ # exposes, and the chat still answers, just without any data.+ # Say so in the log: otherwise the only symptom is an assistant+ # claiming it can't see anything.+ logger.warning(+ "MCP server %r unreachable at %s (%s). Its tools are "+ "unavailable for this conversation.",+ client.name,+ client.url,+ exc,+ )continueout.extend(tools)return out@@ -203,7 +217,14 @@ class MCPRegistry:for server_name, client in self._servers.items():try:handles = await client.list_tools(token=token)- except Exception:+ except Exception as exc:+ logger.warning(+ "MCP server %r unreachable at %s (%s) while resolving tool %r.",+ server_name,+ client.url,+ exc,+ wire_name,+ )continueif any(h.name == bare for h in handles):return await client.call_tool(name=bare, arguments=arguments, token=token)diff --git a/backend/app/agents/services/connection_service.py b/backend/app/agents/services/connection_service.pyindex 27ac36fc..ae2732d3 100644--- a/backend/app/agents/services/connection_service.py+++ b/backend/app/agents/services/connection_service.py@@ -2,7 +2,10 @@from __future__ import annotationsimport uuid+from ipaddress import ip_address+from pathlib import Pathfrom typing import Any, Optional+from urllib.parse import urlparsefrom sqlalchemy import select, updatefrom sqlalchemy.ext.asyncio import AsyncSession@@ -132,6 +135,54 @@ def build_provider_for_connection(conn: LlmConnection):)+def _in_container() -> bool:+ """Whether this process runs inside a container. Only used to decide+ whether a routing hint would help; never to change behaviour."""+ return Path("/.dockerenv").exists() or Path("/run/.containerenv").exists()+++def _routing_hint(url: str) -> str:+ """Point at the usual cause when a container can't reach a model server+ the user can reach from their browser: the address is theirs, not the+ container's."""+ if not url or not _in_container():+ return ""+ host = (urlparse(url).hostname or "").strip("[]").lower()+ if not host:+ return ""+ port = urlparse(url).port+ suggestion = f"http://host.docker.internal:{port}" if port else "http://host.docker.internal"++ if host == "localhost":+ loopback, private = True, False+ else:+ try:+ ip = ip_address(host)+ except ValueError:+ return ""+ loopback, private = ip.is_loopback, ip.is_private++ if loopback:+ return (+ f". Securo runs in a container, where localhost is the container itself. "+ f"Use {suggestion} when the model server runs on the host machine."+ )+ if private:+ return (+ f". Securo runs in a container, which often can't reach the host's LAN address. "+ f"Use {suggestion} when the model server runs on the host machine."+ )+ return ""+++def _unreachable(url: str, exc: Exception) -> dict[str, Any]:+ """Never return a bare 'unreachable:'. Connect timeouts stringify to an+ empty message, which left the UI showing no reason at all."""+ reason = str(exc).strip() or type(exc).__name__+ where = f" at {url}" if url else ""+ return {"ok": False, "detail": f"unreachable{where}: {reason}{_routing_hint(url)}"}++async def test_connection(conn: LlmConnection) -> dict[str, Any]:"""Live probe. For Ollama we list models; for OpenAI/Anthropic we doa tiny embedding or a 1-token chat. Returns {ok, detail, models?}."""@@ -139,10 +190,13 @@ async def test_connection(conn: LlmConnection) -> dict[str, Any]:api_key = decrypt(conn.api_key_encrypted) or ""timeout = httpx.Timeout(10.0, connect=5.0)+ # Tracked so the failure path can name the address that was tried.+ attempted = conn.base_url or ""try:if conn.kind == "ollama":url = (conn.base_url or "http://ollama:11434").rstrip("/") + "/api/tags"+ attempted = urlasync with httpx.AsyncClient(timeout=timeout) as client:r = await client.get(url)r.raise_for_status()@@ -156,6 +210,7 @@ async def test_connection(conn: LlmConnection) -> dict[str, Any]:from app.agents.providers.openai import normalize_openai_base_urlbase = normalize_openai_base_url(conn.base_url or "https://api.openai.com/v1")+ attempted = baseheaders = {"Authorization": f"Bearer {api_key}"} if api_key else {}async with httpx.AsyncClient(timeout=timeout) as client:r = await client.get(f"{base}/models", headers=headers)@@ -180,6 +235,7 @@ async def test_connection(conn: LlmConnection) -> dict[str, Any]:if conn.kind == "anthropic":base = (conn.base_url or "https://api.anthropic.com/v1").rstrip("/")+ attempted = baseheaders = {"x-api-key": api_key, "anthropic-version": "2023-06-01"}async with httpx.AsyncClient(timeout=timeout) as client:r = await client.get(f"{base}/models", headers=headers)@@ -193,6 +249,6 @@ async def test_connection(conn: LlmConnection) -> dict[str, Any]:return {"ok": False, "detail": f"unknown kind: {conn.kind}"}except httpx.HTTPError as exc:- return {"ok": False, "detail": f"unreachable: {exc}"}+ return _unreachable(attempted, exc)except Exception as exc: # noqa: BLE001- return {"ok": False, "detail": str(exc)}+ return {"ok": False, "detail": str(exc).strip() or type(exc).__name__}diff --git a/backend/app/core/config.py b/backend/app/core/config.pyindex b8ae10e2..7608cf57 100644--- a/backend/app/core/config.py+++ b/backend/app/core/config.py@@ -125,9 +125,17 @@ class Settings(BaseSettings):# same file no matter which working directory each service is launched# from (a systemd unit without WorkingDirectory= used to leave the worker# with default settings, silently disabling every bank-sync provider).+ #+ # extra="ignore" because the same .env is shared with Docker Compose and with+ # the optional modules: it legitimately holds keys this model doesn't declare+ # (COMPOSE_PROFILES, FRONTEND_PORT, AGENTS_*, ...). Unknown keys coming from+ # the process environment are ignored by pydantic-settings anyway; without+ # this, the very same key written into the .env file aborted startup with+ # "Extra inputs are not permitted".model_config = SettingsConfigDict(env_file=(".env", Path(__file__).resolve().parents[2] / ".env"),secrets_dir=CREDENTIALS_DIRECTORY,+ extra="ignore",)diff --git a/backend/tests/test_agents_connection_service.py b/backend/tests/test_agents_connection_service.pyindex 234a6347..f042872a 100644--- a/backend/tests/test_agents_connection_service.py+++ b/backend/tests/test_agents_connection_service.py@@ -309,3 +309,42 @@ async def test_probe_swallows_httpx_errors(monkeypatch):out = await cs.test_connection(_conn(kind="ollama"))assert out["ok"] is Falseassert "unreachable" in out["detail"]+++async def test_probe_names_the_error_when_it_stringifies_empty(monkeypatch):+ """httpx connect timeouts carry no message, which used to render as a+ bare 'unreachable:' with no reason and no address in the UI."""+ import httpx++ class _TimesOut:+ def __init__(self, *_, **__):+ pass++ async def __aenter__(self):+ raise httpx.ConnectTimeout("")++ async def __aexit__(self, *exc):+ return False++ monkeypatch.setattr(httpx, "AsyncClient", _TimesOut)+ out = await cs.test_connection(+ _conn(kind="openai_compatible", base_url="http://192.168.1.152:1234")+ )+ assert out["ok"] is False+ assert "ConnectTimeout" in out["detail"]+ assert "192.168.1.152:1234" in out["detail"]+++def test_routing_hint_only_fires_inside_a_container(monkeypatch):+ """A LAN or loopback address the browser can reach is often unroutable+ from the container, so suggest the host gateway. Bare-metal installs+ reach those addresses fine and must not get the hint."""+ monkeypatch.setattr(cs, "_in_container", lambda: True)+ assert "host.docker.internal:1234" in cs._routing_hint("http://192.168.1.152:1234")+ assert "host.docker.internal:11434" in cs._routing_hint("http://localhost:11434")+ # Public hosts are reachable from anywhere; nothing to suggest.+ assert cs._routing_hint("https://api.openai.com/v1") == ""++ monkeypatch.setattr(cs, "_in_container", lambda: False)+ assert cs._routing_hint("http://192.168.1.152:1234") == ""diff --git a/backend/tests/test_agents_mcp_client.py b/backend/tests/test_agents_mcp_client.pyindex 219cedf4..e09b053d 100644--- a/backend/tests/test_agents_mcp_client.py+++ b/backend/tests/test_agents_mcp_client.py@@ -284,6 +284,26 @@ async def test_registry_discover_swallows_per_server_errors(monkeypatch):assert {h.name for h in handles} == {"a"}+async def test_registry_discover_warns_when_a_server_is_unreachable(monkeypatch, caplog):+ """An unreachable server leaves the agent with no tools while the chat+ still answers, so the log has to name the server and its URL. Otherwise+ the only symptom is an assistant saying it can't see any data."""+ from app.agents.config import get_agent_settings++ monkeypatch.setattr(get_agent_settings(), "extra_mcp_servers", "")+ monkeypatch.setattr(get_agent_settings(), "builtin_mcp_url", "http://mcp-server:8765/mcp")+ _FakeAsyncClient.queue.append(_FakeResponse(status_code=500, json_body={}))++ reg = MCPRegistry()+ with caplog.at_level("WARNING", logger="app.agents.mcp.client"):+ handles = await reg.discover(user_id=uuid.uuid4())++ assert handles == []+ assert "http://mcp-server:8765/mcp" in caplog.text+ assert "securo" in caplog.text++@pytest.mark.asyncioasync def test_registry_call_routes_via_namespaced_name():_FakeAsyncClient.queue.append(_FakeResponse(json_body={diff --git a/backend/tests/test_config.py b/backend/tests/test_config.pyindex 5a7c7c83..13b306ba 100644--- a/backend/tests/test_config.py+++ b/backend/tests/test_config.py@@ -25,6 +25,34 @@ def test_env_file_is_anchored_to_backend_dir():assert backend_env in tuple(Path(p) for p in env_files)+def test_env_file_with_unknown_keys_still_loads(tmp_path: Path, secrets: Path):+ """The .env is shared with Docker Compose and the optional modules, so it+ holds keys Settings doesn't declare (COMPOSE_PROFILES, FRONTEND_PORT,+ AGENTS_*). Unknown keys must be ignored, not abort startup."""+ env_file = tmp_path / ".env"+ env_file.write_text(+ "SECRET_KEY=from-env-file\n"+ "AGENTS_ENABLED=true\n"+ "COMPOSE_PROFILES=agents\n"+ "FRONTEND_PORT=3132\n"+ )++ settings = Settings(_env_file=str(env_file), _secrets_dir=str(secrets))++ assert settings.secret_key.get_secret_value() == "from-env-file"+++def test_agent_settings_env_file_is_anchored_to_backend_dir():+ """Agents settings must resolve the same .env as the main Settings, so a+ worker started outside backend/ doesn't silently fall back to defaults."""+ from app.agents.config import AgentSettings++ env_files = AgentSettings.model_config["env_file"]+ backend_env = Path(__file__).resolve().parents[1] / ".env"+ assert isinstance(env_files, tuple)+ assert backend_env in tuple(Path(p) for p in env_files)++def test_oidc_secret_reads_from_secrets_dir(secrets: Path):write(secrets, "oidc_client_secret", "file-secret")diff --git a/docker-compose.prod.yml b/docker-compose.prod.ymlindex dfcf5671..96ad9c71 100644--- a/docker-compose.prod.yml+++ b/docker-compose.prod.yml@@ -68,6 +68,11 @@ x-backend-env: &backend-envx-backend-base: &backend-baseimage: ghcr.io/securo-finance/securo-backend:latestenvironment: *backend-env+ # Lets a connection point at a model server running on the host (LM Studio,+ # Ollama, llama.cpp) as http://host.docker.internal:<port>. Docker Desktop+ # resolves the name on its own; this makes it work on Linux too.+ extra_hosts:+ - "host.docker.internal:host-gateway"depends_on:db: { condition: service_healthy }redis: { condition: service_healthy }diff --git a/docker-compose.yml b/docker-compose.ymlindex be28d9a1..dc438cd4 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -74,6 +74,11 @@ x-backend-env: &backend-envx-backend-base: &backend-basebuild: ./backendenvironment: *backend-env+ # Lets a connection point at a model server running on the host (LM Studio,+ # Ollama, llama.cpp) as http://host.docker.internal:<port>. Docker Desktop+ # resolves the name on its own; this makes it work on Linux too.+ extra_hosts:+ - "host.docker.internal:host-gateway"depends_on:db: { condition: service_healthy }redis: { condition: service_healthy }diff --git a/frontend/src/locales/de.json b/frontend/src/locales/de.jsonindex 99ace48d..c954e9e2 100644--- a/frontend/src/locales/de.json+++ b/frontend/src/locales/de.json@@ -1644,7 +1644,7 @@"saving": "Wird gespeichert…","saved": "Werkzeuge aktualisiert","saveFailed": "Fehler beim Speichern der Werkzeugauswahl",- "empty": "Keine Werkzeuge gefunden. Läuft der MCP-Server? (`docker compose --profile agents up -d`)",+ "empty": "Keine Werkzeuge gefunden. Läuft der MCP-Server? Mit Docker: `docker compose --profile agents up -d`.","proposeBadge": "vorschlagen","loading": "Werkzeuge werden geladen…"},diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.jsonindex 801f0460..985ce235 100644--- a/frontend/src/locales/en.json+++ b/frontend/src/locales/en.json@@ -1644,7 +1644,7 @@"saving": "Saving…","saved": "Tools updated","saveFailed": "Failed to save tool selection",- "empty": "No tools discovered. Is the MCP server running? (`docker compose --profile agents up -d`)",+ "empty": "No tools discovered. Is the MCP server running? With Docker: `docker compose --profile agents up -d`.","proposeBadge": "propose","loading": "Loading tools…"},diff --git a/frontend/src/locales/es.json b/frontend/src/locales/es.jsonindex 46c2eeea..de6dd308 100644--- a/frontend/src/locales/es.json+++ b/frontend/src/locales/es.json@@ -1629,7 +1629,7 @@"saving": "Guardando…","saved": "Herramientas actualizadas","saveFailed": "No se pudo guardar la selección de herramientas",- "empty": "No se detectaron herramientas. ¿Está en ejecución el servidor MCP? (`docker compose --profile agents up -d`)",+ "empty": "No se detectaron herramientas. ¿Está en ejecución el servidor MCP? Con Docker: `docker compose --profile agents up -d`.","proposeBadge": "proponer","loading": "Cargando herramientas…"},diff --git a/frontend/src/locales/fr.json b/frontend/src/locales/fr.jsonindex 3563c124..92375888 100644--- a/frontend/src/locales/fr.json+++ b/frontend/src/locales/fr.json@@ -1644,7 +1644,7 @@"saving": "Enregistrement…","saved": "Outils mis à jour","saveFailed": "Échec de l'enregistrement de la sélection d'outils",- "empty": "Aucun outil découvert. Le serveur MCP est-il en cours d'exécution ? (`docker compose --profile agents up -d`)",+ "empty": "Aucun outil découvert. Le serveur MCP est-il en cours d'exécution ? Avec Docker : `docker compose --profile agents up -d`.","proposeBadge": "proposer","loading": "Chargement des outils…"},diff --git a/frontend/src/locales/it.json b/frontend/src/locales/it.jsonindex f1bcae01..7c0c8d66 100644--- a/frontend/src/locales/it.json+++ b/frontend/src/locales/it.json@@ -1629,7 +1629,7 @@"saving": "Salvataggio…","saved": "Strumenti aggiornati","saveFailed": "Impossibile salvare la selezione degli strumenti",- "empty": "Nessuno strumento rilevato. Il server MCP è in esecuzione? (`docker compose --profile agents up -d`)",+ "empty": "Nessuno strumento rilevato. Il server MCP è in esecuzione? Con Docker: `docker compose --profile agents up -d`.","proposeBadge": "proponi","loading": "Caricamento strumenti…"},diff --git a/frontend/src/locales/pl.json b/frontend/src/locales/pl.jsonindex e7bb15cc..20470340 100644--- a/frontend/src/locales/pl.json+++ b/frontend/src/locales/pl.json@@ -1722,7 +1722,7 @@"saving": "Zapisywanie…","saved": "Zaktualizowano narzędzia","saveFailed": "Nie udało się zapisać wyboru narzędzi",- "empty": "Nie wykryto narzędzi. Czy serwer MCP działa? (`docker compose --profile agents up -d`)",+ "empty": "Nie wykryto narzędzi. Czy serwer MCP działa? W Dockerze: `docker compose --profile agents up -d`.","proposeBadge": "propozycja","loading": "Wczytywanie narzędzi…"},diff --git a/frontend/src/locales/pt-BR.json b/frontend/src/locales/pt-BR.jsonindex d7352ef5..c442da3d 100644--- a/frontend/src/locales/pt-BR.json+++ b/frontend/src/locales/pt-BR.json@@ -1629,7 +1629,7 @@"saving": "Salvando…","saved": "Ferramentas atualizadas","saveFailed": "Falha ao salvar a seleção de ferramentas",- "empty": "Nenhuma ferramenta descoberta. O servidor MCP está rodando? (`docker compose --profile agents up -d`)",+ "empty": "Nenhuma ferramenta descoberta. O servidor MCP está rodando? Com Docker: `docker compose --profile agents up -d`.","proposeBadge": "proposta","loading": "Carregando ferramentas…"},diff --git a/frontend/src/locales/pt-PT.json b/frontend/src/locales/pt-PT.jsonindex fc5f8412..a74d2358 100644--- a/frontend/src/locales/pt-PT.json+++ b/frontend/src/locales/pt-PT.json@@ -1629,7 +1629,7 @@"saving": "A guardar…","saved": "Ferramentas atualizadas","saveFailed": "Falha ao guardar a seleção de ferramentas",- "empty": "Nenhuma ferramenta descoberta. O servidor MCP está a correr? (`docker compose --profile agents up -d`)",+ "empty": "Nenhuma ferramenta descoberta. O servidor MCP está a correr? Com Docker: `docker compose --profile agents up -d`.","proposeBadge": "proposta","loading": "A carregar ferramentas…"},diff --git a/frontend/src/locales/ru.json b/frontend/src/locales/ru.jsonindex b057b6d0..3ed7a893 100644--- a/frontend/src/locales/ru.json+++ b/frontend/src/locales/ru.json@@ -1650,7 +1650,7 @@"saving": "Сохранение…","saved": "Инструменты обновлены","saveFailed": "Не удалось сохранить выбор инструментов",- "empty": "Инструменты не найдены. MCP-сервер запущен? (`docker compose --profile agents up -d`)",+ "empty": "Инструменты не найдены. MCP-сервер запущен? В Docker: `docker compose --profile agents up -d`.","proposeBadge": "предложить","loading": "Загрузка инструментов…"},diff --git a/frontend/src/locales/uk.json b/frontend/src/locales/uk.jsonindex 1c6172b9..fef07269 100644--- a/frontend/src/locales/uk.json+++ b/frontend/src/locales/uk.json@@ -1650,7 +1650,7 @@"saving": "Збереження…","saved": "Інструменти оновлено","saveFailed": "Не вдалося зберегти вибір інструментів",- "empty": "Інструменти не знайдено. MCP-сервер запущено? (`docker compose --profile agents up -d`)",+ "empty": "Інструменти не знайдено. MCP-сервер запущено? У Docker: `docker compose --profile agents up -d`.","proposeBadge": "запропонувати","loading": "Завантаження інструментів…"},