Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(credentials): server-owned provider registry + idempotency + OAuth circuit breaker
trohitg697c67623bf7
diff --git a/server/config/credential_providers.json b/server/config/credential_providers.jsonnew file mode 100644index 00000000..fa322667--- /dev/null+++ b/server/config/credential_providers.json@@ -0,0 +1,392 @@+{+ "version": "2026.04",+ "last_updated": "2026-04-11",+ "_description": "Credential provider catalogue. Loaded by server/services/credential_registry.py at startup; served to clients via the get_credential_catalogue WebSocket handler. Supports `extends` inheritance with array-merge-by-key semantics (see credential_registry.resolve_extends). `_abstract: true` entries are resolved but excluded from get_all_providers(). Color tokens are resolved client-side via the theme helper.",++ "categories": {+ "ai": { "label": "AI Providers", "order": 1 },+ "social": { "label": "Social Media", "order": 2 },+ "productivity": { "label": "Productivity", "order": 3 },+ "email": { "label": "Email", "order": 4 },+ "android": { "label": "Android", "order": 5 },+ "search": { "label": "Search", "order": 6 },+ "scrapers": { "label": "Scrapers", "order": 7 },+ "services": { "label": "Services", "order": 8 }+ },++ "providers": {+ "_ai_base": {+ "_abstract": true,+ "category": "ai",+ "category_label": "AI Providers",+ "kind": "apiKey",+ "has_defaults": true,+ "icon_ref": "ai:${id}",+ "fields": [+ {+ "key": "apiKey",+ "label": "API Key",+ "type": "password",+ "secret": true,+ "required": true+ }+ ]+ },++ "openai": {+ "extends": "_ai_base",+ "name": "OpenAI",+ "color": "dracula.green",+ "fields": [+ { "key": "apiKey", "placeholder": "sk-..." }+ ]+ },+ "anthropic": {+ "extends": "_ai_base",+ "name": "Anthropic",+ "color": "dracula.orange",+ "fields": [+ { "key": "apiKey", "placeholder": "sk-ant-..." }+ ]+ },+ "gemini": {+ "extends": "_ai_base",+ "name": "Gemini",+ "color": "solarized.blue",+ "fields": [+ { "key": "apiKey", "placeholder": "AIza..." }+ ]+ },+ "groq": {+ "extends": "_ai_base",+ "name": "Groq",+ "color": "dracula.red",+ "fields": [+ { "key": "apiKey", "placeholder": "gsk_..." }+ ]+ },+ "cerebras": {+ "extends": "_ai_base",+ "name": "Cerebras",+ "color": "dracula.orange",+ "fields": [+ { "key": "apiKey", "placeholder": "csk-..." }+ ]+ },+ "openrouter": {+ "extends": "_ai_base",+ "name": "OpenRouter",+ "color": "solarized.violet",+ "fields": [+ { "key": "apiKey", "placeholder": "sk-or-..." }+ ]+ },+ "deepseek": {+ "extends": "_ai_base",+ "name": "DeepSeek",+ "color": "dracula.cyan",+ "fields": [+ { "key": "apiKey", "placeholder": "sk-..." }+ ]+ },+ "kimi": {+ "extends": "_ai_base",+ "name": "Kimi",+ "color": "dracula.purple",+ "fields": [+ { "key": "apiKey", "placeholder": "sk-..." }+ ]+ },+ "mistral": {+ "extends": "_ai_base",+ "name": "Mistral",+ "color": "dracula.pink",+ "fields": [+ { "key": "apiKey", "placeholder": "sk-..." }+ ]+ },++ "whatsapp_personal": {+ "name": "WhatsApp Personal",+ "category": "social",+ "category_label": "Social Media",+ "color": "theme.categoryWhatsapp",+ "kind": "qrPairing",+ "icon_ref": "data_uri:WHATSAPP_CONNECT_ICON",+ "status_hook": "whatsapp",+ "has_rate_limits": true,+ "status_rows": [+ { "key": "status", "label": "Status", "ok_field": "connected", "true_text": "Connected", "false_text": "Disconnected" },+ { "key": "session", "label": "Session", "ok_field": "has_session", "true_text": "Active", "false_text": "Inactive" },+ { "key": "service", "label": "Service", "ok_field": "running", "true_text": "Running", "false_text": "Stopped" }+ ],+ "actions": [+ { "key": "start", "label": "Start", "theme_color": "green" },+ { "key": "restart", "label": "Restart", "theme_color": "orange" },+ { "key": "refresh", "label": "Refresh", "theme_color": "cyan" }+ ],+ "qr": {+ "qr_field": "qr",+ "is_connected_field": "connected",+ "connected_title": "Already Connected!",+ "connected_subtitle_static": "No QR code needed",+ "is_loading_fields": ["running", "!qr", "!connected"],+ "empty_text_running": "Waiting for QR code...",+ "empty_text_stopped": "Start service to get QR code",+ "scan_text": "Scan with WhatsApp mobile app"+ }+ },++ "twitter": {+ "name": "Twitter/X",+ "category": "social",+ "category_label": "Social Media",+ "color": "theme.categoryTrigger",+ "kind": "oauth",+ "icon_ref": "data_uri:TWITTER_ICON",+ "status_hook": "twitter",+ "usage_service": "twitter",+ "ws": {+ "login": "twitter_oauth_login",+ "logout": "twitter_logout",+ "status": "twitter_oauth_status"+ },+ "fields": [+ {+ "key": "twitter_client_id",+ "label": "Client ID",+ "type": "string",+ "placeholder": "Client ID (from X Developer Portal)",+ "required": true+ },+ {+ "key": "twitter_client_secret",+ "label": "Client Secret",+ "type": "password",+ "secret": true,+ "placeholder": "Client Secret (optional for PKCE)"+ }+ ],+ "callback_url": "http://localhost:3010/api/twitter/callback",+ "instructions": "Get credentials from the X Developer Portal. Create an app with OAuth 2.0 enabled."+ },++ "telegram": {+ "name": "Telegram Bot",+ "category": "social",+ "category_label": "Social Media",+ "color": "theme.categoryChat",+ "kind": "oauth",+ "icon_ref": "raw_svg:TelegramIcons.telegram",+ "status_hook": "telegram",+ "ws": {+ "login": "telegram_connect",+ "logout": "telegram_disconnect",+ "status": "telegram_status"+ },+ "fields": [+ {+ "key": "telegram_bot_token",+ "label": "Bot Token",+ "type": "password",+ "secret": true,+ "placeholder": "123456789:ABCdefGHIjklMNOpqrSTUvwxYZ",+ "required": true+ }+ ],+ "instructions": "Get a bot token from @BotFather on Telegram. After connecting, send any message to your bot to register as the owner."+ },++ "gmail": {+ "name": "Google Workspace",+ "category": "productivity",+ "category_label": "Productivity",+ "color": "theme.categoryAI",+ "kind": "oauth",+ "icon_ref": "raw_svg:GoogleIcons.gmail",+ "status_hook": "google",+ "usage_service": "google_workspace",+ "ws": {+ "login": "google_oauth_login",+ "logout": "google_logout",+ "status": "google_oauth_status"+ },+ "fields": [+ {+ "key": "google_client_id",+ "label": "Client ID",+ "type": "string",+ "placeholder": "Client ID (from Google Cloud Console)",+ "required": true+ },+ {+ "key": "google_client_secret",+ "label": "Client Secret",+ "type": "password",+ "secret": true,+ "placeholder": "Client Secret (required)",+ "required": true+ }+ ],+ "callback_url": "http://localhost:3010/api/google/callback",+ "instructions": "Get credentials from Google Cloud Console. Enable Gmail, Calendar, Drive, Sheets, Tasks, and People APIs."+ },++ "email_himalaya": {+ "name": "Email (IMAP/SMTP)",+ "category": "email",+ "category_label": "Email",+ "color": "theme.categoryChat",+ "kind": "email",+ "icon_ref": "raw_svg:EmailIcons.read"+ },++ "android_remote": {+ "name": "Android Device",+ "category": "android",+ "category_label": "Android",+ "color": "theme.categoryAndroid",+ "kind": "qrPairing",+ "icon_ref": "emoji:π±",+ "status_hook": "android",+ "fields": [+ {+ "key": "android_remote",+ "label": "API Key",+ "type": "password",+ "secret": true,+ "placeholder": "Enter Android Relay API key...",+ "required": true+ }+ ],+ "status_rows": [+ { "key": "device", "label": "Device", "ok_field": "paired", "true_text": "Paired", "false_text": "Not Paired" },+ { "key": "relay", "label": "Relay", "ok_field": "connected", "true_text": "Connected", "false_text": "Disconnected" },+ { "key": "pairing", "label": "Pairing", "ok_field": "paired", "true_text": "Paired", "false_text": "Waiting", "warn": true }+ ],+ "actions": [+ {+ "key": "connect",+ "label": "Connect",+ "theme_color": "green",+ "disabled_when": ["paired", "!stored"]+ },+ {+ "key": "disconnect",+ "label": "Disconnect",+ "theme_color": "pink",+ "disabled_when": ["!connected"]+ }+ ],+ "qr": {+ "qr_field": "qr_data",+ "is_connected_field": "paired",+ "connected_title": "Device Paired!",+ "connected_subtitle_field": "device_name",+ "connected_subtitle_fallback": "Android Device",+ "is_loading_fields": ["connected", "!qr_data", "!paired"],+ "empty_text_no_key": "Add API key first",+ "empty_text_running": "Waiting for QR code...",+ "empty_text_stopped": "Click Connect to start",+ "scan_text": "Scan with Android companion app"+ }+ },++ "brave_search": {+ "name": "Brave Search",+ "category": "search",+ "category_label": "Search",+ "color": "theme.categoryLocation",+ "kind": "apiKey",+ "icon_ref": "raw_svg:SearchIcons.brave",+ "fields": [+ {+ "key": "apiKey",+ "label": "API Key",+ "type": "password",+ "secret": true,+ "placeholder": "BSA...",+ "required": true+ }+ ]+ },++ "perplexity": {+ "name": "Perplexity",+ "category": "search",+ "category_label": "Search",+ "color": "theme.categoryUtil",+ "kind": "apiKey",+ "icon_ref": "raw_svg:SearchIcons.perplexity",+ "fields": [+ {+ "key": "apiKey",+ "label": "API Key",+ "type": "password",+ "secret": true,+ "placeholder": "pplx-...",+ "required": true+ }+ ]+ },++ "apify": {+ "name": "Apify",+ "category": "scrapers",+ "category_label": "Scrapers",+ "color": "theme.categoryAI",+ "kind": "apiKey",+ "icon_ref": "raw_svg:ApifyIcons.apify",+ "validate_as": "apify",+ "fields": [+ {+ "key": "apiKey",+ "label": "API Key",+ "type": "password",+ "secret": true,+ "placeholder": "apify_api_...",+ "required": true+ }+ ]+ },++ "serper": {+ "name": "Serper",+ "category": "scrapers",+ "category_label": "Scrapers",+ "color": "theme.categoryAI",+ "kind": "apiKey",+ "icon_ref": "raw_svg:SearchIcons.serper",+ "fields": [+ {+ "key": "apiKey",+ "label": "API Key",+ "type": "password",+ "secret": true,+ "required": true+ }+ ]+ },++ "google_maps": {+ "name": "Google Maps",+ "category": "services",+ "category_label": "Services",+ "color": "theme.categoryLocation",+ "kind": "apiKey",+ "icon_ref": "emoji:πΊοΈ",+ "validate_as": "google_maps",+ "usage_service": "google_maps",+ "fields": [+ {+ "key": "apiKey",+ "label": "API Key",+ "type": "password",+ "secret": true,+ "placeholder": "AIza...",+ "required": true+ }+ ]+ }+ }+}diff --git a/server/main.py b/server/main.pyindex 696dee53..5e50256d 100644--- a/server/main.py+++ b/server/main.py@@ -39,7 +39,7 @@ _startup_log("Importing settings + logging...")from core.config import Settingsfrom core.logging import configure_logging, get_logger, setup_websocket_logging, shutdown_websocket_logging_startup_log("Importing routers...")-from routers import workflow, database, maps, nodejs_compat, android, websocket, webhook, auth, twitter, google+from routers import workflow, database, maps, nodejs_compat, android, websocket, webhook, auth, twitter, google, credentials_startup_log("All imports complete")# Initialize settings and logging@@ -397,6 +397,7 @@ app.include_router(websocket.router)app.include_router(webhook.router)app.include_router(twitter.router) # Twitter/X OAuth routesapp.include_router(google.router) # Google Workspace OAuth routes (Gmail, Calendar, Drive, Sheets, Tasks, Contacts)+app.include_router(credentials.router) # Credentials panel - lazy per-tile icon endpoint (n8n pattern)@app.get("/health")diff --git a/server/routers/credentials.py b/server/routers/credentials.pynew file mode 100644index 00000000..00eab247--- /dev/null+++ b/server/routers/credentials.py@@ -0,0 +1,88 @@+"""Credentials routes.++Phase A of the credentials-scaling plan: lazy per-tile icon endpoint.++Serves SVG icons for credential providers out of+``server/static/credential_icons/<provider_id>.svg`` with long-lived+``Cache-Control: immutable`` headers. If no file is present, returns 404+so the client falls back to its existing in-bundle icon resolution (see+``client/src/components/credentials/providers.tsx`` and+``client/src/components/icons/AIProviderIcons.tsx``).++This follows n8n's lazy-icon pattern: the frontend does not need to bundle+icons as data URIs, and the backend can later serve per-customer or+user-uploaded provider icons without a frontend redeploy.+"""++from __future__ import annotations++from pathlib import Path++from fastapi import APIRouter, HTTPException+from fastapi.responses import FileResponse, Response++from core.logging import get_logger++logger = get_logger(__name__)++router = APIRouter(prefix="/api/credentials", tags=["credentials"])++# Resolve the icons directory relative to the server package root so it+# works under both local dev and Docker (where /app is the cwd).+_STATIC_ICONS_DIR = Path(__file__).parent.parent / "static" / "credential_icons"++# Only allow a conservative subset of characters in the provider_id to+# prevent path traversal. The credential registry uses lowercase+# alphanumerics and underscores throughout.+_ALLOWED_CHARS = set("abcdefghijklmnopqrstuvwxyz0123456789_-")++# 1 year, immutable: SVG content never changes for a given provider_id.+# If we ever need to invalidate, bump the version in+# credential_providers.json and the frontend will cache-bust via the+# catalogue version hash it already holds.+_IMMUTABLE_CACHE = "public, max-age=31536000, immutable"+++def _safe_icon_path(provider_id: str) -> Path | None:+ """Return an absolute path to the icon file if it's safe and exists.++ Returns None if the provider_id is malformed, points outside the+ icons directory, or the file is missing.+ """+ if not provider_id or len(provider_id) > 64:+ return None+ if not all(ch in _ALLOWED_CHARS for ch in provider_id):+ return None++ candidate = (_STATIC_ICONS_DIR / f"{provider_id}.svg").resolve()+ try:+ # Ensure the resolved path is still inside the icons dir.+ candidate.relative_to(_STATIC_ICONS_DIR.resolve())+ except ValueError:+ logger.warning("credentials.icon: path traversal rejected for %r", provider_id)+ return None++ if not candidate.is_file():+ return None+ return candidate++[email protected]("/icon/{provider_id}")+async def get_credential_icon(provider_id: str) -> Response:+ """Serve a credential provider icon SVG with long-lived caching.++ The frontend fetches this lazily per-tile as the credentials palette+ scrolls (n8n pattern). Missing icons return 404 and the client falls+ back to its in-bundle icon resolution, so this endpoint is optional+ for the current 20-provider catalogue but is wired now so the shape+ is stable for the 5000-provider target.+ """+ path = _safe_icon_path(provider_id)+ if path is None:+ raise HTTPException(status_code=404, detail="icon not found")++ return FileResponse(+ path,+ media_type="image/svg+xml",+ headers={"Cache-Control": _IMMUTABLE_CACHE},+ )diff --git a/server/routers/websocket.py b/server/routers/websocket.pyindex f3302cc4..1dbd434f 100644--- a/server/routers/websocket.py+++ b/server/routers/websocket.py@@ -233,6 +233,32 @@ async def handle_get_all_tool_schemas(data: Dict[str, Any], websocket: WebSocketreturn {"success": True, "schemas": schemas}+# ============================================================================+# Credential Registry Handler (Nango-style bulk fetch for 20 -> 5000 providers)+# ============================================================================++@ws_handler()+async def handle_get_credential_catalogue(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:+ """Return the full credential provider catalogue.++ Response shape: {providers, categories, version}. The version is a+ content-sha256 of the resolved payload; clients warm-start from+ IndexedDB and only re-fetch when the version changes. Supports+ Nango-style bulk fetch at modal-open time (one roundtrip, cached for+ the app session via TanStack Query + experimental_createPersister).+ """+ from services.credential_registry import get_credential_registry++ registry = get_credential_registry()+ # Optional conditional fetch: if the client already has the current+ # version in IndexedDB it can send `since` and get a 304-style response.+ since = data.get("since")+ version = registry.get_version()+ if since and since == version:+ return {"unchanged": True, "version": version}+ return registry.get_catalogue()++# ============================================================================# Node Execution Handlers# ============================================================================@@ -1021,19 +1047,48 @@ async def handle_get_stored_api_key(data: Dict[str, Any], websocket: WebSocket)@ws_handler("provider", "api_key")async def handle_save_api_key(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:- """Save an API key (without validation)."""- auth_service = container.auth_service()- await auth_service.store_api_key(provider=data["provider"].lower(), api_key=data["api_key"].strip(),- models=data.get("models", []), session_id=data.get("session_id", "default"))- return {"provider": data["provider"]}+ """Save an API key (without validation).++ Supports client-side idempotency: if the client supplies a+ `request_id` (opaque UUID), duplicate calls within 60 s return the+ cached result instead of re-running the mutation. Prevents+ double-writes on retry / reconnect / double-click.+ """+ from services.idempotency import get_idempotency_store++ store = get_idempotency_store("credentials")+ provider = data["provider"].lower()++ async def _do_save() -> Dict[str, Any]:+ auth_service = container.auth_service()+ await auth_service.store_api_key(+ provider=provider,+ api_key=data["api_key"].strip(),+ models=data.get("models", []),+ session_id=data.get("session_id", "default"),+ )+ return {"provider": data["provider"]}++ return await store.run(data.get("request_id"), _do_save)@ws_handler("provider")async def handle_delete_api_key(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:- """Delete stored API key."""- auth_service = container.auth_service()- await auth_service.remove_api_key(data["provider"].lower(), data.get("session_id", "default"))- return {"provider": data["provider"]}+ """Delete stored API key.++ Idempotent on `request_id` β see `handle_save_api_key`.+ """+ from services.idempotency import get_idempotency_store++ store = get_idempotency_store("credentials")+ provider = data["provider"].lower()++ async def _do_delete() -> Dict[str, Any]:+ auth_service = container.auth_service()+ await auth_service.remove_api_key(provider, data.get("session_id", "default"))+ return {"provider": data["provider"]}++ return await store.run(data.get("request_id"), _do_delete)# ============================================================================@@ -3058,6 +3113,9 @@ MESSAGE_HANDLERS: Dict[str, MessageHandler] = {"delete_tool_schema": handle_delete_tool_schema,"get_all_tool_schemas": handle_get_all_tool_schemas,+ # Credential registry (Nango-style bulk catalogue for credentials panel)+ "get_credential_catalogue": handle_get_credential_catalogue,+# Node execution"execute_node": handle_execute_node,"execute_workflow": handle_execute_workflow,diff --git a/server/services/auth.py b/server/services/auth.pyindex 0927d46b..938a489f 100644--- a/server/services/auth.py+++ b/server/services/auth.py@@ -285,6 +285,92 @@ class AuthService:logger.error("Failed to get OAuth tokens", provider=provider, error=str(e))return None+ async def refresh_oauth_tokens_with_breaker(+ self,+ provider: str,+ refresh_fn,+ ) -> Dict[str, Any]:+ """Run an OAuth refresh call under a circuit breaker.++ Phase 7.5c of the credentials-scaling plan. Protects downstream+ OAuth providers from cascading failures: after 3 consecutive+ refresh failures within 60 s for a given provider, the breaker+ opens and all subsequent calls short-circuit for 30 s with a+ clear error instead of piling on the upstream.++ Args:+ provider: Provider name (e.g. 'google', 'twitter') β used as+ the breaker scope so one provider failing does not trip+ the breaker for another.+ refresh_fn: Zero-arg callable returning the refresh result+ dict. Can be sync or async. Typical shape:+ ``{"success": True, "access_token": "..."}``+ or+ ``{"success": False, "error": "..."}``.++ Returns:+ The refresh_fn result on success, or a dict of+ ``{"success": False, "error": "...", "circuit_open": True,+ "retry_after_seconds": N}`` when the breaker is open.++ This method is opt-in: legacy call sites that call their OAuth+ helpers directly are unaffected. New/migrated call sites gain+ breaker protection by routing through here instead of invoking+ the refresh helper directly.+ """+ import asyncio+ from services.circuit_breaker import get_circuit_breaker, CircuitBreakerOpen++ breaker = get_circuit_breaker(+ f"{provider}_oauth_refresh",+ failure_threshold=3,+ failure_window=60.0,+ cooldown_seconds=30.0,+ )++ async def _call():+ if asyncio.iscoroutinefunction(refresh_fn):+ result = await refresh_fn()+ else:+ # Run sync callables in a thread to avoid blocking the+ # event loop when the refresh does a blocking HTTP call+ # (e.g. google-auth's Credentials.refresh()).+ result = await asyncio.to_thread(refresh_fn)++ # Treat dict-shaped {"success": False} as a breaker-relevant+ # failure too, not just raised exceptions. Most OAuth helpers+ # in this codebase return {"success": False, "error": "..."}+ # instead of raising.+ if isinstance(result, dict) and result.get("success") is False:+ raise RuntimeError(+ result.get("error") or "oauth refresh reported failure"+ )+ return result++ try:+ return await breaker.run(_call)+ except CircuitBreakerOpen as e:+ logger.warning(+ "auth: OAuth refresh breaker open for %s (retry after %.0fs)",+ provider,+ e.retry_after_seconds,+ )+ return {+ "success": False,+ "error": str(e),+ "circuit_open": True,+ "retry_after_seconds": e.retry_after_seconds,+ "needs_reauth": False,+ }+ except Exception as e: # noqa: BLE001 β surface non-breaker errors+ logger.error("auth: OAuth refresh failed for %s: %s", provider, e)+ return {+ "success": False,+ "error": str(e),+ "circuit_open": False,+ "needs_reauth": True,+ }+async def remove_oauth_tokens(self,provider: str,diff --git a/server/services/circuit_breaker.py b/server/services/circuit_breaker.pynew file mode 100644index 00000000..01fb0a6a--- /dev/null+++ b/server/services/circuit_breaker.py@@ -0,0 +1,258 @@+"""Simple in-process circuit breaker.++Phase 7.5c of the credentials-scaling plan. Generalizes to any upstream+call where repeated failures should stop cascading into the rest of the+system (OAuth refresh, third-party API calls, LLM provider endpoints).++Semantics+---------+A breaker has three states:++- **closed** β normal operation; calls pass through to the wrapped+ coroutine. On each failure the breaker increments a per-key counter.+- **open** β all calls short-circuit immediately with+ :class:`CircuitBreakerOpen`. Opens when the failure counter crosses+ ``failure_threshold`` within a ``failure_window`` rolling window.+- **half_open**β after ``cooldown_seconds``, the next call is allowed+ through as a probe. Success closes the breaker; failure reopens it+ and resets the cooldown.++Design decisions+----------------+- **Per-key**: each breaker instance is keyed on a free-form scope+ (e.g. ``"twitter"``, ``"google"``). One provider failing does not trip+ the breaker for another provider.+- **Process-local**: single-instance self-hosted tool. No Redis, no+ distributed consensus. At deployment scale, upgrade to Redis-backed+ counters.+- **Opt-in**: services call ``breaker.run(func)`` explicitly. Legacy+ code paths that don't opt in get the old behavior unchanged.+- **Exception caching**: when open, the breaker raises+ :class:`CircuitBreakerOpen` with the last recorded failure as+ ``__cause__``, so callers can introspect why it opened.++Usage+-----+ from services.circuit_breaker import get_circuit_breaker, CircuitBreakerOpen++ async def refresh_google_tokens() -> TokenPair:+ breaker = get_circuit_breaker("google_oauth_refresh")+ try:+ return await breaker.run(_actually_refresh_with_google)+ except CircuitBreakerOpen as e:+ logger.warning("google OAuth refresh circuit open: %s", e)+ raise++Tunables (defaults)+-------------------+- ``failure_threshold``: 3+- ``failure_window``: 60.0 seconds+- ``cooldown_seconds``: 30.0 seconds+"""++from __future__ import annotations++import asyncio+import time+from dataclasses import dataclass, field+from enum import Enum+from typing import Awaitable, Callable, Dict, List, Optional, TypeVar++from core.logging import get_logger++logger = get_logger(__name__)++T = TypeVar("T")+++class CircuitState(str, Enum):+ CLOSED = "closed"+ OPEN = "open"+ HALF_OPEN = "half_open"+++class CircuitBreakerOpen(RuntimeError):+ """Raised when a call short-circuits because the breaker is open."""++ def __init__(self, scope: str, retry_after_seconds: float) -> None:+ super().__init__(+ f"circuit breaker {scope!r} is open; retry after ~{retry_after_seconds:.0f}s"+ )+ self.scope = scope+ self.retry_after_seconds = retry_after_seconds+++@dataclass+class _CircuitState:+ state: CircuitState = CircuitState.CLOSED+ failures: List[float] = field(default_factory=list)+ opened_at: Optional[float] = None+ last_exception: Optional[BaseException] = None+++class CircuitBreaker:+ """One breaker instance scoped to a named upstream."""++ def __init__(+ self,+ scope: str,+ *,+ failure_threshold: int = 3,+ failure_window: float = 60.0,+ cooldown_seconds: float = 30.0,+ ) -> None:+ self._scope = scope+ self._failure_threshold = failure_threshold+ self._failure_window = failure_window+ self._cooldown = cooldown_seconds+ self._state = _CircuitState()+ self._lock = asyncio.Lock()++ # ----- public API -----++ async def run(self, func: Callable[[], Awaitable[T]]) -> T:+ """Execute ``func`` under breaker supervision.++ - Closed or half-open: runs normally. Success resets the counter+ and closes the breaker. Failure increments the counter and may+ open the breaker.+ - Open: short-circuits immediately with+ :class:`CircuitBreakerOpen`, unless the cooldown has elapsed+ (in which case the breaker moves to half-open and runs the+ call as a probe).+ """+ async with self._lock:+ self._maybe_half_open(time.monotonic())+ if self._state.state is CircuitState.OPEN:+ retry_after = self._retry_after()+ exc = CircuitBreakerOpen(self._scope, retry_after)+ if self._state.last_exception is not None:+ exc.__cause__ = self._state.last_exception+ raise exc++ try:+ result = await func()+ except BaseException as exc: # noqa: BLE001 β re-raised below+ async with self._lock:+ self._record_failure(time.monotonic(), exc)+ raise++ async with self._lock:+ self._record_success()+ return result++ def state(self) -> CircuitState:+ return self._state.state++ def stats(self) -> Dict[str, object]:+ now = time.monotonic()+ cutoff = now - self._failure_window+ recent_failures = sum(1 for t in self._state.failures if t >= cutoff)+ return {+ "scope": self._scope,+ "state": self._state.state.value,+ "failure_threshold": self._failure_threshold,+ "failure_window_seconds": self._failure_window,+ "cooldown_seconds": self._cooldown,+ "recent_failures": recent_failures,+ "opened_at": self._state.opened_at,+ "retry_after_seconds": self._retry_after() if self._state.state is CircuitState.OPEN else 0,+ }++ def reset(self) -> None:+ """Force-close the breaker. Tests or manual admin endpoint only."""+ self._state = _CircuitState()++ # ----- internals -----++ def _maybe_half_open(self, now: float) -> None:+ if self._state.state is not CircuitState.OPEN:+ return+ if self._state.opened_at is None:+ return+ if now - self._state.opened_at < self._cooldown:+ return+ # Cooldown elapsed β allow one probe call.+ logger.info(+ "circuit_breaker[%s]: cooldown elapsed, moving to half_open", self._scope+ )+ self._state.state = CircuitState.HALF_OPEN++ def _record_failure(self, now: float, exc: BaseException) -> None:+ self._state.last_exception = exc+ # Prune failures outside the rolling window.+ cutoff = now - self._failure_window+ self._state.failures = [t for t in self._state.failures if t >= cutoff]+ self._state.failures.append(now)++ if self._state.state is CircuitState.HALF_OPEN:+ # Probe failed β reopen immediately and restart cooldown.+ logger.warning(+ "circuit_breaker[%s]: half_open probe failed, reopening: %s",+ self._scope,+ type(exc).__name__,+ )+ self._state.state = CircuitState.OPEN+ self._state.opened_at = now+ return++ if len(self._state.failures) >= self._failure_threshold:+ logger.warning(+ "circuit_breaker[%s]: threshold reached (%d/%d failures in %ds), opening",+ self._scope,+ len(self._state.failures),+ self._failure_threshold,+ int(self._failure_window),+ )+ self._state.state = CircuitState.OPEN+ self._state.opened_at = now++ def _record_success(self) -> None:+ if self._state.state in (CircuitState.HALF_OPEN, CircuitState.OPEN):+ logger.info("circuit_breaker[%s]: closed after successful call", self._scope)+ self._state.state = CircuitState.CLOSED+ self._state.failures.clear()+ self._state.opened_at = None+ self._state.last_exception = None++ def _retry_after(self) -> float:+ if self._state.opened_at is None:+ return 0.0+ elapsed = time.monotonic() - self._state.opened_at+ return max(0.0, self._cooldown - elapsed)+++# ----- module-level registry -----++_BREAKERS: Dict[str, CircuitBreaker] = {}+++def get_circuit_breaker(+ scope: str,+ *,+ failure_threshold: int = 3,+ failure_window: float = 60.0,+ cooldown_seconds: float = 30.0,+) -> CircuitBreaker:+ """Return (creating on first call) the shared breaker for ``scope``.++ Tunable parameters only apply on first creation; subsequent calls+ with the same scope return the existing instance and ignore new+ params. This keeps scope-to-breaker identity stable across a+ process lifetime.+ """+ breaker = _BREAKERS.get(scope)+ if breaker is None:+ breaker = CircuitBreaker(+ scope,+ failure_threshold=failure_threshold,+ failure_window=failure_window,+ cooldown_seconds=cooldown_seconds,+ )+ _BREAKERS[scope] = breaker+ return breaker+++def all_breaker_stats() -> List[Dict[str, object]]:+ """Return stats for every known breaker (observability endpoint)."""+ return [b.stats() for b in _BREAKERS.values()]diff --git a/server/services/credential_registry.py b/server/services/credential_registry.pynew file mode 100644index 00000000..2349d609--- /dev/null+++ b/server/services/credential_registry.py@@ -0,0 +1,271 @@+"""Credential registry - loads server/config/credential_providers.json.++Lazy singleton service backed by a single JSON file. Supports `extends`+inheritance with array-merge-by-key semantics (port of n8n's+NodeHelpers.mergeNodeProperties pattern).++Served to clients via the `get_credential_catalogue` WebSocket handler,+which returns `{providers, categories, version}`. The version is a+content-sha256 of the resolved catalogue so the client can warm-start from+IndexedDB and revalidate only when the server content changes.++Follows the same lazy-singleton pattern as services/email_service.py.+"""++from __future__ import annotations++import copy+import hashlib+import json+from pathlib import Path+from typing import Any, Dict, List, Optional, Set++from core.logging import get_logger++logger = get_logger(__name__)++_CONFIG_PATH = Path(__file__).parent.parent / "config" / "credential_providers.json"++# Arrays that are merged by `key` instead of replaced wholesale when a child+# extends a parent. Matches the shape defined in credential_providers.json.+_MERGE_BY_KEY_ARRAYS: Set[str] = {"fields", "status_rows", "actions"}+++class CredentialRegistryError(RuntimeError):+ """Raised for malformed registry files or resolution cycles."""+++class CredentialRegistry:+ """Lazy singleton wrapper around credential_providers.json."""++ _instance: Optional["CredentialRegistry"] = None++ def __init__(self) -> None:+ self._raw: Optional[Dict[str, Any]] = None+ self._resolved_providers: Optional[Dict[str, Dict[str, Any]]] = None+ self._version: Optional[str] = None++ @classmethod+ def get_instance(cls) -> "CredentialRegistry":+ if cls._instance is None:+ cls._instance = cls()+ return cls._instance++ # ----- loading -----++ def _load_raw(self) -> Dict[str, Any]:+ """Parse the JSON file once, cache in memory for app lifetime."""+ if self._raw is None:+ try:+ with _CONFIG_PATH.open(encoding="utf-8") as f:+ self._raw = json.load(f)+ except FileNotFoundError as e:+ raise CredentialRegistryError(+ f"credential_providers.json not found at {_CONFIG_PATH}"+ ) from e+ except json.JSONDecodeError as e:+ raise CredentialRegistryError(+ f"credential_providers.json is not valid JSON: {e}"+ ) from e+ return self._raw++ def _resolve_all(self) -> Dict[str, Dict[str, Any]]:+ """Resolve every provider's `extends` chain into flat dicts."""+ if self._resolved_providers is not None:+ return self._resolved_providers++ raw = self._load_raw().get("providers", {})+ if not isinstance(raw, dict):+ raise CredentialRegistryError("providers must be a dict")++ resolved: Dict[str, Dict[str, Any]] = {}+ for provider_id in raw:+ resolved[provider_id] = self._resolve_one(provider_id, raw, visiting=set())++ self._resolved_providers = resolved+ return resolved++ def _resolve_one(+ self,+ provider_id: str,+ raw: Dict[str, Dict[str, Any]],+ visiting: Set[str],+ ) -> Dict[str, Any]:+ """Walk the extends chain, deep-merge parent then child overrides."""+ if provider_id in visiting:+ raise CredentialRegistryError(+ f"extends cycle detected involving provider {provider_id!r}: "+ f"{' -> '.join(sorted(visiting))} -> {provider_id}"+ )+ if provider_id not in raw:+ raise CredentialRegistryError(+ f"provider {provider_id!r} referenced via extends but not defined"+ )++ entry = raw[provider_id]+ extends = entry.get("extends")+ if not extends:+ merged = copy.deepcopy(entry)+ merged["id"] = provider_id+ return merged++ # Resolve parent first (recursive), then overlay child.+ parent_visiting = visiting | {provider_id}+ parent = self._resolve_one(extends, raw, parent_visiting)++ merged = _deep_merge(parent, entry)+ merged["id"] = provider_id+ # `extends` itself is an implementation detail; strip from output.+ merged.pop("extends", None)+ # `_abstract` must not be inherited β only concrete entries that set+ # it explicitly should be treated as abstract. Otherwise every child+ # of `_ai_base` would silently become abstract and vanish from+ # get_all_providers().+ if "_abstract" not in entry:+ merged.pop("_abstract", None)+ return merged++ # ----- public API -----++ def get_all_providers(self) -> List[Dict[str, Any]]:+ """Return all concrete (non-abstract) providers in JSON order."""+ resolved = self._resolve_all()+ # Preserve insertion order of the raw providers dict.+ raw = self._load_raw().get("providers", {})+ out: List[Dict[str, Any]] = []+ for provider_id in raw:+ entry = resolved[provider_id]+ if entry.get("_abstract"):+ continue+ out.append(entry)+ return out++ def get_provider(self, provider_id: str) -> Optional[Dict[str, Any]]:+ """Return a single resolved provider by id, or None if abstract/missing."""+ resolved = self._resolve_all()+ entry = resolved.get(provider_id)+ if entry is None or entry.get("_abstract"):+ return None+ return entry++ def get_categories(self) -> List[Dict[str, Any]]:+ """Return ordered category metadata derived from the raw file."""+ raw = self._load_raw()+ categories = raw.get("categories", {})+ if not isinstance(categories, dict):+ return []+ out = [+ {"key": key, "label": cfg.get("label", key), "order": cfg.get("order", 0)}+ for key, cfg in categories.items()+ ]+ out.sort(key=lambda c: (c["order"], c["key"]))+ return out++ def get_version(self) -> str:+ """Content-sha256 of the resolved catalogue (providers + categories).++ Used by clients for warm-start cache invalidation: when the hash+ changes, the client fetches a fresh catalogue; otherwise it serves+ from IndexedDB with zero network traffic.+ """+ if self._version is None:+ payload = {+ "providers": self.get_all_providers(),+ "categories": self.get_categories(),+ }+ encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")+ self._version = hashlib.sha256(encoded).hexdigest()+ return self._version++ def get_catalogue(self) -> Dict[str, Any]:+ """Full payload returned by the `get_credential_catalogue` WebSocket handler."""+ return {+ "providers": self.get_all_providers(),+ "categories": self.get_categories(),+ "version": self.get_version(),+ }++ def reload(self) -> None:+ """Drop all caches; next call re-parses the JSON file."""+ self._raw = None+ self._resolved_providers = None+ self._version = None+++# ----- deep-merge helper (n8n mergeNodeProperties pattern) -----+++def _deep_merge(parent: Dict[str, Any], child: Dict[str, Any]) -> Dict[str, Any]:+ """Return a new dict: parent deep-merged with child overrides.++ Rules:+ - Nested dicts merge recursively.+ - Arrays named in _MERGE_BY_KEY_ARRAYS merge by the `key` field: child+ entries replace parent entries with the same key; unmatched child+ entries append to the parent list.+ - All other arrays and scalars are replaced wholesale by the child.+ - The child's `extends` key is ignored (resolved at a higher level).+ """+ result: Dict[str, Any] = copy.deepcopy(parent)+ for key, child_value in child.items():+ if key == "extends":+ continue+ parent_value = result.get(key)+ if isinstance(parent_value, dict) and isinstance(child_value, dict):+ result[key] = _deep_merge(parent_value, child_value)+ elif (+ key in _MERGE_BY_KEY_ARRAYS+ and isinstance(parent_value, list)+ and isinstance(child_value, list)+ ):+ result[key] = _merge_array_by_key(parent_value, child_value)+ else:+ result[key] = copy.deepcopy(child_value)+ return result+++def _merge_array_by_key(parent: List[Any], child: List[Any]) -> List[Any]:+ """Merge two lists of dicts by their `key` field.++ Entries in the child that share a `key` with a parent entry recursively+ deep-merge (child wins on scalar conflicts). Unmatched child entries+ append to the end. Entries without a `key` field on either side are+ treated as identity replacements (parent dropped, child appended).+ """+ by_key: Dict[str, Dict[str, Any]] = {}+ order: List[str] = []++ def _item_key(item: Any, prefix: str, index: int) -> str:+ if isinstance(item, dict) and "key" in item:+ return str(item["key"])+ return f"{prefix}:{index}"++ for i, item in enumerate(parent):+ k = _item_key(item, "parent", i)+ by_key[k] = copy.deepcopy(item) if isinstance(item, dict) else item+ order.append(k)++ for i, item in enumerate(child):+ if isinstance(item, dict) and "key" in item:+ k = str(item["key"])+ existing = by_key.get(k)+ if isinstance(existing, dict):+ by_key[k] = _deep_merge(existing, item)+ else:+ by_key[k] = copy.deepcopy(item)+ if k not in order:+ order.append(k)+ else:+ k = f"child:{i}"+ by_key[k] = copy.deepcopy(item) if isinstance(item, dict) else item+ order.append(k)++ return [by_key[k] for k in order]+++# ----- module-level singleton accessor -----+++def get_credential_registry() -> CredentialRegistry:+ """Return the process-wide CredentialRegistry singleton."""+ return CredentialRegistry.get_instance()diff --git a/server/services/idempotency.py b/server/services/idempotency.pynew file mode 100644index 00000000..4d22c2fe--- /dev/null+++ b/server/services/idempotency.py@@ -0,0 +1,198 @@+"""Idempotency key service.++Phase 7.5b of the credentials-scaling plan. Generalizes to any mutation+handler in the platform refactor (control plane RFC, Phase C).++Client mutations include an opaque `request_id` (UUID generated in the+browser via `crypto.randomUUID()`). The server caches the result per+request_id with a short TTL. If the same request_id arrives twice (retry,+WebSocket reconnect, user double-click, etc.), the server returns the+cached result instead of re-running the mutation. Prevents double-writes+and makes every mutation handler safe to retry.++Design decisions+----------------+- Process-local dict, not Redis β at our scale (single-user / small-team+ self-hosted) process-local is enough. Survives handler-internal retries+ but not server restarts; that's the correct tradeoff for a 60 s window.+- Positive and negative caching: both successful results and raised+ exceptions are cached, so a retried mutation that would have failed+ the first time still fails on retry (rather than silently succeeding).+- TTL is in wall-clock seconds; entries are pruned lazily on access,+ avoiding a background sweeper task.+- Thread/async safe via a per-scope asyncio.Lock; lock is only held for+ the cache-miss window, not for the duration of the wrapped function.++Usage+-----+ from services.idempotency import get_idempotency_store++ store = get_idempotency_store("credentials")++ async def handle_save_credential(data, websocket):+ request_id = data.get("request_id")+ return await store.run(+ request_id,+ lambda: _do_save(data),+ )++Routers that omit `request_id` fall through to a direct call with no+caching β callers are not forced to opt in.+"""++from __future__ import annotations++import asyncio+import time+from dataclasses import dataclass+from typing import Any, Awaitable, Callable, Dict, Optional++from core.logging import get_logger++logger = get_logger(__name__)+++@dataclass+class _CacheEntry:+ result: Any+ exception: Optional[BaseException]+ expires_at: float+++class IdempotencyStore:+ """Per-scope TTL-bounded idempotency cache.++ Each router or module typically owns one store, keyed by the+ handler's logical scope (e.g. ``"credentials"``, ``"workflow"``),+ so request_id collisions across scopes are impossible.+ """++ def __init__(self, scope: str, ttl_seconds: float = 60.0) -> None:+ self._scope = scope+ self._ttl = ttl_seconds+ self._entries: Dict[str, _CacheEntry] = {}+ # Per-request_id locks prevent two concurrent calls with the same+ # request_id from both running the underlying function. First+ # caller executes; second caller awaits the same lock and then+ # hits the cache.+ self._locks: Dict[str, asyncio.Lock] = {}+ self._global_lock = asyncio.Lock()++ # ----- public API -----++ async def run(+ self,+ request_id: Optional[str],+ func: Callable[[], Awaitable[Any]],+ ) -> Any:+ """Run ``func`` and cache the result by ``request_id``.++ If ``request_id`` is falsy, ``func`` runs directly with no+ caching. This keeps the opt-in contract one-sided: clients that+ supply a request_id get dedupe; clients that don't get normal+ behavior.+ """+ if not request_id:+ return await func()++ # Fast path: hit the cache without acquiring a per-key lock.+ cached = self._lookup(request_id)+ if cached is not None:+ return self._materialize(cached)++ # Slow path: ensure only one coroutine per request_id runs at a+ # time. Concurrent duplicate requests queue on the same lock.+ lock = await self._get_lock(request_id)+ async with lock:+ cached = self._lookup(request_id)+ if cached is not None:+ return self._materialize(cached)++ try:+ result = await func()+ except BaseException as exc: # noqa: BLE001 β we re-raise below+ self._store_failure(request_id, exc)+ raise+ else:+ self._store_success(request_id, result)+ return result++ def stats(self) -> Dict[str, Any]:+ """Return observability stats for logging/debug endpoints."""+ now = time.monotonic()+ live = sum(1 for e in self._entries.values() if e.expires_at > now)+ return {+ "scope": self._scope,+ "ttl_seconds": self._ttl,+ "entries_total": len(self._entries),+ "entries_live": live,+ }++ def clear(self) -> None:+ """Drop all entries. Tests or manual admin use only."""+ self._entries.clear()+ self._locks.clear()++ # ----- internals -----++ def _lookup(self, request_id: str) -> Optional[_CacheEntry]:+ entry = self._entries.get(request_id)+ if entry is None:+ return None+ if entry.expires_at <= time.monotonic():+ # Lazy eviction β drop the stale entry on access.+ self._entries.pop(request_id, None)+ return None+ return entry++ def _materialize(self, entry: _CacheEntry) -> Any:+ if entry.exception is not None:+ raise entry.exception+ return entry.result++ def _store_success(self, request_id: str, result: Any) -> None:+ self._entries[request_id] = _CacheEntry(+ result=result,+ exception=None,+ expires_at=time.monotonic() + self._ttl,+ )+ logger.debug("idempotency[%s]: cached success for %s", self._scope, request_id)++ def _store_failure(self, request_id: str, exc: BaseException) -> None:+ self._entries[request_id] = _CacheEntry(+ result=None,+ exception=exc,+ expires_at=time.monotonic() + self._ttl,+ )+ logger.debug(+ "idempotency[%s]: cached failure for %s: %s",+ self._scope,+ request_id,+ type(exc).__name__,+ )++ async def _get_lock(self, request_id: str) -> asyncio.Lock:+ async with self._global_lock:+ lock = self._locks.get(request_id)+ if lock is None:+ lock = asyncio.Lock()+ self._locks[request_id] = lock+ return lock+++# ----- module-level registry -----++_STORES: Dict[str, IdempotencyStore] = {}+++def get_idempotency_store(scope: str, ttl_seconds: float = 60.0) -> IdempotencyStore:+ """Return (creating on first call) the shared store for ``scope``.++ Scopes are free-form namespaces. Pick one per router or per logical+ domain β e.g. ``"credentials"``, ``"workflow"``, ``"nodespec"``.+ """+ store = _STORES.get(scope)+ if store is None:+ store = IdempotencyStore(scope, ttl_seconds=ttl_seconds)+ _STORES[scope] = store+ return store