Skip to content

Self-improving AI Employees for everyone

feat(credentials): server-owned provider registry + idempotency + OAuth circuit breaker

trohitg697c67623bf7
diff --git a/server/config/credential_providers.json b/server/config/credential_providers.json
new file mode 100644
index 00000000..fa322667
--- /dev/null
+++ b/server/config/credential_providers.json
@@ -0,0 +1,392 @@
+{
+ "version": "2026.04",
+ "last_updated": "2026-04-11",
+ "_description": "Credential provider catalogue. Loaded by server/services/credential_registry.py at startup; served to clients via the get_credential_catalogue WebSocket handler. Supports `extends` inheritance with array-merge-by-key semantics (see credential_registry.resolve_extends). `_abstract: true` entries are resolved but excluded from get_all_providers(). Color tokens are resolved client-side via the theme helper.",
+
+ "categories": {
+ "ai": { "label": "AI Providers", "order": 1 },
+ "social": { "label": "Social Media", "order": 2 },
+ "productivity": { "label": "Productivity", "order": 3 },
+ "email": { "label": "Email", "order": 4 },
+ "android": { "label": "Android", "order": 5 },
+ "search": { "label": "Search", "order": 6 },
+ "scrapers": { "label": "Scrapers", "order": 7 },
+ "services": { "label": "Services", "order": 8 }
+ },
+
+ "providers": {
+ "_ai_base": {
+ "_abstract": true,
+ "category": "ai",
+ "category_label": "AI Providers",
+ "kind": "apiKey",
+ "has_defaults": true,
+ "icon_ref": "ai:${id}",
+ "fields": [
+ {
+ "key": "apiKey",
+ "label": "API Key",
+ "type": "password",
+ "secret": true,
+ "required": true
+ }
+ ]
+ },
+
+ "openai": {
+ "extends": "_ai_base",
+ "name": "OpenAI",
+ "color": "dracula.green",
+ "fields": [
+ { "key": "apiKey", "placeholder": "sk-..." }
+ ]
+ },
+ "anthropic": {
+ "extends": "_ai_base",
+ "name": "Anthropic",
+ "color": "dracula.orange",
+ "fields": [
+ { "key": "apiKey", "placeholder": "sk-ant-..." }
+ ]
+ },
+ "gemini": {
+ "extends": "_ai_base",
+ "name": "Gemini",
+ "color": "solarized.blue",
+ "fields": [
+ { "key": "apiKey", "placeholder": "AIza..." }
+ ]
+ },
+ "groq": {
+ "extends": "_ai_base",
+ "name": "Groq",
+ "color": "dracula.red",
+ "fields": [
+ { "key": "apiKey", "placeholder": "gsk_..." }
+ ]
+ },
+ "cerebras": {
+ "extends": "_ai_base",
+ "name": "Cerebras",
+ "color": "dracula.orange",
+ "fields": [
+ { "key": "apiKey", "placeholder": "csk-..." }
+ ]
+ },
+ "openrouter": {
+ "extends": "_ai_base",
+ "name": "OpenRouter",
+ "color": "solarized.violet",
+ "fields": [
+ { "key": "apiKey", "placeholder": "sk-or-..." }
+ ]
+ },
+ "deepseek": {
+ "extends": "_ai_base",
+ "name": "DeepSeek",
+ "color": "dracula.cyan",
+ "fields": [
+ { "key": "apiKey", "placeholder": "sk-..." }
+ ]
+ },
+ "kimi": {
+ "extends": "_ai_base",
+ "name": "Kimi",
+ "color": "dracula.purple",
+ "fields": [
+ { "key": "apiKey", "placeholder": "sk-..." }
+ ]
+ },
+ "mistral": {
+ "extends": "_ai_base",
+ "name": "Mistral",
+ "color": "dracula.pink",
+ "fields": [
+ { "key": "apiKey", "placeholder": "sk-..." }
+ ]
+ },
+
+ "whatsapp_personal": {
+ "name": "WhatsApp Personal",
+ "category": "social",
+ "category_label": "Social Media",
+ "color": "theme.categoryWhatsapp",
+ "kind": "qrPairing",
+ "icon_ref": "data_uri:WHATSAPP_CONNECT_ICON",
+ "status_hook": "whatsapp",
+ "has_rate_limits": true,
+ "status_rows": [
+ { "key": "status", "label": "Status", "ok_field": "connected", "true_text": "Connected", "false_text": "Disconnected" },
+ { "key": "session", "label": "Session", "ok_field": "has_session", "true_text": "Active", "false_text": "Inactive" },
+ { "key": "service", "label": "Service", "ok_field": "running", "true_text": "Running", "false_text": "Stopped" }
+ ],
+ "actions": [
+ { "key": "start", "label": "Start", "theme_color": "green" },
+ { "key": "restart", "label": "Restart", "theme_color": "orange" },
+ { "key": "refresh", "label": "Refresh", "theme_color": "cyan" }
+ ],
+ "qr": {
+ "qr_field": "qr",
+ "is_connected_field": "connected",
+ "connected_title": "Already Connected!",
+ "connected_subtitle_static": "No QR code needed",
+ "is_loading_fields": ["running", "!qr", "!connected"],
+ "empty_text_running": "Waiting for QR code...",
+ "empty_text_stopped": "Start service to get QR code",
+ "scan_text": "Scan with WhatsApp mobile app"
+ }
+ },
+
+ "twitter": {
+ "name": "Twitter/X",
+ "category": "social",
+ "category_label": "Social Media",
+ "color": "theme.categoryTrigger",
+ "kind": "oauth",
+ "icon_ref": "data_uri:TWITTER_ICON",
+ "status_hook": "twitter",
+ "usage_service": "twitter",
+ "ws": {
+ "login": "twitter_oauth_login",
+ "logout": "twitter_logout",
+ "status": "twitter_oauth_status"
+ },
+ "fields": [
+ {
+ "key": "twitter_client_id",
+ "label": "Client ID",
+ "type": "string",
+ "placeholder": "Client ID (from X Developer Portal)",
+ "required": true
+ },
+ {
+ "key": "twitter_client_secret",
+ "label": "Client Secret",
+ "type": "password",
+ "secret": true,
+ "placeholder": "Client Secret (optional for PKCE)"
+ }
+ ],
+ "callback_url": "http://localhost:3010/api/twitter/callback",
+ "instructions": "Get credentials from the X Developer Portal. Create an app with OAuth 2.0 enabled."
+ },
+
+ "telegram": {
+ "name": "Telegram Bot",
+ "category": "social",
+ "category_label": "Social Media",
+ "color": "theme.categoryChat",
+ "kind": "oauth",
+ "icon_ref": "raw_svg:TelegramIcons.telegram",
+ "status_hook": "telegram",
+ "ws": {
+ "login": "telegram_connect",
+ "logout": "telegram_disconnect",
+ "status": "telegram_status"
+ },
+ "fields": [
+ {
+ "key": "telegram_bot_token",
+ "label": "Bot Token",
+ "type": "password",
+ "secret": true,
+ "placeholder": "123456789:ABCdefGHIjklMNOpqrSTUvwxYZ",
+ "required": true
+ }
+ ],
+ "instructions": "Get a bot token from @BotFather on Telegram. After connecting, send any message to your bot to register as the owner."
+ },
+
+ "gmail": {
+ "name": "Google Workspace",
+ "category": "productivity",
+ "category_label": "Productivity",
+ "color": "theme.categoryAI",
+ "kind": "oauth",
+ "icon_ref": "raw_svg:GoogleIcons.gmail",
+ "status_hook": "google",
+ "usage_service": "google_workspace",
+ "ws": {
+ "login": "google_oauth_login",
+ "logout": "google_logout",
+ "status": "google_oauth_status"
+ },
+ "fields": [
+ {
+ "key": "google_client_id",
+ "label": "Client ID",
+ "type": "string",
+ "placeholder": "Client ID (from Google Cloud Console)",
+ "required": true
+ },
+ {
+ "key": "google_client_secret",
+ "label": "Client Secret",
+ "type": "password",
+ "secret": true,
+ "placeholder": "Client Secret (required)",
+ "required": true
+ }
+ ],
+ "callback_url": "http://localhost:3010/api/google/callback",
+ "instructions": "Get credentials from Google Cloud Console. Enable Gmail, Calendar, Drive, Sheets, Tasks, and People APIs."
+ },
+
+ "email_himalaya": {
+ "name": "Email (IMAP/SMTP)",
+ "category": "email",
+ "category_label": "Email",
+ "color": "theme.categoryChat",
+ "kind": "email",
+ "icon_ref": "raw_svg:EmailIcons.read"
+ },
+
+ "android_remote": {
+ "name": "Android Device",
+ "category": "android",
+ "category_label": "Android",
+ "color": "theme.categoryAndroid",
+ "kind": "qrPairing",
+ "icon_ref": "emoji:πŸ“±",
+ "status_hook": "android",
+ "fields": [
+ {
+ "key": "android_remote",
+ "label": "API Key",
+ "type": "password",
+ "secret": true,
+ "placeholder": "Enter Android Relay API key...",
+ "required": true
+ }
+ ],
+ "status_rows": [
+ { "key": "device", "label": "Device", "ok_field": "paired", "true_text": "Paired", "false_text": "Not Paired" },
+ { "key": "relay", "label": "Relay", "ok_field": "connected", "true_text": "Connected", "false_text": "Disconnected" },
+ { "key": "pairing", "label": "Pairing", "ok_field": "paired", "true_text": "Paired", "false_text": "Waiting", "warn": true }
+ ],
+ "actions": [
+ {
+ "key": "connect",
+ "label": "Connect",
+ "theme_color": "green",
+ "disabled_when": ["paired", "!stored"]
+ },
+ {
+ "key": "disconnect",
+ "label": "Disconnect",
+ "theme_color": "pink",
+ "disabled_when": ["!connected"]
+ }
+ ],
+ "qr": {
+ "qr_field": "qr_data",
+ "is_connected_field": "paired",
+ "connected_title": "Device Paired!",
+ "connected_subtitle_field": "device_name",
+ "connected_subtitle_fallback": "Android Device",
+ "is_loading_fields": ["connected", "!qr_data", "!paired"],
+ "empty_text_no_key": "Add API key first",
+ "empty_text_running": "Waiting for QR code...",
+ "empty_text_stopped": "Click Connect to start",
+ "scan_text": "Scan with Android companion app"
+ }
+ },
+
+ "brave_search": {
+ "name": "Brave Search",
+ "category": "search",
+ "category_label": "Search",
+ "color": "theme.categoryLocation",
+ "kind": "apiKey",
+ "icon_ref": "raw_svg:SearchIcons.brave",
+ "fields": [
+ {
+ "key": "apiKey",
+ "label": "API Key",
+ "type": "password",
+ "secret": true,
+ "placeholder": "BSA...",
+ "required": true
+ }
+ ]
+ },
+
+ "perplexity": {
+ "name": "Perplexity",
+ "category": "search",
+ "category_label": "Search",
+ "color": "theme.categoryUtil",
+ "kind": "apiKey",
+ "icon_ref": "raw_svg:SearchIcons.perplexity",
+ "fields": [
+ {
+ "key": "apiKey",
+ "label": "API Key",
+ "type": "password",
+ "secret": true,
+ "placeholder": "pplx-...",
+ "required": true
+ }
+ ]
+ },
+
+ "apify": {
+ "name": "Apify",
+ "category": "scrapers",
+ "category_label": "Scrapers",
+ "color": "theme.categoryAI",
+ "kind": "apiKey",
+ "icon_ref": "raw_svg:ApifyIcons.apify",
+ "validate_as": "apify",
+ "fields": [
+ {
+ "key": "apiKey",
+ "label": "API Key",
+ "type": "password",
+ "secret": true,
+ "placeholder": "apify_api_...",
+ "required": true
+ }
+ ]
+ },
+
+ "serper": {
+ "name": "Serper",
+ "category": "scrapers",
+ "category_label": "Scrapers",
+ "color": "theme.categoryAI",
+ "kind": "apiKey",
+ "icon_ref": "raw_svg:SearchIcons.serper",
+ "fields": [
+ {
+ "key": "apiKey",
+ "label": "API Key",
+ "type": "password",
+ "secret": true,
+ "required": true
+ }
+ ]
+ },
+
+ "google_maps": {
+ "name": "Google Maps",
+ "category": "services",
+ "category_label": "Services",
+ "color": "theme.categoryLocation",
+ "kind": "apiKey",
+ "icon_ref": "emoji:πŸ—ΊοΈ",
+ "validate_as": "google_maps",
+ "usage_service": "google_maps",
+ "fields": [
+ {
+ "key": "apiKey",
+ "label": "API Key",
+ "type": "password",
+ "secret": true,
+ "placeholder": "AIza...",
+ "required": true
+ }
+ ]
+ }
+ }
+}
diff --git a/server/main.py b/server/main.py
index 696dee53..5e50256d 100644
--- a/server/main.py
+++ b/server/main.py
@@ -39,7 +39,7 @@ _startup_log("Importing settings + logging...")
from core.config import Settings
from core.logging import configure_logging, get_logger, setup_websocket_logging, shutdown_websocket_logging
_startup_log("Importing routers...")
-from routers import workflow, database, maps, nodejs_compat, android, websocket, webhook, auth, twitter, google
+from routers import workflow, database, maps, nodejs_compat, android, websocket, webhook, auth, twitter, google, credentials
_startup_log("All imports complete")
# Initialize settings and logging
@@ -397,6 +397,7 @@ app.include_router(websocket.router)
app.include_router(webhook.router)
app.include_router(twitter.router) # Twitter/X OAuth routes
app.include_router(google.router) # Google Workspace OAuth routes (Gmail, Calendar, Drive, Sheets, Tasks, Contacts)
+app.include_router(credentials.router) # Credentials panel - lazy per-tile icon endpoint (n8n pattern)
@app.get("/health")
diff --git a/server/routers/credentials.py b/server/routers/credentials.py
new file mode 100644
index 00000000..00eab247
--- /dev/null
+++ b/server/routers/credentials.py
@@ -0,0 +1,88 @@
+"""Credentials routes.
+
+Phase A of the credentials-scaling plan: lazy per-tile icon endpoint.
+
+Serves SVG icons for credential providers out of
+``server/static/credential_icons/<provider_id>.svg`` with long-lived
+``Cache-Control: immutable`` headers. If no file is present, returns 404
+so the client falls back to its existing in-bundle icon resolution (see
+``client/src/components/credentials/providers.tsx`` and
+``client/src/components/icons/AIProviderIcons.tsx``).
+
+This follows n8n's lazy-icon pattern: the frontend does not need to bundle
+icons as data URIs, and the backend can later serve per-customer or
+user-uploaded provider icons without a frontend redeploy.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import FileResponse, Response
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+router = APIRouter(prefix="/api/credentials", tags=["credentials"])
+
+# Resolve the icons directory relative to the server package root so it
+# works under both local dev and Docker (where /app is the cwd).
+_STATIC_ICONS_DIR = Path(__file__).parent.parent / "static" / "credential_icons"
+
+# Only allow a conservative subset of characters in the provider_id to
+# prevent path traversal. The credential registry uses lowercase
+# alphanumerics and underscores throughout.
+_ALLOWED_CHARS = set("abcdefghijklmnopqrstuvwxyz0123456789_-")
+
+# 1 year, immutable: SVG content never changes for a given provider_id.
+# If we ever need to invalidate, bump the version in
+# credential_providers.json and the frontend will cache-bust via the
+# catalogue version hash it already holds.
+_IMMUTABLE_CACHE = "public, max-age=31536000, immutable"
+
+
+def _safe_icon_path(provider_id: str) -> Path | None:
+ """Return an absolute path to the icon file if it's safe and exists.
+
+ Returns None if the provider_id is malformed, points outside the
+ icons directory, or the file is missing.
+ """
+ if not provider_id or len(provider_id) > 64:
+ return None
+ if not all(ch in _ALLOWED_CHARS for ch in provider_id):
+ return None
+
+ candidate = (_STATIC_ICONS_DIR / f"{provider_id}.svg").resolve()
+ try:
+ # Ensure the resolved path is still inside the icons dir.
+ candidate.relative_to(_STATIC_ICONS_DIR.resolve())
+ except ValueError:
+ logger.warning("credentials.icon: path traversal rejected for %r", provider_id)
+ return None
+
+ if not candidate.is_file():
+ return None
+ return candidate
+
+
[email protected]("/icon/{provider_id}")
+async def get_credential_icon(provider_id: str) -> Response:
+ """Serve a credential provider icon SVG with long-lived caching.
+
+ The frontend fetches this lazily per-tile as the credentials palette
+ scrolls (n8n pattern). Missing icons return 404 and the client falls
+ back to its in-bundle icon resolution, so this endpoint is optional
+ for the current 20-provider catalogue but is wired now so the shape
+ is stable for the 5000-provider target.
+ """
+ path = _safe_icon_path(provider_id)
+ if path is None:
+ raise HTTPException(status_code=404, detail="icon not found")
+
+ return FileResponse(
+ path,
+ media_type="image/svg+xml",
+ headers={"Cache-Control": _IMMUTABLE_CACHE},
+ )
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index f3302cc4..1dbd434f 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -233,6 +233,32 @@ async def handle_get_all_tool_schemas(data: Dict[str, Any], websocket: WebSocket
return {"success": True, "schemas": schemas}
+# ============================================================================
+# Credential Registry Handler (Nango-style bulk fetch for 20 -> 5000 providers)
+# ============================================================================
+
+@ws_handler()
+async def handle_get_credential_catalogue(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """Return the full credential provider catalogue.
+
+ Response shape: {providers, categories, version}. The version is a
+ content-sha256 of the resolved payload; clients warm-start from
+ IndexedDB and only re-fetch when the version changes. Supports
+ Nango-style bulk fetch at modal-open time (one roundtrip, cached for
+ the app session via TanStack Query + experimental_createPersister).
+ """
+ from services.credential_registry import get_credential_registry
+
+ registry = get_credential_registry()
+ # Optional conditional fetch: if the client already has the current
+ # version in IndexedDB it can send `since` and get a 304-style response.
+ since = data.get("since")
+ version = registry.get_version()
+ if since and since == version:
+ return {"unchanged": True, "version": version}
+ return registry.get_catalogue()
+
+
# ============================================================================
# Node Execution Handlers
# ============================================================================
@@ -1021,19 +1047,48 @@ async def handle_get_stored_api_key(data: Dict[str, Any], websocket: WebSocket)
@ws_handler("provider", "api_key")
async def handle_save_api_key(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
- """Save an API key (without validation)."""
- auth_service = container.auth_service()
- await auth_service.store_api_key(provider=data["provider"].lower(), api_key=data["api_key"].strip(),
- models=data.get("models", []), session_id=data.get("session_id", "default"))
- return {"provider": data["provider"]}
+ """Save an API key (without validation).
+
+ Supports client-side idempotency: if the client supplies a
+ `request_id` (opaque UUID), duplicate calls within 60 s return the
+ cached result instead of re-running the mutation. Prevents
+ double-writes on retry / reconnect / double-click.
+ """
+ from services.idempotency import get_idempotency_store
+
+ store = get_idempotency_store("credentials")
+ provider = data["provider"].lower()
+
+ async def _do_save() -> Dict[str, Any]:
+ auth_service = container.auth_service()
+ await auth_service.store_api_key(
+ provider=provider,
+ api_key=data["api_key"].strip(),
+ models=data.get("models", []),
+ session_id=data.get("session_id", "default"),
+ )
+ return {"provider": data["provider"]}
+
+ return await store.run(data.get("request_id"), _do_save)
@ws_handler("provider")
async def handle_delete_api_key(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
- """Delete stored API key."""
- auth_service = container.auth_service()
- await auth_service.remove_api_key(data["provider"].lower(), data.get("session_id", "default"))
- return {"provider": data["provider"]}
+ """Delete stored API key.
+
+ Idempotent on `request_id` β€” see `handle_save_api_key`.
+ """
+ from services.idempotency import get_idempotency_store
+
+ store = get_idempotency_store("credentials")
+ provider = data["provider"].lower()
+
+ async def _do_delete() -> Dict[str, Any]:
+ auth_service = container.auth_service()
+ await auth_service.remove_api_key(provider, data.get("session_id", "default"))
+ return {"provider": data["provider"]}
+
+ return await store.run(data.get("request_id"), _do_delete)
# ============================================================================
@@ -3058,6 +3113,9 @@ MESSAGE_HANDLERS: Dict[str, MessageHandler] = {
"delete_tool_schema": handle_delete_tool_schema,
"get_all_tool_schemas": handle_get_all_tool_schemas,
+ # Credential registry (Nango-style bulk catalogue for credentials panel)
+ "get_credential_catalogue": handle_get_credential_catalogue,
+
# Node execution
"execute_node": handle_execute_node,
"execute_workflow": handle_execute_workflow,
diff --git a/server/services/auth.py b/server/services/auth.py
index 0927d46b..938a489f 100644
--- a/server/services/auth.py
+++ b/server/services/auth.py
@@ -285,6 +285,92 @@ class AuthService:
logger.error("Failed to get OAuth tokens", provider=provider, error=str(e))
return None
+ async def refresh_oauth_tokens_with_breaker(
+ self,
+ provider: str,
+ refresh_fn,
+ ) -> Dict[str, Any]:
+ """Run an OAuth refresh call under a circuit breaker.
+
+ Phase 7.5c of the credentials-scaling plan. Protects downstream
+ OAuth providers from cascading failures: after 3 consecutive
+ refresh failures within 60 s for a given provider, the breaker
+ opens and all subsequent calls short-circuit for 30 s with a
+ clear error instead of piling on the upstream.
+
+ Args:
+ provider: Provider name (e.g. 'google', 'twitter') β€” used as
+ the breaker scope so one provider failing does not trip
+ the breaker for another.
+ refresh_fn: Zero-arg callable returning the refresh result
+ dict. Can be sync or async. Typical shape:
+ ``{"success": True, "access_token": "..."}``
+ or
+ ``{"success": False, "error": "..."}``.
+
+ Returns:
+ The refresh_fn result on success, or a dict of
+ ``{"success": False, "error": "...", "circuit_open": True,
+ "retry_after_seconds": N}`` when the breaker is open.
+
+ This method is opt-in: legacy call sites that call their OAuth
+ helpers directly are unaffected. New/migrated call sites gain
+ breaker protection by routing through here instead of invoking
+ the refresh helper directly.
+ """
+ import asyncio
+ from services.circuit_breaker import get_circuit_breaker, CircuitBreakerOpen
+
+ breaker = get_circuit_breaker(
+ f"{provider}_oauth_refresh",
+ failure_threshold=3,
+ failure_window=60.0,
+ cooldown_seconds=30.0,
+ )
+
+ async def _call():
+ if asyncio.iscoroutinefunction(refresh_fn):
+ result = await refresh_fn()
+ else:
+ # Run sync callables in a thread to avoid blocking the
+ # event loop when the refresh does a blocking HTTP call
+ # (e.g. google-auth's Credentials.refresh()).
+ result = await asyncio.to_thread(refresh_fn)
+
+ # Treat dict-shaped {"success": False} as a breaker-relevant
+ # failure too, not just raised exceptions. Most OAuth helpers
+ # in this codebase return {"success": False, "error": "..."}
+ # instead of raising.
+ if isinstance(result, dict) and result.get("success") is False:
+ raise RuntimeError(
+ result.get("error") or "oauth refresh reported failure"
+ )
+ return result
+
+ try:
+ return await breaker.run(_call)
+ except CircuitBreakerOpen as e:
+ logger.warning(
+ "auth: OAuth refresh breaker open for %s (retry after %.0fs)",
+ provider,
+ e.retry_after_seconds,
+ )
+ return {
+ "success": False,
+ "error": str(e),
+ "circuit_open": True,
+ "retry_after_seconds": e.retry_after_seconds,
+ "needs_reauth": False,
+ }
+ except Exception as e: # noqa: BLE001 β€” surface non-breaker errors
+ logger.error("auth: OAuth refresh failed for %s: %s", provider, e)
+ return {
+ "success": False,
+ "error": str(e),
+ "circuit_open": False,
+ "needs_reauth": True,
+ }
+
async def remove_oauth_tokens(
self,
provider: str,
diff --git a/server/services/circuit_breaker.py b/server/services/circuit_breaker.py
new file mode 100644
index 00000000..01fb0a6a
--- /dev/null
+++ b/server/services/circuit_breaker.py
@@ -0,0 +1,258 @@
+"""Simple in-process circuit breaker.
+
+Phase 7.5c of the credentials-scaling plan. Generalizes to any upstream
+call where repeated failures should stop cascading into the rest of the
+system (OAuth refresh, third-party API calls, LLM provider endpoints).
+
+Semantics
+---------
+A breaker has three states:
+
+- **closed** β€” normal operation; calls pass through to the wrapped
+ coroutine. On each failure the breaker increments a per-key counter.
+- **open** β€” all calls short-circuit immediately with
+ :class:`CircuitBreakerOpen`. Opens when the failure counter crosses
+ ``failure_threshold`` within a ``failure_window`` rolling window.
+- **half_open**β€” after ``cooldown_seconds``, the next call is allowed
+ through as a probe. Success closes the breaker; failure reopens it
+ and resets the cooldown.
+
+Design decisions
+----------------
+- **Per-key**: each breaker instance is keyed on a free-form scope
+ (e.g. ``"twitter"``, ``"google"``). One provider failing does not trip
+ the breaker for another provider.
+- **Process-local**: single-instance self-hosted tool. No Redis, no
+ distributed consensus. At deployment scale, upgrade to Redis-backed
+ counters.
+- **Opt-in**: services call ``breaker.run(func)`` explicitly. Legacy
+ code paths that don't opt in get the old behavior unchanged.
+- **Exception caching**: when open, the breaker raises
+ :class:`CircuitBreakerOpen` with the last recorded failure as
+ ``__cause__``, so callers can introspect why it opened.
+
+Usage
+-----
+ from services.circuit_breaker import get_circuit_breaker, CircuitBreakerOpen
+
+ async def refresh_google_tokens() -> TokenPair:
+ breaker = get_circuit_breaker("google_oauth_refresh")
+ try:
+ return await breaker.run(_actually_refresh_with_google)
+ except CircuitBreakerOpen as e:
+ logger.warning("google OAuth refresh circuit open: %s", e)
+ raise
+
+Tunables (defaults)
+-------------------
+- ``failure_threshold``: 3
+- ``failure_window``: 60.0 seconds
+- ``cooldown_seconds``: 30.0 seconds
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Awaitable, Callable, Dict, List, Optional, TypeVar
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+T = TypeVar("T")
+
+
+class CircuitState(str, Enum):
+ CLOSED = "closed"
+ OPEN = "open"
+ HALF_OPEN = "half_open"
+
+
+class CircuitBreakerOpen(RuntimeError):
+ """Raised when a call short-circuits because the breaker is open."""
+
+ def __init__(self, scope: str, retry_after_seconds: float) -> None:
+ super().__init__(
+ f"circuit breaker {scope!r} is open; retry after ~{retry_after_seconds:.0f}s"
+ )
+ self.scope = scope
+ self.retry_after_seconds = retry_after_seconds
+
+
+@dataclass
+class _CircuitState:
+ state: CircuitState = CircuitState.CLOSED
+ failures: List[float] = field(default_factory=list)
+ opened_at: Optional[float] = None
+ last_exception: Optional[BaseException] = None
+
+
+class CircuitBreaker:
+ """One breaker instance scoped to a named upstream."""
+
+ def __init__(
+ self,
+ scope: str,
+ *,
+ failure_threshold: int = 3,
+ failure_window: float = 60.0,
+ cooldown_seconds: float = 30.0,
+ ) -> None:
+ self._scope = scope
+ self._failure_threshold = failure_threshold
+ self._failure_window = failure_window
+ self._cooldown = cooldown_seconds
+ self._state = _CircuitState()
+ self._lock = asyncio.Lock()
+
+ # ----- public API -----
+
+ async def run(self, func: Callable[[], Awaitable[T]]) -> T:
+ """Execute ``func`` under breaker supervision.
+
+ - Closed or half-open: runs normally. Success resets the counter
+ and closes the breaker. Failure increments the counter and may
+ open the breaker.
+ - Open: short-circuits immediately with
+ :class:`CircuitBreakerOpen`, unless the cooldown has elapsed
+ (in which case the breaker moves to half-open and runs the
+ call as a probe).
+ """
+ async with self._lock:
+ self._maybe_half_open(time.monotonic())
+ if self._state.state is CircuitState.OPEN:
+ retry_after = self._retry_after()
+ exc = CircuitBreakerOpen(self._scope, retry_after)
+ if self._state.last_exception is not None:
+ exc.__cause__ = self._state.last_exception
+ raise exc
+
+ try:
+ result = await func()
+ except BaseException as exc: # noqa: BLE001 β€” re-raised below
+ async with self._lock:
+ self._record_failure(time.monotonic(), exc)
+ raise
+
+ async with self._lock:
+ self._record_success()
+ return result
+
+ def state(self) -> CircuitState:
+ return self._state.state
+
+ def stats(self) -> Dict[str, object]:
+ now = time.monotonic()
+ cutoff = now - self._failure_window
+ recent_failures = sum(1 for t in self._state.failures if t >= cutoff)
+ return {
+ "scope": self._scope,
+ "state": self._state.state.value,
+ "failure_threshold": self._failure_threshold,
+ "failure_window_seconds": self._failure_window,
+ "cooldown_seconds": self._cooldown,
+ "recent_failures": recent_failures,
+ "opened_at": self._state.opened_at,
+ "retry_after_seconds": self._retry_after() if self._state.state is CircuitState.OPEN else 0,
+ }
+
+ def reset(self) -> None:
+ """Force-close the breaker. Tests or manual admin endpoint only."""
+ self._state = _CircuitState()
+
+ # ----- internals -----
+
+ def _maybe_half_open(self, now: float) -> None:
+ if self._state.state is not CircuitState.OPEN:
+ return
+ if self._state.opened_at is None:
+ return
+ if now - self._state.opened_at < self._cooldown:
+ return
+ # Cooldown elapsed β†’ allow one probe call.
+ logger.info(
+ "circuit_breaker[%s]: cooldown elapsed, moving to half_open", self._scope
+ )
+ self._state.state = CircuitState.HALF_OPEN
+
+ def _record_failure(self, now: float, exc: BaseException) -> None:
+ self._state.last_exception = exc
+ # Prune failures outside the rolling window.
+ cutoff = now - self._failure_window
+ self._state.failures = [t for t in self._state.failures if t >= cutoff]
+ self._state.failures.append(now)
+
+ if self._state.state is CircuitState.HALF_OPEN:
+ # Probe failed β†’ reopen immediately and restart cooldown.
+ logger.warning(
+ "circuit_breaker[%s]: half_open probe failed, reopening: %s",
+ self._scope,
+ type(exc).__name__,
+ )
+ self._state.state = CircuitState.OPEN
+ self._state.opened_at = now
+ return
+
+ if len(self._state.failures) >= self._failure_threshold:
+ logger.warning(
+ "circuit_breaker[%s]: threshold reached (%d/%d failures in %ds), opening",
+ self._scope,
+ len(self._state.failures),
+ self._failure_threshold,
+ int(self._failure_window),
+ )
+ self._state.state = CircuitState.OPEN
+ self._state.opened_at = now
+
+ def _record_success(self) -> None:
+ if self._state.state in (CircuitState.HALF_OPEN, CircuitState.OPEN):
+ logger.info("circuit_breaker[%s]: closed after successful call", self._scope)
+ self._state.state = CircuitState.CLOSED
+ self._state.failures.clear()
+ self._state.opened_at = None
+ self._state.last_exception = None
+
+ def _retry_after(self) -> float:
+ if self._state.opened_at is None:
+ return 0.0
+ elapsed = time.monotonic() - self._state.opened_at
+ return max(0.0, self._cooldown - elapsed)
+
+
+# ----- module-level registry -----
+
+_BREAKERS: Dict[str, CircuitBreaker] = {}
+
+
+def get_circuit_breaker(
+ scope: str,
+ *,
+ failure_threshold: int = 3,
+ failure_window: float = 60.0,
+ cooldown_seconds: float = 30.0,
+) -> CircuitBreaker:
+ """Return (creating on first call) the shared breaker for ``scope``.
+
+ Tunable parameters only apply on first creation; subsequent calls
+ with the same scope return the existing instance and ignore new
+ params. This keeps scope-to-breaker identity stable across a
+ process lifetime.
+ """
+ breaker = _BREAKERS.get(scope)
+ if breaker is None:
+ breaker = CircuitBreaker(
+ scope,
+ failure_threshold=failure_threshold,
+ failure_window=failure_window,
+ cooldown_seconds=cooldown_seconds,
+ )
+ _BREAKERS[scope] = breaker
+ return breaker
+
+
+def all_breaker_stats() -> List[Dict[str, object]]:
+ """Return stats for every known breaker (observability endpoint)."""
+ return [b.stats() for b in _BREAKERS.values()]
diff --git a/server/services/credential_registry.py b/server/services/credential_registry.py
new file mode 100644
index 00000000..2349d609
--- /dev/null
+++ b/server/services/credential_registry.py
@@ -0,0 +1,271 @@
+"""Credential registry - loads server/config/credential_providers.json.
+
+Lazy singleton service backed by a single JSON file. Supports `extends`
+inheritance with array-merge-by-key semantics (port of n8n's
+NodeHelpers.mergeNodeProperties pattern).
+
+Served to clients via the `get_credential_catalogue` WebSocket handler,
+which returns `{providers, categories, version}`. The version is a
+content-sha256 of the resolved catalogue so the client can warm-start from
+IndexedDB and revalidate only when the server content changes.
+
+Follows the same lazy-singleton pattern as services/email_service.py.
+"""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+_CONFIG_PATH = Path(__file__).parent.parent / "config" / "credential_providers.json"
+
+# Arrays that are merged by `key` instead of replaced wholesale when a child
+# extends a parent. Matches the shape defined in credential_providers.json.
+_MERGE_BY_KEY_ARRAYS: Set[str] = {"fields", "status_rows", "actions"}
+
+
+class CredentialRegistryError(RuntimeError):
+ """Raised for malformed registry files or resolution cycles."""
+
+
+class CredentialRegistry:
+ """Lazy singleton wrapper around credential_providers.json."""
+
+ _instance: Optional["CredentialRegistry"] = None
+
+ def __init__(self) -> None:
+ self._raw: Optional[Dict[str, Any]] = None
+ self._resolved_providers: Optional[Dict[str, Dict[str, Any]]] = None
+ self._version: Optional[str] = None
+
+ @classmethod
+ def get_instance(cls) -> "CredentialRegistry":
+ if cls._instance is None:
+ cls._instance = cls()
+ return cls._instance
+
+ # ----- loading -----
+
+ def _load_raw(self) -> Dict[str, Any]:
+ """Parse the JSON file once, cache in memory for app lifetime."""
+ if self._raw is None:
+ try:
+ with _CONFIG_PATH.open(encoding="utf-8") as f:
+ self._raw = json.load(f)
+ except FileNotFoundError as e:
+ raise CredentialRegistryError(
+ f"credential_providers.json not found at {_CONFIG_PATH}"
+ ) from e
+ except json.JSONDecodeError as e:
+ raise CredentialRegistryError(
+ f"credential_providers.json is not valid JSON: {e}"
+ ) from e
+ return self._raw
+
+ def _resolve_all(self) -> Dict[str, Dict[str, Any]]:
+ """Resolve every provider's `extends` chain into flat dicts."""
+ if self._resolved_providers is not None:
+ return self._resolved_providers
+
+ raw = self._load_raw().get("providers", {})
+ if not isinstance(raw, dict):
+ raise CredentialRegistryError("providers must be a dict")
+
+ resolved: Dict[str, Dict[str, Any]] = {}
+ for provider_id in raw:
+ resolved[provider_id] = self._resolve_one(provider_id, raw, visiting=set())
+
+ self._resolved_providers = resolved
+ return resolved
+
+ def _resolve_one(
+ self,
+ provider_id: str,
+ raw: Dict[str, Dict[str, Any]],
+ visiting: Set[str],
+ ) -> Dict[str, Any]:
+ """Walk the extends chain, deep-merge parent then child overrides."""
+ if provider_id in visiting:
+ raise CredentialRegistryError(
+ f"extends cycle detected involving provider {provider_id!r}: "
+ f"{' -> '.join(sorted(visiting))} -> {provider_id}"
+ )
+ if provider_id not in raw:
+ raise CredentialRegistryError(
+ f"provider {provider_id!r} referenced via extends but not defined"
+ )
+
+ entry = raw[provider_id]
+ extends = entry.get("extends")
+ if not extends:
+ merged = copy.deepcopy(entry)
+ merged["id"] = provider_id
+ return merged
+
+ # Resolve parent first (recursive), then overlay child.
+ parent_visiting = visiting | {provider_id}
+ parent = self._resolve_one(extends, raw, parent_visiting)
+
+ merged = _deep_merge(parent, entry)
+ merged["id"] = provider_id
+ # `extends` itself is an implementation detail; strip from output.
+ merged.pop("extends", None)
+ # `_abstract` must not be inherited β€” only concrete entries that set
+ # it explicitly should be treated as abstract. Otherwise every child
+ # of `_ai_base` would silently become abstract and vanish from
+ # get_all_providers().
+ if "_abstract" not in entry:
+ merged.pop("_abstract", None)
+ return merged
+
+ # ----- public API -----
+
+ def get_all_providers(self) -> List[Dict[str, Any]]:
+ """Return all concrete (non-abstract) providers in JSON order."""
+ resolved = self._resolve_all()
+ # Preserve insertion order of the raw providers dict.
+ raw = self._load_raw().get("providers", {})
+ out: List[Dict[str, Any]] = []
+ for provider_id in raw:
+ entry = resolved[provider_id]
+ if entry.get("_abstract"):
+ continue
+ out.append(entry)
+ return out
+
+ def get_provider(self, provider_id: str) -> Optional[Dict[str, Any]]:
+ """Return a single resolved provider by id, or None if abstract/missing."""
+ resolved = self._resolve_all()
+ entry = resolved.get(provider_id)
+ if entry is None or entry.get("_abstract"):
+ return None
+ return entry
+
+ def get_categories(self) -> List[Dict[str, Any]]:
+ """Return ordered category metadata derived from the raw file."""
+ raw = self._load_raw()
+ categories = raw.get("categories", {})
+ if not isinstance(categories, dict):
+ return []
+ out = [
+ {"key": key, "label": cfg.get("label", key), "order": cfg.get("order", 0)}
+ for key, cfg in categories.items()
+ ]
+ out.sort(key=lambda c: (c["order"], c["key"]))
+ return out
+
+ def get_version(self) -> str:
+ """Content-sha256 of the resolved catalogue (providers + categories).
+
+ Used by clients for warm-start cache invalidation: when the hash
+ changes, the client fetches a fresh catalogue; otherwise it serves
+ from IndexedDB with zero network traffic.
+ """
+ if self._version is None:
+ payload = {
+ "providers": self.get_all_providers(),
+ "categories": self.get_categories(),
+ }
+ encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ self._version = hashlib.sha256(encoded).hexdigest()
+ return self._version
+
+ def get_catalogue(self) -> Dict[str, Any]:
+ """Full payload returned by the `get_credential_catalogue` WebSocket handler."""
+ return {
+ "providers": self.get_all_providers(),
+ "categories": self.get_categories(),
+ "version": self.get_version(),
+ }
+
+ def reload(self) -> None:
+ """Drop all caches; next call re-parses the JSON file."""
+ self._raw = None
+ self._resolved_providers = None
+ self._version = None
+
+
+# ----- deep-merge helper (n8n mergeNodeProperties pattern) -----
+
+
+def _deep_merge(parent: Dict[str, Any], child: Dict[str, Any]) -> Dict[str, Any]:
+ """Return a new dict: parent deep-merged with child overrides.
+
+ Rules:
+ - Nested dicts merge recursively.
+ - Arrays named in _MERGE_BY_KEY_ARRAYS merge by the `key` field: child
+ entries replace parent entries with the same key; unmatched child
+ entries append to the parent list.
+ - All other arrays and scalars are replaced wholesale by the child.
+ - The child's `extends` key is ignored (resolved at a higher level).
+ """
+ result: Dict[str, Any] = copy.deepcopy(parent)
+ for key, child_value in child.items():
+ if key == "extends":
+ continue
+ parent_value = result.get(key)
+ if isinstance(parent_value, dict) and isinstance(child_value, dict):
+ result[key] = _deep_merge(parent_value, child_value)
+ elif (
+ key in _MERGE_BY_KEY_ARRAYS
+ and isinstance(parent_value, list)
+ and isinstance(child_value, list)
+ ):
+ result[key] = _merge_array_by_key(parent_value, child_value)
+ else:
+ result[key] = copy.deepcopy(child_value)
+ return result
+
+
+def _merge_array_by_key(parent: List[Any], child: List[Any]) -> List[Any]:
+ """Merge two lists of dicts by their `key` field.
+
+ Entries in the child that share a `key` with a parent entry recursively
+ deep-merge (child wins on scalar conflicts). Unmatched child entries
+ append to the end. Entries without a `key` field on either side are
+ treated as identity replacements (parent dropped, child appended).
+ """
+ by_key: Dict[str, Dict[str, Any]] = {}
+ order: List[str] = []
+
+ def _item_key(item: Any, prefix: str, index: int) -> str:
+ if isinstance(item, dict) and "key" in item:
+ return str(item["key"])
+ return f"{prefix}:{index}"
+
+ for i, item in enumerate(parent):
+ k = _item_key(item, "parent", i)
+ by_key[k] = copy.deepcopy(item) if isinstance(item, dict) else item
+ order.append(k)
+
+ for i, item in enumerate(child):
+ if isinstance(item, dict) and "key" in item:
+ k = str(item["key"])
+ existing = by_key.get(k)
+ if isinstance(existing, dict):
+ by_key[k] = _deep_merge(existing, item)
+ else:
+ by_key[k] = copy.deepcopy(item)
+ if k not in order:
+ order.append(k)
+ else:
+ k = f"child:{i}"
+ by_key[k] = copy.deepcopy(item) if isinstance(item, dict) else item
+ order.append(k)
+
+ return [by_key[k] for k in order]
+
+
+# ----- module-level singleton accessor -----
+
+
+def get_credential_registry() -> CredentialRegistry:
+ """Return the process-wide CredentialRegistry singleton."""
+ return CredentialRegistry.get_instance()
diff --git a/server/services/idempotency.py b/server/services/idempotency.py
new file mode 100644
index 00000000..4d22c2fe
--- /dev/null
+++ b/server/services/idempotency.py
@@ -0,0 +1,198 @@
+"""Idempotency key service.
+
+Phase 7.5b of the credentials-scaling plan. Generalizes to any mutation
+handler in the platform refactor (control plane RFC, Phase C).
+
+Client mutations include an opaque `request_id` (UUID generated in the
+browser via `crypto.randomUUID()`). The server caches the result per
+request_id with a short TTL. If the same request_id arrives twice (retry,
+WebSocket reconnect, user double-click, etc.), the server returns the
+cached result instead of re-running the mutation. Prevents double-writes
+and makes every mutation handler safe to retry.
+
+Design decisions
+----------------
+- Process-local dict, not Redis β€” at our scale (single-user / small-team
+ self-hosted) process-local is enough. Survives handler-internal retries
+ but not server restarts; that's the correct tradeoff for a 60 s window.
+- Positive and negative caching: both successful results and raised
+ exceptions are cached, so a retried mutation that would have failed
+ the first time still fails on retry (rather than silently succeeding).
+- TTL is in wall-clock seconds; entries are pruned lazily on access,
+ avoiding a background sweeper task.
+- Thread/async safe via a per-scope asyncio.Lock; lock is only held for
+ the cache-miss window, not for the duration of the wrapped function.
+
+Usage
+-----
+ from services.idempotency import get_idempotency_store
+
+ store = get_idempotency_store("credentials")
+
+ async def handle_save_credential(data, websocket):
+ request_id = data.get("request_id")
+ return await store.run(
+ request_id,
+ lambda: _do_save(data),
+ )
+
+Routers that omit `request_id` fall through to a direct call with no
+caching β€” callers are not forced to opt in.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable, Dict, Optional
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class _CacheEntry:
+ result: Any
+ exception: Optional[BaseException]
+ expires_at: float
+
+
+class IdempotencyStore:
+ """Per-scope TTL-bounded idempotency cache.
+
+ Each router or module typically owns one store, keyed by the
+ handler's logical scope (e.g. ``"credentials"``, ``"workflow"``),
+ so request_id collisions across scopes are impossible.
+ """
+
+ def __init__(self, scope: str, ttl_seconds: float = 60.0) -> None:
+ self._scope = scope
+ self._ttl = ttl_seconds
+ self._entries: Dict[str, _CacheEntry] = {}
+ # Per-request_id locks prevent two concurrent calls with the same
+ # request_id from both running the underlying function. First
+ # caller executes; second caller awaits the same lock and then
+ # hits the cache.
+ self._locks: Dict[str, asyncio.Lock] = {}
+ self._global_lock = asyncio.Lock()
+
+ # ----- public API -----
+
+ async def run(
+ self,
+ request_id: Optional[str],
+ func: Callable[[], Awaitable[Any]],
+ ) -> Any:
+ """Run ``func`` and cache the result by ``request_id``.
+
+ If ``request_id`` is falsy, ``func`` runs directly with no
+ caching. This keeps the opt-in contract one-sided: clients that
+ supply a request_id get dedupe; clients that don't get normal
+ behavior.
+ """
+ if not request_id:
+ return await func()
+
+ # Fast path: hit the cache without acquiring a per-key lock.
+ cached = self._lookup(request_id)
+ if cached is not None:
+ return self._materialize(cached)
+
+ # Slow path: ensure only one coroutine per request_id runs at a
+ # time. Concurrent duplicate requests queue on the same lock.
+ lock = await self._get_lock(request_id)
+ async with lock:
+ cached = self._lookup(request_id)
+ if cached is not None:
+ return self._materialize(cached)
+
+ try:
+ result = await func()
+ except BaseException as exc: # noqa: BLE001 β€” we re-raise below
+ self._store_failure(request_id, exc)
+ raise
+ else:
+ self._store_success(request_id, result)
+ return result
+
+ def stats(self) -> Dict[str, Any]:
+ """Return observability stats for logging/debug endpoints."""
+ now = time.monotonic()
+ live = sum(1 for e in self._entries.values() if e.expires_at > now)
+ return {
+ "scope": self._scope,
+ "ttl_seconds": self._ttl,
+ "entries_total": len(self._entries),
+ "entries_live": live,
+ }
+
+ def clear(self) -> None:
+ """Drop all entries. Tests or manual admin use only."""
+ self._entries.clear()
+ self._locks.clear()
+
+ # ----- internals -----
+
+ def _lookup(self, request_id: str) -> Optional[_CacheEntry]:
+ entry = self._entries.get(request_id)
+ if entry is None:
+ return None
+ if entry.expires_at <= time.monotonic():
+ # Lazy eviction β€” drop the stale entry on access.
+ self._entries.pop(request_id, None)
+ return None
+ return entry
+
+ def _materialize(self, entry: _CacheEntry) -> Any:
+ if entry.exception is not None:
+ raise entry.exception
+ return entry.result
+
+ def _store_success(self, request_id: str, result: Any) -> None:
+ self._entries[request_id] = _CacheEntry(
+ result=result,
+ exception=None,
+ expires_at=time.monotonic() + self._ttl,
+ )
+ logger.debug("idempotency[%s]: cached success for %s", self._scope, request_id)
+
+ def _store_failure(self, request_id: str, exc: BaseException) -> None:
+ self._entries[request_id] = _CacheEntry(
+ result=None,
+ exception=exc,
+ expires_at=time.monotonic() + self._ttl,
+ )
+ logger.debug(
+ "idempotency[%s]: cached failure for %s: %s",
+ self._scope,
+ request_id,
+ type(exc).__name__,
+ )
+
+ async def _get_lock(self, request_id: str) -> asyncio.Lock:
+ async with self._global_lock:
+ lock = self._locks.get(request_id)
+ if lock is None:
+ lock = asyncio.Lock()
+ self._locks[request_id] = lock
+ return lock
+
+
+# ----- module-level registry -----
+
+_STORES: Dict[str, IdempotencyStore] = {}
+
+
+def get_idempotency_store(scope: str, ttl_seconds: float = 60.0) -> IdempotencyStore:
+ """Return (creating on first call) the shared store for ``scope``.
+
+ Scopes are free-form namespaces. Pick one per router or per logical
+ domain β€” e.g. ``"credentials"``, ``"workflow"``, ``"nodespec"``.
+ """
+ store = _STORES.get(scope)
+ if store is None:
+ store = IdempotencyStore(scope, ttl_seconds=ttl_seconds)
+ _STORES[scope] = store
+ return store