Skip to content

Self-improving AI Employees for everyone

refactor(credentials): DB as single source of truth + symmetric broadcasts + cache dedup

trohitgc94a610f1681
diff --git a/client/src/components/SquareNode.tsx b/client/src/components/SquareNode.tsx
index 650a5042..8bf1830c 100644
--- a/client/src/components/SquareNode.tsx
+++ b/client/src/components/SquareNode.tsx
@@ -9,6 +9,7 @@ import { useWebSocket, useWhatsAppStatus, useNodeStatus } from '../contexts/WebS
import { getCachedNodeSpec, isNodeInBackendGroup, resolveNodeDescription, useNodeSpec } from '../lib/nodeSpec';
import { NodeIcon } from '../assets/icons';
import { AI_MODEL_PROVIDER_MAP } from '../lib/aiModelProviders';
+import { useProviderStored } from '../hooks/useCatalogueQuery';
// Nodes with 'tool' in their group can connect to AI Agent/Zeenie tool handles
const hasToolGroup = (definition: any): boolean => {
@@ -145,11 +146,12 @@ const SquareNode: React.FC<NodeProps<NodeData>> = ({ id, type, data, isConnectab
return '';
}, [definition?.credentials, type]);
- // "Is a credential configured for this provider?" is read from the
- // WS-context apiKeyStatuses map — populated synchronously by
- // validateApiKeyAsync + the api_key_status broadcast. The canvas node
- // never fetches decrypted keys; that's the credentials modal's job.
- const hasApiKey = !!(providerId && getApiKeyStatus(providerId)?.hasKey);
+ // "Is a credential configured for this provider?" reads from the
+ // server-driven catalogue (single source of truth — the `stored`
+ // flag is computed from `auth_service.has_valid_key()` on every
+ // catalogue read). The retired `apiKeyStatuses[id].hasKey` mirror
+ // duplicated this answer with no synchronisation contract.
+ const hasApiKey = useProviderStored(providerId);
const isConfigured = hasApiKey && !!data && Object.keys(data).length > 0;
const handleParametersClick = (e: React.MouseEvent) => {
@@ -187,10 +189,11 @@ const SquareNode: React.FC<NodeProps<NodeData>> = ({ id, type, data, isConnectab
return googleMapsKeyStatus.valid ? theme.dracula.green : theme.dracula.red;
}
- // AI Model nodes - use reactive WebSocket API key status
+ // AI Model nodes - "is stored" from catalogue (hasApiKey via
+ // useProviderStored above), "is validated" from apiKeyStatuses.
if (isAIModelNode) {
- if (aiKeyStatus?.valid && aiKeyStatus?.hasKey) return theme.dracula.green;
- if (aiKeyStatus?.hasKey) return theme.dracula.orange;
+ if (aiKeyStatus?.valid && hasApiKey) return theme.dracula.green;
+ if (hasApiKey) return theme.dracula.orange;
return theme.dracula.red;
}
@@ -224,12 +227,14 @@ const SquareNode: React.FC<NodeProps<NodeData>> = ({ id, type, data, isConnectab
? 'Google Maps API key validated'
: `API key invalid: ${googleMapsKeyStatus.message || 'Validation failed'}`;
}
- // AI Model nodes - use reactive WebSocket API key status
+ // AI Model nodes - "is stored" from catalogue (hasApiKey),
+ // "is validated" from apiKeyStatuses. Independent concerns
+ // post-Wave-12; the two sources don't drift across tabs.
if (isAIModelNode) {
- if (aiKeyStatus?.valid && aiKeyStatus?.hasKey) {
+ if (aiKeyStatus?.valid && hasApiKey) {
return `${aiProviderId?.charAt(0).toUpperCase()}${aiProviderId?.slice(1)} API key validated`;
}
- if (aiKeyStatus?.hasKey) {
+ if (hasApiKey) {
return 'API key found, validation pending';
}
return 'API key required - configure in Credentials';
diff --git a/client/src/components/credentials/CredentialsModal.tsx b/client/src/components/credentials/CredentialsModal.tsx
index e96a2fde..ac43c33d 100644
--- a/client/src/components/credentials/CredentialsModal.tsx
+++ b/client/src/components/credentials/CredentialsModal.tsx
@@ -1,24 +1,28 @@
/**
* CredentialsModal — thin shell.
*
- * Phase 4 of the credentials-scaling plan:
- * - Data source is the server-owned `credential_providers.json` via
- * `useCatalogueQuery` (TanStack Query + IndexedDB warm-start).
- * - Sidebar is `CredentialsPalette` (cmdk + fuzzysort + GroupedVirtuoso).
- * - Detail panel is still `PanelRenderer` (unchanged; consumed via the
- * catalogue adapter so panels see the same `ProviderConfig` shape).
+ * Post-Wave-12: the server-owned catalogue (`get_credential_catalogue`
+ * → `useCatalogueQuery`) is the SINGLE source of truth for the
+ * provider list. The retired `providers.tsx` static fallback no longer
+ * exists — adding a new provider is a backend-only change.
*
- * Additive safety: if the server fetch fails or hasn't landed yet, the
- * modal falls back to the client-owned `PROVIDERS` from `providers.tsx`.
- * That file stays in place until Phase 8 verification passes.
+ * Cold-boot UX:
+ * - With IDB hit (return visit): catalogue populated within ~50 ms
+ * via the warm-start path in `useCatalogueQuery`.
+ * - With IDB miss (first visit / cleared storage): a Skeleton
+ * palette renders while the WS catalogue arrives (~200-500 ms).
+ * - Server unreachable: explicit "couldn't reach server" error
+ * state — never a stale fallback list (would mislead the user
+ * about which providers they have configured).
*/
import React, { useMemo } from 'react';
-import { Loader2, ShieldCheck } from 'lucide-react';
+import { Loader2, ShieldCheck, AlertTriangle } from 'lucide-react';
import Modal from '../ui/Modal';
import { Badge } from '@/components/ui/badge';
-import { CATEGORIES as CLIENT_CATEGORIES, PROVIDERS as CLIENT_PROVIDERS } from './providers';
+import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
+import { Skeleton } from '@/components/ui/skeleton';
import PanelRenderer from './PanelRenderer';
import CredentialsPalette from './CredentialsPalette';
import { rehydrateCatalogue } from './catalogueAdapter';
@@ -35,17 +39,20 @@ const CredentialsModal: React.FC<Props> = ({ visible, onClose }) => {
const selectedId = useCredentialRegistry((s) => s.selectedId);
const setSelectedId = useCredentialRegistry((s) => s.setSelectedId);
- // Server-owned catalogue with IDB warm-start.
+ // Server-owned catalogue with IDB warm-start. Single source of truth.
const catalogue = useCatalogueQuery();
- // Rehydrate server JSON → runtime ProviderConfig shape; fall back to
- // the client-owned registry if the server data hasn't arrived (cold
- // start with no IDB cache, server unreachable, etc.).
- const { providers, categories } = useMemo(() => {
- if (catalogue.data) return rehydrateCatalogue(catalogue.data);
- return { providers: CLIENT_PROVIDERS, categories: CLIENT_CATEGORIES };
+ // Rehydrate server JSON → runtime ProviderConfig shape. No client
+ // fallback — if the data isn't here yet we render Skeleton, and if
+ // the fetch errored we render an explicit error state.
+ const rehydrated = useMemo(() => {
+ if (!catalogue.data) return null;
+ return rehydrateCatalogue(catalogue.data);
}, [catalogue.data]);
+ const providers = rehydrated?.providers ?? [];
+ const categories = rehydrated?.categories ?? [];
+
// Default selection: if nothing is selected yet (or the previous
// selection isn't in the current catalogue), pick the first provider.
const effectiveSelectedId = useMemo(() => {
@@ -66,8 +73,8 @@ const CredentialsModal: React.FC<Props> = ({ visible, onClose }) => {
[providers, effectiveSelectedId],
);
- const usingServerData = !!catalogue.data;
const isLoadingServer = catalogue.isLoading && !catalogue.data;
+ const hasServerError = catalogue.isError && !catalogue.data;
const headerActions = (
<div className="flex items-center gap-3">
@@ -75,29 +82,59 @@ const CredentialsModal: React.FC<Props> = ({ visible, onClose }) => {
<ShieldCheck className="h-4 w-4 text-dracula-yellow" />
<span>API Credentials</span>
</div>
- <Badge variant="success">{providers.length} providers</Badge>
+ {rehydrated && (
+ <Badge variant="success">{providers.length} providers</Badge>
+ )}
{isLoadingServer && (
<Badge variant="info" className="gap-1">
<Loader2 className="h-3 w-3 animate-spin" />
loading
</Badge>
)}
- {!usingServerData && !isLoadingServer && (
- <Badge variant="warning" title="Using bundled fallback catalogue">
+ {hasServerError && (
+ <Badge variant="destructive" className="gap-1" title="Server unreachable">
+ <AlertTriangle className="h-3 w-3" />
offline
</Badge>
)}
</div>
);
- return (
- <Modal
- isOpen={visible}
- onClose={onClose}
- maxWidth="95vw"
- maxHeight="95vh"
- headerActions={headerActions}
- >
+ // Body dispatch:
+ // 1. Server error + no cached data → error state, no providers shown.
+ // 2. Loading + no cached data → Skeleton palette + empty detail.
+ // 3. Catalogue available (cached or fresh) → normal UI.
+ let body: React.ReactNode;
+ if (hasServerError) {
+ body = (
+ <div className="flex flex-1 flex-col items-center justify-center p-8">
+ <Alert variant="destructive" className="max-w-md">
+ <AlertTriangle className="h-4 w-4" />
+ <AlertTitle>Couldn't reach the credentials server</AlertTitle>
+ <AlertDescription>
+ The provider list comes from the backend. Check your connection
+ and try again — refreshing the page will retry the fetch.
+ </AlertDescription>
+ </Alert>
+ </div>
+ );
+ } else if (!rehydrated) {
+ body = (
+ <div className="flex h-full overflow-hidden">
+ <div className="flex w-[280px] shrink-0 flex-col gap-2 border-r border-border p-4">
+ {Array.from({ length: 6 }).map((_, i) => (
+ <Skeleton key={i} className="h-9 w-full" />
+ ))}
+ </div>
+ <div className="flex flex-1 flex-col gap-3 bg-background p-6">
+ <Skeleton className="h-6 w-48" />
+ <Skeleton className="h-4 w-72" />
+ <Skeleton className="mt-4 h-32 w-full" />
+ </div>
+ </div>
+ );
+ } else {
+ body = (
<div className="flex h-full overflow-hidden">
<div className="flex w-[280px] shrink-0 flex-col border-r border-border">
<CredentialsPalette
@@ -111,6 +148,18 @@ const CredentialsModal: React.FC<Props> = ({ visible, onClose }) => {
<PanelRenderer config={selected} visible={visible} />
</div>
</div>
+ );
+ }
+
+ return (
+ <Modal
+ isOpen={visible}
+ onClose={onClose}
+ maxWidth="95vw"
+ maxHeight="95vh"
+ headerActions={headerActions}
+ >
+ {body}
</Modal>
);
};
diff --git a/client/src/components/credentials/panels/ApiKeyPanel.tsx b/client/src/components/credentials/panels/ApiKeyPanel.tsx
index 66b52c6c..42ca287b 100644
--- a/client/src/components/credentials/panels/ApiKeyPanel.tsx
+++ b/client/src/components/credentials/panels/ApiKeyPanel.tsx
@@ -101,6 +101,13 @@ const ApiKeyPanel: React.FC<{ config: ProviderConfig; visible: boolean }> = ({ c
placeholder={field.placeholder}
loading={panel.loading === 'validate'}
isStored={validated}
+ // Local-LLM providers declare a non-`apiKey` field key
+ // (e.g. `ollama_proxy`) — the click probes the user's URL
+ // and pulls the model list. Cloud providers validate an
+ // upstream API key. The verb on the button reflects the
+ // semantic difference.
+ saveLabel={field.key === 'apiKey' ? 'Validate' : 'Fetch'}
+ savedLabel={field.key === 'apiKey' ? 'Valid' : 'Connected'}
/>
)}
</CardContent>
diff --git a/client/src/components/credentials/providers.tsx b/client/src/components/credentials/providers.tsx
deleted file mode 100644
index 8f34244b..00000000
--- a/client/src/components/credentials/providers.tsx
+++ /dev/null
@@ -1,195 +0,0 @@
-/**
- * Provider configs — every credential provider as a plain config object.
- *
- * Icons are declared as `iconRef` strings using the prefix-dispatch
- * contract resolved by `<NodeIcon>`:
- * - `lobehub:<brand>` → React component from @lobehub/icons
- * - `asset:<key>` → SVG via the filesystem registry
- * - emoji / text → rendered as-is
- * Colors are theme token keys, resolved at render by the consumer.
- * Status rows are pure data — theme colors are resolved in StatusCard.
- */
-
-import { AI_PROVIDER_META } from '../icons/AIProviderIcons';
-import type { ProviderConfig, CategoryGroup } from './types';
-
-// ============================================================================
-// AI PROVIDERS — derived from centralized AI_PROVIDER_META
-// ============================================================================
-
-const AI_PLACEHOLDERS: Record<string, string> = {
- openai: 'sk-...', anthropic: 'sk-ant-...', gemini: 'AIza...',
- groq: 'gsk_...', cerebras: 'csk-...', openrouter: 'sk-or-...',
- deepseek: 'sk-...', kimi: 'sk-...', mistral: 'sk-...',
- // Local LLM servers — the field stores the base URL, not an API key.
- // Backend reads it from the `{provider}_proxy` credential, the same
- // path cloud providers already use for Ollama-style auth delegation.
- ollama: 'http://localhost:11434/v1',
- lmstudio: 'http://localhost:1234/v1',
-};
-
-// Local LLM servers reuse the existing `{provider}_proxy` credential
-// for their base URL — no new field plumbing. The single-field
-// ApiKeyPanel renders this as "Base URL" via the override below; the
-// backend's _LocalLLM.resolve() returns a placeholder api_key so the
-// "API key required" check in execute_chat passes.
-const LOCAL_PROVIDER_IDS = new Set(['ollama', 'lmstudio']);
-
-const aiProviders: ProviderConfig[] = Object.entries(AI_PROVIDER_META).map(([id, meta]) => {
- const isLocal = LOCAL_PROVIDER_IDS.has(id);
- const fields = isLocal
- ? [{ key: `${id}_proxy`, label: 'Base URL', placeholder: AI_PLACEHOLDERS[id], required: true }]
- : [{ key: 'apiKey', label: 'API Key', secret: true, placeholder: AI_PLACEHOLDERS[id], required: true }];
- return {
- id, name: meta.label, category: 'ai', categoryLabel: 'AI Providers',
- color: meta.color, kind: 'apiKey' as const, iconRef: meta.iconRef, hasDefaults: true,
- fields,
- };
-});
-
-// ============================================================================
-// ALL PROVIDERS — flat array. Categories derived below.
-// ============================================================================
-
-export const PROVIDERS: ProviderConfig[] = [
- ...aiProviders,
-
- // ── Social: WhatsApp (QR pairing) ────────────────────────────────────
- {
- id: 'whatsapp_personal', name: 'WhatsApp Personal',
- category: 'social', categoryLabel: 'Social Media',
- color: 'categoryWhatsapp', kind: 'qrPairing', iconRef: 'asset:whatsapp',
- statusHook: 'whatsapp', hasRateLimits: true,
- statusRows: [
- { label: 'Status', ok: s => !!s?.connected, trueText: 'Connected', falseText: 'Disconnected' },
- { label: 'Session', ok: s => !!s?.has_session, trueText: 'Active', falseText: 'Inactive' },
- { label: 'Service', ok: s => !!s?.running, trueText: 'Running', falseText: 'Stopped' },
- ],
- actions: [
- { key: 'start', label: 'Start', intent: 'run' },
- { key: 'restart', label: 'Restart', intent: 'config' },
- { key: 'refresh', label: 'Refresh', intent: 'save' },
- ],
- qr: {
- qrField: 'qr',
- isConnected: s => !!s?.connected,
- connectedTitle: 'Already Connected!',
- connectedSubtitle: () => 'No QR code needed',
- isLoading: s => !!(s?.running && !s?.qr && !s?.connected),
- emptyText: s => s?.running ? 'Waiting for QR code...' : 'Start service to get QR code',
- scanText: 'Scan with WhatsApp mobile app',
- },
- },
-
- // ── Social: Twitter/X (OAuth) ────────────────────────────────────────
- {
- id: 'twitter', name: 'Twitter/X',
- category: 'social', categoryLabel: 'Social Media',
- color: 'categoryTrigger', kind: 'oauth', iconRef: 'asset:x',
- statusHook: 'twitter', usageService: 'twitter',
- ws: { login: 'twitter_oauth_login', logout: 'twitter_logout', status: 'twitter_oauth_status' },
- fields: [
- { key: 'twitter_client_id', label: 'Client ID', placeholder: 'Client ID (from X Developer Portal)', required: true },
- { key: 'twitter_client_secret', label: 'Client Secret', secret: true, placeholder: 'Client Secret (optional for PKCE)' },
- ],
- callbackUrl: 'http://localhost:3010/api/twitter/callback',
- instructions: 'Get credentials from the X Developer Portal. Create an app with OAuth 2.0 enabled.',
- },
-
- // ── Social: Telegram Bot (OAuth-like token flow) ─────────────────────
- {
- id: 'telegram', name: 'Telegram Bot',
- category: 'social', categoryLabel: 'Social Media',
- color: 'categoryChat', kind: 'oauth', iconRef: 'asset:telegram', statusHook: 'telegram',
- ws: { login: 'telegram_connect', logout: 'telegram_disconnect', status: 'telegram_status' },
- fields: [{ key: 'telegram_bot_token', label: 'Bot Token', secret: true, placeholder: '123456789:ABCdefGHIjklMNOpqrSTUvwxYZ', required: true }],
- instructions: 'Get a bot token from @BotFather on Telegram. After connecting, send any message to your bot to register as the owner.',
- },
-
- // ── Productivity: Google Workspace (OAuth) ───────────────────────────
- {
- id: 'gmail', name: 'Google Workspace',
- category: 'productivity', categoryLabel: 'Productivity',
- color: 'categoryAI', kind: 'oauth', iconRef: 'asset:gmail',
- statusHook: 'google', usageService: 'google_workspace',
- ws: { login: 'google_oauth_login', logout: 'google_logout', status: 'google_oauth_status' },
- fields: [
- { key: 'google_client_id', label: 'Client ID', placeholder: 'Client ID (from Google Cloud Console)', required: true },
- { key: 'google_client_secret', label: 'Client Secret', secret: true, placeholder: 'Client Secret (required)', required: true },
- ],
- callbackUrl: 'http://localhost:3010/api/google/callback',
- instructions: 'Get credentials from Google Cloud Console. Enable Gmail, Calendar, Drive, Sheets, Tasks, and People APIs.',
- },
-
- // ── Email (Himalaya IMAP/SMTP) ───────────────────────────────────────
- {
- id: 'email_himalaya', name: 'Email (IMAP/SMTP)',
- category: 'email', categoryLabel: 'Email',
- color: 'categoryChat', kind: 'email', iconRef: 'asset:read',
- },
-
- // ── Android Device (QR pairing) ──────────────────────────────────────
- {
- id: 'android_remote', name: 'Android Device',
- category: 'android', categoryLabel: 'Android',
- color: 'categoryAndroid', kind: 'qrPairing', iconRef: '📱',
- statusHook: 'android',
- fields: [{ key: 'android_remote', label: 'API Key', secret: true, placeholder: 'Enter Android Relay API key...', required: true }],
- statusRows: [
- { label: 'Device', ok: s => !!s?.paired, trueText: 'Paired', falseText: 'Not Paired' },
- { label: 'Relay', ok: s => !!s?.connected, trueText: 'Connected', falseText: 'Disconnected' },
- { label: 'Pairing', ok: s => !!s?.paired, trueText: 'Paired', falseText: 'Waiting', warn: true },
- ],
- actions: [
- { key: 'connect', label: 'Connect', intent: 'save',
- disabled: (s, stored) => !!s?.paired || !stored },
- { key: 'disconnect', label: 'Disconnect', intent: 'stop',
- disabled: s => !s?.connected },
- ],
- qr: {
- qrField: 'qr_data',
- isConnected: s => !!s?.paired,
- connectedTitle: 'Device Paired!',
- connectedSubtitle: s => s?.device_name || 'Android Device',
- isLoading: s => !!(s?.connected && !s?.qr_data && !s?.paired),
- emptyText: (s, stored) => !stored ? 'Add API key first' : (s?.connected ? 'Waiting for QR code...' : 'Click Connect to start'),
- scanText: 'Scan with Android companion app',
- },
- },
-
- // ── Search ───────────────────────────────────────────────────────────
- { id: 'brave_search', name: 'Brave Search', category: 'search', categoryLabel: 'Search',
- color: 'categoryLocation', kind: 'apiKey', iconRef: 'asset:brave',
- fields: [{ key: 'apiKey', label: 'API Key', secret: true, placeholder: 'BSA...', required: true }] },
- { id: 'perplexity', name: 'Perplexity', category: 'search', categoryLabel: 'Search',
- color: 'categoryUtil', kind: 'apiKey', iconRef: 'asset:perplexity',
- fields: [{ key: 'apiKey', label: 'API Key', secret: true, placeholder: 'pplx-...', required: true }] },
-
- // ── Scrapers ─────────────────────────────────────────────────────────
- { id: 'apify', name: 'Apify', category: 'scrapers', categoryLabel: 'Scrapers',
- color: 'categoryAI', kind: 'apiKey', iconRef: 'asset:apify', validateAs: 'apify',
- fields: [{ key: 'apiKey', label: 'API Key', secret: true, placeholder: 'apify_api_...', required: true }] },
- { id: 'serper', name: 'Serper', category: 'scrapers', categoryLabel: 'Scrapers',
- color: 'categoryAI', kind: 'apiKey', iconRef: 'asset:google',
- fields: [{ key: 'apiKey', label: 'API Key', secret: true, required: true }] },
-
- // ── Services ─────────────────────────────────────────────────────────
- { id: 'google_maps', name: 'Google Maps', category: 'services', categoryLabel: 'Services',
- color: 'categoryLocation', kind: 'apiKey', iconRef: '🗺️',
- validateAs: 'google_maps', usageService: 'google_maps',
- fields: [{ key: 'apiKey', label: 'API Key', secret: true, placeholder: 'AIza...', required: true }] },
-];
-
-// ============================================================================
-// CATEGORIES — derived from PROVIDERS, preserving insertion order
-// ============================================================================
-
-export const CATEGORIES: CategoryGroup[] = (() => {
- const map = new Map<string, CategoryGroup>();
- for (const p of PROVIDERS) {
- let cat = map.get(p.category);
- if (!cat) { cat = { key: p.category, label: p.categoryLabel, items: [] }; map.set(p.category, cat); }
- cat.items.push(p);
- }
- return Array.from(map.values());
-})();
diff --git a/client/src/components/ui/TopToolbar.tsx b/client/src/components/ui/TopToolbar.tsx
index 8f03c662..457d4fb3 100644
--- a/client/src/components/ui/TopToolbar.tsx
+++ b/client/src/components/ui/TopToolbar.tsx
@@ -47,7 +47,7 @@ import { cn } from '@/lib/utils';
import { useTheme } from '../../contexts/ThemeContext';
import { useAuth } from '../../contexts/AuthContext';
import { useApiKeys, GlobalModelState } from '../../hooks/useApiKeys';
-import { useWebSocket } from '../../contexts/WebSocketContext';
+import { useStoredProviderCount } from '../../hooks/useCatalogueQuery';
import { AI_PROVIDER_META } from '../icons/AIProviderIcons';
const Divider = () => <div className="mx-1 h-6 w-px bg-border" />;
@@ -112,11 +112,13 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
// Global Model Selector state
const { getValidatedAiProviders, saveGlobalModel, isConnected: apiKeysConnected } = useApiKeys();
- const { apiKeyStatuses } = useWebSocket();
const [globalModelState, setGlobalModelState] = useState<GlobalModelState>({ providers: [], global_provider: null, global_model: null });
- // Re-fetch validated providers when API key statuses change
- const apiKeyCount = Object.values(apiKeyStatuses).filter(s => s.hasKey).length;
+ // Re-fetch validated providers whenever the count of stored
+ // credentials changes. Read from the catalogue (single source of
+ // truth — `provider.stored` flag); the retired
+ // `apiKeyStatuses[id].hasKey` mirror duplicated this answer.
+ const apiKeyCount = useStoredProviderCount();
useEffect(() => {
if (!apiKeysConnected) return;
getValidatedAiProviders().then(state => setGlobalModelState(state));
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index 10342be5..21b11c79 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -173,9 +173,17 @@ export interface RateLimitStats {
pause_reason?: string;
}
+/**
+ * In-memory validation-result cache for an API-key provider.
+ *
+ * NOT a "do we have a stored key" answer — that comes from the
+ * `useCatalogueQuery['credentialCatalogue']` `provider.stored` field
+ * (single source of truth, derived from the encrypted DB on each
+ * `get_credential_catalogue` round-trip). Mixing the two flags caused
+ * cross-tab drift; the duplicated `hasKey` field was retired.
+ */
export interface ApiKeyStatus {
valid: boolean;
- hasKey?: boolean;
message?: string;
models?: string[];
timestamp?: number;
@@ -1259,9 +1267,11 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
`init_${provider}`,
);
if (response.hasKey) {
+ // Only seed the validation cache; "is stored" lives on
+ // the catalogue's `provider.stored` flag (single source).
setApiKeyStatuses(prev => ({
...prev,
- [provider]: { hasKey: true, valid: true },
+ [provider]: { valid: true },
}));
}
} catch {
@@ -2089,11 +2099,12 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
models: response.models
};
- // Update apiKeyStatuses on successful validation
+ // Update apiKeyStatuses on successful validation. "is stored"
+ // lives on catalogue.provider.stored — don't duplicate it here.
if (result.valid) {
setApiKeyStatuses(prev => ({
...prev,
- [provider]: { hasKey: true, valid: true, models: result.models }
+ [provider]: { valid: true, models: result.models }
}));
}
@@ -2121,12 +2132,13 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
models: response.models,
};
- // Mirror into apiKeyStatuses so consumers reading from context
- // stay in sync without an extra round-trip.
+ // Mirror models into apiKeyStatuses so consumers reading from
+ // context stay in sync without an extra round-trip. "is stored"
+ // lives on the catalogue's provider.stored, not here.
if (result.hasKey) {
setApiKeyStatuses(prev => ({
...prev,
- [provider]: { hasKey: true, valid: true, models: result.models }
+ [provider]: { valid: true, models: result.models }
}));
}
@@ -2150,11 +2162,14 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
});
const success = response.success !== false;
- // Update apiKeyStatuses on successful save
+ // Update apiKeyStatuses on successful save. The 'valid: true'
+ // here is optimistic — save_api_key doesn't actually validate
+ // upstream. Catalogue refetch (via the credential.api_key.saved
+ // broadcast) is the truthful source for "is stored".
if (success) {
setApiKeyStatuses(prev => ({
...prev,
- [provider]: { hasKey: true, valid: true, models }
+ [provider]: { valid: true, models }
}));
}
diff --git a/client/src/hooks/useCatalogueQuery.ts b/client/src/hooks/useCatalogueQuery.ts
index f686c07e..3b092656 100644
--- a/client/src/hooks/useCatalogueQuery.ts
+++ b/client/src/hooks/useCatalogueQuery.ts
@@ -339,3 +339,30 @@ export function useCatalogueQuery(): UseCatalogueQueryResult {
return useMemo(() => ({ ...query, refresh }), [query, refresh]);
}
+
+/**
+ * Single-provider "is stored?" selector derived from the catalogue.
+ *
+ * Replaces the retired ``apiKeyStatuses[id].hasKey`` duplication —
+ * `provider.stored` on the server-driven catalogue is the canonical
+ * answer (computed from `auth_service.has_valid_key` on every catalogue
+ * read). Consumers re-render only when this provider's `stored` flag
+ * actually flips, not on every credential mutation, because TanStack
+ * Query produces a new array reference per refetch and React's
+ * referential-equality short-circuits the boolean derivation.
+ */
+export function useProviderStored(providerId: string | null | undefined): boolean {
+ const { data } = useCatalogueQuery();
+ if (!providerId || !data?.providers) return false;
+ return Boolean(data.providers.find((p) => p.id === providerId)?.stored);
+}
+
+/**
+ * Count of providers with a stored credential (any kind — API key,
+ * OAuth, paired QR). Read from the catalogue's `stored` flag.
+ */
+export function useStoredProviderCount(): number {
+ const { data } = useCatalogueQuery();
+ if (!data?.providers) return 0;
+ return data.providers.filter((p) => p.stored).length;
+}
diff --git a/client/src/lib/queryPersist.ts b/client/src/lib/queryPersist.ts
index cd1d61f8..aad8493a 100644
--- a/client/src/lib/queryPersist.ts
+++ b/client/src/lib/queryPersist.ts
@@ -62,11 +62,19 @@ export const queryPersistMaxAge = 24 * 60 * 60 * 1000;
const PERSISTED_KEY_PREFIXES = [
'nodeSpec',
'nodeGroups',
- // credentialValues: panel form fields prefilled across reloads.
- 'credentialValues',
- // credentialCatalogue is NOT here -- it has its own IDB warm-start in
- // useCatalogueQuery.ts (idb-keyval) so localStorage persistence would
- // duplicate the bytes for no benefit. skillContent stays in-memory only.
+ // credentialValues was previously persisted here for the cred-panel
+ // form-field warm-start. Removed per OWASP HTML5 Security Cheat Sheet
+ // / ASVS V9.9: a credentialValues entry contains the decrypted API
+ // key value (populated by getStoredApiKey on the WS roundtrip), and
+ // localStorage stores it in plaintext readable via DevTools on shared
+ // / compromised machines. The in-memory TanStack Query cache (still
+ // gcTime: FOREVER, see queryClient.ts) keeps the form populated for
+ // the session lifetime; on reload the panel refetches — one
+ // roundtrip cost, fine because the modal only opens on user action.
+ //
+ // credentialCatalogue is also NOT here -- it has its own IDB
+ // warm-start in useCatalogueQuery.ts (idb-keyval). skillContent
+ // stays in-memory only.
];
export function shouldPersistQuery(query: Query): boolean {
diff --git a/docs-internal/credentials_encryption.md b/docs-internal/credentials_encryption.md
index 28fbb871..3ebcb9ab 100644
--- a/docs-internal/credentials_encryption.md
+++ b/docs-internal/credentials_encryption.md
@@ -183,6 +183,42 @@ If `API_KEY_ENCRYPTION_KEY` is missing or changed, existing ciphertext becomes u
- **Salt per install**: different MachinaOS installs have different salts, so ciphertext is not portable across installs even with the same server key.
- **Wipes on shutdown**: `EncryptionService.clear()` zeroes the Fernet reference, preventing cold-boot recovery of the derived key.
+## Source of Truth
+
+`CredentialsDatabase` (encrypted SQLite at `credentials.db`) is the **canonical source** for every credential. Two derived in-memory caches exist for performance, both invalidated atomically on every DB write/delete:
+
+- **Backend** (`server/services/auth.py`):
+ - `_api_key_cache: Dict[str, ApiKeyCacheEntry]` keyed by `{session}_{provider}`. Single dataclass entry per provider carries decrypted key + models + `stored_at`. Replaces the previous pair of `_memory_cache` (key) + `_models_cache` (models) which shared the same key shape but had separate write/evict sites — invitation to drift.
+ - `_oauth_cache: Dict[str, Dict]` keyed by `{customer}_{provider}`. Holds **only** access token + display fields (`email`, `name`, `scopes`). Per [RFC 9700](https://datatracker.ietf.org/doc/rfc9700/) (OAuth 2.0 Security BCP, 2024) §5.1 the **refresh token is not memory-cached** — `AuthService.get_oauth_refresh_token(provider, customer_id)` reads from the encrypted DB on every call (rare path; refresh tokens are accessed only at access-token renewal + on revoke / logout).
+
+- **Frontend in-memory**:
+ - `useCatalogueQuery['credentialCatalogue']` — provider list + per-provider `stored: boolean` flag. Single source for "do we have a credential for X?". Replaces the retired `apiKeyStatuses[id].hasKey` mirror that duplicated this answer.
+ - `apiKeyStatuses[id]` — narrowly the **validation result** (`valid`, `models`, `message`, `timestamp`). NOT a duplicate of `provider.stored` — it answers "does the stored key still validate against upstream?".
+
+- **Frontend warm-start**:
+ - IndexedDB key `credentials:catalogue:current` for the provider list (idb-keyval; ~50 ms hydration on return visit).
+ - **localStorage holds NO decrypted credential values.** Per [OWASP HTML5 Security Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/HTML5_Security_Cheat_Sheet.html) and ASVS V9.9, plaintext credentials must not live in `localStorage`. The previous `'credentialValues'` prefix was removed from `PERSISTED_KEY_PREFIXES` in `client/src/lib/queryPersist.ts`. The in-memory TanStack Query cache (`gcTime: ∞`) keeps the form populated for the session lifetime; on reload the panel refetches via WS — one round-trip cost, fine because the modal only opens on user action.
+
+## Broadcast Contract
+
+Every backend handler that mutates credential state MUST emit one or both of:
+
+- **`api_key_status`** — per-provider validation state change. Payload: `{valid, models, message, timestamp}`. Used for validation results and to clear `apiKeyStatuses[provider]` on every connected client (e.g. after `delete_api_key`).
+- **`credential_catalogue_updated`** — refetch signal carrying a CloudEvents v1.0 envelope. Body shape: `WorkflowEvent` from [`server/services/events/envelope.py`](../server/services/events/envelope.py) — same envelope the Wave 12 EventSource framework already uses. CloudEvents `type` follows the convention `credential.<area>.<action>` (e.g. `credential.api_key.saved`, `credential.api_key.deleted`, `credential.oauth.disconnected`). The wire-format outer `type` stays `credential_catalogue_updated` for frontend back-compat; future external interop (EventBridge / Knative) is a JSON-schema swap rather than a rewrite.
+
+Helper: `broadcaster.broadcast_credential_event(event_type, *, provider, customer_id=None)` in `services/status_broadcaster.py` wraps `WorkflowEvent` with `source="machinaos://services/credentials"` and `subject=provider`.
+
+Delete-style mutations emit **both** events. The frontend's `WebSocketContext` handles both and refreshes `apiKeyStatuses` plus the `useCatalogueQuery` cache. The 300 ms debounce in `invalidateCatalogue(queryClient)` (`client/src/hooks/useCatalogueQuery.ts`) coalesces simultaneous events into one refetch.
+
+Pytest invariant `server/tests/credentials/test_credential_broadcasts.py` locks the contract:
+- Each canonical handler (`handle_validate_api_key`, `handle_save_api_key`, `handle_delete_api_key`, `handle_twitter_logout`, `handle_google_logout`) must contain a call to `broadcaster.update_api_key_status(...)` or `broadcaster.broadcast_credential_event(...)`.
+- `delete_api_key` must contain BOTH (clears the in-memory map AND invalidates the catalogue).
+- `AuthService.store_*` / `remove_*` must call `credentials_db.<method>` (canonical) and `_oauth_cache` entries must NOT carry `refresh_token`.
+
+## No Hand-Maintained Frontend Provider Lists
+
+All credential providers come from the backend `get_credential_catalogue` handler (which reads `server/config/credential_providers.json`). The retired `client/src/components/credentials/providers.tsx` static fallback is gone — adding a new provider is a backend-only change. On cold-boot the `CredentialsModal` renders a `<Skeleton>` palette while the WS catalogue arrives; on server-unreachable it shows an explicit error state, never stale fallback data.
+
## Related Docs
- [DESIGN.md](DESIGN.md) - overall security posture
diff --git a/server/routers/twitter.py b/server/routers/twitter.py
index 5cf3981c..6506677f 100644
--- a/server/routers/twitter.py
+++ b/server/routers/twitter.py
@@ -185,7 +185,6 @@ async def get_twitter_status(request: Request):
}
access_token = tokens.get("access_token")
- refresh_token = tokens.get("refresh_token")
# Get stored client credentials (these remain in auth_service as they're app-level)
client_id = await auth_service.get_api_key("twitter_client_id") or ""
@@ -202,7 +201,11 @@ async def get_twitter_status(request: Request):
user_info = await oauth.get_user_info(access_token)
if not user_info.get("success"):
- # Token may be expired, try to refresh
+ # Token may be expired, try to refresh — refresh_token is read
+ # from the DB on demand (RFC 9700; not cached in memory).
+ refresh_token = await auth_service.get_oauth_refresh_token(
+ "twitter", customer_id="owner"
+ )
if refresh_token:
refresh_result = await oauth.refresh_access_token(refresh_token)
if refresh_result.get("success"):
@@ -252,7 +255,10 @@ async def twitter_logout(request: Request):
if tokens:
access_token = tokens.get("access_token")
- refresh_token = tokens.get("refresh_token")
+ # refresh_token is not cached in memory (RFC 9700) — read from DB.
+ refresh_token = await auth_service.get_oauth_refresh_token(
+ "twitter", customer_id="owner"
+ )
# Get client credentials for revocation (app-level, in auth_service)
client_id = await auth_service.get_api_key("twitter_client_id") or ""
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index dcbcf2ec..d17a8ebe 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -61,7 +61,10 @@ MessageHandler = Callable[[Dict[str, Any], WebSocket], Awaitable[Dict[str, Any]]
def ws_handler(*required_fields: str):
"""Simple decorator for WebSocket handlers. Validates required fields and wraps errors."""
+ import functools
+
def decorator(func: MessageHandler) -> MessageHandler:
+ @functools.wraps(func)
async def wrapper(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
for field in required_fields:
if not data.get(field):
@@ -1283,12 +1286,20 @@ async def handle_save_api_key(data: Dict[str, Any], websocket: WebSocket) -> Dic
async def _do_save() -> Dict[str, Any]:
auth_service = container.auth_service()
+ broadcaster = get_status_broadcaster()
await auth_service.store_api_key(
provider=provider,
api_key=data["api_key"].strip(),
models=data.get("models", []),
session_id=data.get("session_id", "default"),
)
+ # Symmetric broadcast: tells every connected client to refetch
+ # the catalogue so the `stored` flag flips on this provider.
+ # Don't claim validity here — save_api_key doesn't validate.
+ await broadcaster.broadcast_credential_event(
+ "credential.api_key.saved",
+ provider=provider,
+ )
return {"provider": data["provider"]}
return await store.run(data.get("request_id"), _do_save)
@@ -1307,7 +1318,20 @@ async def handle_delete_api_key(data: Dict[str, Any], websocket: WebSocket) -> D
async def _do_delete() -> Dict[str, Any]:
auth_service = container.auth_service()
+ broadcaster = get_status_broadcaster()
await auth_service.remove_api_key(provider, data.get("session_id", "default"))
+ # Two broadcasts: api_key_status clears `apiKeyStatuses[provider]`
+ # on every connected client (in-memory validation cache); the
+ # CloudEvents-typed credential.api_key.deleted invalidates the
+ # catalogue so the `stored` flag flips. Both go through the
+ # 300 ms invalidateCatalogue debounce — one refetch.
+ await broadcaster.update_api_key_status(
+ provider, valid=False, has_key=False, message="deleted", models=[],
+ )
+ await broadcaster.broadcast_credential_event(
+ "credential.api_key.deleted",
+ provider=provider,
+ )
return {"provider": data["provider"]}
return await store.run(data.get("request_id"), _do_delete)
@@ -1462,10 +1486,13 @@ async def handle_twitter_logout(data: Dict[str, Any], websocket: WebSocket) -> D
auth_service = container.auth_service()
- # Get tokens from OAuth system
+ # Get tokens from OAuth system. refresh_token is read from DB
+ # directly (RFC 9700; not cached in memory).
tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")
access_token = tokens.get("access_token") if tokens else None
- refresh_token = tokens.get("refresh_token") if tokens else None
+ refresh_token = await auth_service.get_oauth_refresh_token(
+ "twitter", customer_id="owner"
+ ) if tokens else None
# Get client credentials (correctly stored as API keys)
client_id = await auth_service.get_api_key("twitter_client_id") or ""
@@ -1497,6 +1524,15 @@ async def handle_twitter_logout(data: Dict[str, Any], websocket: WebSocket) -> D
except Exception:
pass
+ # Symmetric credential broadcast — flips the catalogue's `stored`
+ # flag for "twitter" on every connected client.
+ broadcaster = get_status_broadcaster()
+ await broadcaster.broadcast_credential_event(
+ "credential.oauth.disconnected",
+ provider="twitter",
+ customer_id="owner",
+ )
+
return {"success": True, "message": "Twitter disconnected"}
@@ -1568,7 +1604,10 @@ async def handle_google_oauth_status(data: Dict[str, Any], websocket: WebSocket)
email = tokens.get("email")
name = tokens.get("name")
- refresh_token = tokens.get("refresh_token")
+ # refresh_token is read from DB directly (RFC 9700; not cached).
+ refresh_token = await auth_service.get_oauth_refresh_token(
+ "google", customer_id="owner"
+ )
# Try to refresh token proactively
try:
@@ -1625,6 +1664,17 @@ async def handle_google_logout(data: Dict[str, Any], websocket: WebSocket) -> Di
"data": {"connected": False, "email": None, "name": None},
})
+ # Symmetric credential broadcast — flips the catalogue's `stored`
+ # flag for "google" on every connected client. The google_status
+ # event above also triggers invalidateCatalogue on the frontend
+ # ([WebSocketContext.tsx:722]) but the credential event keeps the
+ # contract uniform with twitter / telegram disconnect paths.
+ await broadcaster.broadcast_credential_event(
+ "credential.oauth.disconnected",
+ provider="google",
+ customer_id="owner",
+ )
+
return {"success": True, "message": "Google Workspace disconnected"}
diff --git a/server/services/auth.py b/server/services/auth.py
index 938a489f..88b6d1ed 100644
--- a/server/services/auth.py
+++ b/server/services/auth.py
@@ -1,8 +1,32 @@
-"""API key management service with encrypted credentials database."""
+"""API key management service with encrypted credentials database.
+
+Source of truth: ``CredentialsDatabase`` (encrypted SQLite at credentials.db).
+Two derived in-memory caches exist for performance — both keyed by composite
+strings, both invalidated atomically on every DB write/delete:
+
+- ``_api_key_cache``: ``Dict[str, ApiKeyCacheEntry]`` keyed by ``{session}_{provider}``.
+ One entry carries decrypted key + models + stored_at. Replaces the previous
+ pair of ``_memory_cache`` (key) + ``_models_cache`` (models) which shared the
+ same key shape but had separate write/evict sites — invitation to drift.
+- ``_oauth_cache``: ``Dict[str, Dict]`` keyed by ``{customer}_{provider}``.
+ Different namespace, different shape, different lifecycle — kept separate.
+ Per RFC 9700 (OAuth 2.0 BCP 2024) refresh tokens are NOT cached here;
+ ``get_oauth_refresh_token()`` reads from the DB on every call.
+
+Async-safety: every read / write is await-free across the cache check, so
+the GIL guarantees atomicity at the bytecode level — no locks needed. Lazy
+DB fallback inside ``get_api_key`` introduces an await, but the only race
+is "two concurrent reads both miss cache and both fetch from DB" — benign,
+since both writes set the same value.
+
+Neither cache has a TTL today; eviction is on explicit ``remove_*`` or
+``clear_cache()`` (logout). External revocation by an upstream provider is
+not detected.
+"""
import hashlib
-import json
-from pathlib import Path
+import time
+from dataclasses import dataclass, field
from typing import Dict, Any, Optional, List
from core.config import Settings
@@ -14,6 +38,20 @@ from core.logging import get_logger
logger = get_logger(__name__)
+@dataclass
+class ApiKeyCacheEntry:
+ """One in-memory cache entry per ``{session}_{provider}``.
+
+ Carries the decrypted API key, the models list discovered at validation
+ time, and a monotonic timestamp for future TTL support. Single struct
+ so the two values can never drift — one write site, one evict site.
+ """
+
+ key: str
+ models: List[str] = field(default_factory=list)
+ stored_at: float = field(default_factory=time.monotonic)
+
+
class AuthService:
"""API key management service using encrypted credentials database.
@@ -32,11 +70,15 @@ class AuthService:
self.cache = cache # Kept for backward compatibility, not used for API keys
self.database = database # Kept for backward compatibility
self.settings = settings
- # Memory-only cache for decrypted API keys (never persisted to Redis/disk)
- self._memory_cache: Dict[str, str] = {}
- # Memory-only cache for models list
- self._models_cache: Dict[str, List[str]] = {}
- # Memory-only cache for OAuth tokens
+ # Memory-only cache for decrypted API keys + models (never persisted
+ # to Redis/disk). Single entry per provider; one write site, one
+ # evict site — see module docstring.
+ self._api_key_cache: Dict[str, ApiKeyCacheEntry] = {}
+ # Memory-only cache for OAuth tokens. Different key namespace
+ # (``{customer}_{provider}``); kept separate from API keys.
+ # Per RFC 9700 the refresh_token is NOT cached here — only
+ # access_token + display fields. Refresh-token access goes via
+ # ``get_oauth_refresh_token()`` which reads from the DB.
self._oauth_cache: Dict[str, Dict[str, Any]] = {}
def hash_api_key(self, api_key: str) -> str:
@@ -66,7 +108,7 @@ class AuthService:
logger.info(f"Storing API key for provider: {provider}, session: {session_id}")
- # Store in encrypted credentials database
+ # 1. DB write first (canonical source).
await self.credentials_db.save_api_key(
provider=provider,
api_key=api_key,
@@ -74,9 +116,12 @@ class AuthService:
session_id=session_id
)
- # Cache decrypted key in memory only (for quick access)
- self._memory_cache[cache_key] = api_key
- self._models_cache[cache_key] = models
+ # 2. Cache update only after DB write succeeds. One entry,
+ # not two parallel dicts — see ApiKeyCacheEntry docstring.
+ self._api_key_cache[cache_key] = ApiKeyCacheEntry(
+ key=api_key,
+ models=list(models),
+ )
logger.info(f"Stored and cached API key for {provider}")
return True
@@ -100,15 +145,23 @@ class AuthService:
try:
cache_key = f"{session_id}_{provider}"
- # Check memory cache first (fastest, most secure)
- if cache_key in self._memory_cache:
- return self._memory_cache[cache_key]
+ # Check memory cache first (fastest, most secure).
+ entry = self._api_key_cache.get(cache_key)
+ if entry is not None:
+ return entry.key
- # Fallback to encrypted database
+ # Fallback to encrypted database. The lazy fetch also pulls
+ # the models list so we populate the cache entry fully — no
+ # second roundtrip on the next get_stored_models() call.
api_key = await self.credentials_db.get_api_key(provider, session_id)
if api_key:
- # Cache in memory for quick subsequent access
- self._memory_cache[cache_key] = api_key
+ models = await self.credentials_db.get_api_key_models(
+ provider, session_id
+ ) or []
+ self._api_key_cache[cache_key] = ApiKeyCacheEntry(
+ key=api_key,
+ models=models,
+ )
return api_key
return None
@@ -131,13 +184,21 @@ class AuthService:
cache_key = f"{session_id}_{provider}"
# Check memory cache first
- if cache_key in self._models_cache:
- return self._models_cache[cache_key]
+ entry = self._api_key_cache.get(cache_key)
+ if entry is not None:
+ return entry.models
- # Fallback to encrypted database
+ # Fallback to encrypted database. Pull the key alongside so
+ # the cache entry is populated fully — symmetric with
+ # get_api_key()'s lazy-populate path.
models = await self.credentials_db.get_api_key_models(provider, session_id)
if models:
- self._models_cache[cache_key] = models
+ api_key = await self.credentials_db.get_api_key(provider, session_id)
+ if api_key:
+ self._api_key_cache[cache_key] = ApiKeyCacheEntry(
+ key=api_key,
+ models=list(models),
+ )
return models
return []
@@ -159,13 +220,12 @@ class AuthService:
try:
cache_key = f"{session_id}_{provider}"
- # Remove from memory cache
- self._memory_cache.pop(cache_key, None)
- self._models_cache.pop(cache_key, None)
-
- # Remove from encrypted database
+ # 1. DB delete first (canonical source).
await self.credentials_db.delete_api_key(provider, session_id)
+ # 2. Cache evict only after DB succeeds. One pop, not two.
+ self._api_key_cache.pop(cache_key, None)
+
logger.info(f"Removed API key for {provider}")
return True
@@ -192,8 +252,7 @@ class AuthService:
Should be called on user logout to ensure decrypted keys
don't persist in memory longer than necessary.
"""
- self._memory_cache.clear()
- self._models_cache.clear()
+ self._api_key_cache.clear()
self._oauth_cache.clear()
logger.debug("Cleared all credential memory caches")
@@ -226,6 +285,7 @@ class AuthService:
try:
cache_key = f"{customer_id}_{provider}"
+ # 1. DB write first (canonical source).
await self.credentials_db.save_oauth_tokens(
provider=provider,
access_token=access_token,
@@ -236,13 +296,15 @@ class AuthService:
customer_id=customer_id
)
- # Cache in memory
+ # 2. Cache only the access token + display fields. Refresh
+ # tokens are long-lived secrets per RFC 9700 (OAuth 2.0
+ # BCP 2024) and are NOT cached in memory — readers go
+ # through ``get_oauth_refresh_token()`` which hits the DB.
self._oauth_cache[cache_key] = {
"access_token": access_token,
- "refresh_token": refresh_token,
"email": email,
"name": name,
- "scopes": scopes
+ "scopes": scopes,
}
logger.info(f"Stored OAuth tokens for {provider}")
@@ -257,27 +319,42 @@ class AuthService:
provider: str,
customer_id: str = "owner"
) -> Optional[Dict[str, Any]]:
- """Get OAuth tokens from cache or encrypted database.
+ """Get OAuth display tokens from cache or encrypted database.
+
+ Returns ``{access_token, email, name, scopes}`` only — the
+ refresh token is intentionally NOT included. Callers that need
+ the refresh token (token-refresh + revoke flows) must call
+ :meth:`get_oauth_refresh_token` explicitly. This split implements
+ RFC 9700 (OAuth 2.0 BCP 2024) §5.1 — refresh tokens are
+ long-lived secrets and must not live in process memory.
Args:
provider: OAuth provider name
customer_id: Customer identifier
Returns:
- Dict with access_token, refresh_token, email, name, scopes or None
+ Dict with ``access_token``, ``email``, ``name``, ``scopes`` or
+ ``None`` if no tokens are stored.
"""
try:
cache_key = f"{customer_id}_{provider}"
- # Check memory cache first
+ # Check memory cache first.
if cache_key in self._oauth_cache:
return self._oauth_cache[cache_key]
- # Fallback to encrypted database
+ # Fallback to encrypted database. Strip refresh_token before
+ # caching so the in-memory copy is short-lived-display-only.
tokens = await self.credentials_db.get_oauth_tokens(provider, customer_id)
if tokens:
- self._oauth_cache[cache_key] = tokens
- return tokens
+ display = {
+ "access_token": tokens.get("access_token"),
+ "email": tokens.get("email"),
+ "name": tokens.get("name"),
+ "scopes": tokens.get("scopes"),
+ }
+ self._oauth_cache[cache_key] = display
+ return display
return None
@@ -285,6 +362,38 @@ class AuthService:
logger.error("Failed to get OAuth tokens", provider=provider, error=str(e))
return None
+ async def get_oauth_refresh_token(
+ self,
+ provider: str,
+ customer_id: str = "owner",
+ ) -> Optional[str]:
+ """Read the OAuth refresh token directly from the encrypted DB.
+
+ Per RFC 9700 (OAuth 2.0 BCP 2024) §5.1 the refresh token is a
+ long-lived bearer secret and must not be cached in process
+ memory. Every call decrypts from disk; this is acceptable
+ because refresh tokens are accessed rarely (only at access-token
+ renewal + on revoke / logout).
+
+ Args:
+ provider: OAuth provider name
+ customer_id: Customer identifier
+
+ Returns:
+ The decrypted refresh token, or ``None`` if no tokens are
+ stored for ``(provider, customer_id)``.
+ """
+ try:
+ tokens = await self.credentials_db.get_oauth_tokens(provider, customer_id)
+ if tokens:
+ return tokens.get("refresh_token")
+ return None
+ except Exception as e:
+ logger.error(
+ "Failed to get OAuth refresh token", provider=provider, error=str(e)
+ )
+ return None
+
async def refresh_oauth_tokens_with_breaker(
self,
provider: str,
diff --git a/server/services/handlers/google_auth.py b/server/services/handlers/google_auth.py
index 93f093af..71cfd2d0 100644
--- a/server/services/handlers/google_auth.py
+++ b/server/services/handlers/google_auth.py
@@ -70,7 +70,10 @@ async def get_google_credentials(
)
access_token = tokens["access_token"]
- refresh_token = tokens.get("refresh_token")
+ # refresh_token is read from DB directly (RFC 9700; not cached).
+ refresh_token = await auth_service.get_oauth_refresh_token(
+ "google", customer_id="owner"
+ )
auth_service = container.auth_service()
client_id = await auth_service.get_api_key("google_client_id") or ""
diff --git a/server/services/status_broadcaster.py b/server/services/status_broadcaster.py
index 48a02620..777ec9cc 100644
--- a/server/services/status_broadcaster.py
+++ b/server/services/status_broadcaster.py
@@ -230,6 +230,54 @@ class StatusBroadcaster:
"""Get API key validation status for a provider."""
return self._status["api_keys"].get(provider)
+ # =========================================================================
+ # Credential Mutation Broadcasts (CloudEvents v1.0)
+ # =========================================================================
+ #
+ # Every backend handler that mutates credential state (store / remove API
+ # keys, OAuth login / logout) MUST emit a credential event via this helper
+ # so the frontend `useCatalogueQuery` cache stays coherent across all
+ # connected clients. Wire-format key stays `credential_catalogue_updated`
+ # for FE back-compat; the body is the CloudEvents-shaped `WorkflowEvent`
+ # so future EventBridge / Knative interop is a JSON-schema swap.
+ #
+ # Pytest invariant `test_credential_broadcasts.py` locks the contract.
+
+ async def broadcast_credential_event(
+ self,
+ event_type: str,
+ *,
+ provider: str,
+ customer_id: Optional[str] = None,
+ ) -> None:
+ """Emit a CloudEvents-typed credential-mutation broadcast.
+
+ Args:
+ event_type: CloudEvents `type` field. Convention:
+ ``"credential.api_key.saved"`` / ``".deleted"``,
+ ``"credential.oauth.disconnected"``.
+ provider: Provider id (e.g. ``"openai"``, ``"twitter"``).
+ customer_id: For multi-tenant OAuth flows. Default omitted.
+ """
+ # Local import keeps the broadcaster module independent of the
+ # event framework's load order during startup.
+ from services.events import WorkflowEvent
+
+ event = WorkflowEvent(
+ source="machinaos://services/credentials",
+ type=event_type,
+ subject=provider,
+ data={
+ "provider": provider,
+ **({"customer_id": customer_id} if customer_id else {}),
+ },
+ )
+
+ await self.broadcast({
+ "type": "credential_catalogue_updated",
+ "data": event.model_dump(mode="json"),
+ })
+
# =========================================================================
# Android Status Updates
# =========================================================================
diff --git a/server/tests/credentials/test_auth_service.py b/server/tests/credentials/test_auth_service.py
index ded216af..57a29a67 100644
--- a/server/tests/credentials/test_auth_service.py
+++ b/server/tests/credentials/test_auth_service.py
@@ -3,7 +3,9 @@
Locks in invariants 7 and 8 from docs-internal/credentials_panel.md:
- store_api_key requires models=[]
- Memory cache hit path does not query DB on every call
- - clear_cache wipes _memory_cache, _models_cache, _oauth_cache
+ - clear_cache wipes _api_key_cache, _oauth_cache (one merged cache
+ per RFC 9700 + post-Wave-12 dedup; previously the API-key half was
+ split across _memory_cache + _models_cache).
"""
from __future__ import annotations
@@ -91,8 +93,7 @@ class TestMemoryCache:
await auth_service.store_api_key("openai", "sk-foo", models=[])
# Wipe memory cache to simulate fresh process
- auth_service._memory_cache.clear()
- auth_service._models_cache.clear()
+ auth_service._api_key_cache.clear()
call_count = {"n": 0}
original = auth_service.credentials_db.get_api_key
@@ -111,19 +112,17 @@ class TestMemoryCache:
assert await auth_service.get_api_key("openai") == "sk-foo"
assert call_count["n"] == 1
- async def test_clear_cache_wipes_all_three_caches(self, auth_service):
+ async def test_clear_cache_wipes_both_caches(self, auth_service):
await auth_service.store_api_key("openai", "sk-foo", models=["gpt-4"])
await auth_service.store_oauth_tokens("google", "a", "r", email="[email protected]")
# Sanity: caches populated
- assert auth_service._memory_cache
- assert auth_service._models_cache
+ assert auth_service._api_key_cache
assert auth_service._oauth_cache
auth_service.clear_cache()
- assert auth_service._memory_cache == {}
- assert auth_service._models_cache == {}
+ assert auth_service._api_key_cache == {}
assert auth_service._oauth_cache == {}
async def test_clear_cache_does_not_delete_db_data(self, auth_service):
@@ -149,10 +148,18 @@ class TestOAuthOps:
tokens = await auth_service.get_oauth_tokens("google")
assert tokens is not None
assert tokens["access_token"] == "access-1"
- assert tokens["refresh_token"] == "refresh-1"
+ # Per RFC 9700 the refresh token is NOT returned by
+ # get_oauth_tokens — read via get_oauth_refresh_token() instead.
+ assert "refresh_token" not in tokens
assert tokens["email"] == "[email protected]"
assert tokens["scopes"] == "openid email"
+ # The refresh token is reachable through the dedicated helper
+ # that always reads from the encrypted DB (no in-memory cache).
+ assert (
+ await auth_service.get_oauth_refresh_token("google") == "refresh-1"
+ )
+
async def test_remove_oauth_tokens(self, auth_service):
await auth_service.store_oauth_tokens("google", "a", "r")
ok = await auth_service.remove_oauth_tokens("google")
diff --git a/server/tests/credentials/test_credential_broadcasts.py b/server/tests/credentials/test_credential_broadcasts.py
new file mode 100644
index 00000000..0624ed65
--- /dev/null
+++ b/server/tests/credentials/test_credential_broadcasts.py
@@ -0,0 +1,209 @@
+"""Pytest invariant: every credential-mutation handler MUST broadcast.
+
+The frontend's `useCatalogueQuery` cache + `apiKeyStatuses` map both go
+stale unless the backend explicitly invalidates via:
+
+- ``broadcaster.update_api_key_status(...)`` — per-provider validation
+ state change (carries the validation result payload).
+- ``broadcaster.broadcast_credential_event("credential.*", ...)`` —
+ CloudEvents-typed mutation (refetch signal). Wraps ``WorkflowEvent``
+ from ``services.events.envelope``.
+
+Cross-tab visibility breaks the moment a handler omits both. This test
+locks the contract by reading each handler's source via
+``inspect.getsource`` and asserting it contains at least one of the two
+broadcast call patterns. New credential mutations that forget to
+broadcast fail CI before they ship.
+
+Companion: ``test_auth_service.py`` locks the DB-write-then-cache-update
+ordering inside AuthService.
+"""
+
+from __future__ import annotations
+
+import inspect
+import re
+
+import pytest
+
+
+pytestmark = pytest.mark.credentials
+
+
+# Match either:
+# broadcaster.update_api_key_status(...)
+# broadcaster.broadcast_credential_event("credential.<anything>", ...)
+# Anchor on `.update_api_key_status(` or `.broadcast_credential_event(`
+# so renames force a test update (intentional).
+_BROADCAST_PATTERN = re.compile(
+ r"\.(?:update_api_key_status|broadcast_credential_event)\s*\("
+)
+
+
+def _handler_source(handler) -> str:
+ """Return the unwrapped source of a handler decorated by `@ws_handler`."""
+ fn = handler
+ while hasattr(fn, "__wrapped__"):
+ fn = fn.__wrapped__
+ return inspect.getsource(fn)
+
+
+class TestCredentialMutationHandlersBroadcast:
+ """Every handler that mutates credential state MUST broadcast.
+
+ Failing this test means a frontend cache will go stale across tabs
+ after the mutation; users will see the wrong stored / connected
+ state until they manually refresh.
+ """
+
+ def test_validate_api_key_broadcasts(self):
+ """Sanity — already correct path; confirms the matcher works."""
+ from routers import websocket as ws_module
+
+ src = _handler_source(ws_module.handle_validate_api_key)
+ assert _BROADCAST_PATTERN.search(src), (
+ "handle_validate_api_key must broadcast api_key_status "
+ "or credential_catalogue_updated"
+ )
+
+ def test_save_api_key_broadcasts(self):
+ from routers import websocket as ws_module
+
+ src = _handler_source(ws_module.handle_save_api_key)
+ assert _BROADCAST_PATTERN.search(src), (
+ "handle_save_api_key must broadcast credential.api_key.saved "
+ "via broadcast_credential_event so the catalogue's stored "
+ "flag flips on every connected client"
+ )
+
+ def test_delete_api_key_broadcasts(self):
+ from routers import websocket as ws_module
+
+ src = _handler_source(ws_module.handle_delete_api_key)
+ # Delete must trigger BOTH broadcasts: api_key_status (clears
+ # apiKeyStatuses) + credential.api_key.deleted (catalogue refetch).
+ matches = _BROADCAST_PATTERN.findall(src)
+ assert len(matches) >= 2, (
+ "handle_delete_api_key must broadcast both update_api_key_status "
+ "(clears apiKeyStatuses[provider]) AND broadcast_credential_event "
+ "('credential.api_key.deleted') (catalogue refetch). "
+ f"Found {len(matches)} broadcast call(s)."
+ )
+
+ def test_twitter_logout_broadcasts(self):
+ from routers import websocket as ws_module
+
+ src = _handler_source(ws_module.handle_twitter_logout)
+ assert _BROADCAST_PATTERN.search(src), (
+ "handle_twitter_logout must broadcast credential.oauth.disconnected"
+ )
+
+ def test_google_logout_broadcasts(self):
+ from routers import websocket as ws_module
+
+ src = _handler_source(ws_module.handle_google_logout)
+ assert _BROADCAST_PATTERN.search(src), (
+ "handle_google_logout must broadcast credential.oauth.disconnected"
+ )
+
+
+class TestCredentialEventCloudEventsShape:
+ """The credential broadcast helper wraps `WorkflowEvent` so the
+ on-the-wire body is a CloudEvents v1.0 envelope. Locks the spec
+ fields so future refactors don't quietly drop required keys.
+ """
+
+ @pytest.fixture
+ def envelope(self):
+ from services.events.envelope import WorkflowEvent
+
+ event = WorkflowEvent(
+ source="machinaos://services/credentials",
+ type="credential.api_key.saved",
+ subject="openai",
+ data={"provider": "openai"},
+ )
+ return event.model_dump(mode="json")
+
+ def test_specversion_pinned_to_1(self, envelope):
+ assert envelope["specversion"] == "1.0"
+
+ def test_required_fields_present(self, envelope):
+ # CloudEvents 1.0 mandatory: id, source, specversion, type
+ for field in ("id", "source", "specversion", "type"):
+ assert envelope.get(field), f"missing required field: {field}"
+
+ def test_credential_subject_is_provider(self, envelope):
+ assert envelope["subject"] == "openai"
+
+ def test_event_type_namespaced(self, envelope):
+ # Convention: credential.<area>.<action>
+ assert envelope["type"].startswith("credential."), envelope["type"]
+
+
+class TestAuthServiceDbCanonicalInvariant:
+ """AuthService.store_*/remove_* must do the DB call before touching
+ the in-memory cache. Reverse order leaves a stale-cache window.
+
+ Static-source check; doesn't run the methods.
+ """
+
+ @pytest.fixture
+ def auth_source(self):
+ from services import auth as auth_module
+
+ return inspect.getsource(auth_module.AuthService)
+
+ @pytest.mark.parametrize(
+ "method_name,db_call",
+ [
+ ("store_api_key", "credentials_db.save_api_key"),
+ ("remove_api_key", "credentials_db.delete_api_key"),
+ ("store_oauth_tokens", "credentials_db.save_oauth_tokens"),
+ ("remove_oauth_tokens", "credentials_db.delete_oauth_tokens"),
+ ],
+ )
+ def test_method_calls_db(self, auth_source, method_name, db_call):
+ """Every store_*/remove_* method must call the canonical DB
+ method. Invariant breaks if a refactor accidentally bypasses
+ the DB and only updates the cache."""
+ # Find the method body in the AuthService source. Anchor end on
+ # the next method/class start OR end-of-string (the last method
+ # in the class has no trailing sibling).
+ method_re = re.compile(
+ rf"async def {method_name}\b.*?(?=\n async def |\n def |\nclass |\Z)",
+ re.DOTALL,
+ )
+ match = method_re.search(auth_source)
+ assert match, f"AuthService.{method_name} not found in source"
+ body = match.group(0)
+ assert db_call in body, (
+ f"AuthService.{method_name} must call self.{db_call}() "
+ f"before touching the in-memory cache (DB is canonical "
+ f"source of truth)"
+ )
+
+ def test_oauth_cache_does_not_carry_refresh_token(self, auth_source):
+ """Per RFC 9700, refresh tokens must not live in process memory.
+ store_oauth_tokens caches only the access token + display fields.
+ """
+ match = re.search(
+ r"async def store_oauth_tokens\b.*?(?=\n async def |\n def |\nclass )",
+ auth_source,
+ re.DOTALL,
+ )
+ assert match, "AuthService.store_oauth_tokens not found"
+ body = match.group(0)
+
+ # Find the cache-write block. It assigns to self._oauth_cache[...].
+ cache_block = re.search(
+ r"self\._oauth_cache\[\w+\]\s*=\s*\{[^}]+\}", body, re.DOTALL
+ )
+ assert cache_block, (
+ "AuthService.store_oauth_tokens must populate _oauth_cache"
+ )
+ assert '"refresh_token"' not in cache_block.group(0), (
+ "RFC 9700 violation: _oauth_cache entry must not carry "
+ "refresh_token. Use get_oauth_refresh_token() helper that "
+ "always reads from the encrypted DB."
+ )
diff --git a/server/tests/credentials/test_websocket_handlers.py b/server/tests/credentials/test_websocket_handlers.py
index 2f67a318..b6e2f27c 100644
--- a/server/tests/credentials/test_websocket_handlers.py
+++ b/server/tests/credentials/test_websocket_handlers.py
@@ -60,6 +60,10 @@ def patched_container(monkeypatch, auth_service):
# not on services.status_broadcaster (the bound name in the handler module).
fake_broadcaster = MagicMock()
fake_broadcaster.update_api_key_status = AsyncMock()
+ # Wave-12 credential broadcast helper used by save / delete /
+ # oauth-logout handlers. Wraps WorkflowEvent (CloudEvents v1.0)
+ # and broadcasts as `credential_catalogue_updated`.
+ fake_broadcaster.broadcast_credential_event = AsyncMock()
fake_broadcaster.broadcast = AsyncMock()
fake_broadcaster._status = {}
monkeypatch.setattr(ws_module, "get_status_broadcaster", lambda: fake_broadcaster)