Skip to content

Self-improving AI Employees for everyone

feat(workflow): pre-execute validator + credential runtime CloudEvent + import-time missing-credential prompt

trohitg7dcaebad9b9b
diff --git a/client/src/Dashboard.tsx b/client/src/Dashboard.tsx
index 80361760..c16f5345 100644
--- a/client/src/Dashboard.tsx
+++ b/client/src/Dashboard.tsx
@@ -57,6 +57,7 @@ import {
generateWorkflowId
} from './utils/workflow';
import { importWorkflowFromFile } from './utils/workflowExport';
+import { CATALOGUE_QUERY_KEY, type CatalogueResponse } from './hooks/useCatalogueQuery';
import { buildCanvasStyles } from './styles/canvasAnimations';
import 'reactflow/dist/style.css';
@@ -804,6 +805,31 @@ const DashboardContent: React.FC = () => {
const importedWorkflow = await importWorkflowFromFile(file);
+ // Cross-reference the export's `requirements.credentials` against
+ // the catalogue's `provider.stored` flag. Warn the user about
+ // missing credentials before saving — they can import anyway and
+ // configure later, or cancel. Older exports without `requirements`
+ // skip this step transparently.
+ const reqCreds = importedWorkflow.requirements?.credentials ?? [];
+ if (reqCreds.length > 0) {
+ const catalogue = queryClient.getQueryData<CatalogueResponse>(CATALOGUE_QUERY_KEY);
+ const providers = catalogue?.providers ?? [];
+ const missing = reqCreds
+ .map(need => providers.find(p => p.id === need.provider_id))
+ .filter((p): p is NonNullable<typeof p> => !!p && p.stored !== true)
+ .map(p => `- ${p.name} (${p.id})`);
+ if (missing.length > 0) {
+ const proceed = window.confirm(
+ `This workflow needs ${missing.length} credential${missing.length === 1 ? '' : 's'} ` +
+ `that ${missing.length === 1 ? 'is' : 'are'} not configured:\n\n` +
+ `${missing.join('\n')}\n\n` +
+ `You can import it anyway and configure ${missing.length === 1 ? 'it' : 'them'} later. ` +
+ `Click OK to import, or Cancel to abort.`,
+ );
+ if (!proceed) return;
+ }
+ }
+
// Check for name conflict with existing workflows
const existingNames = savedWorkflows.map(w => w.name.toLowerCase());
let finalName = importedWorkflow.name;
diff --git a/client/src/hooks/useWorkflowValidation.ts b/client/src/hooks/useWorkflowValidation.ts
new file mode 100644
index 00000000..17de2051
--- /dev/null
+++ b/client/src/hooks/useWorkflowValidation.ts
@@ -0,0 +1,142 @@
+/**
+ * useWorkflowValidation — debounced workflow validation against the backend.
+ *
+ * Calls the `validate_workflow` WebSocket handler (server/services/workflow_validator.py)
+ * whenever `nodes` or `edges` change, with a 500ms trailing-edge debounce.
+ * Returns the latest `{errors, warnings}` report for editor live-lint.
+ *
+ * The same WS handler is reused at execute-time gating
+ * (`handle_execute_workflow` with `force=false`) and at import-time
+ * dry-run, so the editor preview matches what the runtime gate sees.
+ */
+
+import { useEffect, useRef, useState } from 'react';
+import type { Node, Edge } from 'reactflow';
+import { useWebSocket } from '../contexts/WebSocketContext';
+
+export interface ValidationIssue {
+ code:
+ | 'CYCLE'
+ | 'DANGLING_EDGE'
+ | 'UNKNOWN_NODE_TYPE'
+ | 'INVALID_PARAM'
+ | 'MISSING_CREDENTIAL';
+ node_id: string | null;
+ message: string;
+ node_type?: string;
+ provider_id?: string;
+ remediation?: 'add_key' | 'reconnect';
+ path?: Array<string | number>;
+ nodes?: string[];
+}
+
+export interface ValidationReport {
+ errors: ValidationIssue[];
+ warnings: ValidationIssue[];
+}
+
+const EMPTY_REPORT: ValidationReport = { errors: [], warnings: [] };
+
+const DEBOUNCE_MS = 500;
+
+export interface UseWorkflowValidationResult {
+ report: ValidationReport;
+ isValidating: boolean;
+ /** Manually trigger a validate (skips the debounce). Returns the fresh report. */
+ validate: () => Promise<ValidationReport>;
+}
+
+/**
+ * Validate the workflow on every change to `nodes` / `edges`.
+ *
+ * @param nodes ReactFlow nodes
+ * @param edges ReactFlow edges
+ * @param parametersById Optional map of node_id -> parameters for INVALID_PARAM
+ * detection. The frontend can pass the parameter-panel's in-flight state so
+ * the lint reflects unsaved edits; if omitted, the backend falls back to
+ * `node.data.parameters` (rarely populated since parameters live in the DB).
+ * @param enabled Set false to suspend validation (e.g. during workflow load).
+ */
+export function useWorkflowValidation(
+ nodes: Node[],
+ edges: Edge[],
+ parametersById?: Record<string, Record<string, any>>,
+ enabled: boolean = true,
+): UseWorkflowValidationResult {
+ const { sendRequest, isReady } = useWebSocket();
+ const [report, setReport] = useState<ValidationReport>(EMPTY_REPORT);
+ const [isValidating, setIsValidating] = useState(false);
+
+ // Latest-write-wins guard so a slow in-flight request doesn't overwrite a
+ // newer report with a stale one.
+ const requestSeqRef = useRef(0);
+
+ const runValidate = useRef(async (): Promise<ValidationReport> => EMPTY_REPORT);
+ runValidate.current = async () => {
+ if (!isReady || !enabled) return EMPTY_REPORT;
+ const seq = ++requestSeqRef.current;
+ setIsValidating(true);
+ try {
+ const result = await sendRequest<{ success: boolean; report?: ValidationReport }>(
+ 'validate_workflow',
+ {
+ nodes: nodes.map(n => ({
+ id: n.id,
+ type: n.type ?? '',
+ data: n.data ?? {},
+ })),
+ edges: edges.map(e => ({
+ id: e.id,
+ source: e.source,
+ target: e.target,
+ sourceHandle: e.sourceHandle ?? undefined,
+ targetHandle: e.targetHandle ?? undefined,
+ })),
+ parameters_by_id: parametersById,
+ },
+ );
+ const fresh = result?.report ?? EMPTY_REPORT;
+ // Drop the response if a newer request has already started.
+ if (seq === requestSeqRef.current) {
+ setReport(fresh);
+ }
+ return fresh;
+ } catch (err) {
+ // Surface as an error issue so the editor doesn't silently mask the
+ // backend round-trip failing.
+ const errorReport: ValidationReport = {
+ errors: [
+ {
+ code: 'UNKNOWN_NODE_TYPE',
+ node_id: null,
+ message: `Validation request failed: ${err instanceof Error ? err.message : String(err)}`,
+ },
+ ],
+ warnings: [],
+ };
+ if (seq === requestSeqRef.current) {
+ setReport(errorReport);
+ }
+ return errorReport;
+ } finally {
+ if (seq === requestSeqRef.current) {
+ setIsValidating(false);
+ }
+ }
+ };
+
+ useEffect(() => {
+ if (!enabled || !isReady) return;
+ const timer = setTimeout(() => {
+ void runValidate.current();
+ }, DEBOUNCE_MS);
+ return () => clearTimeout(timer);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [nodes, edges, parametersById, enabled, isReady]);
+
+ return {
+ report,
+ isValidating,
+ validate: () => runValidate.current(),
+ };
+}
diff --git a/client/src/utils/workflowExport.ts b/client/src/utils/workflowExport.ts
index 8ad8bee0..7c53e7b3 100644
--- a/client/src/utils/workflowExport.ts
+++ b/client/src/utils/workflowExport.ts
@@ -9,6 +9,7 @@ import { Node } from 'reactflow';
import { WorkflowData } from '../store/useAppStore';
import { validateWorkflow, serializeWorkflow, deserializeWorkflow } from '../schemas/workflowSchema';
import { sanitizeParameters } from './parameterSanitizer';
+import { getCachedNodeSpec } from '../lib/nodeSpec';
// Injected by Vite define from root package.json (see vite.config.js)
declare const __APP_VERSION__: string;
const APP_VERSION = typeof __APP_VERSION__ !== 'undefined' ? __APP_VERSION__ : '0.0.0';
@@ -17,9 +18,51 @@ const APP_VERSION = typeof __APP_VERSION__ !== 'undefined' ? __APP_VERSION__ : '
// that belongs in the DB and should not leak into exported JSON files.
export const UI_DATA_FIELDS = new Set(['label', 'disabled', 'condition']);
+/**
+ * Declarative manifest of what an exported workflow needs to run on the
+ * target instance. Populated from each node's cached NodeSpec at export time;
+ * read by the import flow to surface a "missing credentials / unknown
+ * plugin" prompt before the workflow is saved.
+ *
+ * - `credentials`: the set of credential provider IDs the workflow references
+ * (from each `NodeSpec.credentials`). Carries IDs only — values are stripped
+ * by `parameterSanitizer`.
+ * - `nodes`: the plugin types and versions the workflow was authored against.
+ * Used informationally for drift detection; not enforced.
+ *
+ * Older exports without this field import cleanly — its presence is the
+ * discriminator (no `format_version` bump needed).
+ */
+export interface RequirementsManifest {
+ credentials: Array<{ provider_id: string }>;
+ nodes: Array<{ type: string; version: number }>;
+}
+
/** Workflow data with optional embedded node parameters (from new-format exports). */
export interface ImportedWorkflow extends WorkflowData {
nodeParameters?: Record<string, Record<string, any>>;
+ requirements?: RequirementsManifest;
+}
+
+/**
+ * Build the per-export `requirements` manifest by walking each node's
+ * cached NodeSpec. Returns undefined if no metadata is available
+ * (cold-start before specs warm — rare).
+ */
+function buildRequirements(nodes: Node[]): RequirementsManifest | undefined {
+ const credSet = new Set<string>();
+ const nodeReqs: Array<{ type: string; version: number }> = [];
+ for (const n of nodes) {
+ const spec = getCachedNodeSpec(n.type ?? '');
+ if (!spec) continue;
+ nodeReqs.push({ type: spec.type, version: spec.version ?? 1 });
+ for (const credId of spec.credentials ?? []) credSet.add(credId);
+ }
+ if (credSet.size === 0 && nodeReqs.length === 0) return undefined;
+ return {
+ credentials: [...credSet].map(provider_id => ({ provider_id })),
+ nodes: nodeReqs,
+ };
}
/**
@@ -83,6 +126,11 @@ export function exportWorkflowToFile(
workflowJSON.nodeParameters = exportParams;
}
+ const requirements = buildRequirements(workflow.nodes);
+ if (requirements) {
+ workflowJSON.requirements = requirements;
+ }
+
const jsonString = serializeWorkflow(workflowJSON);
const blob = new Blob([jsonString], { type: 'application/json' });
const url = URL.createObjectURL(blob);
@@ -109,8 +157,11 @@ export function importWorkflowFromFile(file: File): Promise<ImportedWorkflow> {
const jsonString = event.target?.result as string;
const raw = JSON.parse(jsonString);
- // Extract nodeParameters before standard deserialization
+ // Extract embedded nodeParameters + requirements manifest before
+ // standard deserialization. Both are optional — older exports
+ // without them import cleanly.
const nodeParameters = raw.nodeParameters || undefined;
+ const requirements: RequirementsManifest | undefined = raw.requirements || undefined;
const workflow = deserializeWorkflow(jsonString);
@@ -120,7 +171,7 @@ export function importWorkflowFromFile(file: File): Promise<ImportedWorkflow> {
return;
}
- resolve({ ...workflow, nodeParameters });
+ resolve({ ...workflow, nodeParameters, requirements });
} catch (error) {
reject(error);
}
@@ -161,6 +212,11 @@ export function exportWorkflowToJSON(
workflowJSON.nodeParameters = exportParams;
}
+ const requirements = buildRequirements(workflow.nodes);
+ if (requirements) {
+ workflowJSON.requirements = requirements;
+ }
+
return serializeWorkflow(workflowJSON);
}
@@ -171,6 +227,7 @@ export function exportWorkflowToJSON(
export function importWorkflowFromJSON(jsonString: string): ImportedWorkflow {
const raw = JSON.parse(jsonString);
const nodeParameters = raw.nodeParameters || undefined;
+ const requirements: RequirementsManifest | undefined = raw.requirements || undefined;
const workflow = deserializeWorkflow(jsonString);
@@ -179,7 +236,7 @@ export function importWorkflowFromJSON(jsonString: string): ImportedWorkflow {
throw new Error(`Invalid workflow JSON: ${validation.errors.join(', ')}`);
}
- return { ...workflow, nodeParameters };
+ return { ...workflow, nodeParameters, requirements };
}
/**
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index 4f5d09e4..1c4eaf4f 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -789,6 +789,32 @@ async def handle_clear_node_output(data: Dict[str, Any], websocket: WebSocket) -
"db_cleared": db_cleared, "broadcaster_cleared": broadcaster_cleared}
+@ws_handler()
+async def handle_validate_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """Run pre-execution validation on a workflow graph.
+
+ Serves three call sites: editor live-lint (debounced), pre-execute gate
+ (called internally from ``handle_execute_workflow``), and import dry-run
+ (called by the frontend after ``importWorkflowFromFile``).
+
+ Expects:
+ nodes: List of workflow nodes with {id, type, data}
+ edges: List of edges with {id, source, target}
+ parameters_by_id: Optional map of node_id -> parameters for INVALID_PARAM
+ check. Defaults to ``node.data.parameters`` (rarely populated).
+
+ Returns:
+ ``{"success": True, "report": {"errors": [...], "warnings": [...]}}``.
+ """
+ from services.workflow_validator import validate_workflow
+ report = await validate_workflow(
+ nodes=data.get("nodes", []),
+ edges=data.get("edges", []),
+ parameters_by_id=data.get("parameters_by_id"),
+ )
+ return {"report": report}
+
+
@ws_handler()
async def handle_execute_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Execute entire workflow from start node to end.
@@ -798,6 +824,9 @@ async def handle_execute_workflow(data: Dict[str, Any], websocket: WebSocket) ->
nodes: List of workflow nodes with {id, type, data}
edges: List of edges with {id, source, target}
session_id: Optional session identifier
+ force: Optional bool — bypass validation errors. When False/missing,
+ validation errors short-circuit execution and the response carries
+ the report so the frontend can prompt the user.
Returns:
Workflow execution result with all node outputs
@@ -809,10 +838,26 @@ async def handle_execute_workflow(data: Dict[str, Any], websocket: WebSocket) ->
nodes = data.get("nodes", [])
edges = data.get("edges", [])
session_id = data.get("session_id", "default")
+ force = bool(data.get("force"))
if not nodes:
return {"success": False, "error": "No nodes provided"}
+ # Pre-execute validation gate. force=True overrides errors (matches the
+ # "Run anyway" UX in Windmill); warnings never block.
+ if not force:
+ from services.workflow_validator import validate_workflow
+ report = await validate_workflow(
+ nodes=nodes, edges=edges,
+ parameters_by_id=data.get("parameters_by_id"),
+ )
+ if report["errors"]:
+ return {
+ "success": False,
+ "error": "validation_failed",
+ "report": report,
+ }
+
# Mark this workflow active so the toolbar Start->Stop reflects whole-workflow runs
await broadcaster.workflow_run_started(workflow_id)
@@ -907,6 +952,21 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->
if not workflow_id:
return {"success": False, "error": "workflow_id is required for deployment"}
+ # Pre-deploy validation gate. Deploy never honors a force-override —
+ # a broken workflow running on a schedule is far worse than a failed
+ # one-shot manual run.
+ from services.workflow_validator import validate_workflow
+ deploy_report = await validate_workflow(
+ nodes=nodes, edges=edges,
+ parameters_by_id=data.get("parameters_by_id"),
+ )
+ if deploy_report["errors"]:
+ return {
+ "success": False,
+ "error": "validation_failed",
+ "report": deploy_report,
+ }
+
# Check if THIS specific workflow is already deployed (per-workflow isolation)
if workflow_service.is_workflow_deployed(workflow_id):
status = workflow_service.get_deployment_status(workflow_id)
@@ -2657,6 +2717,7 @@ MESSAGE_HANDLERS: Dict[str, MessageHandler] = {
# Node execution
"execute_node": handle_execute_node,
"execute_workflow": handle_execute_workflow,
+ "validate_workflow": handle_validate_workflow,
"cancel_execution": handle_cancel_execution,
"get_workflow_status": handle_get_workflow_status,
"get_node_output": handle_get_node_output,
diff --git a/server/services/example_loader.py b/server/services/example_loader.py
index fdb4bca2..3ed48ea7 100644
--- a/server/services/example_loader.py
+++ b/server/services/example_loader.py
@@ -33,24 +33,60 @@ def get_example_workflows() -> List[Dict[str, Any]]:
async def import_examples_for_user(database) -> int:
"""Import all examples using existing database.save_workflow().
+ Dry-runs the workflow validator (services.workflow_validator) on each
+ example before saving. Validation ERRORS skip the example (a broken
+ example shipped on disk is a bug); WARNINGS are logged but allowed
+ through — first-launch missing credentials are expected.
+
Returns count of workflows imported.
"""
+ from services.workflow_validator import validate_workflow
+
examples = get_example_workflows()
imported = 0
for example in examples:
# Use ID from JSON, prefixed with 'example_' for clarity
workflow_id = f"example_{example.get('id', 'unknown')}"
+ nodes = example.get("nodes", [])
+ edges = example.get("edges", [])
+ node_parameters = example.get("nodeParameters", {})
+
+ # Pre-save validation. parameters_by_id from the JSON's
+ # ``nodeParameters`` block lets INVALID_PARAM check see the
+ # configured defaults instead of empty dicts.
+ try:
+ report = await validate_workflow(
+ nodes=nodes, edges=edges,
+ parameters_by_id=node_parameters,
+ )
+ except Exception as exc:
+ logger.warning(
+ "Skipping example %r: validator raised %s",
+ example.get("name"), exc,
+ )
+ continue
+
+ if report["errors"]:
+ logger.warning(
+ "Skipping example %r: %d validation errors %s",
+ example.get("name"), len(report["errors"]),
+ [iss.get("code") for iss in report["errors"]],
+ )
+ continue
+ if report["warnings"]:
+ # Expected on first launch — credentials not yet configured.
+ logger.info(
+ "Example %r has %d warnings (likely first-launch credential gaps)",
+ example.get("name"), len(report["warnings"]),
+ )
# Reuse existing save_workflow method
success = await database.save_workflow(
workflow_id=workflow_id,
name=example.get("name", "Example Workflow"),
description=example.get("description"),
- data={
- "nodes": example.get("nodes", []),
- "edges": example.get("edges", [])
- }
+ data={"nodes": nodes, "edges": edges},
)
if success:
@@ -58,7 +94,6 @@ async def import_examples_for_user(database) -> int:
logger.info(f"Imported example: {example.get('name')}")
# Save embedded nodeParameters if present
- node_parameters = example.get("nodeParameters", {})
for node_id, params in node_parameters.items():
if params:
try:
diff --git a/server/services/workflow_validator.py b/server/services/workflow_validator.py
new file mode 100644
index 00000000..1b721636
--- /dev/null
+++ b/server/services/workflow_validator.py
@@ -0,0 +1,199 @@
+"""Pre-execution workflow validation.
+
+Returns plain dicts so callers can JSON-serialize directly; no dataclasses,
+no enums, no new exception types. Reuses:
+
+- ``services.node_registry.get_node_class`` to look up the plugin class for
+ parameter validation and credential introspection.
+- The plugin's ``Params`` Pydantic model for INVALID_PARAM detection.
+- ``auth_service.has_valid_key`` for MISSING_CREDENTIAL detection.
+- Kahn's algorithm pattern from ``services/execution/executor.py`` for CYCLE
+ detection — duplicated here as a small local helper (~15 lines) to keep
+ this module dependency-free.
+
+Issue shape (flat dict):
+
+ {
+ "code": "CYCLE" | "DANGLING_EDGE" | "UNKNOWN_NODE_TYPE"
+ | "INVALID_PARAM" | "MISSING_CREDENTIAL",
+ "node_id": str | None, # node where the issue surfaces, if applicable
+ "message": str, # user-facing description
+ # plus code-specific extras:
+ "node_type": str?, # INVALID_PARAM / UNKNOWN_NODE_TYPE
+ "provider_id": str?, # MISSING_CREDENTIAL
+ "remediation": str?, # MISSING_CREDENTIAL ("add_key" / "reconnect")
+ }
+
+The validator is called from three places:
+
+1. ``handle_validate_workflow`` WS handler — editor live-lint.
+2. ``handle_execute_workflow`` / ``handle_deploy_workflow`` — pre-flight gate.
+3. ``example_loader.import_examples_for_user`` — log-only warning on first launch.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from core.logging import get_logger
+from services.node_registry import get_node_class
+
+logger = get_logger(__name__)
+
+
+async def validate_workflow(
+ nodes: List[Dict[str, Any]],
+ edges: List[Dict[str, Any]],
+ *,
+ parameters_by_id: Optional[Dict[str, Dict[str, Any]]] = None,
+) -> Dict[str, List[Dict[str, Any]]]:
+ """Validate a workflow graph + per-node params + credential presence.
+
+ Args:
+ nodes: ReactFlow-shaped nodes ``[{"id", "type", "data": {...}}, ...]``.
+ edges: ReactFlow-shaped edges ``[{"source", "target", ...}, ...]``.
+ parameters_by_id: Optional override map of node_id -> parameters dict.
+ When omitted, the validator reads ``node["data"]["parameters"]``
+ (which is usually empty since parameters live in the DB — callers
+ who want INVALID_PARAM warnings must hydrate from DB and pass them).
+
+ Returns:
+ ``{"errors": [...], "warnings": [...]}``. ``errors`` block execution
+ when ``force=False``; ``warnings`` are surfaced to the user but don't
+ block.
+ """
+ errors: List[Dict[str, Any]] = []
+ warnings: List[Dict[str, Any]] = []
+ nodes = nodes or []
+ edges = edges or []
+ parameters_by_id = parameters_by_id or {}
+
+ node_by_id: Dict[str, Dict[str, Any]] = {n["id"]: n for n in nodes if n.get("id")}
+
+ # 1. Dangling edges — references to non-existent nodes. Errors.
+ for e in edges:
+ src = e.get("source")
+ tgt = e.get("target")
+ if src and src not in node_by_id:
+ errors.append({
+ "code": "DANGLING_EDGE",
+ "node_id": None,
+ "message": f"Edge source {src!r} does not match any node",
+ })
+ if tgt and tgt not in node_by_id:
+ errors.append({
+ "code": "DANGLING_EDGE",
+ "node_id": None,
+ "message": f"Edge target {tgt!r} does not match any node",
+ })
+
+ # 2. Unknown node types — plugin not installed on this instance. Errors.
+ # 3. Per-node Pydantic param validation. Warnings (matches runtime
+ # soft-fail at node_executor._prepare_parameters).
+ # 4. Missing credentials. Warnings (a workflow can be saved with missing
+ # credentials and configured later — only at force=True execute does
+ # it become a runtime error).
+ auth_service = None
+ for n in nodes:
+ node_id = n.get("id")
+ node_type = n.get("type", "")
+ if not node_id:
+ continue
+
+ cls = get_node_class(node_type)
+ if cls is None:
+ errors.append({
+ "code": "UNKNOWN_NODE_TYPE",
+ "node_id": node_id,
+ "node_type": node_type,
+ "message": f"Plugin type {node_type!r} is not installed",
+ })
+ continue
+
+ # Param validation — prefer caller-supplied params, fall back to
+ # node.data.parameters (rarely populated since params live in DB).
+ params = parameters_by_id.get(node_id)
+ if params is None:
+ params = (n.get("data") or {}).get("parameters") or {}
+ try:
+ cls.Params.model_validate(params)
+ except Exception as exc: # Pydantic ValidationError or anything Params raises
+ # Surface the first error's message — matches the runtime
+ # envelope produced by BaseNode.execute on ValidationError.
+ first_err = None
+ try:
+ errs = exc.errors() # type: ignore[attr-defined]
+ if errs:
+ first_err = errs[0]
+ except Exception:
+ pass
+ msg = (
+ first_err.get("msg")
+ if isinstance(first_err, dict) and first_err.get("msg")
+ else str(exc)
+ )
+ warnings.append({
+ "code": "INVALID_PARAM",
+ "node_id": node_id,
+ "node_type": node_type,
+ "message": msg,
+ "path": list(first_err.get("loc", ())) if isinstance(first_err, dict) else [],
+ })
+
+ # Credential presence — for each Credential subclass declared on
+ # the plugin, ask AuthService whether a key/token is stored.
+ for cred_cls in getattr(cls, "credentials", ()) or ():
+ if auth_service is None:
+ # Lazy import to avoid a hard dependency on the container at
+ # module import time (lets validator be unit-tested with a
+ # fake auth_service injected via the parameter — see
+ # tests/test_workflow_validator.py).
+ from core.container import container
+ auth_service = container.auth_service()
+ try:
+ stored = await auth_service.has_valid_key(cred_cls.id)
+ except Exception:
+ logger.debug(
+ "[workflow_validator] has_valid_key failed for %s",
+ cred_cls.id, exc_info=True,
+ )
+ stored = False
+ if not stored:
+ warnings.append({
+ "code": "MISSING_CREDENTIAL",
+ "node_id": node_id,
+ "node_type": node_type,
+ "provider_id": cred_cls.id,
+ "message": f"Credential {cred_cls.id!r} is not configured",
+ "remediation": "add_key",
+ })
+
+ # 5. Cycle detection via Kahn's algorithm. Errors when unresolved nodes
+ # remain after the topological pass. Mirrors the warn-and-continue
+ # pattern in execution/executor.py._compute_execution_layers but
+ # promotes the warning to an error (validator's job is to block).
+ in_degree: Dict[str, int] = {n["id"]: 0 for n in nodes if n.get("id")}
+ for e in edges:
+ tgt = e.get("target")
+ if tgt in in_degree:
+ in_degree[tgt] += 1
+ queue: List[str] = [nid for nid, d in in_degree.items() if d == 0]
+ seen = 0
+ while queue:
+ nid = queue.pop()
+ seen += 1
+ for e in edges:
+ if e.get("source") == nid and e.get("target") in in_degree:
+ in_degree[e["target"]] -= 1
+ if in_degree[e["target"]] == 0:
+ queue.append(e["target"])
+ if seen < len(in_degree):
+ unresolved = sorted(nid for nid, d in in_degree.items() if d > 0)
+ errors.append({
+ "code": "CYCLE",
+ "node_id": None,
+ "message": f"Cycle detected involving nodes: {unresolved}",
+ "nodes": unresolved,
+ })
+
+ return {"errors": errors, "warnings": warnings}
diff --git a/server/tests/credentials/test_credential_broadcasts.py b/server/tests/credentials/test_credential_broadcasts.py
index 214e0d46..e79e09f6 100644
--- a/server/tests/credentials/test_credential_broadcasts.py
+++ b/server/tests/credentials/test_credential_broadcasts.py
@@ -218,3 +218,99 @@ class TestAuthServiceDbCanonicalInvariant:
"refresh_token. Use get_oauth_refresh_token() helper that "
"always reads from the encrypted DB."
)
+
+
+class TestCredentialRuntimeFailureBroadcast:
+ """Runtime credential failures (workflow tries to use a credential that
+ isn't configured) must surface to connected clients via a CloudEvents
+ broadcast. Without this the user fixes the credential but the
+ Credentials modal keeps showing it as missing until they reconnect.
+
+ The broadcast site is ``BaseNode.execute``'s ``PermissionError`` branch.
+ ``Credential.resolve()`` annotates the exception with ``.provider`` and
+ ``.reason``; ``BaseNode`` reads them and emits
+ ``credential.{api_key|oauth}.runtime_failed``.
+ """
+
+ def test_credential_resolvers_annotate_permission_error(self):
+ """Both resolvers must attach ``.provider`` so BaseNode can
+ broadcast the catalogue refresh. Without these attributes,
+ runtime failures fall to the generic exception branch and
+ the user gets no UI feedback."""
+ from services.plugin import credential as cred_module
+
+ oauth_src = inspect.getsource(cred_module.OAuth2Credential.resolve)
+ apikey_src = inspect.getsource(cred_module.ApiKeyCredential.resolve)
+
+ for label, src in (("OAuth2Credential", oauth_src), ("ApiKeyCredential", apikey_src)):
+ assert "err.provider" in src, (
+ f"{label}.resolve must annotate PermissionError with .provider "
+ "so BaseNode.execute can surface the failing credential."
+ )
+ assert "err.reason" in src, (
+ f"{label}.resolve must annotate PermissionError with .reason "
+ "('missing' / 'expired' / 'invalid')."
+ )
+ assert "err.auth" in src, (
+ f"{label}.resolve must annotate PermissionError with .auth "
+ "('api_key' or 'oauth2') so BaseNode can build the correct "
+ "CloudEvents type (credential.{auth}.runtime_failed)."
+ )
+
+ def test_basenode_permission_branch_broadcasts(self):
+ """BaseNode.execute's PermissionError branch must call
+ broadcast_credential_event (CloudEvents path) so the wire emits
+ a credential.<auth>.runtime_failed event the frontend can route."""
+ from services.plugin import base as base_module
+
+ src = inspect.getsource(base_module.BaseNode.execute)
+ assert _BROADCAST_PATTERN.search(src), (
+ "BaseNode.execute must broadcast on PermissionError so missing "
+ "credentials surface to the Credentials modal. Reuses the "
+ "existing broadcast contract — same regex as mutation handlers."
+ )
+ # Type string convention: credential.<auth>.runtime_failed
+ assert "runtime_failed" in src, (
+ "BaseNode.execute should emit credential.<auth>.runtime_failed "
+ "(matches the credential.oauth.connected naming convention)."
+ )
+
+ def test_broadcast_credential_event_carries_workflow_id_and_extras(self):
+ """The broadcaster helper must accept arbitrary extra data fields
+ (used for runtime failure events: reason, node_id, error)."""
+ import asyncio
+
+ from services.status_broadcaster import StatusBroadcaster
+
+ broadcaster = StatusBroadcaster()
+ captured: list[dict] = []
+
+ async def fake_broadcast(payload):
+ captured.append(payload)
+
+ broadcaster.broadcast = fake_broadcast # type: ignore[method-assign]
+
+ asyncio.get_event_loop().run_until_complete(
+ broadcaster.broadcast_credential_event(
+ event_type="credential.api_key.runtime_failed",
+ provider="telegram_bot_token",
+ workflow_id="wf-test",
+ reason="missing",
+ node_id="telegramSend-1",
+ error="No API key for 'telegram_bot_token'.",
+ )
+ )
+
+ assert len(captured) == 1
+ payload = captured[0]
+ assert payload["type"] == "credential_catalogue_updated"
+ envelope = payload["data"]
+ assert envelope["specversion"] == "1.0"
+ assert envelope["type"] == "credential.api_key.runtime_failed"
+ assert envelope["subject"] == "telegram_bot_token"
+ assert envelope["workflow_id"] == "wf-test"
+ data = envelope["data"]
+ assert data["provider"] == "telegram_bot_token"
+ assert data["reason"] == "missing"
+ assert data["node_id"] == "telegramSend-1"
+ assert "error" in data
diff --git a/server/tests/test_workflow_validator.py b/server/tests/test_workflow_validator.py
new file mode 100644
index 00000000..cebef0d7
--- /dev/null
+++ b/server/tests/test_workflow_validator.py
@@ -0,0 +1,307 @@
+"""Contract tests for ``services.workflow_validator.validate_workflow``.
+
+Locks the six issue codes the validator surfaces:
+
+- ``DANGLING_EDGE`` — edge references a node id that isn't in the graph.
+- ``UNKNOWN_NODE_TYPE`` — plugin not installed on this instance.
+- ``INVALID_PARAM`` — Pydantic ``Params.model_validate`` raises.
+- ``MISSING_CREDENTIAL`` — declared credential not stored.
+- ``CYCLE`` — Kahn's algorithm leaves nodes unresolved.
+- Empty report — valid workflow returns ``{errors: [], warnings: []}``.
+
+Used to gate ``handle_execute_workflow`` (force=False), all
+``handle_deploy_workflow`` calls, and ``example_loader.import_examples_for_user``.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from pydantic import BaseModel, Field
+
+
+pytestmark = pytest.mark.asyncio
+
+
+# ---------------------------------------------------------------------------
+# Fakes — a small in-memory plugin registry so tests are deterministic and
+# don't depend on which plugins happen to be installed at run time.
+# ---------------------------------------------------------------------------
+
+
+class _FakeCredential:
+ id = "fake_provider"
+ auth = "api_key"
+
+
+class _FakeParams(BaseModel):
+ text: str = Field(..., min_length=1)
+
+
+class _FakeNodeClass:
+ type = "fakeNode"
+ Params = _FakeParams
+ credentials = (_FakeCredential,)
+
+
+class _FakeNodelessClass:
+ """Plugin with no credentials and a permissive Params (every workflow
+ invariant is honored)."""
+
+ type = "fakeNodeless"
+
+ class Params(BaseModel):
+ pass
+
+ credentials = ()
+
+
+def _patch_registry(monkeypatch, mapping: dict[str, object]) -> None:
+ """Patch ``get_node_class`` to return our fakes."""
+ monkeypatch.setattr(
+ "services.workflow_validator.get_node_class",
+ lambda node_type: mapping.get(node_type),
+ )
+
+
+def _patch_auth(monkeypatch, has_valid_key_return: bool) -> None:
+ """Patch the container so ``auth_service.has_valid_key`` returns a fixed bool."""
+ fake_auth = MagicMock()
+ fake_auth.has_valid_key = AsyncMock(return_value=has_valid_key_return)
+ fake_container = MagicMock()
+ fake_container.auth_service.return_value = fake_auth
+ monkeypatch.setattr("core.container.container", fake_container)
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+async def test_dangling_edge_target_is_error(monkeypatch):
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {"fakeNodeless": _FakeNodelessClass})
+ _patch_auth(monkeypatch, True)
+
+ report = await validate_workflow(
+ nodes=[{"id": "n1", "type": "fakeNodeless", "data": {}}],
+ edges=[{"id": "e1", "source": "n1", "target": "ghost"}],
+ )
+ codes = [iss["code"] for iss in report["errors"]]
+ assert "DANGLING_EDGE" in codes
+
+
+async def test_dangling_edge_source_is_error(monkeypatch):
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {"fakeNodeless": _FakeNodelessClass})
+ _patch_auth(monkeypatch, True)
+
+ report = await validate_workflow(
+ nodes=[{"id": "n1", "type": "fakeNodeless", "data": {}}],
+ edges=[{"id": "e1", "source": "ghost", "target": "n1"}],
+ )
+ codes = [iss["code"] for iss in report["errors"]]
+ assert "DANGLING_EDGE" in codes
+
+
+async def test_unknown_node_type_is_error(monkeypatch):
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {})
+ _patch_auth(monkeypatch, True)
+
+ report = await validate_workflow(
+ nodes=[{"id": "n1", "type": "doesNotExist", "data": {}}],
+ edges=[],
+ )
+ codes = [iss["code"] for iss in report["errors"]]
+ assert "UNKNOWN_NODE_TYPE" in codes
+ issue = next(iss for iss in report["errors"] if iss["code"] == "UNKNOWN_NODE_TYPE")
+ assert issue["node_id"] == "n1"
+ assert issue["node_type"] == "doesNotExist"
+
+
+async def test_invalid_param_is_warning(monkeypatch):
+ """INVALID_PARAM is a WARNING — matches the runtime soft-fail at
+ ``node_executor._prepare_parameters`` (logs WARN, continues)."""
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {"fakeNode": _FakeNodeClass})
+ _patch_auth(monkeypatch, True)
+
+ # text is required + min_length=1; empty triggers ValidationError.
+ report = await validate_workflow(
+ nodes=[{"id": "n1", "type": "fakeNode", "data": {"parameters": {"text": ""}}}],
+ edges=[],
+ )
+ codes = [iss["code"] for iss in report["warnings"]]
+ assert "INVALID_PARAM" in codes
+ assert all(iss["code"] != "INVALID_PARAM" for iss in report["errors"]), (
+ "INVALID_PARAM must not block execution; only deploy-time errors block."
+ )
+
+
+async def test_missing_credential_is_warning(monkeypatch):
+ """MISSING_CREDENTIAL is a WARNING so the workflow can be saved/imported
+ and credentials configured afterward. Runtime failure (different path,
+ different broadcast) is what surfaces to the user during execution."""
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {"fakeNode": _FakeNodeClass})
+ _patch_auth(monkeypatch, False) # credential NOT stored
+
+ report = await validate_workflow(
+ nodes=[{"id": "n1", "type": "fakeNode", "data": {"parameters": {"text": "ok"}}}],
+ edges=[],
+ )
+ codes = [iss["code"] for iss in report["warnings"]]
+ assert "MISSING_CREDENTIAL" in codes
+ issue = next(iss for iss in report["warnings"] if iss["code"] == "MISSING_CREDENTIAL")
+ assert issue["provider_id"] == "fake_provider"
+ assert issue["remediation"] == "add_key"
+ assert issue["node_id"] == "n1"
+
+
+async def test_cycle_is_error(monkeypatch):
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {"fakeNodeless": _FakeNodelessClass})
+ _patch_auth(monkeypatch, True)
+
+ # Two-node cycle: n1 -> n2 -> n1
+ report = await validate_workflow(
+ nodes=[
+ {"id": "n1", "type": "fakeNodeless", "data": {}},
+ {"id": "n2", "type": "fakeNodeless", "data": {}},
+ ],
+ edges=[
+ {"id": "e1", "source": "n1", "target": "n2"},
+ {"id": "e2", "source": "n2", "target": "n1"},
+ ],
+ )
+ codes = [iss["code"] for iss in report["errors"]]
+ assert "CYCLE" in codes
+ cycle = next(iss for iss in report["errors"] if iss["code"] == "CYCLE")
+ assert set(cycle["nodes"]) == {"n1", "n2"}
+
+
+async def test_valid_workflow_returns_empty_report(monkeypatch):
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {"fakeNodeless": _FakeNodelessClass})
+ _patch_auth(monkeypatch, True)
+
+ report = await validate_workflow(
+ nodes=[
+ {"id": "n1", "type": "fakeNodeless", "data": {}},
+ {"id": "n2", "type": "fakeNodeless", "data": {}},
+ ],
+ edges=[{"id": "e1", "source": "n1", "target": "n2"}],
+ )
+ assert report == {"errors": [], "warnings": []}
+
+
+async def test_parameters_by_id_overrides_node_data(monkeypatch):
+ """When parameters_by_id is supplied, it wins over node.data.parameters.
+ Used by the WS execute handler (hydrating from DB) and the
+ example_loader (which holds params in the JSON's nodeParameters block)."""
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {"fakeNode": _FakeNodeClass})
+ _patch_auth(monkeypatch, True)
+
+ # node.data has valid params, parameters_by_id has invalid → invalid wins.
+ report = await validate_workflow(
+ nodes=[{"id": "n1", "type": "fakeNode", "data": {"parameters": {"text": "good"}}}],
+ edges=[],
+ parameters_by_id={"n1": {"text": ""}},
+ )
+ codes = [iss["code"] for iss in report["warnings"]]
+ assert "INVALID_PARAM" in codes
+
+
+async def test_report_shape_is_plain_dicts(monkeypatch):
+ """Issues must be plain dicts (no dataclasses / enums) so the report
+ serializes directly to JSON without custom encoders."""
+ import json
+
+ from services.workflow_validator import validate_workflow
+
+ _patch_registry(monkeypatch, {})
+ _patch_auth(monkeypatch, True)
+
+ report = await validate_workflow(
+ nodes=[{"id": "n1", "type": "unknown", "data": {}}],
+ edges=[],
+ )
+ # Should round-trip through stdlib JSON without errors.
+ serialized = json.dumps(report)
+ assert "UNKNOWN_NODE_TYPE" in serialized
+
+
+# ---------------------------------------------------------------------------
+# WS-handler gating invariants
+# ---------------------------------------------------------------------------
+
+
+class TestExecuteAndDeployHandlersGate:
+ """Static-source contract: ``handle_execute_workflow`` and
+ ``handle_deploy_workflow`` MUST call ``validate_workflow`` before
+ handing the graph off to ``WorkflowService``. Anchor on the function
+ name string so a rename forces a deliberate test update.
+ """
+
+ @staticmethod
+ def _handler_source(handler) -> str:
+ import inspect
+
+ fn = handler
+ while hasattr(fn, "__wrapped__"):
+ fn = fn.__wrapped__
+ return inspect.getsource(fn)
+
+ def test_execute_workflow_calls_validator(self):
+ from routers import websocket as ws_module
+
+ src = self._handler_source(ws_module.handle_execute_workflow)
+ assert "validate_workflow" in src, (
+ "handle_execute_workflow must call validate_workflow so "
+ "broken workflows are blocked at the gate (force=true overrides)."
+ )
+ assert "force" in src, (
+ "handle_execute_workflow must support the force=true override "
+ "so 'Run anyway' can bypass warnings (Windmill pattern)."
+ )
+
+ def test_deploy_workflow_calls_validator_unconditionally(self):
+ from routers import websocket as ws_module
+
+ src = self._handler_source(ws_module.handle_deploy_workflow)
+ assert "validate_workflow" in src, (
+ "handle_deploy_workflow must call validate_workflow — a broken "
+ "workflow deployed on a schedule is much worse than a failed "
+ "manual run, so deploy never honors a force-override."
+ )
+
+ def test_validate_workflow_handler_registered(self):
+ from routers import websocket as ws_module
+
+ assert "validate_workflow" in ws_module.MESSAGE_HANDLERS, (
+ "validate_workflow message type must be in MESSAGE_HANDLERS "
+ "so the frontend live-lint and import dry-run paths can reach it."
+ )
+
+ def test_example_loader_calls_validator(self):
+ """First-launch example import must run validator to skip
+ malformed examples (errors) and log credential gaps (warnings)."""
+ import inspect
+ from services import example_loader
+
+ src = inspect.getsource(example_loader.import_examples_for_user)
+ assert "validate_workflow" in src, (
+ "import_examples_for_user must run validate_workflow before "
+ "save_workflow — broken examples shipped on disk are bugs."
+ )