Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
refactor(workflow_import): backend-authoritative import orchestrator + workflow.imported CloudEvent
trohitge29354f3c3cf
diff --git a/client/src/Dashboard.tsx b/client/src/Dashboard.tsxindex c16f5345..6d294c20 100644--- a/client/src/Dashboard.tsx+++ b/client/src/Dashboard.tsx@@ -54,12 +54,34 @@ import { useNodeStatusStore } from './stores/nodeStatusStore';import {sanitizeNodesForComparison,sanitizeEdgesForComparison,- generateWorkflowId} from './utils/workflow';import { importWorkflowFromFile } from './utils/workflowExport';-import { CATALOGUE_QUERY_KEY, type CatalogueResponse } from './hooks/useCatalogueQuery';+import type { ValidationIssue } from './hooks/useWorkflowValidation';import { buildCanvasStyles } from './styles/canvasAnimations';+/** Wire shape returned by the backend ``import_workflow`` WS handler.+ * Mirrors ``services.workflow_import.import_workflow`` — flat optional+ * fields rather than a discriminated union, since callers branch on+ * ``response.success`` + ``response.preview`` anyway. */+interface ImportWorkflowResponse {+ success: boolean;+ preview?: boolean;+ workflow_id?: string;+ name?: string;+ node_count?: number;+ edge_count?: number;+ saved_parameters?: number;+ error?: string;+ report?: { errors: ValidationIssue[]; warnings: ValidationIssue[] };+ missing_credentials?: Array<{ provider_id: string; display_name: string; kind: string }>;+ name_conflict?: boolean;+ suggested_name?: string | null;+ requirements?: {+ credentials: Array<{ provider_id: string }>;+ nodes: Array<{ type: string; version: number }>;+ };+}+import 'reactflow/dist/style.css';// Wave 10.D step 2: React-component dispatch table.@@ -135,7 +157,6 @@ const DashboardContent: React.FC = () => {const updateWorkflow = useAppStore((s) => s.updateWorkflow);const loadWorkflow = useAppStore((s) => s.loadWorkflow);const createNewWorkflow = useAppStore((s) => s.createNewWorkflow);- const saveWorkflow = useAppStore((s) => s.saveWorkflow);const deleteWorkflow = useAppStore((s) => s.deleteWorkflow);const migrateCurrentWorkflow = useAppStore((s) => s.migrateCurrentWorkflow);const toggleSidebar = useAppStore((s) => s.toggleSidebar);@@ -144,7 +165,6 @@ const DashboardContent: React.FC = () => {const toggleProMode = useAppStore((s) => s.toggleProMode);const exportWorkflowToJSON = useAppStore((s) => s.exportWorkflowToJSON);const exportWorkflowToFile = useAppStore((s) => s.exportWorkflowToFile);- const setCurrentWorkflow = useAppStore((s) => s.setCurrentWorkflow);const selectedNode = useAppStore((s) => s.selectedNode);const setSelectedNode = useAppStore((s) => s.setSelectedNode);const renamingNodeId = useAppStore((s) => s.renamingNodeId);@@ -803,108 +823,90 @@ const DashboardContent: React.FC = () => {const file = (e.target as HTMLInputElement).files?.[0];if (!file) return;- const importedWorkflow = await importWorkflowFromFile(file);-- // Cross-reference the export's `requirements.credentials` against- // the catalogue's `provider.stored` flag. Warn the user about- // missing credentials before saving — they can import anyway and- // configure later, or cancel. Older exports without `requirements`- // skip this step transparently.- const reqCreds = importedWorkflow.requirements?.credentials ?? [];- if (reqCreds.length > 0) {- const catalogue = queryClient.getQueryData<CatalogueResponse>(CATALOGUE_QUERY_KEY);- const providers = catalogue?.providers ?? [];- const missing = reqCreds- .map(need => providers.find(p => p.id === need.provider_id))- .filter((p): p is NonNullable<typeof p> => !!p && p.stored !== true)- .map(p => `- ${p.name} (${p.id})`);+ // Frontend parses the file (nice error message on malformed JSON);+ // everything else — validation, name-conflict check, missing-+ // credential cross-reference, node-id remap, save — lives in the+ // backend `import_workflow` orchestrator. This UI is reduced to+ // file pick + user prompts + view switch.+ const imported = await importWorkflowFromFile(file);+ const proposedName = imported.name || 'Imported Workflow';++ let response = await sendRequest<ImportWorkflowResponse>('import_workflow', {+ workflow: imported,+ });++ if (!response.success) {+ const reportLines = (response.report?.errors ?? [])+ .map((err: ValidationIssue) => `- ${err.code}: ${err.message}`)+ .join('\n');+ alert(+ `Failed to import workflow: ${response.error ?? 'unknown error'}` ++ (reportLines ? `\n\nValidation errors:\n${reportLines}` : ''),+ );+ return;+ }++ // Preview branch — backend wants user confirmation before saving.+ if (response.preview) {+ let forceCredentials = false;+ let finalName = proposedName;++ // Missing credentials prompt+ const missing = response.missing_credentials ?? [];if (missing.length > 0) {+ const lines = missing+ .map((c: { provider_id: string; display_name: string }) =>+ `- ${c.display_name} (${c.provider_id})`)+ .join('\n');const proceed = window.confirm(`This workflow needs ${missing.length} credential${missing.length === 1 ? '' : 's'} ` +- `that ${missing.length === 1 ? 'is' : 'are'} not configured:\n\n` +- `${missing.join('\n')}\n\n` +- `You can import it anyway and configure ${missing.length === 1 ? 'it' : 'them'} later. ` +- `Click OK to import, or Cancel to abort.`,+ `that ${missing.length === 1 ? 'is' : 'are'} not configured:\n\n${lines}\n\n` ++ `Click OK to import anyway and configure ${missing.length === 1 ? 'it' : 'them'} later, ` ++ `or Cancel to abort.`,);if (!proceed) return;+ forceCredentials = true;}- }- // Check for name conflict with existing workflows- const existingNames = savedWorkflows.map(w => w.name.toLowerCase());- let finalName = importedWorkflow.name;-- if (existingNames.includes(finalName.toLowerCase())) {- // Name conflict detected - prompt user for new name- const suggestedName = `${importedWorkflow.name} (imported)`;- const userInput = window.prompt(- `A workflow named "${importedWorkflow.name}" already exists.\n\nEnter a new name for the imported workflow:`,- suggestedName- );-- if (userInput === null) {- // User cancelled the import- return;+ // Name conflict prompt+ if (response.name_conflict) {+ const suggested = response.suggested_name ?? `${proposedName} (imported)`;+ const userInput = window.prompt(+ `A workflow named "${proposedName}" already exists.\n\nEnter a new name for the imported workflow:`,+ suggested,+ );+ if (userInput === null) return;+ finalName = userInput.trim();+ if (!finalName) {+ alert('Workflow name cannot be empty');+ return;+ }}- finalName = userInput.trim();-- if (!finalName) {- alert('Workflow name cannot be empty');- return;- }+ // Commit with confirmations+ response = await sendRequest<ImportWorkflowResponse>('import_workflow', {+ workflow: imported,+ name: finalName,+ force_credentials: forceCredentials,+ });- // Check if the new name also conflicts- if (existingNames.includes(finalName.toLowerCase())) {- alert(`A workflow named "${finalName}" also exists. Please try again with a different name.`);+ if (!response.success) {+ alert(`Failed to import workflow: ${response.error ?? 'unknown error'}`);return;}}- const workflow = {- ...importedWorkflow,- name: finalName,- id: generateWorkflowId(),- createdAt: new Date(),- lastModified: new Date()- };-- console.log('Importing workflow:', workflow);-- // Save embedded nodeParameters from new-format exports- if (importedWorkflow.nodeParameters) {- for (const [nodeId, params] of Object.entries(importedWorkflow.nodeParameters)) {- if (params && Object.keys(params).length > 0) {- try {- await saveNodeParameters(nodeId, params);- console.log(`Saved parameters for node ${nodeId} (from nodeParameters)`);- } catch (error) {- console.error(`Failed to save parameters for node ${nodeId}:`, error);- }- }- }- } else {- // Fallback for old-format exports: save node.data if it has non-UI fields- for (const node of workflow.nodes) {- if (node.data && Object.keys(node.data).length > 0) {- try {- await saveNodeParameters(node.id, node.data);- console.log(`Saved parameters for node ${node.id} (from node.data fallback)`);- } catch (error) {- console.error(`Failed to save parameters for node ${node.id}:`, error);- }- }- }+ if (!response.preview && response.workflow_id) {+ // Saved server-side. The backend already broadcast+ // `workflow.imported` (handled in WebSocketContext), so other+ // tabs refetch the workflows list automatically. This tab+ // switches the editor view to the new workflow.+ await loadWorkflow(response.workflow_id);+ alert(+ `Workflow "${response.name}" imported with ${response.node_count} nodes ` ++ `and ${response.edge_count} connections`,+ );}-- // Set as current workflow first- setCurrentWorkflow(workflow);-- // Auto-save to database so it appears in sidebar immediately- await saveWorkflow();-- console.log('Workflow imported and saved successfully');- alert(`Workflow "${workflow.name}" imported with ${workflow.nodes.length} nodes and ${workflow.edges.length} connections`);} catch (error: any) {console.error('Import error:', error);alert(`Failed to import workflow: ${error.message}`);diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsxindex 726956f9..5c820e0a 100644--- a/client/src/contexts/WebSocketContext.tsx+++ b/client/src/contexts/WebSocketContext.tsx@@ -19,6 +19,7 @@ import { queryClient } from '../lib/queryClient';import { queryKeys } from '../lib/queryConfig';import { invalidateCatalogue } from '../hooks/useCatalogueQuery';import { nodeParamsQueryKey } from '../hooks/useNodeParamsQuery';+import { WORKFLOWS_QUERY_KEY } from '../hooks/useWorkflowsQuery';import type { WorkflowEvent } from '../types/cloudEvents';import { WS_CLOSE, WS_RECONNECT } from '../lib/connectionConfig';import {@@ -712,6 +713,23 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chibreak;}+ case 'workflow_lifecycle': {+ // CloudEvents-typed workflow lifecycle (today: workflow.imported).+ // Frontend action: invalidate the workflows query so the sidebar+ // picks up the new entry on every connected client. Other stages+ // (deployment.started, lock.acquired, ...) are read for future+ // dispatch but currently have side effects elsewhere.+ const event = data as WorkflowEvent<{+ name?: string;+ node_count?: number;+ edge_count?: number;+ }>;+ if (event?.type?.endsWith('.imported')) {+ void queryClient.invalidateQueries({ queryKey: WORKFLOWS_QUERY_KEY });+ }+ break;+ }+case 'android_status':setAndroidStatus(data || defaultAndroidStatus);break;diff --git a/client/src/utils/workflow.ts b/client/src/utils/workflow.tsindex 460dcb6d..7c7cf609 100644--- a/client/src/utils/workflow.ts+++ b/client/src/utils/workflow.ts@@ -26,5 +26,11 @@ export const snapToGrid = (position: { x: number; y: number }, gridSize = 20) =>y: Math.round(position.y / gridSize) * gridSize,});-export const getDefaultNodePosition = (nodeCount: number): { x: number; y: number } =>- nodeCount === 0 ? { x: 100, y: 200 } : { x: 0, y: 0 };\ No newline at end of file+export const getDefaultNodePosition = (nodeCount: number): { x: number; y: number } =>+ nodeCount === 0 ? { x: 100, y: 200 } : { x: 0, y: 0 };++// Node-id remap on import lives backend-side in+// ``server/services/workflow_import.py::remap_node_ids`` — the import+// path is now backend-authoritative (``import_workflow`` WS handler),+// so there's no need for a duplicate frontend implementation. The+// in-canvas copy/paste path uses its own id scheme in ``useCopyPaste``.\ No newline at end of filediff --git a/server/routers/websocket.py b/server/routers/websocket.pyindex 1c4eaf4f..de89ce76 100644--- a/server/routers/websocket.py+++ b/server/routers/websocket.py@@ -1565,6 +1565,40 @@ async def handle_save_workflow(data: Dict[str, Any], websocket: WebSocket) -> Direturn {"success": success, "workflow_id": data["workflow_id"]}+@ws_handler()+async def handle_import_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:+ """Import a workflow JSON. Two-step UX:++ First call with just the workflow object returns a preview if+ confirmations are needed (name conflict, missing credentials). The+ frontend prompts the user, then re-calls with ``name`` set and+ ``force_credentials=True`` to commit.++ Body fields:+ workflow: Raw workflow dict (nodes, edges, optional nodeParameters).+ name: User-confirmed final workflow name; omit on first call to+ let the server report a name conflict.+ force_credentials: Skip the missing-credential preview gate when+ the user has acknowledged the warning.++ See ``services.workflow_import.import_workflow`` for the full+ orchestrator contract.+ """+ from services.workflow_import import import_workflow++ workflow_payload = data.get("workflow")+ if not isinstance(workflow_payload, dict):+ return {"success": False, "error": "workflow payload required"}++ return await import_workflow(+ workflow_payload,+ name=data.get("name"),+ force_credentials=bool(data.get("force_credentials")),+ auth_service=container.auth_service(),+ database=container.database(),+ )++async def handle_get_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:"""Get workflow by ID."""database = container.database()@@ -2777,6 +2811,7 @@ MESSAGE_HANDLERS: Dict[str, MessageHandler] = {# Workflow storage operations"save_workflow": handle_save_workflow,+ "import_workflow": handle_import_workflow,"get_workflow": handle_get_workflow,"get_all_workflows": handle_get_all_workflows,"delete_workflow": handle_delete_workflow,diff --git a/server/services/events/envelope.py b/server/services/events/envelope.pyindex 888879df..1e8c322d 100644--- a/server/services/events/envelope.py+++ b/server/services/events/envelope.py@@ -187,6 +187,7 @@ class WorkflowEvent(BaseModel):"lock.released","execution.started","execution.stopped",+ "imported",],*,workflow_id: str,@@ -195,7 +196,13 @@ class WorkflowEvent(BaseModel):"""Workflow lifecycle event. Carries ``workflow_id`` both as``subject`` (CloudEvents convention) and as the``workflow_id`` extension attribute (existing reader- contract)."""+ contract).++ ``imported`` fires from ``services.workflow_import.import_workflow``+ after the new workflow is persisted. Frontend listeners invalidate+ the workflows query so the sidebar picks up the new entry across+ all connected clients (browser tabs).+ """return cls(source="machinaos://services/workflow",type=f"{_TYPE_PREFIX}workflow.{stage}",diff --git a/server/services/example_loader.py b/server/services/example_loader.pyindex 3ed48ea7..bceb0477 100644--- a/server/services/example_loader.py+++ b/server/services/example_loader.py@@ -1,4 +1,10 @@-"""Example workflow loader - reuses existing database.save_workflow()"""+"""Example workflow loader - reuses existing database.save_workflow().++Most of the import business logic (validation, remap, requirements+extraction, credential cross-check, name conflict) lives in+``services.workflow_import`` and is shared with the WS ``import_workflow``+handler. This module is the first-launch-only convenience wrapper.+"""import jsonimport loggingfrom pathlib import Path@@ -31,15 +37,17 @@ def get_example_workflows() -> List[Dict[str, Any]]:async def import_examples_for_user(database) -> int:- """Import all examples using existing database.save_workflow().+ """Import all examples using shared ``services.workflow_import`` helpers.- Dry-runs the workflow validator (services.workflow_validator) on each- example before saving. Validation ERRORS skip the example (a broken- example shipped on disk is a bug); WARNINGS are logged but allowed- through — first-launch missing credentials are expected.+ Dry-runs the workflow validator on each example, remaps node ids+ (mandatory: today's two example workflows share 6 node ids), then+ saves the workflow + per-node parameters. Errors skip the example+ (broken examples shipped on disk are bugs); warnings are logged but+ allowed through — first-launch missing credentials are expected.- Returns count of workflows imported.+ Returns the count of workflows imported."""+ from services.workflow_import import remap_node_idsfrom services.workflow_validator import validate_workflowexamples = get_example_workflows()@@ -48,13 +56,18 @@ async def import_examples_for_user(database) -> int:for example in examples:# Use ID from JSON, prefixed with 'example_' for clarityworkflow_id = f"example_{example.get('id', 'unknown')}"- nodes = example.get("nodes", [])- edges = example.get("edges", [])- node_parameters = example.get("nodeParameters", {})- # Pre-save validation. parameters_by_id from the JSON's- # ``nodeParameters`` block lets INVALID_PARAM check see the- # configured defaults instead of empty dicts.+ # Rewrite every node id so two examples that share ids don't+ # overwrite each other's parameters in the node_parameters table.+ nodes, edges, node_parameters = remap_node_ids(+ example.get("nodes", []),+ example.get("edges", []),+ example.get("nodeParameters", {}),+ )++ # Pre-save validation. parameters_by_id is the freshly-remapped+ # map so INVALID_PARAM check sees the configured defaults instead+ # of empty dicts.try:report = await validate_workflow(nodes=nodes, edges=edges,@@ -93,7 +106,7 @@ async def import_examples_for_user(database) -> int:imported += 1logger.info(f"Imported example: {example.get('name')}")- # Save embedded nodeParameters if present+ # Save embedded nodeParameters under the remapped node ids.for node_id, params in node_parameters.items():if params:try:diff --git a/server/services/status_broadcaster.py b/server/services/status_broadcaster.pyindex 018e0c28..4a5080dd 100644--- a/server/services/status_broadcaster.py+++ b/server/services/status_broadcaster.py@@ -328,6 +328,34 @@ class StatusBroadcaster:"data": event.model_dump(mode="json"),})+ async def broadcast_workflow_lifecycle(+ self,+ stage: str,+ *,+ workflow_id: str,+ **data_extra: Any,+ ) -> None:+ """Emit a CloudEvents-typed workflow.{stage} broadcast.++ Wraps :meth:`WorkflowEvent.workflow_lifecycle` so callers don't+ hand-build envelopes. ``stage`` matches the factory's Literal+ (``imported`` / ``deployment.started`` / etc.). Wire-format key+ is ``workflow_lifecycle`` — the frontend's ``WebSocketContext``+ routes on this key and invalidates the workflows query so the+ sidebar refreshes across all connected clients.+ """+ from services.events import WorkflowEvent++ event = WorkflowEvent.workflow_lifecycle(+ stage=stage, # type: ignore[arg-type]+ workflow_id=workflow_id,+ data=data_extra or None,+ )+ await self.broadcast({+ "type": "workflow_lifecycle",+ "data": event.model_dump(mode="json"),+ })+async def broadcast_node_parameters_updated(self,node_id: str,diff --git a/server/services/workflow_import.py b/server/services/workflow_import.pynew file mode 100644index 00000000..f640859c--- /dev/null+++ b/server/services/workflow_import.py@@ -0,0 +1,346 @@+"""Workflow import — backend-authoritative orchestrator.++Owns every business decision the import flow needs to make. The frontend+is reduced to a thin pipeline (file picker -> WS call -> user prompts on+the returned preview -> WS call again with confirmations). All of the+following are server-side:++- **Parse + validation** — runs :func:`services.workflow_validator.validate_workflow`+ on the raw graph; errors block the import entirely.+- **Requirements extraction** — walks the plugin registry to build a+ canonical ``{credentials, nodes}`` manifest from the actual node types,+ rather than trusting the embedded ``requirements`` block in the file.+- **Credential cross-check** — diffs the manifest against+ ``auth_service.has_valid_key``.+- **Name conflict detection** — diffs the proposed name against+ ``database.get_all_workflows()`` and synthesizes a suggested alternative.+- **Node-id remap** — rewrites every node id (and edge refs ++ nodeParameters keys) so two imports of the same JSON don't collide on+ the ``node_parameters`` table's unique key.+- **Persistence** — single ``database.save_workflow`` + per-node+ ``save_node_parameters`` calls under the freshly-remapped ids.++The two-step UX: a first call returns ``preview=True`` with whatever+needs user confirmation (missing credentials, name conflict). The+frontend prompts; the second call passes the confirmed name + a+``force_credentials`` flag and the same orchestrator branches to the+save path.+"""++from __future__ import annotations++import secrets+import time+from typing import Any, Dict, List, Optional, Tuple++from core.logging import get_logger+from services.node_registry import get_node_class+from services.workflow_validator import validate_workflow++logger = get_logger(__name__)+++# ---------------------------------------------------------------------------+# Node-id remap (was example_loader._remap_node_ids — moved here so both+# the example loader and the WS import handler share one implementation).+# ---------------------------------------------------------------------------+++def remap_node_ids(+ nodes: List[Dict[str, Any]],+ edges: List[Dict[str, Any]],+ node_parameters: Optional[Dict[str, Dict[str, Any]]] = None,+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], Dict[str, Dict[str, Any]]]:+ """Rewrite every node id to a fresh unique id; remap edge refs and+ nodeParameters keys to match.++ Without this, two workflows that share node ids (today: ``AI Assistant``+ and ``Claude Assistant`` share 6 trigger/console/memory nodes) collide+ in the ``node_parameters`` table — keyed unique on ``node_id`` — and+ the second import silently overwrites the first's parameters.++ New ids follow the same convention as ``useDragAndDrop``'s on-drop+ scheme: ``{node_type}-{timestamp}{index}-{random}``. The random salt+ protects against two clients re-importing the same file within one+ millisecond.++ Returns a new ``(nodes, edges, node_parameters)`` triple — inputs+ untouched (shallow copy at the dict level).+ """+ node_parameters = node_parameters or {}+ now = int(time.time() * 1000)+ salt = secrets.token_hex(3)++ id_map: Dict[str, str] = {}+ for index, node in enumerate(nodes):+ prefix = node.get("type") or "node"+ old_id = node.get("id")+ if old_id:+ id_map[old_id] = f"{prefix}-{now}{index}-{salt}"++ new_nodes = [+ {**node, "id": id_map.get(node.get("id"), node.get("id"))}+ for node in nodes+ ]++ new_edges: List[Dict[str, Any]] = []+ for edge in edges:+ src = id_map.get(edge.get("source"), edge.get("source"))+ tgt = id_map.get(edge.get("target"), edge.get("target"))+ new_edges.append({+ **edge,+ "source": src,+ "target": tgt,+ "id": f"e-{src}-{tgt}-{now}-{salt}",+ })++ new_params: Dict[str, Dict[str, Any]] = {}+ for old_id, params in node_parameters.items():+ new_id = id_map.get(old_id)+ if new_id:+ new_params[new_id] = params+ # Orphans (param entry whose node isn't in the graph) are dropped.++ return new_nodes, new_edges, new_params+++# ---------------------------------------------------------------------------+# Requirements + credential cross-check + name conflict+# ---------------------------------------------------------------------------+++def extract_requirements(nodes: List[Dict[str, Any]]) -> Dict[str, Any]:+ """Build a canonical ``{credentials, nodes}`` manifest from the actual+ nodes in the graph.++ Replaces the frontend's ``buildRequirements`` helper for the import+ path — the backend has authoritative ``NodeMetadata`` via the plugin+ registry, so we do NOT trust whatever ``requirements`` block was+ embedded in the imported file (a hand-edited export could lie).+ """+ cred_ids: set[str] = set()+ node_reqs: List[Dict[str, Any]] = []+ for node in nodes:+ cls = get_node_class(node.get("type", ""))+ if cls is None:+ continue+ node_reqs.append({"type": cls.type, "version": getattr(cls, "version", 1)})+ for cred_cls in getattr(cls, "credentials", ()) or ():+ cred_ids.add(cred_cls.id)+ return {+ "credentials": [{"provider_id": pid} for pid in sorted(cred_ids)],+ "nodes": node_reqs,+ }+++async def cross_check_credentials(+ requirements: Dict[str, Any], auth_service+) -> List[Dict[str, Any]]:+ """For each credential in the manifest, ask ``auth_service`` whether+ it's stored. Returns the MISSING ones with display info pulled from+ ``CREDENTIAL_REGISTRY`` so the frontend can render a friendly list.++ Display info shape: ``{provider_id, display_name, kind}``. ``kind``+ is ``"api_key"`` or ``"oauth2"`` (mirrors ``Credential.auth``).+ """+ from services.plugin.credential import CREDENTIAL_REGISTRY++ missing: List[Dict[str, Any]] = []+ for need in requirements.get("credentials", []):+ pid = need.get("provider_id")+ if not pid:+ continue+ try:+ stored = await auth_service.has_valid_key(pid)+ except Exception:+ logger.debug(+ "[workflow_import] has_valid_key failed for %s", pid, exc_info=True,+ )+ stored = False+ if stored:+ continue+ cred_cls = CREDENTIAL_REGISTRY.get(pid)+ missing.append({+ "provider_id": pid,+ "display_name": getattr(cred_cls, "display_name", pid) if cred_cls else pid,+ "kind": getattr(cred_cls, "auth", "api_key") if cred_cls else "api_key",+ })+ return missing+++async def check_name_conflict(name: str, database) -> Dict[str, Any]:+ """Returns ``{has_conflict, suggested_name}`` for a proposed workflow+ name. Suggestion format: ``"<name> (imported)"`` with a numeric suffix+ if that itself collides — mirrors the conflict-resolution pattern in+ ``client/src/utils/workflow.ts`` callers.+ """+ existing = await database.get_all_workflows()+ existing_names = {(getattr(w, "name", "") or "").lower() for w in existing}+ has_conflict = (name or "").lower() in existing_names+ suggested: Optional[str] = None+ if has_conflict:+ base = f"{name} (imported)"+ candidate = base+ counter = 2+ while candidate.lower() in existing_names:+ candidate = f"{base} {counter}"+ counter += 1+ suggested = candidate+ return {"has_conflict": has_conflict, "suggested_name": suggested}+++# ---------------------------------------------------------------------------+# Orchestrator+# ---------------------------------------------------------------------------+++async def import_workflow(+ workflow: Dict[str, Any],+ *,+ name: Optional[str] = None,+ force_credentials: bool = False,+ auth_service,+ database,+) -> Dict[str, Any]:+ """Orchestrate the full import: validate -> cross-check -> name check+ -> (preview if confirmations needed) -> remap -> save.++ Args:+ workflow: Raw workflow dict as deserialized from the import JSON.+ Must contain ``nodes`` + ``edges`` at minimum; ``nodeParameters``+ and ``name`` are optional.+ name: User-confirmed final name. When ``None``, the orchestrator+ uses ``workflow.name`` and reports a name conflict (if any)+ as part of the preview envelope.+ force_credentials: When ``True``, missing credentials don't block+ the save (user confirmed via UI). When ``False``, missing+ credentials cause a preview response.+ auth_service: Injected ``AuthService`` (for credential checks).+ database: Injected ``Database`` (for name conflicts + persistence).++ Returns:+ On validation failure::++ {"success": False, "error": "validation_failed", "report": {...}}++ On preview (needs user confirmation)::++ {+ "success": True, "preview": True,+ "report": {...},+ "missing_credentials": [...],+ "name_conflict": bool,+ "suggested_name": str | None,+ "requirements": {...},+ }++ On save success::++ {+ "success": True, "preview": False,+ "workflow_id": str, "name": str,+ "node_count": int, "edge_count": int,+ "report": {...},+ }+ """+ nodes = workflow.get("nodes") or []+ edges = workflow.get("edges") or []+ node_parameters = workflow.get("nodeParameters") or {}+ proposed_name = name or workflow.get("name") or "Imported Workflow"++ # 1. Validate. Errors block the import; warnings flow through.+ report = await validate_workflow(+ nodes=nodes, edges=edges, parameters_by_id=node_parameters,+ )+ if report["errors"]:+ return {+ "success": False,+ "error": "validation_failed",+ "report": report,+ }++ # 2. Canonical requirements (don't trust the embedded manifest).+ requirements = extract_requirements(nodes)++ # 3. Cross-check stored credentials.+ missing_credentials = await cross_check_credentials(requirements, auth_service)++ # 4. Name conflict check.+ name_check = await check_name_conflict(proposed_name, database)++ # 5. If anything needs user confirmation, return preview without saving.+ needs_preview = (+ bool(missing_credentials) and not force_credentials+ ) or name_check["has_conflict"]+ if needs_preview:+ return {+ "success": True,+ "preview": True,+ "report": report,+ "missing_credentials": missing_credentials,+ "name_conflict": name_check["has_conflict"],+ "suggested_name": name_check["suggested_name"],+ "requirements": requirements,+ }++ # 6. Remap node ids — protects the node_parameters table from+ # duplicate-id collisions across imports.+ remapped_nodes, remapped_edges, remapped_params = remap_node_ids(+ nodes, edges, node_parameters,+ )++ # 7. Save. Workflow id format matches the frontend's generateWorkflowId.+ workflow_id = f"workflow-{int(time.time() * 1000)}-{secrets.token_hex(4)}"+ saved = await database.save_workflow(+ workflow_id=workflow_id,+ name=proposed_name,+ description=workflow.get("description"),+ data={"nodes": remapped_nodes, "edges": remapped_edges},+ )+ if not saved:+ return {"success": False, "error": "save_failed", "report": report}++ # 8. Per-node parameter saves under the remapped ids.+ saved_params = 0+ for node_id, params in remapped_params.items():+ if not params:+ continue+ try:+ await database.save_node_parameters(node_id, params)+ saved_params += 1+ except Exception as e:+ logger.error(+ "[workflow_import] save_node_parameters failed for %s: %s",+ node_id, e,+ )++ # 9. CloudEvents broadcast — ``workflow.imported`` envelope. Other+ # connected clients (browser tabs) listen for this and invalidate+ # their workflows query so the sidebar picks up the new entry+ # without a manual refresh. Broadcast failure must never fail the+ # import — the workflow IS saved either way.+ try:+ from services.status_broadcaster import get_status_broadcaster+ await get_status_broadcaster().broadcast_workflow_lifecycle(+ "imported",+ workflow_id=workflow_id,+ name=proposed_name,+ node_count=len(remapped_nodes),+ edge_count=len(remapped_edges),+ )+ except Exception:+ logger.debug(+ "[workflow_import] broadcast_workflow_lifecycle failed",+ exc_info=True,+ )++ return {+ "success": True,+ "preview": False,+ "workflow_id": workflow_id,+ "name": proposed_name,+ "node_count": len(remapped_nodes),+ "edge_count": len(remapped_edges),+ "saved_parameters": saved_params,+ "report": report,+ }diff --git a/server/tests/test_workflow_validator.py b/server/tests/test_workflow_validator.pyindex cebef0d7..b35a9562 100644--- a/server/tests/test_workflow_validator.py+++ b/server/tests/test_workflow_validator.py@@ -305,3 +305,510 @@ class TestExecuteAndDeployHandlersGate:"import_examples_for_user must run validate_workflow before ""save_workflow — broken examples shipped on disk are bugs.")++ def test_example_loader_remaps_node_ids(self):+ """Example loader MUST rewrite node ids before save. Without this,+ two example workflows that share a node id (today: AI Assistant ++ Claude Assistant share 6 trigger/console/memory nodes) collide in+ the node_parameters table (keyed unique on node_id), and the+ second example silently overwrites the first's parameters."""+ import inspect+ from services import example_loader++ src = inspect.getsource(example_loader.import_examples_for_user)+ assert "remap_node_ids" in src, (+ "import_examples_for_user must call workflow_import.remap_node_ids "+ "before save_workflow / save_node_parameters."+ )+++class TestRemapNodeIds:+ """Regression contract for ``services.workflow_import.remap_node_ids``.+ Two example workflows in ``workflows/`` share 6 node ids — this helper+ is what prevents parameter overwrites at import time."""++ def test_remap_eliminates_duplicate_ids(self):+ """Two graphs with identical input ids must emit disjoint output+ id sets after independent remap calls."""+ from services.workflow_import import remap_node_ids++ nodes_a = [{"id": "shared-1", "type": "telegramSend", "data": {}}]+ nodes_b = [{"id": "shared-1", "type": "telegramSend", "data": {}}]+ a_nodes, _, _ = remap_node_ids(nodes_a, [], {"shared-1": {"x": 1}})+ b_nodes, _, _ = remap_node_ids(nodes_b, [], {"shared-1": {"x": 2}})++ a_ids = {n["id"] for n in a_nodes}+ b_ids = {n["id"] for n in b_nodes}+ assert a_ids.isdisjoint(b_ids), (+ f"Remap must produce disjoint id sets, got overlap: {a_ids & b_ids}"+ )++ def test_remap_rewrites_edge_refs(self):+ """Edges must point at the new node ids; no dangling references."""+ from services.workflow_import import remap_node_ids++ nodes = [+ {"id": "n1", "type": "start", "data": {}},+ {"id": "n2", "type": "httpRequest", "data": {}},+ ]+ edges = [{"id": "e1", "source": "n1", "target": "n2"}]+ new_nodes, new_edges, _ = remap_node_ids(nodes, edges, {})++ node_id_set = {n["id"] for n in new_nodes}+ assert len(new_edges) == 1+ assert new_edges[0]["source"] in node_id_set+ assert new_edges[0]["target"] in node_id_set++ def test_remap_rekeys_parameters(self):+ """nodeParameters keys must be remapped to match the new node ids."""+ from services.workflow_import import remap_node_ids++ nodes = [{"id": "orig-1", "type": "telegramSend", "data": {}}]+ new_nodes, _, new_params = remap_node_ids(+ nodes, [], {"orig-1": {"text": "hello"}}+ )+ assert "orig-1" not in new_params+ assert new_nodes[0]["id"] in new_params+ assert new_params[new_nodes[0]["id"]] == {"text": "hello"}++ def test_remap_drops_orphan_parameters(self):+ """A parameter entry whose node isn't in the graph is dead data+ (rare, only happens when an export was hand-edited). Dropped."""+ from services.workflow_import import remap_node_ids++ _, _, new_params = remap_node_ids(+ [{"id": "n1", "type": "start", "data": {}}],+ [],+ {"n1": {"x": 1}, "ghost": {"y": 2}},+ )+ assert len(new_params) == 1, "orphan parameter entry should be dropped"++ def test_remap_real_examples_no_collisions(self):+ """Spot check against the actual example workflow files shipped in+ ``workflows/``: today AI Assistant + Claude Assistant share 6+ node ids; after remap, the union must be collision-free."""+ import collections+ import glob+ import json++ from services.example_loader import EXAMPLES_DIR+ from services.workflow_import import remap_node_ids++ all_ids = collections.defaultdict(list)+ for path in sorted(glob.glob(str(EXAMPLES_DIR / "*.json"))):+ with open(path, encoding="utf-8") as fh:+ wf = json.load(fh)+ nodes, _, _ = remap_node_ids(+ wf.get("nodes", []),+ wf.get("edges", []),+ wf.get("nodeParameters", {}),+ )+ for node in nodes:+ all_ids[node["id"]].append(path)++ collisions = {k: v for k, v in all_ids.items() if len(v) > 1}+ assert not collisions, (+ f"Remap should produce disjoint ids across all example "+ f"workflows; found collisions: {collisions}"+ )+++# ---------------------------------------------------------------------------+# workflow_import orchestrator tests+# ---------------------------------------------------------------------------+++class _FakeWorkflow:+ """Mimics the SQLModel.Workflow rows returned by get_all_workflows."""++ def __init__(self, name: str):+ self.name = name+++def _fake_database(existing_names: list[str] | None = None):+ """In-memory database double for the orchestrator tests."""+ saved_workflows: list[dict] = []+ saved_params: dict[str, dict] = {}++ db = MagicMock()+ db.get_all_workflows = AsyncMock(+ return_value=[_FakeWorkflow(n) for n in (existing_names or [])]+ )++ async def save_workflow(**kwargs):+ saved_workflows.append(kwargs)+ return True++ async def save_node_parameters(node_id, params):+ saved_params[node_id] = params+ return True++ db.save_workflow = AsyncMock(side_effect=save_workflow)+ db.save_node_parameters = AsyncMock(side_effect=save_node_parameters)+ db._saved_workflows = saved_workflows # test-readable+ db._saved_params = saved_params+ return db+++def _fake_auth(has_valid_key_return: bool = True):+ auth = MagicMock()+ auth.has_valid_key = AsyncMock(return_value=has_valid_key_return)+ return auth+++class TestImportWorkflowOrchestrator:+ """Contract for ``services.workflow_import.import_workflow``. Locks the+ two-step UX: first call returns preview when confirmations are needed+ (name conflict / missing creds), second call with confirmations saves."""++ async def test_save_path_with_no_conflicts_no_missing_creds(self, monkeypatch):+ """Happy path: valid workflow, no name conflict, no missing creds+ → preview is False, workflow saved, workflow_id returned."""+ from services.workflow_import import import_workflow++ # Patch the registry so the synthetic node type validates.+ _patch_registry_in_validator(monkeypatch)++ db = _fake_database(existing_names=[])+ auth = _fake_auth(has_valid_key_return=True)++ result = await import_workflow(+ {+ "name": "My Workflow",+ "nodes": [{"id": "n1", "type": "fakeNodeless", "data": {}}],+ "edges": [],+ },+ auth_service=auth,+ database=db,+ )+ assert result["success"] is True+ assert result["preview"] is False+ assert result["workflow_id"].startswith("workflow-")+ assert result["name"] == "My Workflow"+ assert len(db._saved_workflows) == 1++ async def test_preview_when_name_conflicts(self, monkeypatch):+ """Name conflict → preview=True with suggested_name; no save yet."""+ from services.workflow_import import import_workflow++ _patch_registry_in_validator(monkeypatch)+ db = _fake_database(existing_names=["My Workflow"])+ auth = _fake_auth(has_valid_key_return=True)++ result = await import_workflow(+ {+ "name": "My Workflow",+ "nodes": [{"id": "n1", "type": "fakeNodeless", "data": {}}],+ "edges": [],+ },+ auth_service=auth,+ database=db,+ )+ assert result["success"] is True+ assert result["preview"] is True+ assert result["name_conflict"] is True+ assert result["suggested_name"] == "My Workflow (imported)"+ assert len(db._saved_workflows) == 0 # NOT saved yet++ async def test_preview_when_credentials_missing(self, monkeypatch):+ """Missing credentials (without force flag) → preview=True with+ missing_credentials list; no save yet."""+ from services.workflow_import import import_workflow++ _patch_registry_in_validator(monkeypatch)+ # Patch CREDENTIAL_REGISTRY so cross-check can pull display info.+ monkeypatch.setattr(+ "services.plugin.credential.CREDENTIAL_REGISTRY",+ {"fake_provider": _FakeCredential},+ )+ db = _fake_database(existing_names=[])+ auth = _fake_auth(has_valid_key_return=False) # no creds stored++ result = await import_workflow(+ {+ "name": "Need Creds",+ "nodes": [{"id": "n1", "type": "fakeNode", "data": {"parameters": {"text": "ok"}}}],+ "edges": [],+ },+ auth_service=auth,+ database=db,+ )+ assert result["preview"] is True+ assert len(result["missing_credentials"]) == 1+ assert result["missing_credentials"][0]["provider_id"] == "fake_provider"+ assert len(db._saved_workflows) == 0++ async def test_force_credentials_skips_preview(self, monkeypatch):+ """force_credentials=True bypasses the missing-credential preview."""+ from services.workflow_import import import_workflow++ _patch_registry_in_validator(monkeypatch)+ monkeypatch.setattr(+ "services.plugin.credential.CREDENTIAL_REGISTRY",+ {"fake_provider": _FakeCredential},+ )+ # Mock the broadcaster — save path emits workflow.imported.+ _patch_broadcaster(monkeypatch)+ db = _fake_database(existing_names=[])+ auth = _fake_auth(has_valid_key_return=False)++ result = await import_workflow(+ {+ "name": "Force",+ "nodes": [{"id": "n1", "type": "fakeNode", "data": {"parameters": {"text": "ok"}}}],+ "edges": [],+ },+ force_credentials=True,+ auth_service=auth,+ database=db,+ )+ assert result["success"] is True+ assert result["preview"] is False+ assert len(db._saved_workflows) == 1++ async def test_validation_errors_block_import(self, monkeypatch):+ """Validation errors (e.g. unknown node type, cycle) short-circuit+ with success=False — no preview, no save."""+ from services.workflow_import import import_workflow++ _patch_registry_in_validator(monkeypatch)+ db = _fake_database()+ auth = _fake_auth()++ result = await import_workflow(+ {+ "name": "Bad",+ "nodes": [{"id": "n1", "type": "doesNotExist", "data": {}}],+ "edges": [],+ },+ auth_service=auth,+ database=db,+ )+ assert result["success"] is False+ assert result["error"] == "validation_failed"+ assert any(i["code"] == "UNKNOWN_NODE_TYPE" for i in result["report"]["errors"])+ assert len(db._saved_workflows) == 0++ async def test_save_path_remaps_node_ids(self, monkeypatch):+ """Save path applies remap_node_ids so the saved workflow has+ fresh ids regardless of the input. Two saves of the same JSON+ produce disjoint id sets."""+ from services.workflow_import import import_workflow++ _patch_registry_in_validator(monkeypatch)+ _patch_broadcaster(monkeypatch)+ db = _fake_database(existing_names=[])+ auth = _fake_auth(has_valid_key_return=True)++ payload = {+ "name": "Remap A",+ "nodes": [{"id": "orig-1", "type": "fakeNodeless", "data": {}}],+ "edges": [],+ "nodeParameters": {"orig-1": {"x": 1}},+ }+ r1 = await import_workflow(payload, auth_service=auth, database=db)+ r2 = await import_workflow(+ {**payload, "name": "Remap B"}, auth_service=auth, database=db+ )++ # Saved node ids must differ across the two imports.+ saved_a_nodes = db._saved_workflows[0]["data"]["nodes"]+ saved_b_nodes = db._saved_workflows[1]["data"]["nodes"]+ a_ids = {n["id"] for n in saved_a_nodes}+ b_ids = {n["id"] for n in saved_b_nodes}+ assert a_ids.isdisjoint(b_ids), (+ f"Two imports of the same JSON must produce disjoint node ids; "+ f"overlap: {a_ids & b_ids}"+ )+ # Parameters saved under the NEW ids, not the original.+ assert "orig-1" not in db._saved_params+ assert len(db._saved_params) >= 2 # both r1 and r2 saved their params+ # Sanity: r1 and r2 both succeeded.+ assert r1["success"] and r2["success"]++ async def test_save_path_broadcasts_workflow_imported(self, monkeypatch):+ """Save path must emit a CloudEvents-typed workflow.imported event+ via broadcast_workflow_lifecycle. Other connected clients listen+ for this and invalidate the workflows query."""+ from services.workflow_import import import_workflow++ _patch_registry_in_validator(monkeypatch)+ broadcaster, calls = _patch_broadcaster(monkeypatch)+ db = _fake_database()+ auth = _fake_auth()++ await import_workflow(+ {+ "name": "Broadcast Test",+ "nodes": [{"id": "n1", "type": "fakeNodeless", "data": {}}],+ "edges": [],+ },+ auth_service=auth,+ database=db,+ )+ assert len(calls) == 1, (+ "import_workflow save path must call broadcast_workflow_lifecycle "+ "exactly once on success."+ )+ stage, kwargs = calls[0]+ assert stage == "imported"+ assert kwargs["workflow_id"].startswith("workflow-")+ assert kwargs["name"] == "Broadcast Test"+++def _patch_registry_in_validator(monkeypatch):+ """Patch ``services.workflow_validator.get_node_class`` (the lookup+ used by the validator)."""+ registry = {"fakeNodeless": _FakeNodelessClass, "fakeNode": _FakeNodeClass}+ monkeypatch.setattr(+ "services.workflow_validator.get_node_class",+ lambda t: registry.get(t),+ )+ # workflow_import.extract_requirements imports get_node_class too.+ monkeypatch.setattr(+ "services.workflow_import.get_node_class",+ lambda t: registry.get(t),+ )+++def _patch_broadcaster(monkeypatch):+ """Replace get_status_broadcaster with a stub that records lifecycle+ calls so tests can assert on them without touching real WebSockets."""+ calls: list[tuple[str, dict]] = []++ class _StubBroadcaster:+ async def broadcast_workflow_lifecycle(self, stage: str, **kwargs):+ calls.append((stage, kwargs))++ stub = _StubBroadcaster()+ monkeypatch.setattr(+ "services.status_broadcaster.get_status_broadcaster", lambda: stub,+ )+ return stub, calls+++# ---------------------------------------------------------------------------+# extract_requirements + cross_check_credentials + check_name_conflict+# ---------------------------------------------------------------------------+++class TestExtractRequirements:+ def test_pulls_credential_ids_and_node_versions(self, monkeypatch):+ from services.workflow_import import extract_requirements++ monkeypatch.setattr(+ "services.workflow_import.get_node_class",+ lambda t: {+ "fakeNode": _FakeNodeClass,+ "fakeNodeless": _FakeNodelessClass,+ }.get(t),+ )++ reqs = extract_requirements([+ {"id": "n1", "type": "fakeNode"},+ {"id": "n2", "type": "fakeNodeless"},+ ])+ cred_ids = [c["provider_id"] for c in reqs["credentials"]]+ types = [n["type"] for n in reqs["nodes"]]+ assert "fake_provider" in cred_ids+ assert "fakeNode" in types+ assert "fakeNodeless" in types++ def test_ignores_unknown_types(self, monkeypatch):+ from services.workflow_import import extract_requirements++ monkeypatch.setattr(+ "services.workflow_import.get_node_class",+ lambda t: None,+ )+ reqs = extract_requirements([{"id": "n1", "type": "unknown"}])+ assert reqs == {"credentials": [], "nodes": []}+++class TestCrossCheckCredentials:+ async def test_reports_missing_with_display_info(self, monkeypatch):+ from services.workflow_import import cross_check_credentials++ monkeypatch.setattr(+ "services.plugin.credential.CREDENTIAL_REGISTRY",+ {"fake_provider": _FakeCredential},+ )+ auth = _fake_auth(has_valid_key_return=False)++ missing = await cross_check_credentials(+ {"credentials": [{"provider_id": "fake_provider"}]}, auth+ )+ assert len(missing) == 1+ assert missing[0]["provider_id"] == "fake_provider"+ assert "display_name" in missing[0]+ assert missing[0]["kind"] == "api_key"++ async def test_empty_when_all_stored(self, monkeypatch):+ from services.workflow_import import cross_check_credentials++ auth = _fake_auth(has_valid_key_return=True)+ missing = await cross_check_credentials(+ {"credentials": [{"provider_id": "x"}]}, auth+ )+ assert missing == []+++class TestCheckNameConflict:+ async def test_detects_conflict_and_suggests_name(self):+ from services.workflow_import import check_name_conflict++ db = _fake_database(existing_names=["My Flow", "Other"])+ result = await check_name_conflict("My Flow", db)+ assert result["has_conflict"] is True+ assert result["suggested_name"] == "My Flow (imported)"++ async def test_increments_suffix_when_suggestion_also_conflicts(self):+ from services.workflow_import import check_name_conflict++ db = _fake_database(existing_names=["A", "A (imported)"])+ result = await check_name_conflict("A", db)+ assert result["suggested_name"] == "A (imported) 2"++ async def test_no_conflict_returns_none(self):+ from services.workflow_import import check_name_conflict++ db = _fake_database(existing_names=["Other"])+ result = await check_name_conflict("Unique", db)+ assert result["has_conflict"] is False+ assert result["suggested_name"] is None+++class TestCloudEventsBroadcastShape:+ """The save path emits a real WorkflowEvent envelope. Static-source+ check that the broadcaster method uses the typed factory."""++ def test_broadcaster_uses_workflow_lifecycle_factory(self):+ import inspect++ from services import status_broadcaster++ src = inspect.getsource(status_broadcaster.StatusBroadcaster.broadcast_workflow_lifecycle)+ assert "WorkflowEvent.workflow_lifecycle" in src, (+ "broadcast_workflow_lifecycle must build the envelope via the "+ "typed factory so the dataschema URI + reverse-DNS type prefix "+ "are consistent with the rest of the CloudEvents surface."+ )+ assert '"workflow_lifecycle"' in src, (+ "wire-format key must be 'workflow_lifecycle' so the frontend "+ "WebSocketContext.tsx case statement routes correctly."+ )++ def test_import_workflow_emits_imported_lifecycle(self):+ import inspect++ from services import workflow_import++ src = inspect.getsource(workflow_import.import_workflow)+ assert "broadcast_workflow_lifecycle" in src, (+ "import_workflow save path must broadcast via "+ "broadcast_workflow_lifecycle so connected clients refresh."+ )+ assert '"imported"' in src, (+ "import_workflow must use the 'imported' lifecycle stage "+ "(matches WorkflowEvent.workflow_lifecycle Literal)."+ )