Skip to content

Self-improving AI Employees for everyone

feat(workflow_ops): protocol for backend-driven canvas mutations

trohitga69d07f817fd
diff --git a/CLAUDE.md b/CLAUDE.md
index b8152463..d765ffd0 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -600,6 +600,8 @@ The project uses WebSocket as the primary communication method between frontend
- `server/routers/websocket.py` - WebSocket endpoint with 89 message handlers
- `server/services/status_broadcaster.py` - Connection management and broadcasting
+**Canvas mutations from the backend** -- any handler that needs to add / move / delete nodes or edges (e.g., auto-add-skill on tool connect, future workflow-template features, AI-suggested edits) returns a workflow-ops batch (`{operations: [...]}`) and the frontend applies it through `applyOperations` in [client/src/lib/workflowOps.ts](./client/src/lib/workflowOps.ts). Backend builders live in [server/services/workflow_ops.py](./server/services/workflow_ops.py). Full spec: [docs-internal/workflow_ops_protocol.md](./docs-internal/workflow_ops_protocol.md). Do not invent a parallel "modify the canvas" RPC for new features -- target this protocol.
+
## Implemented Node Types
The following 92 nodes are currently implemented and functional:
diff --git a/client/src/lib/workflowOps.ts b/client/src/lib/workflowOps.ts
new file mode 100644
index 00000000..0c70939e
--- /dev/null
+++ b/client/src/lib/workflowOps.ts
@@ -0,0 +1,291 @@
+/**
+ * Workflow Operations protocol -- frontend applier.
+ *
+ * The standard wire format any backend service uses to mutate the
+ * React Flow canvas. Backend services return `{operations: [...]}`
+ * (see server/services/workflow_ops.py); this module walks the list,
+ * generating real React Flow node ids, resolving cross-op references
+ * via `client_ref` placeholders, and persisting parameter changes
+ * through `saveNodeParameters`.
+ *
+ * Full protocol spec: docs-internal/workflow_ops_protocol.md
+ *
+ * Adding a new operation type:
+ * 1. Mirror the TypedDict from server/services/workflow_ops.py
+ * below.
+ * 2. Add the apply branch to applyOperations.
+ * 3. Document the new op in workflow_ops_protocol.md.
+ */
+
+import type { Node, Edge } from 'reactflow';
+
+// ---------------------------------------------------------------------------
+// Wire types -- mirror server/services/workflow_ops.py 1:1
+// ---------------------------------------------------------------------------
+
+export type AbsolutePosition = { x: number; y: number };
+export type AnchoredPosition = {
+ anchor_node_id: string;
+ offset?: { x?: number; y?: number };
+ fallback?: AbsolutePosition;
+};
+export type PositionSpec = AbsolutePosition | AnchoredPosition;
+
+export type NodeRef = string | { client_ref: string };
+
+export interface AddNodeOp {
+ type: 'add_node';
+ client_ref: string;
+ node_type: string;
+ parameters: Record<string, any>;
+ label?: string;
+ position?: PositionSpec;
+}
+
+export interface AddEdgeOp {
+ type: 'add_edge';
+ source: NodeRef;
+ target: NodeRef;
+ source_handle?: string;
+ target_handle?: string;
+}
+
+export interface SetNodeParametersOp {
+ type: 'set_node_parameters';
+ node_id: string;
+ parameters: Record<string, any>;
+}
+
+export interface DeleteNodeOp {
+ type: 'delete_node';
+ node_id: string;
+}
+
+export interface DeleteEdgeOp {
+ type: 'delete_edge';
+ edge_id: string;
+}
+
+export interface MoveNodeOp {
+ type: 'move_node';
+ node_id: string;
+ position: PositionSpec;
+}
+
+export interface ReplaceNodeOp {
+ type: 'replace_node';
+ node_id: string;
+ node_type: string;
+ parameters: Record<string, any>;
+ label?: string;
+ preserve_edges?: boolean;
+}
+
+export type WorkflowOperation =
+ | AddNodeOp
+ | AddEdgeOp
+ | SetNodeParametersOp
+ | DeleteNodeOp
+ | DeleteEdgeOp
+ | MoveNodeOp
+ | ReplaceNodeOp;
+
+// ---------------------------------------------------------------------------
+// Apply context + result
+// ---------------------------------------------------------------------------
+
+export interface ApplyContext {
+ /** Current React Flow nodes (read-only -- used for anchor resolution). */
+ nodes: Node[];
+ /** Current React Flow edges (read-only -- used for delete-cascade). */
+ edges: Edge[];
+ setNodes: (updater: (ns: Node[]) => Node[]) => void;
+ setEdges: (updater: (es: Edge[]) => Edge[]) => void;
+ /** Persist params for a node id; backend WS round-trip. */
+ saveNodeParameters: (nodeId: string, parameters: Record<string, any>) => Promise<boolean>;
+ /** Optional fallback position when an anchor node is missing. */
+ defaultPosition?: AbsolutePosition;
+}
+
+export interface OperationError {
+ op: WorkflowOperation;
+ message: string;
+}
+
+export interface ApplyResult {
+ applied: number;
+ errors: OperationError[];
+ /** client_ref -> generated React Flow node id, populated by add_node ops. */
+ refMap: Record<string, string>;
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+function newId(nodeType: string): string {
+ return `${nodeType}-${Date.now()}-${Math.floor(Math.random() * 1000)}`;
+}
+
+function resolveNodeRef(ref: NodeRef, refMap: Record<string, string>): string | null {
+ if (typeof ref === 'string') return ref;
+ return refMap[ref.client_ref] ?? null;
+}
+
+function resolvePosition(
+ spec: PositionSpec | undefined,
+ ctx: ApplyContext,
+ liveNodes: Node[],
+): AbsolutePosition {
+ if (!spec) return ctx.defaultPosition ?? { x: 200, y: 200 };
+ if ('x' in spec && 'y' in spec) return { x: spec.x, y: spec.y };
+ const anchor = liveNodes.find(n => n.id === spec.anchor_node_id);
+ if (anchor) {
+ return {
+ x: anchor.position.x + (spec.offset?.x ?? 0),
+ y: anchor.position.y + (spec.offset?.y ?? 0),
+ };
+ }
+ return spec.fallback ?? ctx.defaultPosition ?? { x: 200, y: 200 };
+}
+
+// ---------------------------------------------------------------------------
+// Apply
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply a workflow-ops batch to the canvas.
+ *
+ * Operations apply in order. `client_ref` placeholders in `add_node`
+ * ops produce real ids that subsequent ops can reference via
+ * `{client_ref: "..."}`. Failures are collected per op; the function
+ * does not throw -- callers inspect the returned `errors` array.
+ */
+export async function applyOperations(
+ ops: WorkflowOperation[],
+ ctx: ApplyContext,
+): Promise<ApplyResult> {
+ const result: ApplyResult = { applied: 0, errors: [], refMap: {} };
+
+ // Local mutable copies so anchor-resolution sees nodes added
+ // earlier in the batch (setNodes is async w.r.t. React).
+ let liveNodes = [...ctx.nodes];
+ let liveEdges = [...ctx.edges];
+
+ for (const op of ops) {
+ try {
+ switch (op.type) {
+ case 'add_node': {
+ const id = newId(op.node_type);
+ result.refMap[op.client_ref] = id;
+ const position = resolvePosition(op.position, ctx, liveNodes);
+ const label = op.label ?? op.parameters?.label ?? op.node_type;
+ const node: Node = {
+ id,
+ type: op.node_type,
+ position,
+ data: { label },
+ };
+ liveNodes = liveNodes.concat(node);
+ ctx.setNodes(ns => ns.concat(node));
+ await ctx.saveNodeParameters(id, { label, ...op.parameters });
+ break;
+ }
+
+ case 'add_edge': {
+ const source = resolveNodeRef(op.source, result.refMap);
+ const target = resolveNodeRef(op.target, result.refMap);
+ if (!source || !target) {
+ throw new Error('add_edge: unresolved client_ref');
+ }
+ const edge: Edge = {
+ id: `e-${source}-${target}-${Date.now()}-${Math.floor(Math.random() * 1000)}`,
+ source,
+ target,
+ sourceHandle: op.source_handle ?? null,
+ targetHandle: op.target_handle ?? null,
+ };
+ liveEdges = liveEdges.concat(edge);
+ ctx.setEdges(es => es.concat(edge));
+ break;
+ }
+
+ case 'set_node_parameters': {
+ const ok = await ctx.saveNodeParameters(op.node_id, op.parameters);
+ if (!ok) throw new Error(`saveNodeParameters returned false for ${op.node_id}`);
+ break;
+ }
+
+ case 'delete_node': {
+ liveNodes = liveNodes.filter(n => n.id !== op.node_id);
+ liveEdges = liveEdges.filter(e => e.source !== op.node_id && e.target !== op.node_id);
+ ctx.setNodes(ns => ns.filter(n => n.id !== op.node_id));
+ ctx.setEdges(es => es.filter(e => e.source !== op.node_id && e.target !== op.node_id));
+ break;
+ }
+
+ case 'delete_edge': {
+ liveEdges = liveEdges.filter(e => e.id !== op.edge_id);
+ ctx.setEdges(es => es.filter(e => e.id !== op.edge_id));
+ break;
+ }
+
+ case 'move_node': {
+ const position = resolvePosition(op.position, ctx, liveNodes);
+ liveNodes = liveNodes.map(n => (n.id === op.node_id ? { ...n, position } : n));
+ ctx.setNodes(ns => ns.map(n => (n.id === op.node_id ? { ...n, position } : n)));
+ break;
+ }
+
+ case 'replace_node': {
+ const existing = liveNodes.find(n => n.id === op.node_id);
+ if (!existing) throw new Error(`replace_node: node ${op.node_id} not found`);
+ const newNodeId = newId(op.node_type);
+ const label = op.label ?? op.parameters?.label ?? op.node_type;
+ const replacement: Node = {
+ id: newNodeId,
+ type: op.node_type,
+ position: existing.position,
+ data: { label },
+ };
+ // Drop old, add new.
+ liveNodes = liveNodes.filter(n => n.id !== op.node_id).concat(replacement);
+ ctx.setNodes(ns => ns.filter(n => n.id !== op.node_id).concat(replacement));
+ // Edges: rewire to the new id (preserve_edges defaults to true).
+ if (op.preserve_edges !== false) {
+ liveEdges = liveEdges.map(e => {
+ if (e.source === op.node_id) return { ...e, source: newNodeId };
+ if (e.target === op.node_id) return { ...e, target: newNodeId };
+ return e;
+ });
+ ctx.setEdges(es => es.map(e => {
+ if (e.source === op.node_id) return { ...e, source: newNodeId };
+ if (e.target === op.node_id) return { ...e, target: newNodeId };
+ return e;
+ }));
+ } else {
+ liveEdges = liveEdges.filter(e => e.source !== op.node_id && e.target !== op.node_id);
+ ctx.setEdges(es => es.filter(e => e.source !== op.node_id && e.target !== op.node_id));
+ }
+ await ctx.saveNodeParameters(newNodeId, { label, ...op.parameters });
+ break;
+ }
+
+ default: {
+ // Exhaustiveness check -- TS will yell if a new op type is
+ // added without an apply branch.
+ const _exhaustive: never = op;
+ throw new Error(`Unknown op type: ${(_exhaustive as any)?.type}`);
+ }
+ }
+ result.applied += 1;
+ } catch (err) {
+ result.errors.push({
+ op,
+ message: err instanceof Error ? err.message : String(err),
+ });
+ }
+ }
+
+ return result;
+}
diff --git a/docs-internal/workflow_ops_protocol.md b/docs-internal/workflow_ops_protocol.md
new file mode 100644
index 00000000..c4baf1e8
--- /dev/null
+++ b/docs-internal/workflow_ops_protocol.md
@@ -0,0 +1,135 @@
+# Workflow Operations Protocol
+
+The standard wire format any backend service uses to mutate the React
+Flow canvas. A service returns `{operations: [...]}`; the frontend
+walks the list with a single applier (`client/src/lib/workflowOps.ts`)
+that creates / removes nodes + edges and persists parameter changes.
+
+**Why a protocol**: keeps domain rules in the backend (visuals.json
+lookups, plugin-registry checks, canonical config shapes) without
+forcing each new feature to invent its own RPC + frontend glue.
+"Auto-add Skill on tool connect" is the first consumer; future
+features (workflow templates, AI-suggested edits, automation recipes)
+just emit op batches.
+
+## Wire shape
+
+```json
+{
+ "operations": [
+ { "type": "add_node", ... },
+ { "type": "add_edge", ... }
+ ]
+}
+```
+
+Operations apply **in order**. Earlier ops can produce React Flow node
+ids that later ops reference via `client_ref` placeholders.
+
+## Operation types (v1)
+
+| Type | Required | Optional | Purpose |
+|---|---|---|---|
+| `add_node` | `client_ref`, `node_type`, `parameters` | `label`, `position` | Create a new node + persist its initial params. `client_ref` is a batch-local id later ops can target. |
+| `add_edge` | `source`, `target` | `source_handle`, `target_handle` | Wire two nodes. `source` / `target` are either existing node ids (string) or `{client_ref}` references. |
+| `set_node_parameters` | `node_id`, `parameters` | -- | Shallow-merge `parameters` into the node's existing params via `saveNodeParameters`. |
+| `delete_node` | `node_id` | -- | Remove node; edges to/from it cascade-delete. |
+| `delete_edge` | `edge_id` | -- | Remove a single edge. |
+| `move_node` | `node_id`, `position` | -- | Reposition without changing identity. |
+| `replace_node` | `node_id`, `node_type`, `parameters` | `label`, `preserve_edges` (default `true`) | Atomic delete-and-add at the same position. With `preserve_edges`, edges to/from the old id rewire to the new id. |
+
+### `PositionSpec`
+
+Either absolute or anchored:
+
+```ts
+type PositionSpec =
+ | { x: number; y: number }
+ | { anchor_node_id: string;
+ offset?: { x?: number; y?: number };
+ fallback?: { x: number; y: number } };
+```
+
+Anchored positions stay backend-agnostic about pixel coords -- the
+frontend resolves the anchor against current React Flow state. If the
+anchor doesn't exist, `fallback` is used (or a sensible default).
+
+### `NodeRef`
+
+```ts
+type NodeRef = string | { client_ref: string };
+```
+
+A string is an existing React Flow node id. A `{client_ref}` is a
+placeholder that resolves to the id of an `add_node` op earlier in
+the same batch.
+
+## Application semantics
+
+- **Order**: ops apply sequentially. Anchor lookups and `client_ref`
+ resolution see the cumulative state of earlier ops in the batch.
+- **Best-effort**: a failed op (e.g. `delete_node` on a missing id) is
+ logged and reported in the result; subsequent ops still apply. v1
+ has no rollback. Backend services should write op sequences that are
+ robust to partial application.
+- **Not a diff reconciler**: re-applying a batch is not assumed safe.
+ The protocol is a one-shot mutation wire format.
+
+## Backend usage
+
+```python
+from services import workflow_ops
+
+ref = "new_master"
+return {
+ "operations": [
+ workflow_ops.add_node(
+ ref, "masterSkill",
+ {"skillsConfig": {...}},
+ label="Master Skill",
+ position=workflow_ops.anchored(agent_id, offset_x=-60, offset_y=220),
+ ),
+ workflow_ops.add_edge(
+ {"client_ref": ref}, agent_id,
+ source_handle="output-tool", target_handle="input-skill",
+ ),
+ ],
+}
+```
+
+Helpers live in `server/services/workflow_ops.py` (TypedDicts +
+builder functions). Empty result: `workflow_ops.empty()` returns
+`{"operations": []}`.
+
+## Frontend usage
+
+```ts
+import { applyOperations } from '@/lib/workflowOps';
+
+const result = await applyOperations(operations, {
+ nodes, edges, setNodes, setEdges, saveNodeParameters,
+});
+// result.applied -> count of successful ops
+// result.errors -> [{op, message}, ...]
+// result.refMap -> { client_ref: generated_node_id, ... }
+```
+
+`applyOperations` does not throw; callers inspect `errors` to surface
+failures (toast, log, retry, etc.).
+
+## Current consumers
+
+| Service | WS handler | Module |
+|---|---|---|
+| Auto-add Skill on tool connect | `evaluate_auto_skill` | `server/services/auto_skill.py` |
+
+## Adding a new consumer
+
+1. Write a backend service that returns `{operations: [...]}` using
+ the builder helpers in `services/workflow_ops`.
+2. Add a thin `@ws_handler` that calls into the service.
+3. On the frontend, send the WS request and pass the returned
+ `operations` into `applyOperations`. No new applier code.
+4. If you need a new op type, follow the steps in the docstring of
+ `server/services/workflow_ops.py` (mirror the TypedDict in TS,
+ add an apply branch, document here).
diff --git a/server/services/workflow_ops.py b/server/services/workflow_ops.py
new file mode 100644
index 00000000..8a3cc7b8
--- /dev/null
+++ b/server/services/workflow_ops.py
@@ -0,0 +1,241 @@
+"""Workflow Operations protocol.
+
+The standard wire format any backend service uses to mutate the
+React Flow canvas. A service returns ``{"operations": [...]}`` and the
+frontend's single applier (``client/src/lib/workflowOps.ts``) walks
+the list, generating real React Flow node ids, resolving cross-op
+references, and persisting parameter changes through
+``saveNodeParameters``.
+
+See ``docs-internal/workflow_ops_protocol.md`` for the full spec.
+
+Protocol notes:
+
+* Operations apply **in order**. Earlier ops can produce ids that
+ later ops reference via ``client_ref`` placeholders.
+* Failures are logged + reported per op; subsequent ops still apply.
+ v1 has no rollback -- write op sequences that are robust to partial
+ application.
+* The protocol is a declarative wire format, not a diff reconciler.
+ Re-applying a batch is not assumed to be idempotent.
+
+Adding a new operation type:
+
+1. Add the TypedDict below + extend ``WorkflowOperation`` union.
+2. Add a builder helper below.
+3. Mirror the type in ``client/src/lib/workflowOps.ts``.
+4. Add the apply branch to ``applyOperations`` in that same TS file.
+5. Document the new op in ``docs-internal/workflow_ops_protocol.md``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
+
+
+# ---------------------------------------------------------------------------
+# Position spec
+# ---------------------------------------------------------------------------
+
+class AbsolutePosition(TypedDict):
+ x: float
+ y: float
+
+
+class _AnchorOffset(TypedDict, total=False):
+ x: float
+ y: float
+
+
+class AnchoredPosition(TypedDict, total=False):
+ anchor_node_id: str
+ offset: _AnchorOffset
+ fallback: AbsolutePosition
+
+
+# Either {x,y} or {anchor_node_id, offset?, fallback?}. The frontend
+# resolves the anchor against current React Flow node positions.
+PositionSpec = Union[AbsolutePosition, AnchoredPosition]
+
+
+# ---------------------------------------------------------------------------
+# Node references
+# ---------------------------------------------------------------------------
+
+class _ClientRef(TypedDict):
+ client_ref: str
+
+
+# Existing-id strings mix freely with batch-local ``{client_ref}``
+# placeholders so an add_edge op can target a node added earlier in
+# the same batch before its real id exists.
+NodeRef = Union[str, _ClientRef]
+
+
+# ---------------------------------------------------------------------------
+# Operation TypedDicts
+# ---------------------------------------------------------------------------
+
+class AddNodeOp(TypedDict, total=False):
+ type: Literal["add_node"]
+ client_ref: str
+ node_type: str
+ parameters: Dict[str, Any]
+ label: str
+ position: PositionSpec
+
+
+class AddEdgeOp(TypedDict, total=False):
+ type: Literal["add_edge"]
+ source: NodeRef
+ target: NodeRef
+ source_handle: str
+ target_handle: str
+
+
+class SetNodeParametersOp(TypedDict, total=False):
+ type: Literal["set_node_parameters"]
+ node_id: str
+ parameters: Dict[str, Any]
+
+
+class DeleteNodeOp(TypedDict):
+ type: Literal["delete_node"]
+ node_id: str
+
+
+class DeleteEdgeOp(TypedDict):
+ type: Literal["delete_edge"]
+ edge_id: str
+
+
+class MoveNodeOp(TypedDict):
+ type: Literal["move_node"]
+ node_id: str
+ position: PositionSpec
+
+
+class ReplaceNodeOp(TypedDict, total=False):
+ type: Literal["replace_node"]
+ node_id: str
+ node_type: str
+ parameters: Dict[str, Any]
+ label: str
+ preserve_edges: bool
+
+
+WorkflowOperation = Union[
+ AddNodeOp,
+ AddEdgeOp,
+ SetNodeParametersOp,
+ DeleteNodeOp,
+ DeleteEdgeOp,
+ MoveNodeOp,
+ ReplaceNodeOp,
+]
+
+
+# ---------------------------------------------------------------------------
+# Builder helpers
+# ---------------------------------------------------------------------------
+
+def add_node(
+ client_ref: str,
+ node_type: str,
+ parameters: Optional[Dict[str, Any]] = None,
+ *,
+ label: Optional[str] = None,
+ position: Optional[PositionSpec] = None,
+) -> AddNodeOp:
+ op: AddNodeOp = {
+ "type": "add_node",
+ "client_ref": client_ref,
+ "node_type": node_type,
+ "parameters": parameters or {},
+ }
+ if label is not None:
+ op["label"] = label
+ if position is not None:
+ op["position"] = position
+ return op
+
+
+def add_edge(
+ source: NodeRef,
+ target: NodeRef,
+ *,
+ source_handle: Optional[str] = None,
+ target_handle: Optional[str] = None,
+) -> AddEdgeOp:
+ op: AddEdgeOp = {
+ "type": "add_edge",
+ "source": source,
+ "target": target,
+ }
+ if source_handle is not None:
+ op["source_handle"] = source_handle
+ if target_handle is not None:
+ op["target_handle"] = target_handle
+ return op
+
+
+def set_node_parameters(node_id: str, parameters: Dict[str, Any]) -> SetNodeParametersOp:
+ return {
+ "type": "set_node_parameters",
+ "node_id": node_id,
+ "parameters": parameters,
+ }
+
+
+def delete_node(node_id: str) -> DeleteNodeOp:
+ return {"type": "delete_node", "node_id": node_id}
+
+
+def delete_edge(edge_id: str) -> DeleteEdgeOp:
+ return {"type": "delete_edge", "edge_id": edge_id}
+
+
+def move_node(node_id: str, position: PositionSpec) -> MoveNodeOp:
+ return {"type": "move_node", "node_id": node_id, "position": position}
+
+
+def replace_node(
+ node_id: str,
+ node_type: str,
+ parameters: Optional[Dict[str, Any]] = None,
+ *,
+ label: Optional[str] = None,
+ preserve_edges: bool = True,
+) -> ReplaceNodeOp:
+ op: ReplaceNodeOp = {
+ "type": "replace_node",
+ "node_id": node_id,
+ "node_type": node_type,
+ "parameters": parameters or {},
+ "preserve_edges": preserve_edges,
+ }
+ if label is not None:
+ op["label"] = label
+ return op
+
+
+def anchored(
+ anchor_node_id: str,
+ *,
+ offset_x: float = 0,
+ offset_y: float = 0,
+ fallback: Optional[AbsolutePosition] = None,
+) -> AnchoredPosition:
+ """Convenience builder for ``{anchor_node_id, offset, fallback}``."""
+ pos: AnchoredPosition = {
+ "anchor_node_id": anchor_node_id,
+ "offset": {"x": offset_x, "y": offset_y},
+ }
+ if fallback is not None:
+ pos["fallback"] = fallback
+ return pos
+
+
+def empty() -> Dict[str, List[WorkflowOperation]]:
+ """Standard empty response (``{"operations": []}``)."""
+ return {"operations": []}