Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(workflow_ops): protocol for backend-driven canvas mutations
trohitga69d07f817fd
diff --git a/CLAUDE.md b/CLAUDE.mdindex b8152463..d765ffd0 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -600,6 +600,8 @@ The project uses WebSocket as the primary communication method between frontend- `server/routers/websocket.py` - WebSocket endpoint with 89 message handlers- `server/services/status_broadcaster.py` - Connection management and broadcasting+**Canvas mutations from the backend** -- any handler that needs to add / move / delete nodes or edges (e.g., auto-add-skill on tool connect, future workflow-template features, AI-suggested edits) returns a workflow-ops batch (`{operations: [...]}`) and the frontend applies it through `applyOperations` in [client/src/lib/workflowOps.ts](./client/src/lib/workflowOps.ts). Backend builders live in [server/services/workflow_ops.py](./server/services/workflow_ops.py). Full spec: [docs-internal/workflow_ops_protocol.md](./docs-internal/workflow_ops_protocol.md). Do not invent a parallel "modify the canvas" RPC for new features -- target this protocol.+## Implemented Node TypesThe following 92 nodes are currently implemented and functional:diff --git a/client/src/lib/workflowOps.ts b/client/src/lib/workflowOps.tsnew file mode 100644index 00000000..0c70939e--- /dev/null+++ b/client/src/lib/workflowOps.ts@@ -0,0 +1,291 @@+/**+ * Workflow Operations protocol -- frontend applier.+ *+ * The standard wire format any backend service uses to mutate the+ * React Flow canvas. Backend services return `{operations: [...]}`+ * (see server/services/workflow_ops.py); this module walks the list,+ * generating real React Flow node ids, resolving cross-op references+ * via `client_ref` placeholders, and persisting parameter changes+ * through `saveNodeParameters`.+ *+ * Full protocol spec: docs-internal/workflow_ops_protocol.md+ *+ * Adding a new operation type:+ * 1. Mirror the TypedDict from server/services/workflow_ops.py+ * below.+ * 2. Add the apply branch to applyOperations.+ * 3. Document the new op in workflow_ops_protocol.md.+ */++import type { Node, Edge } from 'reactflow';++// ---------------------------------------------------------------------------+// Wire types -- mirror server/services/workflow_ops.py 1:1+// ---------------------------------------------------------------------------++export type AbsolutePosition = { x: number; y: number };+export type AnchoredPosition = {+ anchor_node_id: string;+ offset?: { x?: number; y?: number };+ fallback?: AbsolutePosition;+};+export type PositionSpec = AbsolutePosition | AnchoredPosition;++export type NodeRef = string | { client_ref: string };++export interface AddNodeOp {+ type: 'add_node';+ client_ref: string;+ node_type: string;+ parameters: Record<string, any>;+ label?: string;+ position?: PositionSpec;+}++export interface AddEdgeOp {+ type: 'add_edge';+ source: NodeRef;+ target: NodeRef;+ source_handle?: string;+ target_handle?: string;+}++export interface SetNodeParametersOp {+ type: 'set_node_parameters';+ node_id: string;+ parameters: Record<string, any>;+}++export interface DeleteNodeOp {+ type: 'delete_node';+ node_id: string;+}++export interface DeleteEdgeOp {+ type: 'delete_edge';+ edge_id: string;+}++export interface MoveNodeOp {+ type: 'move_node';+ node_id: string;+ position: PositionSpec;+}++export interface ReplaceNodeOp {+ type: 'replace_node';+ node_id: string;+ node_type: string;+ parameters: Record<string, any>;+ label?: string;+ preserve_edges?: boolean;+}++export type WorkflowOperation =+ | AddNodeOp+ | AddEdgeOp+ | SetNodeParametersOp+ | DeleteNodeOp+ | DeleteEdgeOp+ | MoveNodeOp+ | ReplaceNodeOp;++// ---------------------------------------------------------------------------+// Apply context + result+// ---------------------------------------------------------------------------++export interface ApplyContext {+ /** Current React Flow nodes (read-only -- used for anchor resolution). */+ nodes: Node[];+ /** Current React Flow edges (read-only -- used for delete-cascade). */+ edges: Edge[];+ setNodes: (updater: (ns: Node[]) => Node[]) => void;+ setEdges: (updater: (es: Edge[]) => Edge[]) => void;+ /** Persist params for a node id; backend WS round-trip. */+ saveNodeParameters: (nodeId: string, parameters: Record<string, any>) => Promise<boolean>;+ /** Optional fallback position when an anchor node is missing. */+ defaultPosition?: AbsolutePosition;+}++export interface OperationError {+ op: WorkflowOperation;+ message: string;+}++export interface ApplyResult {+ applied: number;+ errors: OperationError[];+ /** client_ref -> generated React Flow node id, populated by add_node ops. */+ refMap: Record<string, string>;+}++// ---------------------------------------------------------------------------+// Internal helpers+// ---------------------------------------------------------------------------++function newId(nodeType: string): string {+ return `${nodeType}-${Date.now()}-${Math.floor(Math.random() * 1000)}`;+}++function resolveNodeRef(ref: NodeRef, refMap: Record<string, string>): string | null {+ if (typeof ref === 'string') return ref;+ return refMap[ref.client_ref] ?? null;+}++function resolvePosition(+ spec: PositionSpec | undefined,+ ctx: ApplyContext,+ liveNodes: Node[],+): AbsolutePosition {+ if (!spec) return ctx.defaultPosition ?? { x: 200, y: 200 };+ if ('x' in spec && 'y' in spec) return { x: spec.x, y: spec.y };+ const anchor = liveNodes.find(n => n.id === spec.anchor_node_id);+ if (anchor) {+ return {+ x: anchor.position.x + (spec.offset?.x ?? 0),+ y: anchor.position.y + (spec.offset?.y ?? 0),+ };+ }+ return spec.fallback ?? ctx.defaultPosition ?? { x: 200, y: 200 };+}++// ---------------------------------------------------------------------------+// Apply+// ---------------------------------------------------------------------------++/**+ * Apply a workflow-ops batch to the canvas.+ *+ * Operations apply in order. `client_ref` placeholders in `add_node`+ * ops produce real ids that subsequent ops can reference via+ * `{client_ref: "..."}`. Failures are collected per op; the function+ * does not throw -- callers inspect the returned `errors` array.+ */+export async function applyOperations(+ ops: WorkflowOperation[],+ ctx: ApplyContext,+): Promise<ApplyResult> {+ const result: ApplyResult = { applied: 0, errors: [], refMap: {} };++ // Local mutable copies so anchor-resolution sees nodes added+ // earlier in the batch (setNodes is async w.r.t. React).+ let liveNodes = [...ctx.nodes];+ let liveEdges = [...ctx.edges];++ for (const op of ops) {+ try {+ switch (op.type) {+ case 'add_node': {+ const id = newId(op.node_type);+ result.refMap[op.client_ref] = id;+ const position = resolvePosition(op.position, ctx, liveNodes);+ const label = op.label ?? op.parameters?.label ?? op.node_type;+ const node: Node = {+ id,+ type: op.node_type,+ position,+ data: { label },+ };+ liveNodes = liveNodes.concat(node);+ ctx.setNodes(ns => ns.concat(node));+ await ctx.saveNodeParameters(id, { label, ...op.parameters });+ break;+ }++ case 'add_edge': {+ const source = resolveNodeRef(op.source, result.refMap);+ const target = resolveNodeRef(op.target, result.refMap);+ if (!source || !target) {+ throw new Error('add_edge: unresolved client_ref');+ }+ const edge: Edge = {+ id: `e-${source}-${target}-${Date.now()}-${Math.floor(Math.random() * 1000)}`,+ source,+ target,+ sourceHandle: op.source_handle ?? null,+ targetHandle: op.target_handle ?? null,+ };+ liveEdges = liveEdges.concat(edge);+ ctx.setEdges(es => es.concat(edge));+ break;+ }++ case 'set_node_parameters': {+ const ok = await ctx.saveNodeParameters(op.node_id, op.parameters);+ if (!ok) throw new Error(`saveNodeParameters returned false for ${op.node_id}`);+ break;+ }++ case 'delete_node': {+ liveNodes = liveNodes.filter(n => n.id !== op.node_id);+ liveEdges = liveEdges.filter(e => e.source !== op.node_id && e.target !== op.node_id);+ ctx.setNodes(ns => ns.filter(n => n.id !== op.node_id));+ ctx.setEdges(es => es.filter(e => e.source !== op.node_id && e.target !== op.node_id));+ break;+ }++ case 'delete_edge': {+ liveEdges = liveEdges.filter(e => e.id !== op.edge_id);+ ctx.setEdges(es => es.filter(e => e.id !== op.edge_id));+ break;+ }++ case 'move_node': {+ const position = resolvePosition(op.position, ctx, liveNodes);+ liveNodes = liveNodes.map(n => (n.id === op.node_id ? { ...n, position } : n));+ ctx.setNodes(ns => ns.map(n => (n.id === op.node_id ? { ...n, position } : n)));+ break;+ }++ case 'replace_node': {+ const existing = liveNodes.find(n => n.id === op.node_id);+ if (!existing) throw new Error(`replace_node: node ${op.node_id} not found`);+ const newNodeId = newId(op.node_type);+ const label = op.label ?? op.parameters?.label ?? op.node_type;+ const replacement: Node = {+ id: newNodeId,+ type: op.node_type,+ position: existing.position,+ data: { label },+ };+ // Drop old, add new.+ liveNodes = liveNodes.filter(n => n.id !== op.node_id).concat(replacement);+ ctx.setNodes(ns => ns.filter(n => n.id !== op.node_id).concat(replacement));+ // Edges: rewire to the new id (preserve_edges defaults to true).+ if (op.preserve_edges !== false) {+ liveEdges = liveEdges.map(e => {+ if (e.source === op.node_id) return { ...e, source: newNodeId };+ if (e.target === op.node_id) return { ...e, target: newNodeId };+ return e;+ });+ ctx.setEdges(es => es.map(e => {+ if (e.source === op.node_id) return { ...e, source: newNodeId };+ if (e.target === op.node_id) return { ...e, target: newNodeId };+ return e;+ }));+ } else {+ liveEdges = liveEdges.filter(e => e.source !== op.node_id && e.target !== op.node_id);+ ctx.setEdges(es => es.filter(e => e.source !== op.node_id && e.target !== op.node_id));+ }+ await ctx.saveNodeParameters(newNodeId, { label, ...op.parameters });+ break;+ }++ default: {+ // Exhaustiveness check -- TS will yell if a new op type is+ // added without an apply branch.+ const _exhaustive: never = op;+ throw new Error(`Unknown op type: ${(_exhaustive as any)?.type}`);+ }+ }+ result.applied += 1;+ } catch (err) {+ result.errors.push({+ op,+ message: err instanceof Error ? err.message : String(err),+ });+ }+ }++ return result;+}diff --git a/docs-internal/workflow_ops_protocol.md b/docs-internal/workflow_ops_protocol.mdnew file mode 100644index 00000000..c4baf1e8--- /dev/null+++ b/docs-internal/workflow_ops_protocol.md@@ -0,0 +1,135 @@+# Workflow Operations Protocol++The standard wire format any backend service uses to mutate the React+Flow canvas. A service returns `{operations: [...]}`; the frontend+walks the list with a single applier (`client/src/lib/workflowOps.ts`)+that creates / removes nodes + edges and persists parameter changes.++**Why a protocol**: keeps domain rules in the backend (visuals.json+lookups, plugin-registry checks, canonical config shapes) without+forcing each new feature to invent its own RPC + frontend glue.+"Auto-add Skill on tool connect" is the first consumer; future+features (workflow templates, AI-suggested edits, automation recipes)+just emit op batches.++## Wire shape++```json+{+ "operations": [+ { "type": "add_node", ... },+ { "type": "add_edge", ... }+ ]+}+```++Operations apply **in order**. Earlier ops can produce React Flow node+ids that later ops reference via `client_ref` placeholders.++## Operation types (v1)++| Type | Required | Optional | Purpose |+|---|---|---|---|+| `add_node` | `client_ref`, `node_type`, `parameters` | `label`, `position` | Create a new node + persist its initial params. `client_ref` is a batch-local id later ops can target. |+| `add_edge` | `source`, `target` | `source_handle`, `target_handle` | Wire two nodes. `source` / `target` are either existing node ids (string) or `{client_ref}` references. |+| `set_node_parameters` | `node_id`, `parameters` | -- | Shallow-merge `parameters` into the node's existing params via `saveNodeParameters`. |+| `delete_node` | `node_id` | -- | Remove node; edges to/from it cascade-delete. |+| `delete_edge` | `edge_id` | -- | Remove a single edge. |+| `move_node` | `node_id`, `position` | -- | Reposition without changing identity. |+| `replace_node` | `node_id`, `node_type`, `parameters` | `label`, `preserve_edges` (default `true`) | Atomic delete-and-add at the same position. With `preserve_edges`, edges to/from the old id rewire to the new id. |++### `PositionSpec`++Either absolute or anchored:++```ts+type PositionSpec =+ | { x: number; y: number }+ | { anchor_node_id: string;+ offset?: { x?: number; y?: number };+ fallback?: { x: number; y: number } };+```++Anchored positions stay backend-agnostic about pixel coords -- the+frontend resolves the anchor against current React Flow state. If the+anchor doesn't exist, `fallback` is used (or a sensible default).++### `NodeRef`++```ts+type NodeRef = string | { client_ref: string };+```++A string is an existing React Flow node id. A `{client_ref}` is a+placeholder that resolves to the id of an `add_node` op earlier in+the same batch.++## Application semantics++- **Order**: ops apply sequentially. Anchor lookups and `client_ref`+ resolution see the cumulative state of earlier ops in the batch.+- **Best-effort**: a failed op (e.g. `delete_node` on a missing id) is+ logged and reported in the result; subsequent ops still apply. v1+ has no rollback. Backend services should write op sequences that are+ robust to partial application.+- **Not a diff reconciler**: re-applying a batch is not assumed safe.+ The protocol is a one-shot mutation wire format.++## Backend usage++```python+from services import workflow_ops++ref = "new_master"+return {+ "operations": [+ workflow_ops.add_node(+ ref, "masterSkill",+ {"skillsConfig": {...}},+ label="Master Skill",+ position=workflow_ops.anchored(agent_id, offset_x=-60, offset_y=220),+ ),+ workflow_ops.add_edge(+ {"client_ref": ref}, agent_id,+ source_handle="output-tool", target_handle="input-skill",+ ),+ ],+}+```++Helpers live in `server/services/workflow_ops.py` (TypedDicts ++builder functions). Empty result: `workflow_ops.empty()` returns+`{"operations": []}`.++## Frontend usage++```ts+import { applyOperations } from '@/lib/workflowOps';++const result = await applyOperations(operations, {+ nodes, edges, setNodes, setEdges, saveNodeParameters,+});+// result.applied -> count of successful ops+// result.errors -> [{op, message}, ...]+// result.refMap -> { client_ref: generated_node_id, ... }+```++`applyOperations` does not throw; callers inspect `errors` to surface+failures (toast, log, retry, etc.).++## Current consumers++| Service | WS handler | Module |+|---|---|---|+| Auto-add Skill on tool connect | `evaluate_auto_skill` | `server/services/auto_skill.py` |++## Adding a new consumer++1. Write a backend service that returns `{operations: [...]}` using+ the builder helpers in `services/workflow_ops`.+2. Add a thin `@ws_handler` that calls into the service.+3. On the frontend, send the WS request and pass the returned+ `operations` into `applyOperations`. No new applier code.+4. If you need a new op type, follow the steps in the docstring of+ `server/services/workflow_ops.py` (mirror the TypedDict in TS,+ add an apply branch, document here).diff --git a/server/services/workflow_ops.py b/server/services/workflow_ops.pynew file mode 100644index 00000000..8a3cc7b8--- /dev/null+++ b/server/services/workflow_ops.py@@ -0,0 +1,241 @@+"""Workflow Operations protocol.++The standard wire format any backend service uses to mutate the+React Flow canvas. A service returns ``{"operations": [...]}`` and the+frontend's single applier (``client/src/lib/workflowOps.ts``) walks+the list, generating real React Flow node ids, resolving cross-op+references, and persisting parameter changes through+``saveNodeParameters``.++See ``docs-internal/workflow_ops_protocol.md`` for the full spec.++Protocol notes:++* Operations apply **in order**. Earlier ops can produce ids that+ later ops reference via ``client_ref`` placeholders.+* Failures are logged + reported per op; subsequent ops still apply.+ v1 has no rollback -- write op sequences that are robust to partial+ application.+* The protocol is a declarative wire format, not a diff reconciler.+ Re-applying a batch is not assumed to be idempotent.++Adding a new operation type:++1. Add the TypedDict below + extend ``WorkflowOperation`` union.+2. Add a builder helper below.+3. Mirror the type in ``client/src/lib/workflowOps.ts``.+4. Add the apply branch to ``applyOperations`` in that same TS file.+5. Document the new op in ``docs-internal/workflow_ops_protocol.md``.+"""++from __future__ import annotations++from typing import Any, Dict, List, Literal, Optional, TypedDict, Union+++# ---------------------------------------------------------------------------+# Position spec+# ---------------------------------------------------------------------------++class AbsolutePosition(TypedDict):+ x: float+ y: float+++class _AnchorOffset(TypedDict, total=False):+ x: float+ y: float+++class AnchoredPosition(TypedDict, total=False):+ anchor_node_id: str+ offset: _AnchorOffset+ fallback: AbsolutePosition+++# Either {x,y} or {anchor_node_id, offset?, fallback?}. The frontend+# resolves the anchor against current React Flow node positions.+PositionSpec = Union[AbsolutePosition, AnchoredPosition]+++# ---------------------------------------------------------------------------+# Node references+# ---------------------------------------------------------------------------++class _ClientRef(TypedDict):+ client_ref: str+++# Existing-id strings mix freely with batch-local ``{client_ref}``+# placeholders so an add_edge op can target a node added earlier in+# the same batch before its real id exists.+NodeRef = Union[str, _ClientRef]+++# ---------------------------------------------------------------------------+# Operation TypedDicts+# ---------------------------------------------------------------------------++class AddNodeOp(TypedDict, total=False):+ type: Literal["add_node"]+ client_ref: str+ node_type: str+ parameters: Dict[str, Any]+ label: str+ position: PositionSpec+++class AddEdgeOp(TypedDict, total=False):+ type: Literal["add_edge"]+ source: NodeRef+ target: NodeRef+ source_handle: str+ target_handle: str+++class SetNodeParametersOp(TypedDict, total=False):+ type: Literal["set_node_parameters"]+ node_id: str+ parameters: Dict[str, Any]+++class DeleteNodeOp(TypedDict):+ type: Literal["delete_node"]+ node_id: str+++class DeleteEdgeOp(TypedDict):+ type: Literal["delete_edge"]+ edge_id: str+++class MoveNodeOp(TypedDict):+ type: Literal["move_node"]+ node_id: str+ position: PositionSpec+++class ReplaceNodeOp(TypedDict, total=False):+ type: Literal["replace_node"]+ node_id: str+ node_type: str+ parameters: Dict[str, Any]+ label: str+ preserve_edges: bool+++WorkflowOperation = Union[+ AddNodeOp,+ AddEdgeOp,+ SetNodeParametersOp,+ DeleteNodeOp,+ DeleteEdgeOp,+ MoveNodeOp,+ ReplaceNodeOp,+]+++# ---------------------------------------------------------------------------+# Builder helpers+# ---------------------------------------------------------------------------++def add_node(+ client_ref: str,+ node_type: str,+ parameters: Optional[Dict[str, Any]] = None,+ *,+ label: Optional[str] = None,+ position: Optional[PositionSpec] = None,+) -> AddNodeOp:+ op: AddNodeOp = {+ "type": "add_node",+ "client_ref": client_ref,+ "node_type": node_type,+ "parameters": parameters or {},+ }+ if label is not None:+ op["label"] = label+ if position is not None:+ op["position"] = position+ return op+++def add_edge(+ source: NodeRef,+ target: NodeRef,+ *,+ source_handle: Optional[str] = None,+ target_handle: Optional[str] = None,+) -> AddEdgeOp:+ op: AddEdgeOp = {+ "type": "add_edge",+ "source": source,+ "target": target,+ }+ if source_handle is not None:+ op["source_handle"] = source_handle+ if target_handle is not None:+ op["target_handle"] = target_handle+ return op+++def set_node_parameters(node_id: str, parameters: Dict[str, Any]) -> SetNodeParametersOp:+ return {+ "type": "set_node_parameters",+ "node_id": node_id,+ "parameters": parameters,+ }+++def delete_node(node_id: str) -> DeleteNodeOp:+ return {"type": "delete_node", "node_id": node_id}+++def delete_edge(edge_id: str) -> DeleteEdgeOp:+ return {"type": "delete_edge", "edge_id": edge_id}+++def move_node(node_id: str, position: PositionSpec) -> MoveNodeOp:+ return {"type": "move_node", "node_id": node_id, "position": position}+++def replace_node(+ node_id: str,+ node_type: str,+ parameters: Optional[Dict[str, Any]] = None,+ *,+ label: Optional[str] = None,+ preserve_edges: bool = True,+) -> ReplaceNodeOp:+ op: ReplaceNodeOp = {+ "type": "replace_node",+ "node_id": node_id,+ "node_type": node_type,+ "parameters": parameters or {},+ "preserve_edges": preserve_edges,+ }+ if label is not None:+ op["label"] = label+ return op+++def anchored(+ anchor_node_id: str,+ *,+ offset_x: float = 0,+ offset_y: float = 0,+ fallback: Optional[AbsolutePosition] = None,+) -> AnchoredPosition:+ """Convenience builder for ``{anchor_node_id, offset, fallback}``."""+ pos: AnchoredPosition = {+ "anchor_node_id": anchor_node_id,+ "offset": {"x": offset_x, "y": offset_y},+ }+ if fallback is not None:+ pos["fallback"] = fallback+ return pos+++def empty() -> Dict[str, List[WorkflowOperation]]:+ """Standard empty response (``{"operations": []}``)."""+ return {"operations": []}