Skip to content

Self-improving AI Employees for everyone

59.0 KB
Raw Blame
"""Temporal workflow - Pure orchestrator for distributed node execution.

The workflow ONLY orchestrates:
- Parses graph structure
- Filters config nodes (tools, memory, services)
- Determines execution order based on dependencies
- Schedules node activities (can run on ANY worker)
- Collects results and routes outputs to dependent nodes

NO business logic in workflow - all execution happens in activities.
This enables massive horizontal scaling and multi-tenant distribution.
"""

from datetime import timedelta
from typing import Any, Dict, List, Optional, Set

from temporalio import workflow

from ._exits import is_workflow_bug
from ._retry_policies import DEFAULT_ACTIVITY_RETRY, QUICK_ACTIVITY_RETRY
from .execution_control import ExecutionControl

# ``conditions`` is pure -- ``re`` + comparisons, no IO, no clock, no
# randomness -- so it is safe to evaluate inside a workflow.
#
# Importing the submodule still executes ``services/execution/__init__.py``
# first (Python always initialises the parent package), which drags in the
# executor, cache, recovery sweeper and DLQ. That is tolerated, not avoided:
# nothing in that subtree imports back into ``services.temporal``, so there is
# no cycle, and none of it pulls redis or sqlalchemy at import time. The
# pass-through keeps the sandbox from re-importing the chain per workflow.
with workflow.unsafe.imports_passed_through():
    from services.execution.conditions import evaluate_condition

# Conditional edges were not evaluated on this path at all before this patch --
# a condition set in the editor rendered a label and did nothing once execution
# routed through Temporal. Gating the skip keeps replay deterministic for
# histories recorded while every node was scheduled unconditionally.
CONDITIONAL_EDGES_PATCH = "machina-conditional-edges-v1"

# Each finished trigger-spawned run schedules a record activity so Normal mode
# can say how much an employee did today. Gated so histories recorded before
# it (which end without that activity) still replay.
RUN_RECORD_PATCH = "machina-run-record-v1"

# A run that the owner's chat message spawned claims its chat run when it
# starts and finishes it at its end (services/chat/ledger.py), so Talk and
# the editor's chat follow the answer while it is worked on. Gated: the two
# activities are new commands.
CHAT_RUN_PATCH = "machina-chat-run-v1"
# The source and type services/chat/events.py gives the owner's message. A
# chat run id is trusted only from an event with both, never from what
# another producer put in its event data.
CHAT_MESSAGE_SOURCE = "opencompany://services/chat"
CHAT_MESSAGE_TYPE = "com.opencompany.chat.message.received"

# Config handles - nodes connecting via these are config nodes (not executed)
# AI Agent handles: input-context, input-tools, input-model, input-task, input-teammates
# Zeenie handles: input-skill, input-tools
CONFIG_HANDLES = {
    "input-context",
    "input-tools",
    "input-memory",  # replay/import compatibility for legacy input-memory graphs
    "input-model",
    "input-skill",
    "input-task",
    "input-teammates",
}


def _is_config_edge(edge: Dict[str, Any], node_map: Dict[str, Dict[str, Any]]) -> bool:
    """Return whether an edge attaches configuration rather than runtime data.

    ``input-task`` is normally a configuration handle, but a taskTrigger emits
    runtime completion data through that handle.  Treating that edge as config
    removes both the pre-executed trigger and its payload before dependency
    resolution, leaving the downstream agent with an empty prompt.
    """
    handle = edge.get("targetHandle") or edge.get("target_handle") or ""
    if handle not in CONFIG_HANDLES:
        return False
    source = node_map.get(edge.get("source"), {})
    return not (handle == "input-task" and source.get("type") == "taskTrigger")

# Trigger node types — event listeners that should never be scheduled
# as blocking activities. Imported from constants to avoid drift (was
# previously redefined here with a "keep in sync" comment — Wave 11.E.2).
# Android service types follow the same pattern: imported from constants
# so the canonical 16-entry list (Wave 11.I, milestone P -- the local
# 6-entry copy that lived here was a stale subset).
from constants import (
    ANDROID_SERVICE_NODE_TYPES as ANDROID_SERVICE_TYPES,
    WORKFLOW_TRIGGER_TYPES as TRIGGER_NODE_TYPES,
)

# Skill node types (connect to Zeenie's input-skill, not executed directly)
SKILL_NODE_TYPES = {
    "masterSkill",
}

# The Approval node (services/approvals/contract.py). In a generation that
# waits for the owner (approval_wait_version 1) it waits outside Stop's
# checkpoint, so Stop completes while it waits (RFC-0007 §4.8).
APPROVAL_GATE_TYPE = "approvalGate"

# F4.B: agent types that migrate to AgentWorkflow (Temporal child workflow).
# When ``temporal_agent_workflow_enabled`` is True the orchestrator schedules
# AgentWorkflow for these node types instead of an activity. Tool calls
# inside the agent loop become per-type activities (F4.A path).
#
# Excluded: ``rlm_agent``, ``claude_code_agent``.
# Their internal session state (RLM REPL / Claude CLI --resume with stable
# cwd) requires single-process continuity and breaks
# across activity boundaries. They continue via F4.A per-type activities.
AGENT_WORKFLOW_TYPES = frozenset(
    [
        "aiAgent",
        "chatAgent",
        # Specialized agents (11)
        "android_agent",
        "browser_agent",
        "coding_agent",
        "web_agent",
        "task_agent",
        "social_agent",
        "travel_agent",
        "tool_agent",
        "productivity_agent",
        "payments_agent",
        "consumer_agent",
        "autonomous_agent",
        # Team leads (2)
        "orchestrator_agent",
        "ai_employee",
    ]
)

AGENT_CONTEXT_GRAPH_VERSION = 2
# Root dispatch consumes an immutable routing snapshot carried in workflow
# input rather than process-local Settings, so flipping an environment
# value can never change how an in-flight run dispatches.
TEMPORAL_ROUTING_INPUT_KEY = "_temporal_routing_v1"
TEMPORAL_ROUTING_INPUT_VERSION = 1
# Direct Temporal callers predating the input helper may omit the snapshot.
# Keep that path deterministic and on the manager's queue while retaining the
# shipped child/per-type protocols. Normal executor/deployment starts always
# provide an explicit snapshot.
_SAFE_FROZEN_ROUTING = {
    "version": TEMPORAL_ROUTING_INPUT_VERSION,
    "agent_workflow_enabled": True,
    "per_type_dispatch_enabled": True,
    "worker_pool_enabled": False,
}
# Agent child workflows used to carry 1h execution/run timeouts, and node
# activities a 10-minute start_to_close. A run may legitimately execute —
# or stay cooperatively paused — for months, and Temporal's timeout
# timers keep ticking through a pause, so the caps silently terminated
# long/paused runs. New executions start agent children unbounded and
# give node activities a generous start_to_close; liveness is enforced
# by the 2-minute heartbeat timeout (activities self-heartbeat every
# 30s), not by lifetime caps.
# Circuit breaker: a failed trigger-spawned run schedules the
# workflow_control.pause_on_failure activity so the deployment pauses
# (user fixes + resumes) instead of the trigger firing into the same
# error indefinitely. The WORKFLOW_CONTROL_PAUSE_ON_FAILURE knob is
# evaluated on the activity side, so flipping config never touches
# recorded workflow commands. The scheduling itself is not behind a
# workflow.patched guard.
# start_to_close for node activities. Generous by
# design: a single node step (agent turn batch, browser op, long shell
# command) may run for hours; worker-death detection is the heartbeat
# timeout's job, so a long ceiling costs nothing in liveness.
_NODE_ACTIVITY_START_TO_CLOSE = timedelta(hours=24)


def _agent_child_workflow_id(root_workflow_id: str, agent_node_id: str) -> str:
    """Return a deterministic child id unique to one canvas agent and run."""
    return f"{root_workflow_id}-agent-{agent_node_id}"


#: Selections frozen per generation at Start (RFC-0005 §24.1 rule 2): the
#: workflow code branches on these, never on live settings, so a change
#: behind one replays older histories without a patch marker
#: (tests/temporal/test_workflow_identity_golden.py). ``approval_wait_version``:
#: what asks is the mode and the class, and the agent answering a chat run
#: waits for the owner (RFC-0007).
GENERATION_FLAGS = ("pi_harness_version", "approval_wait_version")


def _frozen_routing_from_input(
    workflow_data: Dict[str, Any],
) -> Dict[str, Any]:
    """Normalize the immutable routing snapshot without consulting Settings."""
    raw = workflow_data.get(TEMPORAL_ROUTING_INPUT_KEY)
    if (
        not isinstance(raw, dict)
        or raw.get("version") != TEMPORAL_ROUTING_INPUT_VERSION
    ):
        raw = _SAFE_FROZEN_ROUTING
    if raw.get("pi_harness_version") not in {None, 1}:
        raise ValueError("Unsupported recorded Coding Agent harness version")
    return {
        "version": TEMPORAL_ROUTING_INPUT_VERSION,
        "agent_workflow_enabled": raw.get("agent_workflow_enabled") is True,
        "per_type_dispatch_enabled": (
            raw.get("per_type_dispatch_enabled") is True
        ),
        "worker_pool_enabled": raw.get("worker_pool_enabled") is True,
        **{flag: 1 for flag in GENERATION_FLAGS if raw.get(flag) == 1},
    }


# Durable protocol identifier: existing Temporal histories and child-workflow
# start calls replay against this exact type string. It intentionally retains
# the pre-rebrand name even though the product is now OpenCompany.
@workflow.defn(name="MachinaWorkflow", sandboxed=False)
class MachinaWorkflow:
    """Distributed workflow orchestrator.

    This workflow ONLY orchestrates - all execution happens in activities
    that can run on any worker in the cluster.

    Features:
    - Continuous scheduling (FIRST_COMPLETED pattern)
    - Per-node retry policies
    - Config node filtering (tools, memory, services)
    - Multi-tenant support via tenant_id in context
    - Wave 12 A7: event-framework signal handler — receives Temporal
      Signals from ``services/events/dispatch.py:emit`` and parks
      pending events in ``_matched_events`` for trigger nodes to
      consume via ``workflow.wait_condition(_has_event_matching, ...)``.
      Per-run dedup via ``_seen_event_ids`` (reconstructed
      deterministically from Event History on replay).
    """

    @workflow.init
    def __init__(self, workflow_data: Optional[Dict[str, Any]] = None) -> None:
        # Wave 12 A7: per-run event-framework state. Populated by the
        # ``on_event`` signal handler; consumed by Phase C1's
        # ``wait_condition`` predicate in the trigger-waiter rewrite.
        # Today these stay empty unless the event-framework feature
        # flag is on AND a producer signals this workflow.
        self._seen_event_ids: Set[str] = set()
        self._matched_events: List[Dict[str, Any]] = []
        self._control_paused = False
        self._execution_control = ExecutionControl()
        if workflow_data is not None:
            self._execution_control.bind(workflow_data)
        self._action_contexts: Dict[str, Any] = {}

    @workflow.signal
    async def pause(self) -> None:
        """Cooperatively gate new node/activity scheduling."""
        if self._execution_control.enabled:
            return
        self._control_paused = True

    @workflow.signal
    async def resume(self) -> None:
        if self._execution_control.enabled:
            return
        self._control_paused = False

    @workflow.update
    async def set_control_state(self, payload: Dict[str, Any]) -> Dict[str, Any]:
        return await self._execution_control.set_control_state(payload)

    @set_control_state.validator
    def validate_set_control_state(self, payload: Dict[str, Any]) -> None:
        self._execution_control.validate_control(payload)

    @workflow.update
    async def wait_for_checkpoint(self, payload: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
        return await self._execution_control.wait_for_checkpoint(payload)

    @wait_for_checkpoint.validator
    def validate_wait_for_checkpoint(self, payload: Optional[Dict[str, Any]] = None) -> None:
        self._execution_control.validate_control(payload)

    @workflow.query
    def execution_control_status(self) -> Dict[str, Any]:
        return self._execution_control.status()

    async def _wait_until_resumed(self) -> None:
        if self._execution_control.enabled:
            await self._execution_control.wait_until_running()
            return
        if self._control_paused:
            await workflow.wait_condition(lambda: not self._control_paused)

    async def _pause_deployment_on_failure(
        self,
        workflow_data: Dict[str, Any],
        nodes: List[Dict[str, Any]],
        errors: List[Dict],
    ) -> None:
        """Circuit breaker: a failed trigger-spawned run pauses its deployment.

        Only deployment-spawned runs qualify — they always carry a
        ``_pre_executed`` firing trigger, which one-off manual canvas runs
        never do, so a failed test run cannot pause a live deployment.
        Legacy uncontrolled deployments no-op inside the activity (no
        running control row). Non-fatal by design: the run's own failure
        result is the primary outcome either way.
        """
        workflow_id = workflow_data.get("workflow_id")
        spawned_by_trigger = any(node.get("_pre_executed") for node in nodes)
        if not workflow_id or not spawned_by_trigger:
            return
        failure = errors[0] or {}
        recovery = {}
        if failure.get("requires_user_action") is True:
            recovery = {
                "requires_user_action": True,
                "hint": failure.get("hint"),
                "generation": workflow_data.get("generation"),
            }
        try:
            await workflow.execute_activity(
                "workflow_control.pause_on_failure",
                {
                    "workflow_id": workflow_id,
                    "reason": str((errors[0] or {}).get("error", "run_failed"))[:500],
                    **recovery,
                },
                start_to_close_timeout=timedelta(seconds=30),
                retry_policy=QUICK_ACTIVITY_RETRY,
            )
        except Exception as exc:  # noqa: BLE001 — cosmetic relative to the run result
            workflow.logger.warning(f"pause-on-failure activity failed (non-fatal): {exc}")

    async def _record_run_completion(
        self,
        workflow_data: Dict[str, Any],
        nodes: List[Dict[str, Any]],
        success: bool,
    ) -> None:
        """Record a finished trigger-spawned run (services/employees/runs.py).

        Same eligibility as the circuit breaker: deployment-spawned runs
        carry a ``_pre_executed`` firing trigger, manual canvas runs never
        do. Behind RUN_RECORD_PATCH; the patch check comes after the
        eligibility test, which depends only on the workflow input, so it
        is deterministic. Non-fatal: a missed record only undercounts.
        """
        workflow_id = workflow_data.get("workflow_id")
        spawned_by_trigger = any(node.get("_pre_executed") for node in nodes)
        if not workflow_id or not spawned_by_trigger:
            return
        if not workflow.patched(RUN_RECORD_PATCH):
            return
        try:
            info = workflow.info()
            await workflow.execute_activity(
                "workflow_runs.record_completion",
                {
                    "workflow_id": workflow_id,
                    "run_id": f"{info.workflow_id}:{info.run_id}",
                    "generation": int(workflow_data.get("generation") or 0),
                    "status": "success" if success else "failed",
                },
                start_to_close_timeout=timedelta(seconds=30),
                retry_policy=QUICK_ACTIVITY_RETRY,
            )
        except Exception as exc:  # noqa: BLE001 — cosmetic relative to the run result
            workflow.logger.warning(f"run-record activity failed (non-fatal): {exc}")

    @staticmethod
    def _chat_run_id(nodes: List[Dict[str, Any]]) -> Optional[str]:
        """The chat run the firing trigger's event started, or None.

        Read from the event envelope the trigger listener attached to the
        trigger's output, and only when the chat service produced it. Depends
        on the workflow input alone, so it is deterministic."""
        for node in nodes:
            if not node.get("_pre_executed"):
                continue
            output = node.get("_trigger_output")
            if not isinstance(output, dict) or output.get("not_triggered"):
                continue
            envelope = output.get("_event_envelope")
            if not isinstance(envelope, dict):
                continue
            if envelope.get("source") != CHAT_MESSAGE_SOURCE or envelope.get("type") != CHAT_MESSAGE_TYPE:
                continue
            data = envelope.get("data")
            run_id = data.get("run_id") if isinstance(data, dict) else None
            if isinstance(run_id, str) and run_id:
                return run_id
        return None

    async def _start_chat_run(self, run_id: str, session_id: str) -> Optional[Dict[str, Any]]:
        """Claim the chat run; returns the run scope node contexts carry, or
        None when this run does not hold it (another chat trigger in the
        graph claimed it, or it ended) and runs untracked."""
        info = workflow.info()
        try:
            claimed = await workflow.execute_activity(
                "chat_run.start",
                {"run_id": run_id, "temporal_workflow_id": info.workflow_id, "temporal_run_id": info.run_id},
                start_to_close_timeout=timedelta(seconds=30),
                retry_policy=DEFAULT_ACTIVITY_RETRY,
            )
        except Exception as exc:  # noqa: BLE001 — the answer matters more than its tracking
            workflow.logger.warning(f"chat run start failed (running untracked): {exc}")
            return None
        if not (isinstance(claimed, dict) and claimed.get("claimed")):
            return None
        # The chat's session comes from the run's row: this workflow's own
        # session is its execution's ("<workflow>:execution:<n>" on a deployed
        # run). A claim recorded before it carried the session keeps the old
        # value; only activity inputs change, so histories replay as before.
        chat_session = claimed.get("session_id")
        return {"run_id": run_id, "session_id": chat_session if isinstance(chat_session, str) and chat_session else session_id}

    async def _finish_chat_run(self, run_scope: Dict[str, Any], success: bool, errors: List[Dict]) -> None:
        """End the claimed chat run with the run's outcome. Non-fatal: the
        watchdog ends a run whose finish never arrived."""
        info = workflow.info()
        payload: Dict[str, Any] = {
            "run_id": run_scope["run_id"],
            "temporal_workflow_id": info.workflow_id,
            "temporal_run_id": info.run_id,
            "success": success,
        }
        if errors:
            failure = errors[0] or {}
            payload["error"] = str(failure.get("error") or "run_failed")[:500]
            for key in ("hint", "requires_user_action"):
                if failure.get(key) is not None:
                    payload[key] = failure[key]
        try:
            await workflow.execute_activity(
                "chat_run.finish",
                payload,
                start_to_close_timeout=timedelta(seconds=30),
                retry_policy=DEFAULT_ACTIVITY_RETRY,
            )
        except Exception as exc:  # noqa: BLE001 — cosmetic relative to the run result
            workflow.logger.warning(f"chat run finish failed (non-fatal): {exc}")

    @workflow.signal
    async def on_event(self, event_payload: Dict[str, Any]) -> None:
        """Wave 12 A7: receive an event from ``services.events.dispatch.emit``.

        Drops duplicates by ``event.id`` (per-run dedup — the set is
        reconstructed deterministically from Event History on replay).
        Matching events are queued for the workflow body to consume via
        :meth:`_has_event_matching` in a ``wait_condition`` (wired in
        Phase C1).
        """
        event_id = event_payload.get("id")
        if not event_id:
            workflow.logger.warning("on_event: skipping malformed envelope without 'id'")
            return
        if event_id in self._seen_event_ids:
            workflow.logger.debug(f"on_event: dedup hit for event.id={event_id}")
            return
        self._seen_event_ids.add(event_id)
        self._matched_events.append(event_payload)
        workflow.logger.info(
            f"on_event: queued event.id={event_id} " f"type={event_payload.get('type', '?')} " f"(matched={len(self._matched_events)})"
        )

    def _has_event_matching(self, predicate=None) -> bool:
        """Wave 12 A7: ``wait_condition`` predicate for trigger nodes.

        Returns True when at least one queued event matches the optional
        ``predicate(event_payload) -> bool``. Defaults to "any event
        queued" — most trigger nodes filter further by inspecting the
        envelope after pop.

        Used by Phase C1's trigger-waiter rewrite to replace the
        in-process ``asyncio.Future`` waiter:

            await workflow.wait_condition(
                lambda: self._has_event_matching(my_filter),
                timeout=...,
            )
            event = self._pop_matching_event(my_filter)
        """
        if not self._matched_events:
            return False
        if predicate is None:
            return True
        return any(predicate(e) for e in self._matched_events)

    def _pop_matching_event(self, predicate=None) -> Optional[Dict[str, Any]]:
        """Wave 12 A7: dequeue the first event that satisfies ``predicate``.

        Pairs with :meth:`_has_event_matching` — Phase C1 trigger
        waiters call ``wait_condition`` on the predicate, then pop the
        actual envelope here. FIFO order: the predicate is evaluated
        against the head of ``_matched_events`` first, so events
        arrive in the order Temporal delivered the signals (and on
        replay, the order Event History records them — deterministic).

        Returns ``None`` if no queued event matches; callers should
        only reach this method after ``wait_condition`` resolves, in
        which case at least one match exists. The ``None`` return path
        exists for defensive symmetry — never silently raises mid-
        replay.

        Determinism note: list mutation inside a workflow is safe per
        Temporal's Python SDK contract (the workflow's local state is
        rebuilt from Event History on replay; the same signals deliver
        the same envelopes, so the same pops happen in the same order).
        """
        for idx, event in enumerate(self._matched_events):
            if predicate is None or predicate(event):
                return self._matched_events.pop(idx)
        return None

    @workflow.run
    async def run(self, workflow_data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
        workflow_data = workflow_data or {}
        self._execution_control.bind(workflow_data)
        if not self._execution_control.enabled:
            return await self._run_graph(workflow_data)
        await self._execution_control.register_root()
        bug = False
        try:
            return await self._run_graph(workflow_data)
        except Exception as exc:
            bug = is_workflow_bug(exc)
            raise
        finally:
            # A bug fails the task and records nothing on the way out (_exits).
            if not bug:
                for admission in self._action_contexts.values():
                    await admission.__aexit__(None, None, None)
                self._action_contexts.clear()
                await self._execution_control.unregister_root()
                await workflow.wait_condition(workflow.all_handlers_finished)

    async def _run_graph(self, workflow_data: Dict[str, Any]) -> Dict[str, Any]:
        """Execute workflow by orchestrating node activities.

        Args:
            workflow_data: Dict containing:
                - nodes: List of node definitions from React Flow
                - edges: List of edge definitions from React Flow
                - session_id: Session identifier
                - workflow_id: Workflow ID for tracking
                - tenant_id: Tenant identifier for multi-tenancy

        Returns:
            Dict with success, outputs, execution_trace, errors, and
            total_nodes (the size of the executable graph)
        """
        nodes = workflow_data.get("nodes", [])
        edges = workflow_data.get("edges", [])
        session_id = workflow_data.get("session_id", "default")
        workflow_id = workflow_data.get("workflow_id")
        # Human-readable slug for child workflow IDs visible in the
        # Temporal Web UI. Falls back to workflow_id when missing
        # (one-off Runs without a saved DB row).
        workflow_slug = workflow_data.get("workflow_slug") or workflow_id
        tenant_id = workflow_data.get("tenant_id")
        graph_version = int(
            workflow_data.get("graphVersion")
            or workflow_data.get("graph_version")
            or 0
        )
        generation = int(workflow_data.get("generation") or 0)
        # Stable per-run execution id. The executor passes the same value
        # it uses as this workflow's Temporal id (``<slug>-<uuid8>``), so
        # the fallback to ``workflow.info().workflow_id`` is identical by
        # construction. Threaded into every node activity context so
        # session-keyed nodes (browser) share one instance per run instead
        # of minting a fresh uuid per call (node_executor.py fallback).
        execution_id = workflow_data.get("execution_id") or workflow.info().workflow_id
        # Dispatch flags come from the routing snapshot the starter froze
        # into the workflow input (``_temporal_routing_v1``), never from
        # Settings, so an environment change cannot alter how a run that
        # is already in flight dispatches.
        frozen_routing = _frozen_routing_from_input(workflow_data)
        from services.coding_agent_runtime import coding_agent_ancestors
        pi_delegators = coding_agent_ancestors(nodes, edges) if frozen_routing.get("pi_harness_version") == 1 else set()

        workflow.logger.info(f"Starting workflow orchestration: {len(nodes)} nodes, {len(edges)} edges")

        if not nodes:
            return {
                "success": False,
                "error": "No nodes provided",
                "outputs": {},
                "execution_trace": [],
            }

        # 1. Filter out config nodes (tools, memory, services)
        exec_nodes, exec_edges = self._filter_executable_graph(nodes, edges)

        workflow.logger.info(
            f"After filtering: {len(exec_nodes)} executable nodes " f"(filtered {len(nodes) - len(exec_nodes)} config nodes)"
        )

        # 2. Build dependency maps
        deps, node_map = self._build_dependency_maps(exec_nodes, exec_edges)
        conditional_edges = self._build_conditional_edge_map(exec_edges, node_map)
        use_conditional_edges = workflow.patched(CONDITIONAL_EDGES_PATCH)

        # 3. Initialize state
        outputs: Dict[str, Any] = {}  # node_id -> result
        completed: Set[str] = set()
        running: Dict[str, Any] = {}  # node_id -> activity handle
        errors: List[Dict] = []
        execution_trace: List[str] = []

        # 4. Handle pre-executed triggers (already have their output)
        pre_executed_count = 0
        for node in exec_nodes:
            if node.get("_pre_executed"):
                node_id = node["id"]
                trigger_output = node.get("_trigger_output", {})
                outputs[node_id] = {
                    "success": True,
                    "result": trigger_output,
                    "pre_executed": True,
                }
                completed.add(node_id)
                execution_trace.append(node_id)
                pre_executed_count += 1
                workflow.logger.info(f"Pre-executed trigger: {node_id}")

                # Persist the trigger output to the workflow output
                # store so ParameterResolver can resolve
                # ``{{triggerNodeName.field}}`` in downstream nodes.
                # Pre-executed nodes bypass NodeExecutor.execute (the
                # normal write site), so without this activity the
                # legacy path's ``_store_output(trigger_node_id, ...)``
                # behaviour is missing on the canary path and templates
                # against the trigger silently resolve to empty.
                # Skip the persist for non-firing siblings — their
                # ``_trigger_output`` is ``{not_triggered: True}``, no
                # downstream template should resolve against them.
                if not trigger_output.get("not_triggered"):
                    await self._wait_until_resumed()
                    await workflow.execute_activity(
                        "store_node_output_activity",
                        {
                            "node_id": node_id,
                            "session_id": session_id,
                            "result": trigger_output,
                        },
                        start_to_close_timeout=timedelta(seconds=10),
                    )

        workflow.logger.info(f"Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")

        # The owner's chat message started a chat run: claim it. The patch
        # check comes after the eligibility test, which reads only the
        # workflow input, so it is deterministic.
        run_scope: Optional[Dict[str, Any]] = None
        chat_run_id = self._chat_run_id(nodes)
        if chat_run_id is not None and workflow.patched(CHAT_RUN_PATCH):
            await self._wait_until_resumed()
            run_scope = await self._start_chat_run(chat_run_id, session_id)

        # 5. Retry policy for node activities. Wave 12 D1: imported
        # shared constant declares ``non_retryable_error_types`` so
        # NodeUserError fails fast instead of burning 3 retries on
        # user-correctable failures.
        retry_policy = DEFAULT_ACTIVITY_RETRY

        # 6. Continuous scheduling loop
        loop_count = 0
        while True:
            loop_count += 1
            # Avoid emitting a redundant wait command on the normal running
            # path.  Once paused, the condition is durable and replay-safe;
            # the resume signal flips the flag and releases scheduling.
            if self._control_paused and not self._execution_control.enabled:
                await workflow.wait_condition(lambda: not self._control_paused)
            # Find ready nodes (all deps completed, not running/completed)
            ready = self._find_ready_nodes(deps, completed, running, node_map)
            if self._execution_control.enabled and errors:
                ready = []
            if self._execution_control.enabled and self._execution_control.status()["state"] != "running" and running:
                # Results and graph bookkeeping settle even when admission is
                # closed. Waiting for Resume here would deadlock checkpoint.
                ready = []
            elif self._execution_control.enabled and self._execution_control.status().get("producers_held") and running:
                ready = []
            workflow.logger.debug(f"Loop {loop_count}: ready={len(ready)}, running={len(running)}, completed={len(completed)}")

            # Triggers auto-completed by the skip branch below unblock their
            # downstream nodes without putting anything into ``running`` —
            # the exit check must re-evaluate readiness in that case instead
            # of concluding the graph is drained (direct canvas-Run path:
            # ``start`` is NOT pre-executed, so without this the loop broke
            # after the start node and downstream never ran).
            auto_completed_this_pass = 0

            # Start activities for ready nodes
            for node_id in ready:
                node = node_map[node_id]
                node_type = node.get("type", "unknown")

                # Safety: auto-complete trigger nodes that weren't pre-executed.
                # Trigger nodes are event listeners - scheduling them as activities
                # would block indefinitely waiting for external events.
                if node_type in TRIGGER_NODE_TYPES and not node.get("_pre_executed"):
                    workflow.logger.warning(f"Skipping non-pre-executed trigger: {node_id} ({node_type})")
                    outputs[node_id] = {
                        "success": True,
                        "result": {"not_triggered": True},
                        "skipped_trigger": True,
                    }
                    completed.add(node_id)
                    execution_trace.append(node_id)
                    auto_completed_this_pass += 1
                    continue

                # Every dependency is done, so any incoming condition is now
                # decidable. Mirrors WorkflowExecutor._find_ready_nodes: OR-any
                # across the conditional edges, and a skip still counts as
                # "completed" for dependency purposes, so skipping is NOT
                # transitive -- an unconditional downstream node still runs.
                if use_conditional_edges and node_id in conditional_edges:
                    if not self._conditions_met(node_id, conditional_edges[node_id], outputs):
                        workflow.logger.info(f"Skipping {node_id}: no incoming edge condition matched")
                        completed.add(node_id)
                        execution_trace.append(node_id)
                        auto_completed_this_pass += 1
                        continue

                # Build immutable context for this node
                context = {
                    "node_id": node_id,
                    "node_type": node.get("type", "unknown"),
                    "node_data": (workflow_data.get("parameter_snapshot") or {}).get(node_id, node.get("data", {})),
                    "inputs": self._get_node_inputs(node_id, deps, outputs),
                    "workflow_id": workflow_id,
                    "workflow_slug": workflow_slug,
                    "tenant_id": tenant_id,
                    "session_id": session_id,
                    "execution_id": execution_id,
                    "nodes": nodes,  # Full list for tool/memory detection
                    "edges": edges,  # Full list for tool/memory detection
                    **({"parameter_snapshot": workflow_data["parameter_snapshot"]} if workflow_data.get("parameter_snapshot") else {}),
                    # Include pre-executed info if applicable
                    "pre_executed": node.get("_pre_executed", False),
                    "trigger_output": node.get("_trigger_output"),
                }
                if self._execution_control.enabled:
                    context.update({key: value for key, value in self._execution_control.carry().items()
                                    if key not in {"execution_control_root_run_id", "execution_control_released_revision"}})
                if (
                    graph_version >= AGENT_CONTEXT_GRAPH_VERSION
                    and generation > 0
                ):
                    context.update(
                        {
                            "graphVersion": graph_version,
                            "generation": generation,
                            "root_execution_id": workflow_data.get(
                                "root_execution_id"
                            )
                            or execution_id,
                            "data_scope_id": workflow_data.get(
                                "data_scope_id"
                            ),
                            # Context thread identity is distinct from the
                            # output/data namespace. Trigger listeners provide
                            # a per-firing execution id and, for chat events,
                            # the explicit conversation session.
                            "context_execution_id": workflow_data.get(
                                "context_execution_id"
                            ),
                            "context_session_id": workflow_data.get(
                                "context_session_id"
                            ),
                        }
                    )
                if "user_id" in workflow_data:
                    context["user_id"] = workflow_data.get("user_id")
                if run_scope is not None:
                    context["run_scope"] = dict(run_scope)
                context["temporal_worker_pool_enabled"] = bool(
                    frozen_routing.get("worker_pool_enabled")
                )
                if frozen_routing.get("pi_harness_version") == 1:
                    context["pi_harness_version"] = 1
                    for key in ("pi_local_control_version", "pi_local_control_state", "pi_local_control_revision"):
                        if key in workflow_data:
                            context[key] = workflow_data[key]
                if frozen_routing.get("approval_wait_version") == 1:
                    context["approval_wait_version"] = 1
                if workflow_data.get("browser_routing_version") == 1:
                    context.update(browser_routing_version=1, browser_bindings=workflow_data.get("browser_bindings") or {})
                    binding = context["browser_bindings"].get(node_id)
                    if node_type == "browser" and binding:
                        context.update(_browser_owner=binding, _browser_task_id=str(workflow.info().workflow_id))

                # F4.B: agent-as-child-workflow takes precedence over the
                # activity path for the 15 migrating agent types when its
                # flag is on. Tool calls inside the agent loop become
                # per-type activities (F4.A path) automatically.
                dispatch = self._resolve_dispatch(
                    node_type,
                    graph_version=graph_version,
                    generation=generation,
                    context_enabled=(
                        graph_version >= AGENT_CONTEXT_GRAPH_VERSION
                        and generation > 0
                    ),
                    routing_snapshot=frozen_routing,
                    pi_delegator=node_id in pi_delegators,
                )
                if node_type == "browser" and context.get("_browser_owner"):
                    from services.node_registry import get_node_class
                    cls = get_node_class("browser")
                    dispatch = {"kind": "activity", "name": f"node.browser.v{cls.version}",
                                "queue": context["_browser_owner"].get("task_queue")}
                # A preceding child start yields to the workflow event loop,
                # so a pause signal may have landed since ``ready`` was
                # computed. Re-admit every command in the batch.
                if self._execution_control.enabled and running and (
                    self._execution_control.state != "running" or self._execution_control.producers_held
                ):
                    break
                await self._wait_until_resumed()
                if dispatch["kind"] == "child_workflow":
                    # ``workflow.start_child_workflow`` is ``async def`` —
                    # awaiting it returns the ``ChildWorkflowHandle`` once
                    # the child has been started by the server (fast; not
                    # blocking on child completion). The handle is a
                    # Task-like with ``.done()`` so it slots into the same
                    # FIRST_COMPLETED loop as activity handles.
                    # Not patch-gated: every run, new or replayed, derives the
                    # id from the actual root Temporal workflow id plus the
                    # exact canvas node id (labels are mutable and need not
                    # be unique). There is no ``workflow.patched`` marker
                    # and no label-derived fallback for this id.
                    child_workflow_id = _agent_child_workflow_id(
                        workflow.info().workflow_id,
                        node_id,
                    )
                    # Unbounded by design: a run may execute — or stay
                    # cooperatively paused — for months, and Temporal's
                    # timeout timers keep ticking through a pause.
                    child_start_kwargs: Dict[str, Any] = dict(
                        args=[context],
                        id=child_workflow_id,
                    )
                    if self._execution_control.enabled:
                        async with self._execution_control.child_start():
                            handle = await workflow.start_child_workflow(dispatch["name"], **child_start_kwargs)
                    else:
                        handle = await workflow.start_child_workflow(dispatch["name"], **child_start_kwargs)
                    running[node_id] = handle
                    workflow.logger.info(f"Scheduled child workflow for node: {node_id} " f"(workflow={dispatch['name']})")
                else:
                    # F4.A activity path: per-type when the plugin class is
                    # registered AND the per-type flag is on; legacy
                    # execute_node_activity otherwise. ``activity_id`` =
                    # node_id so the Temporal Web UI history tab labels
                    # each activity by the canvas node it ran.
                    # Honor each plugin's declared RetryPolicy.
                    activity_retry_policy = retry_policy
                    from services.node_registry import get_node_class

                    node_cls = get_node_class(node_type)
                    declared_retry = getattr(node_cls, "retry_policy", None)
                    if declared_retry is not None:
                        activity_retry_policy = declared_retry.to_temporal()
                    start_kwargs: Dict[str, Any] = dict(
                        args=[context],
                        activity_id=node_id,
                        # Heartbeat is the liveness mechanism (activities
                        # self-heartbeat every 30s); start_to_close only
                        # bounds a single legitimate step. Unconditional:
                        # no ``workflow.patched`` marker gates this value.
                        start_to_close_timeout=_NODE_ACTIVITY_START_TO_CLOSE,
                        heartbeat_timeout=timedelta(minutes=2),
                        retry_policy=activity_retry_policy,
                    )
                    if getattr(node_cls, "workspace_task", False) and workflow.patched("workspace-node-cancellation-v1"):
                        start_kwargs.update(
                            start_to_close_timeout=node_cls.start_to_close_timeout,
                            heartbeat_timeout=node_cls.heartbeat_timeout,
                            cancellation_type=workflow.ActivityCancellationType.WAIT_CANCELLATION_COMPLETED,
                        )
                    if dispatch.get("queue") is not None:
                        start_kwargs["task_queue"] = dispatch["queue"]

                    # The Approval node of a generation that waits for the
                    # owner is no admitted action; the node after it is
                    # admitted after Resume like any other.
                    waits_outside_stop = node_type == APPROVAL_GATE_TYPE and frozen_routing.get("approval_wait_version") == 1
                    if self._execution_control.enabled and not waits_outside_stop:
                        admission = self._execution_control.action()
                        await admission.__aenter__()
                        self._action_contexts[node_id] = admission
                    handle = workflow.start_activity(dispatch["name"], **start_kwargs)
                    running[node_id] = handle
                    workflow.logger.info(
                        f"Scheduled activity for node: {node_id} "
                        f"(activity={dispatch['name']}, queue={dispatch.get('queue') or 'default'})"
                    )

            # Exit only when nothing is running AND this pass made no
            # progress. Auto-completed triggers count as progress — they
            # may have unblocked downstream nodes that the next
            # _find_ready_nodes pass will pick up.
            if not running:
                if auto_completed_this_pass:
                    continue
                break

            # Wait for ANY activity to complete (FIRST_COMPLETED pattern)
            done_id, result = await self._wait_any_complete(running)

            if result.get("success"):
                outputs[done_id] = result
                completed.add(done_id)
                execution_trace.append(done_id)
                workflow.logger.info(f"Node completed: {done_id}")
            else:
                # Node failed after all retries
                error_info = {
                    "node_id": done_id,
                    "error": result.get("error", "Unknown error"),
                    **{key: result[key] for key in ("hint", "requires_user_action", "retryable") if key in result},
                }
                errors.append(error_info)
                workflow.logger.error(f"Node failed: {done_id} - {error_info['error']}")
                if done_id in self._action_contexts:
                    await self._action_contexts.pop(done_id).__aexit__(None, None, None)

                # Stop workflow on failure
                # TODO: Could add option to continue with partial results
                if not self._execution_control.enabled:
                    break

            if done_id in self._action_contexts:
                await self._action_contexts.pop(done_id).__aexit__(None, None, None)

        # Build final result
        success = len(errors) == 0 and len(completed) == len(node_map)

        if run_scope is not None:
            await self._finish_chat_run(run_scope, success, errors)
        if errors:
            await self._pause_deployment_on_failure(workflow_data, nodes, errors)
        await self._record_run_completion(workflow_data, nodes, success)

        workflow.logger.info(f"Workflow complete: success={success}, " f"executed={len(execution_trace)}/{len(node_map)}")

        return {
            "success": success,
            "outputs": outputs,
            "execution_trace": execution_trace,
            "errors": errors if errors else None,
            "total_nodes": len(node_map),
        }

    def _get_node_inputs(
        self,
        node_id: str,
        deps: Dict[str, Set[str]],
        outputs: Dict[str, Any],
    ) -> Dict[str, Any]:
        """Get outputs from upstream nodes as inputs for this node."""
        inputs = {}
        for dep_id in deps.get(node_id, set()):
            if dep_id in outputs:
                inputs[dep_id] = outputs[dep_id].get("result", {})
        return inputs

    def _resolve_dispatch(
        self,
        node_type: str,
        *,
        graph_version: int = 0,
        generation: int = 0,
        context_enabled: bool = False,
        routing_snapshot: Optional[Dict[str, Any]] = None,
        pi_delegator: bool = False,
    ) -> Dict[str, Any]:
        """Resolve dispatch kind for a node type.

        Returns one of:
          - ``{"kind": "child_workflow", "name": "AgentWorkflow"}`` — when
            F4.B is enabled AND node_type is in ``AGENT_WORKFLOW_TYPES``.
          - ``{"kind": "activity", "name": <activity_name>, "queue": <queue|None>}``
            — F4.A per-type activity OR legacy fallback, depending on
            ``temporal_per_type_dispatch``.

        ``MachinaWorkflow.run`` always passes ``routing_snapshot`` from the
        workflow input (``_frozen_routing_from_input`` substitutes
        ``_SAFE_FROZEN_ROUTING`` when the input carries none), so no mutable
        Settings value can alter a run's command shape. ``None`` falls back
        to reading Settings and is reached only by direct callers such as
        unit tests.
        """
        if routing_snapshot is None:
            from core.config import Settings

            settings = Settings()
            agent_workflow_enabled = getattr(
                settings,
                "temporal_agent_workflow_enabled",
                False,
            )
            per_type_dispatch_enabled = getattr(
                settings,
                "temporal_per_type_dispatch",
                False,
            )
            worker_pool_enabled = getattr(
                settings,
                "temporal_worker_pool_enabled",
                False,
            )
        else:
            agent_workflow_enabled = (
                routing_snapshot.get("agent_workflow_enabled") is True
            )
            per_type_dispatch_enabled = (
                routing_snapshot.get("per_type_dispatch_enabled") is True
            )
            worker_pool_enabled = (
                routing_snapshot.get("worker_pool_enabled") is True
            )
        # Runtime selection is recorded before admission. Old snapshots lacking
        # this marker keep their native commands even after worker upgrades.
        if node_type == "coding_agent" and (routing_snapshot or {}).get("pi_harness_version") == 1:
            return {"kind": "child_workflow", "name": "PiCodingAgentWorkflow"}
        if pi_delegator and node_type in AGENT_WORKFLOW_TYPES and (routing_snapshot or {}).get("pi_harness_version") == 1:
            return {"kind": "child_workflow", "name": "AgentWorkflow"}
        # This new capability has no historical single-Activity implementation
        # to replay. Its task-lifetime Browser ownership requires native dispatch.
        if node_type == "browser_agent" or (agent_workflow_enabled and node_type in AGENT_WORKFLOW_TYPES):
            return {"kind": "child_workflow", "name": "AgentWorkflow"}

        name, queue = self._resolve_activity(
            node_type,
            per_type_dispatch_enabled=per_type_dispatch_enabled,
            worker_pool_enabled=worker_pool_enabled,
        )
        return {"kind": "activity", "name": name, "queue": queue}

    def _resolve_activity(
        self,
        node_type: str,
        *,
        per_type_dispatch_enabled: Optional[bool] = None,
        worker_pool_enabled: Optional[bool] = None,
    ) -> tuple[str, str | None]:
        """Resolve (activity_name, task_queue) for a node type.

        F4.A: when per-type dispatch is on AND the plugin class is
        registered, returns the per-type activity name
        ``node.{type}.v{version}``.

        Wave 16.3: the returned queue is ``cls.task_queue`` when the worker
        pool is on (each declared queue then has a dedicated
        ``TemporalWorkerPool`` worker polling it — started by
        ``services/temporal/lifecycle.py::_start_execution_engine`` right
        after the manager), or ``None`` otherwise so the activity stays on
        the workflow's default queue, which the single
        ``TemporalWorkerManager`` polls. The flag is the rollback channel,
        but only for runs started after the flip: ``MachinaWorkflow.run``
        passes both flags from the run's frozen routing snapshot, and only
        an omitted argument falls back to reading Settings.

        Falls back to ``("execute_node_activity", None)`` when:
          - the flag is off (preserves pre-F4.A behavior exactly), OR
          - the node type isn't registered as a BaseNode subclass
            (covers legacy types still on the metadata-only path).

        Determinism: lookups go through the frozen module-level
        ``_NODE_CLASS_REGISTRY`` and the caller-supplied flags — no I/O. Safe inside
        ``MachinaWorkflow.run`` per the workflow-definition contract.
        Imports are inside the method to keep the workflow module's
        top-level import set minimal and to avoid import-cycle drift.
        """
        from services.node_registry import get_node_class

        if (
            per_type_dispatch_enabled is None
            or worker_pool_enabled is None
        ):
            from core.config import Settings

            settings = Settings()
            if per_type_dispatch_enabled is None:
                per_type_dispatch_enabled = (
                    settings.temporal_per_type_dispatch
                )
            if worker_pool_enabled is None:
                worker_pool_enabled = (
                    settings.temporal_worker_pool_enabled
                )

        if not per_type_dispatch_enabled:
            return "execute_node_activity", None

        cls = get_node_class(node_type)
        if cls is None:
            return "execute_node_activity", None

        queue = cls.task_queue if worker_pool_enabled else None
        return f"node.{cls.type}.v{cls.version}", queue

    @staticmethod
    def _node_failure(error: Exception) -> Dict[str, Any]:
        """Keep safe recovery metadata through child/activity wrappers."""
        if workflow.patched("workflow-node-user-action-error-v1"):
            cause = error
            for _ in range(10):
                kind = str(getattr(cause, "type", ""))
                if kind.startswith("LLMError.") or kind == "MissingAgentProviderCredential":
                    details = getattr(cause, "details", ())
                    if details and isinstance(details[0], dict) and details[0].get("requires_user_action") is True:
                        return {
                            "success": False, "error_type": "NodeUserError",
                            "error": cause.message, "hint": details[0].get("hint"),
                            "requires_user_action": True, "retryable": False,
                        }
                cause = getattr(cause, "cause", None)
                if cause is None:
                    break
        return {"success": False, "error": str(error)}

    async def _wait_any_complete(self, running: Dict[str, Any]) -> tuple:
        """Wait for any activity to complete, return (node_id, result).

        Uses Temporal's native wait mechanism for efficient polling.
        """
        # Convert to list for iteration
        items = list(running.items())

        # Check if any already done
        for node_id, handle in items:
            if handle.done():
                del running[node_id]
                try:
                    result = await handle
                    return node_id, result
                except Exception as e:
                    return node_id, self._node_failure(e)

        # Wait for first completion using Temporal's wait
        await workflow.wait_condition(lambda: any(h.done() for _, h in items))

        # Find the completed one
        for node_id, handle in items:
            if handle.done():
                del running[node_id]
                try:
                    result = await handle
                    return node_id, result
                except Exception as e:
                    return node_id, self._node_failure(e)

        # Should not reach here
        raise RuntimeError("No activity completed after wait")

    def _filter_executable_graph(
        self,
        nodes: List[Dict],
        edges: List[Dict],
    ) -> tuple:
        """Filter out config nodes based on edge handles.

        Config nodes (tools, memory, model configs) connect via special handles
        and are consumed by their target nodes, not executed independently.

        Returns:
            Tuple of (executable_nodes, executable_edges)
        """
        node_map = {n["id"]: n for n in nodes}
        config_ids = set()

        for edge in edges:
            source_id = edge.get("source")

            # Edges to config handles mean source is a config node
            if _is_config_edge(edge, node_map):
                config_ids.add(source_id)

            # Android services connected as direct tools
            source_node = node_map.get(source_id, {})
            if source_node.get("type") in ANDROID_SERVICE_TYPES:
                config_ids.add(source_id)

            # Skill nodes (always config, connect to Zeenie)
            if source_node.get("type") in SKILL_NODE_TYPES:
                config_ids.add(source_id)

        # Filter nodes and edges
        exec_nodes = [n for n in nodes if n["id"] not in config_ids]
        exec_edges = [
            e
            for e in edges
            if e.get("source") not in config_ids
            and e.get("target") not in config_ids
            and not _is_config_edge(e, node_map)
        ]

        return exec_nodes, exec_edges

    def _build_dependency_maps(
        self,
        nodes: List[Dict],
        edges: List[Dict],
    ) -> tuple:
        """Build dependency graph from nodes and edges.

        Returns:
            Tuple of (dependencies_map, node_map)
            - dependencies_map: node_id -> set of node IDs it depends on
            - node_map: node_id -> node definition
        """
        node_map = {n["id"]: n for n in nodes}
        node_ids = set(node_map.keys())

        deps = {nid: set() for nid in node_ids}

        for edge in edges:
            src, tgt = edge.get("source"), edge.get("target")
            if src in node_ids and tgt in node_ids:
                deps[tgt].add(src)

        return deps, node_map

    def _build_conditional_edge_map(
        self,
        edges: List[Dict],
        node_map: Dict[str, Dict],
    ) -> Dict[str, List[Dict]]:
        """Group incoming edges that carry a condition, keyed by target node.

        Only targets that appear here are gated; a node with no conditional
        incoming edge keeps the unconditional "all deps done" rule.
        """
        conditional: Dict[str, List[Dict]] = {}
        for edge in edges:
            target = edge.get("target")
            if target not in node_map:
                continue
            if not (edge.get("data") or {}).get("condition"):
                continue
            conditional.setdefault(target, []).append(edge)
        return conditional

    def _conditions_met(
        self,
        target_node_id: str,
        edges: List[Dict],
        outputs: Dict[str, Any],
    ) -> bool:
        """Return whether any conditional incoming edge admits this node.

        OR-any, matching ``WorkflowExecutor._evaluate_incoming_conditions``.
        A source that produced no output evaluates against ``{}`` rather than
        raising, so a skipped upstream cannot wedge the graph.
        """
        for edge in edges:
            condition = (edge.get("data") or {}).get("condition")
            if not condition:
                continue
            source_output = outputs.get(edge.get("source")) or {}
            if evaluate_condition(condition, source_output):
                return True
        return False

    def _find_ready_nodes(
        self,
        deps: Dict[str, Set[str]],
        completed: Set[str],
        running: Dict[str, Any],
        node_map: Dict[str, Dict],
    ) -> List[str]:
        """Find nodes ready to execute.

        A node is ready when:
        - All its dependencies have completed
        - It's not already running
        - It's not already completed
        """
        ready = []
        for node_id in node_map:
            if node_id in completed or node_id in running:
                continue
            if deps[node_id] <= completed:
                ready.append(node_id)
        return ready