Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(temporal): the Approval node waits outside Stop's checkpoint (RFC-0007 AP.5)
trohitg2fabb9ea4f63
diff --git a/docs-internal/TEMPORAL_ARCHITECTURE.md b/docs-internal/TEMPORAL_ARCHITECTURE.mdindex c5f3ea07..b1e2e753 100644--- a/docs-internal/TEMPORAL_ARCHITECTURE.md+++ b/docs-internal/TEMPORAL_ARCHITECTURE.md@@ -873,7 +873,9 @@ Declined, expired or cancelled, it reads why nothing ran. Every change to such await has not read while the generation runs (`reconcile.renudge_turns`). A cancelled wait ends a pending row`cancelled`; a Reset terminates the waits with their turns and cancels the rows they waited on(`waiting.cancel_waits`). It takes no part in generation control, so Stop's tree walk leaves it out-(`execution_control.NON_PARTICIPANTS`).+(`execution_control.NON_PARTICIPANTS`). In such a generation the Approval node's activity is no admitted action+either (`workflow.APPROVAL_GATE_TYPE`, RFC-0007 §4.8): Stop completes while it waits for the owner, and the node after+it is admitted after Resume like any other. In older generations it still holds Stop until the owner decides.A send that broke off midway outside `ApprovedToolCallWorkflow` leaves a card the same way (`outcome: unknown`, see"Worker crashes mid-execution" below). A graph node's card keeps what the node sent, its templates resolved and its secrets left out;diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex fc44a82e..69a38217 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -97,6 +97,11 @@ SKILL_NODE_TYPES = {"masterSkill",}+# The Approval node (services/approvals/contract.py). In a generation that+# waits for the owner (approval_wait_version 1) it waits outside Stop's+# checkpoint, so Stop completes while it waits (RFC-0007 §4.8).+APPROVAL_GATE_TYPE = "approvalGate"+# F4.B: agent types that migrate to AgentWorkflow (Temporal child workflow).# When ``temporal_agent_workflow_enabled`` is True the orchestrator schedules# AgentWorkflow for these node types instead of an activity. Tool calls@@ -873,7 +878,11 @@ class MachinaWorkflow:if dispatch.get("queue") is not None:start_kwargs["task_queue"] = dispatch["queue"]- if self._execution_control.enabled:+ # The Approval node of a generation that waits for the+ # owner is no admitted action; the node after it is+ # admitted after Resume like any other.+ waits_outside_stop = node_type == APPROVAL_GATE_TYPE and frozen_routing.get("approval_wait_version") == 1+ if self._execution_control.enabled and not waits_outside_stop:admission = self._execution_control.action()await admission.__aenter__()self._action_contexts[node_id] = admissiondiff --git a/server/tests/temporal/test_approval_gate_stop_integration.py b/server/tests/temporal/test_approval_gate_stop_integration.pynew file mode 100644index 00000000..2b340e88--- /dev/null+++ b/server/tests/temporal/test_approval_gate_stop_integration.py@@ -0,0 +1,143 @@+"""The Approval node and Stop (RFC-0007 §4.8), on the real SDK with stub+activities: in a generation that waits for the owner, Stop completes while+the gate waits, and the node after it runs only after Resume; in an older+generation the gate holds Stop until the owner decides. Every history+replays."""++from __future__ import annotations++import asyncio+import subprocess+import sys+from pathlib import Path+from uuid import uuid4++from temporalio import activity+from temporalio.client import WorkflowHistory+from temporalio.testing import WorkflowEnvironment+from temporalio.worker import Replayer, Worker++from services.temporal.workflow import APPROVAL_GATE_TYPE, MachinaWorkflow++QUEUE = "approval-gate-stop"+CALLS: list[str] = []+GATE: dict[str, asyncio.Event] = {}++[email protected](name="execute_node_activity")+async def _execute_node(context: dict) -> dict:+ node_id = context["node_id"]+ CALLS.append(node_id)+ if node_id == "gate":+ GATE["started"].set()+ await GATE["decided"].wait()+ return {"success": True, "node_id": node_id, "result": {"done": node_id}}++[email protected](name="execution_control.register")+async def _register(payload: dict) -> dict:+ return {"participant_state": "running", "participant_revision": 0}++[email protected](name="execution_control.unregister")+async def _unregister(payload: dict) -> dict:+ return {}+++ACTIVITIES = [_execute_node, _register, _unregister]+PAUSED = {"state": "paused", "revision": 1, "generation": 7}+RUNNING = {"state": "running", "revision": 2, "generation": 7, "producers_held": False}+++def _input(waits: bool) -> dict:+ routing = {"version": 1, "agent_workflow_enabled": False, "per_type_dispatch_enabled": False, "worker_pool_enabled": False}+ if waits:+ routing["approval_wait_version"] = 1+ return {+ "nodes": [{"id": "gate", "type": APPROVAL_GATE_TYPE, "data": {}}, {"id": "reply", "type": "controlProbe", "data": {}}],+ "edges": [{"source": "gate", "target": "reply"}],+ "workflow_id": "graph",+ "workflow_slug": "graph",+ "execution_control_version": 1,+ "generation": 7,+ "graphVersion": 2,+ "controller_workflow_id": "controller",+ "_temporal_routing_v1": routing,+ }+++def _reset() -> None:+ CALLS.clear()+ GATE.update(started=asyncio.Event(), decided=asyncio.Event())+++async def _replay(history: WorkflowHistory) -> None:+ result = await Replayer(workflows=[MachinaWorkflow]).replay_workflow(history)+ assert result.replay_failure is None+++async def _stop_completes_while_the_gate_waits(environment: WorkflowEnvironment) -> None:+ _reset()+ async with Worker(environment.client, task_queue=QUEUE, workflows=[MachinaWorkflow], activities=ACTIVITIES, max_cached_workflows=0):+ handle = await environment.client.start_workflow("MachinaWorkflow", _input(waits=True), id=f"waits-{uuid4()}", task_queue=QUEUE)+ await asyncio.wait_for(GATE["started"].wait(), 10)+ await handle.execute_update("set_control_state", PAUSED)+ drained = await asyncio.wait_for(handle.execute_update("wait_for_checkpoint", PAUSED), 10)+ assert drained["checkpoint"] and drained["active_actions"] == 0+ # Decided while stopped: the node after it runs only after Resume.+ GATE["decided"].set()+ await asyncio.sleep(0.3)+ assert CALLS == ["gate"]+ await handle.execute_update("set_control_state", RUNNING)+ result = await asyncio.wait_for(handle.result(), 10)+ history = await handle.fetch_history()+ assert result["success"] is True and CALLS == ["gate", "reply"]+ await _replay(history)+++async def _an_older_generation_s_gate_holds_stop(environment: WorkflowEnvironment) -> None:+ _reset()+ async with Worker(environment.client, task_queue=QUEUE, workflows=[MachinaWorkflow], activities=ACTIVITIES, max_cached_workflows=0):+ handle = await environment.client.start_workflow("MachinaWorkflow", _input(waits=False), id=f"holds-{uuid4()}", task_queue=QUEUE)+ await asyncio.wait_for(GATE["started"].wait(), 10)+ await handle.execute_update("set_control_state", PAUSED)+ drain = asyncio.ensure_future(handle.execute_update("wait_for_checkpoint", PAUSED))+ await asyncio.sleep(0.3)+ assert not drain.done()+ GATE["decided"].set()+ assert (await asyncio.wait_for(drain, 10))["checkpoint"]+ await handle.execute_update("set_control_state", RUNNING)+ result = await asyncio.wait_for(handle.result(), 10)+ history = await handle.fetch_history()+ assert result["success"] is True and CALLS == ["gate", "reply"]+ await _replay(history)+++async def _main() -> None:+ async with await WorkflowEnvironment.start_time_skipping() as environment:+ with environment.auto_time_skipping_disabled():+ await _stop_completes_while_the_gate_waits(environment)+ await _an_older_generation_s_gate_holds_stop(environment)+++def test_the_workflow_names_the_gate_as_the_approvals_contract_does() -> None:+ from services.approvals.contract import APPROVAL_GATE_TYPE as CONTRACT_GATE_TYPE++ assert APPROVAL_GATE_TYPE == CONTRACT_GATE_TYPE+++def test_stop_completes_while_the_gate_waits_for_the_owner() -> None:+ result = subprocess.run(+ [sys.executable, "-m", "tests.temporal.test_approval_gate_stop_integration"],+ cwd=Path(__file__).parents[2],+ stdin=subprocess.DEVNULL,+ stdout=subprocess.PIPE,+ stderr=subprocess.STDOUT,+ timeout=120,+ check=False,+ )+ assert result.returncode == 0, result.stdout.decode(errors="replace")[-12000:]+++if __name__ == "__main__":+ asyncio.run(_main())