Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(control): stop reset generations from probing and wedging future runs
trohitg91d56ae6f0c6
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.pyindex 8f5acf96..5f753305 100644--- a/server/services/deployment/handlers.py+++ b/server/services/deployment/handlers.py@@ -30,7 +30,11 @@ from fastapi import WebSocketfrom core.logging import get_loggerfrom services.ws_handler_registry import ws_handler-from services.deployment.control import WorkflowControlService, serialize_control+from services.deployment.control import (+ ACTIVE_STATES,+ WorkflowControlService,+ serialize_control,+)# ``core.container`` and ``services.status_broadcaster`` are lazy-imported# inside each handler body. This module is imported transitively via@@ -50,6 +54,17 @@ class TemporalControlAckMismatch(RuntimeError):"""Temporal completed an Update without returning the requested state."""+class ControllerExecutionMissing(RuntimeError):+ """The controller execution this generation names no longer exists.++ Distinct from :class:`TemporalControlUnavailable`, and the distinction is+ the whole point: "unavailable" is transient and worth waiting out, whereas+ a deleted execution never comes back. Treating the second as the first is+ what let a generation sit in ``pausing`` forever, which in turn blocked+ every future Start on that workflow.+ """++# Per-workflow deployment tasks for proper cancellation (Temporal/n8n pattern).# Maps workflow_id -> asyncio.Task for parallel workflow deployments._deployment_tasks: Dict[str, asyncio.Task] = {}@@ -487,7 +502,13 @@ async def _update_controller_state(async def _query_controller_state(control) -> Optional[Dict[str, Any]]:- """Return controller state when reachable; status reads remain resilient."""+ """Return controller state when reachable; status reads remain resilient.++ Raises :class:`ControllerExecutionMissing` when Temporal reports the+ execution does not exist, rather than folding that into the ``None``+ that means "could not reach it". Callers need to tell the two apart:+ one resolves itself, the other never will.+ """handle = _controller_handle(control)if handle is None:return None@@ -495,6 +516,19 @@ async def _query_controller_state(control) -> Optional[Dict[str, Any]]:result = await handle.query("status")return result if isinstance(result, dict) else Noneexcept Exception as exc:+ if _temporal_target_already_gone(exc):+ # Expected and unremarkable: a controller closes when its+ # generation ends, and Temporal deletes closed executions once+ # the namespace retention window passes. Not a warning.+ logger.debug(+ "Workflow controller execution no longer exists",+ workflow_id=control.workflow_id,+ controller_workflow_id=control.controller_workflow_id,+ status=control.status,+ )+ raise ControllerExecutionMissing(+ str(control.controller_workflow_id)+ ) from exclogger.warning("Workflow controller status query failed",workflow_id=control.workflow_id,@@ -504,6 +538,44 @@ async def _query_controller_state(control) -> Optional[Dict[str, Any]]:return None+# A missing controller is only actionable for a generation the database still+# believes is alive. ``resetting`` is excluded deliberately: it already has an+# explicit retry path through Reset, and auto-failing it here could race a+# reset running concurrently in another request.+_MISSING_CONTROLLER_FAILS = frozenset(+ {"starting", "running", "pausing", "paused", "resuming"}+)+++async def _fail_missing_controller(service: WorkflowControlService, control):+ """Converge a generation whose controller has vanished to ``failed``.++ Without this the row keeps whatever live status it had, and because+ ``begin_generation`` refuses to open a new generation unless the latest+ one is ``reset``, the workflow could never be started again. ``failed`` is+ both honest and recoverable -- Reset accepts it, and Reset is what returns+ the workflow to ``ready``.+ """+ if control.status not in _MISSING_CONTROLLER_FAILS:+ return control+ logger.warning(+ "Workflow controller execution is gone; failing the generation so it "+ "can be reset",+ workflow_id=control.workflow_id,+ controller_workflow_id=control.controller_workflow_id,+ status=control.status,+ generation=control.generation,+ )+ try:+ failed = await service.fail(control, "controller_execution_missing")+ except ValueError:+ # Lost the CAS to a concurrent writer, which means someone else has+ # already moved this row. Their transition wins; report what we have.+ return control+ await _broadcast_control(failed)+ return failed++def _generation_visibility_query(control) -> str:"""Visibility query for controller descendants and standalone trigger roots."""if control.controller_workflow_id:@@ -796,7 +868,19 @@ async def _reconcile_control(service: WorkflowControlService, control):"""Finish an interrupted DB transition from acknowledged Temporal state."""from core.container import container- controller_status = await _query_controller_state(control)+ # A generation that has been reset (or failed) closed its controller on+ # purpose, and Temporal deletes closed executions once the namespace+ # retention window passes. Probing one is therefore a guaranteed-failing+ # RPC on every single status read, forever. Terminal generations are+ # answered from the database, which is authoritative for them anyway.+ if control.status not in ACTIVE_STATES:+ return control, None++ try:+ controller_status = await _query_controller_state(control)+ except ControllerExecutionMissing:+ return await _fail_missing_controller(service, control), None+transition_target = {"pausing": ("paused", "paused"),"resuming": ("running", "running"),@@ -1058,7 +1142,13 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> D# Reaching ``running`` commits successful deployment setup. Projection# failures after that point must not tear down a live durable generation;# the client can recover the committed state through its status resync.- controller_status = await _query_controller_state(control)+ try:+ controller_status = await _query_controller_state(control)+ except ControllerExecutionMissing:+ # Same rule as the comment above: this is a projection read taken+ # after the generation committed. Reporting no controller state is+ # acceptable; propagating and unwinding a live generation is not.+ controller_status = Nonepayload = await _broadcast_control(control,controller_status=controller_status,diff --git a/server/tests/services/test_workflow_control_handlers.py b/server/tests/services/test_workflow_control_handlers.pyindex a5830312..479ddb1c 100644--- a/server/tests/services/test_workflow_control_handlers.py+++ b/server/tests/services/test_workflow_control_handlers.py@@ -1140,3 +1140,134 @@ async def test_generation_fanout_expands_tagged_standalone_root_trees():"('cron-firing-root', 'workflow-control-wf-g1') ""AND ExecutionStatus='Running'")+++# ============================================================================+# A controller execution that no longer exists+#+# Temporal deletes closed executions once the namespace retention window+# passes (24h on the dev server). A generation's controller closes when that+# generation is reset, so every control row eventually names an execution+# Temporal has forgotten. Two things must hold: terminal generations must not+# be probed at all, and a *live* generation whose controller vanished must+# converge instead of waiting for something that will never answer.+# ============================================================================+++class _Gone(Exception):+ """Shaped like the Temporal error, matched by _temporal_target_already_gone."""++ def __str__(self) -> str:+ return (+ 'workflow execution not found for workflow ID '+ '"workflow-control-wf-g1" and run ID "run-1"'+ )+++def _reconcile_service(**overrides):+ database = SimpleNamespace(**overrides.pop("database", {}))+ return SimpleNamespace(database=database, **overrides)++[email protected]("terminal_status", ["reset", "failed"])+async def test_terminal_generation_is_never_probed(monkeypatch, terminal_status):+ """The reported symptom: a doomed RPC on every status read, forever."""+ probe = AsyncMock(return_value={"state": "running"})+ monkeypatch.setattr(handlers, "_query_controller_state", probe)++ control = _control(terminal_status, 6)+ result, controller_status = await handlers._reconcile_control(+ _reconcile_service(), control+ )++ probe.assert_not_awaited()+ assert result is control+ assert controller_status is None+++async def test_live_generation_with_missing_controller_is_failed(monkeypatch):+ """Otherwise it stays non-'reset' forever and blocks every future Start."""+ failed = _control("failed", 7, terminal_reason="controller_execution_missing")+ fail = AsyncMock(return_value=failed)+ broadcast = AsyncMock(return_value={})++ monkeypatch.setattr(+ handlers,+ "_query_controller_state",+ AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),+ )+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)++ control = _control("pausing", 6)+ result, controller_status = await handlers._reconcile_control(+ _reconcile_service(fail=fail), control+ )++ assert result is failed+ assert controller_status is None+ assert fail.await_args.args[1] == "controller_execution_missing"+ broadcast.assert_awaited_once()+++async def test_resetting_is_not_auto_failed(monkeypatch):+ """Reset owns its own retry; auto-failing here could race a live reset."""+ fail = AsyncMock()+ monkeypatch.setattr(+ handlers,+ "_query_controller_state",+ AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),+ )++ control = _control("resetting", 5)+ result, _ = await handlers._reconcile_control(+ _reconcile_service(fail=fail), control+ )++ fail.assert_not_awaited()+ assert result is control+++async def test_a_transient_query_failure_does_not_fail_the_generation(monkeypatch):+ """'Unreachable' must stay recoverable -- only 'gone' is terminal."""+ fail = AsyncMock()+ monkeypatch.setattr(+ handlers, "_query_controller_state", AsyncMock(return_value=None)+ )++ control = _control("running", 4)+ result, controller_status = await handlers._reconcile_control(+ _reconcile_service(fail=fail), control+ )++ fail.assert_not_awaited()+ assert result is control+ assert controller_status is None+++async def test_missing_execution_is_classified_as_gone_not_as_an_outage():+ """Locks the string match against the real Temporal error text."""+ assert handlers._temporal_target_already_gone(_Gone()) is True+ assert handlers._temporal_target_already_gone(RuntimeError("deadline")) is False+++async def test_query_raises_missing_rather_than_returning_none(monkeypatch):+ """The distinction the whole fix rests on."""+ handle = SimpleNamespace(query=AsyncMock(side_effect=_Gone()))+ monkeypatch.setattr(handlers, "_controller_handle", lambda _c: handle)++ with pytest.raises(handlers.ControllerExecutionMissing):+ await handlers._query_controller_state(_control("running", 1))+++async def test_query_still_returns_none_for_a_real_outage(monkeypatch):+ handle = SimpleNamespace(query=AsyncMock(side_effect=RuntimeError("deadline")))+ monkeypatch.setattr(handlers, "_controller_handle", lambda _c: handle)++ assert await handlers._query_controller_state(_control("running", 1)) is None