Skip to content

Self-improving AI Employees for everyone

fix(control): stop reset generations from probing and wedging future runs

trohitg91d56ae6f0c6
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index 8f5acf96..5f753305 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -30,7 +30,11 @@ from fastapi import WebSocket
from core.logging import get_logger
from services.ws_handler_registry import ws_handler
-from services.deployment.control import WorkflowControlService, serialize_control
+from services.deployment.control import (
+ ACTIVE_STATES,
+ WorkflowControlService,
+ serialize_control,
+)
# ``core.container`` and ``services.status_broadcaster`` are lazy-imported
# inside each handler body. This module is imported transitively via
@@ -50,6 +54,17 @@ class TemporalControlAckMismatch(RuntimeError):
"""Temporal completed an Update without returning the requested state."""
+class ControllerExecutionMissing(RuntimeError):
+ """The controller execution this generation names no longer exists.
+
+ Distinct from :class:`TemporalControlUnavailable`, and the distinction is
+ the whole point: "unavailable" is transient and worth waiting out, whereas
+ a deleted execution never comes back. Treating the second as the first is
+ what let a generation sit in ``pausing`` forever, which in turn blocked
+ every future Start on that workflow.
+ """
+
+
# Per-workflow deployment tasks for proper cancellation (Temporal/n8n pattern).
# Maps workflow_id -> asyncio.Task for parallel workflow deployments.
_deployment_tasks: Dict[str, asyncio.Task] = {}
@@ -487,7 +502,13 @@ async def _update_controller_state(
async def _query_controller_state(control) -> Optional[Dict[str, Any]]:
- """Return controller state when reachable; status reads remain resilient."""
+ """Return controller state when reachable; status reads remain resilient.
+
+ Raises :class:`ControllerExecutionMissing` when Temporal reports the
+ execution does not exist, rather than folding that into the ``None``
+ that means "could not reach it". Callers need to tell the two apart:
+ one resolves itself, the other never will.
+ """
handle = _controller_handle(control)
if handle is None:
return None
@@ -495,6 +516,19 @@ async def _query_controller_state(control) -> Optional[Dict[str, Any]]:
result = await handle.query("status")
return result if isinstance(result, dict) else None
except Exception as exc:
+ if _temporal_target_already_gone(exc):
+ # Expected and unremarkable: a controller closes when its
+ # generation ends, and Temporal deletes closed executions once
+ # the namespace retention window passes. Not a warning.
+ logger.debug(
+ "Workflow controller execution no longer exists",
+ workflow_id=control.workflow_id,
+ controller_workflow_id=control.controller_workflow_id,
+ status=control.status,
+ )
+ raise ControllerExecutionMissing(
+ str(control.controller_workflow_id)
+ ) from exc
logger.warning(
"Workflow controller status query failed",
workflow_id=control.workflow_id,
@@ -504,6 +538,44 @@ async def _query_controller_state(control) -> Optional[Dict[str, Any]]:
return None
+# A missing controller is only actionable for a generation the database still
+# believes is alive. ``resetting`` is excluded deliberately: it already has an
+# explicit retry path through Reset, and auto-failing it here could race a
+# reset running concurrently in another request.
+_MISSING_CONTROLLER_FAILS = frozenset(
+ {"starting", "running", "pausing", "paused", "resuming"}
+)
+
+
+async def _fail_missing_controller(service: WorkflowControlService, control):
+ """Converge a generation whose controller has vanished to ``failed``.
+
+ Without this the row keeps whatever live status it had, and because
+ ``begin_generation`` refuses to open a new generation unless the latest
+ one is ``reset``, the workflow could never be started again. ``failed`` is
+ both honest and recoverable -- Reset accepts it, and Reset is what returns
+ the workflow to ``ready``.
+ """
+ if control.status not in _MISSING_CONTROLLER_FAILS:
+ return control
+ logger.warning(
+ "Workflow controller execution is gone; failing the generation so it "
+ "can be reset",
+ workflow_id=control.workflow_id,
+ controller_workflow_id=control.controller_workflow_id,
+ status=control.status,
+ generation=control.generation,
+ )
+ try:
+ failed = await service.fail(control, "controller_execution_missing")
+ except ValueError:
+ # Lost the CAS to a concurrent writer, which means someone else has
+ # already moved this row. Their transition wins; report what we have.
+ return control
+ await _broadcast_control(failed)
+ return failed
+
+
def _generation_visibility_query(control) -> str:
"""Visibility query for controller descendants and standalone trigger roots."""
if control.controller_workflow_id:
@@ -796,7 +868,19 @@ async def _reconcile_control(service: WorkflowControlService, control):
"""Finish an interrupted DB transition from acknowledged Temporal state."""
from core.container import container
- controller_status = await _query_controller_state(control)
+ # A generation that has been reset (or failed) closed its controller on
+ # purpose, and Temporal deletes closed executions once the namespace
+ # retention window passes. Probing one is therefore a guaranteed-failing
+ # RPC on every single status read, forever. Terminal generations are
+ # answered from the database, which is authoritative for them anyway.
+ if control.status not in ACTIVE_STATES:
+ return control, None
+
+ try:
+ controller_status = await _query_controller_state(control)
+ except ControllerExecutionMissing:
+ return await _fail_missing_controller(service, control), None
+
transition_target = {
"pausing": ("paused", "paused"),
"resuming": ("running", "running"),
@@ -1058,7 +1142,13 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
# Reaching ``running`` commits successful deployment setup. Projection
# failures after that point must not tear down a live durable generation;
# the client can recover the committed state through its status resync.
- controller_status = await _query_controller_state(control)
+ try:
+ controller_status = await _query_controller_state(control)
+ except ControllerExecutionMissing:
+ # Same rule as the comment above: this is a projection read taken
+ # after the generation committed. Reporting no controller state is
+ # acceptable; propagating and unwinding a live generation is not.
+ controller_status = None
payload = await _broadcast_control(
control,
controller_status=controller_status,
diff --git a/server/tests/services/test_workflow_control_handlers.py b/server/tests/services/test_workflow_control_handlers.py
index a5830312..479ddb1c 100644
--- a/server/tests/services/test_workflow_control_handlers.py
+++ b/server/tests/services/test_workflow_control_handlers.py
@@ -1140,3 +1140,134 @@ async def test_generation_fanout_expands_tagged_standalone_root_trees():
"('cron-firing-root', 'workflow-control-wf-g1') "
"AND ExecutionStatus='Running'"
)
+
+
+# ============================================================================
+# A controller execution that no longer exists
+#
+# Temporal deletes closed executions once the namespace retention window
+# passes (24h on the dev server). A generation's controller closes when that
+# generation is reset, so every control row eventually names an execution
+# Temporal has forgotten. Two things must hold: terminal generations must not
+# be probed at all, and a *live* generation whose controller vanished must
+# converge instead of waiting for something that will never answer.
+# ============================================================================
+
+
+class _Gone(Exception):
+ """Shaped like the Temporal error, matched by _temporal_target_already_gone."""
+
+ def __str__(self) -> str:
+ return (
+ 'workflow execution not found for workflow ID '
+ '"workflow-control-wf-g1" and run ID "run-1"'
+ )
+
+
+def _reconcile_service(**overrides):
+ database = SimpleNamespace(**overrides.pop("database", {}))
+ return SimpleNamespace(database=database, **overrides)
+
+
[email protected]("terminal_status", ["reset", "failed"])
+async def test_terminal_generation_is_never_probed(monkeypatch, terminal_status):
+ """The reported symptom: a doomed RPC on every status read, forever."""
+ probe = AsyncMock(return_value={"state": "running"})
+ monkeypatch.setattr(handlers, "_query_controller_state", probe)
+
+ control = _control(terminal_status, 6)
+ result, controller_status = await handlers._reconcile_control(
+ _reconcile_service(), control
+ )
+
+ probe.assert_not_awaited()
+ assert result is control
+ assert controller_status is None
+
+
+async def test_live_generation_with_missing_controller_is_failed(monkeypatch):
+ """Otherwise it stays non-'reset' forever and blocks every future Start."""
+ failed = _control("failed", 7, terminal_reason="controller_execution_missing")
+ fail = AsyncMock(return_value=failed)
+ broadcast = AsyncMock(return_value={})
+
+ monkeypatch.setattr(
+ handlers,
+ "_query_controller_state",
+ AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),
+ )
+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
+
+ control = _control("pausing", 6)
+ result, controller_status = await handlers._reconcile_control(
+ _reconcile_service(fail=fail), control
+ )
+
+ assert result is failed
+ assert controller_status is None
+ assert fail.await_args.args[1] == "controller_execution_missing"
+ broadcast.assert_awaited_once()
+
+
+async def test_resetting_is_not_auto_failed(monkeypatch):
+ """Reset owns its own retry; auto-failing here could race a live reset."""
+ fail = AsyncMock()
+ monkeypatch.setattr(
+ handlers,
+ "_query_controller_state",
+ AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),
+ )
+
+ control = _control("resetting", 5)
+ result, _ = await handlers._reconcile_control(
+ _reconcile_service(fail=fail), control
+ )
+
+ fail.assert_not_awaited()
+ assert result is control
+
+
+async def test_a_transient_query_failure_does_not_fail_the_generation(monkeypatch):
+ """'Unreachable' must stay recoverable -- only 'gone' is terminal."""
+ fail = AsyncMock()
+ monkeypatch.setattr(
+ handlers, "_query_controller_state", AsyncMock(return_value=None)
+ )
+
+ control = _control("running", 4)
+ result, controller_status = await handlers._reconcile_control(
+ _reconcile_service(fail=fail), control
+ )
+
+ fail.assert_not_awaited()
+ assert result is control
+ assert controller_status is None
+
+
+async def test_missing_execution_is_classified_as_gone_not_as_an_outage():
+ """Locks the string match against the real Temporal error text."""
+ assert handlers._temporal_target_already_gone(_Gone()) is True
+ assert handlers._temporal_target_already_gone(RuntimeError("deadline")) is False
+
+
+async def test_query_raises_missing_rather_than_returning_none(monkeypatch):
+ """The distinction the whole fix rests on."""
+ handle = SimpleNamespace(query=AsyncMock(side_effect=_Gone()))
+ monkeypatch.setattr(handlers, "_controller_handle", lambda _c: handle)
+
+ with pytest.raises(handlers.ControllerExecutionMissing):
+ await handlers._query_controller_state(_control("running", 1))
+
+
+async def test_query_still_returns_none_for_a_real_outage(monkeypatch):
+ handle = SimpleNamespace(query=AsyncMock(side_effect=RuntimeError("deadline")))
+ monkeypatch.setattr(handlers, "_controller_handle", lambda _c: handle)
+
+ assert await handlers._query_controller_state(_control("running", 1)) is None