Skip to content

Self-improving AI Employees for everyone

48.2 KB
Raw Blame
"""Control-handler contract tests for acknowledged Temporal lifecycle changes."""

from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock

import pytest

from services.deployment import handlers


@pytest.fixture(autouse=True)
def no_workspace_tasks(monkeypatch):
    # These lifecycle contracts use ordinary graph nodes. Workspace-specific
    # admission and cleanup are covered in test_workspace_task_reset.py.
    monkeypatch.setattr(handlers, "_workspace_runtime_status", AsyncMock(return_value={}))
    monkeypatch.setattr(handlers, "_reset_workspace_tasks", AsyncMock(return_value={"present": False}))


def _control(
    status: str,
    revision: int,
    *,
    terminal_reason: str | None = None,
):
    return SimpleNamespace(
        id="workflow-control:wf:1",
        workflow_id="wf",
        generation=1,
        execution_id="execution-1",
        root_execution_id="execution-1",
        data_scope_id="execution-1",
        controller_workflow_id="workflow-control-wf-g1",
        controller_run_id="controller-run-1",
        status=status,
        revision=revision,
        created_at=None,
        updated_at=None,
        terminal_reason=terminal_reason,
    )


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("latest", "expected_revision", "state", "revision"),
    [
        (_control("running", 2), 0, "running", 2),
        (_control("reset", 4), 3, "ready", 4),
        (None, 3, "never_started", 0),
    ],
)
async def test_stale_start_returns_current_status_without_starting_or_error_trace(
    monkeypatch, caplog, latest, expected_revision, state, revision,
):
    service = SimpleNamespace(
        database=SimpleNamespace(
            get_workflow_control_by_idempotency_key=AsyncMock(return_value=None),
            get_latest_workflow_control=AsyncMock(return_value=latest),
        ),
        begin_generation=AsyncMock(),
    )
    start_controller = AsyncMock()
    deploy = AsyncMock()
    snapshot = {"workflow_id": "wf", "state": state, "revision": revision}
    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(handlers, "_control_payload", AsyncMock(return_value=snapshot))
    monkeypatch.setattr(handlers, "_with_runtime_counts", AsyncMock(return_value=snapshot))
    monkeypatch.setattr(handlers, "_start_controller", start_controller)
    monkeypatch.setattr(handlers, "handle_deploy_workflow", deploy)

    result = await handlers.handle_start_workflow(
        {"workflow_id": "wf", "expected_revision": expected_revision,
         "idempotency_key": "stale-start"},
        None,
    )

    assert result == {"success": False, "error": "control_revision_conflict", **snapshot}
    service.begin_generation.assert_not_awaited()
    start_controller.assert_not_awaited()
    deploy.assert_not_awaited()
    assert not any(record.levelno >= 40 or record.exc_info for record in caplog.records)


@pytest.mark.asyncio
async def test_start_waits_for_deployment_setup_before_publishing_running(monkeypatch):
    starting = _control("starting", 0)
    starting_with_run = _control("starting", 1)
    running = _control("running", 2)
    order: list[str] = []

    async def transition(_control_value, **kwargs):
        order.append(f"db:{kwargs['status']}")
        return (
            starting_with_run
            if kwargs["status"] == "starting"
            else running
        )

    service = SimpleNamespace(
        database=SimpleNamespace(
            get_workflow_control_by_idempotency_key=AsyncMock(return_value=None),
            get_latest_workflow_control=AsyncMock(return_value=None),
            update_workflow_run_data_scope=AsyncMock(return_value=True),
        ),
        begin_generation=AsyncMock(return_value=(starting, True)),
        transition=AsyncMock(side_effect=transition),
    )

    async def broadcast(control, **_kwargs):
        order.append(f"broadcast:{control.status}:{control.revision}")
        return {"state": control.status, "revision": control.revision}

    async def deployment_setup(_workflow_id):
        order.append("deployment:ready")
        return {"success": True}

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
    monkeypatch.setattr(
        handlers,
        "_start_controller",
        AsyncMock(return_value="controller-run-1"),
    )
    monkeypatch.setattr(
        handlers,
        "handle_deploy_workflow",
        AsyncMock(return_value={"success": True}),
    )
    monkeypatch.setattr(handlers, "_await_deployment_setup", deployment_setup)
    monkeypatch.setattr(
        handlers,
        "_query_controller_state",
        AsyncMock(return_value={"state": "running", "revision": 0}),
    )

    result = await handlers.handle_start_workflow(
        {
            "workflow_id": "wf",
            "nodes": [{"id": "start-1", "type": "start", "data": {}}],
            "edges": [],
            "expected_revision": 0,
            "idempotency_key": "start-request-1",
        },
        None,
    )

    assert result["success"] is True
    assert result["state"] == "running"
    assert order == [
        "broadcast:starting:0",
        "db:starting",
        "broadcast:starting:1",
        "deployment:ready",
        "db:running",
        "broadcast:running:2",
    ]


@pytest.mark.asyncio
async def test_start_post_commit_broadcast_failure_does_not_tear_down_running(
    monkeypatch,
):
    starting = _control("starting", 0)
    starting_with_run = _control("starting", 1)
    running = _control("running", 2)

    async def transition(_control_value, **kwargs):
        if kwargs["status"] == "starting":
            return starting_with_run
        return running

    async def broadcast(control, **_kwargs):
        if control.status == "running":
            raise RuntimeError("status broadcast unavailable")
        return {"state": control.status, "revision": control.revision}

    service = SimpleNamespace(
        database=SimpleNamespace(
            get_workflow_control_by_idempotency_key=AsyncMock(
                return_value=None,
            ),
            get_latest_workflow_control=AsyncMock(return_value=None),
            update_workflow_run_data_scope=AsyncMock(return_value=True),
        ),
        begin_generation=AsyncMock(return_value=(starting, True)),
        transition=AsyncMock(side_effect=transition),
        fail=AsyncMock(),
    )
    signal_controller = AsyncMock()
    terminate_generation = AsyncMock()

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
    monkeypatch.setattr(
        handlers,
        "_start_controller",
        AsyncMock(return_value="controller-run-1"),
    )
    monkeypatch.setattr(
        handlers,
        "handle_deploy_workflow",
        AsyncMock(return_value={"success": True}),
    )
    monkeypatch.setattr(
        handlers,
        "_await_deployment_setup",
        AsyncMock(return_value={"success": True}),
    )
    monkeypatch.setattr(
        handlers,
        "_query_controller_state",
        AsyncMock(return_value={"state": "running", "revision": 0}),
    )
    monkeypatch.setattr(handlers, "_signal_controller", signal_controller)
    monkeypatch.setattr(
        handlers,
        "_terminate_generation_workflows",
        terminate_generation,
    )

    result = await handlers.handle_start_workflow(
        {
            "workflow_id": "wf",
            "nodes": [{"id": "start-1", "type": "start", "data": {}}],
            "edges": [],
            "expected_revision": 0,
            "idempotency_key": "start-request-1",
        },
        None,
    )

    assert result == {
        "success": False,
        "error": "status broadcast unavailable",
    }
    assert service.transition.await_count == 2
    assert service.transition.await_args.kwargs["status"] == "running"
    service.fail.assert_not_awaited()
    signal_controller.assert_not_awaited()
    terminate_generation.assert_not_awaited()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("control", "expected_error"),
    [
        (_control("starting", 1), "workflow_start_pending"),
        (
            _control(
                "failed",
                2,
                terminal_reason="deployment_setup_failed",
            ),
            "deployment_setup_failed",
        ),
    ],
)
async def test_duplicate_start_reports_non_successful_durable_outcome(
    monkeypatch,
    control,
    expected_error,
):
    service = SimpleNamespace(
        database=SimpleNamespace(
            get_workflow_control_by_idempotency_key=AsyncMock(
                return_value=control,
            ),
        ),
    )
    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_reconcile_control",
        AsyncMock(return_value=(control, None)),
    )
    monkeypatch.setattr(
        handlers,
        "_control_payload",
        AsyncMock(
            return_value={
                "state": control.status,
                "revision": control.revision,
            }
        ),
    )

    result = await handlers.handle_start_workflow(
        {
            "workflow_id": "wf",
            "idempotency_key": "start-request-1",
        },
        None,
    )

    assert result == {
        "success": False,
        "error": expected_error,
        "idempotent": True,
        "state": control.status,
        "revision": control.revision,
    }


@pytest.mark.asyncio
async def test_concurrent_duplicate_start_reports_pending_winner(monkeypatch):
    starting = _control("starting", 1)
    service = SimpleNamespace(
        database=SimpleNamespace(
            get_workflow_control_by_idempotency_key=AsyncMock(
                return_value=None,
            ),
            get_latest_workflow_control=AsyncMock(return_value=None),
        ),
        begin_generation=AsyncMock(return_value=(starting, False)),
    )
    start_controller = AsyncMock()
    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_reconcile_control",
        AsyncMock(return_value=(starting, None)),
    )
    monkeypatch.setattr(
        handlers,
        "_control_payload",
        AsyncMock(return_value={"state": "starting", "revision": 1}),
    )
    monkeypatch.setattr(handlers, "_start_controller", start_controller)

    result = await handlers.handle_start_workflow(
        {
            "workflow_id": "wf",
            "expected_revision": 0,
            "idempotency_key": "start-request-1",
        },
        None,
    )

    assert result["success"] is False
    assert result["error"] == "workflow_start_pending"
    assert result["idempotent"] is True
    start_controller.assert_not_awaited()


@pytest.mark.asyncio
async def test_controller_state_change_waits_for_acknowledged_update(monkeypatch):
    handle = SimpleNamespace(
        execute_update=AsyncMock(return_value={
            "state": "paused",
            "revision": 3,
            "queued_events": 2,
        }),
    )
    monkeypatch.setattr(handlers, "_controller_handle", lambda _control: handle)

    result = await handlers._update_controller_state(
        _control("pausing", 2),
        "paused",
        update_id="request-1:paused",
    )

    assert result["state"] == "paused"
    handle.execute_update.assert_awaited_once_with(
        "set_control_state",
        "paused",
        id="request-1:paused",
    )


@pytest.mark.asyncio
async def test_reconcile_completes_interrupted_pause_from_temporal(monkeypatch):
    from core import container as container_module

    pausing = _control("pausing", 3)
    paused = _control("paused", 4)
    service = SimpleNamespace(
        transition=AsyncMock(return_value=paused),
        database=SimpleNamespace(get_latest_workflow_control=AsyncMock()),
    )
    update = AsyncMock(return_value={
        "state": "paused",
        "revision": 1,
        "queued_events": 0,
    })
    broadcast = AsyncMock(return_value={})
    workflow_service = SimpleNamespace(
        pause_deployment=MagicMock(),
        update_trigger_pause_status=AsyncMock(return_value=1),
    )
    monkeypatch.setattr(
        handlers,
        "_query_controller_state",
        AsyncMock(return_value={"state": "running", "revision": 0}),
    )
    monkeypatch.setattr(handlers, "_update_controller_state", update)
    monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
    monkeypatch.setattr(handlers, "_set_cron_pause", AsyncMock(return_value=1))
    monkeypatch.setattr(
        handlers,
        "_signal_generation_workflows",
        AsyncMock(return_value=2),
    )
    monkeypatch.setattr(
        container_module.container,
        "workflow_service",
        lambda: workflow_service,
    )

    reconciled, temporal_status = await handlers._reconcile_control(
        service,
        pausing,
    )

    assert reconciled is paused
    assert temporal_status["state"] == "paused"
    update.assert_awaited_once_with(
        pausing,
        "paused",
        update_id="reconcile:workflow-control:wf:1:3:paused",
    )
    service.transition.assert_awaited_once_with(
        pausing,
        expected_revision=3,
        from_statuses={"pausing"},
        status="paused",
    )
    broadcast.assert_awaited_once_with(
        paused,
        controller_status=temporal_status,
        extra={
            "signalled_executions": 2,
            "paused_schedules": 1,
            "paused_triggers": 1,
        },
    )


@pytest.mark.asyncio
async def test_pause_response_is_final_only_after_temporal_ack(monkeypatch):
    from core import container as container_module

    running = _control("running", 2)
    pausing = _control("pausing", 3)
    paused = _control("paused", 4)
    order: list[str] = []
    update_calls: list[tuple[object, str, str]] = []

    async def transition(control, **kwargs):
        order.append(f"db:{kwargs['status']}")
        return pausing if kwargs["status"] == "pausing" else paused

    service = SimpleNamespace(
        database=SimpleNamespace(
            get_latest_workflow_control=AsyncMock(return_value=running),
        ),
        transition=AsyncMock(side_effect=transition),
    )
    workflow_service = SimpleNamespace(
        pause_deployment=MagicMock(side_effect=lambda _workflow_id: order.append("local:pause")),
        resume_deployment=AsyncMock(),
        update_trigger_pause_status=AsyncMock(return_value=1),
        get_deployment_status=MagicMock(return_value={
            "active_runs": 0,
            "queued_events": 0,
        }),
    )

    async def update_controller(control, requested_state, *, update_id):
        order.append("temporal:ack")
        update_calls.append((control, requested_state, update_id))
        return {"state": "paused", "revision": 1, "queued_events": 0}

    async def broadcast(control, **_kwargs):
        order.append(f"broadcast:{control.status}")
        return {
            "state": control.status,
            "revision": control.revision,
            "can_pause": control.status == "running",
            "can_resume": control.status == "paused",
        }

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_reconcile_control",
        AsyncMock(return_value=(running, {"state": "running", "revision": 0})),
    )
    monkeypatch.setattr(handlers, "_update_controller_state", update_controller)
    monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
    monkeypatch.setattr(handlers, "_set_cron_pause", AsyncMock(return_value=1))
    monkeypatch.setattr(
        handlers,
        "_signal_generation_workflows",
        AsyncMock(return_value=2),
    )
    monkeypatch.setattr(
        container_module.container,
        "workflow_service",
        lambda: workflow_service,
    )

    result = await handlers.handle_pause_workflow(
        {
            "workflow_id": "wf",
            "expected_revision": 2,
            "idempotency_key": "request-1",
        },
        None,
    )

    assert result["success"] is True
    assert result["state"] == "paused"
    assert order == [
        "db:pausing",
        "broadcast:pausing",
        "local:pause",
        "temporal:ack",
        "db:paused",
        "broadcast:paused",
    ]
    assert update_calls == [
        (
            pausing,
            "paused",
            "workflow-control:wf:1:3:request-1:paused",
        )
    ]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    (
        "operation",
        "initial_status",
        "transitional_status",
        "requested_state",
        "signal_name",
        "cron_paused",
    ),
    [
        ("pause", "running", "pausing", "paused", "pause", True),
        ("resume", "paused", "resuming", "running", "resume", False),
    ],
)
async def test_stable_state_is_not_published_when_strict_fanout_fails(
    monkeypatch,
    operation,
    initial_status,
    transitional_status,
    requested_state,
    signal_name,
    cron_paused,
):
    from core import container as container_module

    initial = _control(initial_status, 2)
    transitional = _control(transitional_status, 3)
    service = SimpleNamespace(
        database=SimpleNamespace(
            get_latest_workflow_control=AsyncMock(return_value=initial),
        ),
        transition=AsyncMock(return_value=transitional),
    )
    workflow_service = SimpleNamespace(
        pause_deployment=MagicMock(),
        resume_deployment=AsyncMock(return_value=0),
        update_trigger_pause_status=AsyncMock(return_value=1),
    )
    set_cron_pause = AsyncMock(return_value=1)
    fanout = AsyncMock(
        side_effect=RuntimeError("workflow_signal_failed:1"),
    )

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_reconcile_control",
        AsyncMock(
            return_value=(
                initial,
                {"state": initial_status, "revision": 0},
            )
        ),
    )
    monkeypatch.setattr(
        handlers,
        "_update_controller_state",
        AsyncMock(
            return_value={
                "state": requested_state,
                "revision": 1,
                "queued_events": 0,
            }
        ),
    )
    monkeypatch.setattr(
        handlers,
        "_broadcast_control",
        AsyncMock(return_value={}),
    )
    monkeypatch.setattr(handlers, "_set_cron_pause", set_cron_pause)
    monkeypatch.setattr(
        handlers,
        "_signal_generation_workflows",
        fanout,
    )
    monkeypatch.setattr(
        container_module.container,
        "workflow_service",
        lambda: workflow_service,
    )

    handler = (
        handlers.handle_pause_workflow
        if operation == "pause"
        else handlers.handle_resume_workflow
    )
    result = await handler(
        {
            "workflow_id": "wf",
            "expected_revision": 2,
            "idempotency_key": "request-1",
        },
        None,
    )

    assert result == {
        "success": False,
        "error": "workflow_signal_failed:1",
    }
    service.transition.assert_awaited_once_with(
        initial,
        expected_revision=2,
        from_statuses={initial_status},
        status=transitional_status,
    )
    set_cron_pause.assert_awaited_once_with(
        "wf",
        paused=cron_paused,
        strict=True,
    )
    fanout.assert_awaited_once_with(
        transitional,
        signal_name,
        strict=True,
    )


@pytest.mark.asyncio
async def test_pause_temporal_failure_restores_running_projection(monkeypatch):
    from core import container as container_module

    running = _control("running", 2)
    pausing = _control("pausing", 3)
    service = SimpleNamespace(
        database=SimpleNamespace(
            get_latest_workflow_control=AsyncMock(return_value=running),
        ),
        transition=AsyncMock(return_value=pausing),
    )
    workflow_service = SimpleNamespace(
        pause_deployment=MagicMock(),
        resume_deployment=AsyncMock(return_value=0),
    )
    restore = AsyncMock(return_value=running)

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_reconcile_control",
        AsyncMock(return_value=(running, {"state": "running", "revision": 0})),
    )
    monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))
    monkeypatch.setattr(
        handlers,
        "_update_controller_state",
        AsyncMock(
            side_effect=handlers.TemporalControlUnavailable(
                "temporal_control_unavailable"
            )
        ),
    )
    monkeypatch.setattr(
        handlers,
        "_restore_control_after_failed_update",
        restore,
    )
    monkeypatch.setattr(
        container_module.container,
        "workflow_service",
        lambda: workflow_service,
    )

    result = await handlers.handle_pause_workflow(
        {"workflow_id": "wf", "expected_revision": 2},
        None,
    )

    assert result == {
        "success": False,
        "error": "temporal_control_unavailable",
    }
    workflow_service.resume_deployment.assert_awaited_once_with("wf")
    restore.assert_awaited_once_with(
        service,
        pausing,
        transitional_state="pausing",
        stable_state="running",
    )


@pytest.mark.asyncio
async def test_unknown_update_outcome_stays_transitional_for_reconciliation(
    monkeypatch,
):
    from core import container as container_module

    running = _control("running", 2)
    pausing = _control("pausing", 3)
    service = SimpleNamespace(
        database=SimpleNamespace(
            get_latest_workflow_control=AsyncMock(return_value=running),
        ),
        transition=AsyncMock(return_value=pausing),
    )
    workflow_service = SimpleNamespace(
        pause_deployment=MagicMock(),
        resume_deployment=AsyncMock(return_value=0),
    )
    restore = AsyncMock()

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_reconcile_control",
        AsyncMock(return_value=(running, {"state": "running", "revision": 0})),
    )
    monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))
    monkeypatch.setattr(
        handlers,
        "_update_controller_state",
        AsyncMock(side_effect=TimeoutError("update outcome unknown")),
    )
    monkeypatch.setattr(
        handlers,
        "_restore_control_after_failed_update",
        restore,
    )
    monkeypatch.setattr(
        container_module.container,
        "workflow_service",
        lambda: workflow_service,
    )

    result = await handlers.handle_pause_workflow(
        {"workflow_id": "wf", "expected_revision": 2},
        None,
    )

    assert result == {"success": False, "error": "update outcome unknown"}
    workflow_service.resume_deployment.assert_not_awaited()
    restore.assert_not_awaited()


@pytest.mark.asyncio
async def test_runtime_counts_sum_controller_and_local_queues(monkeypatch):
    from core import container as container_module

    workflow_service = SimpleNamespace(
        get_deployment_status=MagicMock(
            return_value={
                "active_runs": 4,
                "queued_events": 3,
            }
        ),
    )
    monkeypatch.setattr(
        container_module.container,
        "workflow_service",
        lambda: workflow_service,
    )

    result = await handlers._with_runtime_counts(
        {"queued_count": 2},
        "wf",
    )

    assert result["active_count"] == 4
    assert result["in_flight_count"] == 4
    assert result["queued_count"] == 5


@pytest.mark.asyncio
async def test_reset_cleanup_failure_leaves_control_resetting(monkeypatch):
    running = _control("running", 2)
    resetting = _control("resetting", 3)
    cleanup_order: list[str] = []
    database = SimpleNamespace(
        get_latest_workflow_control=AsyncMock(return_value=running),
        update_workflow_run_data_scope=AsyncMock(return_value=True),
    )
    service = SimpleNamespace(
        database=database,
        transition=AsyncMock(return_value=resetting),
    )

    async def delete_schedules_impl(*_args, **_kwargs):
        cleanup_order.append("delete_schedules")
        return 0

    async def cancel_deployment_impl(*_args, **_kwargs):
        cleanup_order.append("cancel_local")
        return {"success": True}

    async def terminate_impl(*_args, **_kwargs):
        cleanup_order.append("terminate")
        raise RuntimeError("workflow_termination_failed:1")

    def close_local_admission(_workflow_id):
        cleanup_order.append("close_local_admission")

    async def signal_controller(*_args, **_kwargs):
        cleanup_order.append("close_controller")

    delete_schedules = AsyncMock(side_effect=delete_schedules_impl)
    cancel_deployment = AsyncMock(side_effect=cancel_deployment_impl)
    terminate = AsyncMock(side_effect=terminate_impl)

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_broadcast_control",
        AsyncMock(return_value={}),
    )
    monkeypatch.setattr(
        handlers,
        "_close_local_admission",
        close_local_admission,
    )
    monkeypatch.setattr(
        handlers,
        "_signal_controller",
        AsyncMock(side_effect=signal_controller),
    )
    monkeypatch.setattr(
        handlers,
        "_terminate_generation_workflows",
        terminate,
    )
    monkeypatch.setattr(
        handlers,
        "_delete_cron_schedules",
        delete_schedules,
    )
    monkeypatch.setattr(
        handlers,
        "handle_cancel_deployment",
        cancel_deployment,
    )

    result = await handlers.handle_reset_workflow(
        {"workflow_id": "wf", "expected_revision": 2},
        None,
    )

    assert result == {
        "success": False,
        "error": "workflow_termination_failed:1",
    }
    assert service.transition.await_count == 1
    first_transition = service.transition.await_args
    assert first_transition.args == (running,)
    assert first_transition.kwargs["expected_revision"] == 2
    assert first_transition.kwargs["status"] == "resetting"
    assert cleanup_order == [
        "close_local_admission",
        "close_controller",
        "delete_schedules",
        "cancel_local",
        "terminate",
    ]
    delete_schedules.assert_awaited_once_with("wf", strict=True)
    cancel_deployment.assert_awaited_once_with(
        {"workflow_id": "wf"},
        None,
    )
    terminate.assert_awaited_once_with(resetting, strict=True)
    database.update_workflow_run_data_scope.assert_not_awaited()


@pytest.mark.asyncio
async def test_reset_local_cleanup_failure_leaves_control_resetting(
    monkeypatch,
):
    resetting = _control("resetting", 3)
    database = SimpleNamespace(
        get_latest_workflow_control=AsyncMock(return_value=resetting),
        update_workflow_run_data_scope=AsyncMock(return_value=True),
    )
    service = SimpleNamespace(
        database=database,
        transition=AsyncMock(),
    )

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_broadcast_control",
        AsyncMock(return_value={}),
    )
    monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())
    monkeypatch.setattr(
        handlers,
        "_terminate_generation_workflows",
        AsyncMock(return_value=2),
    )
    monkeypatch.setattr(
        handlers,
        "_delete_cron_schedules",
        AsyncMock(return_value=1),
    )
    monkeypatch.setattr(
        handlers,
        "handle_cancel_deployment",
        AsyncMock(
            return_value={
                "success": False,
                "error": "listener teardown failed",
            }
        ),
    )

    result = await handlers.handle_reset_workflow(
        {"workflow_id": "wf"},
        None,
    )

    assert result == {
        "success": False,
        "error": (
            "workflow_local_cleanup_failed:listener teardown failed"
        ),
    }
    service.transition.assert_not_awaited()
    database.update_workflow_run_data_scope.assert_not_awaited()


@pytest.mark.asyncio
async def test_already_reset_is_idempotent_without_repeating_cleanup(
    monkeypatch,
):
    reset = _control("reset", 4)
    database = SimpleNamespace(
        get_latest_workflow_control=AsyncMock(return_value=reset),
        update_workflow_run_data_scope=AsyncMock(),
    )
    service = SimpleNamespace(
        database=database,
        transition=AsyncMock(),
    )
    close_local_admission = MagicMock()
    signal_controller = AsyncMock()
    delete_schedules = AsyncMock()
    cancel_deployment = AsyncMock()
    terminate = AsyncMock()
    control_payload = AsyncMock(
        return_value={
            "state": "ready",
            "revision": 4,
            "generation": 1,
        }
    )

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_close_local_admission",
        close_local_admission,
    )
    monkeypatch.setattr(handlers, "_signal_controller", signal_controller)
    monkeypatch.setattr(
        handlers,
        "_delete_cron_schedules",
        delete_schedules,
    )
    monkeypatch.setattr(
        handlers,
        "handle_cancel_deployment",
        cancel_deployment,
    )
    monkeypatch.setattr(
        handlers,
        "_terminate_generation_workflows",
        terminate,
    )
    monkeypatch.setattr(handlers, "_control_payload", control_payload)

    result = await handlers.handle_reset_workflow(
        {
            "workflow_id": "wf",
            "expected_revision": 4,
            "idempotency_key": "reset-request-1",
        },
        None,
    )

    assert result == {
        "success": True,
        "idempotent": True,
        "state": "ready",
        "revision": 4,
        "generation": 1,
    }
    control_payload.assert_awaited_once_with(reset)
    service.transition.assert_not_awaited()
    close_local_admission.assert_not_called()
    signal_controller.assert_not_awaited()
    delete_schedules.assert_not_awaited()
    cancel_deployment.assert_not_awaited()
    terminate.assert_not_awaited()
    database.update_workflow_run_data_scope.assert_not_awaited()


@pytest.mark.asyncio
async def test_reset_retry_resumes_cleanup_before_marking_ready(monkeypatch):
    from services.deployment import runtime_state
    from services import status_broadcaster

    resetting = _control("resetting", 3)
    reset = _control("reset", 4)
    database = SimpleNamespace(
        get_latest_workflow_control=AsyncMock(return_value=resetting),
        update_workflow_run_data_scope=AsyncMock(return_value=True),
    )
    service = SimpleNamespace(
        database=database,
        transition=AsyncMock(return_value=reset),
    )
    terminate = AsyncMock(return_value=2)
    delete_schedules = AsyncMock(return_value=3)
    cancel_deployment = AsyncMock(
        return_value={
            "success": False,
            "message": "Workflow wf is not deployed",
        }
    )
    archive_nodes = AsyncMock(
        return_value={
            "archived_nodes": 2,
            "reset_nodes": ["agent-1"],
        }
    )
    broadcaster = SimpleNamespace(broadcast=AsyncMock())

    async def broadcast_control(control, **_kwargs):
        return {
            "state": "ready" if control.status == "reset" else control.status,
            "revision": control.revision,
        }

    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(
        handlers,
        "_broadcast_control",
        AsyncMock(side_effect=broadcast_control),
    )
    monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())
    monkeypatch.setattr(
        handlers,
        "_terminate_generation_workflows",
        terminate,
    )
    # The calls the reset turns waited on end cancelled (RFC-0007 §4.6).
    cancel_waits = AsyncMock(return_value=0)
    monkeypatch.setattr("services.approvals.waiting.cancel_waits", cancel_waits)
    # The generation's open team work and requests are closed (1b.21).
    close_open_work = AsyncMock(return_value=None)
    monkeypatch.setattr("services.employees.safe_apply.close_open_work", close_open_work)
    monkeypatch.setattr(
        handlers,
        "_delete_cron_schedules",
        delete_schedules,
    )
    monkeypatch.setattr(
        handlers,
        "handle_cancel_deployment",
        cancel_deployment,
    )
    monkeypatch.setattr(
        runtime_state,
        "archive_and_reset_node_state",
        archive_nodes,
    )
    monkeypatch.setattr(
        status_broadcaster,
        "get_status_broadcaster",
        lambda: broadcaster,
    )

    result = await handlers.handle_reset_workflow(
        {"workflow_id": "wf"},
        None,
    )

    assert result["success"] is True
    assert result["state"] == "ready"
    assert result["revision"] == 4
    cancel_waits.assert_awaited_once()
    assert cancel_waits.await_args.args[1:] == ("wf", resetting.generation)
    close_open_work.assert_awaited_once()
    assert close_open_work.await_args.args[1:] == ("wf",)
    assert close_open_work.await_args.kwargs["reason"] == "The workflow was reset"
    assert "queued" in close_open_work.await_args.kwargs["job_states"]
    assert result["terminated_executions"] == 2
    assert result["deleted_schedules"] == 3
    assert result["local_cleanup_completed"] is False
    terminate.assert_awaited_once_with(resetting, strict=True)
    delete_schedules.assert_awaited_once_with("wf", strict=True)
    cancel_deployment.assert_awaited_once_with(
        {"workflow_id": "wf"},
        None,
    )
    database.update_workflow_run_data_scope.assert_awaited_once()
    archive_nodes.assert_awaited_once_with(
        resetting,
        database,
        broadcaster,
    )
    service.transition.assert_awaited_once()
    final_transition = service.transition.await_args
    assert final_transition.args == (resetting,)
    assert final_transition.kwargs["expected_revision"] == 3
    assert final_transition.kwargs["from_statuses"] == {"resetting"}
    assert final_transition.kwargs["status"] == "reset"


def test_generation_fanout_uses_temporal_root_tree():
    query = handlers._generation_visibility_query(_control("running", 2))

    assert query == (
        "(RootWorkflowId='workflow-control-wf-g1' OR "
        "EventWorkflowId='wf') "
        "AND ExecutionStatus='Running'"
    )


@pytest.mark.asyncio
async def test_generation_fanout_expands_tagged_standalone_root_trees():
    queries: list[str] = []
    controller = SimpleNamespace(
        id="workflow-control-wf-g1",
        run_id="controller-run",
        root_id="workflow-control-wf-g1",
    )
    cron_graph = SimpleNamespace(
        id="cron-graph",
        run_id="graph-run",
        root_id="cron-firing-root",
    )
    cron_agent = SimpleNamespace(
        id="cron-agent",
        run_id="agent-run",
        root_id="cron-firing-root",
    )

    def list_workflows(*, query):
        queries.append(query)

        async def iterate():
            values = (
                [controller, cron_graph]
                if query.startswith("(RootWorkflowId=")
                else [controller, cron_graph, cron_agent]
            )
            for value in values:
                yield value

        return iterate()

    client = SimpleNamespace(list_workflows=list_workflows)

    executions = await handlers._list_generation_workflows(
        client,
        _control("running", 2),
    )

    assert {
        execution.id
        for execution in executions
    } == {
        "workflow-control-wf-g1",
        "cron-graph",
        "cron-agent",
    }
    assert queries[0] == (
        "(RootWorkflowId='workflow-control-wf-g1' OR "
        "EventWorkflowId='wf') "
        "AND ExecutionStatus='Running'"
    )
    assert queries[1] == (
        "RootWorkflowId IN "
        "('cron-firing-root', 'workflow-control-wf-g1') "
        "AND ExecutionStatus='Running'"
    )


# ============================================================================
# A controller execution that no longer exists
#
# Temporal deletes closed executions once the namespace retention window
# passes (24h on the dev server). A generation's controller closes when that
# generation is reset, so every control row eventually names an execution
# Temporal has forgotten. Two things must hold: terminal generations must not
# be probed at all, and a *live* generation whose controller vanished must
# converge instead of waiting for something that will never answer.
# ============================================================================


class _Gone(Exception):
    """Shaped like the Temporal error, matched by _temporal_target_already_gone."""

    def __str__(self) -> str:
        return (
            'workflow execution not found for workflow ID '
            '"workflow-control-wf-g1" and run ID "run-1"'
        )


def _reconcile_service(**overrides):
    database = SimpleNamespace(**overrides.pop("database", {}))
    return SimpleNamespace(database=database, **overrides)


@pytest.mark.asyncio
@pytest.mark.parametrize("terminal_status", ["reset", "failed"])
async def test_terminal_generation_is_never_probed(monkeypatch, terminal_status):
    """The reported symptom: a doomed RPC on every status read, forever."""
    probe = AsyncMock(return_value={"state": "running"})
    monkeypatch.setattr(handlers, "_query_controller_state", probe)

    control = _control(terminal_status, 6)
    result, controller_status = await handlers._reconcile_control(
        _reconcile_service(), control
    )

    probe.assert_not_awaited()
    assert result is control
    assert controller_status is None


@pytest.mark.asyncio
async def test_live_generation_with_missing_controller_pauses_by_default(monkeypatch):
    """WORKFLOW_CONTROL_MISSING_CONTROLLER=pause (default): a killed
    controller leaves the generation user-resumable — Resume rebuilds it —
    instead of forcing a Reset that archives conversation state."""
    paused = _control("paused", 7)
    transition = AsyncMock(return_value=paused)
    fail = AsyncMock()
    broadcast = AsyncMock(return_value={})

    monkeypatch.setattr(handlers, "_missing_controller_policy", lambda: "pause")
    monkeypatch.setattr(
        handlers,
        "_query_controller_state",
        AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),
    )
    monkeypatch.setattr(handlers, "_broadcast_control", broadcast)

    control = _control("pausing", 6)
    result, controller_status = await handlers._reconcile_control(
        _reconcile_service(fail=fail, transition=transition), control
    )

    assert result is paused
    assert controller_status is None
    assert transition.await_args.kwargs["status"] == "paused"
    fail.assert_not_awaited()
    broadcast.assert_awaited_once()


@pytest.mark.asyncio
async def test_already_paused_generation_with_missing_controller_is_left_alone(
    monkeypatch,
):
    """Paused is already the recoverable posture; Resume performs the rebuild."""
    transition = AsyncMock()
    fail = AsyncMock()
    monkeypatch.setattr(handlers, "_missing_controller_policy", lambda: "pause")
    monkeypatch.setattr(
        handlers,
        "_query_controller_state",
        AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),
    )

    control = _control("paused", 6)
    result, _ = await handlers._reconcile_control(
        _reconcile_service(fail=fail, transition=transition), control
    )

    assert result is control
    transition.assert_not_awaited()
    fail.assert_not_awaited()


@pytest.mark.asyncio
async def test_live_generation_with_missing_controller_fails_under_legacy_policy(
    monkeypatch,
):
    """WORKFLOW_CONTROL_MISSING_CONTROLLER=fail preserves the Reset-only
    behaviour; otherwise the row stays non-'reset' forever and blocks Start."""
    failed = _control("failed", 7, terminal_reason="controller_execution_missing")
    fail = AsyncMock(return_value=failed)
    broadcast = AsyncMock(return_value={})

    monkeypatch.setattr(handlers, "_missing_controller_policy", lambda: "fail")
    monkeypatch.setattr(
        handlers,
        "_query_controller_state",
        AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),
    )
    monkeypatch.setattr(handlers, "_broadcast_control", broadcast)

    control = _control("pausing", 6)
    result, controller_status = await handlers._reconcile_control(
        _reconcile_service(fail=fail), control
    )

    assert result is failed
    assert controller_status is None
    assert fail.await_args.args[1] == "controller_execution_missing"
    broadcast.assert_awaited_once()


@pytest.mark.asyncio
async def test_starting_with_missing_controller_still_fails_under_pause_policy(
    monkeypatch,
):
    """Nothing durable runs yet mid-start; Reset + Start rebuilds cleanly."""
    failed = _control("failed", 7, terminal_reason="controller_execution_missing")
    fail = AsyncMock(return_value=failed)
    monkeypatch.setattr(handlers, "_missing_controller_policy", lambda: "pause")
    monkeypatch.setattr(
        handlers,
        "_query_controller_state",
        AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),
    )
    monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))

    control = _control("starting", 6)
    result, _ = await handlers._reconcile_control(
        _reconcile_service(fail=fail), control
    )

    assert result is failed
    assert fail.await_args.args[1] == "controller_execution_missing"


@pytest.mark.asyncio
async def test_resetting_is_not_auto_failed(monkeypatch):
    """Reset owns its own retry; auto-failing here could race a live reset."""
    fail = AsyncMock()
    monkeypatch.setattr(
        handlers,
        "_query_controller_state",
        AsyncMock(side_effect=handlers.ControllerExecutionMissing("gone")),
    )

    control = _control("resetting", 5)
    result, _ = await handlers._reconcile_control(
        _reconcile_service(fail=fail), control
    )

    fail.assert_not_awaited()
    assert result is control


@pytest.mark.asyncio
async def test_a_transient_query_failure_does_not_fail_the_generation(monkeypatch):
    """'Unreachable' must stay recoverable -- only 'gone' is terminal."""
    fail = AsyncMock()
    monkeypatch.setattr(
        handlers, "_query_controller_state", AsyncMock(return_value=None)
    )

    control = _control("running", 4)
    result, controller_status = await handlers._reconcile_control(
        _reconcile_service(fail=fail), control
    )

    fail.assert_not_awaited()
    assert result is control
    assert controller_status is None


@pytest.mark.asyncio
async def test_missing_execution_is_classified_as_gone_not_as_an_outage():
    """Locks the string match against the real Temporal error text."""
    assert handlers._temporal_target_already_gone(_Gone()) is True
    assert handlers._temporal_target_already_gone(RuntimeError("deadline")) is False


@pytest.mark.asyncio
async def test_query_raises_missing_rather_than_returning_none(monkeypatch):
    """The distinction the whole fix rests on."""
    handle = SimpleNamespace(query=AsyncMock(side_effect=_Gone()))
    monkeypatch.setattr(handlers, "_controller_handle", lambda _c: handle)

    with pytest.raises(handlers.ControllerExecutionMissing):
        await handlers._query_controller_state(_control("running", 1))


@pytest.mark.asyncio
async def test_query_still_returns_none_for_a_real_outage(monkeypatch):
    handle = SimpleNamespace(query=AsyncMock(side_effect=RuntimeError("deadline")))
    monkeypatch.setattr(handlers, "_controller_handle", lambda _c: handle)

    assert await handlers._query_controller_state(_control("running", 1)) is None


@pytest.mark.asyncio
async def test_start_admits_the_employee_team_plan_on_its_lead(monkeypatch):
    """The saved graph keeps node data to UI fields; Start copies the plan in (#148)."""
    plan = {"version": 2, "lead_node_id": "wf:ai_employee:1", "talk_node_id": "wf:aiAgent:1"}
    running = _control("running", 2)

    async def transition(_value, **kwargs):
        return _control("starting", 1) if kwargs["status"] == "starting" else running

    async def broadcast(control, **_kwargs):
        return {"state": control.status, "revision": control.revision}

    service = SimpleNamespace(
        database=SimpleNamespace(
            get_workflow_control_by_idempotency_key=AsyncMock(return_value=None),
            get_latest_workflow_control=AsyncMock(return_value=None),
            update_workflow_run_data_scope=AsyncMock(return_value=True),
        ),
        begin_generation=AsyncMock(return_value=(_control("starting", 0), True)),
        transition=AsyncMock(side_effect=transition),
    )
    monkeypatch.setattr(handlers, "_control_service", lambda: service)
    monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
    monkeypatch.setattr(handlers, "_start_controller", AsyncMock(return_value="controller-run-1"))
    monkeypatch.setattr(handlers, "handle_deploy_workflow", AsyncMock(return_value={"success": True}))
    monkeypatch.setattr(handlers, "_await_deployment_setup", AsyncMock(return_value={"success": True}))
    monkeypatch.setattr(handlers, "_query_controller_state", AsyncMock(return_value={"state": "running", "revision": 0}))
    monkeypatch.setattr("services.workflow_validator.validate_workflow", AsyncMock(return_value={"errors": []}))
    monkeypatch.setattr("services.employees.store.get_by_workflow", AsyncMock(return_value=SimpleNamespace(team_plan=plan)))

    result = await handlers.handle_start_workflow(
        {
            "workflow_id": "wf",
            "nodes": [
                # A client cannot put a plan on a node; Start drops it and admits the employee's.
                {"id": "wf:ai_employee:1", "type": "ai_employee", "data": {"label": "Lead", "employee_team_plan": {"version": 2}}},
                {"id": "wf:aiAgent:1", "type": "aiAgent", "data": {"label": "Talk"}},
            ],
            "edges": [],
            "parameters_by_id": {},
            "expected_revision": 0,
            "idempotency_key": "start-employee",
        },
        None,
    )

    assert result["success"] is True
    admitted = {node["id"]: node["data"] for node in service.begin_generation.await_args.kwargs["nodes"]}
    assert admitted["wf:ai_employee:1"] == {"label": "Lead", "employee_recipe_version": 2, "employee_team_plan": plan}
    assert admitted["wf:aiAgent:1"] == {"label": "Talk"}


@pytest.mark.asyncio
async def test_deploy_admits_the_employee_team_plan_on_its_lead(monkeypatch):
    """Start and the restart re-arm deploy through here, which sanitizes the graph again (#148)."""
    plan = {"version": 2, "lead_node_id": "wf:ai_employee:1", "talk_node_id": "wf:aiAgent:1"}
    workflow_service = SimpleNamespace(
        is_workflow_deployed=lambda _workflow_id: False,
        deploy_workflow=AsyncMock(return_value={"success": True, "triggers_setup": [], "deployment_id": "deployment"}),
    )
    broadcaster = SimpleNamespace(
        lock_workflow=AsyncMock(return_value=True), unlock_workflow=AsyncMock(), update_node_status=AsyncMock(),
        update_workflow_status=AsyncMock(), update_deployment_status=AsyncMock(),
    )
    monkeypatch.setattr("core.container.container", SimpleNamespace(workflow_service=lambda: workflow_service,
                                                                    database=lambda: SimpleNamespace()))
    monkeypatch.setattr("services.status_broadcaster.get_status_broadcaster", lambda: broadcaster)
    monkeypatch.setattr("services.workflow_validator.validate_workflow", AsyncMock(return_value={"errors": []}))
    monkeypatch.setattr("services.employees.store.get_by_workflow", AsyncMock(return_value=SimpleNamespace(team_plan=plan)))
    monkeypatch.setattr(handlers, "_deployment_tasks", {})

    result = await handlers.handle_deploy_workflow(
        {
            "workflow_id": "wf",
            "nodes": [
                # A plan the caller sends is dropped; the employee's is deployed.
                {"id": "wf:ai_employee:1", "type": "ai_employee",
                 "data": {"label": "Lead", "employee_recipe_version": 2, "employee_team_plan": {"version": 2, "lead_node_id": "forged"}}},
                {"id": "wf:aiAgent:1", "type": "aiAgent", "data": {"label": "Talk"}},
            ],
            "edges": [],
            "parameters_by_id": {},
            "generation": 1,
        },
        None,
    )
    await handlers._deployment_tasks["wf"]

    assert result["success"] is True
    deployed = {node["id"]: node["data"] for node in workflow_service.deploy_workflow.await_args.kwargs["nodes"]}
    assert deployed["wf:ai_employee:1"] == {"label": "Lead", "employee_recipe_version": 2, "employee_team_plan": plan}
    assert deployed["wf:aiAgent:1"] == {"label": "Talk"}