Skip to content

Automate browser based workflows with AI

29.4 KB
Raw Blame
"""Tests for cancellation / webhook-tolerance helpers on WorkflowService.

- ``mark_workflow_run_as_canceled_if_not_final`` delegates to the conditional
  DB update and is a no-op when the DB reports no row was affected - either
  because the row was already in a terminal state or because no row with that
  id exists.
- ``mark_workflow_run_as_canceled`` rejects transitions to ``canceled`` when
  the run has already reached a terminal state (SKY-9188).
- ``execute_workflow_webhook`` returns cleanly when the workflow row has been
  soft-deleted mid-run — it must not raise ``WorkflowNotFound`` from the
  cleanup path.
- The cancellation-safe finalize pattern used in ``execute_workflow``'s outer
  ``finally`` runs ``_finalize_workflow_run_status`` via ``asyncio.shield``
  so an outer cancel mid-body still restores the real terminal status.
"""

from __future__ import annotations

import asyncio
from datetime import UTC, datetime, timedelta, tzinfo
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock

import pytest

from skyvern.exceptions import BrowserSessionClosed, BrowserSessionStartupTimeout, WorkflowNotFound, WorkflowRunNotFound
from skyvern.forge import app
from skyvern.forge.sdk.workflow import service as service_module
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.forge.sdk.workflow.retry_policy import TERMINAL_RELEASE_RETRY_MAX_ATTEMPTS
from skyvern.forge.sdk.workflow.service import (
    WorkflowBrowserCleanupResult,
    WorkflowService,
    _browser_lease_failure_category,
)
from skyvern.services import run_service


@pytest.mark.asyncio
async def test_mark_canceled_if_not_final_returns_conditional_result(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    updated_row = MagicMock()
    updated_row.status = WorkflowRunStatus.canceled

    delegate = AsyncMock(return_value=updated_row)
    monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run_if_not_final", delegate)

    svc = WorkflowService()
    result = await svc.mark_workflow_run_as_canceled_if_not_final(workflow_run_id="wr_1")

    assert result is updated_row
    delegate.assert_awaited_once_with(
        workflow_run_id="wr_1",
        status=WorkflowRunStatus.canceled,
        failure_reason=None,
    )


def _patch_service_log(monkeypatch: pytest.MonkeyPatch, **methods: object) -> None:
    """Replace the service module's LOG rather than a method on it. Undoing a patched method on
    structlog's lazy proxy leaves a bound logger stuck to the proxy, and later tests in the same
    process then miss that module's lines in structlog.testing.capture_logs."""
    log = MagicMock(wraps=service_module.LOG)
    for name, method in methods.items():
        setattr(log, name, method)
    monkeypatch.setattr(service_module, "LOG", log)


def _make_updated_row(now: datetime | None = None) -> MagicMock:
    now = now or datetime.now(UTC)
    row = MagicMock()
    row.status = WorkflowRunStatus.canceled
    row.created_at = now - timedelta(seconds=30)
    row.queued_at = None
    row.started_at = now - timedelta(seconds=20)
    row.workflow_id = "wf_abc"
    row.organization_id = "org_abc"
    row.run_with = None
    row.failure_category = None
    row.ai_fallback = False
    row.trigger_type = None
    row.workflow_schedule_id = None
    return row


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "final_status",
    [WorkflowRunStatus.failed, WorkflowRunStatus.terminated, WorkflowRunStatus.timed_out],
)
async def test_completion_tags_write_final_status_execution_mode_and_primary_failure_category(
    monkeypatch: pytest.MonkeyPatch,
    final_status: WorkflowRunStatus,
) -> None:
    workflow_run = _make_updated_row()
    workflow_run.workflow_run_id = f"wr_{final_status.value}"
    workflow_run.status = final_status
    workflow_run.run_with = "code"
    workflow_run.failure_category = [{"category": "PARAMETER_BINDING_ERROR"}]
    apply_tags = AsyncMock()
    monkeypatch.setattr(app.DATABASE.tags, "apply_system_run_tag_changes", apply_tags)

    await WorkflowService._apply_completion_run_tags_best_effort(workflow_run)

    apply_tags.assert_awaited_once_with(
        workflow_run_id=workflow_run.workflow_run_id,
        organization_id="org_abc",
        sets={
            "skyvern.status": final_status.value,
            "skyvern.execution_mode": "code",
            "skyvern.failure_category": "PARAMETER_BINDING_ERROR",
        },
        caller_id="system:completion-tagging",
    )


@pytest.mark.asyncio
async def test_completion_tags_write_status_and_execution_mode_for_completed_run(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow_run = _make_updated_row()
    workflow_run.workflow_run_id = "wr_completed"
    workflow_run.status = "completed"
    workflow_run.run_with = "agent"
    apply_tags = AsyncMock()
    monkeypatch.setattr(app.DATABASE.tags, "apply_system_run_tag_changes", apply_tags)

    await WorkflowService._apply_completion_run_tags_best_effort(workflow_run)

    apply_tags.assert_awaited_once_with(
        workflow_run_id="wr_completed",
        organization_id="org_abc",
        sets={"skyvern.status": "completed", "skyvern.execution_mode": "agent"},
        caller_id="system:completion-tagging",
    )


@pytest.mark.asyncio
async def test_completion_tag_write_failure_does_not_interrupt_finalization(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow_run = _make_updated_row()
    workflow_run.workflow_run_id = "wr_completion"
    monkeypatch.setattr(
        app.DATABASE.tags,
        "apply_system_run_tag_changes",
        AsyncMock(side_effect=RuntimeError("tag write failed")),
    )

    await WorkflowService._apply_completion_run_tags_best_effort(workflow_run)


@pytest.mark.asyncio
async def test_conditional_cancel_writes_completion_tags(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    updated_row = _make_updated_row()
    updated_row.workflow_run_id = "wr_canceled"
    monkeypatch.setattr(
        app.DATABASE.workflow_runs,
        "update_workflow_run_if_not_final",
        AsyncMock(return_value=updated_row),
    )
    completion_tags = AsyncMock()
    monkeypatch.setattr(WorkflowService, "_apply_completion_run_tags_best_effort", completion_tags)
    monkeypatch.setattr(WorkflowService, "_sync_task_run_from_workflow_run", AsyncMock())

    await WorkflowService().mark_workflow_run_as_canceled_if_not_final(workflow_run_id="wr_canceled")

    completion_tags.assert_awaited_once_with(updated_row)


@pytest.mark.asyncio
async def test_mark_canceled_if_not_final_logs_duration_metrics(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Terminal-transition parity with ``_update_workflow_run_status``: a
    successful conditional cancel must emit the ``Workflow run duration metrics``
    log with the same structured fields, so the metric stays comparable across
    statuses. ``recorded_seconds`` must equal the sample handed to
    ``record_run_duration`` so a log query keyed on it reconciles with the metric.
    """
    fixed_now = datetime(2026, 1, 1, tzinfo=UTC)
    updated_row = _make_updated_row(fixed_now)
    updated_row.workflow_run_id = "wr_live"
    updated_row.parent_workflow_run_id = None

    class FrozenDateTime(datetime):
        @classmethod
        def now(cls, tz: tzinfo | None = None) -> datetime:
            if tz is None:
                return fixed_now.replace(tzinfo=None)
            return fixed_now.astimezone(tz)

    monkeypatch.setattr(service_module, "datetime", FrozenDateTime)
    monkeypatch.setattr(
        app.DATABASE.workflow_runs,
        "update_workflow_run_if_not_final",
        AsyncMock(return_value=updated_row),
    )
    monkeypatch.setattr(
        WorkflowService,
        "_sync_task_run_from_workflow_run",
        AsyncMock(return_value=None),
    )
    recorder = AsyncMock()
    monkeypatch.setattr(app.AGENT_FUNCTION, "record_run_duration", recorder)

    info_calls: list[tuple[str, dict]] = []

    def fake_info(event: str, **kwargs: object) -> None:
        info_calls.append((event, dict(kwargs)))

    _patch_service_log(monkeypatch, info=fake_info)

    svc = WorkflowService()
    result = await svc.mark_workflow_run_as_canceled_if_not_final(workflow_run_id="wr_live")

    assert result is updated_row
    metrics_events = [kwargs for event, kwargs in info_calls if event == "Workflow run duration metrics"]
    assert len(metrics_events) == 1
    metrics = metrics_events[0]
    assert metrics["workflow_run_id"] == "wr_live"
    assert metrics["workflow_id"] == "wf_abc"
    assert metrics["organization_id"] == "org_abc"
    assert metrics["workflow_run_status"] == WorkflowRunStatus.canceled
    assert metrics["queued_seconds"] == pytest.approx(10.0, abs=1.0)
    assert metrics["duration_seconds"] == pytest.approx(20.0, abs=1.0)
    assert recorder.await_count == 1
    assert metrics["recorded_seconds"] == recorder.await_args.kwargs["duration_seconds"]


@pytest.mark.asyncio
async def test_mark_canceled_if_not_final_syncs_task_runs(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Terminal-transition parity: a successful conditional cancel must spawn
    the ``_sync_task_run_from_workflow_run`` write-through so downstream
    consumers reading ``task_runs`` see the cancel event.
    """
    updated_row = _make_updated_row()
    updated_row.workflow_run_id = "wr_sync"
    monkeypatch.setattr(
        app.DATABASE.workflow_runs,
        "update_workflow_run_if_not_final",
        AsyncMock(return_value=updated_row),
    )
    sync_mock = AsyncMock(return_value=None)
    monkeypatch.setattr(WorkflowService, "_sync_task_run_from_workflow_run", sync_mock)

    svc = WorkflowService()
    await svc.mark_workflow_run_as_canceled_if_not_final(workflow_run_id="wr_sync")

    # Let the fire-and-forget background task drain.
    for _ in range(3):
        await asyncio.sleep(0)

    sync_mock.assert_awaited_once()
    call_args = sync_mock.await_args
    assert call_args is not None
    assert call_args.args[0] is updated_row
    assert call_args.args[1] == "wr_sync"
    assert call_args.args[2] == WorkflowRunStatus.canceled


@pytest.mark.asyncio
async def test_mark_canceled_if_not_final_skips_side_effects_on_terminal_row(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """When the conditional update returns ``None`` (row already terminal),
    neither the duration-metrics log nor the task_runs sync may fire — the
    terminal status's own ``_update_workflow_run_status`` call already emitted
    them.
    """
    monkeypatch.setattr(
        app.DATABASE.workflow_runs,
        "update_workflow_run_if_not_final",
        AsyncMock(return_value=None),
    )
    sync_mock = AsyncMock(side_effect=AssertionError("_sync_task_run_from_workflow_run must not run on no-op"))
    monkeypatch.setattr(WorkflowService, "_sync_task_run_from_workflow_run", sync_mock)

    info_calls: list[str] = []

    def fake_info(event: str, **kwargs: object) -> None:
        info_calls.append(event)

    _patch_service_log(monkeypatch, info=fake_info)

    svc = WorkflowService()
    result = await svc.mark_workflow_run_as_canceled_if_not_final(workflow_run_id="wr_already_done")

    assert result is None
    assert "Workflow run duration metrics" not in info_calls
    sync_mock.assert_not_awaited()


@pytest.mark.asyncio
async def test_mark_canceled_if_not_final_is_noop_on_terminal_row(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """DB returns None when the row is already terminal — helper must propagate
    that as None instead of raising or writing a conflicting row."""
    delegate = AsyncMock(return_value=None)
    monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run_if_not_final", delegate)

    svc = WorkflowService()
    result = await svc.mark_workflow_run_as_canceled_if_not_final(workflow_run_id="wr_done")

    assert result is None
    delegate.assert_awaited_once()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "terminal_status",
    [
        WorkflowRunStatus.completed,
        WorkflowRunStatus.failed,
        WorkflowRunStatus.terminated,
        WorkflowRunStatus.timed_out,
        WorkflowRunStatus.canceled,
    ],
)
async def test_mark_canceled_rejects_transition_on_terminal_row(
    monkeypatch: pytest.MonkeyPatch,
    terminal_status: WorkflowRunStatus,
) -> None:
    """SKY-9188: ``mark_workflow_run_as_canceled`` must not overwrite a
    finalized status. The underlying conditional update returns ``None`` for
    terminal rows; the service helper must propagate the existing row back to
    callers without writing ``canceled``.
    """
    existing_row = MagicMock()
    existing_row.status = terminal_status

    conditional_update = AsyncMock(return_value=None)
    monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run_if_not_final", conditional_update)
    get_workflow_run = AsyncMock(return_value=existing_row)
    monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run", get_workflow_run)
    # A guard that accidentally falls through to the unconditional write would
    # call ``update_workflow_run`` — fail loudly if that happens.
    unconditional_update = AsyncMock(side_effect=AssertionError("unconditional update_workflow_run must not be called"))
    monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run", unconditional_update)

    svc = WorkflowService()
    result = await svc.mark_workflow_run_as_canceled(workflow_run_id="wr_final")

    assert result is existing_row
    assert result.status == terminal_status
    conditional_update.assert_awaited_once_with(
        workflow_run_id="wr_final",
        status=WorkflowRunStatus.canceled,
        failure_reason=None,
    )
    unconditional_update.assert_not_called()


@pytest.mark.asyncio
async def test_mark_canceled_writes_when_row_is_non_terminal(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """SKY-9188: a non-terminal run still transitions to ``canceled`` through
    the conditional update path.
    """
    canceled_row = MagicMock()
    canceled_row.status = WorkflowRunStatus.canceled

    conditional_update = AsyncMock(return_value=canceled_row)
    monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run_if_not_final", conditional_update)
    get_workflow_run = AsyncMock(side_effect=AssertionError("get_workflow_run must not be called on the happy path"))
    monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run", get_workflow_run)

    svc = WorkflowService()
    result = await svc.mark_workflow_run_as_canceled(workflow_run_id="wr_running")

    assert result is canceled_row
    conditional_update.assert_awaited_once_with(
        workflow_run_id="wr_running",
        status=WorkflowRunStatus.canceled,
        failure_reason=None,
    )
    get_workflow_run.assert_not_called()


@pytest.mark.asyncio
async def test_mark_canceled_raises_when_row_missing(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """When the conditional update rejects (returns ``None``) AND the row no
    longer exists, there is no sensible ``WorkflowRun`` to return — raise
    ``WorkflowRunNotFound`` rather than silently pretending the cancel
    succeeded.
    """
    conditional_update = AsyncMock(return_value=None)
    monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run_if_not_final", conditional_update)
    get_workflow_run = AsyncMock(return_value=None)
    monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run", get_workflow_run)

    svc = WorkflowService()

    with pytest.raises(WorkflowRunNotFound):
        await svc.mark_workflow_run_as_canceled(workflow_run_id="wr_missing")


@pytest.mark.asyncio
async def test_cleanup_after_cancellation_emits_only_canceled_webhook(monkeypatch: pytest.MonkeyPatch) -> None:
    running_run = _make_updated_row()
    running_run.workflow_run_id = "wr_canceled_during_cleanup"
    running_run.status = WorkflowRunStatus.running
    canceled_run = _make_updated_row()
    canceled_run.workflow_run_id = running_run.workflow_run_id

    svc = WorkflowService()
    monkeypatch.setattr(svc, "_grade_completion_contract", AsyncMock(return_value=None))
    conditional_finalize = AsyncMock(return_value=None)
    monkeypatch.setattr(svc, "_update_workflow_run_status_if_not_final", conditional_finalize)
    monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=canceled_run))

    webhook_statuses: list[WorkflowRunStatus] = []

    async def record_webhook(workflow_run: MagicMock, _api_key: str | None = None, **_claim: object) -> None:
        webhook_statuses.append(workflow_run.status)

    monkeypatch.setattr(svc, "execute_workflow_webhook", record_webhook)
    monkeypatch.setattr(app.AGENT_FUNCTION, "on_workflow_run_terminal", AsyncMock())
    monkeypatch.setattr(app.ARTIFACT_MANAGER, "wait_for_upload_aiotasks", AsyncMock())
    monkeypatch.setattr(app.STORAGE, "save_downloaded_files", AsyncMock())
    monkeypatch.setattr(service_module.uploaded_file_service, "delete_files_attached_to_run", AsyncMock())
    monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "remove_workflow_run_context", MagicMock())

    finalized_run = await svc._finalize_workflow_run_status(
        workflow_run_id=running_run.workflow_run_id,
        workflow_run=running_run,
        pre_finally_status=WorkflowRunStatus.running,
        pre_finally_failure_reason=None,
    )
    await svc.clean_up_workflow(
        workflow=SimpleNamespace(workflow_definition=SimpleNamespace(retry_policy=None)),
        workflow_run=finalized_run,
        browser_cleanup_result=WorkflowBrowserCleanupResult(
            browser_state=None,
            tasks=[],
            all_workflow_task_ids=[],
            child_workflow_run_ids=[],
            close_browser_on_completion=True,
        ),
        schedule_credential_fallback_retry=False,
    )

    conditional_finalize.assert_awaited_once_with(
        workflow_run_id=running_run.workflow_run_id,
        status=WorkflowRunStatus.completed,
    )
    assert finalized_run is canceled_run
    assert webhook_statuses == [WorkflowRunStatus.canceled]


@pytest.mark.asyncio
async def test_cancel_policy_run_leaves_attachment_deletion_to_terminal_side_effects(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow_run = _make_updated_row()
    workflow_run.workflow_run_id = "wr_policy_cancel"
    workflow_run.status = WorkflowRunStatus.running
    canceled_run = _make_updated_row()
    canceled_run.workflow_run_id = workflow_run.workflow_run_id
    attempts = [MagicMock(attempt_number=1)]
    monkeypatch.setattr(
        run_service.app.DATABASE.workflow_runs,
        "get_workflow_run",
        AsyncMock(return_value=workflow_run),
    )
    monkeypatch.setattr(
        run_service.app.DATABASE.workflow_runs,
        "get_workflow_runs_by_parent_workflow_run_id",
        AsyncMock(return_value=[]),
    )
    monkeypatch.setattr(
        run_service.app.DATABASE.workflow_run_attempts, "get_attempts", AsyncMock(return_value=attempts)
    )
    monkeypatch.setattr(
        run_service.app.WORKFLOW_SERVICE, "mark_workflow_run_as_canceled", AsyncMock(return_value=canceled_run)
    )
    decision = run_service.RetryDecision(False, 1, 0, True, "cancel")
    monkeypatch.setattr(run_service, "get_recorded_decision", AsyncMock(return_value=decision))
    terminal_effects = AsyncMock()
    monkeypatch.setattr(run_service.app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", terminal_effects)
    delete = AsyncMock()
    monkeypatch.setattr(run_service.uploaded_file_service, "delete_files_attached_to_run", delete)
    webhook = AsyncMock()
    monkeypatch.setattr(run_service.app.WORKFLOW_SERVICE, "execute_workflow_webhook", webhook)

    await run_service.cancel_workflow_run(workflow_run.workflow_run_id, organization_id="org_abc")

    delete.assert_not_awaited()
    terminal_effects.assert_awaited_once_with(canceled_run, decision, api_key=None)
    webhook.assert_not_awaited()


@pytest.mark.asyncio
async def test_cancel_policy_run_retries_failed_terminal_release_without_raising(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow_run = _make_updated_row()
    workflow_run.workflow_run_id = "wr_policy_cancel_effect_failed"
    workflow_run.status = WorkflowRunStatus.running
    canceled_run = _make_updated_row()
    canceled_run.workflow_run_id = workflow_run.workflow_run_id
    attempt = SimpleNamespace(
        attempt_number=1,
        retry_decision="final",
        decision_reason="cancel",
        side_effects_released_at=None,
        webhook_sent_at=None,
        interim_webhook_sent_at=None,
    )
    monkeypatch.setattr(
        run_service.app.DATABASE.workflow_runs,
        "get_workflow_run",
        AsyncMock(return_value=workflow_run),
    )
    monkeypatch.setattr(
        run_service.app.DATABASE.workflow_runs,
        "get_workflow_runs_by_parent_workflow_run_id",
        AsyncMock(return_value=[]),
    )
    monkeypatch.setattr(
        run_service.app.DATABASE.workflow_run_attempts,
        "get_attempts",
        AsyncMock(return_value=[attempt]),
    )
    svc = WorkflowService()
    monkeypatch.setattr(svc, "mark_workflow_run_as_canceled", AsyncMock(return_value=canceled_run))
    monkeypatch.setattr(run_service.app, "WORKFLOW_SERVICE", svc)
    decision = run_service.RetryDecision(False, 1, 0, True, "cancel")
    monkeypatch.setattr(run_service, "get_recorded_decision", AsyncMock(return_value=decision))

    release_calls = 0

    async def fail_release(*_args: object, **_kwargs: object) -> str:
        nonlocal release_calls
        release_calls += 1
        return "effect_failed"

    monkeypatch.setattr(svc, "run_terminal_side_effects", fail_release)
    warning = MagicMock()
    _patch_service_log(monkeypatch, warning=warning)

    # The cancel operation itself succeeded; a failed best-effort release is retried and left
    # for the durable recovery owner instead of being raised through the API path.
    await run_service.cancel_workflow_run(workflow_run.workflow_run_id, organization_id="org_abc")

    assert release_calls == TERMINAL_RELEASE_RETRY_MAX_ATTEMPTS
    assert any(
        call.args
        and "exhausted" in str(call.args[0])
        and call.kwargs.get("workflow_run_id") == workflow_run.workflow_run_id
        and call.kwargs.get("attempt_number") == 1
        and call.kwargs.get("outcome") == "effect_failed"
        for call in warning.call_args_list
    )


@pytest.mark.asyncio
async def test_cancel_no_policy_run_deletes_attachments_before_webhook(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow_run = _make_updated_row()
    workflow_run.workflow_run_id = "wr_no_policy_cancel"
    workflow_run.status = WorkflowRunStatus.queued
    canceled_run = _make_updated_row()
    canceled_run.workflow_run_id = workflow_run.workflow_run_id
    monkeypatch.setattr(
        run_service.app.DATABASE.workflow_runs,
        "get_workflow_run",
        AsyncMock(return_value=workflow_run),
    )
    monkeypatch.setattr(
        run_service.app.DATABASE.workflow_runs,
        "get_workflow_runs_by_parent_workflow_run_id",
        AsyncMock(return_value=[]),
    )
    monkeypatch.setattr(run_service.app.DATABASE.workflow_run_attempts, "get_attempts", AsyncMock(return_value=[]))
    monkeypatch.setattr(
        run_service.app.WORKFLOW_SERVICE, "mark_workflow_run_as_canceled", AsyncMock(return_value=canceled_run)
    )
    order: list[str] = []
    delete = AsyncMock(side_effect=lambda **_kwargs: order.append("delete"))
    webhook = AsyncMock(side_effect=lambda *args, **kwargs: order.append("webhook"))
    monkeypatch.setattr(run_service.uploaded_file_service, "delete_files_attached_to_run", delete)
    monkeypatch.setattr(run_service.app.WORKFLOW_SERVICE, "execute_workflow_webhook", webhook)

    await run_service.cancel_workflow_run(workflow_run.workflow_run_id, organization_id="org_abc")

    assert order == ["delete", "webhook"]
    delete.assert_awaited_once_with(run_id=workflow_run.workflow_run_id)


@pytest.mark.asyncio
async def test_execute_workflow_webhook_tolerates_soft_deleted_workflow(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """If the workflow row has been soft-deleted by the time cleanup runs,
    ``execute_workflow_webhook`` should log a warning and return — not raise.
    """
    svc = WorkflowService()

    async def raise_not_found(*args: object, **kwargs: object) -> None:
        raise WorkflowNotFound(workflow_permanent_id="wpid_gone")

    monkeypatch.setattr(svc, "build_workflow_run_status_response", raise_not_found)

    run = MagicMock()
    run.workflow_permanent_id = "wpid_gone"
    # Must complete cleanly without propagating the exception.
    await svc.execute_workflow_webhook(workflow_run=run)


@pytest.mark.asyncio
async def test_build_status_response_uses_filter_deleted_false_when_allowed(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """``allow_deleted=True`` goes through the run-joined repository lookup with
    ``filter_deleted=False`` so soft-deleted workflows still resolve.
    """
    svc = WorkflowService()

    by_run = AsyncMock(return_value=MagicMock())
    monkeypatch.setattr(app.DATABASE.workflows, "get_workflow_for_workflow_run", by_run)

    # Short-circuit the rest of build_workflow_run_status_response by making
    # subsequent DB calls raise the first caught thing — we only care about
    # the lookup kwargs here.
    async def immediately_raise(*args: object, **kwargs: object) -> None:
        raise RuntimeError("short-circuit")

    monkeypatch.setattr(svc, "get_workflow_run", immediately_raise)

    with pytest.raises(RuntimeError, match="short-circuit"):
        await svc.build_workflow_run_status_response(
            workflow_permanent_id="wpid_soft_deleted",
            workflow_run_id="wr_x",
            organization_id="org_1",
            allow_deleted=True,
        )

    by_run.assert_awaited_once()
    assert by_run.call_args.kwargs["filter_deleted"] is False


@pytest.mark.asyncio
async def test_shielded_finalize_runs_when_outer_cancelled_mid_body() -> None:
    """Contract test for the ``execute_workflow`` cancellation-safe pattern:
    when the try body is cancelled after ``pre_finally_status`` is captured,
    the outer ``finally`` must still run ``_finalize_workflow_run_status``
    via ``asyncio.shield`` so the row ends up terminal rather than stuck as
    transient ``running``. Mirrors the structure of
    ``WorkflowService.execute_workflow``; if anyone removes the ``shield`` or
    moves finalize back into the try, this test breaks.
    """

    finalize_calls: list[WorkflowRunStatus] = []
    clean_up_called = False
    body_entered = asyncio.Event()

    async def finalize(status: WorkflowRunStatus) -> None:
        # Simulate a non-trivial DB write so shield cancellation-protection
        # matters rather than being invisible.
        await asyncio.sleep(0.05)
        finalize_calls.append(status)

    async def clean_up() -> None:
        nonlocal clean_up_called
        clean_up_called = True

    async def simulated_execute_workflow() -> None:
        pre_finally_status: WorkflowRunStatus | None = None
        try:
            pre_finally_status = WorkflowRunStatus.failed
            body_entered.set()
            # Simulate the finally-block execution phase that our copilot
            # cancel lands inside of.
            await asyncio.sleep(10)
        finally:
            if pre_finally_status is not None:
                try:
                    await asyncio.shield(finalize(pre_finally_status))
                except Exception:
                    pass
            await clean_up()

    task = asyncio.create_task(simulated_execute_workflow())
    await body_entered.wait()
    task.cancel()

    with pytest.raises(asyncio.CancelledError):
        await task

    assert finalize_calls == [WorkflowRunStatus.failed], (
        "shielded finalize must run with the captured pre_finally_status"
    )
    assert clean_up_called, "clean_up_workflow must still run in the outer finally"


@pytest.mark.asyncio
async def test_shielded_finalize_skipped_when_pre_finally_status_unset() -> None:
    """If cancellation lands before block execution captures
    ``pre_finally_status``, there's no intended terminal state to restore —
    the outer ``finally`` must skip finalize, not call it with ``None``.
    """

    finalize_called = False

    async def finalize(status: WorkflowRunStatus) -> None:
        nonlocal finalize_called
        finalize_called = True

    async def simulated_execute_workflow() -> None:
        pre_finally_status: WorkflowRunStatus | None = None
        try:
            await asyncio.sleep(10)
            pre_finally_status = WorkflowRunStatus.failed  # pragma: no cover
        finally:
            if pre_finally_status is not None:
                await asyncio.shield(finalize(pre_finally_status))

    task = asyncio.create_task(simulated_execute_workflow())
    await asyncio.sleep(0)  # let the task enter its body
    task.cancel()

    with pytest.raises(asyncio.CancelledError):
        await task

    assert not finalize_called, "finalize must not run when pre_finally_status is unset"


def test_lease_failure_category_persists_only_typed_browser_loss() -> None:
    workflow_run = SimpleNamespace(created_at=datetime.now(UTC))
    closed = _browser_lease_failure_category(BrowserSessionClosed("pbs_x"), workflow_run)
    timeout = _browser_lease_failure_category(BrowserSessionStartupTimeout("pbs_x"), workflow_run)

    assert closed is not None and closed[0]["category"] == "BROWSER_ERROR"
    assert closed[0]["reason_code"] == "browser_session_closed"
    assert timeout is not None and timeout[0]["reason_code"] == "browser_session_startup_timeout"
    assert (
        _browser_lease_failure_category(RuntimeError("a required workflow parameter was not provided"), workflow_run)
        is None
    )