Skip to content

Automate browser based workflows with AI

Deliver the run webhook when a run without an attempt row fails before execution (SKY-16380) (#8566)

Shuchang Zheng64b59479c987
diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.py
index 877b404c6..e2fc34c31 100644
--- a/skyvern/forge/sdk/executor/background_task_executor.py
+++ b/skyvern/forge/sdk/executor/background_task_executor.py
@@ -609,6 +609,8 @@ class BackgroundTaskExecutor(AsyncExecutor):
if await fail_run_without_attempt_row(
attempt.workflow_run_id,
f"Workflow run initialization failed before execution: {type(exc).__name__}: {exc}",
+ api_key=execution_kwargs.get("api_key"),
+ need_call_webhook=execution_kwargs.get("need_call_webhook", True),
):
self._retry_resumes_needing_recovery.discard(key)
LOG.warning(
@@ -820,6 +822,22 @@ class BackgroundTaskExecutor(AsyncExecutor):
decision,
api_key=api_key,
)
+ else:
+ try:
+ terminal_run = await app.DATABASE.workflow_runs.get_workflow_run(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization.organization_id,
+ )
+ if terminal_run is not None:
+ await app.WORKFLOW_SERVICE.execute_workflow_webhook(
+ terminal_run, api_key=api_key, claim_kind=None
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to deliver workflow webhook after rejecting a sequential credential run",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
raise BackgroundSequentialCredentialUnsupported(workflow_run_id)
attempt = next(
diff --git a/skyvern/forge/sdk/workflow/retry_policy.py b/skyvern/forge/sdk/workflow/retry_policy.py
index 604456569..ccccfd246 100644
--- a/skyvern/forge/sdk/workflow/retry_policy.py
+++ b/skyvern/forge/sdk/workflow/retry_policy.py
@@ -337,17 +337,27 @@ async def ensure_attempt_row(
return True
-async def fail_run_without_attempt_row(workflow_run_id: str, failure_reason: str) -> bool:
+async def fail_run_without_attempt_row(
+ workflow_run_id: str, failure_reason: str, *, api_key: str | None = None, need_call_webhook: bool = True
+) -> bool:
"""Fail a queued run that no recovery sweep can see. Returns False when an attempt row exists."""
if await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id):
return False
try:
- await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ workflow_run = await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
workflow_run_id=workflow_run_id,
failure_reason=failure_reason,
)
- except Exception:
- workflow_run = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id)
+ except Exception as finalization_error:
+ try:
+ workflow_run = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id)
+ except Exception:
+ LOG.warning(
+ "Failed to re-read workflow run after a finalization error; leaving the run recoverable",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
+ raise finalization_error
if workflow_run is not None and not workflow_run.status.is_final():
raise
LOG.warning(
@@ -355,6 +365,19 @@ async def fail_run_without_attempt_row(workflow_run_id: str, failure_reason: str
workflow_run_id=workflow_run_id,
exc_info=True,
)
+ if workflow_run is not None and (
+ workflow_run.status != WorkflowRunStatus.failed or workflow_run.failure_reason != failure_reason
+ ):
+ workflow_run = None
+ if workflow_run is not None and need_call_webhook:
+ try:
+ await app.WORKFLOW_SERVICE.execute_workflow_webhook(workflow_run, api_key=api_key, claim_kind=None)
+ except Exception:
+ LOG.warning(
+ "Failed to deliver workflow webhook after initialization failure",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
return True
diff --git a/skyvern/services/workflow_schedule_service.py b/skyvern/services/workflow_schedule_service.py
index a1a181a39..071dc6fa9 100644
--- a/skyvern/services/workflow_schedule_service.py
+++ b/skyvern/services/workflow_schedule_service.py
@@ -236,6 +236,8 @@ class LocalWorkflowScheduleScheduler:
if await fail_run_without_attempt_row(
workflow_run.workflow_run_id,
f"Workflow run initialization failed before execution: {type(exc).__name__}: {exc}",
+ api_key=None,
+ need_call_webhook=True,
):
LOG.warning(
"Scheduled workflow initialization failed without an attempt row; run is terminal",
diff --git a/tests/unit/test_background_task_executor.py b/tests/unit/test_background_task_executor.py
index f82e304be..111a66883 100644
--- a/tests/unit/test_background_task_executor.py
+++ b/tests/unit/test_background_task_executor.py
@@ -14,7 +14,11 @@ from fastapi import BackgroundTasks
from sqlalchemy import insert, literal, select, update
from skyvern.config import settings
-from skyvern.exceptions import SkyvernException, WorkflowRetryAttemptLookupError
+from skyvern.exceptions import (
+ BackgroundSequentialCredentialUnsupported,
+ SkyvernException,
+ WorkflowRetryAttemptLookupError,
+)
from skyvern.forge import app
from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.sdk.artifact.models import ArtifactType
@@ -2870,9 +2874,12 @@ async def test_execute_workflow_dispatches_from_latest_retry_attempt(monkeypatch
@pytest.mark.asyncio
[email protected]("scenario", ["state_file", "llm_runtime", "lookup_failure", "attempt_row"])
[email protected]("need_call_webhook", [True, False])
+ "scenario", ["state_file", "llm_runtime", "lookup_failure", "attempt_row", "webhook_preparation"]
+)
async def test_no_policy_initializer_failure_fails_the_run_durably(
- interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, scenario: str
+ interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, scenario: str, need_call_webhook: bool
) -> None:
database = interim_retry_db
async with database.Session() as session:
@@ -2904,6 +2911,14 @@ async def test_no_policy_initializer_failure_fails_the_run_durably(
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
fail_run = AsyncMock(wraps=app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", fail_run)
+ webhook = AsyncMock(wraps=app.WORKFLOW_SERVICE.execute_workflow_webhook)
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_webhook", webhook)
+ deliver = AsyncMock(return_value=True)
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "deliver_prepared_workflow_webhook", deliver)
+ if scenario == "webhook_preparation":
+ monkeypatch.setattr(
+ app.WORKFLOW_SERVICE, "prepare_workflow_webhook", AsyncMock(side_effect=RuntimeError("preparation failed"))
+ )
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
organization = await database.organizations.get_organization("org_test")
@@ -2916,12 +2931,13 @@ async def test_no_policy_initializer_failure_fails_the_run_durably(
workflow_run_id="wr_retry",
workflow_permanent_id="wpid_retry",
max_steps_override=None,
- api_key=None,
+ api_key="test-dispatch-key",
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
_background_tasks, dispatch, attempt, execution_kwargs = executor._schedule.call_args.args
+ execution_kwargs["need_call_webhook"] = need_call_webhook
get_attempts = database.workflow_run_attempts.get_attempts
if scenario == "lookup_failure":
monkeypatch.setattr(
@@ -2938,6 +2954,7 @@ async def test_no_policy_initializer_failure_fails_the_run_durably(
assert run.status == WorkflowRunStatus.queued
assert run.failure_reason is None
fail_run.assert_not_awaited()
+ webhook.assert_not_awaited()
assert key in executor._retry_dispatches_needing_recovery
assert key in executor._retry_resumes_needing_recovery
if scenario == "attempt_row":
@@ -2951,6 +2968,10 @@ async def test_no_policy_initializer_failure_fails_the_run_durably(
reason = f"Workflow run initialization failed before execution: RuntimeError: {error}"
assert run.failure_reason == reason
fail_run.assert_awaited_once_with(workflow_run_id="wr_retry", failure_reason=reason)
+ if need_call_webhook:
+ webhook.assert_awaited_once_with(run, api_key="test-dispatch-key", claim_kind=None)
+ else:
+ webhook.assert_not_awaited()
assert not attempts
assert not executor._retry_dispatches_needing_recovery
assert not executor._retry_resumes_needing_recovery
@@ -2958,10 +2979,90 @@ async def test_no_policy_initializer_failure_fails_the_run_durably(
await executor._recover_pending_retries_once(resume_fresh=True)
executor._schedule.assert_not_called()
execute.assert_not_awaited()
+ if need_call_webhook and scenario in {"state_file", "llm_runtime"}:
+ deliver.assert_awaited_once()
+ else:
+ deliver.assert_not_awaited()
if app.WORKFLOW_SERVICE._background_tasks:
await asyncio.gather(*app.WORKFLOW_SERVICE._background_tasks)
+ "scenario",
+ [
+ "duplicate",
+ "terminal_reread",
+ "foreign_terminal",
+ "nonfinal_reread",
+ "reread_failure",
+ "missing_reread",
+ "delivery_failure",
+ ],
+)
+async def test_fail_run_without_attempt_row_finalization_outcomes(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, scenario: str
+) -> None:
+ run = SimpleNamespace(
+ workflow_run_id="wr_test", status=WorkflowRunStatus.failed, failure_reason="initialization failed"
+ )
+ finalization_error = RuntimeError("finalization failed")
+ fail_run = AsyncMock(return_value=run)
+ reread = AsyncMock(return_value=run)
+ webhook = AsyncMock()
+ if scenario == "duplicate":
+ fail_run.return_value = None
+ elif scenario == "delivery_failure":
+ webhook.side_effect = RuntimeError("delivery failed")
+ else:
+ fail_run.side_effect = finalization_error
+ if scenario == "nonfinal_reread":
+ run.status = WorkflowRunStatus.queued
+ elif scenario == "foreign_terminal":
+ run.status = WorkflowRunStatus.canceled
+ elif scenario == "reread_failure":
+ reread.side_effect = RuntimeError("re-read failed")
+ elif scenario == "missing_reread":
+ reread.return_value = None
+ get_attempts = AsyncMock(return_value=[])
+ monkeypatch.setattr(
+ retry_policy_module,
+ "app",
+ SimpleNamespace(
+ DATABASE=SimpleNamespace(
+ workflow_run_attempts=SimpleNamespace(get_attempts=get_attempts),
+ workflow_runs=SimpleNamespace(get_workflow_run=reread),
+ ),
+ WORKFLOW_SERVICE=SimpleNamespace(
+ mark_workflow_run_as_failed_if_not_final=fail_run,
+ execute_workflow_webhook=webhook,
+ ),
+ ),
+ )
+ if scenario in {"nonfinal_reread", "reread_failure"}:
+ with pytest.raises(RuntimeError) as exc_info:
+ await retry_policy_module.fail_run_without_attempt_row("wr_test", "initialization failed")
+ assert exc_info.value is finalization_error
+ if scenario == "reread_failure":
+ assert exc_info.value.__context__ is reread.side_effect
+ else:
+ assert await retry_policy_module.fail_run_without_attempt_row("wr_test", "initialization failed") is True
+ get_attempts.assert_awaited_once_with("wr_test")
+ fail_run.assert_awaited_once_with(workflow_run_id="wr_test", failure_reason="initialization failed")
+ if scenario in {"duplicate", "delivery_failure"}:
+ reread.assert_not_awaited()
+ else:
+ reread.assert_awaited_once_with("wr_test")
+ if scenario in {"terminal_reread", "delivery_failure"}:
+ webhook.assert_awaited_once_with(run, api_key=None, claim_kind=None)
+ else:
+ webhook.assert_not_awaited()
+ if scenario == "reread_failure":
+ assert "leaving the run recoverable" in caplog.text
+ elif scenario == "delivery_failure":
+ assert "Failed to deliver workflow webhook after initialization failure" in caplog.text
+
+
@pytest.mark.asyncio
async def test_first_dispatch_initializer_failure_is_retried_by_the_sweep(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
@@ -3552,19 +3653,40 @@ async def test_scheduled_task_is_retained_until_done() -> None:
@pytest.mark.asyncio
-async def test_execute_workflow_fails_closed_for_stamped_sequential_credential(monkeypatch: pytest.MonkeyPatch) -> None:
[email protected]("delivery_case", ["no_attempt_rows", "attempt_rows", "webhook_error"])
+async def test_execute_workflow_fails_closed_for_stamped_sequential_credential(
+ monkeypatch: pytest.MonkeyPatch, delivery_case: str
+) -> None:
workflow_run = SimpleNamespace(sequential_credential_id="cred_sequential", browser_session_id=None)
+ terminal_run = SimpleNamespace(
+ status=WorkflowRunStatus.failed,
+ failure_reason="Sequential credential execution is unavailable in the background executor",
+ finished_at=datetime.now(UTC),
+ )
+ get_run = AsyncMock(side_effect=[workflow_run, terminal_run])
monkeypatch.setattr(
app.DATABASE.workflow_runs,
"get_workflow_run",
- AsyncMock(return_value=workflow_run),
+ get_run,
+ )
+ monkeypatch.setattr(
+ app.DATABASE.workflow_run_attempts,
+ "get_attempts",
+ AsyncMock(return_value=[SimpleNamespace(attempt_number=1)] if delivery_case == "attempt_rows" else []),
)
mark_failed = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+ decision = RetryDecision(False, 1, 0, True, "sequential_credential_unsupported")
+ finalize = AsyncMock(return_value=decision)
+ monkeypatch.setattr(background_task_executor_module, "finalize_abandoned_attempt", finalize)
+ release = AsyncMock()
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", release)
+ webhook = AsyncMock(side_effect=RuntimeError("delivery failed") if delivery_case == "webhook_error" else None)
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_webhook", webhook)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
- with pytest.raises(SkyvernException, match="background executor"):
+ with pytest.raises(BackgroundSequentialCredentialUnsupported, match="background executor"):
await executor.execute_workflow(
request=None,
background_tasks=None,
@@ -3573,7 +3695,7 @@ async def test_execute_workflow_fails_closed_for_stamped_sequential_credential(m
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
- api_key=None,
+ api_key="test-api-key",
browser_session_id=None,
block_labels=None,
block_outputs=None,
@@ -3583,6 +3705,22 @@ async def test_execute_workflow_fails_closed_for_stamped_sequential_credential(m
mark_failed.assert_awaited_once()
assert mark_failed.await_args.kwargs["workflow_run_id"] == "wr_test"
assert mark_failed.await_args.kwargs["cascade_children"] is True
+ get_run.assert_awaited_with(workflow_run_id="wr_test", organization_id="org_test")
+ if delivery_case == "attempt_rows":
+ finalize.assert_awaited_once_with(
+ workflow_run_id="wr_test",
+ organization_id="org_test",
+ reason="sequential_credential_unsupported",
+ status=terminal_run.status,
+ failure_reason=terminal_run.failure_reason,
+ finished_at=terminal_run.finished_at,
+ )
+ release.assert_awaited_once_with(terminal_run, decision, api_key="test-api-key")
+ webhook.assert_not_awaited()
+ else:
+ webhook.assert_awaited_once_with(terminal_run, api_key="test-api-key", claim_kind=None)
+ release.assert_not_awaited()
+ finalize.assert_not_awaited()
@pytest.mark.asyncio
diff --git a/tests/unit/test_workflow_schedule_service.py b/tests/unit/test_workflow_schedule_service.py
index ea5d9587c..ad3938511 100644
--- a/tests/unit/test_workflow_schedule_service.py
+++ b/tests/unit/test_workflow_schedule_service.py
@@ -170,7 +170,9 @@ async def test_run_schedule_no_policy_initializer_failure_fails_the_run(
previous_fire_time = datetime(2026, 6, 2, 10, 0, tzinfo=UTC)
run_id = schedule_service.build_scheduled_workflow_run_id(schedule.workflow_schedule_id, previous_fire_time)
execute = AsyncMock()
- fail_run = AsyncMock()
+ failed_run = SimpleNamespace(workflow_run_id=run_id, status=WorkflowRunStatus.failed)
+ fail_run = AsyncMock(return_value=failed_run)
+ webhook = AsyncMock()
get_attempts = AsyncMock(return_value=[SimpleNamespace(attempt_number=1)] if has_attempt_row else [])
fake_app = SimpleNamespace(
DATABASE=SimpleNamespace(
@@ -183,6 +185,7 @@ async def test_run_schedule_no_policy_initializer_failure_fails_the_run(
WORKFLOW_SERVICE=SimpleNamespace(
execute_workflow_with_retries=execute,
mark_workflow_run_as_failed_if_not_final=fail_run,
+ execute_workflow_webhook=webhook,
),
)
monkeypatch.setattr(schedule_service, "app", fake_app)
@@ -202,12 +205,14 @@ async def test_run_schedule_no_policy_initializer_failure_fails_the_run(
with pytest.raises(RuntimeError, match=f"{initializer} unavailable"):
await scheduler._run_schedule(due)
fail_run.assert_not_awaited()
+ webhook.assert_not_awaited()
else:
await scheduler._run_schedule(due)
fail_run.assert_awaited_once_with(
workflow_run_id=run_id,
failure_reason=f"Workflow run initialization failed before execution: RuntimeError: {error}",
)
+ webhook.assert_awaited_once_with(failed_run, api_key=None, claim_kind=None)
get_attempts.assert_awaited_once_with(run_id)
fake_app.DATABASE.workflow_runs.queue_initial_dispatch.assert_awaited_once_with(run_id, 1)
execute.assert_not_awaited()