Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
import asyncio
import json
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import ANY, AsyncMock, MagicMock, Mock
import httpx
import pytest
import pytest_asyncio
from fastapi import BackgroundTasks
from sqlalchemy import insert, literal, select, update
from skyvern.config import settings
from skyvern.exceptions import (
BackgroundSequentialCredentialUnsupported,
SkyvernException,
WorkflowRetryAttemptLookupError,
)
from skyvern.forge import app
from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.agent_db import AgentDB, _build_engine
from skyvern.forge.sdk.db.models import (
ArtifactModel,
Base,
OrganizationModel,
OutputParameterModel,
StepModel,
TaskModel,
WorkflowModel,
WorkflowRunAttemptModel,
WorkflowRunBlockModel,
WorkflowRunModel,
WorkflowRunOutputParameterModel,
)
from skyvern.forge.sdk.db.repositories import workflow_run_attempts as attempts_repository_module
from skyvern.forge.sdk.db.repositories import workflow_runs as workflow_runs_repository_module
from skyvern.forge.sdk.db.repositories.workflow_runs import PrepareNextAttemptResult
from skyvern.forge.sdk.executor import background_task_executor as background_task_executor_module
from skyvern.forge.sdk.executor.background_task_executor import BackgroundTaskExecutor
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
from skyvern.forge.sdk.workflow import retry_policy as retry_policy_module
from skyvern.forge.sdk.workflow import service as service_module
from skyvern.forge.sdk.workflow.constants import INTERIM_OUTPUT_SNAPSHOT_MAX_BYTES
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
from skyvern.forge.sdk.workflow.retry_policy import RETRY_DECISION_GRACE_SECONDS, RetryDecision, mark_attempt_started
from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.schemas.browser_session_close import BrowserSessionCloseReason
from skyvern.schemas.run_enums import WebhookDeliveryStatus
from skyvern.services import workflow_schedule_service as schedule_service_module
from skyvern.services.webhook_delivery import PreparedWorkflowWebhook
from tests.unit.scoped_asyncio import ScopedAsyncio
@pytest.fixture(autouse=True)
def workflow_service(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(app, "WORKFLOW_SERVICE", WorkflowService())
monkeypatch.setattr(
app.DATABASE.workflow_run_attempts, "list_stale_dispatch_claims", AsyncMock(return_value=[]), raising=False
)
monkeypatch.setattr(app.AGENT_FUNCTION, "get_workflow_run_execution_status", AsyncMock(return_value="absent"))
@pytest_asyncio.fixture
async def sqlite_db(tmp_path: Path) -> AsyncIterator[AgentDB]:
url = f"sqlite+aiosqlite:///{tmp_path / 'recovery.db'}"
engine = _build_engine(url)
async with engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
try:
yield AgentDB(url, db_engine=engine)
finally:
await engine.dispose()
@pytest_asyncio.fixture
async def interim_retry_db(sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch) -> AgentDB:
monkeypatch.setattr(app, "DATABASE", sqlite_db)
now = datetime.now(UTC).replace(tzinfo=None)
async with sqlite_db.Session() as session:
session.add(OrganizationModel(organization_id="org_test", organization_name="Test Organization"))
await session.commit()
session.add_all(
[
WorkflowModel(
workflow_id="wf_retry",
workflow_permanent_id="wpid_retry",
organization_id="org_test",
title="Retry delivery",
version=1,
workflow_definition={
"blocks": [],
"parameters": [],
"retry_policy": {
"max_retries": 1,
"delay_seconds": 5,
"retry_on": [{"status": "failed"}],
"webhook_on_retry": "every_attempt",
},
},
),
WorkflowRunModel(
workflow_run_id="wr_retry",
workflow_id="wf_retry",
workflow_permanent_id="wpid_retry",
organization_id="org_test",
status="failed",
finished_at=now,
),
WorkflowRunAttemptModel(
workflow_run_id="wr_retry",
organization_id="org_test",
attempt_number=1,
status="failed",
retry_decision="retry",
finished_at=now,
next_attempt_at=now + timedelta(seconds=5),
),
]
)
await session.commit()
svc = app.WORKFLOW_SERVICE
monkeypatch.setattr(
svc,
"prepare_workflow_webhook",
AsyncMock(
return_value=PreparedWorkflowWebhook(
workflow_id="wf_retry",
workflow_run_id="wr_retry",
organization_id="org_test",
webhook_callback_url="https://example.com/hook",
signed_payload="{}",
headers={},
)
),
)
monkeypatch.setattr(service_module.uploaded_file_service, "delete_files_attached_to_run", AsyncMock())
monkeypatch.setattr(app.AGENT_FUNCTION, "on_workflow_run_final", AsyncMock())
monkeypatch.setattr(svc, "_start_credential_fallback_retry_best_effort", AsyncMock())
monkeypatch.setattr(svc, "_apply_completion_run_tags_best_effort", AsyncMock())
monkeypatch.setattr(svc, "_run_workflow_run_terminal_hooks", AsyncMock())
monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "remove_workflow_run_context", MagicMock())
monkeypatch.setattr(sqlite_db.tasks, "get_tasks_by_workflow_run_id", AsyncMock(return_value=[]))
monkeypatch.setattr(sqlite_db.observer, "get_workflow_run_blocks", AsyncMock(return_value=[]))
return sqlite_db
@pytest_asyncio.fixture
async def prepared_dispatch_db(interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch) -> AgentDB:
database = interim_retry_db
monkeypatch.setattr(settings, "RETRY_DISPATCH_GRACE_SECONDS", 600)
async with database.Session() as session:
previous = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
assert previous is not None
previous.interim_webhook_sent_at = datetime.now(UTC).replace(tzinfo=None)
await session.commit()
preparation = await database.workflow_runs.prepare_next_attempt_atomic("wr_retry", "org_test", 1, "failed", None)
assert preparation.status == "inserted"
monkeypatch.setattr(
database.organizations, "get_valid_org_auth_token", AsyncMock(return_value=SimpleNamespace(token="api-token"))
)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
return database
@pytest.mark.asyncio
@pytest.mark.parametrize("resume_fresh", [True, False], ids=["startup", "periodic"])
@pytest.mark.parametrize("configured_grace", [0, 600, 2 * service_module.LEASE_TAKEOVER_SECONDS])
@pytest.mark.parametrize("scenario", ["stale", "young", "boundary", "running"])
async def test_claimed_undispatched_retry_recovers_after_process_loss(
prepared_dispatch_db: AgentDB,
monkeypatch: pytest.MonkeyPatch,
scenario: str,
resume_fresh: bool,
configured_grace: int,
) -> None:
database = prepared_dispatch_db
assert await database.workflow_run_attempts.claim_prepared_attempt_execution("wr_retry", "org_test", 2)
attempt = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
original_stamp = attempt.started_at
assert original_stamp is not None
monkeypatch.setattr(settings, "RETRY_DISPATCH_GRACE_SECONDS", configured_grace)
grace = max(600, service_module.LEASE_TAKEOVER_SECONDS, configured_grace)
age = grace + {"stale": 1, "young": -1, "boundary": 0, "running": 1}[scenario]
recovery_time = attempt.modified_at + timedelta(seconds=age)
for module in (service_module, background_task_executor_module, attempts_repository_module):
monkeypatch.setattr(module, "naive_utc_now", lambda: recovery_time)
if scenario == "running":
async with database.Session() as session:
run_model = await session.get(WorkflowRunModel, "wr_retry")
assert run_model is not None
run_model.status = "running"
await session.commit()
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
run = await database.workflow_runs.get_workflow_run("wr_retry", "org_test")
assert run is not None and run.started_at is not None
assert attempt.status == "running" and attempt.retry_decision is None
assert await AgentFunction.get_workflow_run_execution_status(app.AGENT_FUNCTION, run) == "absent"
# Cloud/Temporal's existing recovery query must continue to exclude dispatch claims.
assert (
await database.workflow_run_attempts.list_attempts_needing_recovery(
recovery_time - timedelta(seconds=service_module.LEASE_TAKEOVER_SECONDS)
)
== []
)
executor = BackgroundTaskExecutor()
assert not executor._retry_dispatches_needing_recovery
if resume_fresh:
monkeypatch.setattr(executor, "_start_retry_recovery_sweep", MagicMock())
await executor.recover_pending_retries()
else:
await executor._recover_pending_retries_once(resume_fresh=False)
if executor._background_tasks:
await asyncio.gather(*executor._background_tasks)
current = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
if scenario == "stale":
execute.assert_awaited_once()
assert execute.await_args is not None
assert execute.await_args.kwargs["attempt_number"] == 2
assert execute.await_args.kwargs["prepared_attempt_claimed"] is True
assert current.started_at == recovery_time != original_stamp
assert current.status == "running" and current.retry_decision is None
else:
execute.assert_not_awaited()
assert current.started_at == original_stamp
assert current.modified_at == attempt.modified_at
def _hold_after_dispatch_claim(
database: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> tuple[asyncio.Event, asyncio.Event]:
"""Pause a prepared resume right after it claims the dispatch, before execution starts."""
claimed = asyncio.Event()
resume = asyncio.Event()
claim = database.workflow_run_attempts.claim_prepared_attempt_execution
async def claim_and_wait(*args: object, **kwargs: object) -> datetime | None:
stamp = await claim(*args, **kwargs)
claimed.set()
await resume.wait()
return stamp
monkeypatch.setattr(database.workflow_run_attempts, "claim_prepared_attempt_execution", claim_and_wait)
return claimed, resume
@pytest.mark.asyncio
async def test_retry_dispatch_fences_claim_replaced_before_dispatch(
prepared_dispatch_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
database = prepared_dispatch_db
entered, resume = _hold_after_dispatch_claim(database, monkeypatch)
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
pending = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
executor = BackgroundTaskExecutor()
executor._schedule_retry_resume(pending)
tasks = list(executor._background_tasks)
try:
await asyncio.wait_for(entered.wait(), timeout=5)
claimed = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert claimed.started_at is not None
# Emulate the durable result of a competing recovery release/reclaim before this dispatch runs.
replacement_stamp = claimed.started_at + timedelta(seconds=2 * service_module.LEASE_TAKEOVER_SECONDS)
async with database.Session() as session:
await session.execute(
update(WorkflowRunAttemptModel)
.where(
WorkflowRunAttemptModel.workflow_run_id == "wr_retry", WorkflowRunAttemptModel.attempt_number == 2
)
.values(started_at=replacement_stamp, modified_at=replacement_stamp)
)
await session.execute(
update(WorkflowRunModel)
.where(WorkflowRunModel.workflow_run_id == "wr_retry")
.values(started_at=replacement_stamp, modified_at=replacement_stamp)
)
await session.commit()
finally:
resume.set()
await asyncio.gather(*tasks)
execute.assert_not_awaited()
current = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert current.started_at == replacement_stamp
assert current.retry_decision is None
assert not executor._retry_dispatches_needing_recovery
assert not executor._scheduled_retry_resumes
@pytest.mark.asyncio
@pytest.mark.parametrize("initializer", ["state_file", "llm_runtime"])
@pytest.mark.parametrize("recovers", [True, False], ids=["transient", "persistent"])
async def test_prepared_retry_initializer_failure_keeps_the_attempt_recoverable(
prepared_dispatch_db: AgentDB, monkeypatch: pytest.MonkeyPatch, initializer: str, recovers: bool
) -> None:
database = prepared_dispatch_db
error = RuntimeError("initializer unavailable")
failing = AsyncMock(side_effect=[error, None, None] if recovers else error)
monkeypatch.setattr(
background_task_executor_module,
"initialize_skyvern_state_file" if initializer == "state_file" else "prepare_org_llm_runtime",
failing,
)
real_now = background_task_executor_module.naive_utc_now
clock_offset = timedelta()
async def sleep(seconds: float) -> None:
nonlocal clock_offset
assert 0 < seconds <= background_task_executor_module.LEASE_TAKEOVER_SECONDS
if not recovers:
clock_offset += timedelta(seconds=background_task_executor_module.LEASE_TAKEOVER_SECONDS + 1)
monkeypatch.setattr(background_task_executor_module, "naive_utc_now", lambda: real_now() + clock_offset)
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=sleep))
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
pending = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
executor = BackgroundTaskExecutor()
executor._schedule_retry_resume(pending)
await asyncio.gather(*executor._background_tasks)
current = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
workflow_run = await database.workflow_runs.get_workflow_run(workflow_run_id="wr_retry", organization_id="org_test")
assert workflow_run is not None
assert current.retry_decision is None
assert not executor._scheduled_retry_resumes
if recovers:
execute.assert_awaited_once()
assert execute.await_args is not None
assert execute.await_args.kwargs["attempt_number"] == 2
assert execute.await_args.kwargs["prepared_attempt_claimed"] is True
assert current.started_at is not None
else:
execute.assert_not_awaited()
assert current.status == "queued" and current.started_at is None
assert workflow_run.status == WorkflowRunStatus.queued
assert ("wr_retry", 2) in executor._retry_resumes_needing_recovery
@pytest.mark.asyncio
async def test_periodic_recovery_skips_its_own_stale_dispatch(
prepared_dispatch_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
database = prepared_dispatch_db
entered, resume = _hold_after_dispatch_claim(database, monkeypatch)
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
pending = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
executor = BackgroundTaskExecutor()
executor._schedule_retry_resume(pending)
tasks = list(executor._background_tasks)
try:
await asyncio.wait_for(entered.wait(), timeout=5)
claimed = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert claimed.started_at is not None
recovery_time = claimed.modified_at + timedelta(seconds=2 * service_module.LEASE_TAKEOVER_SECONDS)
for module in (service_module, background_task_executor_module, attempts_repository_module):
monkeypatch.setattr(module, "naive_utc_now", lambda: recovery_time)
await executor._recover_pending_retries_once(resume_fresh=False)
current = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert current.started_at == claimed.started_at
assert current.modified_at == claimed.modified_at
execute.assert_not_awaited()
finally:
resume.set()
await asyncio.gather(*tasks)
execute.assert_awaited_once()
@pytest.mark.asyncio
async def test_recovery_skips_dispatch_if_release_cas_loses(
prepared_dispatch_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
database = prepared_dispatch_db
assert await database.workflow_run_attempts.claim_prepared_attempt_execution("wr_retry", "org_test", 2)
claimed = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
recovery_time = claimed.modified_at + timedelta(seconds=2 * service_module.LEASE_TAKEOVER_SECONDS)
for module in (service_module, background_task_executor_module, attempts_repository_module):
monkeypatch.setattr(module, "naive_utc_now", lambda: recovery_time)
release = database.workflow_run_attempts.release_stale_dispatch_claim
async def lose_release(*args: object, **kwargs: object) -> bool:
async with database.Session() as session:
await session.execute(
update(WorkflowRunModel).where(WorkflowRunModel.workflow_run_id == "wr_retry").values(status="running")
)
await session.commit()
return await release(*args, **kwargs)
monkeypatch.setattr(database.workflow_run_attempts, "release_stale_dispatch_claim", lose_release)
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
executor = BackgroundTaskExecutor()
await executor._recover_pending_retries_once(resume_fresh=False)
if executor._background_tasks:
await asyncio.gather(*executor._background_tasks)
execute.assert_not_awaited()
current = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert current.started_at == claimed.started_at
assert not executor._scheduled_retry_resumes
@pytest_asyncio.fixture
async def undecided_terminal_db(interim_retry_db: AgentDB) -> AgentDB:
old = datetime.now(UTC).replace(tzinfo=None) - timedelta(seconds=service_module.LEASE_TAKEOVER_SECONDS + 1)
async with interim_retry_db.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.started_at = old - timedelta(seconds=10)
run.finished_at = old
run.failure_reason = "persisted terminal failure"
run.failure_category = [{"category": "navigation"}]
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.status = "running"
attempt.retry_decision = None
attempt.started_at = run.started_at
attempt.finished_at = None
attempt.next_attempt_at = None
attempt.modified_at = old
await session.commit()
return interim_retry_db
@pytest.mark.asyncio
@pytest.mark.parametrize("sweep", ["startup", "periodic", "release_only"])
@pytest.mark.parametrize("status", ["failed", "terminated"])
async def test_recovery_reconstructs_undecided_terminal_attempt(
undecided_terminal_db: AgentDB, monkeypatch: pytest.MonkeyPatch, sweep: str, status: str
) -> None:
database = undecided_terminal_db
svc = app.WORKFLOW_SERVICE
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.status = status
await session.commit()
deliver = AsyncMock(return_value=True)
execute = AsyncMock()
monkeypatch.setattr(svc, "deliver_prepared_workflow_webhook", deliver)
monkeypatch.setattr(svc, "execute_workflow_with_retries", execute)
monkeypatch.setattr(service_module, "_get_recovery_api_key", AsyncMock(return_value="test-key"))
monkeypatch.setattr(BackgroundTaskExecutor, "_get_valid_api_key", AsyncMock(return_value="test-key"))
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))
executor = BackgroundTaskExecutor()
if sweep == "release_only":
await svc.recover_pending_workflow_attempts(release_only=True)
else:
await executor._recover_pending_retries_once(resume_fresh=sweep == "startup")
await asyncio.gather(*executor._background_tasks)
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
first = attempts[0]
assert first.status == status
assert first.retry_decision == ("retry" if status == "failed" else "final")
assert first.failure_reason == "persisted terminal failure"
assert first.failure_category == [{"category": "navigation"}]
assert first.finished_at is not None
deliver.assert_awaited_once()
if status == "failed":
assert first.next_attempt_at is not None
assert first.interim_webhook_sent_at is not None
assert first.webhook_sent_at is None
if sweep == "release_only":
assert len(attempts) == 1
execute.assert_not_awaited()
else:
assert first.next_attempt_prepared_at is not None
assert len(attempts) == 2
assert attempts[1].status == "queued"
assert execute.await_args.kwargs["attempt_number"] == 2
else:
assert len(attempts) == 1
assert first.webhook_sent_at is not None
assert first.next_attempt_at is None
execute.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("sweep", ["background", "release_only"])
@pytest.mark.parametrize("ownership", ["absent", "running", "unknown"])
async def test_recovery_leaves_concurrent_terminal_decision_and_live_owner_alone(
undecided_terminal_db: AgentDB, monkeypatch: pytest.MonkeyPatch, sweep: str, ownership: str
) -> None:
database = undecided_terminal_db
svc = app.WORKFLOW_SERVICE
async def execution_status(run: WorkflowRun) -> str:
if ownership == "absent":
await service_module.on_terminal_transition(
run, run.status, run.failure_reason, run.failure_category, attempt_number=1
)
return ownership
monkeypatch.setattr(app.AGENT_FUNCTION, "get_workflow_run_execution_status", execution_status)
deliver = AsyncMock(return_value=True)
monkeypatch.setattr(svc, "deliver_prepared_workflow_webhook", deliver)
executor = BackgroundTaskExecutor()
if sweep == "release_only":
await svc.recover_pending_workflow_attempts(release_only=True)
else:
await executor._recover_pending_retries_once(resume_fresh=False)
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert len(attempts) == 1
assert attempts[0].retry_decision == ("retry" if ownership == "absent" else None)
assert attempts[0].side_effects_released_at is None
assert attempts[0].interim_webhook_sent_at is None
assert attempts[0].webhook_sent_at is None
assert not executor._scheduled_retry_resumes
deliver.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("race_stage", ["read", "write"])
async def test_undecided_recovery_is_noop_when_live_finalizer_wins(
undecided_terminal_db: AgentDB, monkeypatch: pytest.MonkeyPatch, race_stage: str
) -> None:
database = undecided_terminal_db
svc = app.WORKFLOW_SERVICE
read_attempts = database.workflow_run_attempts.get_attempts
transition = service_module.on_terminal_transition
winning_attempt: WorkflowRunAttemptModel | None = None
finalize_ready = asyncio.Event()
async def finalize_live() -> None:
nonlocal winning_attempt
await finalize_ready.wait()
winning_attempt = await database.workflow_run_attempts.finalize_attempt(
"wr_retry",
1,
status="failed",
failure_reason="live finalizer failure",
failure_category=None,
error_codes=[],
finished_at=datetime.now(UTC),
retry_decision="final",
decision_reason="live finalizer decision",
next_attempt_at=None,
)
live_finalizer = asyncio.create_task(finalize_live())
async def transition_after_live(*args: object, **kwargs: object) -> object:
finalize_ready.set()
await live_finalizer
return await transition(*args, **kwargs)
async def tasks_after_live(*args: object, **kwargs: object) -> list:
finalize_ready.set()
await live_finalizer
return []
if race_stage == "read":
monkeypatch.setattr(service_module, "on_terminal_transition", transition_after_live)
else:
monkeypatch.setattr(database.tasks, "get_tasks_by_workflow_run_id", tasks_after_live)
deliver = AsyncMock(return_value=True)
monkeypatch.setattr(svc, "deliver_prepared_workflow_webhook", deliver)
executor = BackgroundTaskExecutor()
try:
await executor._recover_pending_retries_once(resume_fresh=False)
finally:
live_finalizer.cancel()
await asyncio.gather(live_finalizer, return_exceptions=True)
assert winning_attempt is not None
attempts = await read_attempts("wr_retry")
assert len(attempts) == 1
current = attempts[0]
assert current.retry_decision == "final"
assert current.failure_reason == "live finalizer failure"
assert current.decision_reason == "live finalizer decision"
assert current.modified_at == winning_attempt.modified_at
assert current.side_effects_released_at is None
assert current.webhook_sent_at is None
assert not executor._scheduled_retry_resumes
deliver.assert_not_awaited()
@pytest.mark.asyncio
async def test_recovery_sweep_paginates_undecided_attempts_after_finalizing_each_page(
undecided_terminal_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
database = undecided_terminal_db
svc = app.WORKFLOW_SERVICE
old = datetime.now(UTC).replace(tzinfo=None) - timedelta(seconds=service_module.LEASE_TAKEOVER_SECONDS + 1)
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.status = "terminated"
for index in range(4):
run_id = f"wr_page_{index}"
session.add(
WorkflowRunModel(
workflow_run_id=run_id,
organization_id="org_test",
workflow_id="wf_retry",
workflow_permanent_id="wpid_retry",
status="terminated",
started_at=old,
finished_at=old,
)
)
session.add(
WorkflowRunAttemptModel(
workflow_run_id=run_id,
organization_id="org_test",
attempt_number=1,
status="running",
started_at=old,
modified_at=old,
)
)
await session.commit()
read_page = database.workflow_run_attempts.list_attempts_needing_recovery
cursors: list[tuple[datetime, str, int] | None] = []
async def small_page(
cutoff: datetime, *, cursor: tuple[datetime, str, int] | None = None
) -> list[WorkflowRunAttemptModel]:
cursors.append(cursor)
return await read_page(cutoff, cursor=cursor, limit=2)
monkeypatch.setattr(database.workflow_run_attempts, "list_attempts_needing_recovery", small_page)
monkeypatch.setattr(service_module, "ATTEMPT_RECOVERY_BATCH_SIZE", 2)
monkeypatch.setattr(service_module, "_get_recovery_api_key", AsyncMock(return_value="test-key"))
deliver = AsyncMock(return_value=True)
monkeypatch.setattr(svc, "deliver_prepared_workflow_webhook", deliver)
await svc.recover_pending_workflow_attempts(release_only=True)
assert len(cursors) == 3
for run_id in ["wr_retry", *(f"wr_page_{index}" for index in range(4))]:
attempt = (await database.workflow_run_attempts.get_attempts(run_id))[0]
assert attempt.retry_decision == "final"
assert attempt.webhook_sent_at is not None
assert await read_page(datetime.now(UTC)) == []
assert deliver.await_count == 5
@pytest.mark.asyncio
async def test_policy_terminal_winner_survives_late_failed_writer(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.status = "running"
run.finished_at = None
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.status = "running"
attempt.retry_decision = None
attempt.finished_at = None
attempt.next_attempt_at = None
await session.commit()
finalize_attempt = AsyncMock(wraps=database.workflow_run_attempts.finalize_attempt)
monkeypatch.setattr(database.workflow_run_attempts, "finalize_attempt", finalize_attempt)
sync_status = AsyncMock()
monkeypatch.setattr(svc, "_sync_task_run_from_workflow_run", sync_status)
before = set(svc._background_tasks)
completed = await svc.mark_workflow_run_as_completed("wr_retry")
late_failure = await svc.mark_workflow_run_as_failed("wr_retry", failure_reason="late failure")
stored = await svc.get_workflow_run("wr_retry", "org_test")
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
await asyncio.gather(*(svc._background_tasks - before))
assert completed.status == late_failure.status == stored.status == WorkflowRunStatus.completed
assert late_failure.failure_reason == stored.failure_reason is None
assert late_failure.failure_category == stored.failure_category is None
assert late_failure.finished_at == stored.finished_at == completed.finished_at
assert len(attempts) == 1
assert attempts[0].status == "completed"
assert attempts[0].retry_decision == "final"
assert attempts[0].finished_at is not None
assert attempts[0].failure_reason is None
assert attempts[0].next_attempt_at is None
assert attempts[0].next_attempt_prepared_at is None
finalize_attempt.assert_awaited_once()
sync_status.assert_awaited_once()
assert sync_status.await_args.args[2] == WorkflowRunStatus.completed
@pytest.mark.asyncio
@pytest.mark.parametrize("escaped_exception", [False, True])
@pytest.mark.parametrize("final_delivery_succeeds", [False, True])
async def test_in_process_retry_continues_after_interim_delivery_exhaustion(
interim_retry_db: AgentDB,
monkeypatch: pytest.MonkeyPatch,
escaped_exception: bool,
final_delivery_succeeds: bool,
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
events: list[str] = []
deliveries: list[int] = []
delivery_projections: list[tuple[WebhookDeliveryStatus | None, WebhookDeliveryStatus | None]] = []
async def deliver(
_webhook: PreparedWorkflowWebhook,
*,
delivered_projection: WebhookDeliveryStatus | None = None,
exhausted_projection: WebhookDeliveryStatus | None = None,
**_claim: object,
) -> bool:
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
number = attempts[-1].attempt_number
deliveries.append(number)
delivery_projections.append((delivered_projection, exhausted_projection))
return number == 2 and final_delivery_succeeds
async def execute(**kwargs: object) -> WorkflowRun:
number = kwargs["attempt_number"]
events.append(f"execute:{number}")
if number == 1:
if escaped_exception:
raise RuntimeError("attempt failed")
return await svc.get_workflow_run("wr_retry", "org_test")
assert number == 2
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert attempts[0].interim_webhook_sent_at is not None
assert attempts[0].next_attempt_prepared_at is not None
completed = await database.workflow_runs.update_workflow_run_if_not_final(
workflow_run_id="wr_retry", status=WorkflowRunStatus.completed
)
assert completed is not None
await service_module.on_terminal_transition(completed, completed.status, None, None, attempt_number=2)
return completed
async def sleep(seconds: float) -> None:
# The owner sleeps only what is left of the 5 s delay after the interim effects ran.
assert 0 <= seconds <= 5
assert deliveries == [1] * service_module.TERMINAL_RELEASE_RETRY_MAX_ATTEMPTS
events.append("sleep")
monkeypatch.setattr(svc, "execute_workflow", execute)
monkeypatch.setattr(svc, "deliver_prepared_workflow_webhook", deliver)
monkeypatch.setattr(service_module, "asyncio", ScopedAsyncio(sleep=sleep))
result = await svc.execute_workflow_with_retries(
"wr_retry", api_key="test-key", organization=SimpleNamespace(organization_id="org_test")
)
assert result.status == WorkflowRunStatus.completed
assert events == ["execute:1", "sleep", "execute:2"]
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert len(attempts) == 2
assert attempts[0].interim_webhook_sent_at is not None
assert attempts[0].side_effects_released_at is not None
assert attempts[0].interim_side_effects_progress["webhook_delivery_exhausted_at"]
assert not attempts[0].interim_side_effects_progress.get("webhook_delivery_attempted")
assert attempts[0].retry_decision == "retry"
assert attempts[1].retry_decision == "final"
assert attempts[1].webhook_sent_at is not None
assert bool(attempts[1].final_side_effects_progress.get("webhook_delivery_attempted")) == final_delivery_succeeds
assert bool(attempts[1].final_side_effects_progress.get("webhook_delivery_exhausted_at")) != final_delivery_succeeds
assert deliveries.count(2) == (1 if final_delivery_succeeds else service_module.TERMINAL_RELEASE_RETRY_MAX_ATTEMPTS)
expected_final_projections: list[tuple[WebhookDeliveryStatus, WebhookDeliveryStatus | None]] = [
(WebhookDeliveryStatus.delivered, None)
] * deliveries.count(2)
if not final_delivery_succeeds:
expected_final_projections[-1] = (
WebhookDeliveryStatus.delivered,
WebhookDeliveryStatus.exhausted_unattributed,
)
interim_delivery_count = service_module.TERMINAL_RELEASE_RETRY_MAX_ATTEMPTS
assert delivery_projections[:interim_delivery_count] == [(None, None)] * interim_delivery_count
assert delivery_projections[interim_delivery_count:] == expected_final_projections
@pytest.mark.asyncio
@pytest.mark.parametrize("escaped_exception", [False, True])
async def test_in_process_retry_continues_after_interim_payload_failure(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, escaped_exception: bool
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
events: list[str] = []
release_outcomes: list[str] = []
release = svc._run_terminal_side_effects_with_retries
async def record_release(*args: object, **kwargs: object) -> str:
outcome = await release(*args, **kwargs)
release_outcomes.append(outcome)
return outcome
async def prepare(run: WorkflowRun, *args: object, **kwargs: object) -> object:
if run.status == WorkflowRunStatus.failed:
raise RuntimeError("payload unavailable")
return object()
async def execute(**kwargs: object) -> WorkflowRun:
number = kwargs["attempt_number"]
events.append(f"execute:{number}")
if number == 1:
if escaped_exception:
raise RuntimeError("attempt failed")
return await svc.get_workflow_run("wr_retry", "org_test")
assert number == 2
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert attempts[0].next_attempt_prepared_at is not None
completed = await database.workflow_runs.update_workflow_run_if_not_final(
workflow_run_id="wr_retry", status=WorkflowRunStatus.completed
)
assert completed is not None
await service_module.on_terminal_transition(completed, completed.status, None, None, attempt_number=2)
return completed
async def sleep(seconds: float) -> None:
# The owner sleeps only what is left of the 5 s delay after the interim effects ran.
assert 0 <= seconds <= 5
assert release_outcomes == ["effect_failed"]
attempt = (await database.workflow_run_attempts.get_attempts("wr_retry"))[0]
assert attempt.interim_webhook_sent_at is None
assert attempt.side_effects_released_at is None
assert (attempt.interim_side_effects_progress or {}).get("webhook_delivery_attempts", 0) == 0
events.append("sleep")
monkeypatch.setattr(svc, "_run_terminal_side_effects_with_retries", record_release)
monkeypatch.setattr(svc, "prepare_workflow_webhook", prepare)
monkeypatch.setattr(svc, "deliver_prepared_workflow_webhook", AsyncMock(return_value=True))
monkeypatch.setattr(svc, "execute_workflow", execute)
monkeypatch.setattr(service_module, "asyncio", ScopedAsyncio(sleep=sleep))
result = await svc.execute_workflow_with_retries(
"wr_retry", api_key="test-key", organization=SimpleNamespace(organization_id="org_test")
)
assert result.status == WorkflowRunStatus.completed
assert events == ["execute:1", "sleep", "execute:2"]
assert release_outcomes == ["effect_failed", "released"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("artifact_link", "sweep", "current_status", "latest_has_screenshot", "latest_has_recording"),
[
("task", "background", "running", False, False),
("task", "cloud", "completed", True, True),
("step", "background", "running", True, True),
("step", "cloud", "completed", False, False),
("block", "background", "completed", False, True),
("block", "cloud", "running", True, False),
("time", "background", "completed", True, False),
("time", "cloud", "running", False, True),
],
)
@pytest.mark.usefixtures("public_dns")
async def test_recovered_interim_payload_describes_target_attempt(
interim_retry_db: AgentDB,
monkeypatch: pytest.MonkeyPatch,
sweep: str,
current_status: str,
artifact_link: str,
latest_has_screenshot: bool,
latest_has_recording: bool,
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
monkeypatch.setattr(svc, "prepare_workflow_webhook", WorkflowService.prepare_workflow_webhook.__get__(svc))
monkeypatch.setattr(
database.tasks,
"get_tasks_by_workflow_run_id",
type(database.tasks).get_tasks_by_workflow_run_id.__get__(database.tasks),
)
async def links(artifacts: list) -> list[str]:
return [artifact.uri for artifact in artifacts]
monkeypatch.setattr(app.ARTIFACT_MANAGER, "get_share_links_with_bundle_support", links)
monkeypatch.setattr(app.ARTIFACT_MANAGER, "is_recording_archived", AsyncMock(return_value=False))
def attribution(number: str) -> dict[str, str]:
return {
"task": {"task_id": f"task_{number}"},
"step": {"step_id": f"step_{number}"},
"block": {"workflow_run_block_id": f"block_{number}"},
"time": {},
}[artifact_link]
downloads: list[FileInfo] = []
monkeypatch.setattr(app.STORAGE, "get_downloaded_files", AsyncMock(side_effect=lambda **kwargs: list(downloads)))
monkeypatch.setattr(service_module, "_get_recovery_api_key", AsyncMock(return_value="test-key"))
deliver = AsyncMock(return_value=httpx.Response(200))
monkeypatch.setattr(app.AGENT_FUNCTION, "deliver_webhook", deliver)
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.webhook_callback_url = "https://example.com/hook"
run.failure_reason = "first attempt failed"
run.started_at = run.finished_at - timedelta(seconds=10)
run.queued_at = run.started_at - timedelta(seconds=1)
run.failure_category = [{"category": "navigation"}]
run.credits_used = 10
run.cached_credits_used = 2
first = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
first.failure_category = run.failure_category
first.failure_reason = run.failure_reason
first.error_codes = ["FIRST_ERROR"]
first.started_at = run.started_at
session.add(OutputParameterModel(output_parameter_id="op_retry", workflow_id="wf_retry", key="result"))
session.add(
WorkflowRunOutputParameterModel(
workflow_run_id="wr_retry",
output_parameter_id="op_retry",
value={
"value": "first",
"legacy": {
"task_id": "task_first",
"workflow_screenshots": ["old-url"],
},
},
)
)
session.add(
TaskModel(
task_id="task_first",
organization_id="org_test",
workflow_run_id="wr_retry",
status="failed",
url="https://example.com",
attempt_number=None,
)
)
await session.flush()
session.add(
StepModel(
step_id="step_first", task_id="task_first", organization_id="org_test", status="completed", order=0
)
)
session.add(
WorkflowRunBlockModel(
workflow_run_block_id="block_first",
workflow_run_id="wr_retry",
organization_id="org_test",
status="failed",
block_type="task",
attempt_number=None,
)
)
for kind, suffix in [
(ArtifactType.SCREENSHOT_FINAL, "png"),
(ArtifactType.DOWNLOAD, "pdf"),
(ArtifactType.RECORDING, "webm"),
]:
session.add(
ArtifactModel(
artifact_id=f"first_{suffix}",
organization_id="org_test",
run_id="wr_retry",
workflow_run_id="wr_retry",
**attribution("first"),
artifact_type=kind,
uri=(
"https://example.com/run_recordings/attempt1.webm"
if kind == ArtifactType.RECORDING
else f"https://example.com/attempt1.{suffix}"
),
created_at=run.started_at,
modified_at=run.started_at,
)
)
downloads.append(
FileInfo(
url="https://example.com/attempt1.pdf",
filename="attempt1.pdf",
artifact_id="first_pdf",
modified_at=run.started_at,
)
)
await session.commit()
run = await svc.get_workflow_run("wr_retry", "org_test")
original = await svc.prepare_workflow_webhook(run, "test-key")
assert original is not None
expected = json.loads(original.signed_payload)
assert expected["retry_pending"] is True
preparation = await database.workflow_runs.prepare_next_attempt_atomic("wr_retry", "org_test", 1, "failed", None)
assert preparation.status == "inserted"
assert await database.workflow_runs.get_workflow_run_output_parameters("wr_retry") == []
await database.workflow_runs.create_or_update_workflow_run_output_parameter(
"wr_retry", "op_retry", {"value": "second"}
)
async with database.Session() as session:
run_row = await session.get(WorkflowRunModel, "wr_retry")
run_row.status = current_status
run_row.credits_used = 30
run_row.cached_credits_used = 6
run_row.started_at = datetime.now(UTC).replace(tzinfo=None)
run_row.finished_at = run_row.started_at if current_status == "completed" else None
second = await session.get(WorkflowRunAttemptModel, ("wr_retry", 2))
second.status = current_status
second.started_at = run_row.started_at
second.finished_at = run_row.finished_at
second.retry_decision = "final" if current_status == "completed" else None
second.side_effects_released_at = run_row.started_at if current_status == "completed" else None
session.add(
TaskModel(
task_id="task_second",
organization_id="org_test",
workflow_run_id="wr_retry",
status=current_status,
url="https://example.com",
attempt_number=2,
)
)
await session.flush()
session.add(
StepModel(
step_id="step_second", task_id="task_second", organization_id="org_test", status="completed", order=0
)
)
session.add(
WorkflowRunBlockModel(
workflow_run_block_id="block_second",
workflow_run_id="wr_retry",
organization_id="org_test",
status="failed",
block_type="task",
attempt_number=2,
)
)
for kind, suffix in [
(ArtifactType.SCREENSHOT_FINAL, "png"),
(ArtifactType.DOWNLOAD, "pdf"),
(ArtifactType.RECORDING, "webm"),
]:
session.add(
ArtifactModel(
artifact_id=f"second_{suffix}",
organization_id="org_test",
run_id="wr_retry",
workflow_run_id="wr_retry",
**attribution("second"),
artifact_type=kind,
uri=(
"https://example.com/run_recordings/attempt2.webm"
if kind == ArtifactType.RECORDING
else f"https://example.com/attempt2.{suffix}"
),
created_at=run_row.started_at,
modified_at=run_row.started_at,
)
)
if not latest_has_recording:
await session.flush()
recording = await session.get(ArtifactModel, "second_webm")
await session.delete(recording)
if not latest_has_screenshot:
await session.flush()
screenshot = await session.get(ArtifactModel, "second_png")
await session.delete(screenshot)
downloads.extend(
[
FileInfo(
url="https://example.com/attempt2.pdf",
filename="attempt2.pdf",
artifact_id="second_pdf",
modified_at=downloads[0].modified_at,
),
FileInfo(
url="https://example.com/unattributed2.pdf",
filename="unattributed2.pdf",
modified_at=run_row.started_at,
),
]
)
await session.commit()
if sweep == "background":
await BackgroundTaskExecutor()._recover_pending_retries_once(resume_fresh=False)
else:
await svc.recover_pending_workflow_attempts(release_only=True)
assert deliver.await_count == 1
payload = json.loads(deliver.call_args.kwargs["payload"])
compared_fields = [
"attempt",
"status",
"failure_reason",
"failure_category",
"queued_at",
"credits_used",
"cached_credits_used",
"screenshot_urls",
"recording_url",
"recording_urls",
"downloaded_files",
"downloaded_file_urls",
"total_steps",
"step_count",
"started_at",
"finished_at",
"attempts",
"retry_pending",
"next_attempt_at",
"outputs",
"output",
]
mismatched_fields = [field for field in compared_fields if payload[field] != expected[field]]
assert not mismatched_fields, mismatched_fields
assert payload["attempts"][0]["error_codes"] == ["FIRST_ERROR"]
assert payload["screenshot_urls"] == ["https://example.com/attempt1.png"]
assert payload["recording_url"] == "https://example.com/run_recordings/attempt1.webm"
assert payload["step_count"] == 1
assert payload["credits_used"] == 10
monkeypatch.setattr(
app.AGENT_FUNCTION,
"calculate_workflow_run_total_cost",
AsyncMock(side_effect=lambda **kwargs: kwargs["credits_used"] + kwargs["cached_credits_used"]),
)
response = await svc.build_workflow_run_status_response(
"wpid_retry",
"wr_retry",
organization_id="org_test",
target_attempt_number=1,
include_cost=True,
)
assert response.total_cost == 12
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert attempts[0].interim_webhook_sent_at is not None
assert attempts[0].webhook_sent_at is None
assert attempts[1].interim_webhook_sent_at is None
assert attempts[1].webhook_sent_at is None
assert "output_parameters" not in attempts[0].interim_side_effects_progress
historical = (await database.workflow_run_attempts.get_interim_payload_snapshot("wr_retry", 1))["output_parameters"]
assert historical[0]["value"]["value"] == "first"
assert payload["output"]["result"]["legacy"]["workflow_screenshots"] == ["https://example.com/attempt1.png"]
current = await database.workflow_runs.get_workflow_run_output_parameters("wr_retry")
assert [output.value for output in current] == [{"value": "second"}]
async with database.Session() as session:
run_row = await session.get(WorkflowRunModel, "wr_retry")
run_row.status = "completed"
run_row.finished_at = datetime.now(UTC).replace(tzinfo=None)
second = await session.get(WorkflowRunAttemptModel, ("wr_retry", 2))
second.status = "completed"
second.finished_at = run_row.finished_at
second.retry_decision = "final"
second.side_effects_released_at = None
await session.commit()
run = await svc.get_workflow_run("wr_retry", "org_test")
ordinary = await svc.prepare_workflow_webhook(run, "test-key")
targeted = await svc.prepare_workflow_webhook(run, "test-key", target_attempt_number=2)
assert json.loads(ordinary.signed_payload) == json.loads(targeted.signed_payload)
latest = json.loads(ordinary.signed_payload)
expected_screenshots = ["https://example.com/attempt2.png"] if latest_has_screenshot else None
assert latest["screenshot_urls"] == expected_screenshots
response = await svc.build_workflow_run_status_response("wpid_retry", "wr_retry", organization_id="org_test")
assert response.screenshot_urls == expected_screenshots
expected_recordings = ["https://example.com/run_recordings/attempt2.webm"] if latest_has_recording else None
assert response.recording_urls == expected_recordings
assert latest["recording_urls"] == expected_recordings
assert response.recording_url == (expected_recordings[-1] if expected_recordings else None)
assert latest["recording_url"] == response.recording_url
# Steps are attempt-scoped like errors and screenshots; credits accumulate across attempts.
assert latest["step_count"] == 1
assert latest["credits_used"] == 30
decision = RetryDecision(False, 2, 0, True, "no_match")
for expected_outcome in ["released", "already_released"]:
assert await svc._run_terminal_side_effects_with_retries(run, decision, api_key="test-key") == expected_outcome
payloads = [json.loads(call.kwargs["payload"]) for call in deliver.call_args_list]
assert [(item["attempt"], item["status"]) for item in payloads] == [(1, "failed"), (2, "completed")]
assert payloads[1]["output"]["result"] == {"value": "second"}
assert payloads[1]["screenshot_urls"] == expected_screenshots
assert payloads[1]["recording_urls"] == expected_recordings
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert attempts[1].webhook_sent_at is not None
@pytest.mark.asyncio
@pytest.mark.parametrize("has_attempt_row", [False, True])
async def test_single_attempt_preserves_screenshots_and_recordings(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, has_attempt_row: bool
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
start = datetime.now(UTC).replace(tzinfo=None) - timedelta(minutes=10)
async with database.Session() as session:
workflow = await session.get(WorkflowModel, "wf_retry")
workflow.workflow_definition = {"blocks": [], "parameters": []}
run = await session.get(WorkflowRunModel, "wr_retry")
run.webhook_callback_url = "https://example.com/hook"
if not has_attempt_row:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
await session.delete(attempt)
for index in range(2):
session.add(
TaskModel(
task_id=f"legacy_task_{index}",
organization_id="org_test",
workflow_run_id="wr_retry",
status="failed",
url="https://example.com",
attempt_number=None,
created_at=start + timedelta(minutes=index),
)
)
await session.flush()
for index in range(3):
session.add(
ArtifactModel(
artifact_id=f"legacy_png_{index}",
organization_id="org_test",
run_id="wr_retry",
workflow_run_id="wr_retry",
task_id=f"legacy_task_{index}" if index < 2 else None,
artifact_type=ArtifactType.SCREENSHOT_FINAL,
uri=f"https://example.com/legacy{index}.png",
created_at=start + timedelta(minutes=index),
)
)
session.add(
ArtifactModel(
artifact_id="legacy_webm",
organization_id="org_test",
run_id="wr_retry",
workflow_run_id="wr_retry",
artifact_type=ArtifactType.RECORDING,
uri="https://example.com/run_recordings/legacy.webm",
created_at=start,
)
)
await session.commit()
monkeypatch.setattr(
database.tasks,
"get_tasks_by_workflow_run_id",
type(database.tasks).get_tasks_by_workflow_run_id.__get__(database.tasks),
)
monkeypatch.setattr(app.STORAGE, "get_downloaded_files", AsyncMock(return_value=[]))
monkeypatch.setattr(
app.ARTIFACT_MANAGER,
"get_share_links_with_bundle_support",
AsyncMock(side_effect=lambda artifacts: [artifact.uri for artifact in artifacts]),
)
monkeypatch.setattr(app.ARTIFACT_MANAGER, "is_recording_archived", AsyncMock(return_value=False))
expected = [f"https://example.com/legacy{index}.png" for index in (1, 0, 2)]
response = await svc.build_workflow_run_status_response("wpid_retry", "wr_retry", organization_id="org_test")
assert response.screenshot_urls == expected
assert response.recording_urls == ["https://example.com/run_recordings/legacy.webm"]
payload = await WorkflowService.prepare_workflow_webhook(
svc, await svc.get_workflow_run("wr_retry", "org_test"), "test-key"
)
assert json.loads(payload.signed_payload)["screenshot_urls"] == expected
assert json.loads(payload.signed_payload)["recording_urls"] == response.recording_urls
@pytest.mark.asyncio
@pytest.mark.parametrize("interim_complete", [False, True])
@pytest.mark.parametrize("null_progress", [False, True])
async def test_interim_snapshot_is_bounded_and_survives_completed_delivery(
interim_retry_db: AgentDB,
interim_complete: bool,
null_progress: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
database = interim_retry_db
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.interim_webhook_sent_at = datetime.now(UTC).replace(tzinfo=None) if interim_complete else None
attempt.interim_side_effects_progress = (
None
if null_progress
else {
"completed_effects": ["cleanup"],
"webhook_delivery_attempts": 1,
}
)
for index in range(3):
session.add(
OutputParameterModel(output_parameter_id=f"large_{index}", workflow_id="wf_retry", key=f"large_{index}")
)
session.add(
WorkflowRunOutputParameterModel(
workflow_run_id="wr_retry",
output_parameter_id=f"large_{index}",
value="x" * (INTERIM_OUTPUT_SNAPSHOT_MAX_BYTES // 2 + index),
)
)
await session.flush()
if null_progress:
await session.execute(
update(WorkflowRunAttemptModel)
.where(
WorkflowRunAttemptModel.workflow_run_id == "wr_retry",
)
.values(interim_side_effects_progress=literal("null"))
)
await session.commit()
result = await database.workflow_runs.prepare_next_attempt_atomic("wr_retry", "org_test", 1, "failed", None)
assert result.status == "inserted"
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
progress = attempt.interim_side_effects_progress or {}
if not null_progress:
assert progress["completed_effects"] == ["cleanup"]
assert progress["webhook_delivery_attempts"] == 1
# A completed interim delivery no longer skips the snapshot: the historical response needs it.
entries = progress["output_parameters"]
assert len(json.dumps(entries).encode("utf-8")) <= INTERIM_OUTPUT_SNAPSHOT_MAX_BYTES
assert [entry["output_parameter_id"] for entry in entries] == [f"large_{index}" for index in range(3)]
assert all(entry["created_at"] for entry in entries)
assert entries[1]["value"]["truncated"] is True
assert entries[2]["value"]["truncated"] is True
metadata = await database.workflow_run_attempts.get_attempts("wr_retry")
assert "output_parameters" not in metadata[0].interim_side_effects_progress
monkeypatch.setattr(app.STORAGE, "get_downloaded_files", AsyncMock(return_value=[]))
response = await app.WORKFLOW_SERVICE.build_workflow_run_status_response(
"wpid_retry",
"wr_retry",
organization_id="org_test",
target_attempt_number=1,
)
assert response.outputs["large_2"]["truncated"] is True
@pytest.mark.asyncio
@pytest.mark.parametrize("artifact_link", ["task", "step", "block"])
@pytest.mark.parametrize("ancestry", ["block", "chain", "fallback"])
@pytest.mark.parametrize("child_attempt", [None, 1])
async def test_historical_child_artifacts_use_parent_attempt(
interim_retry_db: AgentDB,
monkeypatch: pytest.MonkeyPatch,
artifact_link: str,
ancestry: str,
child_attempt: int | None,
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
start = datetime.now(UTC).replace(tzinfo=None) - timedelta(minutes=30)
downloads = []
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.webhook_callback_url = "https://example.com/hook"
for number in (1, 2):
when = start + timedelta(minutes=10 * (number - 1))
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", number))
if attempt is None:
attempt = WorkflowRunAttemptModel(
workflow_run_id="wr_retry", organization_id="org_test", attempt_number=number
)
session.add(attempt)
attempt.status = "failed"
attempt.started_at = when
attempt.finished_at = when + timedelta(minutes=1)
attempt.retry_decision = "retry"
attempt.interim_side_effects_progress = {
"credits_used": 0,
"cached_credits_used": 0,
"output_parameters": [
{
"output_parameter_id": "op_children",
"created_at": when.isoformat(),
"value": {
"workflow_screenshot_artifact_ids": ["child_1_png", "child_2_png"],
"downloaded_file_artifact_ids": ["child_1_pdf", "child_2_pdf"],
},
}
],
}
enclosing_id = f"wr_enclosing_{number}"
child_id = f"wr_child_{number}"
if ancestry == "chain":
session.add(
WorkflowRunModel(
workflow_run_id=enclosing_id,
workflow_permanent_id="wpid_retry",
workflow_id="wf_retry",
organization_id="org_test",
parent_workflow_run_id="wr_retry",
status="completed",
)
)
session.add(
WorkflowRunModel(
workflow_run_id=child_id,
workflow_permanent_id="wpid_retry",
workflow_id="wf_retry",
organization_id="org_test",
parent_workflow_run_id=enclosing_id if ancestry == "chain" else "wr_retry",
status="completed",
)
)
if ancestry != "fallback":
session.add(
WorkflowRunBlockModel(
workflow_run_block_id=f"spawn_{number}",
workflow_run_id="wr_retry",
block_workflow_run_id=enclosing_id if ancestry == "chain" else child_id,
organization_id="org_test",
block_type="task_v2",
status="completed",
attempt_number=number,
)
)
await session.flush()
session.add(
TaskModel(
task_id=f"child_task_{number}",
workflow_run_id=child_id,
organization_id="org_test",
status="completed",
url="https://example.com",
attempt_number=child_attempt,
)
)
await session.flush()
session.add(
StepModel(
step_id=f"child_step_{number}",
task_id=f"child_task_{number}",
organization_id="org_test",
status="completed",
order=0,
)
)
session.add(
WorkflowRunBlockModel(
workflow_run_block_id=f"child_block_{number}",
workflow_run_id=child_id,
organization_id="org_test",
block_type="task",
status="completed",
attempt_number=child_attempt,
)
)
link = {
"task": {"task_id": f"child_task_{number}"},
"step": {"step_id": f"child_step_{number}"},
"block": {"workflow_run_block_id": f"child_block_{number}"},
}[artifact_link]
# Late uploads retain the spawning attempt when an ancestor block is available.
uploaded_at = when if ancestry == "fallback" else start + timedelta(minutes=25)
for kind, suffix in [
(ArtifactType.DOWNLOAD, "pdf"),
(ArtifactType.SCREENSHOT_FINAL, "png"),
(ArtifactType.RECORDING, "webm"),
]:
artifact_id = f"child_{number}_{suffix}"
url = f"https://example.com/{artifact_id}"
session.add(
ArtifactModel(
artifact_id=artifact_id,
organization_id="org_test",
run_id="wr_retry",
workflow_run_id=child_id,
artifact_type=kind,
uri=url,
created_at=uploaded_at,
modified_at=uploaded_at,
**link,
)
)
if suffix == "pdf":
downloads.append(
FileInfo(url=url, filename=artifact_id, artifact_id=artifact_id, modified_at=uploaded_at)
)
session.add(OutputParameterModel(output_parameter_id="op_children", workflow_id="wf_retry", key="children"))
session.add(
WorkflowRunAttemptModel(
workflow_run_id="wr_retry",
organization_id="org_test",
attempt_number=3,
status="running",
started_at=start + timedelta(minutes=20),
)
)
await session.commit()
monkeypatch.setattr(app.STORAGE, "get_downloaded_files", AsyncMock(return_value=downloads))
monkeypatch.setattr(
app.ARTIFACT_MANAGER,
"get_share_links_with_bundle_support",
AsyncMock(side_effect=lambda artifacts: [a.uri for a in artifacts]),
)
monkeypatch.setattr(
app.ARTIFACT_MANAGER, "resolve_share_url", AsyncMock(side_effect=lambda artifact, **kwargs: artifact.uri)
)
monkeypatch.setattr(app.ARTIFACT_MANAGER, "resolve_artifact_url_expiry_seconds", AsyncMock(return_value=3600))
monkeypatch.setattr(app.ARTIFACT_MANAGER, "is_recording_archived", AsyncMock(return_value=False))
lookup = AsyncMock(wraps=database.workflow_runs.get_artifacts_for_attempt)
monkeypatch.setattr(database.workflow_runs, "get_artifacts_for_attempt", lookup)
for number in (1, 2):
payload = await WorkflowService.prepare_workflow_webhook(
svc, await svc.get_workflow_run("wr_retry", "org_test"), "test-key", target_attempt_number=number
)
body = json.loads(payload.signed_payload)
assert body["screenshot_urls"] == [f"https://example.com/child_{number}_png"]
assert body["recording_url"] == f"https://example.com/child_{number}_webm"
assert body["downloaded_file_urls"] == [f"https://example.com/child_{number}_pdf"]
assert body["outputs"]["children"]["downloaded_file_urls"] == [f"https://example.com/child_{number}_pdf"]
assert lookup.await_count == number
artifacts = await database.workflow_runs.get_artifacts_for_attempt(
"wr_retry", "org_test", 1, start, start + timedelta(minutes=10)
)
legacy = await svc._refresh_output_urls(
{
"workflow_screenshot_artifact_ids": [],
"downloaded_files": [
{"filename": "child_1_pdf"},
{"filename": "child_2_pdf"},
],
},
"org_test",
"wr_retry",
attempt_rows=await database.workflow_run_attempts.get_attempts("wr_retry"),
attempt_number=1,
historical_artifacts=artifacts,
)
assert legacy["downloaded_file_urls"] == ["https://example.com/child_1_pdf"]
assert lookup.await_count == 3
async with database.Session() as session:
await session.execute(
update(ArtifactModel)
.where(ArtifactModel.artifact_type == ArtifactType.DOWNLOAD)
.values(run_id="observer_storage")
)
await session.commit()
monkeypatch.setattr(
app.STORAGE,
"get_downloaded_files",
AsyncMock(side_effect=lambda **kwargs: downloads if kwargs["run_id"] == "observer_storage" else []),
)
files, urls = await svc._fetch_downloaded_files(
await svc.get_workflow_run("wr_retry", "org_test"),
SimpleNamespace(observer_cruise_id="observer_storage"),
attempt_rows=await database.workflow_run_attempts.get_attempts("wr_retry"),
attempt_number=1,
historical_artifacts=[],
)
assert files is not None
assert [file.artifact_id for file in files] == ["child_1_pdf"]
assert urls == ["https://example.com/child_1_pdf"]
assert lookup.await_count == 4
assert lookup.call_args.args[0] == "wr_retry"
assert lookup.call_args.kwargs == {"artifact_run_id": "observer_storage"}
@pytest.mark.asyncio
@pytest.mark.parametrize("usage", ["null_columns", "missing_keys", "null_keys"])
async def test_historical_usage_defaults_to_zero(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, usage: str
) -> None:
database = interim_retry_db
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.credits_used = None
run.cached_credits_used = None
run.webhook_callback_url = "https://example.com/hook"
await session.commit()
result = await database.workflow_runs.prepare_next_attempt_atomic("wr_retry", "org_test", 1, "failed", None)
assert result.status == "inserted"
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.credits_used = 91
run.cached_credits_used = 17
if usage != "null_columns":
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.interim_side_effects_progress = (
{"output_parameters": []}
if usage == "missing_keys"
else {"output_parameters": [], "credits_used": None, "cached_credits_used": None}
)
await session.commit()
monkeypatch.setattr(app.STORAGE, "get_downloaded_files", AsyncMock(return_value=[]))
payload = await WorkflowService.prepare_workflow_webhook(
app.WORKFLOW_SERVICE,
await app.WORKFLOW_SERVICE.get_workflow_run("wr_retry", "org_test"),
"test-key",
target_attempt_number=1,
)
body = json.loads(payload.signed_payload)
assert (body["credits_used"], body["cached_credits_used"]) == (0, 0)
if usage == "null_columns":
snapshot = await database.workflow_run_attempts.get_interim_payload_snapshot("wr_retry", 1)
assert (snapshot["credits_used"], snapshot["cached_credits_used"]) == (0, 0)
@pytest.mark.asyncio
async def test_interim_snapshot_metadata_overflow_keeps_retry_and_reports_omission(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
database = interim_retry_db
count = 10000
created_at = datetime.now(UTC).replace(tzinfo=None)
ids = [f"op_metadata_overflow_{index:08d}" for index in range(count)]
async with database.Session() as session:
await session.execute(
insert(OutputParameterModel),
[{"output_parameter_id": output_id, "workflow_id": "wf_retry", "key": output_id} for output_id in ids],
)
await session.execute(
insert(WorkflowRunOutputParameterModel),
[
{
"workflow_run_id": "wr_retry",
"output_parameter_id": output_id,
"value": None,
"created_at": created_at + timedelta(microseconds=index),
}
for index, output_id in enumerate(ids)
],
)
run = await session.get(WorkflowRunModel, "wr_retry")
run.webhook_callback_url = "https://example.com/hook"
await session.commit()
result = await database.workflow_runs.prepare_next_attempt_atomic("wr_retry", "org_test", 1, "failed", None)
assert result.status == "inserted"
assert len(await database.workflow_run_attempts.get_attempts("wr_retry")) == 2
snapshot = await database.workflow_run_attempts.get_interim_payload_snapshot("wr_retry", 1)
entries = snapshot["output_parameters"]
assert 0 < len(entries) < count
assert snapshot["omitted_output_count"] == count - len(entries)
assert (
len(json.dumps({key: snapshot[key] for key in ("output_parameters", "omitted_output_count")}).encode())
<= INTERIM_OUTPUT_SNAPSHOT_MAX_BYTES
)
assert [entry["output_parameter_id"] for entry in entries] == ids[: len(entries)]
monkeypatch.setattr(app.STORAGE, "get_downloaded_files", AsyncMock(return_value=[]))
payload = await WorkflowService.prepare_workflow_webhook(
app.WORKFLOW_SERVICE,
await app.WORKFLOW_SERVICE.get_workflow_run("wr_retry", "org_test"),
"test-key",
target_attempt_number=1,
)
body = json.loads(payload.signed_payload)
assert body["outputs"]["_interim_output_snapshot"] == {
"truncated": True,
"reason": "interim_output_snapshot_budget_exceeded",
"omitted_output_count": count - len(entries),
}
assert body["output"] == body["outputs"]
@pytest.mark.asyncio
@pytest.mark.parametrize("target_attempt_number", [1, None])
async def test_attempt_recording_uses_its_shared_session(
interim_retry_db: AgentDB,
monkeypatch: pytest.MonkeyPatch,
target_attempt_number: int | None,
) -> None:
database = interim_retry_db
now = datetime.now(UTC).replace(tzinfo=None)
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.browser_session_id = "session_first"
run.started_at = now - timedelta(seconds=30)
first = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
first.started_at = run.started_at
await session.commit()
result = await database.workflow_runs.prepare_next_attempt_atomic(
"wr_retry",
"org_test",
1,
"failed",
"session_second",
)
assert result.status == "inserted"
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
run.started_at = now
second = await session.get(WorkflowRunAttemptModel, ("wr_retry", 2))
second.started_at = now
await session.commit()
expected_session = "session_first" if target_attempt_number == 1 else "session_second"
async def shared_recordings(*, organization_id: str, browser_session_id: str) -> list[FileInfo]:
assert browser_session_id == expected_session
return [
FileInfo(
url="https://example.com/shared.webm", filename="shared.webm", modified_at=now + timedelta(hours=1)
)
]
monkeypatch.setattr(app.STORAGE, "get_shared_recordings_in_browser_session", shared_recordings)
monkeypatch.setattr(app.STORAGE, "get_downloaded_files", AsyncMock(return_value=[]))
response = await app.WORKFLOW_SERVICE.build_workflow_run_status_response(
"wpid_retry",
"wr_retry",
organization_id="org_test",
target_attempt_number=target_attempt_number,
)
assert response.recording_url == "https://example.com/shared.webm"
assert response.browser_session_id == expected_session
@pytest.mark.asyncio
@pytest.mark.parametrize("sweep", ["background", "cloud"])
@pytest.mark.parametrize("lease", ["none", "stale", "young"])
async def test_recovery_releases_earlier_interim_without_abandoning_retry(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, sweep: str, lease: str
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
now = datetime.now(UTC).replace(tzinfo=None)
cutoff = now - timedelta(seconds=service_module.LEASE_TAKEOVER_SECONDS)
old = cutoff - timedelta(seconds=1)
preparation = await database.workflow_runs.prepare_next_attempt_atomic("wr_retry", "org_test", 1, "failed", None)
assert preparation.status == "inserted"
async with database.Session() as session:
attempt = await session.scalar(
select(WorkflowRunAttemptModel).where(WorkflowRunAttemptModel.attempt_number == 1)
)
attempt.next_attempt_at = old
attempt.side_effects_released_at = {"none": None, "stale": old, "young": now}[lease]
await session.commit()
assert [a.attempt_number for a in await database.workflow_run_attempts.list_attempts_needing_recovery(cutoff)] == (
[] if lease == "young" else [1]
)
delivery = AsyncMock(return_value=False)
monkeypatch.setattr(svc, "deliver_prepared_workflow_webhook", delivery)
monkeypatch.setattr(service_module, "_get_recovery_api_key", AsyncMock(return_value="test-key"))
for succeeds in [False, True]:
delivery.return_value = succeeds
if sweep == "background":
await BackgroundTaskExecutor()._recover_pending_retries_once(resume_fresh=False)
else:
await svc.recover_pending_workflow_attempts(release_only=True)
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert [a.retry_decision for a in attempts] == ["retry", None]
assert (attempts[0].interim_webhook_sent_at is not None) == (lease != "young")
if lease != "young":
assert attempts[0].interim_side_effects_progress["webhook_delivery_exhausted_at"]
assert not attempts[0].interim_side_effects_progress.get("webhook_delivery_attempted")
assert attempts[0].webhook_sent_at is None
assert attempts[1].started_at is None
assert delivery.await_count == (0 if lease == "young" else service_module.WORKFLOW_WEBHOOK_DELIVERY_MAX_ATTEMPTS)
assert await database.workflow_run_attempts.list_attempts_needing_recovery(cutoff) == []
@pytest.mark.asyncio
@pytest.mark.parametrize("sweep", ["background", "cloud"])
@pytest.mark.parametrize("run_terminal", [True, False])
async def test_recovery_finalizes_never_started_attempts_of_terminal_runs(
monkeypatch: pytest.MonkeyPatch, sweep: str, run_terminal: bool
) -> None:
# A cancel before mark_attempt_started leaves a created row with no started_at; the terminal run
# still owes its cleanup and webhook. A queued run with the same row is not terminal and stays put.
old = datetime.now(UTC).replace(tzinfo=None) - timedelta(seconds=service_module.LEASE_TAKEOVER_SECONDS + 1)
attempt = SimpleNamespace(
workflow_run_id="wr_never_started",
organization_id="org_test",
attempt_number=1,
status="created",
retry_decision=None,
started_at=None,
next_attempt_at=None,
next_attempt_prepared_at=None,
modified_at=old,
)
run = SimpleNamespace(
workflow_run_id="wr_never_started",
organization_id="org_test",
status=WorkflowRunStatus.canceled if run_terminal else WorkflowRunStatus.queued,
)
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=AsyncMock(side_effect=[[attempt], []]),
list_stale_dispatch_claims=AsyncMock(return_value=[]),
),
workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(return_value=run)),
),
)
recover = AsyncMock(return_value=None)
monkeypatch.setattr(service_module, "_recover_undecided_terminal_attempt", recover)
schedule_resume = MagicMock()
if sweep == "background":
executor = BackgroundTaskExecutor()
executor._schedule_retry_resume = schedule_resume # type: ignore[method-assign]
await executor._recover_pending_retries_once(resume_fresh=False)
else:
await app.WORKFLOW_SERVICE.recover_pending_workflow_attempts(release_only=True)
if run_terminal:
recover.assert_awaited_once_with(attempt, workflow_run=run)
else:
recover.assert_not_awaited()
schedule_resume.assert_not_called()
@pytest.mark.asyncio
async def test_undecided_recovery_reuses_the_run_the_sweep_loaded(monkeypatch: pytest.MonkeyPatch) -> None:
# workflow_runs is high-traffic: the sweep's finality read is the only read per candidate.
attempt = SimpleNamespace(workflow_run_id="wr_loaded", organization_id="org_test", attempt_number=1)
run = SimpleNamespace(
workflow_run_id="wr_loaded",
organization_id="org_test",
status=WorkflowRunStatus.canceled,
failure_reason=None,
failure_category=None,
)
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(side_effect=AssertionError("re-read"))),
),
)
monkeypatch.setattr(
app, "AGENT_FUNCTION", SimpleNamespace(get_workflow_run_execution_status=AsyncMock(return_value="running"))
)
assert await service_module._recover_undecided_terminal_attempt(attempt, workflow_run=run) is None
@pytest.mark.asyncio
async def test_schedule_runs_work_without_background_tasks() -> None:
"""Without a FastAPI BackgroundTasks the work used to be dropped silently."""
ran = asyncio.Event()
async def work(value: str, *, keyword: str) -> None:
assert value == "positional"
assert keyword == "keyword"
ran.set()
BackgroundTaskExecutor()._schedule(None, work, "positional", keyword="keyword")
await asyncio.wait_for(ran.wait(), timeout=1)
@pytest.mark.asyncio
async def test_schedule_defers_to_background_tasks_when_present() -> None:
calls: list[tuple[str, str]] = []
async def work(value: str, *, keyword: str) -> None:
calls.append((value, keyword))
background_tasks = BackgroundTasks()
BackgroundTaskExecutor()._schedule(background_tasks, work, "positional", keyword="keyword")
# Queued on the request's BackgroundTasks rather than started eagerly.
assert calls == []
await background_tasks()
assert calls == [("positional", "keyword")]
@pytest.mark.asyncio
async def test_recover_pending_retries_abandons_only_stale_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
stale_attempt = SimpleNamespace(
workflow_run_id="wr_stale",
organization_id="org_test",
attempt_number=1,
retry_decision="retry",
next_attempt_at=datetime.now(UTC) - timedelta(seconds=RETRY_DECISION_GRACE_SECONDS + 1),
next_attempt_prepared_at=None,
)
fresh_attempt = SimpleNamespace(workflow_run_id="wr_fresh", organization_id="org_test", attempt_number=1)
stale_run = SimpleNamespace(
workflow_run_id="wr_stale",
organization_id="org_test",
status=WorkflowRunStatus.failed,
failure_reason="failed",
finished_at=SimpleNamespace(),
)
list_recovery = AsyncMock(return_value=[stale_attempt])
get_workflow_run = AsyncMock(return_value=stale_run)
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=list_recovery, list_stale_dispatch_claims=AsyncMock(return_value=[])
),
workflow_runs=SimpleNamespace(get_workflow_run=get_workflow_run),
),
)
abandoned_decision = RetryDecision(False, 1, 0, True, "process_restart")
finalize_abandoned = AsyncMock(return_value=abandoned_decision)
monkeypatch.setattr(
"skyvern.forge.sdk.executor.background_task_executor.finalize_abandoned_attempt",
finalize_abandoned,
)
side_effects = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", side_effects)
monkeypatch.setattr(BackgroundTaskExecutor, "_start_retry_recovery_sweep", MagicMock())
await BackgroundTaskExecutor().recover_pending_retries()
list_recovery.assert_awaited_once()
assert get_workflow_run.await_args.kwargs["workflow_run_id"] == stale_attempt.workflow_run_id
assert fresh_attempt.workflow_run_id not in {
call.kwargs["workflow_run_id"] for call in get_workflow_run.await_args_list
}
finalize_abandoned.assert_awaited_once_with(
workflow_run_id="wr_stale",
organization_id="org_test",
reason="process_restart",
status=WorkflowRunStatus.failed,
failure_reason="failed",
finished_at=stale_run.finished_at,
attempt_number=1,
)
side_effects.assert_awaited_once_with(stale_run, abandoned_decision)
@pytest.mark.asyncio
async def test_recovery_sweep_does_not_start_final_effects_for_a_young_retry_lease(
monkeypatch: pytest.MonkeyPatch,
) -> None:
stale_attempt = SimpleNamespace(
workflow_run_id="wr_young_lease",
organization_id="org_test",
attempt_number=1,
retry_decision="retry",
next_attempt_at=datetime.now(UTC) - timedelta(seconds=RETRY_DECISION_GRACE_SECONDS + 1),
next_attempt_prepared_at=None,
side_effects_released_at=datetime.now(UTC),
)
stale_run = SimpleNamespace(
workflow_run_id="wr_young_lease",
organization_id="org_test",
status=WorkflowRunStatus.failed,
failure_reason="failed",
finished_at=SimpleNamespace(),
)
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=AsyncMock(return_value=[stale_attempt]),
list_stale_dispatch_claims=AsyncMock(return_value=[]),
),
workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(return_value=stale_run)),
),
)
skipped_decision = RetryDecision(True, 1, 0, True, "stale_retry_lease_active")
finalize_abandoned = AsyncMock(return_value=skipped_decision)
monkeypatch.setattr(
"skyvern.forge.sdk.executor.background_task_executor.finalize_abandoned_attempt",
finalize_abandoned,
)
side_effects = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", side_effects)
await BackgroundTaskExecutor()._recover_pending_retries_once(resume_fresh=False)
finalize_abandoned.assert_awaited_once()
side_effects.assert_not_awaited()
@pytest.mark.asyncio
async def test_recover_pending_retries_starts_the_sweep_when_the_initial_pass_raises(
monkeypatch: pytest.MonkeyPatch,
) -> None:
executor = BackgroundTaskExecutor()
passes: list[bool] = []
outcomes = iter([RuntimeError("database unavailable"), RuntimeError("database unavailable"), None, None])
async def recover_once(*, resume_fresh: bool) -> None:
passes.append(resume_fresh)
outcome = next(outcomes)
if outcome is not None:
raise outcome
ticks = asyncio.Semaphore(0)
async def controlled_sleep(_seconds: float) -> None:
await ticks.acquire()
async def wait_for_pass(count: int) -> None:
while len(passes) < count:
await asyncio.sleep(0)
monkeypatch.setattr(executor, "_recover_pending_retries_once", recover_once)
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=controlled_sleep))
with pytest.raises(RuntimeError, match="database unavailable"):
await executor.recover_pending_retries()
sweep_task = executor._retry_recovery_sweep_task
assert sweep_task is not None and not sweep_task.done()
try:
for count in range(2, 5):
ticks.release()
await asyncio.wait_for(wait_for_pass(count), timeout=1)
finally:
await executor.stop_retry_recovery()
# The sweep keeps resuming fresh rows until one pass succeeds, then leaves them to in-process owners.
assert passes == [True, True, True, False]
@pytest.mark.asyncio
async def test_stop_retry_recovery_cancels_the_sweep_and_scheduled_resumes(monkeypatch: pytest.MonkeyPatch) -> None:
executor = BackgroundTaskExecutor()
monkeypatch.setattr(executor, "_recover_pending_retries_once", AsyncMock())
async def wait_for_next_attempt(_attempt: object) -> None:
await asyncio.Event().wait()
monkeypatch.setattr(executor, "_resume_pending_retry", wait_for_next_attempt)
await executor.recover_pending_retries()
executor._schedule_retry_resume(SimpleNamespace(workflow_run_id="wr_retry", attempt_number=1))
sweep_task = executor._retry_recovery_sweep_task
assert sweep_task is not None and not sweep_task.done()
(resume_task,) = executor._retry_resume_tasks
assert not resume_task.done()
await executor.stop_retry_recovery()
assert sweep_task.cancelled()
assert resume_task.cancelled()
assert executor._retry_recovery_sweep_task is None
assert not executor._retry_resume_tasks
await executor.stop_retry_recovery()
@pytest.mark.asyncio
async def test_recover_pending_retries_resumes_fresh_attempt_after_delay(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = datetime.now(UTC)
pending_attempt = SimpleNamespace(
workflow_run_id="wr_fresh",
organization_id="org_test",
attempt_number=1,
retry_decision="retry",
next_attempt_at=now + timedelta(seconds=5),
next_attempt_prepared_at=None,
)
organization = SimpleNamespace(organization_id="org_test")
sleep_started = asyncio.Event()
release_sleep = asyncio.Event()
execution_done = asyncio.Event()
async def controlled_sleep(delay_seconds: float) -> None:
assert delay_seconds > 0
sleep_started.set()
await release_sleep.wait()
async def execute_workflow_with_retries(**kwargs: object) -> None:
assert kwargs["attempt_number"] == 2
assert kwargs["browser_session_id"] == "pinned-session"
assert kwargs["api_key"] == "api-token"
execution_done.set()
prepare = AsyncMock(
return_value=PrepareNextAttemptResult(
status="inserted",
pinned_browser_session_id="pinned-session",
)
)
get_valid_org_auth_token = AsyncMock(return_value=SimpleNamespace(token="api-token"))
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=AsyncMock(return_value=[pending_attempt]),
list_stale_dispatch_claims=AsyncMock(return_value=[]),
),
organizations=SimpleNamespace(
get_organization=AsyncMock(return_value=organization),
get_valid_org_auth_token=get_valid_org_auth_token,
),
),
)
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=controlled_sleep))
monkeypatch.setattr(background_task_executor_module, "prepare_next_attempt_result", prepare)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
monkeypatch.setattr(
app.WORKFLOW_SERVICE,
"execute_workflow_with_retries",
execute_workflow_with_retries,
)
monkeypatch.setattr(BackgroundTaskExecutor, "_start_retry_recovery_sweep", MagicMock())
executor = BackgroundTaskExecutor()
await executor.recover_pending_retries()
await asyncio.wait_for(sleep_started.wait(), timeout=1)
prepare.assert_not_awaited()
release_sleep.set()
await asyncio.wait_for(execution_done.wait(), timeout=1)
get_valid_org_auth_token.assert_awaited_once_with(
organization_id="org_test",
token_type="api",
)
prepare.assert_awaited_once_with(
workflow_run_id="wr_fresh",
organization_id="org_test",
from_attempt=1,
clear_browser_address=False,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"lookup, failure",
[
("token", "exception"),
("token", "none"),
("organization", "exception"),
("organization", "none"),
("initialization", "exception"),
],
)
async def test_startup_retry_recovers_transient_initialization_failure_without_another_sweep(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, lookup: str, failure: str
) -> None:
database = interim_retry_db
now = datetime.now(UTC).replace(tzinfo=None)
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.next_attempt_at = now
attempt.interim_webhook_sent_at = now
await session.commit()
organization = await database.organizations.get_organization("org_test")
token = SimpleNamespace(token="api-token")
error = RuntimeError("temporary lookup failure") if failure == "exception" else None
monkeypatch.setattr(
database.organizations,
"get_organization",
AsyncMock(side_effect=[error, organization] if lookup == "organization" else None, return_value=organization),
)
monkeypatch.setattr(
database.organizations,
"get_valid_org_auth_token",
AsyncMock(side_effect=[error, token] if lookup == "token" else None, return_value=token),
)
monkeypatch.setattr(
background_task_executor_module,
"initialize_skyvern_state_file",
AsyncMock(side_effect=[error, None, None] if lookup == "initialization" else None),
)
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
sleeps: list[float] = []
async def sleep(seconds: float) -> None:
sleeps.append(seconds)
assert 0 < seconds <= background_task_executor_module.LEASE_TAKEOVER_SECONDS
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert len(attempts) == 1
assert attempts[0].next_attempt_prepared_at is None
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=sleep))
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
executor = BackgroundTaskExecutor()
await executor._recover_pending_retries_once(resume_fresh=True)
await asyncio.gather(*executor._background_tasks)
assert sleeps
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert [row.attempt_number for row in attempts] == [1, 2]
assert attempts[0].next_attempt_prepared_at is not None
execute.assert_awaited_once()
assert execute.await_args.kwargs["attempt_number"] == 2
assert execute.await_args.kwargs["api_key"] == "api-token"
assert not executor._scheduled_retry_resumes
@pytest.mark.asyncio
@pytest.mark.parametrize("cancelled", [False, True])
async def test_startup_retry_backoff_preserves_sweep_recovery_or_cancellation(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, cancelled: bool
) -> None:
database = interim_retry_db
now = datetime.now(UTC).replace(tzinfo=None)
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.next_attempt_at = now
attempt.interim_webhook_sent_at = now
await session.commit()
monkeypatch.setattr(background_task_executor_module, "naive_utc_now", lambda: now)
token = AsyncMock(side_effect=RuntimeError("database unavailable"))
monkeypatch.setattr(database.organizations, "get_valid_org_auth_token", token)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
executor = BackgroundTaskExecutor()
sleeps: list[float] = []
async def sleep(seconds: float) -> None:
nonlocal now
sleeps.append(seconds)
if cancelled:
raise asyncio.CancelledError
now += timedelta(seconds=seconds)
await executor._recover_pending_retries_once(resume_fresh=False)
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert len(attempts) == 1
assert attempts[0].retry_decision == "retry"
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=sleep))
await executor._recover_pending_retries_once(resume_fresh=True)
if cancelled:
with pytest.raises(asyncio.CancelledError):
await asyncio.gather(*executor._background_tasks)
else:
await asyncio.gather(*executor._background_tasks)
execute.assert_not_awaited()
assert not executor._scheduled_retry_resumes
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert len(attempts) == 1
assert attempts[0].retry_decision == "retry"
if cancelled:
assert not executor._retry_resumes_needing_recovery
return
assert sum(sleeps) == background_task_executor_module.LEASE_TAKEOVER_SECONDS
assert max(sleeps) <= 30
assert ("wr_retry", 1) in executor._retry_resumes_needing_recovery
now += timedelta(seconds=1)
token.side_effect = None
token.return_value = SimpleNamespace(token="api-token")
await executor._recover_pending_retries_once(resume_fresh=False)
await asyncio.gather(*executor._background_tasks)
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert [row.attempt_number for row in attempts] == [1, 2]
execute.assert_awaited_once()
assert execute.await_args.kwargs["attempt_number"] == 2
assert not executor._retry_resumes_needing_recovery
@pytest.mark.asyncio
@pytest.mark.usefixtures("public_dns")
async def test_startup_retry_with_definitively_missing_token_remains_abandonable(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
database = interim_retry_db
now = datetime.now(UTC).replace(tzinfo=None)
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.next_attempt_at = now
await session.commit()
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
executor = BackgroundTaskExecutor()
await executor._recover_pending_retries_once(resume_fresh=True)
await asyncio.gather(*executor._background_tasks)
assert not executor._scheduled_retry_resumes
execute.assert_not_awaited()
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.next_attempt_at = now - timedelta(seconds=background_task_executor_module.LEASE_TAKEOVER_SECONDS + 1)
await session.commit()
await executor._recover_pending_retries_once(resume_fresh=False)
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
assert len(attempts) == 1
assert attempts[0].retry_decision == "abandoned"
execute.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("dependency", ["token", "organization"])
@pytest.mark.parametrize("transient", [False, True])
async def test_prepared_retry_missing_dependency_abandons_or_recovers_on_next_sweep(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, dependency: str, transient: bool
) -> None:
database = interim_retry_db
now = datetime.now(UTC).replace(tzinfo=None)
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.interim_webhook_sent_at = now
await session.commit()
preparation = await database.workflow_runs.prepare_next_attempt_atomic("wr_retry", "org_test", 1, "failed", None)
assert preparation.status == "inserted"
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 2))
attempt.modified_at = now - timedelta(seconds=background_task_executor_module.LEASE_TAKEOVER_SECONDS + 1)
await session.commit()
monkeypatch.setattr(background_task_executor_module, "LEASE_TAKEOVER_SECONDS", 3)
monkeypatch.setattr(background_task_executor_module, "naive_utc_now", lambda: now)
organization = await database.organizations.get_organization("org_test")
lookup = AsyncMock(side_effect=RuntimeError("database unavailable") if transient else None, return_value=None)
monkeypatch.setattr(
database.organizations,
"get_valid_org_auth_token" if dependency == "token" else "get_organization",
lookup,
)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
monkeypatch.setattr(app.WORKFLOW_SERVICE, "prepare_workflow_webhook", AsyncMock(return_value=None))
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
executor = BackgroundTaskExecutor()
async def sleep(seconds: float) -> None:
nonlocal now
current = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert current.status == "queued"
assert current.retry_decision is None
now += timedelta(seconds=seconds)
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=sleep))
await executor._recover_pending_retries_once(resume_fresh=True)
await asyncio.gather(*executor._background_tasks)
execute.assert_not_awaited()
assert not executor._scheduled_retry_resumes
current = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
run = await database.workflow_runs.get_workflow_run("wr_retry", "org_test")
assert run is not None
if transient:
assert current.status == "queued"
assert current.retry_decision is None
assert current.started_at is None
assert run.status == WorkflowRunStatus.queued
assert ("wr_retry", 2) in executor._retry_resumes_needing_recovery
lookup.side_effect = None
lookup.return_value = SimpleNamespace(token="api-token") if dependency == "token" else organization
if dependency == "organization":
monkeypatch.setattr(
database.organizations,
"get_valid_org_auth_token",
AsyncMock(return_value=SimpleNamespace(token="api-token")),
)
await executor._recover_pending_retries_once(resume_fresh=False)
await asyncio.gather(*executor._background_tasks)
current = (await database.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert current.started_at is not None
execute.assert_awaited_once()
assert execute.await_args.kwargs["attempt_number"] == 2
assert not executor._retry_resumes_needing_recovery
else:
assert current.status == "failed"
assert current.retry_decision == "abandoned"
assert current.decision_reason == "prepared_attempt_never_started"
assert run.status == WorkflowRunStatus.failed
assert ("API token" if dependency == "token" else "organization") in run.failure_reason
assert current.webhook_sent_at is not None
assert current.final_side_effects_progress["completed_effects"]
assert not executor._retry_resumes_needing_recovery
schedule = MagicMock()
monkeypatch.setattr(executor, "_schedule_retry_resume", schedule)
await executor._recover_pending_retries_once(resume_fresh=False)
schedule.assert_not_called()
execute.assert_not_awaited()
@pytest.mark.asyncio
async def test_retry_recovery_sweep_never_resumes_a_retry_while_its_loop_is_sleeping(
monkeypatch: pytest.MonkeyPatch,
) -> None:
pending_attempt = SimpleNamespace(
workflow_run_id="wr_sleeping",
organization_id="org_test",
attempt_number=1,
retry_decision="retry",
next_attempt_at=datetime.now(UTC) + timedelta(seconds=30),
next_attempt_prepared_at=None,
)
sweep_sleep_started = asyncio.Event()
release_sweep = asyncio.Event()
recovery_finished = asyncio.Event()
async def controlled_sleep(_delay_seconds: float) -> None:
sweep_sleep_started.set()
await release_sweep.wait()
await asyncio.sleep(0)
async def list_recovery(_cutoff: datetime) -> list[SimpleNamespace]:
recovery_finished.set()
return [pending_attempt]
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=list_recovery, list_stale_dispatch_claims=AsyncMock(return_value=[])
),
),
)
monkeypatch.setattr(background_task_executor_module, "asyncio", ScopedAsyncio(sleep=controlled_sleep))
schedule_resume = MagicMock()
executor = BackgroundTaskExecutor()
executor._fresh_retries_recovered = True
executor._schedule_retry_resume = schedule_resume # type: ignore[method-assign]
sweep_task = asyncio.create_task(executor._retry_recovery_sweep())
await asyncio.wait_for(sweep_sleep_started.wait(), timeout=1)
release_sweep.set()
await asyncio.wait_for(recovery_finished.wait(), timeout=1)
sweep_task.cancel()
with pytest.raises(asyncio.CancelledError):
await sweep_task
schedule_resume.assert_not_called()
@pytest.mark.asyncio
async def test_two_retry_resumers_only_the_inserting_preparer_executes_the_next_attempt(
monkeypatch: pytest.MonkeyPatch,
) -> None:
pending_attempt = SimpleNamespace(
workflow_run_id="wr_two_callers",
organization_id="org_test",
attempt_number=1,
next_attempt_at=datetime.now(UTC) - timedelta(seconds=1),
)
organization = SimpleNamespace(organization_id="org_test")
prepare = AsyncMock(
side_effect=[
PrepareNextAttemptResult(status="inserted", pinned_browser_session_id="pinned-session"),
PrepareNextAttemptResult(status="already_prepared", pinned_browser_session_id="pinned-session"),
]
)
execute = AsyncMock()
monkeypatch.setattr(background_task_executor_module, "prepare_next_attempt_result", prepare)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
organizations=SimpleNamespace(
get_organization=AsyncMock(return_value=organization),
get_valid_org_auth_token=AsyncMock(return_value=SimpleNamespace(token="api-token")),
),
),
)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
executor = BackgroundTaskExecutor()
await asyncio.gather(
executor._resume_pending_retry(pending_attempt),
executor._resume_pending_retry(pending_attempt),
)
assert prepare.await_count == 2
execute.assert_awaited_once()
assert execute.await_args.kwargs["attempt_number"] == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("persistent", [False, True])
async def test_retry_entry_preserves_policy_when_attempt_lookup_fails(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, persistent: bool
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
organization = await database.organizations.get_organization("org_test")
attempts = await database.workflow_run_attempts.get_attempts("wr_retry")
lookup = AsyncMock(
side_effect=RuntimeError("attempt lookup failed")
if persistent
else [RuntimeError("temporary failure"), attempts]
)
monkeypatch.setattr(database.workflow_run_attempts, "get_attempts", lookup)
sleep = AsyncMock()
monkeypatch.setattr(service_module, "asyncio", ScopedAsyncio(sleep=sleep))
run = await svc.get_workflow_run("wr_retry", "org_test")
execute = AsyncMock(return_value=run)
monkeypatch.setattr(svc, "execute_workflow", execute)
decision = RetryDecision(False, 1, 0, True, "budget_exhausted")
monkeypatch.setattr(service_module, "get_recorded_decision", AsyncMock(return_value=decision))
release = AsyncMock(return_value="released")
monkeypatch.setattr(svc, "_run_terminal_side_effects_with_retries", release)
if persistent:
with pytest.raises(WorkflowRetryAttemptLookupError) as raised:
await svc.execute_workflow_with_retries("wr_retry", api_key="api-token", organization=organization)
assert raised.value.__cause__ is lookup.side_effect
assert lookup.await_count == 3
assert [call.args[0] for call in sleep.await_args_list] == [0.1, 0.2]
execute.assert_not_awaited()
release.assert_not_awaited()
else:
await svc.execute_workflow_with_retries("wr_retry", api_key="api-token", organization=organization)
assert lookup.await_count == 2
sleep.assert_awaited_once_with(0.1)
execute.assert_awaited_once()
assert execute.await_args.kwargs["need_call_webhook"] is False
release.assert_awaited_once()
assert release.await_args.args[1] == decision
@pytest.mark.asyncio
@pytest.mark.parametrize("outage", ["transient", "persistent"])
async def test_retry_owner_retries_next_attempt_preparation_until_the_sweep_cutoff(
monkeypatch: pytest.MonkeyPatch, outage: str
) -> None:
svc = app.WORKFLOW_SERVICE
decision = RetryDecision(True, 1, 5, False, "retry_on_failed")
error = RuntimeError("database unavailable")
read = AsyncMock(side_effect=error if outage == "persistent" else [error, decision])
monkeypatch.setattr(service_module, "get_recorded_decision", read)
prepared = SimpleNamespace(status="inserted", pinned_browser_session_id=None, serialized_identity=False)
prepare = AsyncMock(return_value=prepared)
monkeypatch.setattr(service_module, "prepare_next_attempt_result", prepare)
sleep = AsyncMock()
monkeypatch.setattr(service_module, "asyncio", ScopedAsyncio(sleep=sleep))
# Each clock read advances two thirds of the takeover window, so the second failure is past the deadline.
clock = datetime.now(UTC).replace(tzinfo=None)
step = timedelta(seconds=service_module.LEASE_TAKEOVER_SECONDS * 2 / 3)
def now() -> datetime:
nonlocal clock
clock += step
return clock - step
monkeypatch.setattr(service_module, "naive_utc_now", now)
if outage == "persistent":
with pytest.raises(RuntimeError, match="database unavailable"):
await svc._prepare_recorded_retry("wr_retry", "org_test", decision)
prepare.assert_not_awaited()
else:
assert await svc._prepare_recorded_retry("wr_retry", "org_test", decision) is prepared
prepare.assert_awaited_once()
assert prepare.await_args.kwargs["from_attempt"] == 1
assert prepare.await_args.kwargs["clear_browser_address"] is False
assert read.await_count == 2
sleep.assert_awaited_once_with(1.0)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failure_phase",
[
"preparation",
"entry",
"prepared_entry",
"prepared_run_lookup",
"transient_entry",
"execution",
"prepared_execution",
],
)
async def test_retry_resumer_recovers_attempt_lookup_failure_on_next_sweep(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, failure_phase: str
) -> None:
database = interim_retry_db
svc = app.WORKFLOW_SERVICE
now = datetime.now(UTC).replace(tzinfo=None)
async with database.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
attempt.next_attempt_at = now - timedelta(seconds=1)
attempt.interim_webhook_sent_at = now
await session.commit()
if failure_phase.startswith("prepared_"):
preparation = await database.workflow_runs.prepare_next_attempt_atomic(
"wr_retry", "org_test", 1, "failed", None
)
assert preparation.status == "inserted"
read_attempts = database.workflow_run_attempts.get_attempts
pending = (await read_attempts("wr_retry"))[-1]
monkeypatch.setattr(
database.organizations, "get_valid_org_auth_token", AsyncMock(return_value=SimpleNamespace(token="api-token"))
)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
monkeypatch.setattr(service_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))
lookup = AsyncMock(wraps=read_attempts)
monkeypatch.setattr(database.workflow_run_attempts, "get_attempts", lookup)
claim = AsyncMock(wraps=database.workflow_run_attempts.claim_prepared_attempt_execution)
monkeypatch.setattr(database.workflow_run_attempts, "claim_prepared_attempt_execution", claim)
abandon = AsyncMock()
executor = BackgroundTaskExecutor()
monkeypatch.setattr(executor, "_abandon_retry", abandon)
release = AsyncMock(return_value="released")
monkeypatch.setattr(svc, "_run_terminal_side_effects_with_retries", release)
decision = RetryDecision(False, 2, 0, True, "budget_exhausted")
monkeypatch.setattr(service_module, "get_recorded_decision", AsyncMock(return_value=decision))
execute = AsyncMock(return_value=await svc.get_workflow_run("wr_retry", "org_test"))
monkeypatch.setattr(svc, "execute_workflow", execute)
read_run = AsyncMock(wraps=svc.get_workflow_run)
monkeypatch.setattr(svc, "get_workflow_run", read_run)
real_entry = svc.execute_workflow_with_retries
before_failure: list[dict[str, object]] = []
async def snapshot() -> list[dict[str, object]]:
return [
{column.name: getattr(row, column.name) for column in WorkflowRunAttemptModel.__table__.columns}
for row in await read_attempts("wr_retry")
]
async def entry(**kwargs: object) -> WorkflowRun:
nonlocal before_failure
before_failure = await snapshot()
if failure_phase in {"entry", "prepared_entry"}:
lookup.side_effect = RuntimeError("attempt lookup failed")
elif failure_phase == "prepared_run_lookup":
read_run.side_effect = RuntimeError("workflow run lookup failed before execution")
elif failure_phase == "transient_entry":
lookup.side_effect = [RuntimeError("temporary failure"), await read_attempts("wr_retry")]
return await real_entry(**kwargs)
if failure_phase == "preparation":
before_failure = await snapshot()
lookup.side_effect = RuntimeError("attempt lookup failed")
if failure_phase in {"execution", "prepared_execution"}:
release.side_effect = RuntimeError("terminal effects failed after execution")
if failure_phase == "prepared_execution":
execute.side_effect = RuntimeError("workflow execution failed")
dispatch = AsyncMock(side_effect=entry)
monkeypatch.setattr(svc, "execute_workflow_with_retries", dispatch)
executor._schedule_retry_resume(pending)
await asyncio.gather(*executor._background_tasks)
key = ("wr_retry", pending.attempt_number)
assert not executor._scheduled_retry_resumes
abandon.assert_not_awaited()
if failure_phase.startswith("prepared_"):
claimed_attempt = (await read_attempts("wr_retry"))[-1]
assert claimed_attempt.started_at is not None
assert claim.await_count == 1
if failure_phase in {"transient_entry", "execution", "prepared_execution"}:
execute.assert_awaited_once()
assert execute.await_args.kwargs["need_call_webhook"] is False
assert key not in executor._retry_resumes_needing_recovery
assert key not in executor._retry_dispatches_needing_recovery
lookup.side_effect = None
await executor._recover_pending_retries_once(resume_fresh=False)
assert not executor._scheduled_retry_resumes
return
execute.assert_not_awaited()
assert await snapshot() == before_failure
assert (await read_attempts("wr_retry"))[0].retry_decision == "retry"
assert key in executor._retry_resumes_needing_recovery
if failure_phase != "preparation":
assert key in executor._retry_dispatches_needing_recovery
if failure_phase in {"entry", "prepared_entry"}:
assert lookup.await_count >= 3
lookup.side_effect = None
read_run.side_effect = None
dispatch.side_effect = real_entry
await executor._recover_pending_retries_once(resume_fresh=False)
await asyncio.gather(*executor._background_tasks)
execute.assert_awaited_once()
assert execute.await_args.kwargs["attempt_number"] == 2
assert execute.await_args.kwargs["need_call_webhook"] is False
assert claim.await_count == 1
assert key not in executor._retry_resumes_needing_recovery
assert key not in executor._retry_dispatches_needing_recovery
assert not executor._scheduled_retry_resumes
assert (await read_attempts("wr_retry"))[0].retry_decision == "retry"
abandon.assert_not_awaited()
@pytest.mark.asyncio
async def test_periodic_recovery_leaves_a_fresh_prepared_queued_attempt_alone(
monkeypatch: pytest.MonkeyPatch,
) -> None:
prepared_attempt = SimpleNamespace(
workflow_run_id="wr_prepared_queued",
organization_id="org_test",
attempt_number=2,
retry_decision=None,
status=WorkflowRunStatus.queued.value,
started_at=None,
modified_at=datetime.now(UTC),
)
failed_run = SimpleNamespace(
workflow_run_id="wr_prepared_queued",
organization_id="org_test",
status=WorkflowRunStatus.failed,
failure_reason="Workflow retry attempt was prepared but never started before recovery.",
finished_at=datetime.now(UTC),
)
mark_failed = AsyncMock()
mark_failed.return_value = failed_run
finalize_abandoned = AsyncMock(return_value=RetryDecision(False, 2, 0, True, "prepared_attempt_never_started"))
side_effects = AsyncMock()
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=AsyncMock(side_effect=[[prepared_attempt], []]),
list_stale_dispatch_claims=AsyncMock(return_value=[]),
),
workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(return_value=failed_run)),
),
)
monkeypatch.setattr(background_task_executor_module, "finalize_abandoned_attempt", finalize_abandoned)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", mark_failed)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", side_effects)
executor = BackgroundTaskExecutor()
await executor._recover_pending_retries_once(resume_fresh=False)
await executor._recover_pending_retries_once(resume_fresh=False)
mark_failed.assert_not_awaited()
finalize_abandoned.assert_not_awaited()
side_effects.assert_not_awaited()
@pytest.mark.asyncio
async def test_recover_pending_retries_releases_unreleased_terminal_effects(
monkeypatch: pytest.MonkeyPatch,
) -> None:
terminal_attempt = SimpleNamespace(
workflow_run_id="wr_abandoned",
organization_id="org_test",
attempt_number=2,
retry_decision="abandoned",
decision_reason="process_restart",
side_effects_released_at=None,
webhook_sent_at=None,
modified_at=datetime.now(UTC) - timedelta(seconds=RETRY_DECISION_GRACE_SECONDS + 1),
)
terminal_run = SimpleNamespace(
workflow_run_id="wr_abandoned",
organization_id="org_test",
status=WorkflowRunStatus.failed,
)
list_recovery = AsyncMock(return_value=[terminal_attempt])
get_workflow_run = AsyncMock(return_value=terminal_run)
side_effects = AsyncMock()
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=list_recovery, list_stale_dispatch_claims=AsyncMock(return_value=[])
),
workflow_runs=SimpleNamespace(get_workflow_run=get_workflow_run),
),
)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", side_effects)
monkeypatch.setattr(BackgroundTaskExecutor, "_start_retry_recovery_sweep", MagicMock())
await BackgroundTaskExecutor().recover_pending_retries()
side_effects.assert_awaited_once()
decision = side_effects.await_args.args[1]
assert decision == RetryDecision(False, 2, 0, True, "process_restart")
@pytest.mark.asyncio
async def test_recovery_reacquires_expired_side_effect_lease_after_effect_failure(
monkeypatch: pytest.MonkeyPatch,
) -> None:
old_claim = datetime.now(UTC) - timedelta(seconds=RETRY_DECISION_GRACE_SECONDS + 1)
attempts = [
SimpleNamespace(
workflow_run_id="wr_lease",
organization_id="org_test",
attempt_number=1,
retry_decision="final",
decision_reason="budget_exhausted",
side_effects_released_at=old_claim,
webhook_sent_at=None,
modified_at=old_claim,
),
SimpleNamespace(
workflow_run_id="wr_lease",
organization_id="org_test",
attempt_number=1,
retry_decision="final",
decision_reason="budget_exhausted",
side_effects_released_at=old_claim,
webhook_sent_at=None,
modified_at=old_claim,
),
]
terminal_run = SimpleNamespace(
workflow_run_id="wr_lease",
organization_id="org_test",
status=WorkflowRunStatus.failed,
)
list_recovery = AsyncMock(side_effect=[[attempts[0]], [attempts[1]]])
get_workflow_run = AsyncMock(return_value=terminal_run)
release_calls: list[dict[str, object]] = []
async def release_side_effects(*_args: object, **kwargs: object) -> None:
release_calls.append(kwargs)
if len(release_calls) == 1:
raise RuntimeError("logical terminal effect failed")
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=list_recovery, list_stale_dispatch_claims=AsyncMock(return_value=[])
),
workflow_runs=SimpleNamespace(get_workflow_run=get_workflow_run),
),
)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", release_side_effects)
executor = BackgroundTaskExecutor()
await executor._recover_pending_retries_once(resume_fresh=False)
await executor._recover_pending_retries_once(resume_fresh=False)
assert [call["side_effects_claim_at"] for call in release_calls] == [old_claim, old_claim]
@pytest.mark.asyncio
async def test_scheduled_run_cannot_clobber_the_callers_context() -> None:
"""The caller keeps running after dispatching; both must not write one context object."""
ran = asyncio.Event()
child_context: list[SkyvernContext | None] = []
async def work() -> None:
context = skyvern_context.current()
child_context.append(context)
assert context is not None
# execute_workflow assigns this on whatever context it finds.
context.generate_script = False
context.task_id = "tsk_child"
ran.set()
parent = SkyvernContext(organization_id="org_1", task_id="tsk_parent", generate_script=True)
with skyvern_context.scoped(parent):
BackgroundTaskExecutor()._schedule(None, work)
await asyncio.wait_for(ran.wait(), timeout=1)
assert child_context[0] is not parent
# The caller's context survives the child's writes.
assert parent.task_id == "tsk_parent"
assert parent.generate_script is True
# ...while inherited values still reach the child.
assert child_context[0].organization_id == "org_1"
@pytest.mark.asyncio
async def test_scheduled_run_gets_independent_download_popup_registries() -> None:
"""A fire-and-forget run copied via ``replace(parent)`` must own EMPTY, independent download-popup
lifecycle registries. Otherwise the shallow copy aliases the parent's dicts, so a reset/cleanup on
either context destructively detaches the other live run's listeners and clears its reservations."""
class _FakeBrowserContext:
def __init__(self) -> None:
self.callbacks: list[Any] = []
def on(self, event: str, cb: Any) -> None:
if event == "page":
self.callbacks.append(cb)
def remove_listener(self, event: str, cb: Any) -> None:
if event == "page" and cb in self.callbacks:
self.callbacks.remove(cb)
parent = SkyvernContext(organization_id="org_1", task_id="tsk_parent")
parent_browser_context = _FakeBrowserContext()
parent_page = object()
parent.arm_download_popup_context_listener("tsk_parent", parent_browser_context, lambda page: None)
parent.record_download_popup_claim(
"tsk_parent",
parent_page,
baseline_files=["/d/parent.pdf"],
session_observed=True,
session_baseline_files=["s3://b/parent-session.pdf"],
)
parent.record_download_popup_late_candidate("tsk_parent", object())
parent.stash_pending_download_reservation_release("tsk_parent", ((parent_page,), ()))
parent_sibling = object()
parent.mark_download_popup_claim_delta_siblings("tsk_parent", [parent_page, parent_sibling])
parent.anchor_download_popup_recovery_grace("tsk_parent", parent_page, 0.0)
child_holder: list[SkyvernContext] = []
async def work() -> None:
child = skyvern_context.current()
assert child is not None
child_holder.append(child)
# Assert inside the scope: scoped()'s exit runs _cleanup_outgoing_context on the parent.
with skyvern_context.scoped(parent):
BackgroundTaskExecutor()._schedule(None, work)
for _ in range(100):
if child_holder:
break
await asyncio.sleep(0)
assert child_holder, "the scheduled child run never executed"
child = child_holder[0]
assert child is not parent
# All PR-owned popup lifecycle registries (incl. the recovery-grace anchor) are independent
# objects, empty in the child. The parent recorded a claim and anchored recovery grace.
assert parent.download_popup_recovery_grace_started_at, "parent should hold a recovery-grace anchor"
assert parent.download_popup_claim_baseline, "parent should hold a claim baseline snapshot"
assert parent.download_popup_claim_session_observed, "parent should hold a session-observed flag"
assert parent.download_popup_claim_session_baseline, "parent should hold a session baseline"
assert parent.download_popup_claim_delta_siblings, "parent should hold sibling markers"
for attr in (
"download_popup_claims",
"download_popup_context_listeners",
"download_popup_late_candidates",
"pending_download_reservation_release",
"download_popup_recovery_grace_started_at",
"download_popup_claim_baseline",
"download_popup_claim_session_observed",
"download_popup_claim_session_baseline",
"download_popup_claim_delta_siblings",
):
assert getattr(child, attr) is not getattr(parent, attr), f"{attr} must not alias the parent's dict"
assert getattr(child, attr) == {}, f"{attr} must be empty in the copied child"
# Mutating the child's own sibling markers must not bleed into the parent's (would fail if the
# child aliased the parent's dict for lack of an independent reset).
child.mark_download_popup_claim_delta_siblings("tsk_child_sib", [object()])
assert "tsk_parent" in parent.download_popup_claim_delta_siblings, "child mutation dropped parent's marker"
assert "tsk_child_sib" not in parent.download_popup_claim_delta_siblings, "child marker leaked to the parent"
# A destructive teardown on the child must leave the parent's live ownership state untouched.
child.detach_all_download_popup_context_listeners()
assert parent.has_download_popup_claim("tsk_parent", parent_page), "child teardown cleared the parent's claim"
assert parent_browser_context.callbacks, "child teardown detached the parent's live listener"
assert "tsk_parent" in parent.pending_download_reservation_release, "child teardown cleared parent's pending"
assert parent.download_popup_recovery_grace_started_at, "child teardown cleared the parent's recovery anchor"
assert parent.download_popup_claim_baseline, "child teardown cleared the parent's claim baseline"
assert parent.download_popup_claim_session_observed, "child teardown cleared the parent's session flag"
assert parent.download_popup_claim_session_baseline, "child teardown cleared the parent's session baseline"
assert parent.download_popup_claim_delta_siblings, "child teardown cleared the parent's sibling markers"
# ...and the symmetric direction: a parent teardown must not touch the child's own state.
child_page = object()
child.record_download_popup_claim("tsk_child", child_page)
parent.detach_all_download_popup_context_listeners()
assert child.has_download_popup_claim("tsk_child", child_page), "parent teardown cleared the child's claim"
@pytest.mark.asyncio
async def test_execute_workflow_stamps_org_llm_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
organization = SimpleNamespace(
organization_id="org_test",
default_llm_key="CUSTOM_LLM_oat_smart",
default_secondary_llm_key="CUSTOM_LLM_oat_fast",
created_at=None,
)
monkeypatch.setattr(
app.DATABASE.workflow_runs,
"get_workflow_run",
AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None, status=WorkflowRunStatus.created)),
)
monkeypatch.setattr(
"skyvern.forge.sdk.executor.background_task_executor.initialize_skyvern_state_file",
AsyncMock(),
)
monkeypatch.setattr(
"skyvern.forge.sdk.api.llm.custom_llm_registry.load_custom_llm_configs_for_organization",
AsyncMock(),
)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", AsyncMock())
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_test",
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
_background_tasks, dispatch, attempt, execution_kwargs = executor._schedule.call_args.args
# The dispatch task stamps the defaults on the context the run executes in.
with skyvern_context.scoped(SkyvernContext(organization_id="org_test")) as context:
await dispatch(attempt, execution_kwargs)
assert context.org_default_llm_key == "CUSTOM_LLM_oat_smart"
assert context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"
@pytest.mark.asyncio
async def test_execute_workflow_dispatches_from_latest_retry_attempt(monkeypatch: pytest.MonkeyPatch) -> None:
organization = SimpleNamespace(
organization_id="org_test",
default_llm_key=None,
default_secondary_llm_key=None,
)
monkeypatch.setattr(
app.DATABASE.workflow_runs,
"get_workflow_run",
AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None, status=WorkflowRunStatus.created)),
)
monkeypatch.setattr(
app.DATABASE.workflow_run_attempts,
"get_attempts",
AsyncMock(
return_value=[
SimpleNamespace(attempt_number=1),
SimpleNamespace(attempt_number=2),
]
),
)
monkeypatch.setattr(
"skyvern.forge.sdk.executor.background_task_executor.initialize_skyvern_state_file",
AsyncMock(),
)
monkeypatch.setattr(
"skyvern.forge.sdk.api.llm.custom_llm_registry.load_custom_llm_configs_for_organization",
AsyncMock(),
)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_test",
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
_background_tasks, dispatch, attempt, execution_kwargs = executor._schedule.call_args.args
assert dispatch == executor._dispatch_retry_resume
assert attempt.attempt_number == 2
assert execution_kwargs["attempt_number"] == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("need_call_webhook", [True, False])
@pytest.mark.parametrize(
"scenario",
["state_file", "llm_runtime", "lookup_failure", "attempt_row", "webhook_preparation", "transient_recovers"],
)
async def test_no_policy_initializer_failure_fails_the_run_durably(
interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, scenario: str, need_call_webhook: bool
) -> None:
database = interim_retry_db
async with database.Session() as session:
run = await session.get(WorkflowRunModel, "wr_retry")
assert run is not None
run.status = "created"
run.started_at = None
run.finished_at = None
attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
assert attempt is not None
if scenario == "attempt_row":
attempt.status = "created"
attempt.retry_decision = None
attempt.finished_at = None
attempt.next_attempt_at = None
else:
workflow = await session.get(WorkflowModel, "wf_retry")
assert workflow is not None
workflow.workflow_definition = {"blocks": [], "parameters": []}
await session.delete(attempt)
await session.commit()
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
initializer = "prepare_org_llm_runtime" if scenario == "llm_runtime" else "initialize_skyvern_state_file"
error = RuntimeError(f"{initializer} unavailable")
monkeypatch.setattr(
background_task_executor_module,
initializer,
AsyncMock(side_effect=[error, None] if scenario == "transient_recovers" else error),
)
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
fail_run = AsyncMock(wraps=app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", fail_run)
webhook = AsyncMock(wraps=app.WORKFLOW_SERVICE.execute_workflow_webhook)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_webhook", webhook)
deliver = AsyncMock(return_value=True)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "deliver_prepared_workflow_webhook", deliver)
if scenario == "webhook_preparation":
monkeypatch.setattr(
app.WORKFLOW_SERVICE, "prepare_workflow_webhook", AsyncMock(side_effect=RuntimeError("preparation failed"))
)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
organization = await database.organizations.get_organization("org_test")
assert organization is not None
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_retry",
workflow_run_id="wr_retry",
workflow_permanent_id="wpid_retry",
max_steps_override=None,
api_key="test-dispatch-key",
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
_background_tasks, dispatch, attempt, execution_kwargs = executor._schedule.call_args.args
execution_kwargs["need_call_webhook"] = need_call_webhook
get_attempts = database.workflow_run_attempts.get_attempts
if scenario == "lookup_failure":
monkeypatch.setattr(
database.workflow_run_attempts, "get_attempts", AsyncMock(side_effect=RuntimeError("db down"))
)
await dispatch(attempt, execution_kwargs)
execute.assert_not_awaited()
run = await database.workflow_runs.get_workflow_run("wr_retry")
assert run is not None
assert run.started_at is None
key = ("wr_retry", 1)
assert run.status == WorkflowRunStatus.queued
assert run.failure_reason is None
fail_run.assert_not_awaited()
webhook.assert_not_awaited()
assert key in executor._retry_dispatches_needing_recovery
assert key in executor._retry_resumes_needing_recovery
assert executor._initializer_failures[key] == 1
dispatches = 2 if scenario == "transient_recovers" else background_task_executor_module.INITIALIZER_FAILURE_BUDGET
for dispatch_number in range(2, dispatches + 1):
executor._schedule.reset_mock()
await executor._recover_pending_retries_once(resume_fresh=True)
executor._schedule.assert_called_once()
background_tasks, resume, resume_attempt = executor._schedule.call_args.args
assert background_tasks is None
assert resume == executor._resume_pending_retry
assert resume_attempt == attempt
await resume(resume_attempt)
if dispatch_number < dispatches:
run = await database.workflow_runs.get_workflow_run("wr_retry")
assert run is not None
assert run.status == WorkflowRunStatus.queued
assert run.failure_reason is None
fail_run.assert_not_awaited()
webhook.assert_not_awaited()
execute.assert_not_awaited()
assert key in executor._retry_dispatches_needing_recovery
assert key in executor._retry_resumes_needing_recovery
assert executor._initializer_failures[key] == dispatch_number
run = await database.workflow_runs.get_workflow_run("wr_retry")
assert run is not None
assert run.started_at is None
attempts = await get_attempts("wr_retry")
if scenario in {"lookup_failure", "attempt_row"}:
assert run.status == WorkflowRunStatus.queued
assert run.failure_reason is None
fail_run.assert_not_awaited()
webhook.assert_not_awaited()
assert key in executor._retry_dispatches_needing_recovery
assert key in executor._retry_resumes_needing_recovery
assert executor._initializer_failures[key] == background_task_executor_module.INITIALIZER_FAILURE_BUDGET
execute.assert_not_awaited()
if scenario == "attempt_row":
assert len(attempts) == 1
assert attempts[0].status == "queued"
assert attempts[0].started_at is None
else:
assert not attempts
elif scenario == "transient_recovers":
execute.assert_awaited_once()
assert run.status == WorkflowRunStatus.queued
assert run.failure_reason is None
fail_run.assert_not_awaited()
webhook.assert_not_awaited()
assert not attempts
assert not executor._retry_dispatches_needing_recovery
assert not executor._retry_resumes_needing_recovery
assert key not in executor._initializer_failures
else:
assert run.status == WorkflowRunStatus.failed
reason = f"Workflow run initialization failed before execution: RuntimeError: {error}"
assert run.failure_reason == reason
fail_run.assert_awaited_once_with(workflow_run_id="wr_retry", failure_reason=reason, cascade_children=False)
if need_call_webhook:
webhook.assert_awaited_once_with(
run, api_key="test-dispatch-key", claim_kind=None, finalizer="failed_before_execution"
)
else:
webhook.assert_not_awaited()
assert not attempts
assert not executor._retry_dispatches_needing_recovery
assert not executor._retry_resumes_needing_recovery
assert key not in executor._initializer_failures
executor._schedule.reset_mock()
await executor._recover_pending_retries_once(resume_fresh=True)
executor._schedule.assert_not_called()
execute.assert_not_awaited()
if need_call_webhook and scenario in {"state_file", "llm_runtime"}:
deliver.assert_awaited_once()
assert deliver.await_args.kwargs == {
"delivered_projection": WebhookDeliveryStatus.delivered,
"exhausted_projection": WebhookDeliveryStatus.exhausted_unattributed,
"finalizer": "failed_before_execution",
"attempt_claimed": False,
}
else:
deliver.assert_not_awaited()
if app.WORKFLOW_SERVICE._background_tasks:
await asyncio.gather(*app.WORKFLOW_SERVICE._background_tasks)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"scenario",
[
"duplicate",
"terminal_reread",
"foreign_terminal",
"nonfinal_reread",
"reread_failure",
"missing_reread",
"delivery_failure",
],
)
async def test_fail_run_without_attempt_row_finalization_outcomes(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, scenario: str
) -> None:
run = SimpleNamespace(
workflow_run_id="wr_test", status=WorkflowRunStatus.failed, failure_reason="initialization failed"
)
finalization_error = RuntimeError("finalization failed")
fail_run = AsyncMock(return_value=run)
reread = AsyncMock(return_value=run)
webhook = AsyncMock()
if scenario == "duplicate":
fail_run.return_value = None
elif scenario == "delivery_failure":
webhook.side_effect = RuntimeError("delivery failed")
else:
fail_run.side_effect = finalization_error
if scenario == "nonfinal_reread":
run.status = WorkflowRunStatus.queued
elif scenario == "foreign_terminal":
run.status = WorkflowRunStatus.canceled
elif scenario == "reread_failure":
reread.side_effect = RuntimeError("re-read failed")
elif scenario == "missing_reread":
reread.return_value = None
get_attempts = AsyncMock(return_value=[])
monkeypatch.setattr(
retry_policy_module,
"app",
SimpleNamespace(
DATABASE=SimpleNamespace(
workflow_run_attempts=SimpleNamespace(get_attempts=get_attempts),
workflow_runs=SimpleNamespace(get_workflow_run=reread),
),
WORKFLOW_SERVICE=SimpleNamespace(
mark_workflow_run_as_failed_if_not_final=fail_run,
execute_workflow_webhook=webhook,
),
),
)
if scenario in {"nonfinal_reread", "reread_failure"}:
with pytest.raises(RuntimeError) as exc_info:
await retry_policy_module.fail_run_without_attempt_row("wr_test", "initialization failed")
assert exc_info.value is finalization_error
if scenario == "reread_failure":
assert exc_info.value.__context__ is reread.side_effect
else:
assert await retry_policy_module.fail_run_without_attempt_row("wr_test", "initialization failed") is True
get_attempts.assert_awaited_once_with("wr_test")
fail_run.assert_awaited_once_with(
workflow_run_id="wr_test", failure_reason="initialization failed", cascade_children=False
)
if scenario in {"duplicate", "delivery_failure"}:
reread.assert_not_awaited()
else:
reread.assert_awaited_once_with("wr_test")
if scenario in {"terminal_reread", "delivery_failure"}:
webhook.assert_awaited_once_with(run, api_key=None, claim_kind=None, finalizer="failed_before_execution")
else:
webhook.assert_not_awaited()
if scenario == "reread_failure":
assert "leaving the run recoverable" in caplog.text
elif scenario == "delivery_failure":
assert "Failed to deliver workflow webhook after initialization failure" in caplog.text
@pytest.mark.asyncio
async def test_first_dispatch_initializer_failure_is_retried_by_the_sweep(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
monkeypatch.setattr(app, "DATABASE", sqlite_db)
async with sqlite_db.Session() as session:
session.add_all(
[
WorkflowRunModel(
workflow_run_id="wr_test",
organization_id="org_test",
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
status="created",
),
WorkflowRunAttemptModel(
workflow_run_id="wr_test", organization_id="org_test", attempt_number=1, status="created"
),
]
)
await session.commit()
initializer_calls = 0
async def flaky_initialize(**kwargs: object) -> None:
nonlocal initializer_calls
initializer_calls += 1
if initializer_calls == 1:
raise RuntimeError("state file unavailable")
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", flaky_initialize)
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_test",
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
_background_tasks, dispatch, attempt, execution_kwargs = executor._schedule.call_args.args
# The scheduled dispatch fails to initialize; the run must stay owned by the sweep, not stuck.
await dispatch(attempt, execution_kwargs)
execute.assert_not_awaited()
assert ("wr_test", 1) in executor._retry_dispatches_needing_recovery
await executor._resume_pending_retry(attempt)
execute.assert_awaited_once()
assert execute.await_args is not None
assert execute.await_args.kwargs["attempt_number"] == 1
assert not executor._retry_dispatches_needing_recovery
@pytest.mark.asyncio
@pytest.mark.parametrize("lookup_fails", [False, True], ids=["lookup-ok", "lookup-fails"])
async def test_lost_first_dispatch_is_recovered_after_a_restart(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch, lookup_fails: bool
) -> None:
async with sqlite_db.Session() as session:
session.add_all(
[
WorkflowRunModel(
workflow_run_id="wr_first",
organization_id="org_test",
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
status="created",
),
WorkflowRunAttemptModel(
workflow_run_id="wr_first", organization_id="org_test", attempt_number=1, status="created"
),
]
)
await session.commit()
organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
monkeypatch.setattr(app, "DATABASE", sqlite_db)
monkeypatch.setattr(sqlite_db.organizations, "get_organization", AsyncMock(return_value=organization))
monkeypatch.setattr(app.AGENT_FUNCTION, "get_workflow_run_execution_status", AsyncMock(return_value="absent"))
monkeypatch.setattr(BackgroundTaskExecutor, "_get_valid_api_key", AsyncMock(return_value="token"))
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
if lookup_fails:
real_get_attempts = sqlite_db.workflow_run_attempts.get_attempts
lookups = iter([RuntimeError("temporary database failure")])
async def flaky_get_attempts(workflow_run_id: str) -> list[WorkflowRunAttemptModel]:
failure = next(lookups, None)
if failure is not None:
raise failure
return await real_get_attempts(workflow_run_id)
monkeypatch.setattr(sqlite_db.workflow_run_attempts, "get_attempts", flaky_get_attempts)
dispatcher = BackgroundTaskExecutor()
dispatcher._schedule = MagicMock() # type: ignore[method-assign]
await dispatcher.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_test",
workflow_run_id="wr_first",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
dispatched = (await sqlite_db.workflow_run_attempts.get_attempts("wr_first"))[0]
assert dispatched.status == "queued" and dispatched.started_at is None
queued_run = await sqlite_db.workflow_runs.get_workflow_run("wr_first")
assert queued_run is not None and queued_run.status == WorkflowRunStatus.queued and queued_run.started_at is None
# The process exits before the scheduled dispatch runs; a fresh executor sweeps after the lease window.
later = background_task_executor_module.naive_utc_now() + timedelta(
seconds=2 * service_module.LEASE_TAKEOVER_SECONDS
)
for module in (service_module, background_task_executor_module, attempts_repository_module):
monkeypatch.setattr(module, "naive_utc_now", lambda: later)
recovering = BackgroundTaskExecutor()
await recovering._recover_pending_retries_once(resume_fresh=False)
await asyncio.gather(*recovering._background_tasks)
execute.assert_awaited_once()
assert execute.await_args is not None
assert execute.await_args.kwargs["attempt_number"] == 1
assert execute.await_args.kwargs["prepared_attempt_claimed"] is True
recovered = (await sqlite_db.workflow_run_attempts.get_attempts("wr_first"))[0]
assert recovered.status == "running" and recovered.started_at is not None
@pytest.mark.asyncio
@pytest.mark.parametrize("sweep_claims_first", [False, True], ids=["dispatch_claims", "sweep_claimed_first"])
async def test_first_dispatch_runs_its_attempt_only_when_it_claims_it(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch, sweep_claims_first: bool
) -> None:
"""The initializers run while the attempt is queued. A sweep that claims the attempt in that window
owns it, so the original dispatch must not execute the attempt as well."""
async with sqlite_db.Session() as session:
session.add_all(
[
WorkflowRunModel(
workflow_run_id="wr_first",
organization_id="org_test",
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
status="created",
),
WorkflowRunAttemptModel(
workflow_run_id="wr_first", organization_id="org_test", attempt_number=1, status="created"
),
]
)
await session.commit()
organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
monkeypatch.setattr(app, "DATABASE", sqlite_db)
monkeypatch.setattr(sqlite_db.organizations, "get_organization", AsyncMock(return_value=organization))
monkeypatch.setattr(BackgroundTaskExecutor, "_get_valid_api_key", AsyncMock(return_value="token"))
sweep_stamp: datetime | None = None
async def initialize_state(**_kwargs: Any) -> None:
nonlocal sweep_stamp
if sweep_claims_first:
sweep_stamp = await sqlite_db.workflow_run_attempts.claim_prepared_attempt_execution(
"wr_first", "org_test", 1
)
assert sweep_stamp is not None
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", initialize_state)
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
execute = AsyncMock(return_value=None)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow", execute)
monkeypatch.setattr(
service_module, "get_recorded_decision", AsyncMock(return_value=RetryDecision(False, 1, 0, True, "final"))
)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", AsyncMock())
dispatcher = BackgroundTaskExecutor()
await dispatcher.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_test",
workflow_run_id="wr_first",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
await asyncio.gather(*dispatcher._background_tasks)
attempt = (await sqlite_db.workflow_run_attempts.get_attempts("wr_first"))[0]
assert attempt.status == "running" and attempt.started_at is not None
if sweep_claims_first:
execute.assert_not_awaited()
assert attempt.started_at == sweep_stamp
else:
execute.assert_awaited_once()
assert execute.await_args is not None
assert execute.await_args.kwargs["attempt_number"] == 1
assert execute.await_args.kwargs["dispatch_claim_started_at"] == attempt.started_at
@pytest.mark.asyncio
async def test_scheduled_run_whose_initializer_fails_is_recovered_after_a_restart(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
fire_time = datetime(2026, 6, 2, 10, 0, tzinfo=UTC)
schedule = SimpleNamespace(
workflow_schedule_id="wfs_test",
workflow_permanent_id="wpid_test",
organization_id="org_test",
parameters={},
is_one_time=False,
)
workflow_run_id = schedule_service_module.build_scheduled_workflow_run_id(schedule.workflow_schedule_id, fire_time)
async with sqlite_db.Session() as session:
session.add_all(
[
WorkflowRunModel(
workflow_run_id=workflow_run_id,
organization_id="org_test",
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
status="created",
),
WorkflowRunAttemptModel(
workflow_run_id=workflow_run_id, organization_id="org_test", attempt_number=1, status="created"
),
]
)
await session.commit()
organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
monkeypatch.setattr(app, "DATABASE", sqlite_db)
monkeypatch.setattr(sqlite_db.organizations, "get_organization", AsyncMock(return_value=organization))
monkeypatch.setattr(app.AGENT_FUNCTION, "get_workflow_run_execution_status", AsyncMock(return_value="absent"))
monkeypatch.setattr(BackgroundTaskExecutor, "_get_valid_api_key", AsyncMock(return_value="token"))
monkeypatch.setattr(
schedule_service_module,
"prepare_workflow",
AsyncMock(
return_value=SimpleNamespace(
workflow_run_id=workflow_run_id, browser_session_id=None, status=WorkflowRunStatus.created
)
),
)
monkeypatch.setattr(schedule_service_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(
schedule_service_module, "prepare_org_llm_runtime", AsyncMock(side_effect=RuntimeError("llm registry down"))
)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
scheduler = schedule_service_module.LocalWorkflowScheduleScheduler(poll_interval_seconds=1, max_concurrent_runs=1)
with pytest.raises(RuntimeError, match="llm registry down"):
await scheduler._run_schedule(
schedule_service_module.DueWorkflowSchedule(schedule=schedule, previous_fire_time=fire_time) # type: ignore[arg-type]
)
execute.assert_not_awaited()
dispatched = (await sqlite_db.workflow_run_attempts.get_attempts(workflow_run_id))[0]
assert dispatched.status == "queued" and dispatched.started_at is None
queued_run = await sqlite_db.workflow_runs.get_workflow_run(workflow_run_id)
assert queued_run is not None and queued_run.status == WorkflowRunStatus.queued
# The scheduler now treats the fire as delivered; only the dispatch sweep can run the persisted run.
later = background_task_executor_module.naive_utc_now() + timedelta(
seconds=2 * service_module.LEASE_TAKEOVER_SECONDS
)
for module in (service_module, background_task_executor_module, attempts_repository_module):
monkeypatch.setattr(module, "naive_utc_now", lambda: later)
recovering = BackgroundTaskExecutor()
await recovering._recover_pending_retries_once(resume_fresh=False)
await asyncio.gather(*recovering._background_tasks)
execute.assert_awaited_once()
assert execute.await_args is not None
assert execute.await_args.kwargs["workflow_run_id"] == workflow_run_id
assert execute.await_args.kwargs["attempt_number"] == 1
assert execute.await_args.kwargs["prepared_attempt_claimed"] is True
@pytest.mark.asyncio
@pytest.mark.parametrize("persistent", [False, True], ids=["transient", "persistent"])
async def test_first_dispatch_is_not_scheduled_without_a_durable_queued_attempt(
monkeypatch: pytest.MonkeyPatch, persistent: bool
) -> None:
organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
monkeypatch.setattr(
app.DATABASE.workflow_runs,
"get_workflow_run",
AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None, status=WorkflowRunStatus.created)),
)
monkeypatch.setattr(app.DATABASE.workflow_run_attempts, "get_attempts", AsyncMock(return_value=[]))
failure = RuntimeError("temporary database failure")
queue_write = AsyncMock(side_effect=failure if persistent else [failure, True])
monkeypatch.setattr(app.DATABASE.workflow_runs, "queue_initial_dispatch", queue_write)
sleep = AsyncMock()
monkeypatch.setattr(retry_policy_module, "asyncio", ScopedAsyncio(sleep=sleep))
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
dispatch_kwargs: dict[str, Any] = {
"request": None,
"background_tasks": None,
"organization": organization,
"workflow_id": "wf_test",
"workflow_run_id": "wr_test",
"workflow_permanent_id": "wpid_test",
"max_steps_override": None,
"api_key": None,
"browser_session_id": None,
"block_labels": None,
"block_outputs": None,
}
if persistent:
with pytest.raises(RuntimeError, match="temporary database failure"):
await executor.execute_workflow(**dispatch_kwargs)
assert queue_write.await_count == 3
executor._schedule.assert_not_called()
assert [call.args[0] for call in sleep.await_args_list] == [0.1, 0.2]
else:
await executor.execute_workflow(**dispatch_kwargs)
assert queue_write.await_count == 2
executor._schedule.assert_called_once()
assert [call.args[0] for call in sleep.await_args_list] == [0.1]
@pytest.mark.asyncio
async def test_run_canceled_before_its_first_dispatch_is_not_scheduled(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
async with sqlite_db.Session() as session:
session.add_all(
[
WorkflowRunModel(
workflow_run_id="wr_canceled",
organization_id="org_test",
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
status="canceled",
),
WorkflowRunAttemptModel(
workflow_run_id="wr_canceled",
organization_id="org_test",
attempt_number=1,
status="canceled",
retry_decision="revoked",
),
]
)
await session.commit()
organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
monkeypatch.setattr(app, "DATABASE", sqlite_db)
# The dispatcher's own read still sees the run as created; the cancel commits before the queued write.
stale_run = SimpleNamespace(sequential_credential_id=None, status=WorkflowRunStatus.created)
monkeypatch.setattr(sqlite_db.workflow_runs, "get_workflow_run", AsyncMock(return_value=stale_run))
dispatcher = BackgroundTaskExecutor()
dispatcher._schedule = MagicMock() # type: ignore[method-assign]
await dispatcher.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_test",
workflow_run_id="wr_canceled",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
dispatcher._schedule.assert_not_called()
async with sqlite_db.Session() as session:
run = await session.get(WorkflowRunModel, "wr_canceled")
assert run is not None and run.status == "canceled" and run.queued_at is None
assert (await sqlite_db.workflow_run_attempts.get_attempts("wr_canceled"))[0].status == "canceled"
@pytest.mark.asyncio
async def test_queue_initial_dispatch_leaves_the_run_created_when_the_attempt_write_fails(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
async with sqlite_db.Session() as session:
session.add_all(
[
WorkflowRunModel(
workflow_run_id="wr_partial",
organization_id="org_test",
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
status="created",
),
WorkflowRunAttemptModel(
workflow_run_id="wr_partial", organization_id="org_test", attempt_number=1, status="created"
),
]
)
await session.commit()
real_update = workflow_runs_repository_module.update
def failing_attempt_update(model: Any) -> Any:
if model is WorkflowRunAttemptModel:
raise RuntimeError("database connection lost")
return real_update(model)
monkeypatch.setattr(workflow_runs_repository_module, "update", failing_attempt_update)
with pytest.raises(RuntimeError, match="database connection lost"):
await sqlite_db.workflow_runs.queue_initial_dispatch("wr_partial", 1)
# Neither row moved: a queued run whose attempt stayed created is invisible to dispatch recovery.
async with sqlite_db.Session() as session:
run = await session.get(WorkflowRunModel, "wr_partial")
assert run is not None and run.status == "created" and run.queued_at is None
assert (await sqlite_db.workflow_run_attempts.get_attempts("wr_partial"))[0].status == "created"
@pytest.mark.asyncio
async def test_scoped_inputs_are_marked_unrecoverable_before_the_dispatch_becomes_recoverable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
monkeypatch.setattr(
app.DATABASE.workflow_runs,
"get_workflow_run",
AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None, status=WorkflowRunStatus.created)),
)
monkeypatch.setattr(
app.DATABASE.workflow_run_attempts,
"get_attempts",
AsyncMock(side_effect=RuntimeError("temporary database failure")),
)
writes: list[str] = []
async def mark_unrecoverable(workflow_run_id: str, attempt_number: int) -> None:
writes.append(f"unrecoverable:{workflow_run_id}:{attempt_number}")
async def mark_queued(workflow_run_id: str, attempt_number: int) -> bool:
writes.append(f"queued:{workflow_run_id}:{attempt_number}")
return True
monkeypatch.setattr(app.DATABASE.workflow_run_attempts, "mark_attempt_inputs_unrecoverable", mark_unrecoverable)
monkeypatch.setattr(app.DATABASE.workflow_runs, "queue_initial_dispatch", mark_queued)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_test",
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id=None,
block_labels=["extract"],
block_outputs=None,
)
# A restart-recovered dispatch has no scoped inputs, so the marker must land before the queued write.
assert writes == ["unrecoverable:wr_test:1", "queued:wr_test:1"]
executor._schedule.assert_called_once()
@pytest.mark.asyncio
async def test_execute_workflow_schedules_plain_attempt_when_attempt_lookup_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
organization = SimpleNamespace(
organization_id="org_test",
default_llm_key=None,
default_secondary_llm_key=None,
)
monkeypatch.setattr(
app.DATABASE.workflow_runs,
"get_workflow_run",
AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None, status=WorkflowRunStatus.created)),
)
attempt_lookup = AsyncMock(side_effect=RuntimeError("temporary database failure"))
monkeypatch.setattr(app.DATABASE.workflow_run_attempts, "get_attempts", attempt_lookup)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
# The owner's own bounded lookup fails too; the dispatch must stay owned by the recovery sweep.
owner = AsyncMock(side_effect=WorkflowRetryAttemptLookupError("execution was not started"))
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", owner)
executor = BackgroundTaskExecutor()
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=organization,
workflow_id="wf_test",
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
await asyncio.gather(*executor._background_tasks)
attempt_lookup.assert_awaited_once_with("wr_test")
assert owner.await_args.kwargs["attempt_number"] == 1
pending_attempt, pending_kwargs = executor._retry_dispatches_needing_recovery[("wr_test", 1)]
assert pending_attempt.attempt_number == 1
assert pending_kwargs["attempt_number"] == 1
assert ("wr_test", 1) in executor._retry_resumes_needing_recovery
@pytest.mark.asyncio
async def test_execute_task_v2_stamps_org_llm_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
organization = SimpleNamespace(
organization_id="org_test",
default_llm_key="CUSTOM_LLM_oat_smart",
default_secondary_llm_key="CUSTOM_LLM_oat_fast",
created_at=None,
)
monkeypatch.setattr(app.DATABASE.organizations, "get_organization", AsyncMock(return_value=organization))
monkeypatch.setattr(
app.DATABASE.observer,
"get_task_v2",
AsyncMock(return_value=SimpleNamespace(workflow_run_id="wr_test")),
)
monkeypatch.setattr(app.DATABASE.observer, "update_task_v2", AsyncMock())
monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run", AsyncMock())
monkeypatch.setattr(
"skyvern.forge.sdk.executor.background_task_executor.initialize_skyvern_state_file",
AsyncMock(),
)
load_custom_llms = AsyncMock()
monkeypatch.setattr(
"skyvern.forge.sdk.api.llm.custom_llm_registry.load_custom_llm_configs_for_organization",
load_custom_llms,
)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
with skyvern_context.scoped(SkyvernContext(organization_id="org_test")) as context:
await executor.execute_task_v2(
request=None,
background_tasks=None,
organization_id="org_test",
task_v2_id="task_v2_test",
max_steps_override=None,
browser_session_id=None,
)
assert context.org_default_llm_key == "CUSTOM_LLM_oat_smart"
assert context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"
load_custom_llms.assert_awaited_once_with(app.DATABASE, "org_test")
@pytest.mark.asyncio
async def test_scheduled_task_is_retained_until_done() -> None:
"""A bare create_task reference can be garbage collected mid-flight."""
release = asyncio.Event()
async def work() -> None:
await release.wait()
executor = BackgroundTaskExecutor()
executor._schedule(None, work)
await asyncio.sleep(0)
assert len(executor._background_tasks) == 1
release.set()
await asyncio.sleep(0)
await asyncio.sleep(0)
assert executor._background_tasks == set()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"delivery_case",
[
"no_attempt_rows",
"attempt_rows",
"webhook_error",
"lookup_failure",
"lookup_failure_twice",
"helper_lookup_failure",
"helper_lookup_failure_already_final",
"row_appears",
],
)
async def test_execute_workflow_fails_closed_for_stamped_sequential_credential(
monkeypatch: pytest.MonkeyPatch, delivery_case: str, caplog: pytest.LogCaptureFixture
) -> None:
workflow_run = SimpleNamespace(sequential_credential_id="cred_sequential", browser_session_id=None)
terminal_run = SimpleNamespace(
status=WorkflowRunStatus.failed,
failure_reason="Sequential credential execution is unavailable in the background executor",
finished_at=datetime.now(UTC),
)
get_run = AsyncMock(side_effect=[workflow_run, terminal_run])
monkeypatch.setattr(
app.DATABASE.workflow_runs,
"get_workflow_run",
get_run,
)
monkeypatch.setattr(
app.DATABASE.workflow_run_attempts,
"get_attempts",
AsyncMock(
return_value=[SimpleNamespace(attempt_number=1)] if delivery_case == "attempt_rows" else [],
side_effect={
"lookup_failure": [RuntimeError("db down"), []],
"lookup_failure_twice": [RuntimeError("db down"), RuntimeError("db down")],
"helper_lookup_failure": [[], RuntimeError("db down")],
"helper_lookup_failure_already_final": [[], RuntimeError("db down")],
"row_appears": [[], [SimpleNamespace(attempt_number=1)]],
}.get(delivery_case),
),
)
mark_failed = AsyncMock(
return_value=None if delivery_case == "helper_lookup_failure_already_final" else terminal_run
)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", mark_failed)
decision = RetryDecision(False, 1, 0, True, "sequential_credential_unsupported")
finalize = AsyncMock(return_value=decision)
monkeypatch.setattr(background_task_executor_module, "finalize_abandoned_attempt", finalize)
release = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", release)
webhook = AsyncMock(side_effect=RuntimeError("delivery failed") if delivery_case == "webhook_error" else None)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_webhook", webhook)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
with pytest.raises(BackgroundSequentialCredentialUnsupported, match="background executor"):
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=SimpleNamespace(organization_id="org_test"),
workflow_id="wf_test",
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key="test-api-key",
browser_session_id=None,
block_labels=None,
block_outputs=None,
)
executor._schedule.assert_not_called()
mark_failed.assert_awaited_once_with(
workflow_run_id="wr_test",
failure_reason=(
"Sequential credential execution is unavailable in the background executor; "
"the run failed closed before execution."
),
cascade_children=True,
)
if delivery_case in {"attempt_rows", "row_appears"}:
get_run.assert_awaited_with(workflow_run_id="wr_test", organization_id="org_test")
finalize.assert_awaited_once_with(
workflow_run_id="wr_test",
organization_id="org_test",
reason="sequential_credential_unsupported",
status=terminal_run.status,
failure_reason=terminal_run.failure_reason,
finished_at=terminal_run.finished_at,
)
release.assert_awaited_once_with(terminal_run, decision, api_key="test-api-key")
webhook.assert_not_awaited()
else:
if delivery_case in {"lookup_failure_twice", "helper_lookup_failure_already_final"}:
webhook.assert_not_awaited()
else:
webhook.assert_awaited_once_with(
terminal_run,
api_key="test-api-key",
claim_kind=None,
finalizer="sequential_credential_rejection"
if delivery_case == "helper_lookup_failure"
else "failed_before_execution",
)
release.assert_not_awaited()
finalize.assert_not_awaited()
if delivery_case in {"lookup_failure_twice", "helper_lookup_failure", "helper_lookup_failure_already_final"}:
assert (
"Failed to fail the rejected run through the attempt-less path; writing the failure directly" in caplog.text
)
@pytest.mark.asyncio
async def test_execute_workflow_closes_forced_session_before_credential_fence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
workflow_run = SimpleNamespace(
sequential_credential_id="cred_sequential",
browser_session_id="pbs_forced",
)
monkeypatch.setattr(
app.DATABASE.workflow_runs,
"get_workflow_run",
AsyncMock(return_value=workflow_run),
)
monkeypatch.setattr(
app.DATABASE.browser_sessions,
"get_persistent_browser_session",
AsyncMock(return_value=SimpleNamespace(runnable_type=FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE)),
)
close_session = AsyncMock()
monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "close_session", close_session)
monkeypatch.setattr(
app.WORKFLOW_SERVICE,
"mark_workflow_run_as_failed_if_not_final",
AsyncMock(),
)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
with pytest.raises(SkyvernException, match="background executor"):
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=SimpleNamespace(organization_id="org_test"),
workflow_id="wf_test",
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id="pbs_forced",
block_labels=None,
block_outputs=None,
)
close_session.assert_awaited_once_with("org_test", "pbs_forced", reason=BrowserSessionCloseReason.aborted)
executor._schedule.assert_not_called()
@pytest.mark.asyncio
async def test_execute_workflow_terminalizes_when_forced_session_cleanup_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
workflow_run = SimpleNamespace(
sequential_credential_id="cred_sequential",
browser_session_id="pbs_forced",
)
monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run", AsyncMock(return_value=workflow_run))
monkeypatch.setattr(
app.DATABASE.browser_sessions,
"get_persistent_browser_session",
AsyncMock(return_value=SimpleNamespace(runnable_type=FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE)),
)
monkeypatch.setattr(
app.PERSISTENT_SESSIONS_MANAGER,
"close_session",
AsyncMock(side_effect=RuntimeError("cleanup failed")),
)
mark_failed = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", mark_failed)
executor = BackgroundTaskExecutor()
executor._schedule = MagicMock() # type: ignore[method-assign]
with pytest.raises(SkyvernException, match="background executor"):
await executor.execute_workflow(
request=None,
background_tasks=None,
organization=SimpleNamespace(organization_id="org_test"),
workflow_id="wf_test",
workflow_run_id="wr_test",
workflow_permanent_id="wpid_test",
max_steps_override=None,
api_key=None,
browser_session_id="pbs_forced",
block_labels=None,
block_outputs=None,
)
mark_failed.assert_awaited_once()
executor._schedule.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize("unrecoverable_inputs", [False, True])
async def test_startup_recovers_prepared_retry_without_changing_its_inputs(
monkeypatch: pytest.MonkeyPatch, unrecoverable_inputs: bool
) -> None:
attempt = SimpleNamespace(
workflow_run_id="wr_prepared_resume",
organization_id="org_test",
attempt_number=2,
status="queued",
started_at=None,
retry_decision=None,
decision_reason="unrecoverable_block_outputs" if unrecoverable_inputs else None,
pinned_browser_session_id="pinned-session",
modified_at=datetime.now(UTC) - timedelta(seconds=RETRY_DECISION_GRACE_SECONDS + 1),
)
run = SimpleNamespace(
workflow_run_id=attempt.workflow_run_id,
organization_id=attempt.organization_id,
status=WorkflowRunStatus.queued,
started_at=None,
failure_reason=None,
finished_at=None,
)
organization = SimpleNamespace(organization_id="org_test")
fail = AsyncMock(return_value=True)
claim_stamp = datetime.now(UTC)
database = SimpleNamespace(
workflow_run_attempts=SimpleNamespace(
list_attempts_needing_recovery=AsyncMock(return_value=[attempt]),
list_stale_dispatch_claims=AsyncMock(return_value=[]),
fail_prepared_workflow_run=fail,
claim_prepared_attempt_execution=AsyncMock(return_value=claim_stamp),
get_attempts=AsyncMock(return_value=[attempt]),
),
workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(return_value=run)),
organizations=SimpleNamespace(
get_organization=AsyncMock(return_value=organization),
get_valid_org_auth_token=AsyncMock(return_value=SimpleNamespace(token="api-token")),
),
)
monkeypatch.setattr(app, "DATABASE", database)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
prepare = AsyncMock()
monkeypatch.setattr(background_task_executor_module, "prepare_next_attempt_result", prepare)
abandoned = AsyncMock(return_value=RetryDecision(False, 2, 0, True, "unrecoverable_block_outputs"))
monkeypatch.setattr(background_task_executor_module, "finalize_abandoned_attempt", abandoned)
execute = AsyncMock()
release = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", release)
executor = BackgroundTaskExecutor()
await executor._recover_pending_retries_once(resume_fresh=True)
await asyncio.gather(*executor._background_tasks)
prepare.assert_not_awaited()
if unrecoverable_inputs:
execute.assert_not_awaited()
fail.assert_awaited_once()
abandoned.assert_awaited_once()
release.assert_awaited_once()
else:
fail.assert_not_awaited()
abandoned.assert_not_awaited()
execute.assert_awaited_once_with(
workflow_run_id=attempt.workflow_run_id,
api_key="api-token",
organization=organization,
browser_session_id="pinned-session",
block_labels=None,
block_outputs=None,
need_call_webhook=True,
attempt_number=2,
prepared_attempt_claimed=True,
dispatch_claim_started_at=claim_stamp,
on_execution_start=ANY,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("ownership", ["absent", "running", "unknown"])
async def test_periodic_recovery_resumes_prepared_attempt_after_it_ages(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch, ownership: str
) -> None:
now = datetime.now(UTC).replace(tzinfo=None)
async with sqlite_db.Session() as session:
session.add_all(
[
WorkflowRunModel(
workflow_run_id="wr_aged",
organization_id="org_test",
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
status="queued",
queued_at=now,
),
WorkflowRunAttemptModel(
workflow_run_id="wr_aged",
organization_id="org_test",
attempt_number=2,
status="queued",
modified_at=now,
),
]
)
await session.commit()
monkeypatch.setattr(app, "DATABASE", sqlite_db)
monkeypatch.setattr(app.AGENT_FUNCTION, "get_workflow_run_execution_status", AsyncMock(return_value=ownership))
monkeypatch.setattr(
sqlite_db.organizations, "get_organization", AsyncMock(return_value=SimpleNamespace(organization_id="org_test"))
)
monkeypatch.setattr(BackgroundTaskExecutor, "_get_valid_api_key", AsyncMock(return_value="token"))
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
executor = BackgroundTaskExecutor()
await executor._recover_pending_retries_once(resume_fresh=True)
assert not executor._scheduled_retry_resumes
async with sqlite_db.Session() as session:
attempt = await session.get(WorkflowRunAttemptModel, ("wr_aged", 2))
attempt.modified_at = now - timedelta(seconds=background_task_executor_module.LEASE_TAKEOVER_SECONDS + 1)
await session.commit()
await executor._recover_pending_retries_once(resume_fresh=False)
await asyncio.gather(*executor._background_tasks)
assert execute.await_count == int(ownership == "absent")
@pytest.mark.asyncio
async def test_concurrent_prepared_recovery_dispatches_only_claim_winner(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
async with sqlite_db.Session() as session:
session.add_all(
[
WorkflowRunModel(
workflow_run_id="wr_race",
organization_id="org_test",
workflow_id="wf_test",
workflow_permanent_id="wpid_test",
status="queued",
),
WorkflowRunAttemptModel(
workflow_run_id="wr_race", organization_id="org_test", attempt_number=2, status="queued"
),
]
)
await session.commit()
attempt = (await sqlite_db.workflow_run_attempts.get_attempts("wr_race"))[0]
monkeypatch.setattr(app, "DATABASE", sqlite_db)
monkeypatch.setattr(app.AGENT_FUNCTION, "get_workflow_run_execution_status", AsyncMock(return_value="absent"))
monkeypatch.setattr(
sqlite_db.organizations, "get_organization", AsyncMock(return_value=SimpleNamespace(organization_id="org_test"))
)
both_ready = asyncio.Event()
readers = 0
async def token(_self: BackgroundTaskExecutor, _org: str) -> str:
nonlocal readers
readers += 1
if readers == 2:
both_ready.set()
await both_ready.wait()
return "token"
monkeypatch.setattr(BackgroundTaskExecutor, "_get_valid_api_key", token)
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
execute = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
release = AsyncMock()
monkeypatch.setattr(app.WORKFLOW_SERVICE, "_run_terminal_side_effects_with_retries", release)
await asyncio.wait_for(
asyncio.gather(
BackgroundTaskExecutor()._resume_pending_retry(attempt),
BackgroundTaskExecutor()._resume_pending_retry(attempt),
),
timeout=5,
)
execute.assert_awaited_once()
release.assert_not_awaited()
current = (await sqlite_db.workflow_run_attempts.get_attempts("wr_race"))[0]
assert current.started_at is not None
assert current.retry_decision is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"entrance, escaped_exception, lane, outcome",
[
("in_process", False, "browser_session_id", "clear"),
("in_process", False, "sequential_credential_id", "cancel"),
("in_process", False, "none", "clear"),
("in_process", True, "browser_address", "clear"),
("in_process", True, "sequential_credential_id", "task_cancel"),
("in_process", True, "run_sequentially", "clear"),
("recovery_pending", False, "sequential_credential_id", "clear"),
("recovery_pending", False, "sequential_credential_id", "timeout"),
("recovery_pending", False, "depends_on_workflow_run_id", "clear"),
("recovery_prepared", False, "sequential_key", "clear"),
("recovery_prepared", False, "reuse_bound_key", "clear"),
("recovery_prepared", False, "sequential_credential_id", "query_error"),
],
)
async def test_in_process_retry_reacquires_serialized_lane(
sqlite_db: AgentDB,
monkeypatch: pytest.MonkeyPatch,
entrance: str,
escaped_exception: bool,
lane: str,
outcome: str,
) -> None:
monkeypatch.setattr(app, "DATABASE", sqlite_db)
svc = app.WORKFLOW_SERVICE
now = datetime.now(UTC).replace(tzinfo=None)
identity = {lane: "shared-lane"} if lane not in {"none", "run_sequentially", "depends_on_workflow_run_id"} else {}
async with sqlite_db.Session() as session:
session.add(OrganizationModel(organization_id="org_test", organization_name="Test Organization"))
await session.commit()
session.add_all(
[
WorkflowModel(
workflow_id="wf_lane",
workflow_permanent_id="wpid_lane",
organization_id="org_test",
title="Retry lane",
version=1,
run_sequentially=lane == "run_sequentially",
workflow_definition={
"blocks": [],
"parameters": [],
"retry_policy": {"max_retries": 1, "delay_seconds": 0, "retry_on": [{"status": "failed"}]},
},
),
WorkflowRunModel(
workflow_run_id="wr_retry",
workflow_id="wf_lane",
workflow_permanent_id="wpid_lane",
organization_id="org_test",
status="failed",
created_at=now - timedelta(seconds=40),
started_at=now - timedelta(seconds=30),
finished_at=now,
max_elapsed_time_minutes=1,
depends_on_workflow_run_id="wr_owner" if lane == "depends_on_workflow_run_id" else None,
**identity,
),
WorkflowRunModel(
workflow_run_id="wr_owner",
workflow_id="wf_lane",
workflow_permanent_id="wpid_lane",
organization_id="org_test",
status="running",
queued_at=now,
**identity,
),
WorkflowRunAttemptModel(
workflow_run_id="wr_retry",
organization_id="org_test",
attempt_number=1,
status="failed",
retry_decision="retry",
started_at=now - timedelta(seconds=30),
finished_at=now,
interim_webhook_sent_at=now,
next_attempt_at=now,
pinned_browser_session_id=identity.get("browser_session_id"),
),
]
)
await session.commit()
monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "remove_workflow_run_context", MagicMock())
monkeypatch.setattr(svc, "_run_interim_side_effects_with_retries", AsyncMock(return_value="released"))
final_effects = AsyncMock(return_value="released")
monkeypatch.setattr(svc, "_run_terminal_side_effects_with_retries", final_effects)
monkeypatch.setattr(app.DATABASE.tasks, "get_tasks_by_workflow_run_id", AsyncMock(return_value=[]))
monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=[]))
monkeypatch.setattr(svc, "_cascade_child_entities_on_terminal", AsyncMock())
monkeypatch.setattr(svc, "_sync_task_run_from_workflow_run", AsyncMock())
waiting = asyncio.Event()
release = asyncio.Event()
executed: list[int] = []
async def execute(**kwargs: object) -> WorkflowRun:
number = kwargs["attempt_number"]
assert isinstance(number, int)
executed.append(number)
if number == 1:
if escaped_exception:
raise RuntimeError("attempt failed")
return await svc.get_workflow_run("wr_retry", "org_test")
if lane != "none":
owner = await svc.get_workflow_run("wr_owner", "org_test")
assert owner.status == WorkflowRunStatus.completed, "retry started while the serialized lane was occupied"
completed = await sqlite_db.workflow_runs.update_workflow_run_if_not_final(
workflow_run_id="wr_retry", status=WorkflowRunStatus.completed
)
assert completed is not None
await service_module.on_terminal_transition(completed, completed.status, None, None, attempt_number=number)
return completed
async def sleep(seconds: float) -> None:
if seconds == 0:
await asyncio.sleep(0)
return
waiting.set()
await release.wait()
admission_timeout: asyncio.Timeout | None = None
def timeout(seconds: float | None) -> asyncio.Timeout:
nonlocal admission_timeout
if outcome == "timeout" and seconds == 0.05:
# Arm the real cancellation only after the occupied lane is observed;
# database latency must not decide whether this ordering test passes.
admission_timeout = asyncio.timeout(None)
return admission_timeout
return asyncio.timeout(seconds)
monkeypatch.setattr(svc, "execute_workflow", execute)
monkeypatch.setattr(service_module, "asyncio", ScopedAsyncio(sleep=sleep, timeout=timeout))
clearance_query = sqlite_db.workflow_runs.get_blocking_sequential_workflow_run
query_failed = False
async def check_lane(workflow_run_id: str) -> WorkflowRun | None:
nonlocal query_failed
assert lane != "none", "unserialized retry must not enter the admission gate"
if outcome == "query_error" and not query_failed:
query_failed = True
raise RuntimeError("primary unavailable")
return await clearance_query(workflow_run_id)
monkeypatch.setattr(sqlite_db.workflow_runs, "get_blocking_sequential_workflow_run", check_lane)
remaining_budget = service_module._get_workflow_run_max_elapsed_timeout_seconds
def budget(run: WorkflowRun) -> float:
remaining = remaining_budget(run)
assert 55 < remaining <= 60, "gate must use the prepared attempt's fresh runtime budget"
return 0.05 if outcome == "timeout" else remaining
monkeypatch.setattr(service_module, "_get_workflow_run_max_elapsed_timeout_seconds", budget)
recovering = entrance != "in_process"
if recovering:
monkeypatch.setattr(BackgroundTaskExecutor, "_get_valid_api_key", AsyncMock(return_value="token"))
monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
if entrance == "recovery_prepared":
preparation = await service_module.prepare_next_attempt_result(
"wr_retry", "org_test", 1, clear_browser_address=False
)
assert preparation.status == "inserted"
recovered_attempt = (await sqlite_db.workflow_run_attempts.get_attempts("wr_retry"))[-1]
task = asyncio.create_task(BackgroundTaskExecutor()._resume_pending_retry(recovered_attempt))
else:
task = asyncio.create_task(
svc.execute_workflow_with_retries(
workflow_run_id="wr_retry", api_key=None, organization=SimpleNamespace(organization_id="org_test")
)
)
wait_task = asyncio.create_task(waiting.wait())
try:
if lane == "none":
await asyncio.wait_for(task, timeout=30)
result = await svc.get_workflow_run("wr_retry", "org_test")
assert result.status == WorkflowRunStatus.completed
assert executed == ([2] if recovering else [1, 2])
assert not waiting.is_set()
return
# A generous bound: the assertion is about ordering, and a loaded CI shard can take seconds to get here.
done, _pending = await asyncio.wait({task, wait_task}, timeout=30, return_when=asyncio.FIRST_COMPLETED)
if task in done:
await task
assert wait_task in done, "retry did not wait for the occupied lane"
attempts = await sqlite_db.workflow_run_attempts.get_attempts("wr_retry")
assert attempts[-1].attempt_number == 2
assert (attempts[-1].started_at is not None) == (entrance == "recovery_prepared")
assert executed == ([] if recovering else [1])
if outcome in {"clear", "query_error"}:
await sqlite_db.workflow_runs.update_workflow_run_if_not_final(
workflow_run_id="wr_owner", status=WorkflowRunStatus.completed
)
release.set()
elif outcome == "cancel":
await svc.mark_workflow_run_as_canceled("wr_retry")
release.set()
elif outcome == "task_cancel":
task.cancel()
elif outcome == "timeout":
assert admission_timeout is not None
admission_timeout.reschedule(asyncio.get_running_loop().time())
if outcome == "task_cancel":
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=5)
result = await svc.get_workflow_run("wr_retry", "org_test")
else:
await asyncio.wait_for(task, timeout=5)
result = await svc.get_workflow_run("wr_retry", "org_test")
if outcome in {"clear", "query_error"}:
assert result.status == WorkflowRunStatus.completed
assert executed == ([2] if recovering else [1, 2])
elif outcome == "task_cancel":
# Only shutdown cancels the owner: the prepared attempt stays recoverable, nothing is finalized.
assert not result.status.is_final()
assert executed == ([] if recovering else [1])
attempt = (await sqlite_db.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert attempt.attempt_number == 2
assert attempt.retry_decision is None
assert (attempt.started_at is not None) == (entrance == "recovery_prepared")
final_effects.assert_not_awaited()
else:
assert result.status == (
WorkflowRunStatus.timed_out if outcome == "timeout" else WorkflowRunStatus.canceled
)
assert executed == ([] if recovering else [1])
attempt = (await sqlite_db.workflow_run_attempts.get_attempts("wr_retry"))[-1]
assert (attempt.started_at is not None) == (entrance == "recovery_prepared")
assert attempt.retry_decision in {"final", "revoked", "abandoned"}
assert not await sqlite_db.workflow_run_attempts.claim_prepared_attempt_execution("wr_retry", "org_test", 2)
assert final_effects.await_args.args[1].attempt_number == 2
finally:
for pending in [task, wait_task]:
if not pending.done():
pending.cancel()
await asyncio.gather(task, wait_task, return_exceptions=True)
@pytest.mark.asyncio
async def test_mark_attempt_started_never_revives_a_finalized_attempt(
sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A cancel can finalize the attempt between the run's running write and the attempt's; the
late running write must lose."""
monkeypatch.setattr(app, "DATABASE", sqlite_db)
async with sqlite_db.Session() as session:
session.add(OrganizationModel(organization_id="org_test", organization_name="Test Organization"))
await session.commit()
session.add_all(
[
WorkflowRunAttemptModel(
workflow_run_id="wr_live", organization_id="org_test", attempt_number=1, status="queued"
),
WorkflowRunAttemptModel(
workflow_run_id="wr_canceled",
organization_id="org_test",
attempt_number=1,
status="canceled",
retry_decision="revoked",
decision_reason="cancel",
),
]
)
await session.commit()
assert await mark_attempt_started("wr_live", 1) is True
assert await mark_attempt_started("wr_canceled", 1) is False
live = (await sqlite_db.workflow_run_attempts.get_attempts("wr_live"))[0]
canceled = (await sqlite_db.workflow_run_attempts.get_attempts("wr_canceled"))[0]
assert (live.status, live.started_at is not None) == ("running", True)
assert (canceled.status, canceled.started_at, canceled.retry_decision) == ("canceled", None, "revoked")
@pytest.mark.asyncio
@pytest.mark.parametrize("durable_decision", ["retry", "final"])
async def test_late_owner_failure_leaves_a_recorded_retry_to_the_next_sweep(
monkeypatch: pytest.MonkeyPatch, durable_decision: str
) -> None:
# The decision read after execution is outside the bounded lookup path. Its failure must not
# strand a recorded retry until the periodic sweep abandons it as stale.
executor = BackgroundTaskExecutor()
attempt = SimpleNamespace(workflow_run_id="wr_late", organization_id="org_late", attempt_number=1, started_at=None)
latest = SimpleNamespace(
workflow_run_id="wr_late",
organization_id="org_late",
attempt_number=2,
retry_decision=durable_decision,
next_attempt_prepared_at=None,
next_attempt_at=datetime.now(UTC),
)
monkeypatch.setattr(app.DATABASE.workflow_run_attempts, "get_attempts", AsyncMock(return_value=[attempt, latest]))
async def entry(**kwargs: Any) -> None:
kwargs["on_execution_start"]()
raise RuntimeError("decision read unavailable")
monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", AsyncMock(side_effect=entry))
with pytest.raises(RuntimeError, match="decision read unavailable"):
await executor._dispatch_retry_resume(attempt, {"workflow_run_id": "wr_late", "attempt_number": 2})
key = ("wr_late", 2)
assert key not in executor._retry_dispatches_needing_recovery
assert (key in executor._retry_resumes_needing_recovery) is (durable_decision == "retry")
scheduled = Mock()
monkeypatch.setattr(executor, "_schedule_retry_resume", scheduled)
monkeypatch.setattr(executor, "_recover_stale_dispatch_claims", AsyncMock())
monkeypatch.setattr(executor, "_release_unreleased_terminal_effects", AsyncMock())
async def recover(*, process_attempt: Any) -> None:
await process_attempt(latest, False)
monkeypatch.setattr(app.WORKFLOW_SERVICE, "recover_pending_workflow_attempts", recover)
await executor._recover_pending_retries_once(resume_fresh=False)
if durable_decision == "retry":
scheduled.assert_called_once_with(latest)
else:
scheduled.assert_not_called()
assert key not in executor._retry_resumes_needing_recovery
def test_background_task_set_is_isolated_across_event_loops(monkeypatch: pytest.MonkeyPatch) -> None:
"""A fire-and-forget task scheduled by one WorkflowService under one event loop must never
surface in a freshly built WorkflowService bound to a different loop. When it does, draining
the later service gathers a future from loop A and raises
``ValueError: The future belongs to a different loop`` — the shard-2 CI failure this repairs.
Loop A stays open with its task pending through the ownership assertion; the finally cancels
and awaits that task on loop A so nothing is destroyed while pending.
"""
loop_a = asyncio.new_event_loop()
loop_b = asyncio.new_event_loop()
stale_future = loop_a.create_future()
async def block_until_released(**_: object) -> None:
await stale_future
async def schedule_pending_task() -> WorkflowService:
svc = WorkflowService()
monkeypatch.setattr(
WorkflowService,
"_record_workflow_run_metadata_best_effort",
staticmethod(block_until_released),
)
svc._record_workflow_run_metadata_in_background(
workflow_run_id="wr_loop_a",
organization_id="org_loop_a",
run_metadata={"env": "prod"},
)
return svc
svc_a = loop_a.run_until_complete(schedule_pending_task())
leaked = set(svc_a._background_tasks)
try:
assert leaked, "expected a pending fire-and-forget task on loop A"
# Loop A is still open and its task still pending. A freshly built service (as the autouse
# fixture builds one per test) must not observe loop A's task; when it does, draining its
# own set under loop B gathers loop A's future and raises the cross-loop ValueError from CI.
svc_b = WorkflowService()
assert set(svc_b._background_tasks).isdisjoint(leaked)
async def drain() -> None:
await asyncio.gather(*svc_b._background_tasks)
loop_b.run_until_complete(drain())
finally:
async def cancel_leaked() -> None:
for task in leaked:
task.cancel()
await asyncio.gather(*leaked, return_exceptions=True)
loop_a.run_until_complete(cancel_leaked())
loop_a.close()
loop_b.close()