Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix four workflow-retry follow-ups: durable no-policy failures, Batch attempt resume, download dedupe cutoff, analyzer retry state (SKY-15922) (#8560)
Shuchang Zhenge76ed6644ca5
diff --git a/skyvern/forge/sdk/artifact/storage/azure.py b/skyvern/forge/sdk/artifact/storage/azure.pyindex 9986c24bd..5123dab5f 100644--- a/skyvern/forge/sdk/artifact/storage/azure.py+++ b/skyvern/forge/sdk/artifact/storage/azure.py@@ -663,7 +663,9 @@ class AzureStorage(BaseStorage):if skipped_files:raise DownloadSaveIncompleteError(skipped_files)- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:+ async def get_downloaded_files(+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None+ ) -> list[FileInfo]:# Artifact-first — see s3.py::get_downloaded_files for rationale. When# the keyring isn't configured (OSS default) or no artifact rows exist# (legacy run pre-SKY-8861) we fall back to the legacy listing path so@@ -673,7 +675,9 @@ class AzureStorage(BaseStorage):download_artifacts = await self._list_download_artifacts_safe(organization_id=organization_id, run_id=run_id)- download_artifacts = dedupe_run_scoped_download_artifacts(download_artifacts)+ download_artifacts = dedupe_run_scoped_download_artifacts(+ download_artifacts, attempt_started_at=attempt_started_at+ )if download_artifacts:return await _file_infos_from_download_artifacts(download_artifacts)diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.pyindex cc210759a..416b91332 100644--- a/skyvern/forge/sdk/artifact/storage/base.py+++ b/skyvern/forge/sdk/artifact/storage/base.py@@ -249,7 +249,19 @@ def download_checksums_by_uri(artifacts: list[Artifact]) -> dict[str, str]:return {artifact.uri: artifact.checksum for artifact in artifacts if artifact.uri and artifact.checksum}-def dedupe_run_scoped_download_artifacts(artifacts: list[Artifact]) -> list[Artifact]:+def is_download_timestamp_current(timestamp: datetime | None, attempt_started_at: datetime | None) -> bool:+ if attempt_started_at is None or timestamp is None:+ return True+ if timestamp.tzinfo is None:+ timestamp = timestamp.replace(tzinfo=UTC)+ if attempt_started_at.tzinfo is None:+ attempt_started_at = attempt_started_at.replace(tzinfo=UTC)+ return timestamp >= attempt_started_at+++def dedupe_run_scoped_download_artifacts(+ artifacts: list[Artifact], attempt_started_at: datetime | None = None+) -> list[Artifact]:"""Collapse the two representations of one persistent-session download in a run's DOWNLOAD set.A download can be registered both session-produced (``browser_session_id`` set — run-bound at@@ -258,14 +270,26 @@ def dedupe_run_scoped_download_artifacts(artifacts: list[Artifact]) -> list[Artithe run-scoped one as canonical. Pairing is one-for-one: a checksum with N run-scoped rows drops atmost N session rows, so two legitimate same-content session downloads backed by a single run-scopedrow keep one. Order is preserved; distinct/absent checksums, twinless session rows, and multiple- run-scoped rows are all kept. Run-scoped only — never call on a session listing.+ run-scoped rows are all kept. Rows before the attempt cutoff pass through without pairing.+ Run-scoped only — never call on a session listing."""- run_scoped_checksums = Counter(a.checksum for a in artifacts if a.browser_session_id is None and a.checksum)+ run_scoped_checksums = Counter(+ a.checksum+ for a in artifacts+ if a.browser_session_id is None+ and a.checksum+ and is_download_timestamp_current(a.modified_at or a.created_at, attempt_started_at)+ )if not run_scoped_checksums:return artifactskept: list[Artifact] = []for a in artifacts:- if a.browser_session_id is not None and a.checksum and run_scoped_checksums[a.checksum] > 0:+ if (+ a.browser_session_id is not None+ and a.checksum+ and run_scoped_checksums[a.checksum] > 0+ and is_download_timestamp_current(a.modified_at or a.created_at, attempt_started_at)+ ):run_scoped_checksums[a.checksum] -= 1continuekept.append(a)@@ -508,21 +532,16 @@ class BaseStorage(ABC):return []async def get_current_attempt_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:- files = await self.get_downloaded_files(organization_id=organization_id, run_id=run_id)started_at = await get_download_retry_started_at(organization_id, run_id)- if started_at is None:- return files- current_files = []- for file_info in files:- modified_at = file_info.modified_at- if modified_at is not None and modified_at.tzinfo is None:- modified_at = modified_at.replace(tzinfo=UTC)- if modified_at is None or modified_at >= started_at:- current_files.append(file_info)- return current_files-- @abstractmethod- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:+ files = await self.get_downloaded_files(+ organization_id=organization_id, run_id=run_id, attempt_started_at=started_at+ )+ return [file_info for file_info in files if is_download_timestamp_current(file_info.modified_at, started_at)]++ @abstractmethod+ async def get_downloaded_files(+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None+ ) -> list[FileInfo]:pass@abstractmethoddiff --git a/skyvern/forge/sdk/artifact/storage/gcs.py b/skyvern/forge/sdk/artifact/storage/gcs.pyindex 3a117fc9d..aeedc9b8c 100644--- a/skyvern/forge/sdk/artifact/storage/gcs.py+++ b/skyvern/forge/sdk/artifact/storage/gcs.py@@ -677,7 +677,9 @@ class GcsStorage(BaseStorage):if skipped_files:raise DownloadSaveIncompleteError(skipped_files)- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:+ async def get_downloaded_files(+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None+ ) -> list[FileInfo]:# Artifact-first — see s3.py::get_downloaded_files for rationale. When# the keyring isn't configured (OSS default) or no artifact rows exist# (legacy run) we fall back to the legacy listing path so@@ -687,7 +689,9 @@ class GcsStorage(BaseStorage):download_artifacts = await self._list_download_artifacts_safe(organization_id=organization_id, run_id=run_id)- download_artifacts = dedupe_run_scoped_download_artifacts(download_artifacts)+ download_artifacts = dedupe_run_scoped_download_artifacts(+ download_artifacts, attempt_started_at=attempt_started_at+ )if download_artifacts:return await _file_infos_from_download_artifacts(download_artifacts)diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.pyindex c1cec6e98..2365004de 100644--- a/skyvern/forge/sdk/artifact/storage/local.py+++ b/skyvern/forge/sdk/artifact/storage/local.py@@ -564,7 +564,9 @@ class LocalStorage(BaseStorage):live_uri = f"file://{Path(get_download_dir(run_id=parts[1])) / file_info.filename}"return [to_downloaded_file_signature(file_info.model_copy(update={"url": live_uri}))]- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:+ async def get_downloaded_files(+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None+ ) -> list[FileInfo]:download_dir = get_download_dir(run_id=run_id)artifacts_by_uri: dict[str, Artifact] = {}if run_id is not None:diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.pyindex 8cbae9cc3..0cd0475b7 100644--- a/skyvern/forge/sdk/artifact/storage/s3.py+++ b/skyvern/forge/sdk/artifact/storage/s3.py@@ -936,7 +936,9 @@ class S3Storage(BaseStorage):if skipped_files:raise DownloadSaveIncompleteError(skipped_files)- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:+ async def get_downloaded_files(+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None+ ) -> list[FileInfo]:# Artifact-first: when a run has DOWNLOAD artifact rows, return them as# the source of truth — the row carries enough to build a short signed# /v1/artifacts/{id}/content URL plus the SHA-256 we persisted at save@@ -957,7 +959,9 @@ class S3Storage(BaseStorage):)if not rows_lookup_failed:download_row_count = len(download_artifacts)- download_artifacts = dedupe_run_scoped_download_artifacts(download_artifacts)+ download_artifacts = dedupe_run_scoped_download_artifacts(+ download_artifacts, attempt_started_at=attempt_started_at+ )if download_artifacts:file_infos = await _file_infos_from_download_artifacts(download_artifacts)if not file_infos:diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.pyindex 1cbee40d9..877b404c6 100644--- a/skyvern/forge/sdk/executor/background_task_executor.py+++ b/skyvern/forge/sdk/executor/background_task_executor.py@@ -32,6 +32,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatusfrom skyvern.forge.sdk.workflow.retry_policy import (LEASE_TAKEOVER_SECONDS,RetryDecision,+ fail_run_without_attempt_row,finalize_abandoned_attempt,get_recorded_decision,latest_attempt_awaiting_preparation,@@ -596,12 +597,27 @@ class BackgroundTaskExecutor(AsyncExecutor):):self._retry_resumes_needing_recovery.discard(key)return- # Initialization belongs to the dispatch: a failure here is re-dispatched by the sweep, whereas a- # created attempt that never reached the dispatch matches no recovery predicate.- await initialize_skyvern_state_file(- workflow_run_id=attempt.workflow_run_id, organization_id=attempt.organization_id- )- await prepare_org_llm_runtime(app.DATABASE, attempt.organization_id, execution_kwargs.get("organization"))+ # Attempt rows let the sweep recover initialization failures; attempt-less runs must fail durably.+ try:+ await initialize_skyvern_state_file(+ workflow_run_id=attempt.workflow_run_id, organization_id=attempt.organization_id+ )+ await prepare_org_llm_runtime(+ app.DATABASE, attempt.organization_id, execution_kwargs.get("organization")+ )+ except Exception as exc:+ if await fail_run_without_attempt_row(+ attempt.workflow_run_id,+ f"Workflow run initialization failed before execution: {type(exc).__name__}: {exc}",+ ):+ self._retry_resumes_needing_recovery.discard(key)+ LOG.warning(+ "Workflow run initialization failed without an attempt row; run is terminal",+ workflow_run_id=attempt.workflow_run_id,+ exc_info=True,+ )+ return+ raiseawait app.WORKFLOW_SERVICE.execute_workflow_with_retries(**execution_kwargs, on_execution_start=on_execution_start)diff --git a/skyvern/forge/sdk/workflow/retry_policy.py b/skyvern/forge/sdk/workflow/retry_policy.pyindex b8da38dad..604456569 100644--- a/skyvern/forge/sdk/workflow/retry_policy.py+++ b/skyvern/forge/sdk/workflow/retry_policy.py@@ -337,6 +337,27 @@ async def ensure_attempt_row(return True+async def fail_run_without_attempt_row(workflow_run_id: str, failure_reason: str) -> bool:+ """Fail a queued run that no recovery sweep can see. Returns False when an attempt row exists."""+ if await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id):+ return False+ try:+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(+ workflow_run_id=workflow_run_id,+ failure_reason=failure_reason,+ )+ except Exception:+ workflow_run = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id)+ if workflow_run is not None and not workflow_run.status.is_final():+ raise+ LOG.warning(+ "Workflow run is terminal after initialization failure despite a finalization error",+ workflow_run_id=workflow_run_id,+ exc_info=True,+ )+ return True++async def queue_initial_attempt(workflow_run_id: str, attempt_number: int) -> bool:"""Queue a new run's durable rows before its dispatch, retrying a failed write.diff --git a/skyvern/services/workflow_schedule_service.py b/skyvern/services/workflow_schedule_service.pyindex 4050c737f..a1a181a39 100644--- a/skyvern/services/workflow_schedule_service.py+++ b/skyvern/services/workflow_schedule_service.py@@ -22,7 +22,7 @@ from skyvern.forge.sdk.api.llm.custom_llm_registry import prepare_org_llm_runtimfrom skyvern.forge.sdk.db.enums import WorkflowRunTriggerTypefrom skyvern.forge.sdk.schemas.workflow_schedules import WorkflowSchedulefrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRequestBody-from skyvern.forge.sdk.workflow.retry_policy import queue_initial_attempt+from skyvern.forge.sdk.workflow.retry_policy import fail_run_without_attempt_row, queue_initial_attemptfrom skyvern.forge.sdk.workflow.schedules import compute_previous_fire_timefrom skyvern.services.workflow_service import prepare_workflowfrom skyvern.utils.files import initialize_skyvern_state_file@@ -222,15 +222,29 @@ class LocalWorkflowScheduleScheduler:returnraise- # A persisted run counts as a delivered fire, so the attempt must be recoverable by the dispatch- # sweep before the initializers below can fail.+ # A persisted run counts as a delivered fire. Queue its attempt for sweep recovery;+ # without an attempt row, an initialization failure must make the run terminal.if not await queue_initial_attempt(workflow_run.workflow_run_id, 1):return- await initialize_skyvern_state_file(- workflow_run_id=workflow_run.workflow_run_id,- organization_id=organization.organization_id,- )- await prepare_org_llm_runtime(app.DATABASE, organization.organization_id, organization)+ try:+ await initialize_skyvern_state_file(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization.organization_id,+ )+ await prepare_org_llm_runtime(app.DATABASE, organization.organization_id, organization)+ except Exception as exc:+ if await fail_run_without_attempt_row(+ workflow_run.workflow_run_id,+ f"Workflow run initialization failed before execution: {type(exc).__name__}: {exc}",+ ):+ LOG.warning(+ "Scheduled workflow initialization failed without an attempt row; run is terminal",+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_schedule_id=schedule.workflow_schedule_id,+ exc_info=True,+ )+ return+ raiseawait app.WORKFLOW_SERVICE.execute_workflow_with_retries(workflow_run_id=workflow_run.workflow_run_id,api_key=None,diff --git a/tests/unit/test_background_task_executor.py b/tests/unit/test_background_task_executor.pyindex 5ec5bb43d..f82e304be 100644--- a/tests/unit/test_background_task_executor.py+++ b/tests/unit/test_background_task_executor.py@@ -2870,14 +2870,120 @@ async def test_execute_workflow_dispatches_from_latest_retry_attempt(monkeypatch@pytest.mark.asyncio-async def test_first_dispatch_initializer_failure_is_retried_by_the_sweep(monkeypatch: pytest.MonkeyPatch) -> None:- organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)- monkeypatch.setattr(- app.DATABASE.workflow_runs,- "get_workflow_run",- AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None, status=WorkflowRunStatus.created)),[email protected]("scenario", ["state_file", "llm_runtime", "lookup_failure", "attempt_row"])+async def test_no_policy_initializer_failure_fails_the_run_durably(+ interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, scenario: str+) -> None:+ database = interim_retry_db+ async with database.Session() as session:+ run = await session.get(WorkflowRunModel, "wr_retry")+ assert run is not None+ run.status = "created"+ run.started_at = None+ run.finished_at = None+ attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))+ assert attempt is not None+ if scenario == "attempt_row":+ attempt.status = "created"+ attempt.retry_decision = None+ attempt.finished_at = None+ attempt.next_attempt_at = None+ else:+ workflow = await session.get(WorkflowModel, "wf_retry")+ assert workflow is not None+ workflow.workflow_definition = {"blocks": [], "parameters": []}+ await session.delete(attempt)+ await session.commit()++ monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())+ monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())+ initializer = "prepare_org_llm_runtime" if scenario == "llm_runtime" else "initialize_skyvern_state_file"+ error = RuntimeError(f"{initializer} unavailable")+ monkeypatch.setattr(background_task_executor_module, initializer, AsyncMock(side_effect=error))+ execute = AsyncMock()+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)+ fail_run = AsyncMock(wraps=app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final)+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", fail_run)+ executor = BackgroundTaskExecutor()+ executor._schedule = MagicMock() # type: ignore[method-assign]+ organization = await database.organizations.get_organization("org_test")+ assert organization is not None+ await executor.execute_workflow(+ request=None,+ background_tasks=None,+ organization=organization,+ workflow_id="wf_retry",+ workflow_run_id="wr_retry",+ workflow_permanent_id="wpid_retry",+ max_steps_override=None,+ api_key=None,+ browser_session_id=None,+ block_labels=None,+ block_outputs=None,)- monkeypatch.setattr(app.DATABASE.workflow_run_attempts, "get_attempts", AsyncMock(return_value=[]))+ _background_tasks, dispatch, attempt, execution_kwargs = executor._schedule.call_args.args+ get_attempts = database.workflow_run_attempts.get_attempts+ if scenario == "lookup_failure":+ monkeypatch.setattr(+ database.workflow_run_attempts, "get_attempts", AsyncMock(side_effect=RuntimeError("db down"))+ )+ await dispatch(attempt, execution_kwargs)+ execute.assert_not_awaited()+ run = await database.workflow_runs.get_workflow_run("wr_retry")+ assert run is not None+ assert run.started_at is None+ attempts = await get_attempts("wr_retry")+ key = ("wr_retry", 1)+ if scenario in {"lookup_failure", "attempt_row"}:+ assert run.status == WorkflowRunStatus.queued+ assert run.failure_reason is None+ fail_run.assert_not_awaited()+ assert key in executor._retry_dispatches_needing_recovery+ assert key in executor._retry_resumes_needing_recovery+ if scenario == "attempt_row":+ assert len(attempts) == 1+ assert attempts[0].status == "queued"+ assert attempts[0].started_at is None+ else:+ assert not attempts+ else:+ assert run.status == WorkflowRunStatus.failed+ reason = f"Workflow run initialization failed before execution: RuntimeError: {error}"+ assert run.failure_reason == reason+ fail_run.assert_awaited_once_with(workflow_run_id="wr_retry", failure_reason=reason)+ assert not attempts+ assert not executor._retry_dispatches_needing_recovery+ assert not executor._retry_resumes_needing_recovery+ executor._schedule.reset_mock()+ await executor._recover_pending_retries_once(resume_fresh=True)+ executor._schedule.assert_not_called()+ execute.assert_not_awaited()+ if app.WORKFLOW_SERVICE._background_tasks:+ await asyncio.gather(*app.WORKFLOW_SERVICE._background_tasks)+++async def test_first_dispatch_initializer_failure_is_retried_by_the_sweep(+ sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch+) -> None:+ organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)+ monkeypatch.setattr(app, "DATABASE", sqlite_db)+ async with sqlite_db.Session() as session:+ session.add_all(+ [+ WorkflowRunModel(+ workflow_run_id="wr_test",+ organization_id="org_test",+ workflow_id="wf_test",+ workflow_permanent_id="wpid_test",+ status="created",+ ),+ WorkflowRunAttemptModel(+ workflow_run_id="wr_test", organization_id="org_test", attempt_number=1, status="created"+ ),+ ]+ )+ await session.commit()initializer_calls = 0async def flaky_initialize(**kwargs: object) -> None:diff --git a/tests/unit/test_block_downloaded_file_scoping.py b/tests/unit/test_block_downloaded_file_scoping.pyindex d77ef1deb..93fc46e82 100644--- a/tests/unit/test_block_downloaded_file_scoping.py+++ b/tests/unit/test_block_downloaded_file_scoping.py@@ -358,7 +358,11 @@ async def test_current_attempt_downloads_preserve_unknown_dates_and_include_starassert result == (files[2:] if attempt_number > 1 and started_at is not None else files)resolve_attempt.assert_awaited_once_with("o_1", "wr_child", attempt_number=None)- storage.get_downloaded_files.assert_awaited_once_with(organization_id="o_1", run_id="wr_child")+ storage.get_downloaded_files.assert_awaited_once_with(+ organization_id="o_1",+ run_id="wr_child",+ attempt_started_at=boundary if attempt_number > 1 and started_at else None,+ )@pytest.mark.asynciodiff --git a/tests/unit/test_download_artifact_claim_dedupe.py b/tests/unit/test_download_artifact_claim_dedupe.pyindex b57b419b6..afb1a015b 100644--- a/tests/unit/test_download_artifact_claim_dedupe.py+++ b/tests/unit/test_download_artifact_claim_dedupe.py@@ -10,19 +10,21 @@ These tests drive the real reader on an in-memory DB (schema from the ORM), notfrom __future__ import annotationsimport datetime-from unittest.mock import MagicMock+from unittest.mock import AsyncMock, MagicMockimport pytestimport pytest_asyncio+from sqlalchemy import updatefrom sqlalchemy.ext.asyncio import AsyncEnginefrom skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.artifact.manager import ArtifactManagerfrom skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.artifact.storage import base as base_modulefrom skyvern.forge.sdk.artifact.storage.s3 import S3Storagefrom skyvern.forge.sdk.db.agent_db import AgentDB, _build_engine-from skyvern.forge.sdk.db.models import Base+from skyvern.forge.sdk.db.models import ArtifactModel, Base_DUMMY_KEYRING_JSON = '{"current_kid": "k1", "keys": {"k1": {"secret": "deadbeef"}}}'_WINDOW_START = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - datetime.timedelta(days=1)@@ -49,6 +51,7 @@ async def _make(run_id: str | None,browser_session_id: str | None,checksum: str | None,+ timestamp: datetime.datetime | None = None,) -> str:bucket = "skyvern-artifacts/v1/production" if browser_session_id else "skyvern-uploads/downloads/production"await db.artifacts.create_artifact(@@ -61,6 +64,14 @@ async def _make(checksum=checksum,file_size=192867,)+ if timestamp is not None:+ async with db.artifacts.Session() as session:+ await session.execute(+ update(ArtifactModel)+ .where(ArtifactModel.artifact_id == aid)+ .values(created_at=timestamp, modified_at=timestamp)+ )+ await session.commit()return aid@@ -201,3 +212,45 @@ async def test_run_scoped_read_pairs_same_content_session_rows_one_for_one(assert len(visible) == 2, f"expected the run-scoped row plus exactly one session twin, got {visible}"assert "a_run_scoped" in visible, f"run-scoped canonical must survive, got {visible}"assert len(visible & {"a_session_a", "a_session_b"}) == 1, f"exactly one session twin must survive, got {visible}"+++ ("row_names", "expected_ids"),+ [+ pytest.param(("r1", "s2"), ["a_s2"], id="two-row"),+ pytest.param(("s1", "r1", "s2"), ["a_s2"], id="three-row"),+ pytest.param(("s1", "r1", "s2", "r2"), ["a_r2"], id="four-row"),+ pytest.param(("s1", "r1"), [], id="stale-only"),+ ],+)+async def test_current_attempt_downloads_pair_only_current_artifacts(+ sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch, row_names: tuple[str, ...], expected_ids: list[str]+) -> None:+ org = await sqlite_db.organizations.create_organization("Test")+ org_id, session_id, run_id = org.organization_id, "pbs_retry", "wr_retry"+ cutoff = datetime.datetime(2026, 9, 9, 12, tzinfo=datetime.UTC)+ for row_name in row_names:+ timestamp = cutoff + datetime.timedelta(seconds={"s1": -2, "r1": -1, "s2": 0, "r2": 1}[row_name])+ await _make(+ sqlite_db,+ org_id,+ f"a_{row_name}",+ run_id=run_id,+ browser_session_id=session_id if row_name.startswith("s") else None,+ checksum="same-content",+ timestamp=timestamp.replace(tzinfo=None),+ )+ storage = _install_storage(sqlite_db, monkeypatch)+ monkeypatch.setattr(base_module, "resolve_download_attempt", AsyncMock(return_value=(run_id, 2, cutoff)))+ skip_empty = AsyncMock(wraps=storage._skip_empty_downloads_listing)+ monkeypatch.setattr(storage, "_skip_empty_downloads_listing", skip_empty)++ files = await storage.get_current_attempt_downloaded_files(organization_id=org_id, run_id=run_id)++ assert [file.artifact_id for file in files] == expected_ids+ skip_empty.assert_not_called()+ storage.async_client.list_files.assert_not_called()+ if "r2" in row_names:+ whole_run_files = await storage.get_downloaded_files(organization_id=org_id, run_id=run_id)+ assert {file.artifact_id for file in whole_run_files} == {"a_r1", "a_r2"}diff --git a/tests/unit/test_workflow_schedule_service.py b/tests/unit/test_workflow_schedule_service.pyindex c1d8011ba..ea5d9587c 100644--- a/tests/unit/test_workflow_schedule_service.py+++ b/tests/unit/test_workflow_schedule_service.py@@ -158,3 +158,56 @@ async def test_dispatch_due_schedules_launches_scheduled_workflow(monkeypatch: pneed_call_webhook=True,claim_initial_attempt=True,)++[email protected]("has_attempt_row", [False, True], ids=["no-policy", "policy"])[email protected]("initializer", ["initialize_skyvern_state_file", "prepare_org_llm_runtime"])+async def test_run_schedule_no_policy_initializer_failure_fails_the_run(+ monkeypatch: pytest.MonkeyPatch, has_attempt_row: bool, initializer: str+) -> None:+ schedule = _schedule()+ previous_fire_time = datetime(2026, 6, 2, 10, 0, tzinfo=UTC)+ run_id = schedule_service.build_scheduled_workflow_run_id(schedule.workflow_schedule_id, previous_fire_time)+ execute = AsyncMock()+ fail_run = AsyncMock()+ get_attempts = AsyncMock(return_value=[SimpleNamespace(attempt_number=1)] if has_attempt_row else [])+ fake_app = SimpleNamespace(+ DATABASE=SimpleNamespace(+ organizations=SimpleNamespace(+ get_organization=AsyncMock(return_value=SimpleNamespace(organization_id="org_test"))+ ),+ workflow_runs=SimpleNamespace(queue_initial_dispatch=AsyncMock(return_value=True)),+ workflow_run_attempts=SimpleNamespace(get_attempts=get_attempts),+ ),+ WORKFLOW_SERVICE=SimpleNamespace(+ execute_workflow_with_retries=execute,+ mark_workflow_run_as_failed_if_not_final=fail_run,+ ),+ )+ monkeypatch.setattr(schedule_service, "app", fake_app)+ monkeypatch.setattr(retry_policy_module, "app", fake_app)+ monkeypatch.setattr(+ schedule_service,+ "prepare_workflow",+ AsyncMock(return_value=SimpleNamespace(workflow_run_id=run_id, browser_session_id=None)),+ )+ monkeypatch.setattr(schedule_service, "initialize_skyvern_state_file", AsyncMock())+ monkeypatch.setattr(schedule_service, "prepare_org_llm_runtime", AsyncMock())+ error = RuntimeError(f"{initializer} unavailable")+ monkeypatch.setattr(schedule_service, initializer, AsyncMock(side_effect=error))+ scheduler = schedule_service.LocalWorkflowScheduleScheduler(poll_interval_seconds=1, max_concurrent_runs=1)+ due = schedule_service.DueWorkflowSchedule(schedule=schedule, previous_fire_time=previous_fire_time)+ if has_attempt_row:+ with pytest.raises(RuntimeError, match=f"{initializer} unavailable"):+ await scheduler._run_schedule(due)+ fail_run.assert_not_awaited()+ else:+ await scheduler._run_schedule(due)+ fail_run.assert_awaited_once_with(+ workflow_run_id=run_id,+ failure_reason=f"Workflow run initialization failed before execution: RuntimeError: {error}",+ )+ get_attempts.assert_awaited_once_with(run_id)+ fake_app.DATABASE.workflow_runs.queue_initial_dispatch.assert_awaited_once_with(run_id, 1)+ execute.assert_not_awaited()