Skip to content

Automate browser based workflows with AI

Fix four workflow-retry follow-ups: durable no-policy failures, Batch attempt resume, download dedupe cutoff, analyzer retry state (SKY-15922) (#8560)

Shuchang Zhenge76ed6644ca5
diff --git a/skyvern/forge/sdk/artifact/storage/azure.py b/skyvern/forge/sdk/artifact/storage/azure.py
index 9986c24bd..5123dab5f 100644
--- a/skyvern/forge/sdk/artifact/storage/azure.py
+++ b/skyvern/forge/sdk/artifact/storage/azure.py
@@ -663,7 +663,9 @@ class AzureStorage(BaseStorage):
if skipped_files:
raise DownloadSaveIncompleteError(skipped_files)
- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:
+ async def get_downloaded_files(
+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None
+ ) -> list[FileInfo]:
# Artifact-first — see s3.py::get_downloaded_files for rationale. When
# the keyring isn't configured (OSS default) or no artifact rows exist
# (legacy run pre-SKY-8861) we fall back to the legacy listing path so
@@ -673,7 +675,9 @@ class AzureStorage(BaseStorage):
download_artifacts = await self._list_download_artifacts_safe(
organization_id=organization_id, run_id=run_id
)
- download_artifacts = dedupe_run_scoped_download_artifacts(download_artifacts)
+ download_artifacts = dedupe_run_scoped_download_artifacts(
+ download_artifacts, attempt_started_at=attempt_started_at
+ )
if download_artifacts:
return await _file_infos_from_download_artifacts(download_artifacts)
diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.py
index cc210759a..416b91332 100644
--- a/skyvern/forge/sdk/artifact/storage/base.py
+++ b/skyvern/forge/sdk/artifact/storage/base.py
@@ -249,7 +249,19 @@ def download_checksums_by_uri(artifacts: list[Artifact]) -> dict[str, str]:
return {artifact.uri: artifact.checksum for artifact in artifacts if artifact.uri and artifact.checksum}
-def dedupe_run_scoped_download_artifacts(artifacts: list[Artifact]) -> list[Artifact]:
+def is_download_timestamp_current(timestamp: datetime | None, attempt_started_at: datetime | None) -> bool:
+ if attempt_started_at is None or timestamp is None:
+ return True
+ if timestamp.tzinfo is None:
+ timestamp = timestamp.replace(tzinfo=UTC)
+ if attempt_started_at.tzinfo is None:
+ attempt_started_at = attempt_started_at.replace(tzinfo=UTC)
+ return timestamp >= attempt_started_at
+
+
+def dedupe_run_scoped_download_artifacts(
+ artifacts: list[Artifact], attempt_started_at: datetime | None = None
+) -> list[Artifact]:
"""Collapse the two representations of one persistent-session download in a run's DOWNLOAD set.
A download can be registered both session-produced (``browser_session_id`` set — run-bound at
@@ -258,14 +270,26 @@ def dedupe_run_scoped_download_artifacts(artifacts: list[Artifact]) -> list[Arti
the run-scoped one as canonical. Pairing is one-for-one: a checksum with N run-scoped rows drops at
most N session rows, so two legitimate same-content session downloads backed by a single run-scoped
row keep one. Order is preserved; distinct/absent checksums, twinless session rows, and multiple
- run-scoped rows are all kept. Run-scoped only — never call on a session listing.
+ run-scoped rows are all kept. Rows before the attempt cutoff pass through without pairing.
+ Run-scoped only — never call on a session listing.
"""
- run_scoped_checksums = Counter(a.checksum for a in artifacts if a.browser_session_id is None and a.checksum)
+ run_scoped_checksums = Counter(
+ a.checksum
+ for a in artifacts
+ if a.browser_session_id is None
+ and a.checksum
+ and is_download_timestamp_current(a.modified_at or a.created_at, attempt_started_at)
+ )
if not run_scoped_checksums:
return artifacts
kept: list[Artifact] = []
for a in artifacts:
- if a.browser_session_id is not None and a.checksum and run_scoped_checksums[a.checksum] > 0:
+ if (
+ a.browser_session_id is not None
+ and a.checksum
+ and run_scoped_checksums[a.checksum] > 0
+ and is_download_timestamp_current(a.modified_at or a.created_at, attempt_started_at)
+ ):
run_scoped_checksums[a.checksum] -= 1
continue
kept.append(a)
@@ -508,21 +532,16 @@ class BaseStorage(ABC):
return []
async def get_current_attempt_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:
- files = await self.get_downloaded_files(organization_id=organization_id, run_id=run_id)
started_at = await get_download_retry_started_at(organization_id, run_id)
- if started_at is None:
- return files
- current_files = []
- for file_info in files:
- modified_at = file_info.modified_at
- if modified_at is not None and modified_at.tzinfo is None:
- modified_at = modified_at.replace(tzinfo=UTC)
- if modified_at is None or modified_at >= started_at:
- current_files.append(file_info)
- return current_files
-
- @abstractmethod
- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:
+ files = await self.get_downloaded_files(
+ organization_id=organization_id, run_id=run_id, attempt_started_at=started_at
+ )
+ return [file_info for file_info in files if is_download_timestamp_current(file_info.modified_at, started_at)]
+
+ @abstractmethod
+ async def get_downloaded_files(
+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None
+ ) -> list[FileInfo]:
pass
@abstractmethod
diff --git a/skyvern/forge/sdk/artifact/storage/gcs.py b/skyvern/forge/sdk/artifact/storage/gcs.py
index 3a117fc9d..aeedc9b8c 100644
--- a/skyvern/forge/sdk/artifact/storage/gcs.py
+++ b/skyvern/forge/sdk/artifact/storage/gcs.py
@@ -677,7 +677,9 @@ class GcsStorage(BaseStorage):
if skipped_files:
raise DownloadSaveIncompleteError(skipped_files)
- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:
+ async def get_downloaded_files(
+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None
+ ) -> list[FileInfo]:
# Artifact-first — see s3.py::get_downloaded_files for rationale. When
# the keyring isn't configured (OSS default) or no artifact rows exist
# (legacy run) we fall back to the legacy listing path so
@@ -687,7 +689,9 @@ class GcsStorage(BaseStorage):
download_artifacts = await self._list_download_artifacts_safe(
organization_id=organization_id, run_id=run_id
)
- download_artifacts = dedupe_run_scoped_download_artifacts(download_artifacts)
+ download_artifacts = dedupe_run_scoped_download_artifacts(
+ download_artifacts, attempt_started_at=attempt_started_at
+ )
if download_artifacts:
return await _file_infos_from_download_artifacts(download_artifacts)
diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.py
index c1cec6e98..2365004de 100644
--- a/skyvern/forge/sdk/artifact/storage/local.py
+++ b/skyvern/forge/sdk/artifact/storage/local.py
@@ -564,7 +564,9 @@ class LocalStorage(BaseStorage):
live_uri = f"file://{Path(get_download_dir(run_id=parts[1])) / file_info.filename}"
return [to_downloaded_file_signature(file_info.model_copy(update={"url": live_uri}))]
- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:
+ async def get_downloaded_files(
+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None
+ ) -> list[FileInfo]:
download_dir = get_download_dir(run_id=run_id)
artifacts_by_uri: dict[str, Artifact] = {}
if run_id is not None:
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
index 8cbae9cc3..0cd0475b7 100644
--- a/skyvern/forge/sdk/artifact/storage/s3.py
+++ b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -936,7 +936,9 @@ class S3Storage(BaseStorage):
if skipped_files:
raise DownloadSaveIncompleteError(skipped_files)
- async def get_downloaded_files(self, organization_id: str, run_id: str | None) -> list[FileInfo]:
+ async def get_downloaded_files(
+ self, organization_id: str, run_id: str | None, attempt_started_at: datetime | None = None
+ ) -> list[FileInfo]:
# Artifact-first: when a run has DOWNLOAD artifact rows, return them as
# the source of truth — the row carries enough to build a short signed
# /v1/artifacts/{id}/content URL plus the SHA-256 we persisted at save
@@ -957,7 +959,9 @@ class S3Storage(BaseStorage):
)
if not rows_lookup_failed:
download_row_count = len(download_artifacts)
- download_artifacts = dedupe_run_scoped_download_artifacts(download_artifacts)
+ download_artifacts = dedupe_run_scoped_download_artifacts(
+ download_artifacts, attempt_started_at=attempt_started_at
+ )
if download_artifacts:
file_infos = await _file_infos_from_download_artifacts(download_artifacts)
if not file_infos:
diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.py
index 1cbee40d9..877b404c6 100644
--- a/skyvern/forge/sdk/executor/background_task_executor.py
+++ b/skyvern/forge/sdk/executor/background_task_executor.py
@@ -32,6 +32,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.forge.sdk.workflow.retry_policy import (
LEASE_TAKEOVER_SECONDS,
RetryDecision,
+ fail_run_without_attempt_row,
finalize_abandoned_attempt,
get_recorded_decision,
latest_attempt_awaiting_preparation,
@@ -596,12 +597,27 @@ class BackgroundTaskExecutor(AsyncExecutor):
):
self._retry_resumes_needing_recovery.discard(key)
return
- # Initialization belongs to the dispatch: a failure here is re-dispatched by the sweep, whereas a
- # created attempt that never reached the dispatch matches no recovery predicate.
- await initialize_skyvern_state_file(
- workflow_run_id=attempt.workflow_run_id, organization_id=attempt.organization_id
- )
- await prepare_org_llm_runtime(app.DATABASE, attempt.organization_id, execution_kwargs.get("organization"))
+ # Attempt rows let the sweep recover initialization failures; attempt-less runs must fail durably.
+ try:
+ await initialize_skyvern_state_file(
+ workflow_run_id=attempt.workflow_run_id, organization_id=attempt.organization_id
+ )
+ await prepare_org_llm_runtime(
+ app.DATABASE, attempt.organization_id, execution_kwargs.get("organization")
+ )
+ except Exception as exc:
+ if await fail_run_without_attempt_row(
+ attempt.workflow_run_id,
+ f"Workflow run initialization failed before execution: {type(exc).__name__}: {exc}",
+ ):
+ self._retry_resumes_needing_recovery.discard(key)
+ LOG.warning(
+ "Workflow run initialization failed without an attempt row; run is terminal",
+ workflow_run_id=attempt.workflow_run_id,
+ exc_info=True,
+ )
+ return
+ raise
await app.WORKFLOW_SERVICE.execute_workflow_with_retries(
**execution_kwargs, on_execution_start=on_execution_start
)
diff --git a/skyvern/forge/sdk/workflow/retry_policy.py b/skyvern/forge/sdk/workflow/retry_policy.py
index b8da38dad..604456569 100644
--- a/skyvern/forge/sdk/workflow/retry_policy.py
+++ b/skyvern/forge/sdk/workflow/retry_policy.py
@@ -337,6 +337,27 @@ async def ensure_attempt_row(
return True
+async def fail_run_without_attempt_row(workflow_run_id: str, failure_reason: str) -> bool:
+ """Fail a queued run that no recovery sweep can see. Returns False when an attempt row exists."""
+ if await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id):
+ return False
+ try:
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ workflow_run_id=workflow_run_id,
+ failure_reason=failure_reason,
+ )
+ except Exception:
+ workflow_run = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id)
+ if workflow_run is not None and not workflow_run.status.is_final():
+ raise
+ LOG.warning(
+ "Workflow run is terminal after initialization failure despite a finalization error",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
+ return True
+
+
async def queue_initial_attempt(workflow_run_id: str, attempt_number: int) -> bool:
"""Queue a new run's durable rows before its dispatch, retrying a failed write.
diff --git a/skyvern/services/workflow_schedule_service.py b/skyvern/services/workflow_schedule_service.py
index 4050c737f..a1a181a39 100644
--- a/skyvern/services/workflow_schedule_service.py
+++ b/skyvern/services/workflow_schedule_service.py
@@ -22,7 +22,7 @@ from skyvern.forge.sdk.api.llm.custom_llm_registry import prepare_org_llm_runtim
from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
from skyvern.forge.sdk.schemas.workflow_schedules import WorkflowSchedule
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRequestBody
-from skyvern.forge.sdk.workflow.retry_policy import queue_initial_attempt
+from skyvern.forge.sdk.workflow.retry_policy import fail_run_without_attempt_row, queue_initial_attempt
from skyvern.forge.sdk.workflow.schedules import compute_previous_fire_time
from skyvern.services.workflow_service import prepare_workflow
from skyvern.utils.files import initialize_skyvern_state_file
@@ -222,15 +222,29 @@ class LocalWorkflowScheduleScheduler:
return
raise
- # A persisted run counts as a delivered fire, so the attempt must be recoverable by the dispatch
- # sweep before the initializers below can fail.
+ # A persisted run counts as a delivered fire. Queue its attempt for sweep recovery;
+ # without an attempt row, an initialization failure must make the run terminal.
if not await queue_initial_attempt(workflow_run.workflow_run_id, 1):
return
- await initialize_skyvern_state_file(
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=organization.organization_id,
- )
- await prepare_org_llm_runtime(app.DATABASE, organization.organization_id, organization)
+ try:
+ await initialize_skyvern_state_file(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization.organization_id,
+ )
+ await prepare_org_llm_runtime(app.DATABASE, organization.organization_id, organization)
+ except Exception as exc:
+ if await fail_run_without_attempt_row(
+ workflow_run.workflow_run_id,
+ f"Workflow run initialization failed before execution: {type(exc).__name__}: {exc}",
+ ):
+ LOG.warning(
+ "Scheduled workflow initialization failed without an attempt row; run is terminal",
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_schedule_id=schedule.workflow_schedule_id,
+ exc_info=True,
+ )
+ return
+ raise
await app.WORKFLOW_SERVICE.execute_workflow_with_retries(
workflow_run_id=workflow_run.workflow_run_id,
api_key=None,
diff --git a/tests/unit/test_background_task_executor.py b/tests/unit/test_background_task_executor.py
index 5ec5bb43d..f82e304be 100644
--- a/tests/unit/test_background_task_executor.py
+++ b/tests/unit/test_background_task_executor.py
@@ -2870,14 +2870,120 @@ async def test_execute_workflow_dispatches_from_latest_retry_attempt(monkeypatch
@pytest.mark.asyncio
-async def test_first_dispatch_initializer_failure_is_retried_by_the_sweep(monkeypatch: pytest.MonkeyPatch) -> None:
- organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
- monkeypatch.setattr(
- app.DATABASE.workflow_runs,
- "get_workflow_run",
- AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None, status=WorkflowRunStatus.created)),
[email protected]("scenario", ["state_file", "llm_runtime", "lookup_failure", "attempt_row"])
+async def test_no_policy_initializer_failure_fails_the_run_durably(
+ interim_retry_db: AgentDB, monkeypatch: pytest.MonkeyPatch, scenario: str
+) -> None:
+ database = interim_retry_db
+ async with database.Session() as session:
+ run = await session.get(WorkflowRunModel, "wr_retry")
+ assert run is not None
+ run.status = "created"
+ run.started_at = None
+ run.finished_at = None
+ attempt = await session.get(WorkflowRunAttemptModel, ("wr_retry", 1))
+ assert attempt is not None
+ if scenario == "attempt_row":
+ attempt.status = "created"
+ attempt.retry_decision = None
+ attempt.finished_at = None
+ attempt.next_attempt_at = None
+ else:
+ workflow = await session.get(WorkflowModel, "wf_retry")
+ assert workflow is not None
+ workflow.workflow_definition = {"blocks": [], "parameters": []}
+ await session.delete(attempt)
+ await session.commit()
+
+ monkeypatch.setattr(background_task_executor_module, "initialize_skyvern_state_file", AsyncMock())
+ monkeypatch.setattr(background_task_executor_module, "prepare_org_llm_runtime", AsyncMock())
+ initializer = "prepare_org_llm_runtime" if scenario == "llm_runtime" else "initialize_skyvern_state_file"
+ error = RuntimeError(f"{initializer} unavailable")
+ monkeypatch.setattr(background_task_executor_module, initializer, AsyncMock(side_effect=error))
+ execute = AsyncMock()
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_with_retries", execute)
+ fail_run = AsyncMock(wraps=app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final)
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", fail_run)
+ executor = BackgroundTaskExecutor()
+ executor._schedule = MagicMock() # type: ignore[method-assign]
+ organization = await database.organizations.get_organization("org_test")
+ assert organization is not None
+ await executor.execute_workflow(
+ request=None,
+ background_tasks=None,
+ organization=organization,
+ workflow_id="wf_retry",
+ workflow_run_id="wr_retry",
+ workflow_permanent_id="wpid_retry",
+ max_steps_override=None,
+ api_key=None,
+ browser_session_id=None,
+ block_labels=None,
+ block_outputs=None,
)
- monkeypatch.setattr(app.DATABASE.workflow_run_attempts, "get_attempts", AsyncMock(return_value=[]))
+ _background_tasks, dispatch, attempt, execution_kwargs = executor._schedule.call_args.args
+ get_attempts = database.workflow_run_attempts.get_attempts
+ if scenario == "lookup_failure":
+ monkeypatch.setattr(
+ database.workflow_run_attempts, "get_attempts", AsyncMock(side_effect=RuntimeError("db down"))
+ )
+ await dispatch(attempt, execution_kwargs)
+ execute.assert_not_awaited()
+ run = await database.workflow_runs.get_workflow_run("wr_retry")
+ assert run is not None
+ assert run.started_at is None
+ attempts = await get_attempts("wr_retry")
+ key = ("wr_retry", 1)
+ if scenario in {"lookup_failure", "attempt_row"}:
+ assert run.status == WorkflowRunStatus.queued
+ assert run.failure_reason is None
+ fail_run.assert_not_awaited()
+ assert key in executor._retry_dispatches_needing_recovery
+ assert key in executor._retry_resumes_needing_recovery
+ if scenario == "attempt_row":
+ assert len(attempts) == 1
+ assert attempts[0].status == "queued"
+ assert attempts[0].started_at is None
+ else:
+ assert not attempts
+ else:
+ assert run.status == WorkflowRunStatus.failed
+ reason = f"Workflow run initialization failed before execution: RuntimeError: {error}"
+ assert run.failure_reason == reason
+ fail_run.assert_awaited_once_with(workflow_run_id="wr_retry", failure_reason=reason)
+ assert not attempts
+ assert not executor._retry_dispatches_needing_recovery
+ assert not executor._retry_resumes_needing_recovery
+ executor._schedule.reset_mock()
+ await executor._recover_pending_retries_once(resume_fresh=True)
+ executor._schedule.assert_not_called()
+ execute.assert_not_awaited()
+ if app.WORKFLOW_SERVICE._background_tasks:
+ await asyncio.gather(*app.WORKFLOW_SERVICE._background_tasks)
+
+
+async def test_first_dispatch_initializer_failure_is_retried_by_the_sweep(
+ sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ organization = SimpleNamespace(organization_id="org_test", default_llm_key=None, default_secondary_llm_key=None)
+ monkeypatch.setattr(app, "DATABASE", sqlite_db)
+ async with sqlite_db.Session() as session:
+ session.add_all(
+ [
+ WorkflowRunModel(
+ workflow_run_id="wr_test",
+ organization_id="org_test",
+ workflow_id="wf_test",
+ workflow_permanent_id="wpid_test",
+ status="created",
+ ),
+ WorkflowRunAttemptModel(
+ workflow_run_id="wr_test", organization_id="org_test", attempt_number=1, status="created"
+ ),
+ ]
+ )
+ await session.commit()
initializer_calls = 0
async def flaky_initialize(**kwargs: object) -> None:
diff --git a/tests/unit/test_block_downloaded_file_scoping.py b/tests/unit/test_block_downloaded_file_scoping.py
index d77ef1deb..93fc46e82 100644
--- a/tests/unit/test_block_downloaded_file_scoping.py
+++ b/tests/unit/test_block_downloaded_file_scoping.py
@@ -358,7 +358,11 @@ async def test_current_attempt_downloads_preserve_unknown_dates_and_include_star
assert result == (files[2:] if attempt_number > 1 and started_at is not None else files)
resolve_attempt.assert_awaited_once_with("o_1", "wr_child", attempt_number=None)
- storage.get_downloaded_files.assert_awaited_once_with(organization_id="o_1", run_id="wr_child")
+ storage.get_downloaded_files.assert_awaited_once_with(
+ organization_id="o_1",
+ run_id="wr_child",
+ attempt_started_at=boundary if attempt_number > 1 and started_at else None,
+ )
@pytest.mark.asyncio
diff --git a/tests/unit/test_download_artifact_claim_dedupe.py b/tests/unit/test_download_artifact_claim_dedupe.py
index b57b419b6..afb1a015b 100644
--- a/tests/unit/test_download_artifact_claim_dedupe.py
+++ b/tests/unit/test_download_artifact_claim_dedupe.py
@@ -10,19 +10,21 @@ These tests drive the real reader on an in-memory DB (schema from the ORM), not
from __future__ import annotations
import datetime
-from unittest.mock import MagicMock
+from unittest.mock import AsyncMock, MagicMock
import pytest
import pytest_asyncio
+from sqlalchemy import update
from sqlalchemy.ext.asyncio import AsyncEngine
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.artifact.manager import ArtifactManager
from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.artifact.storage import base as base_module
from skyvern.forge.sdk.artifact.storage.s3 import S3Storage
from skyvern.forge.sdk.db.agent_db import AgentDB, _build_engine
-from skyvern.forge.sdk.db.models import Base
+from skyvern.forge.sdk.db.models import ArtifactModel, Base
_DUMMY_KEYRING_JSON = '{"current_kid": "k1", "keys": {"k1": {"secret": "deadbeef"}}}'
_WINDOW_START = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - datetime.timedelta(days=1)
@@ -49,6 +51,7 @@ async def _make(
run_id: str | None,
browser_session_id: str | None,
checksum: str | None,
+ timestamp: datetime.datetime | None = None,
) -> str:
bucket = "skyvern-artifacts/v1/production" if browser_session_id else "skyvern-uploads/downloads/production"
await db.artifacts.create_artifact(
@@ -61,6 +64,14 @@ async def _make(
checksum=checksum,
file_size=192867,
)
+ if timestamp is not None:
+ async with db.artifacts.Session() as session:
+ await session.execute(
+ update(ArtifactModel)
+ .where(ArtifactModel.artifact_id == aid)
+ .values(created_at=timestamp, modified_at=timestamp)
+ )
+ await session.commit()
return aid
@@ -201,3 +212,45 @@ async def test_run_scoped_read_pairs_same_content_session_rows_one_for_one(
assert len(visible) == 2, f"expected the run-scoped row plus exactly one session twin, got {visible}"
assert "a_run_scoped" in visible, f"run-scoped canonical must survive, got {visible}"
assert len(visible & {"a_session_a", "a_session_b"}) == 1, f"exactly one session twin must survive, got {visible}"
+
+
+ ("row_names", "expected_ids"),
+ [
+ pytest.param(("r1", "s2"), ["a_s2"], id="two-row"),
+ pytest.param(("s1", "r1", "s2"), ["a_s2"], id="three-row"),
+ pytest.param(("s1", "r1", "s2", "r2"), ["a_r2"], id="four-row"),
+ pytest.param(("s1", "r1"), [], id="stale-only"),
+ ],
+)
+async def test_current_attempt_downloads_pair_only_current_artifacts(
+ sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch, row_names: tuple[str, ...], expected_ids: list[str]
+) -> None:
+ org = await sqlite_db.organizations.create_organization("Test")
+ org_id, session_id, run_id = org.organization_id, "pbs_retry", "wr_retry"
+ cutoff = datetime.datetime(2026, 9, 9, 12, tzinfo=datetime.UTC)
+ for row_name in row_names:
+ timestamp = cutoff + datetime.timedelta(seconds={"s1": -2, "r1": -1, "s2": 0, "r2": 1}[row_name])
+ await _make(
+ sqlite_db,
+ org_id,
+ f"a_{row_name}",
+ run_id=run_id,
+ browser_session_id=session_id if row_name.startswith("s") else None,
+ checksum="same-content",
+ timestamp=timestamp.replace(tzinfo=None),
+ )
+ storage = _install_storage(sqlite_db, monkeypatch)
+ monkeypatch.setattr(base_module, "resolve_download_attempt", AsyncMock(return_value=(run_id, 2, cutoff)))
+ skip_empty = AsyncMock(wraps=storage._skip_empty_downloads_listing)
+ monkeypatch.setattr(storage, "_skip_empty_downloads_listing", skip_empty)
+
+ files = await storage.get_current_attempt_downloaded_files(organization_id=org_id, run_id=run_id)
+
+ assert [file.artifact_id for file in files] == expected_ids
+ skip_empty.assert_not_called()
+ storage.async_client.list_files.assert_not_called()
+ if "r2" in row_names:
+ whole_run_files = await storage.get_downloaded_files(organization_id=org_id, run_id=run_id)
+ assert {file.artifact_id for file in whole_run_files} == {"a_r1", "a_r2"}
diff --git a/tests/unit/test_workflow_schedule_service.py b/tests/unit/test_workflow_schedule_service.py
index c1d8011ba..ea5d9587c 100644
--- a/tests/unit/test_workflow_schedule_service.py
+++ b/tests/unit/test_workflow_schedule_service.py
@@ -158,3 +158,56 @@ async def test_dispatch_due_schedules_launches_scheduled_workflow(monkeypatch: p
need_call_webhook=True,
claim_initial_attempt=True,
)
+
+
[email protected]("has_attempt_row", [False, True], ids=["no-policy", "policy"])
[email protected]("initializer", ["initialize_skyvern_state_file", "prepare_org_llm_runtime"])
+async def test_run_schedule_no_policy_initializer_failure_fails_the_run(
+ monkeypatch: pytest.MonkeyPatch, has_attempt_row: bool, initializer: str
+) -> None:
+ schedule = _schedule()
+ previous_fire_time = datetime(2026, 6, 2, 10, 0, tzinfo=UTC)
+ run_id = schedule_service.build_scheduled_workflow_run_id(schedule.workflow_schedule_id, previous_fire_time)
+ execute = AsyncMock()
+ fail_run = AsyncMock()
+ get_attempts = AsyncMock(return_value=[SimpleNamespace(attempt_number=1)] if has_attempt_row else [])
+ fake_app = SimpleNamespace(
+ DATABASE=SimpleNamespace(
+ organizations=SimpleNamespace(
+ get_organization=AsyncMock(return_value=SimpleNamespace(organization_id="org_test"))
+ ),
+ workflow_runs=SimpleNamespace(queue_initial_dispatch=AsyncMock(return_value=True)),
+ workflow_run_attempts=SimpleNamespace(get_attempts=get_attempts),
+ ),
+ WORKFLOW_SERVICE=SimpleNamespace(
+ execute_workflow_with_retries=execute,
+ mark_workflow_run_as_failed_if_not_final=fail_run,
+ ),
+ )
+ monkeypatch.setattr(schedule_service, "app", fake_app)
+ monkeypatch.setattr(retry_policy_module, "app", fake_app)
+ monkeypatch.setattr(
+ schedule_service,
+ "prepare_workflow",
+ AsyncMock(return_value=SimpleNamespace(workflow_run_id=run_id, browser_session_id=None)),
+ )
+ monkeypatch.setattr(schedule_service, "initialize_skyvern_state_file", AsyncMock())
+ monkeypatch.setattr(schedule_service, "prepare_org_llm_runtime", AsyncMock())
+ error = RuntimeError(f"{initializer} unavailable")
+ monkeypatch.setattr(schedule_service, initializer, AsyncMock(side_effect=error))
+ scheduler = schedule_service.LocalWorkflowScheduleScheduler(poll_interval_seconds=1, max_concurrent_runs=1)
+ due = schedule_service.DueWorkflowSchedule(schedule=schedule, previous_fire_time=previous_fire_time)
+ if has_attempt_row:
+ with pytest.raises(RuntimeError, match=f"{initializer} unavailable"):
+ await scheduler._run_schedule(due)
+ fail_run.assert_not_awaited()
+ else:
+ await scheduler._run_schedule(due)
+ fail_run.assert_awaited_once_with(
+ workflow_run_id=run_id,
+ failure_reason=f"Workflow run initialization failed before execution: RuntimeError: {error}",
+ )
+ get_attempts.assert_awaited_once_with(run_id)
+ fake_app.DATABASE.workflow_runs.queue_initial_dispatch.assert_awaited_once_with(run_id, 1)
+ execute.assert_not_awaited()