Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Record run.completed, plan on the organization, and upgrade starts on the portal route (SKY-18287) (#8911)
Shuchang Zhengacbce161a656
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 9d2141da9..b3d69b49c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -204,6 +204,11 @@ from skyvern.forge.taskv3.goal_check import (from skyvern.forge.taskv3.goal_composition import CodeProgressRecordfrom skyvern.forge.taskv3.loop import ACTION_BLOCK_TARGET_ACTION_RESERVE, LoopOutcome, RoundActionfrom skyvern.forge.taskv3.pre_submit_capture import PreSubmitCaptureRing, is_run_sampled, pre_submit_screenshot+from skyvern.forge.taskv3.provider_downloads import (+ PROVIDER_BASELINE_TIMEOUT_SECONDS,+ PROVIDER_DOWNLOAD_WAIT_SECONDS,+ ProviderDownloadRecovery,+)from skyvern.forge.taskv3.run_arms import (DATE_SEGMENT_AIM_FLAG,LOGIN_PACE_FLAG,@@ -2638,6 +2643,31 @@ class ForgeAgent:super().add(name)staged_downloads: set[str] = _StagedDownloads()++ # Frozen before the first action so a provider file an earlier block left is never this block's.+ provider_recovery: ProviderDownloadRecovery | None = None+ provider_source = (+ browser_state.browser_artifacts.get_action_download_source()+ if task_block is not None and task_block.complete_on_download and workflow_download_directory is not None+ else None+ )+ if provider_source is not None and workflow_download_directory is not None:+ provider_recovery = await ProviderDownloadRecovery.begin(+ provider_source,+ download_dir=workflow_download_directory,+ browser_context=browser_state.browser_context,+ deadline=time.monotonic() + min(PROVIDER_BASELINE_TIMEOUT_SECONDS, loop_deadline_at - time.monotonic()),+ )+ # A file that landed while the provider baseline was awaited predates this block's first action,+ # whether or not the baseline read succeeded.+ download_baseline_files = [+ *(download_baseline_files or []),+ *list_files_in_directory(workflow_download_directory, attempt_started_at=attempt_started_at),+ ]++ def _provider_wait_seconds(block: BaseTaskBlock) -> float:+ return min(block.download_timeout or PROVIDER_DOWNLOAD_WAIT_SECONDS, loop_deadline_at - time.monotonic())+completion_probe: CompletionProbe | None = Nonecompletion_blocker: CompletionBlocker | None = Noneif task_block is not None and task_block.complete_on_download and task.workflow_run_id:@@ -2663,6 +2693,16 @@ class ForgeAgent:# The run is being canceled; even if a file landed while we waited, don't# finalize it into a `completed` result out from under the cancellation.return None+ if provider_recovery is not None and await provider_recovery.recover(+ staged=staged,+ timeout=_provider_wait_seconds(complete_on_download_block),+ should_cancel=_should_cancel,+ ):+ return None+ # Either wait can return inside its first poll interval without having checked; a cancel that+ # landed meanwhile must not be finalized into a `completed` result.+ if await _should_cancel():+ return Nonestaged_paths = [str(workflow_download_directory / name) for name in sorted(staged)]new_files = await self._finalize_downloaded_files_for_task(task,diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.pyindex 775d33837..0e4367d11 100644--- a/skyvern/forge/sdk/api/aws.py+++ b/skyvern/forge/sdk/api/aws.py@@ -358,6 +358,27 @@ class _SharedS3Client:await context.__aexit__(None, None, None)+class S3ObjectStream:+ """An open GetObject read in bounded chunks; it holds its shared-client lease until ``aclose``, so retiring that+ client (a session refresh) cannot close the connection the body is still read from."""++ def __init__(self, response: dict[str, Any], lease: contextlib.AsyncExitStack) -> None:+ self.content_length: int = response["ContentLength"]+ self.etag: str | None = response.get("ETag")+ self._body = response["Body"]+ self._lease = lease++ def iter_chunks(self, chunk_size: int) -> AsyncIterator[bytes]:+ return self._body.iter_chunks(chunk_size)++ async def aclose(self) -> None:+ # An unfinished body drops its connection instead of returning it to the pool with unread bytes.+ try:+ self._body.close()+ finally:+ await self._lease.aclose()++def _retire_shared_s3_clients(registry: dict[asyncio.AbstractEventLoop, _SharedS3Client]) -> None:for loop, shared in list(registry.items()):with contextlib.suppress(RuntimeError): # the loop is already closed@@ -1340,22 +1361,47 @@ class AsyncAWSClient:try:return await self._s3_with_retry("download", _op, uri=uri)except Exception as e:- # A missing object is terminal not-found, not a transient failure (e.g. first-run- # profile restore before any archive exists). Log once without a traceback so callers- # fall back immediately instead of surfacing repeated ERROR tracebacks.- if self._is_not_found_error(e):- if log_exception:- LOG.info("S3 object not found", uri=uri)- return None- # An archived object stays unreadable until restored, so a traceback per request adds nothing.- if self._is_archived_object_error(e):- if log_exception:- LOG.warning("S3 object is archived and not restored", uri=uri)- return Noneif log_exception:- LOG.exception("S3 download failed", uri=uri)+ self.log_download_failure(e, uri)return None+ def log_download_failure(self, error: Exception, uri: str) -> None:+ """Log a failed object read; call it from the ``except`` block so a real failure keeps its traceback."""+ # A missing object is terminal not-found, not a transient failure (e.g. first-run+ # profile restore before any archive exists). Log once without a traceback so callers+ # fall back immediately instead of surfacing repeated ERROR tracebacks.+ if self._is_not_found_error(error):+ LOG.info("S3 object not found", uri=uri)+ # An archived object stays unreadable until restored, so a traceback per request adds nothing.+ elif self._is_archived_object_error(error):+ LOG.warning("S3 object is archived and not restored", uri=uri)+ else:+ LOG.exception("S3 download failed", uri=uri)++ async def open_download_stream(+ self, uri: str, *, byte_range: tuple[int, int] | None = None, if_match: str | None = None+ ) -> S3ObjectStream:+ """GetObject without reading the body, which the caller must ``aclose``; ``byte_range`` is inclusive, and an+ ``if_match`` that no longer names the stored object raises ``PreconditionFailed`` like any get_object error."""+ parsed_uri = S3Uri(uri)+ params: dict[str, Any] = {"Bucket": parsed_uri.bucket, "Key": parsed_uri.key}+ if byte_range is not None:+ params["Range"] = f"bytes={byte_range[0]}-{byte_range[1]}"+ if if_match is not None:+ params["IfMatch"] = if_match++ async def _op() -> S3ObjectStream:+ lease = contextlib.AsyncExitStack()+ try:+ client = await lease.enter_async_context(self._s3_client())+ response = await client.get_object(**params)+ except BaseException:+ await lease.aclose()+ raise+ return S3ObjectStream(response, lease)++ return await self._s3_with_retry("download", _op, uri=uri)+async def delete_file(self, uri: str, log_exception: bool = True, raise_on_error: bool = False) -> None:# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/delete_object.htmlasync def _op() -> None:diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.pyindex bf36845c0..22da8bd0a 100644--- a/skyvern/forge/sdk/artifact/storage/base.py+++ b/skyvern/forge/sdk/artifact/storage/base.py@@ -1,9 +1,10 @@import osfrom abc import ABC, abstractmethodfrom collections import Counter-from collections.abc import Awaitable, Callable+from collections.abc import AsyncIterator, Awaitable, Callable+from dataclasses import dataclassfrom datetime import UTC, datetime-from typing import BinaryIO, cast+from typing import BinaryIO, Protocol, castfrom urllib.parse import unquoteimport structlog@@ -342,6 +343,44 @@ FILE_EXTENTSION_MAP: dict[ArtifactType, str] = {}+@dataclass(frozen=True)+class StoredObjectVersion:+ size: int+ # Pins a later ranged read to the version this size was taken from.+ etag: str | None+++class StoredObjectStream(Protocol):+ content_length: int++ def iter_chunks(self, chunk_size: int) -> AsyncIterator[bytes]: ...++ async def aclose(self) -> None: ...+++class StoredObjectChangedError(Exception):+ """The stored object was replaced after it was sized for a ranged read."""+++class StreamingArtifactStorage(ABC):+ """Storage that can serve an artifact's stored bytes in bounded chunks instead of one whole read."""++ @abstractmethod+ def can_stream_artifact(self, artifact: Artifact) -> bool:+ """Whether the artifact's content is its stored bytes unchanged, with no decoding after the read."""++ @abstractmethod+ async def stat_artifact(self, artifact: Artifact) -> StoredObjectVersion | None:+ """The stored object's size and version, or None when it cannot be read."""++ @abstractmethod+ async def open_artifact_stream(+ self, artifact: Artifact, *, byte_range: tuple[int, int] | None = None, etag: str | None = None+ ) -> StoredObjectStream | None:+ """Open the stored bytes or their inclusive ``byte_range``, or return None when they cannot be read; with+ ``etag``, raises ``StoredObjectChangedError`` once the stored object is no longer that version."""++class BaseStorage(ABC):@abstractmethoddef build_uri(self, *, organization_id: str, artifact_id: str, step: Step, artifact_type: ArtifactType) -> str:diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.pyindex 976580ad2..6e7d3a8d6 100644--- a/skyvern/forge/sdk/artifact/storage/s3.py+++ b/skyvern/forge/sdk/artifact/storage/s3.py@@ -15,7 +15,7 @@ from skyvern.config import settingsfrom skyvern.constants import BROWSER_DOWNLOADING_SUFFIX, DOWNLOAD_FILE_PREFIXfrom skyvern.exceptions import DownloadSaveIncompleteErrorfrom skyvern.forge import app-from skyvern.forge.sdk.api.aws import AsyncAWSClient, S3StorageClass, S3Uri, _recording_content_type+from skyvern.forge.sdk.api.aws import AsyncAWSClient, S3ObjectStream, S3StorageClass, S3Uri, _recording_content_typefrom skyvern.forge.sdk.api.files import (calculate_sha256_for_file,create_named_temporary_file,@@ -30,6 +30,9 @@ from skyvern.forge.sdk.artifact.signing import SENSITIVE_ARTIFACT_URL_EXPIRY_SECfrom skyvern.forge.sdk.artifact.storage.base import (FILE_EXTENTSION_MAP,BaseStorage,+ StoredObjectChangedError,+ StoredObjectVersion,+ StreamingArtifactStorage,_file_infos_from_artifacts,_file_infos_from_download_artifacts,dedupe_run_scoped_download_artifacts,@@ -97,7 +100,7 @@ def empty_read_decision(}-class S3Storage(BaseStorage):+class S3Storage(BaseStorage, StreamingArtifactStorage):_PATH_VERSION = "v1"# Caps concurrent head_object calls from S3 listing reads. /browser_sessions/history fans out over up to 100# sessions at once, so the semaphore lives on the instance and every listing call on it shares one cap.@@ -236,6 +239,29 @@ class S3Storage(BaseStorage):) -> S3StorageClass:return S3StorageClass.STANDARD+ def can_stream_artifact(self, artifact: Artifact) -> bool:+ # retrieve_artifact decompresses a zstd HAR and extracts a bundle member from its archive after the read.+ return not artifact.bundle_key and not artifact.uri.endswith(S3_ZSTD_COMPRESSED_SUFFIX)++ async def stat_artifact(self, artifact: Artifact) -> StoredObjectVersion | None:+ try:+ info = await self.async_client.get_object_info(artifact.uri)+ except Exception as e:+ self.async_client.log_download_failure(e, artifact.uri)+ return None+ return StoredObjectVersion(size=info["ContentLength"], etag=info.get("ETag"))++ async def open_artifact_stream(+ self, artifact: Artifact, *, byte_range: tuple[int, int] | None = None, etag: str | None = None+ ) -> S3ObjectStream | None:+ try:+ return await self.async_client.open_download_stream(artifact.uri, byte_range=byte_range, if_match=etag)+ except Exception as e:+ if etag is not None and self.async_client._error_code(e) == "PreconditionFailed":+ raise StoredObjectChangedError(artifact.uri) from e+ self.async_client.log_download_failure(e, artifact.uri)+ return None+async def retrieve_artifact(self, artifact: Artifact) -> bytes | None:data = await self.async_client.download_file(artifact.uri)# Decompress zstd-compressed files (HAR only)diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex d437af17d..0a325ff0a 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -34,6 +34,7 @@ from fastapi.responses import ORJSONResponse, StreamingResponsefrom opentelemetry import tracefrom pydantic import TypeAdapter, ValidationErrorfrom sqlalchemy.exc import IntegrityError, SQLAlchemyError+from starlette.types import Receive, Scope, Sendfrom skyvern import analyticsfrom skyvern._version import __version__@@ -66,7 +67,12 @@ from skyvern.forge.sdk.artifact.signing import (parse_keyring,verify_artifact_signature,)-from skyvern.forge.sdk.artifact.storage.base import artifact_filename_from_uri+from skyvern.forge.sdk.artifact.storage.base import (+ StoredObjectChangedError,+ StoredObjectStream,+ StreamingArtifactStorage,+ artifact_filename_from_uri,+)from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.curl_converter import curl_to_http_request_block_paramsfrom skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory@@ -3433,6 +3439,67 @@ def _artifact_content_response_headers(}+# The most one storage read returns, so a response holds about this much however large the object is.+_ARTIFACT_STREAM_CHUNK_BYTES = 1024 * 1024+++class _StoredArtifactResponse(StreamingResponse):+ """Streams a stored artifact and closes its storage read however the response ends, disconnects included."""++ def __init__(self, stream: StoredObjectStream, **kwargs: Any) -> None:+ super().__init__(stream.iter_chunks(_ARTIFACT_STREAM_CHUNK_BYTES), **kwargs)+ self._stream = stream++ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:+ try:+ await super().__call__(scope, receive, send)+ finally:+ await self._stream.aclose()+++async def _stream_artifact_content(+ storage: StreamingArtifactStorage,+ artifact: Artifact,+ range_header: str | None,+ media_type: str,+ headers: dict[str, str],+) -> Response:+ # A Range header resolves against the stored size exactly as it does against whole bytes, and the read is+ # pinned to the version that was sized; an object replaced in between is sized again.+ for _ in range(2):+ byte_range: tuple[int, int] | None = None+ etag: str | None = None+ content_range: str | None = None+ if range_header:+ version = await storage.stat_artifact(artifact)+ if version is None:+ break+ parsed_range = _parse_range_header(range_header, version.size)+ if parsed_range == _RANGE_UNSATISFIABLE:+ headers["Content-Range"] = f"bytes */{version.size}"+ return Response(status_code=http_status.HTTP_416_RANGE_NOT_SATISFIABLE, headers=headers)+ if parsed_range is not None:+ byte_range, etag = parsed_range, version.etag+ content_range = f"bytes {parsed_range[0]}-{parsed_range[1]}/{version.size}"+ try:+ stream = await storage.open_artifact_stream(artifact, byte_range=byte_range, etag=etag)+ except StoredObjectChangedError:+ continue+ if stream is None:+ break+ headers["Content-Length"] = str(stream.content_length)+ if content_range is None:+ return _StoredArtifactResponse(stream, media_type=media_type, headers=headers)+ headers["Content-Range"] = content_range+ return _StoredArtifactResponse(+ stream, status_code=http_status.HTTP_206_PARTIAL_CONTENT, media_type=media_type, headers=headers+ )+ raise HTTPException(+ status_code=http_status.HTTP_404_NOT_FOUND,+ detail="Artifact content not available",+ )++@base_router.get("/artifacts/{artifact_id}/content",tags=["Artifacts"],@@ -3505,12 +3572,6 @@ async def get_artifact_content(status_code=http_status.HTTP_404_NOT_FOUND,detail=f"Artifact not found {artifact_id}",)- content = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)- if content is None:- raise HTTPException(- status_code=http_status.HTTP_404_NOT_FOUND,- detail="Artifact content not available",- )media_type, content_disposition = _artifact_response_config(artifact)is_signed = sig is not None and expiry is not None and kid is not Nonesigned_expiry_unix: int | None = None@@ -3527,6 +3588,15 @@ async def get_artifact_content(headers["Accept-Ranges"] = "bytes"if artifact.artifact_type in _HTML_ARTIFACT_TYPES:headers["Content-Security-Policy"] = _HTML_ARTIFACT_CSP+ storage = app.STORAGE+ if isinstance(storage, StreamingArtifactStorage) and storage.can_stream_artifact(artifact):+ return await _stream_artifact_content(storage, artifact, request.headers.get("range"), media_type, headers)+ content = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)+ if content is None:+ raise HTTPException(+ status_code=http_status.HTTP_404_NOT_FOUND,+ detail="Artifact content not available",+ )content_length = len(content)parsed_range = _parse_range_header(request.headers.get("range"), content_length)if parsed_range == _RANGE_UNSATISFIABLE:diff --git a/skyvern/forge/taskv3/provider_downloads.py b/skyvern/forge/taskv3/provider_downloads.pynew file mode 100644index 000000000..0484b8eed--- /dev/null+++ b/skyvern/forge/taskv3/provider_downloads.py@@ -0,0 +1,194 @@+"""Read a vendor-kept browser download back into a v3 complete_on_download block's run download directory.++A vendor session can keep the browser's own save while the out-of-band replay of the URL fails, so the+completion probe reads the provider copy and lets the existing finalize path judge what lands."""++from __future__ import annotations++import asyncio+import time+from collections.abc import Awaitable, Callable, Collection+from pathlib import Path+from typing import Any, Protocol++import structlog++from skyvern.constants import BROWSER_DOWNLOADING_SUFFIX+from skyvern.webeye.browser_artifacts import ActionDownloadObservation, ActionDownloadSource++LOG = structlog.get_logger()++PROVIDER_BASELINE_TIMEOUT_SECONDS = 10.0+# The provider lists a file only once the browser finished saving it; a block's download_timeout, when+# set, replaces this per-download wait.+PROVIDER_DOWNLOAD_WAIT_SECONDS = 30.0+POLL_INTERVAL_SECONDS = 1.0+# Lets a downloadWillBegin raised by the action that just returned reach the monitor's counter.+EVENT_ADMISSION_GRACE_SECONDS = 0.25+++class _DownloadMonitor(Protocol):+ @property+ def browser_download_event_count(self) -> int: ...++ def has_pending_download_activity(self) -> bool: ...+++def _download_monitor(browser_context: Any) -> _DownloadMonitor | None:+ # Attached dynamically to the Playwright context by the browser factory.+ monitor = getattr(browser_context, "_skyvern_cdp_download_interceptor", None) if browser_context else None+ count = getattr(monitor, "browser_download_event_count", None)+ if not isinstance(count, int) or not callable(getattr(monitor, "has_pending_download_activity", None)):+ return None+ return monitor+++def _completed_names(directory: Path) -> set[str]:+ try:+ entries = list(directory.iterdir())+ except OSError:+ return set()+ return {entry.name for entry in entries if entry.is_file() and not entry.name.endswith(BROWSER_DOWNLOADING_SUFFIX)}+++class ProviderDownloadRecovery:+ """One v3 block's provider read-back. The download probe awaits it serially, never concurrently."""++ def __init__(+ self,+ observation: ActionDownloadObservation,+ download_dir: Path,+ monitor: _DownloadMonitor | None,+ ) -> None:+ self._observation = observation+ self._download_dir = download_dir+ self._monitor = monitor+ self._events_seen = monitor.browser_download_event_count if monitor is not None else 0+ self._local_seen = _completed_names(download_dir)+ self._owed_read = False++ @classmethod+ async def begin(+ cls,+ source: ActionDownloadSource | None,+ *,+ download_dir: Path,+ browser_context: Any,+ deadline: float,+ ) -> ProviderDownloadRecovery | None:+ """Freeze the provider baseline before the block acts, or ``None`` to leave every download path as is."""+ if source is None:+ return None+ try:+ observation = await source.begin_observation(deadline=deadline)+ except Exception as exc:+ # A provider error can embed a secret-bearing presigned URL, so only its type is logged.+ LOG.warning("Task V3 provider download baseline unavailable", error_type=type(exc).__name__)+ return None+ if observation is None:+ return None+ return cls(observation, download_dir, _download_monitor(browser_context))++ async def recover(+ self,+ *,+ staged: Collection[str],+ timeout: float,+ should_cancel: Callable[[], Awaitable[bool]],+ ) -> bool:+ """Read back files a download the browser began left with the provider; True only when ``should_cancel``+ fired. An admitted download leaves a read owed until a file lands, and completion stays any-file."""+ if timeout <= 0:+ return False+ deadline = time.monotonic() + timeout+ try:+ if self._local_delta(staged):+ self._settle_read()+ return False+ if self._monitor is None:+ # Nothing reports whether a download began, so read the provider once per probe.+ if await self._poll(deadline, should_cancel):+ return True+ self._log_read_back(staged)+ return False+ await asyncio.sleep(min(EVENT_ADMISSION_GRACE_SECONDS, max(0.0, deadline - time.monotonic())))+ new_event = self._monitor.browser_download_event_count != self._events_seen+ if new_event:+ self._events_seen = self._monitor.browser_download_event_count+ self._owed_read = True+ if not self._owed_read:+ return False+ # The monitor's own capture (a replay that works) runs first; reading the provider while it+ # is still in flight could publish the same download twice.+ while self._monitor.has_pending_download_activity():+ if not new_event or time.monotonic() >= deadline:+ return False+ if await should_cancel():+ return True+ await asyncio.sleep(min(POLL_INTERVAL_SECONDS, max(0.0, deadline - time.monotonic())))+ while True:+ if self._local_delta(staged):+ self._settle_read()+ return False+ if time.monotonic() >= deadline:+ return False+ if await self._poll(deadline, should_cancel):+ return True+ if self._log_read_back(staged):+ self._settle_read()+ return False+ if not new_event or time.monotonic() >= deadline:+ return False+ if await should_cancel():+ return True+ await asyncio.sleep(min(POLL_INTERVAL_SECONDS, max(0.0, deadline - time.monotonic())))+ finally:+ self._local_seen = _completed_names(self._download_dir)++ def _settle_read(self) -> None:+ # A file landed locally, so every event admitted so far is accounted for; a later probe must not+ # read one of them as new and publish the provider's copy of the same download beside it.+ self._owed_read = False+ if self._monitor is not None:+ self._events_seen = self._monitor.browser_download_event_count++ def _log_read_back(self, staged: Collection[str]) -> bool:+ landed = self._local_delta(staged)+ if landed:+ LOG.info("Task V3 provider download read back", file_count=len(landed))+ return bool(landed)++ def _local_delta(self, staged: Collection[str]) -> set[str]:+ return _completed_names(self._download_dir) - self._local_seen - set(staged)++ async def _poll(self, deadline: float, should_cancel: Callable[[], Awaitable[bool]]) -> bool:+ poll = asyncio.ensure_future(+ self._observation.poll_and_materialize(destination_dir=self._download_dir, deadline=deadline)+ )+ try:+ while True:+ remaining = deadline - time.monotonic()+ done, _ = await asyncio.wait({poll}, timeout=max(0.0, min(POLL_INTERVAL_SECONDS, remaining)))+ if poll in done or remaining <= 0:+ return False+ if await should_cancel():+ return True+ finally:+ await _join(poll)+++async def _join(poll: asyncio.Future[None]) -> None:+ """Cancel ``poll`` and wait for its own (cooperative, not deadline-bound) cleanup before returning.+ A cancellation of the caller while waiting is re-raised once the poll is done."""+ poll.cancel()+ cancellation: asyncio.CancelledError | None = None+ while not poll.done():+ try:+ await asyncio.wait({poll})+ except asyncio.CancelledError as caught:+ cancellation = caught+ if not poll.cancelled() and (exc := poll.exception()) is not None:+ # A provider error can embed a secret-bearing presigned URL, so only its type is logged.+ LOG.warning("Task V3 provider download poll failed", error_type=type(exc).__name__)+ if cancellation is not None:+ raise cancellationdiff --git a/tests/unit/_download_provider_fakes.py b/tests/unit/_download_provider_fakes.pynew file mode 100644index 000000000..d7311e7d6--- /dev/null+++ b/tests/unit/_download_provider_fakes.py@@ -0,0 +1,144 @@+"""Provider-neutral fakes for an action-scoped remote download source (``ActionDownloadSource``)."""++from __future__ import annotations++import asyncio+import time+from collections.abc import Callable+from pathlib import Path++from pydantic import BaseModel, ValidationError+++class _FakeActionDownloadObservation:+ """Neutral provider observation: drops completed remote files into the run dir on its first poll,+ honoring the ``deadline`` (monotonic) contract strictly (a handler passing per-call ``timeout_seconds``+ materializes nothing -- the anti-terminal-cleanup RED)."""++ def __init__(self, files: list[tuple[str, bytes]], *, empty_polls: int = 0) -> None:+ self._pending = list(files)+ self._empty_polls = empty_polls+ self.poll_deadlines: list[float] = []+ self.poll_started_at: list[float] = []++ async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:+ self.poll_deadlines.append(deadline)+ self.poll_started_at.append(time.monotonic())+ if len(self.poll_deadlines) <= self._empty_polls:+ return+ destination_dir.mkdir(parents=True, exist_ok=True)+ for name, data in self._pending:+ (destination_dir / name).write_bytes(data)+ self._pending = []+++class _RaisingActionDownloadObservation:+ """Provider observation whose poll raises a schema/validation error embedding a presigned URL, mirroring+ the real leak surface (the vendor list body is parsed into typed rows). When ``materialize_first`` is+ set, the first poll drops a completed file (so the handler reaches its final poll) and a later poll raises."""++ def __init__(self, error: BaseException, *, materialize_first: tuple[str, bytes] | None = None) -> None:+ self._error = error+ self._materialize_first = materialize_first+ self.calls = 0++ async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:+ self.calls += 1+ if self._materialize_first is not None and self.calls == 1:+ destination_dir.mkdir(parents=True, exist_ok=True)+ name, data = self._materialize_first+ (destination_dir / name).write_bytes(data)+ return+ raise self._error+++class _StallingActionDownloadObservation:+ """Provider observation whose poll never returns on its own -- only the handler's deadline can end it, so+ if the handler polls it once a local artifact already accounts for the action the stalled provider would+ consume the whole download deadline; the fix must never poll it in that case."""++ def __init__(self) -> None:+ self.poll_calls = 0+ self.poll_deadlines: list[float] = []++ async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:+ self.poll_calls += 1+ self.poll_deadlines.append(deadline)+ await asyncio.Event().wait()+++class _ActionDownloadSource:+ """Neutral provider source: hands back a fixed observation and records begin bookkeeping."""++ def __init__(+ self,+ observation: object,+ *,+ begin_error: Exception | None = None,+ on_begin: Callable[[], object] | None = None,+ ) -> None:+ self._observation = observation+ self._begin_error = begin_error+ self._on_begin = on_begin+ self.begin_calls = 0+ self.begin_deadline: float | None = None++ async def begin_observation(self, *, deadline: float) -> object:+ self.begin_calls += 1+ self.begin_deadline = deadline+ if self._on_begin is not None:+ self._on_begin()+ if self._begin_error is not None:+ raise self._begin_error+ return self._observation+++def _secret_bearing_validation_error(presigned_url: str) -> ValidationError:+ class _Row(BaseModel):+ size: int++ try:+ _Row.model_validate({"size": presigned_url})+ except ValidationError as exc:+ return exc+ raise AssertionError("expected a ValidationError")+++class _FakeBrowserDownloadMonitor:+ """Stand-in for the run's CDP download monitor: an admitted-event counter and a pending flag that+ reads True for ``pending_checks`` calls, then runs ``on_settle`` once (e.g. a replay landing a file)."""++ def __init__(self, *, pending_checks: int = 0, on_settle: object = None) -> None:+ self.browser_download_event_count = 0+ self._pending_checks = pending_checks+ self._on_settle = on_settle+ self.pending_calls = 0++ def has_pending_download_activity(self) -> bool:+ self.pending_calls += 1+ if self.pending_calls <= self._pending_checks:+ return True+ if self._on_settle is not None:+ settle, self._on_settle = self._on_settle, None+ settle() # type: ignore[operator]+ return False+++class _GatedCleanupObservation:+ """Provider observation whose poll never lists anything and whose cancellation cleanup waits on ``gate``,+ so a caller that returns before awaiting the cancelled poll is seen returning with cleanup unfinished."""++ def __init__(self, gate: asyncio.Event) -> None:+ self._gate = gate+ self.started = asyncio.Event()+ self.cleanup_started = asyncio.Event()+ self.cleanup_finished = False++ async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:+ self.started.set()+ try:+ await asyncio.Event().wait()+ finally:+ self.cleanup_started.set()+ await self._gate.wait()+ self.cleanup_finished = Truediff --git a/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py b/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.pyindex a273078b3..60768ae7b 100644--- a/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py+++ b/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py@@ -59,7 +59,8 @@ def mock_browser_session_artifact_create(monkeypatch: pytest.MonkeyPatch) -> Non@pytest.fixture(scope="module")def moto_server() -> Generator[str]:- server = ThreadedMotoServer(port=0)+ # Loopback, not 0.0.0.0: a macOS system proxy bypasses 127.0.0.1 but would otherwise relay the S3 traffic.+ server = ThreadedMotoServer(ip_address="127.0.0.1", port=0)server.start()host, port = server.get_host_and_port()yield f"http://{host}:{port}"diff --git a/tests/unit/forge/sdk/test_artifact_content_range.py b/tests/unit/forge/sdk/test_artifact_content_range.pyindex 1e3ad924a..25e05178e 100644--- a/tests/unit/forge/sdk/test_artifact_content_range.py+++ b/tests/unit/forge/sdk/test_artifact_content_range.py@@ -1,16 +1,31 @@from __future__ import annotations+import asyncio+import contextlib+import io+import uuid+import zipfile+from datetime import datetimefrom types import SimpleNamespace+from typing import Anyfrom unittest.mock import AsyncMock, MagicMock+import boto3import pytest+import zstandard as zstd+from aiobotocore.response import StreamingBodyfrom fastapi import FastAPIfrom fastapi.testclient import TestClient+from skyvern.config import settingsfrom skyvern.forge import app as forge_app-from skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.artifact.manager import ArtifactManager+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType+from skyvern.forge.sdk.artifact.storage.s3 import S3Storage+from skyvern.forge.sdk.routes import agent_protocolfrom skyvern.forge.sdk.routes import routers as routers_modulefrom skyvern.forge.sdk.routes.agent_protocol import _parse_range_header+from tests.unit.forge.sdk.artifact.storage.test_s3_storage_moto import moto_server # noqa: F401CONTENT = b"0123456789" * 100 # 1000 bytes for predictable slicing@@ -168,3 +183,260 @@ class TestArtifactContentRangeRequests:assert response.status_code == 200assert response.content == CONTENTassert response.headers["accept-ranges"] == "bytes"+++STREAM_BUCKET = "artifact-content-route"+++def stream_bucket(moto_server: str, no_env_proxy: None) -> Any: # noqa: F811+ client = boto3.client(+ "s3",+ aws_access_key_id="testing",+ aws_secret_access_key="testing",+ region_name=settings.AWS_REGION,+ endpoint_url=moto_server,+ )+ with contextlib.suppress(client.exceptions.BucketAlreadyOwnedByYou):+ client.create_bucket(Bucket=STREAM_BUCKET)+ return client+++def s3_backed_storage(+ monkeypatch: pytest.MonkeyPatch,+ moto_server: str, # noqa: F811+ stream_bucket: Any,+) -> S3Storage:+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing")+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing")+ monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False)+ monkeypatch.delenv("AWS_PROFILE", raising=False)+ storage = S3Storage(bucket=STREAM_BUCKET, endpoint_url=moto_server)+ monkeypatch.setattr(forge_app, "STORAGE", storage)+ monkeypatch.setattr(forge_app, "ARTIFACT_MANAGER", ArtifactManager())+ return storage+++def storage_bodies(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:+ """Every read of an S3 response body as (requested amount, bytes returned), and how many bodies were closed."""+ observed = SimpleNamespace(reads=[], closes=0)+ real_read = StreamingBody.read++ async def read(self: StreamingBody, amt: int | None = None) -> bytes:+ chunk = await real_read(self, amt)+ observed.reads.append((amt, len(chunk)))+ return chunk++ def close(self: StreamingBody) -> None:+ observed.closes += 1+ self.__wrapped__.close()++ monkeypatch.setattr(StreamingBody, "read", read)+ monkeypatch.setattr(StreamingBody, "close", close, raising=False)+ return observed+++def _store_artifact(+ s3: Any,+ data: bytes,+ *,+ artifact_type: ArtifactType = ArtifactType.DOWNLOAD,+ filename: str = "report.zip",+ bundle_key: str | None = None,+) -> Artifact:+ key = f"objects/{uuid.uuid4().hex}/{filename}"+ s3.put_object(Bucket=STREAM_BUCKET, Key=key, Body=data)+ now = datetime.utcnow()+ return Artifact(+ artifact_id="a_stored",+ artifact_type=artifact_type,+ uri=f"s3://{STREAM_BUCKET}/{key}",+ bundle_key=bundle_key,+ organization_id="org_oss",+ created_at=now,+ modified_at=now,+ )+++def _artifact_route_app(monkeypatch: pytest.MonkeyPatch, artifact: Artifact) -> FastAPI:+ async def _fake_org(**_: object) -> SimpleNamespace:+ return SimpleNamespace(organization_id="org_oss")++ artifacts_db = MagicMock()+ artifacts_db.get_artifact_by_id = AsyncMock(return_value=artifact)+ monkeypatch.setattr(forge_app.DATABASE, "artifacts", artifacts_db)+ monkeypatch.setattr(agent_protocol.org_auth_service, "get_current_org", _fake_org)+ fastapi_app = FastAPI()+ fastapi_app.include_router(routers_module.base_router, prefix="/v1")+ return fastapi_app+++def _asgi_get_scope(path: str) -> dict[str, Any]:+ return {+ "type": "http",+ "asgi": {"version": "3.0", "spec_version": "2.3"},+ "http_version": "1.1",+ "method": "GET",+ "scheme": "http",+ "path": path,+ "raw_path": path.encode(),+ "root_path": "",+ "query_string": b"",+ "headers": [(b"host", b"testserver"), (b"x-api-key", b"synthetic")],+ "client": ("127.0.0.1", 50000),+ "server": ("testserver", 80),+ }+++class TestArtifactContentFromS3:+ @pytest.mark.parametrize(+ ("range_header", "status", "body", "content_range"),+ [+ pytest.param(None, 200, CONTENT, None, id="full"),+ pytest.param("bytes=10-19", 206, CONTENT[10:20], "bytes 10-19/1000", id="range"),+ pytest.param("bytes=900-", 206, CONTENT[900:], "bytes 900-999/1000", id="open-ended"),+ pytest.param("bytes=-50", 206, CONTENT[-50:], "bytes 950-999/1000", id="suffix"),+ pytest.param("bytes=990-5000", 206, CONTENT[990:], "bytes 990-999/1000", id="end-clamped"),+ pytest.param("bytes=5000-6000", 416, b"", "bytes */1000", id="unsatisfiable"),+ pytest.param("bytes=garbage", 200, CONTENT, None, id="ignored"),+ ],+ )+ def test_plain_object_is_read_in_bounded_chunks_with_unchanged_range_semantics(+ self,+ monkeypatch: pytest.MonkeyPatch,+ s3_backed_storage: S3Storage,+ stream_bucket: Any,+ storage_bodies: SimpleNamespace,+ range_header: str | None,+ status: int,+ body: bytes,+ content_range: str | None,+ ) -> None:+ artifact = _store_artifact(stream_bucket, CONTENT)+ with TestClient(_artifact_route_app(monkeypatch, artifact)) as client:+ response = client.get(+ "/v1/artifacts/a_stored/content", headers={"Range": range_header} if range_header else {}+ )++ assert response.status_code == status+ assert response.content == body+ assert response.headers.get("content-range") == content_range+ assert response.headers["content-length"] == str(len(body))+ assert response.headers["accept-ranges"] == "bytes"+ assert response.headers["content-disposition"].startswith("attachment;")+ # GiB-sized downloads killed API tasks: no read may be unbounded, and a slice reads only its bytes.+ assert all(amount is not None for amount, _ in storage_bodies.reads)+ assert sum(size for _, size in storage_bodies.reads) == len(body)++ @pytest.mark.parametrize("range_header", [None, "bytes=5-14"])+ @pytest.mark.parametrize("kind", ["bundle-member", "compressed-har"])+ def test_decoded_artifacts_keep_their_whole_object_semantics(+ self,+ monkeypatch: pytest.MonkeyPatch,+ s3_backed_storage: S3Storage,+ stream_bucket: Any,+ kind: str,+ range_header: str | None,+ ) -> None:+ if kind == "bundle-member":+ served = b"<html><body>bundled member page</body></html>"+ archive = io.BytesIO()+ with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf:+ zf.writestr("scrape.html", served)+ zf.writestr("other.txt", b"another member")+ artifact = _store_artifact(+ stream_bucket,+ archive.getvalue(),+ artifact_type=ArtifactType.HTML_SCRAPE,+ filename="step_archive.zip",+ bundle_key="scrape.html",+ )+ else:+ served = b'{"log": {"version": "1.2", "entries": []}}' * 20+ artifact = _store_artifact(+ stream_bucket,+ zstd.ZstdCompressor(level=3).compress(served),+ artifact_type=ArtifactType.HAR,+ filename="network.har.zst",+ )++ with TestClient(_artifact_route_app(monkeypatch, artifact)) as client:+ response = client.get(+ "/v1/artifacts/a_stored/content", headers={"Range": range_header} if range_header else {}+ )++ if range_header is None:+ assert response.status_code == 200+ assert response.content == served+ else:+ assert response.status_code == 206+ assert response.content == served[5:15]+ assert response.headers["content-range"] == f"bytes 5-14/{len(served)}"++ def test_range_read_of_an_object_replaced_after_sizing_serves_one_version(+ self,+ monkeypatch: pytest.MonkeyPatch,+ s3_backed_storage: S3Storage,+ stream_bucket: Any,+ ) -> None:+ artifact = _store_artifact(stream_bucket, CONTENT)+ replacement = b"abcdefghij" * 300+ sized_versions: list[Any] = []+ stat_artifact = s3_backed_storage.stat_artifact++ async def stat_then_replace(stored: Artifact) -> Any:+ version = await stat_artifact(stored)+ if not sized_versions:+ key = stored.uri.removeprefix(f"s3://{STREAM_BUCKET}/")+ stream_bucket.put_object(Bucket=STREAM_BUCKET, Key=key, Body=replacement)+ sized_versions.append(version)+ return version++ monkeypatch.setattr(s3_backed_storage, "stat_artifact", stat_then_replace)+ with TestClient(_artifact_route_app(monkeypatch, artifact)) as client:+ response = client.get("/v1/artifacts/a_stored/content", headers={"Range": "bytes=0-9"})++ assert response.status_code == 206+ assert response.content == replacement[:10]+ assert response.headers["content-range"] == f"bytes 0-9/{len(replacement)}"+ assert [version.size for version in sized_versions] == [len(CONTENT), len(replacement)]++ @pytest.mark.asyncio+ async def test_client_disconnect_stops_reading_and_closes_the_storage_body(+ self,+ monkeypatch: pytest.MonkeyPatch,+ s3_backed_storage: S3Storage,+ stream_bucket: Any,+ storage_bodies: SimpleNamespace,+ ) -> None:+ large = bytes(range(256)) * 32 * 1024 # 8 MiB+ fastapi_app = _artifact_route_app(monkeypatch, _store_artifact(stream_bucket, large))+ first_body_sent = asyncio.Event()+ sent: list[dict[str, Any]] = []+ request_delivered = False++ async def receive() -> dict[str, Any]:+ nonlocal request_delivered+ if not request_delivered:+ request_delivered = True+ return {"type": "http.request", "body": b"", "more_body": False}+ await first_body_sent.wait()+ return {"type": "http.disconnect"}++ async def send(message: dict[str, Any]) -> None:+ sent.append(message)+ if message["type"] == "http.response.body" and message.get("body"):+ first_body_sent.set()+ # A served send is an await point (BaseHTTPMiddleware passes each message through a memory stream).+ # Without one, anyio skips delivering the disconnect's cancellation while every local S3 read is complete.+ await asyncio.sleep(0)++ await fastapi_app(_asgi_get_scope("/v1/artifacts/a_stored/content"), receive, send)++ assert sent[0]["status"] == 200+ assert 0 < sum(len(message.get("body", b"")) for message in sent[1:]) < len(large)+ assert sum(size for _, size in storage_bodies.reads) < len(large)+ assert storage_bodies.closes == 1+ assert all(shared.leases == 0 for shared in s3_backed_storage.async_client._shared_s3_clients.values())diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex da0e3aea4..c82f61e1e 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -11,6 +11,7 @@ from __future__ import annotationsimport asyncioimport json+import timefrom collections.abc import Awaitable, Callable, Sequencefrom datetime import UTC, datetimefrom pathlib import Path@@ -65,6 +66,7 @@ from skyvern.forge.sdk.workflow.page_derived_templates import CLOSE as PAGE_DERIfrom skyvern.forge.sdk.workflow.page_derived_templates import OPEN as PAGE_DERIVED_OPENfrom skyvern.forge.taskv3 import engine as taskv3_enginefrom skyvern.forge.taskv3 import input_dispatch+from skyvern.forge.taskv3 import provider_downloads as taskv3_provider_downloadsfrom skyvern.forge.taskv3 import tools as taskv3_toolsfrom skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationStatefrom skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop@@ -103,6 +105,14 @@ from skyvern.webeye.actions.actions import (SolveCaptchaAction,UploadFileAction,)+from tests.unit._download_provider_fakes import (+ _ActionDownloadSource,+ _FakeActionDownloadObservation,+ _FakeBrowserDownloadMonitor,+ _GatedCleanupObservation,+ _secret_bearing_validation_error,+ _StallingActionDownloadObservation,+)from tests.unit.helpers import make_action_row, make_browser_state, make_organization, make_step, make_taskfrom tests.unit.scoped_asyncio import ScopedAsynciofrom tests.unit.test_taskv3_engine import _fixed_read_tool, _ReaskAnsweringCaller@@ -2291,6 +2301,10 @@ async def _run_execute_task_v3_download(loop_fn: Callable[[dict[str, Any], dict[str, Any]], Awaitable[LoopOutcome]] | None = None,complete_on_download: bool = True,browser_session_id: str | None = None,+ provider_source: object | None = None,+ download_monitor: object | None = None,+ download_timeout: float | None = None,+ cancel_after_checks: int | None = None,) -> dict[str, Any]:"""Drive _execute_task_v3 for a complete_on_download block, capturing the loop kwargs.@@ -2321,9 +2335,30 @@ async def _run_execute_task_v3_download(browser_state.get_working_page = AsyncMock(return_value=page)browser_state.take_post_action_screenshot = AsyncMock(return_value=b"png-bytes")- block = _make_block(block_cls, complete_on_download=complete_on_download, download_suffix=download_suffix)+ # A real session without a provider-owned download store attaches no source; the bare MagicMock+ # default would otherwise look like one.+ browser_state.browser_artifacts.get_action_download_source = MagicMock(return_value=provider_source)+ browser_state.browser_context._skyvern_cdp_download_interceptor = download_monitor++ block_overrides: dict[str, Any] = {"complete_on_download": complete_on_download, "download_suffix": download_suffix}+ if download_timeout is not None:+ block_overrides["download_timeout"] = download_timeout+ block = _make_block(block_cls, **block_overrides)++ cancel_checks = {"count": 0}- captured: dict[str, Any] = {}+ async def _get_task(**_kwargs: Any) -> Any:+ cancel_checks["count"] += 1+ canceled = cancel_after_checks is not None and cancel_checks["count"] > cancel_after_checks+ return SimpleNamespace(status=TaskStatus.canceled if canceled else TaskStatus.running)++ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task", AsyncMock(side_effect=_get_task))+ monkeypatch.setattr(+ "skyvern.forge.agent.app.DATABASE.workflow_runs.get_workflow_run",+ AsyncMock(return_value=SimpleNamespace(status=WorkflowRunStatus.running)),+ )++ captured: dict[str, Any] = {"cancel_checks": cancel_checks}async def _default_loop_body(**kwargs: Any) -> LoopOutcome:if drop_file:@@ -6776,3 +6811,545 @@ async def test_goal_judge_prompt_never_carries_a_registered_runtime_secret(monkefor _ in range(3):assert redact("verification_code: 482913") == f"verification_code: {REDACTED_SECRET_PLACEHOLDER}"assert secret_set_builds == 1+++# ---------------------------------------------------------------------------+# Provider-owned downloads: a vendor session can keep the browser's own save while the run's+# out-of-band replay of the URL fails. The v3 download probe reads that file back into the run+# download dir so the existing finalize/complete path sees it.+# ---------------------------------------------------------------------------+++def _click_then_probe(+ tmp_path: Path,+ monitor: _FakeBrowserDownloadMonitor | None,+ source: _ActionDownloadSource | None = None,+ *,+ local_file: str | None = None,+ raise_event: bool = True,+) -> Callable[[dict[str, Any], dict[str, Any]], Awaitable[LoopOutcome]]:+ async def _loop(kwargs: dict[str, Any], captured: dict[str, Any]) -> LoopOutcome:+ captured["begin_calls_at_loop_start"] = source.begin_calls if source is not None else None+ if monitor is not None and raise_event:+ monitor.browser_download_event_count += 1+ if local_file is not None:+ (tmp_path / local_file).write_bytes(b"local-bytes")+ probe = kwargs["completion_probe"]+ captured["probe_started_at"] = time.monotonic()+ captured["probe_reason"] = await probe(frozenset())+ captured["probe_returned_at"] = time.monotonic()+ if captured["probe_reason"]:+ return LoopOutcome(status="completed", reason=captured["probe_reason"], billable_actions=["click"])+ return LoopOutcome(status="budget_exhausted", reason="no download detected", billable_actions=[])++ return _loop+++def _record_provider_sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]:+ sleeps: list[float] = []++ async def _record(delay: float) -> None:+ sleeps.append(delay)++ monkeypatch.setattr(taskv3_provider_downloads, "asyncio", ScopedAsyncio(sleep=_record))+ return sleeps+++async def test_v3_download_reads_a_provider_owned_file_back_when_the_replay_saves_nothing(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")], empty_polls=2)+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ block_cls=FileDownloadBlock,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source),+ )++ assert captured["begin_calls_at_loop_start"] == 1+ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert (tmp_path / "invoice.csv").read_bytes() == b"real-export"+ assert len(observation.poll_deadlines) == 3+++async def test_v3_download_does_not_poll_the_provider_when_a_local_capture_landed(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ monitor = _FakeBrowserDownloadMonitor()+ observation = _StallingActionDownloadObservation()+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source, local_file="report.pdf"),+ )++ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert observation.poll_calls == 0+ assert (tmp_path / "invoice.pdf").read_bytes() == b"local-bytes"+++async def test_v3_download_waits_for_the_monitor_replay_before_polling_the_provider(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ # A replay that is still in flight and then lands must win; polling the provider first would+ # publish a second copy of the same download.+ monitor = _FakeBrowserDownloadMonitor(+ pending_checks=2, on_settle=lambda: (tmp_path / "report.pdf").write_bytes(b"replayed")+ )+ observation = _StallingActionDownloadObservation()+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source),+ )++ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert observation.poll_calls == 0+ assert sorted(p.name for p in tmp_path.iterdir() if p.is_file()) == ["invoice.pdf"]+++async def test_v3_download_returns_promptly_without_polling_when_no_download_began(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ sleeps = _record_provider_sleeps(monkeypatch)+ monitor = _FakeBrowserDownloadMonitor()+ observation = _StallingActionDownloadObservation()+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source, raise_event=False),+ )++ assert captured["probe_reason"] is None+ assert observation.poll_calls == 0+ assert sleeps == [taskv3_provider_downloads.EVENT_ADMISSION_GRACE_SECONDS]+++async def test_v3_download_keeps_distinct_provider_files_with_equal_bytes(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([("summary.csv", b"same-bytes"), ("detail.csv", b"same-bytes")])+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source),+ )++ assert captured["probe_reason"] == "download completed (2 new file(s))"+ landed = [p for p in tmp_path.iterdir() if p.is_file()]+ assert len(landed) == 2+ assert all(p.read_bytes() == b"same-bytes" for p in landed)+++async def test_v3_download_credits_only_files_new_since_the_block_began(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ # An earlier block's file is already in the run dir; it is neither a local capture for this+ # block (which would suppress the provider read) nor a file this block delivered.+ (tmp_path / "earlier.pdf").write_bytes(b"earlier")+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source),+ )++ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert (tmp_path / "earlier.pdf").read_bytes() == b"earlier"+ assert (tmp_path / "invoice.csv").read_bytes() == b"real-export"+++async def test_v3_download_stops_reading_the_provider_when_the_run_is_cancelled(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ monitor = _FakeBrowserDownloadMonitor()+ observation = _StallingActionDownloadObservation()+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ cancel_after_checks=0,+ loop_fn=_click_then_probe(tmp_path, monitor, source),+ )++ assert captured["probe_reason"] is None+ assert observation.poll_calls == 1+ # Returned well before the stalled poll's own deadline, so the cancel ended it.+ assert captured["probe_returned_at"] < observation.poll_deadlines[0]+ assert not [p for p in tmp_path.iterdir() if p.is_file()]+++async def test_v3_download_does_not_complete_from_a_provider_file_read_after_the_run_was_cancelled(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ # The cancel lands during the action and the provider answers inside its first wait, so no cancel+ # check runs between the read and finalize unless recover makes one.+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ cancel_after_checks=0,+ loop_fn=_click_then_probe(tmp_path, monitor, source),+ )++ assert len(observation.poll_deadlines) == 1+ assert captured["probe_reason"] is None+++async def test_v3_download_without_a_monitor_does_not_complete_from_a_provider_file_after_a_cancel(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ cancel_after_checks=0,+ loop_fn=_click_then_probe(tmp_path, None, source),+ )++ assert len(observation.poll_deadlines) == 1+ assert captured["probe_reason"] is None++[email protected]("baseline", ["listed", "unavailable", "failed"])+async def test_v3_download_does_not_credit_a_file_that_landed_while_the_provider_baseline_was_read(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, baseline: str+) -> None:+ # An earlier block's download finishes while this block awaits the provider baseline, after the+ # local baseline was already listed; the read may succeed, list nothing usable, or fail.+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([])+ source = _ActionDownloadSource(+ None if baseline == "unavailable" else observation,+ begin_error=RuntimeError("provider down") if baseline == "failed" else None,+ on_begin=lambda: (tmp_path / "earlier.pdf").write_bytes(b"earlier-block"),+ )++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source, raise_event=False),+ )++ assert captured["probe_reason"] is None+ assert (tmp_path / "earlier.pdf").read_bytes() == b"earlier-block"+++async def test_v3_download_does_not_complete_from_a_local_file_after_the_run_was_cancelled(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ # No partial file means the in-flight wait never checks; the cancel must still win over finalize.+ captured = await _run_execute_task_v3_download(monkeypatch, tmp_path, drop_file=True, cancel_after_checks=0)++ assert captured["probe_reason"] is None+++async def test_v3_download_provider_wait_ends_at_the_block_download_timeout(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([("export.csv", b"late")], empty_polls=10_000)+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ download_timeout=2.0,+ loop_fn=_click_then_probe(tmp_path, monitor, source),+ )++ assert captured["probe_reason"] is None+ assert observation.poll_deadlines+ # The block's 2 s timeout, not the 30 s default, set the read deadline; no poll started past it.+ assert observation.poll_deadlines[0] - captured["probe_started_at"] < (+ taskv3_provider_downloads.PROVIDER_DOWNLOAD_WAIT_SECONDS+ )+ assert all(+ started < deadline+ for started, deadline in zip(observation.poll_started_at, observation.poll_deadlines, strict=True)+ )+ assert not [p for p in tmp_path.iterdir() if p.is_file()]+++async def test_v3_download_keeps_the_local_path_when_the_provider_baseline_fails(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ presigned = "https://downloads.example/file.csv?X-Amz-Signature=deadbeefsecret"+ source = _ActionDownloadSource(+ _StallingActionDownloadObservation(), begin_error=_secret_bearing_validation_error(presigned)+ )+ monitor = _FakeBrowserDownloadMonitor()++ with capture_logs() as logs:+ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source, local_file="report.pdf"),+ )++ assert source.begin_calls == 1+ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert "deadbeefsecret" not in json.dumps(logs, default=str)+++async def test_v3_download_does_not_take_a_staged_upload_for_the_admitted_download(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ # A file_upload staged into the run dir in the same round must not settle the provider read.+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])+ source = _ActionDownloadSource(observation)++ async def _loop(kwargs: dict[str, Any], captured: dict[str, Any]) -> LoopOutcome:+ monitor.browser_download_event_count += 1+ (tmp_path / "upload.pdf").write_bytes(b"staged-upload")+ captured["probe_reason"] = await kwargs["completion_probe"](frozenset({"upload.pdf"}))+ return LoopOutcome(status="completed", reason="done", billable_actions=["click"])++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_loop,+ )++ assert len(observation.poll_deadlines) == 1+ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert (tmp_path / "upload.pdf").read_bytes() == b"staged-upload"+++async def test_v3_wait_only_download_does_not_read_the_provider(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ # Only a complete_on_download block owns a provider read-back; a download_timeout-only block keeps its+ # native in-flight wait.+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ complete_on_download=False,+ download_timeout=10.0,+ provider_source=source,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, source),+ )++ assert captured["begin_calls_at_loop_start"] == 0+ assert captured["probe_reason"] is None+ assert observation.poll_deadlines == []+ assert not [p for p in tmp_path.iterdir() if p.is_file()]+++async def test_v3_download_without_a_monitor_reads_the_provider_once_per_probe(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])+ source = _ActionDownloadSource(observation)++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=None,+ loop_fn=_click_then_probe(tmp_path, None, source),+ )++ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert len(observation.poll_deadlines) == 1+++async def test_v3_download_without_a_provider_source_is_unchanged(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ sleeps = _record_provider_sleeps(monkeypatch)+ monitor = _FakeBrowserDownloadMonitor()++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ download_monitor=monitor,+ loop_fn=_click_then_probe(tmp_path, monitor, None),+ )++ assert captured["probe_reason"] is None+ assert sleeps == []+++def _probe_twice(+ monitor: _FakeBrowserDownloadMonitor, between: Callable[[], None]+) -> Callable[[dict[str, Any], dict[str, Any]], Awaitable[LoopOutcome]]:+ # One click, two probes: the second follows an unrelated billable tool, with no new download event.+ async def _loop(kwargs: dict[str, Any], captured: dict[str, Any]) -> LoopOutcome:+ monitor.browser_download_event_count += 1+ probe = kwargs["completion_probe"]+ captured["first_reason"] = await probe(frozenset())+ between()+ captured["probe_reason"] = await probe(frozenset())+ if captured["probe_reason"]:+ return LoopOutcome(status="completed", reason=captured["probe_reason"], billable_actions=["click"])+ return LoopOutcome(status="budget_exhausted", reason="no download detected", billable_actions=[])++ return _loop+++async def test_v3_download_still_reads_the_provider_after_the_monitor_outlasted_the_first_probe(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ monitor = _FakeBrowserDownloadMonitor(pending_checks=10**6)+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])+ source = _ActionDownloadSource(observation)++ def _settle() -> None:+ monitor._pending_checks = 0++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ download_timeout=1.0,+ loop_fn=_probe_twice(monitor, _settle),+ )++ assert captured["first_reason"] is None+ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert (tmp_path / "invoice.csv").read_bytes() == b"real-export"+++async def test_v3_download_still_reads_the_provider_after_an_empty_first_window(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ monitor = _FakeBrowserDownloadMonitor()+ observation = _FakeActionDownloadObservation([("export.csv", b"late-export")], empty_polls=10**6)+ source = _ActionDownloadSource(observation)++ def _list_late() -> None:+ observation._empty_polls = 0++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ download_timeout=1.0,+ loop_fn=_probe_twice(monitor, _list_late),+ )++ assert captured["first_reason"] is None+ assert captured["probe_reason"] == "download completed (1 new file(s))"+ assert (tmp_path / "invoice.csv").read_bytes() == b"late-export"+++async def test_v3_download_cancellation_returns_only_after_the_provider_poll_cleaned_up(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ gate = asyncio.Event()+ observation = _GatedCleanupObservation(gate)+ source = _ActionDownloadSource(observation)+ monitor = _FakeBrowserDownloadMonitor()++ async def _loop(kwargs: dict[str, Any], captured: dict[str, Any]) -> LoopOutcome:+ monitor.browser_download_event_count += 1+ asyncio.get_running_loop().call_later(2.0, gate.set)+ captured["probe_reason"] = await kwargs["completion_probe"](frozenset())+ captured["cleanup_finished_at_return"] = observation.cleanup_finished+ return LoopOutcome(status="budget_exhausted", reason="no download detected", billable_actions=[])++ captured = await _run_execute_task_v3_download(+ monkeypatch,+ tmp_path,+ drop_file=False,+ provider_source=source,+ download_monitor=monitor,+ cancel_after_checks=0,+ loop_fn=_loop,+ )++ assert captured["probe_reason"] is None+ assert captured["cleanup_finished_at_return"] is Truediff --git a/tests/unit/test_aws_client_ttl.py b/tests/unit/test_aws_client_ttl.pyindex 819dbb486..064dee799 100644--- a/tests/unit/test_aws_client_ttl.py+++ b/tests/unit/test_aws_client_ttl.py@@ -582,7 +582,9 @@ async def test_upload_file_from_path_cancelled_mid_multipart_aborts_upload_and_s@contextlib.asynccontextmanager-async def _local_s3(barrier: asyncio.Barrier | None = None, port: int = 0): # type: ignore[no-untyped-def]+async def _local_s3( # type: ignore[no-untyped-def]+ barrier: asyncio.Barrier | None = None, port: int = 0, body: bytes = b"payload"+):"""A plain-HTTP stand-in for S3 that records which connection and credential served each request."""seen: list[tuple[int, str, asyncio.BaseTransport]] = []@@ -593,7 +595,7 @@ async def _local_s3(barrier: asyncio.Barrier | None = None, port: int = 0): # tseen.append((transport.get_extra_info("peername")[1], request.headers.get("Authorization", ""), transport))if barrier is not None:await barrier.wait()- return web.Response(body=b"payload" if request.method == "GET" else b"")+ return web.Response(body=body if request.method == "GET" else b"")app = web.Application()app.router.add_route("*", "/{tail:.*}", handler)@@ -699,3 +701,47 @@ def test_s3_client_keeps_working_across_event_loops() -> None:assert asyncio.run(download_once()) == b"payload"assert asyncio.run(download_once()) == b"payload"+++async def test_open_download_stream_keeps_its_client_open_until_closed_across_a_session_replacement() -> None:+ """A streamed body outlives get_object, so the shared client it came from must not close under it."""+ payload = bytes(range(256)) * 16 * 1024+ async with _local_s3(body=payload) as (endpoint_url, _):+ client = _local_client(endpoint_url)+ stream = await client.open_download_stream("s3://bucket/large.zip")+ replaced_client = client._shared_s3_clients[asyncio.get_running_loop()]+ chunks = stream.iter_chunks(64 * 1024)+ received = [await anext(chunks)]++ client.session = aws.aioboto3.Session(aws_access_key_id="AKIAROTATED", aws_secret_access_key="rotated")+ assert await client.download_file("s3://bucket/k") == payload+ received.extend([chunk async for chunk in chunks])+ assert not replaced_client._owner.done()+ await stream.aclose()++ async with asyncio.timeout(5):+ await replaced_client._owner+ assert stream.content_length == len(payload)+ assert b"".join(received) == payload+++async def test_open_download_stream_releases_its_client_when_closing_the_body_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async with _local_s3() as (endpoint_url, _):+ client = _local_client(endpoint_url)+ stream = await client.open_download_stream("s3://bucket/k")+ shared = client._shared_s3_clients[asyncio.get_running_loop()]+ close_body = stream._body.close++ def close_then_fail() -> None:+ close_body()+ raise RuntimeError("body close failed")++ monkeypatch.setattr(stream._body, "close", close_then_fail)+ with pytest.raises(RuntimeError):+ await stream.aclose()++ assert shared.leases == 0diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.pyindex 78504cbed..80b09c250 100644--- a/tests/unit/test_download_file_action_handler.py+++ b/tests/unit/test_download_file_action_handler.py@@ -16,7 +16,6 @@ from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patchimport pytestimport structlogfrom opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter-from pydantic import BaseModel, ValidationErrorfrom structlog.testing import capture_logsfrom skyvern.config import settings@@ -61,6 +60,13 @@ from skyvern.webeye.browser_factory import (from skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptorfrom skyvern.webeye.scraper.scraped_page import ScrapedPagefrom skyvern.webeye.utils.page import BlobActionFreshness+from tests.unit._download_provider_fakes import (+ _ActionDownloadSource,+ _FakeActionDownloadObservation,+ _RaisingActionDownloadObservation,+ _secret_bearing_validation_error,+ _StallingActionDownloadObservation,+)from tests.unit.helpers import make_organization, make_step, make_task# Five seconds is only a test-side runaway guard. The behavior under test is@@ -7082,87 +7088,6 @@ async def test_retention_torn_down_on_cancelled() -> None:result["teardown"].assert_awaited_once()-class _FakeActionDownloadObservation:- """Neutral provider observation: drops completed remote files into the run dir on its first poll,- honoring the ``deadline`` (monotonic) contract strictly (a handler passing per-call ``timeout_seconds``- materializes nothing -- the anti-terminal-cleanup RED)."""-- def __init__(self, files: list[tuple[str, bytes]], *, empty_polls: int = 0) -> None:- self._pending = list(files)- self._empty_polls = empty_polls- self.poll_deadlines: list[float] = []-- async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:- self.poll_deadlines.append(deadline)- if len(self.poll_deadlines) <= self._empty_polls:- return- destination_dir.mkdir(parents=True, exist_ok=True)- for name, data in self._pending:- (destination_dir / name).write_bytes(data)- self._pending = []---class _RaisingActionDownloadObservation:- """Provider observation whose poll raises a schema/validation error embedding a presigned URL, mirroring- the real leak surface (the vendor list body is parsed into typed rows). When ``materialize_first`` is- set, the first poll drops a completed file (so the handler reaches its final poll) and a later poll raises."""-- def __init__(self, error: BaseException, *, materialize_first: tuple[str, bytes] | None = None) -> None:- self._error = error- self._materialize_first = materialize_first- self.calls = 0-- async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:- self.calls += 1- if self._materialize_first is not None and self.calls == 1:- destination_dir.mkdir(parents=True, exist_ok=True)- name, data = self._materialize_first- (destination_dir / name).write_bytes(data)- return- raise self._error---class _StallingActionDownloadObservation:- """Provider observation whose poll never returns on its own -- only the handler's deadline can end it, so- if the handler polls it once a local artifact already accounts for the action the stalled provider would- consume the whole download deadline; the fix must never poll it in that case."""-- def __init__(self) -> None:- self.poll_calls = 0-- async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:- self.poll_calls += 1- await asyncio.Event().wait()---class _ActionDownloadSource:- """Neutral provider source: hands back a fixed observation and records begin bookkeeping."""-- def __init__(self, observation: object, *, begin_error: Exception | None = None) -> None:- self._observation = observation- self._begin_error = begin_error- self.begin_calls = 0- self.begin_deadline: float | None = None-- async def begin_observation(self, *, deadline: float) -> object:- self.begin_calls += 1- self.begin_deadline = deadline- if self._begin_error is not None:- raise self._begin_error- return self._observation---def _secret_bearing_validation_error(presigned_url: str) -> ValidationError:- class _Row(BaseModel):- size: int-- try:- _Row.model_validate({"size": presigned_url})- except ValidationError as exc:- return exc- raise AssertionError("expected a ValidationError")--@contextlib.contextmanagerdef _capture_handler_logs() -> Iterator[io.StringIO]:import skyvern.webeye.actions.handler as handler_modulediff --git a/tests/unit/test_taskv3_provider_downloads.py b/tests/unit/test_taskv3_provider_downloads.pynew file mode 100644index 000000000..aee161e27--- /dev/null+++ b/tests/unit/test_taskv3_provider_downloads.py@@ -0,0 +1,90 @@+"""ProviderDownloadRecovery owns each provider poll it starts: a recover() call never returns, normally or by+cancellation, while that poll is still running its own cleanup."""++from __future__ import annotations++import asyncio+from collections.abc import Awaitable, Callable+from pathlib import Path++import pytest++from skyvern.forge.taskv3 import provider_downloads+from skyvern.forge.taskv3.provider_downloads import ProviderDownloadRecovery+from tests.unit._download_provider_fakes import _FakeBrowserDownloadMonitor, _GatedCleanupObservation+from tests.unit.scoped_asyncio import ScopedAsyncio+++async def _never_cancel() -> bool:+ return False+++async def _always_cancel() -> bool:+ return True+++def _recovery(tmp_path: Path, observation: object) -> tuple[ProviderDownloadRecovery, _FakeBrowserDownloadMonitor]:+ monitor = _FakeBrowserDownloadMonitor()+ recovery = ProviderDownloadRecovery(observation, tmp_path, monitor) # type: ignore[arg-type]+ monitor.browser_download_event_count += 1+ return recovery, monitor+++def _release_later(gate: asyncio.Event, delay: float) -> None:+ asyncio.get_running_loop().call_later(delay, gate.set)++[email protected]("should_cancel", [_never_cancel, _always_cancel], ids=["deadline", "should_cancel"])+async def test_recover_returns_only_after_its_cancelled_poll_finished_cleanup(+ tmp_path: Path, should_cancel: Callable[[], Awaitable[bool]]+) -> None:+ gate = asyncio.Event()+ observation = _GatedCleanupObservation(gate)+ recovery, _monitor = _recovery(tmp_path, observation)+ _release_later(gate, 1.5)++ await recovery.recover(staged=(), timeout=0.6, should_cancel=should_cancel)++ assert observation.started.is_set()+ assert observation.cleanup_finished+++async def test_outer_cancellation_joins_the_poll_even_when_cancelled_again_during_cleanup(tmp_path: Path) -> None:+ gate = asyncio.Event()+ observation = _GatedCleanupObservation(gate)+ recovery, _monitor = _recovery(tmp_path, observation)+ task = asyncio.create_task(recovery.recover(staged=(), timeout=30.0, should_cancel=_never_cancel))+ await asyncio.wait_for(observation.started.wait(), 5)++ task.cancel()+ await asyncio.wait_for(observation.cleanup_started.wait(), 5)+ task.cancel()+ await asyncio.sleep(0.05)+ assert not task.done(), "recover returned while its cancelled poll was still cleaning up"+ gate.set()++ with pytest.raises(asyncio.CancelledError):+ await task+ assert observation.cleanup_finished+++async def test_admission_grace_never_outlasts_the_probe_budget(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:+ sleeps: list[float] = []++ async def _record(delay: float) -> None:+ sleeps.append(delay)++ monkeypatch.setattr(provider_downloads, "asyncio", ScopedAsyncio(sleep=_record))+ gate = asyncio.Event()+ gate.set()+ observation = _GatedCleanupObservation(gate)+ recovery, _monitor = _recovery(tmp_path, observation)++ await recovery.recover(staged=(), timeout=0.05, should_cancel=_never_cancel)++ assert sleeps+ assert all(delay <= 0.05 for delay in sleeps)+ assert observation.cleanup_finished or not observation.started.is_set()