Skip to content

Automate browser based workflows with AI

Record run.completed, plan on the organization, and upgrade starts on the portal route (SKY-18287) (#8911)

Shuchang Zhengacbce161a656
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 9d2141da9..b3d69b49c 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -204,6 +204,11 @@ from skyvern.forge.taskv3.goal_check import (
from skyvern.forge.taskv3.goal_composition import CodeProgressRecord
from skyvern.forge.taskv3.loop import ACTION_BLOCK_TARGET_ACTION_RESERVE, LoopOutcome, RoundAction
from skyvern.forge.taskv3.pre_submit_capture import PreSubmitCaptureRing, is_run_sampled, pre_submit_screenshot
+from skyvern.forge.taskv3.provider_downloads import (
+ PROVIDER_BASELINE_TIMEOUT_SECONDS,
+ PROVIDER_DOWNLOAD_WAIT_SECONDS,
+ ProviderDownloadRecovery,
+)
from skyvern.forge.taskv3.run_arms import (
DATE_SEGMENT_AIM_FLAG,
LOGIN_PACE_FLAG,
@@ -2638,6 +2643,31 @@ class ForgeAgent:
super().add(name)
staged_downloads: set[str] = _StagedDownloads()
+
+ # Frozen before the first action so a provider file an earlier block left is never this block's.
+ provider_recovery: ProviderDownloadRecovery | None = None
+ provider_source = (
+ browser_state.browser_artifacts.get_action_download_source()
+ if task_block is not None and task_block.complete_on_download and workflow_download_directory is not None
+ else None
+ )
+ if provider_source is not None and workflow_download_directory is not None:
+ provider_recovery = await ProviderDownloadRecovery.begin(
+ provider_source,
+ download_dir=workflow_download_directory,
+ browser_context=browser_state.browser_context,
+ deadline=time.monotonic() + min(PROVIDER_BASELINE_TIMEOUT_SECONDS, loop_deadline_at - time.monotonic()),
+ )
+ # A file that landed while the provider baseline was awaited predates this block's first action,
+ # whether or not the baseline read succeeded.
+ download_baseline_files = [
+ *(download_baseline_files or []),
+ *list_files_in_directory(workflow_download_directory, attempt_started_at=attempt_started_at),
+ ]
+
+ def _provider_wait_seconds(block: BaseTaskBlock) -> float:
+ return min(block.download_timeout or PROVIDER_DOWNLOAD_WAIT_SECONDS, loop_deadline_at - time.monotonic())
+
completion_probe: CompletionProbe | None = None
completion_blocker: CompletionBlocker | None = None
if task_block is not None and task_block.complete_on_download and task.workflow_run_id:
@@ -2663,6 +2693,16 @@ class ForgeAgent:
# The run is being canceled; even if a file landed while we waited, don't
# finalize it into a `completed` result out from under the cancellation.
return None
+ if provider_recovery is not None and await provider_recovery.recover(
+ staged=staged,
+ timeout=_provider_wait_seconds(complete_on_download_block),
+ should_cancel=_should_cancel,
+ ):
+ return None
+ # Either wait can return inside its first poll interval without having checked; a cancel that
+ # landed meanwhile must not be finalized into a `completed` result.
+ if await _should_cancel():
+ return None
staged_paths = [str(workflow_download_directory / name) for name in sorted(staged)]
new_files = await self._finalize_downloaded_files_for_task(
task,
diff --git a/skyvern/forge/sdk/api/aws.py b/skyvern/forge/sdk/api/aws.py
index 775d33837..0e4367d11 100644
--- a/skyvern/forge/sdk/api/aws.py
+++ b/skyvern/forge/sdk/api/aws.py
@@ -358,6 +358,27 @@ class _SharedS3Client:
await context.__aexit__(None, None, None)
+class S3ObjectStream:
+ """An open GetObject read in bounded chunks; it holds its shared-client lease until ``aclose``, so retiring that
+ client (a session refresh) cannot close the connection the body is still read from."""
+
+ def __init__(self, response: dict[str, Any], lease: contextlib.AsyncExitStack) -> None:
+ self.content_length: int = response["ContentLength"]
+ self.etag: str | None = response.get("ETag")
+ self._body = response["Body"]
+ self._lease = lease
+
+ def iter_chunks(self, chunk_size: int) -> AsyncIterator[bytes]:
+ return self._body.iter_chunks(chunk_size)
+
+ async def aclose(self) -> None:
+ # An unfinished body drops its connection instead of returning it to the pool with unread bytes.
+ try:
+ self._body.close()
+ finally:
+ await self._lease.aclose()
+
+
def _retire_shared_s3_clients(registry: dict[asyncio.AbstractEventLoop, _SharedS3Client]) -> None:
for loop, shared in list(registry.items()):
with contextlib.suppress(RuntimeError): # the loop is already closed
@@ -1340,22 +1361,47 @@ class AsyncAWSClient:
try:
return await self._s3_with_retry("download", _op, uri=uri)
except Exception as e:
- # A missing object is terminal not-found, not a transient failure (e.g. first-run
- # profile restore before any archive exists). Log once without a traceback so callers
- # fall back immediately instead of surfacing repeated ERROR tracebacks.
- if self._is_not_found_error(e):
- if log_exception:
- LOG.info("S3 object not found", uri=uri)
- return None
- # An archived object stays unreadable until restored, so a traceback per request adds nothing.
- if self._is_archived_object_error(e):
- if log_exception:
- LOG.warning("S3 object is archived and not restored", uri=uri)
- return None
if log_exception:
- LOG.exception("S3 download failed", uri=uri)
+ self.log_download_failure(e, uri)
return None
+ def log_download_failure(self, error: Exception, uri: str) -> None:
+ """Log a failed object read; call it from the ``except`` block so a real failure keeps its traceback."""
+ # A missing object is terminal not-found, not a transient failure (e.g. first-run
+ # profile restore before any archive exists). Log once without a traceback so callers
+ # fall back immediately instead of surfacing repeated ERROR tracebacks.
+ if self._is_not_found_error(error):
+ LOG.info("S3 object not found", uri=uri)
+ # An archived object stays unreadable until restored, so a traceback per request adds nothing.
+ elif self._is_archived_object_error(error):
+ LOG.warning("S3 object is archived and not restored", uri=uri)
+ else:
+ LOG.exception("S3 download failed", uri=uri)
+
+ async def open_download_stream(
+ self, uri: str, *, byte_range: tuple[int, int] | None = None, if_match: str | None = None
+ ) -> S3ObjectStream:
+ """GetObject without reading the body, which the caller must ``aclose``; ``byte_range`` is inclusive, and an
+ ``if_match`` that no longer names the stored object raises ``PreconditionFailed`` like any get_object error."""
+ parsed_uri = S3Uri(uri)
+ params: dict[str, Any] = {"Bucket": parsed_uri.bucket, "Key": parsed_uri.key}
+ if byte_range is not None:
+ params["Range"] = f"bytes={byte_range[0]}-{byte_range[1]}"
+ if if_match is not None:
+ params["IfMatch"] = if_match
+
+ async def _op() -> S3ObjectStream:
+ lease = contextlib.AsyncExitStack()
+ try:
+ client = await lease.enter_async_context(self._s3_client())
+ response = await client.get_object(**params)
+ except BaseException:
+ await lease.aclose()
+ raise
+ return S3ObjectStream(response, lease)
+
+ return await self._s3_with_retry("download", _op, uri=uri)
+
async def delete_file(self, uri: str, log_exception: bool = True, raise_on_error: bool = False) -> None:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/delete_object.html
async def _op() -> None:
diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.py
index bf36845c0..22da8bd0a 100644
--- a/skyvern/forge/sdk/artifact/storage/base.py
+++ b/skyvern/forge/sdk/artifact/storage/base.py
@@ -1,9 +1,10 @@
import os
from abc import ABC, abstractmethod
from collections import Counter
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
+from dataclasses import dataclass
from datetime import UTC, datetime
-from typing import BinaryIO, cast
+from typing import BinaryIO, Protocol, cast
from urllib.parse import unquote
import structlog
@@ -342,6 +343,44 @@ FILE_EXTENTSION_MAP: dict[ArtifactType, str] = {
}
+@dataclass(frozen=True)
+class StoredObjectVersion:
+ size: int
+ # Pins a later ranged read to the version this size was taken from.
+ etag: str | None
+
+
+class StoredObjectStream(Protocol):
+ content_length: int
+
+ def iter_chunks(self, chunk_size: int) -> AsyncIterator[bytes]: ...
+
+ async def aclose(self) -> None: ...
+
+
+class StoredObjectChangedError(Exception):
+ """The stored object was replaced after it was sized for a ranged read."""
+
+
+class StreamingArtifactStorage(ABC):
+ """Storage that can serve an artifact's stored bytes in bounded chunks instead of one whole read."""
+
+ @abstractmethod
+ def can_stream_artifact(self, artifact: Artifact) -> bool:
+ """Whether the artifact's content is its stored bytes unchanged, with no decoding after the read."""
+
+ @abstractmethod
+ async def stat_artifact(self, artifact: Artifact) -> StoredObjectVersion | None:
+ """The stored object's size and version, or None when it cannot be read."""
+
+ @abstractmethod
+ async def open_artifact_stream(
+ self, artifact: Artifact, *, byte_range: tuple[int, int] | None = None, etag: str | None = None
+ ) -> StoredObjectStream | None:
+ """Open the stored bytes or their inclusive ``byte_range``, or return None when they cannot be read; with
+ ``etag``, raises ``StoredObjectChangedError`` once the stored object is no longer that version."""
+
+
class BaseStorage(ABC):
@abstractmethod
def build_uri(self, *, organization_id: str, artifact_id: str, step: Step, artifact_type: ArtifactType) -> str:
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
index 976580ad2..6e7d3a8d6 100644
--- a/skyvern/forge/sdk/artifact/storage/s3.py
+++ b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -15,7 +15,7 @@ from skyvern.config import settings
from skyvern.constants import BROWSER_DOWNLOADING_SUFFIX, DOWNLOAD_FILE_PREFIX
from skyvern.exceptions import DownloadSaveIncompleteError
from skyvern.forge import app
-from skyvern.forge.sdk.api.aws import AsyncAWSClient, S3StorageClass, S3Uri, _recording_content_type
+from skyvern.forge.sdk.api.aws import AsyncAWSClient, S3ObjectStream, S3StorageClass, S3Uri, _recording_content_type
from skyvern.forge.sdk.api.files import (
calculate_sha256_for_file,
create_named_temporary_file,
@@ -30,6 +30,9 @@ from skyvern.forge.sdk.artifact.signing import SENSITIVE_ARTIFACT_URL_EXPIRY_SEC
from skyvern.forge.sdk.artifact.storage.base import (
FILE_EXTENTSION_MAP,
BaseStorage,
+ StoredObjectChangedError,
+ StoredObjectVersion,
+ StreamingArtifactStorage,
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
dedupe_run_scoped_download_artifacts,
@@ -97,7 +100,7 @@ def empty_read_decision(
}
-class S3Storage(BaseStorage):
+class S3Storage(BaseStorage, StreamingArtifactStorage):
_PATH_VERSION = "v1"
# Caps concurrent head_object calls from S3 listing reads. /browser_sessions/history fans out over up to 100
# sessions at once, so the semaphore lives on the instance and every listing call on it shares one cap.
@@ -236,6 +239,29 @@ class S3Storage(BaseStorage):
) -> S3StorageClass:
return S3StorageClass.STANDARD
+ def can_stream_artifact(self, artifact: Artifact) -> bool:
+ # retrieve_artifact decompresses a zstd HAR and extracts a bundle member from its archive after the read.
+ return not artifact.bundle_key and not artifact.uri.endswith(S3_ZSTD_COMPRESSED_SUFFIX)
+
+ async def stat_artifact(self, artifact: Artifact) -> StoredObjectVersion | None:
+ try:
+ info = await self.async_client.get_object_info(artifact.uri)
+ except Exception as e:
+ self.async_client.log_download_failure(e, artifact.uri)
+ return None
+ return StoredObjectVersion(size=info["ContentLength"], etag=info.get("ETag"))
+
+ async def open_artifact_stream(
+ self, artifact: Artifact, *, byte_range: tuple[int, int] | None = None, etag: str | None = None
+ ) -> S3ObjectStream | None:
+ try:
+ return await self.async_client.open_download_stream(artifact.uri, byte_range=byte_range, if_match=etag)
+ except Exception as e:
+ if etag is not None and self.async_client._error_code(e) == "PreconditionFailed":
+ raise StoredObjectChangedError(artifact.uri) from e
+ self.async_client.log_download_failure(e, artifact.uri)
+ return None
+
async def retrieve_artifact(self, artifact: Artifact) -> bytes | None:
data = await self.async_client.download_file(artifact.uri)
# Decompress zstd-compressed files (HAR only)
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index d437af17d..0a325ff0a 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -34,6 +34,7 @@ from fastapi.responses import ORJSONResponse, StreamingResponse
from opentelemetry import trace
from pydantic import TypeAdapter, ValidationError
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from starlette.types import Receive, Scope, Send
from skyvern import analytics
from skyvern._version import __version__
@@ -66,7 +67,12 @@ from skyvern.forge.sdk.artifact.signing import (
parse_keyring,
verify_artifact_signature,
)
-from skyvern.forge.sdk.artifact.storage.base import artifact_filename_from_uri
+from skyvern.forge.sdk.artifact.storage.base import (
+ StoredObjectChangedError,
+ StoredObjectStream,
+ StreamingArtifactStorage,
+ artifact_filename_from_uri,
+)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.curl_converter import curl_to_http_request_block_params
from skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory
@@ -3433,6 +3439,67 @@ def _artifact_content_response_headers(
}
+# The most one storage read returns, so a response holds about this much however large the object is.
+_ARTIFACT_STREAM_CHUNK_BYTES = 1024 * 1024
+
+
+class _StoredArtifactResponse(StreamingResponse):
+ """Streams a stored artifact and closes its storage read however the response ends, disconnects included."""
+
+ def __init__(self, stream: StoredObjectStream, **kwargs: Any) -> None:
+ super().__init__(stream.iter_chunks(_ARTIFACT_STREAM_CHUNK_BYTES), **kwargs)
+ self._stream = stream
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ try:
+ await super().__call__(scope, receive, send)
+ finally:
+ await self._stream.aclose()
+
+
+async def _stream_artifact_content(
+ storage: StreamingArtifactStorage,
+ artifact: Artifact,
+ range_header: str | None,
+ media_type: str,
+ headers: dict[str, str],
+) -> Response:
+ # A Range header resolves against the stored size exactly as it does against whole bytes, and the read is
+ # pinned to the version that was sized; an object replaced in between is sized again.
+ for _ in range(2):
+ byte_range: tuple[int, int] | None = None
+ etag: str | None = None
+ content_range: str | None = None
+ if range_header:
+ version = await storage.stat_artifact(artifact)
+ if version is None:
+ break
+ parsed_range = _parse_range_header(range_header, version.size)
+ if parsed_range == _RANGE_UNSATISFIABLE:
+ headers["Content-Range"] = f"bytes */{version.size}"
+ return Response(status_code=http_status.HTTP_416_RANGE_NOT_SATISFIABLE, headers=headers)
+ if parsed_range is not None:
+ byte_range, etag = parsed_range, version.etag
+ content_range = f"bytes {parsed_range[0]}-{parsed_range[1]}/{version.size}"
+ try:
+ stream = await storage.open_artifact_stream(artifact, byte_range=byte_range, etag=etag)
+ except StoredObjectChangedError:
+ continue
+ if stream is None:
+ break
+ headers["Content-Length"] = str(stream.content_length)
+ if content_range is None:
+ return _StoredArtifactResponse(stream, media_type=media_type, headers=headers)
+ headers["Content-Range"] = content_range
+ return _StoredArtifactResponse(
+ stream, status_code=http_status.HTTP_206_PARTIAL_CONTENT, media_type=media_type, headers=headers
+ )
+ raise HTTPException(
+ status_code=http_status.HTTP_404_NOT_FOUND,
+ detail="Artifact content not available",
+ )
+
+
@base_router.get(
"/artifacts/{artifact_id}/content",
tags=["Artifacts"],
@@ -3505,12 +3572,6 @@ async def get_artifact_content(
status_code=http_status.HTTP_404_NOT_FOUND,
detail=f"Artifact not found {artifact_id}",
)
- content = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)
- if content is None:
- raise HTTPException(
- status_code=http_status.HTTP_404_NOT_FOUND,
- detail="Artifact content not available",
- )
media_type, content_disposition = _artifact_response_config(artifact)
is_signed = sig is not None and expiry is not None and kid is not None
signed_expiry_unix: int | None = None
@@ -3527,6 +3588,15 @@ async def get_artifact_content(
headers["Accept-Ranges"] = "bytes"
if artifact.artifact_type in _HTML_ARTIFACT_TYPES:
headers["Content-Security-Policy"] = _HTML_ARTIFACT_CSP
+ storage = app.STORAGE
+ if isinstance(storage, StreamingArtifactStorage) and storage.can_stream_artifact(artifact):
+ return await _stream_artifact_content(storage, artifact, request.headers.get("range"), media_type, headers)
+ content = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)
+ if content is None:
+ raise HTTPException(
+ status_code=http_status.HTTP_404_NOT_FOUND,
+ detail="Artifact content not available",
+ )
content_length = len(content)
parsed_range = _parse_range_header(request.headers.get("range"), content_length)
if parsed_range == _RANGE_UNSATISFIABLE:
diff --git a/skyvern/forge/taskv3/provider_downloads.py b/skyvern/forge/taskv3/provider_downloads.py
new file mode 100644
index 000000000..0484b8eed
--- /dev/null
+++ b/skyvern/forge/taskv3/provider_downloads.py
@@ -0,0 +1,194 @@
+"""Read a vendor-kept browser download back into a v3 complete_on_download block's run download directory.
+
+A vendor session can keep the browser's own save while the out-of-band replay of the URL fails, so the
+completion probe reads the provider copy and lets the existing finalize path judge what lands."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Awaitable, Callable, Collection
+from pathlib import Path
+from typing import Any, Protocol
+
+import structlog
+
+from skyvern.constants import BROWSER_DOWNLOADING_SUFFIX
+from skyvern.webeye.browser_artifacts import ActionDownloadObservation, ActionDownloadSource
+
+LOG = structlog.get_logger()
+
+PROVIDER_BASELINE_TIMEOUT_SECONDS = 10.0
+# The provider lists a file only once the browser finished saving it; a block's download_timeout, when
+# set, replaces this per-download wait.
+PROVIDER_DOWNLOAD_WAIT_SECONDS = 30.0
+POLL_INTERVAL_SECONDS = 1.0
+# Lets a downloadWillBegin raised by the action that just returned reach the monitor's counter.
+EVENT_ADMISSION_GRACE_SECONDS = 0.25
+
+
+class _DownloadMonitor(Protocol):
+ @property
+ def browser_download_event_count(self) -> int: ...
+
+ def has_pending_download_activity(self) -> bool: ...
+
+
+def _download_monitor(browser_context: Any) -> _DownloadMonitor | None:
+ # Attached dynamically to the Playwright context by the browser factory.
+ monitor = getattr(browser_context, "_skyvern_cdp_download_interceptor", None) if browser_context else None
+ count = getattr(monitor, "browser_download_event_count", None)
+ if not isinstance(count, int) or not callable(getattr(monitor, "has_pending_download_activity", None)):
+ return None
+ return monitor
+
+
+def _completed_names(directory: Path) -> set[str]:
+ try:
+ entries = list(directory.iterdir())
+ except OSError:
+ return set()
+ return {entry.name for entry in entries if entry.is_file() and not entry.name.endswith(BROWSER_DOWNLOADING_SUFFIX)}
+
+
+class ProviderDownloadRecovery:
+ """One v3 block's provider read-back. The download probe awaits it serially, never concurrently."""
+
+ def __init__(
+ self,
+ observation: ActionDownloadObservation,
+ download_dir: Path,
+ monitor: _DownloadMonitor | None,
+ ) -> None:
+ self._observation = observation
+ self._download_dir = download_dir
+ self._monitor = monitor
+ self._events_seen = monitor.browser_download_event_count if monitor is not None else 0
+ self._local_seen = _completed_names(download_dir)
+ self._owed_read = False
+
+ @classmethod
+ async def begin(
+ cls,
+ source: ActionDownloadSource | None,
+ *,
+ download_dir: Path,
+ browser_context: Any,
+ deadline: float,
+ ) -> ProviderDownloadRecovery | None:
+ """Freeze the provider baseline before the block acts, or ``None`` to leave every download path as is."""
+ if source is None:
+ return None
+ try:
+ observation = await source.begin_observation(deadline=deadline)
+ except Exception as exc:
+ # A provider error can embed a secret-bearing presigned URL, so only its type is logged.
+ LOG.warning("Task V3 provider download baseline unavailable", error_type=type(exc).__name__)
+ return None
+ if observation is None:
+ return None
+ return cls(observation, download_dir, _download_monitor(browser_context))
+
+ async def recover(
+ self,
+ *,
+ staged: Collection[str],
+ timeout: float,
+ should_cancel: Callable[[], Awaitable[bool]],
+ ) -> bool:
+ """Read back files a download the browser began left with the provider; True only when ``should_cancel``
+ fired. An admitted download leaves a read owed until a file lands, and completion stays any-file."""
+ if timeout <= 0:
+ return False
+ deadline = time.monotonic() + timeout
+ try:
+ if self._local_delta(staged):
+ self._settle_read()
+ return False
+ if self._monitor is None:
+ # Nothing reports whether a download began, so read the provider once per probe.
+ if await self._poll(deadline, should_cancel):
+ return True
+ self._log_read_back(staged)
+ return False
+ await asyncio.sleep(min(EVENT_ADMISSION_GRACE_SECONDS, max(0.0, deadline - time.monotonic())))
+ new_event = self._monitor.browser_download_event_count != self._events_seen
+ if new_event:
+ self._events_seen = self._monitor.browser_download_event_count
+ self._owed_read = True
+ if not self._owed_read:
+ return False
+ # The monitor's own capture (a replay that works) runs first; reading the provider while it
+ # is still in flight could publish the same download twice.
+ while self._monitor.has_pending_download_activity():
+ if not new_event or time.monotonic() >= deadline:
+ return False
+ if await should_cancel():
+ return True
+ await asyncio.sleep(min(POLL_INTERVAL_SECONDS, max(0.0, deadline - time.monotonic())))
+ while True:
+ if self._local_delta(staged):
+ self._settle_read()
+ return False
+ if time.monotonic() >= deadline:
+ return False
+ if await self._poll(deadline, should_cancel):
+ return True
+ if self._log_read_back(staged):
+ self._settle_read()
+ return False
+ if not new_event or time.monotonic() >= deadline:
+ return False
+ if await should_cancel():
+ return True
+ await asyncio.sleep(min(POLL_INTERVAL_SECONDS, max(0.0, deadline - time.monotonic())))
+ finally:
+ self._local_seen = _completed_names(self._download_dir)
+
+ def _settle_read(self) -> None:
+ # A file landed locally, so every event admitted so far is accounted for; a later probe must not
+ # read one of them as new and publish the provider's copy of the same download beside it.
+ self._owed_read = False
+ if self._monitor is not None:
+ self._events_seen = self._monitor.browser_download_event_count
+
+ def _log_read_back(self, staged: Collection[str]) -> bool:
+ landed = self._local_delta(staged)
+ if landed:
+ LOG.info("Task V3 provider download read back", file_count=len(landed))
+ return bool(landed)
+
+ def _local_delta(self, staged: Collection[str]) -> set[str]:
+ return _completed_names(self._download_dir) - self._local_seen - set(staged)
+
+ async def _poll(self, deadline: float, should_cancel: Callable[[], Awaitable[bool]]) -> bool:
+ poll = asyncio.ensure_future(
+ self._observation.poll_and_materialize(destination_dir=self._download_dir, deadline=deadline)
+ )
+ try:
+ while True:
+ remaining = deadline - time.monotonic()
+ done, _ = await asyncio.wait({poll}, timeout=max(0.0, min(POLL_INTERVAL_SECONDS, remaining)))
+ if poll in done or remaining <= 0:
+ return False
+ if await should_cancel():
+ return True
+ finally:
+ await _join(poll)
+
+
+async def _join(poll: asyncio.Future[None]) -> None:
+ """Cancel ``poll`` and wait for its own (cooperative, not deadline-bound) cleanup before returning.
+ A cancellation of the caller while waiting is re-raised once the poll is done."""
+ poll.cancel()
+ cancellation: asyncio.CancelledError | None = None
+ while not poll.done():
+ try:
+ await asyncio.wait({poll})
+ except asyncio.CancelledError as caught:
+ cancellation = caught
+ if not poll.cancelled() and (exc := poll.exception()) is not None:
+ # A provider error can embed a secret-bearing presigned URL, so only its type is logged.
+ LOG.warning("Task V3 provider download poll failed", error_type=type(exc).__name__)
+ if cancellation is not None:
+ raise cancellation
diff --git a/tests/unit/_download_provider_fakes.py b/tests/unit/_download_provider_fakes.py
new file mode 100644
index 000000000..d7311e7d6
--- /dev/null
+++ b/tests/unit/_download_provider_fakes.py
@@ -0,0 +1,144 @@
+"""Provider-neutral fakes for an action-scoped remote download source (``ActionDownloadSource``)."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from pydantic import BaseModel, ValidationError
+
+
+class _FakeActionDownloadObservation:
+ """Neutral provider observation: drops completed remote files into the run dir on its first poll,
+ honoring the ``deadline`` (monotonic) contract strictly (a handler passing per-call ``timeout_seconds``
+ materializes nothing -- the anti-terminal-cleanup RED)."""
+
+ def __init__(self, files: list[tuple[str, bytes]], *, empty_polls: int = 0) -> None:
+ self._pending = list(files)
+ self._empty_polls = empty_polls
+ self.poll_deadlines: list[float] = []
+ self.poll_started_at: list[float] = []
+
+ async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:
+ self.poll_deadlines.append(deadline)
+ self.poll_started_at.append(time.monotonic())
+ if len(self.poll_deadlines) <= self._empty_polls:
+ return
+ destination_dir.mkdir(parents=True, exist_ok=True)
+ for name, data in self._pending:
+ (destination_dir / name).write_bytes(data)
+ self._pending = []
+
+
+class _RaisingActionDownloadObservation:
+ """Provider observation whose poll raises a schema/validation error embedding a presigned URL, mirroring
+ the real leak surface (the vendor list body is parsed into typed rows). When ``materialize_first`` is
+ set, the first poll drops a completed file (so the handler reaches its final poll) and a later poll raises."""
+
+ def __init__(self, error: BaseException, *, materialize_first: tuple[str, bytes] | None = None) -> None:
+ self._error = error
+ self._materialize_first = materialize_first
+ self.calls = 0
+
+ async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:
+ self.calls += 1
+ if self._materialize_first is not None and self.calls == 1:
+ destination_dir.mkdir(parents=True, exist_ok=True)
+ name, data = self._materialize_first
+ (destination_dir / name).write_bytes(data)
+ return
+ raise self._error
+
+
+class _StallingActionDownloadObservation:
+ """Provider observation whose poll never returns on its own -- only the handler's deadline can end it, so
+ if the handler polls it once a local artifact already accounts for the action the stalled provider would
+ consume the whole download deadline; the fix must never poll it in that case."""
+
+ def __init__(self) -> None:
+ self.poll_calls = 0
+ self.poll_deadlines: list[float] = []
+
+ async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:
+ self.poll_calls += 1
+ self.poll_deadlines.append(deadline)
+ await asyncio.Event().wait()
+
+
+class _ActionDownloadSource:
+ """Neutral provider source: hands back a fixed observation and records begin bookkeeping."""
+
+ def __init__(
+ self,
+ observation: object,
+ *,
+ begin_error: Exception | None = None,
+ on_begin: Callable[[], object] | None = None,
+ ) -> None:
+ self._observation = observation
+ self._begin_error = begin_error
+ self._on_begin = on_begin
+ self.begin_calls = 0
+ self.begin_deadline: float | None = None
+
+ async def begin_observation(self, *, deadline: float) -> object:
+ self.begin_calls += 1
+ self.begin_deadline = deadline
+ if self._on_begin is not None:
+ self._on_begin()
+ if self._begin_error is not None:
+ raise self._begin_error
+ return self._observation
+
+
+def _secret_bearing_validation_error(presigned_url: str) -> ValidationError:
+ class _Row(BaseModel):
+ size: int
+
+ try:
+ _Row.model_validate({"size": presigned_url})
+ except ValidationError as exc:
+ return exc
+ raise AssertionError("expected a ValidationError")
+
+
+class _FakeBrowserDownloadMonitor:
+ """Stand-in for the run's CDP download monitor: an admitted-event counter and a pending flag that
+ reads True for ``pending_checks`` calls, then runs ``on_settle`` once (e.g. a replay landing a file)."""
+
+ def __init__(self, *, pending_checks: int = 0, on_settle: object = None) -> None:
+ self.browser_download_event_count = 0
+ self._pending_checks = pending_checks
+ self._on_settle = on_settle
+ self.pending_calls = 0
+
+ def has_pending_download_activity(self) -> bool:
+ self.pending_calls += 1
+ if self.pending_calls <= self._pending_checks:
+ return True
+ if self._on_settle is not None:
+ settle, self._on_settle = self._on_settle, None
+ settle() # type: ignore[operator]
+ return False
+
+
+class _GatedCleanupObservation:
+ """Provider observation whose poll never lists anything and whose cancellation cleanup waits on ``gate``,
+ so a caller that returns before awaiting the cancelled poll is seen returning with cleanup unfinished."""
+
+ def __init__(self, gate: asyncio.Event) -> None:
+ self._gate = gate
+ self.started = asyncio.Event()
+ self.cleanup_started = asyncio.Event()
+ self.cleanup_finished = False
+
+ async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:
+ self.started.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ self.cleanup_started.set()
+ await self._gate.wait()
+ self.cleanup_finished = True
diff --git a/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py b/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py
index a273078b3..60768ae7b 100644
--- a/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py
+++ b/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py
@@ -59,7 +59,8 @@ def mock_browser_session_artifact_create(monkeypatch: pytest.MonkeyPatch) -> Non
@pytest.fixture(scope="module")
def moto_server() -> Generator[str]:
- server = ThreadedMotoServer(port=0)
+ # Loopback, not 0.0.0.0: a macOS system proxy bypasses 127.0.0.1 but would otherwise relay the S3 traffic.
+ server = ThreadedMotoServer(ip_address="127.0.0.1", port=0)
server.start()
host, port = server.get_host_and_port()
yield f"http://{host}:{port}"
diff --git a/tests/unit/forge/sdk/test_artifact_content_range.py b/tests/unit/forge/sdk/test_artifact_content_range.py
index 1e3ad924a..25e05178e 100644
--- a/tests/unit/forge/sdk/test_artifact_content_range.py
+++ b/tests/unit/forge/sdk/test_artifact_content_range.py
@@ -1,16 +1,31 @@
from __future__ import annotations
+import asyncio
+import contextlib
+import io
+import uuid
+import zipfile
+from datetime import datetime
from types import SimpleNamespace
+from typing import Any
from unittest.mock import AsyncMock, MagicMock
+import boto3
import pytest
+import zstandard as zstd
+from aiobotocore.response import StreamingBody
from fastapi import FastAPI
from fastapi.testclient import TestClient
+from skyvern.config import settings
from skyvern.forge import app as forge_app
-from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.artifact.manager import ArtifactManager
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.artifact.storage.s3 import S3Storage
+from skyvern.forge.sdk.routes import agent_protocol
from skyvern.forge.sdk.routes import routers as routers_module
from skyvern.forge.sdk.routes.agent_protocol import _parse_range_header
+from tests.unit.forge.sdk.artifact.storage.test_s3_storage_moto import moto_server # noqa: F401
CONTENT = b"0123456789" * 100 # 1000 bytes for predictable slicing
@@ -168,3 +183,260 @@ class TestArtifactContentRangeRequests:
assert response.status_code == 200
assert response.content == CONTENT
assert response.headers["accept-ranges"] == "bytes"
+
+
+STREAM_BUCKET = "artifact-content-route"
+
+
+def stream_bucket(moto_server: str, no_env_proxy: None) -> Any: # noqa: F811
+ client = boto3.client(
+ "s3",
+ aws_access_key_id="testing",
+ aws_secret_access_key="testing",
+ region_name=settings.AWS_REGION,
+ endpoint_url=moto_server,
+ )
+ with contextlib.suppress(client.exceptions.BucketAlreadyOwnedByYou):
+ client.create_bucket(Bucket=STREAM_BUCKET)
+ return client
+
+
+def s3_backed_storage(
+ monkeypatch: pytest.MonkeyPatch,
+ moto_server: str, # noqa: F811
+ stream_bucket: Any,
+) -> S3Storage:
+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing")
+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing")
+ monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False)
+ monkeypatch.delenv("AWS_PROFILE", raising=False)
+ storage = S3Storage(bucket=STREAM_BUCKET, endpoint_url=moto_server)
+ monkeypatch.setattr(forge_app, "STORAGE", storage)
+ monkeypatch.setattr(forge_app, "ARTIFACT_MANAGER", ArtifactManager())
+ return storage
+
+
+def storage_bodies(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
+ """Every read of an S3 response body as (requested amount, bytes returned), and how many bodies were closed."""
+ observed = SimpleNamespace(reads=[], closes=0)
+ real_read = StreamingBody.read
+
+ async def read(self: StreamingBody, amt: int | None = None) -> bytes:
+ chunk = await real_read(self, amt)
+ observed.reads.append((amt, len(chunk)))
+ return chunk
+
+ def close(self: StreamingBody) -> None:
+ observed.closes += 1
+ self.__wrapped__.close()
+
+ monkeypatch.setattr(StreamingBody, "read", read)
+ monkeypatch.setattr(StreamingBody, "close", close, raising=False)
+ return observed
+
+
+def _store_artifact(
+ s3: Any,
+ data: bytes,
+ *,
+ artifact_type: ArtifactType = ArtifactType.DOWNLOAD,
+ filename: str = "report.zip",
+ bundle_key: str | None = None,
+) -> Artifact:
+ key = f"objects/{uuid.uuid4().hex}/{filename}"
+ s3.put_object(Bucket=STREAM_BUCKET, Key=key, Body=data)
+ now = datetime.utcnow()
+ return Artifact(
+ artifact_id="a_stored",
+ artifact_type=artifact_type,
+ uri=f"s3://{STREAM_BUCKET}/{key}",
+ bundle_key=bundle_key,
+ organization_id="org_oss",
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def _artifact_route_app(monkeypatch: pytest.MonkeyPatch, artifact: Artifact) -> FastAPI:
+ async def _fake_org(**_: object) -> SimpleNamespace:
+ return SimpleNamespace(organization_id="org_oss")
+
+ artifacts_db = MagicMock()
+ artifacts_db.get_artifact_by_id = AsyncMock(return_value=artifact)
+ monkeypatch.setattr(forge_app.DATABASE, "artifacts", artifacts_db)
+ monkeypatch.setattr(agent_protocol.org_auth_service, "get_current_org", _fake_org)
+ fastapi_app = FastAPI()
+ fastapi_app.include_router(routers_module.base_router, prefix="/v1")
+ return fastapi_app
+
+
+def _asgi_get_scope(path: str) -> dict[str, Any]:
+ return {
+ "type": "http",
+ "asgi": {"version": "3.0", "spec_version": "2.3"},
+ "http_version": "1.1",
+ "method": "GET",
+ "scheme": "http",
+ "path": path,
+ "raw_path": path.encode(),
+ "root_path": "",
+ "query_string": b"",
+ "headers": [(b"host", b"testserver"), (b"x-api-key", b"synthetic")],
+ "client": ("127.0.0.1", 50000),
+ "server": ("testserver", 80),
+ }
+
+
+class TestArtifactContentFromS3:
+ @pytest.mark.parametrize(
+ ("range_header", "status", "body", "content_range"),
+ [
+ pytest.param(None, 200, CONTENT, None, id="full"),
+ pytest.param("bytes=10-19", 206, CONTENT[10:20], "bytes 10-19/1000", id="range"),
+ pytest.param("bytes=900-", 206, CONTENT[900:], "bytes 900-999/1000", id="open-ended"),
+ pytest.param("bytes=-50", 206, CONTENT[-50:], "bytes 950-999/1000", id="suffix"),
+ pytest.param("bytes=990-5000", 206, CONTENT[990:], "bytes 990-999/1000", id="end-clamped"),
+ pytest.param("bytes=5000-6000", 416, b"", "bytes */1000", id="unsatisfiable"),
+ pytest.param("bytes=garbage", 200, CONTENT, None, id="ignored"),
+ ],
+ )
+ def test_plain_object_is_read_in_bounded_chunks_with_unchanged_range_semantics(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ s3_backed_storage: S3Storage,
+ stream_bucket: Any,
+ storage_bodies: SimpleNamespace,
+ range_header: str | None,
+ status: int,
+ body: bytes,
+ content_range: str | None,
+ ) -> None:
+ artifact = _store_artifact(stream_bucket, CONTENT)
+ with TestClient(_artifact_route_app(monkeypatch, artifact)) as client:
+ response = client.get(
+ "/v1/artifacts/a_stored/content", headers={"Range": range_header} if range_header else {}
+ )
+
+ assert response.status_code == status
+ assert response.content == body
+ assert response.headers.get("content-range") == content_range
+ assert response.headers["content-length"] == str(len(body))
+ assert response.headers["accept-ranges"] == "bytes"
+ assert response.headers["content-disposition"].startswith("attachment;")
+ # GiB-sized downloads killed API tasks: no read may be unbounded, and a slice reads only its bytes.
+ assert all(amount is not None for amount, _ in storage_bodies.reads)
+ assert sum(size for _, size in storage_bodies.reads) == len(body)
+
+ @pytest.mark.parametrize("range_header", [None, "bytes=5-14"])
+ @pytest.mark.parametrize("kind", ["bundle-member", "compressed-har"])
+ def test_decoded_artifacts_keep_their_whole_object_semantics(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ s3_backed_storage: S3Storage,
+ stream_bucket: Any,
+ kind: str,
+ range_header: str | None,
+ ) -> None:
+ if kind == "bundle-member":
+ served = b"<html><body>bundled member page</body></html>"
+ archive = io.BytesIO()
+ with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf:
+ zf.writestr("scrape.html", served)
+ zf.writestr("other.txt", b"another member")
+ artifact = _store_artifact(
+ stream_bucket,
+ archive.getvalue(),
+ artifact_type=ArtifactType.HTML_SCRAPE,
+ filename="step_archive.zip",
+ bundle_key="scrape.html",
+ )
+ else:
+ served = b'{"log": {"version": "1.2", "entries": []}}' * 20
+ artifact = _store_artifact(
+ stream_bucket,
+ zstd.ZstdCompressor(level=3).compress(served),
+ artifact_type=ArtifactType.HAR,
+ filename="network.har.zst",
+ )
+
+ with TestClient(_artifact_route_app(monkeypatch, artifact)) as client:
+ response = client.get(
+ "/v1/artifacts/a_stored/content", headers={"Range": range_header} if range_header else {}
+ )
+
+ if range_header is None:
+ assert response.status_code == 200
+ assert response.content == served
+ else:
+ assert response.status_code == 206
+ assert response.content == served[5:15]
+ assert response.headers["content-range"] == f"bytes 5-14/{len(served)}"
+
+ def test_range_read_of_an_object_replaced_after_sizing_serves_one_version(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ s3_backed_storage: S3Storage,
+ stream_bucket: Any,
+ ) -> None:
+ artifact = _store_artifact(stream_bucket, CONTENT)
+ replacement = b"abcdefghij" * 300
+ sized_versions: list[Any] = []
+ stat_artifact = s3_backed_storage.stat_artifact
+
+ async def stat_then_replace(stored: Artifact) -> Any:
+ version = await stat_artifact(stored)
+ if not sized_versions:
+ key = stored.uri.removeprefix(f"s3://{STREAM_BUCKET}/")
+ stream_bucket.put_object(Bucket=STREAM_BUCKET, Key=key, Body=replacement)
+ sized_versions.append(version)
+ return version
+
+ monkeypatch.setattr(s3_backed_storage, "stat_artifact", stat_then_replace)
+ with TestClient(_artifact_route_app(monkeypatch, artifact)) as client:
+ response = client.get("/v1/artifacts/a_stored/content", headers={"Range": "bytes=0-9"})
+
+ assert response.status_code == 206
+ assert response.content == replacement[:10]
+ assert response.headers["content-range"] == f"bytes 0-9/{len(replacement)}"
+ assert [version.size for version in sized_versions] == [len(CONTENT), len(replacement)]
+
+ @pytest.mark.asyncio
+ async def test_client_disconnect_stops_reading_and_closes_the_storage_body(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ s3_backed_storage: S3Storage,
+ stream_bucket: Any,
+ storage_bodies: SimpleNamespace,
+ ) -> None:
+ large = bytes(range(256)) * 32 * 1024 # 8 MiB
+ fastapi_app = _artifact_route_app(monkeypatch, _store_artifact(stream_bucket, large))
+ first_body_sent = asyncio.Event()
+ sent: list[dict[str, Any]] = []
+ request_delivered = False
+
+ async def receive() -> dict[str, Any]:
+ nonlocal request_delivered
+ if not request_delivered:
+ request_delivered = True
+ return {"type": "http.request", "body": b"", "more_body": False}
+ await first_body_sent.wait()
+ return {"type": "http.disconnect"}
+
+ async def send(message: dict[str, Any]) -> None:
+ sent.append(message)
+ if message["type"] == "http.response.body" and message.get("body"):
+ first_body_sent.set()
+ # A served send is an await point (BaseHTTPMiddleware passes each message through a memory stream).
+ # Without one, anyio skips delivering the disconnect's cancellation while every local S3 read is complete.
+ await asyncio.sleep(0)
+
+ await fastapi_app(_asgi_get_scope("/v1/artifacts/a_stored/content"), receive, send)
+
+ assert sent[0]["status"] == 200
+ assert 0 < sum(len(message.get("body", b"")) for message in sent[1:]) < len(large)
+ assert sum(size for _, size in storage_bodies.reads) < len(large)
+ assert storage_bodies.closes == 1
+ assert all(shared.leases == 0 for shared in s3_backed_storage.async_client._shared_s3_clients.values())
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index da0e3aea4..c82f61e1e 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import asyncio
import json
+import time
from collections.abc import Awaitable, Callable, Sequence
from datetime import UTC, datetime
from pathlib import Path
@@ -65,6 +66,7 @@ from skyvern.forge.sdk.workflow.page_derived_templates import CLOSE as PAGE_DERI
from skyvern.forge.sdk.workflow.page_derived_templates import OPEN as PAGE_DERIVED_OPEN
from skyvern.forge.taskv3 import engine as taskv3_engine
from skyvern.forge.taskv3 import input_dispatch
+from skyvern.forge.taskv3 import provider_downloads as taskv3_provider_downloads
from skyvern.forge.taskv3 import tools as taskv3_tools
from skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState
from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop
@@ -103,6 +105,14 @@ from skyvern.webeye.actions.actions import (
SolveCaptchaAction,
UploadFileAction,
)
+from tests.unit._download_provider_fakes import (
+ _ActionDownloadSource,
+ _FakeActionDownloadObservation,
+ _FakeBrowserDownloadMonitor,
+ _GatedCleanupObservation,
+ _secret_bearing_validation_error,
+ _StallingActionDownloadObservation,
+)
from tests.unit.helpers import make_action_row, make_browser_state, make_organization, make_step, make_task
from tests.unit.scoped_asyncio import ScopedAsyncio
from tests.unit.test_taskv3_engine import _fixed_read_tool, _ReaskAnsweringCaller
@@ -2291,6 +2301,10 @@ async def _run_execute_task_v3_download(
loop_fn: Callable[[dict[str, Any], dict[str, Any]], Awaitable[LoopOutcome]] | None = None,
complete_on_download: bool = True,
browser_session_id: str | None = None,
+ provider_source: object | None = None,
+ download_monitor: object | None = None,
+ download_timeout: float | None = None,
+ cancel_after_checks: int | None = None,
) -> dict[str, Any]:
"""Drive _execute_task_v3 for a complete_on_download block, capturing the loop kwargs.
@@ -2321,9 +2335,30 @@ async def _run_execute_task_v3_download(
browser_state.get_working_page = AsyncMock(return_value=page)
browser_state.take_post_action_screenshot = AsyncMock(return_value=b"png-bytes")
- block = _make_block(block_cls, complete_on_download=complete_on_download, download_suffix=download_suffix)
+ # A real session without a provider-owned download store attaches no source; the bare MagicMock
+ # default would otherwise look like one.
+ browser_state.browser_artifacts.get_action_download_source = MagicMock(return_value=provider_source)
+ browser_state.browser_context._skyvern_cdp_download_interceptor = download_monitor
+
+ block_overrides: dict[str, Any] = {"complete_on_download": complete_on_download, "download_suffix": download_suffix}
+ if download_timeout is not None:
+ block_overrides["download_timeout"] = download_timeout
+ block = _make_block(block_cls, **block_overrides)
+
+ cancel_checks = {"count": 0}
- captured: dict[str, Any] = {}
+ async def _get_task(**_kwargs: Any) -> Any:
+ cancel_checks["count"] += 1
+ canceled = cancel_after_checks is not None and cancel_checks["count"] > cancel_after_checks
+ return SimpleNamespace(status=TaskStatus.canceled if canceled else TaskStatus.running)
+
+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task", AsyncMock(side_effect=_get_task))
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.DATABASE.workflow_runs.get_workflow_run",
+ AsyncMock(return_value=SimpleNamespace(status=WorkflowRunStatus.running)),
+ )
+
+ captured: dict[str, Any] = {"cancel_checks": cancel_checks}
async def _default_loop_body(**kwargs: Any) -> LoopOutcome:
if drop_file:
@@ -6776,3 +6811,545 @@ async def test_goal_judge_prompt_never_carries_a_registered_runtime_secret(monke
for _ in range(3):
assert redact("verification_code: 482913") == f"verification_code: {REDACTED_SECRET_PLACEHOLDER}"
assert secret_set_builds == 1
+
+
+# ---------------------------------------------------------------------------
+# Provider-owned downloads: a vendor session can keep the browser's own save while the run's
+# out-of-band replay of the URL fails. The v3 download probe reads that file back into the run
+# download dir so the existing finalize/complete path sees it.
+# ---------------------------------------------------------------------------
+
+
+def _click_then_probe(
+ tmp_path: Path,
+ monitor: _FakeBrowserDownloadMonitor | None,
+ source: _ActionDownloadSource | None = None,
+ *,
+ local_file: str | None = None,
+ raise_event: bool = True,
+) -> Callable[[dict[str, Any], dict[str, Any]], Awaitable[LoopOutcome]]:
+ async def _loop(kwargs: dict[str, Any], captured: dict[str, Any]) -> LoopOutcome:
+ captured["begin_calls_at_loop_start"] = source.begin_calls if source is not None else None
+ if monitor is not None and raise_event:
+ monitor.browser_download_event_count += 1
+ if local_file is not None:
+ (tmp_path / local_file).write_bytes(b"local-bytes")
+ probe = kwargs["completion_probe"]
+ captured["probe_started_at"] = time.monotonic()
+ captured["probe_reason"] = await probe(frozenset())
+ captured["probe_returned_at"] = time.monotonic()
+ if captured["probe_reason"]:
+ return LoopOutcome(status="completed", reason=captured["probe_reason"], billable_actions=["click"])
+ return LoopOutcome(status="budget_exhausted", reason="no download detected", billable_actions=[])
+
+ return _loop
+
+
+def _record_provider_sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]:
+ sleeps: list[float] = []
+
+ async def _record(delay: float) -> None:
+ sleeps.append(delay)
+
+ monkeypatch.setattr(taskv3_provider_downloads, "asyncio", ScopedAsyncio(sleep=_record))
+ return sleeps
+
+
+async def test_v3_download_reads_a_provider_owned_file_back_when_the_replay_saves_nothing(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")], empty_polls=2)
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ block_cls=FileDownloadBlock,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source),
+ )
+
+ assert captured["begin_calls_at_loop_start"] == 1
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert (tmp_path / "invoice.csv").read_bytes() == b"real-export"
+ assert len(observation.poll_deadlines) == 3
+
+
+async def test_v3_download_does_not_poll_the_provider_when_a_local_capture_landed(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _StallingActionDownloadObservation()
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source, local_file="report.pdf"),
+ )
+
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert observation.poll_calls == 0
+ assert (tmp_path / "invoice.pdf").read_bytes() == b"local-bytes"
+
+
+async def test_v3_download_waits_for_the_monitor_replay_before_polling_the_provider(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ # A replay that is still in flight and then lands must win; polling the provider first would
+ # publish a second copy of the same download.
+ monitor = _FakeBrowserDownloadMonitor(
+ pending_checks=2, on_settle=lambda: (tmp_path / "report.pdf").write_bytes(b"replayed")
+ )
+ observation = _StallingActionDownloadObservation()
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source),
+ )
+
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert observation.poll_calls == 0
+ assert sorted(p.name for p in tmp_path.iterdir() if p.is_file()) == ["invoice.pdf"]
+
+
+async def test_v3_download_returns_promptly_without_polling_when_no_download_began(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ sleeps = _record_provider_sleeps(monkeypatch)
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _StallingActionDownloadObservation()
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source, raise_event=False),
+ )
+
+ assert captured["probe_reason"] is None
+ assert observation.poll_calls == 0
+ assert sleeps == [taskv3_provider_downloads.EVENT_ADMISSION_GRACE_SECONDS]
+
+
+async def test_v3_download_keeps_distinct_provider_files_with_equal_bytes(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([("summary.csv", b"same-bytes"), ("detail.csv", b"same-bytes")])
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source),
+ )
+
+ assert captured["probe_reason"] == "download completed (2 new file(s))"
+ landed = [p for p in tmp_path.iterdir() if p.is_file()]
+ assert len(landed) == 2
+ assert all(p.read_bytes() == b"same-bytes" for p in landed)
+
+
+async def test_v3_download_credits_only_files_new_since_the_block_began(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ # An earlier block's file is already in the run dir; it is neither a local capture for this
+ # block (which would suppress the provider read) nor a file this block delivered.
+ (tmp_path / "earlier.pdf").write_bytes(b"earlier")
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source),
+ )
+
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert (tmp_path / "earlier.pdf").read_bytes() == b"earlier"
+ assert (tmp_path / "invoice.csv").read_bytes() == b"real-export"
+
+
+async def test_v3_download_stops_reading_the_provider_when_the_run_is_cancelled(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _StallingActionDownloadObservation()
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ cancel_after_checks=0,
+ loop_fn=_click_then_probe(tmp_path, monitor, source),
+ )
+
+ assert captured["probe_reason"] is None
+ assert observation.poll_calls == 1
+ # Returned well before the stalled poll's own deadline, so the cancel ended it.
+ assert captured["probe_returned_at"] < observation.poll_deadlines[0]
+ assert not [p for p in tmp_path.iterdir() if p.is_file()]
+
+
+async def test_v3_download_does_not_complete_from_a_provider_file_read_after_the_run_was_cancelled(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ # The cancel lands during the action and the provider answers inside its first wait, so no cancel
+ # check runs between the read and finalize unless recover makes one.
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ cancel_after_checks=0,
+ loop_fn=_click_then_probe(tmp_path, monitor, source),
+ )
+
+ assert len(observation.poll_deadlines) == 1
+ assert captured["probe_reason"] is None
+
+
+async def test_v3_download_without_a_monitor_does_not_complete_from_a_provider_file_after_a_cancel(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ cancel_after_checks=0,
+ loop_fn=_click_then_probe(tmp_path, None, source),
+ )
+
+ assert len(observation.poll_deadlines) == 1
+ assert captured["probe_reason"] is None
+
+
[email protected]("baseline", ["listed", "unavailable", "failed"])
+async def test_v3_download_does_not_credit_a_file_that_landed_while_the_provider_baseline_was_read(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, baseline: str
+) -> None:
+ # An earlier block's download finishes while this block awaits the provider baseline, after the
+ # local baseline was already listed; the read may succeed, list nothing usable, or fail.
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([])
+ source = _ActionDownloadSource(
+ None if baseline == "unavailable" else observation,
+ begin_error=RuntimeError("provider down") if baseline == "failed" else None,
+ on_begin=lambda: (tmp_path / "earlier.pdf").write_bytes(b"earlier-block"),
+ )
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source, raise_event=False),
+ )
+
+ assert captured["probe_reason"] is None
+ assert (tmp_path / "earlier.pdf").read_bytes() == b"earlier-block"
+
+
+async def test_v3_download_does_not_complete_from_a_local_file_after_the_run_was_cancelled(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ # No partial file means the in-flight wait never checks; the cancel must still win over finalize.
+ captured = await _run_execute_task_v3_download(monkeypatch, tmp_path, drop_file=True, cancel_after_checks=0)
+
+ assert captured["probe_reason"] is None
+
+
+async def test_v3_download_provider_wait_ends_at_the_block_download_timeout(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([("export.csv", b"late")], empty_polls=10_000)
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ download_timeout=2.0,
+ loop_fn=_click_then_probe(tmp_path, monitor, source),
+ )
+
+ assert captured["probe_reason"] is None
+ assert observation.poll_deadlines
+ # The block's 2 s timeout, not the 30 s default, set the read deadline; no poll started past it.
+ assert observation.poll_deadlines[0] - captured["probe_started_at"] < (
+ taskv3_provider_downloads.PROVIDER_DOWNLOAD_WAIT_SECONDS
+ )
+ assert all(
+ started < deadline
+ for started, deadline in zip(observation.poll_started_at, observation.poll_deadlines, strict=True)
+ )
+ assert not [p for p in tmp_path.iterdir() if p.is_file()]
+
+
+async def test_v3_download_keeps_the_local_path_when_the_provider_baseline_fails(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ presigned = "https://downloads.example/file.csv?X-Amz-Signature=deadbeefsecret"
+ source = _ActionDownloadSource(
+ _StallingActionDownloadObservation(), begin_error=_secret_bearing_validation_error(presigned)
+ )
+ monitor = _FakeBrowserDownloadMonitor()
+
+ with capture_logs() as logs:
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source, local_file="report.pdf"),
+ )
+
+ assert source.begin_calls == 1
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert "deadbeefsecret" not in json.dumps(logs, default=str)
+
+
+async def test_v3_download_does_not_take_a_staged_upload_for_the_admitted_download(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ # A file_upload staged into the run dir in the same round must not settle the provider read.
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])
+ source = _ActionDownloadSource(observation)
+
+ async def _loop(kwargs: dict[str, Any], captured: dict[str, Any]) -> LoopOutcome:
+ monitor.browser_download_event_count += 1
+ (tmp_path / "upload.pdf").write_bytes(b"staged-upload")
+ captured["probe_reason"] = await kwargs["completion_probe"](frozenset({"upload.pdf"}))
+ return LoopOutcome(status="completed", reason="done", billable_actions=["click"])
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_loop,
+ )
+
+ assert len(observation.poll_deadlines) == 1
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert (tmp_path / "upload.pdf").read_bytes() == b"staged-upload"
+
+
+async def test_v3_wait_only_download_does_not_read_the_provider(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ # Only a complete_on_download block owns a provider read-back; a download_timeout-only block keeps its
+ # native in-flight wait.
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ complete_on_download=False,
+ download_timeout=10.0,
+ provider_source=source,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, source),
+ )
+
+ assert captured["begin_calls_at_loop_start"] == 0
+ assert captured["probe_reason"] is None
+ assert observation.poll_deadlines == []
+ assert not [p for p in tmp_path.iterdir() if p.is_file()]
+
+
+async def test_v3_download_without_a_monitor_reads_the_provider_once_per_probe(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])
+ source = _ActionDownloadSource(observation)
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=None,
+ loop_fn=_click_then_probe(tmp_path, None, source),
+ )
+
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert len(observation.poll_deadlines) == 1
+
+
+async def test_v3_download_without_a_provider_source_is_unchanged(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ sleeps = _record_provider_sleeps(monkeypatch)
+ monitor = _FakeBrowserDownloadMonitor()
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ download_monitor=monitor,
+ loop_fn=_click_then_probe(tmp_path, monitor, None),
+ )
+
+ assert captured["probe_reason"] is None
+ assert sleeps == []
+
+
+def _probe_twice(
+ monitor: _FakeBrowserDownloadMonitor, between: Callable[[], None]
+) -> Callable[[dict[str, Any], dict[str, Any]], Awaitable[LoopOutcome]]:
+ # One click, two probes: the second follows an unrelated billable tool, with no new download event.
+ async def _loop(kwargs: dict[str, Any], captured: dict[str, Any]) -> LoopOutcome:
+ monitor.browser_download_event_count += 1
+ probe = kwargs["completion_probe"]
+ captured["first_reason"] = await probe(frozenset())
+ between()
+ captured["probe_reason"] = await probe(frozenset())
+ if captured["probe_reason"]:
+ return LoopOutcome(status="completed", reason=captured["probe_reason"], billable_actions=["click"])
+ return LoopOutcome(status="budget_exhausted", reason="no download detected", billable_actions=[])
+
+ return _loop
+
+
+async def test_v3_download_still_reads_the_provider_after_the_monitor_outlasted_the_first_probe(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ monitor = _FakeBrowserDownloadMonitor(pending_checks=10**6)
+ observation = _FakeActionDownloadObservation([("export.csv", b"real-export")])
+ source = _ActionDownloadSource(observation)
+
+ def _settle() -> None:
+ monitor._pending_checks = 0
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ download_timeout=1.0,
+ loop_fn=_probe_twice(monitor, _settle),
+ )
+
+ assert captured["first_reason"] is None
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert (tmp_path / "invoice.csv").read_bytes() == b"real-export"
+
+
+async def test_v3_download_still_reads_the_provider_after_an_empty_first_window(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ monitor = _FakeBrowserDownloadMonitor()
+ observation = _FakeActionDownloadObservation([("export.csv", b"late-export")], empty_polls=10**6)
+ source = _ActionDownloadSource(observation)
+
+ def _list_late() -> None:
+ observation._empty_polls = 0
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ download_timeout=1.0,
+ loop_fn=_probe_twice(monitor, _list_late),
+ )
+
+ assert captured["first_reason"] is None
+ assert captured["probe_reason"] == "download completed (1 new file(s))"
+ assert (tmp_path / "invoice.csv").read_bytes() == b"late-export"
+
+
+async def test_v3_download_cancellation_returns_only_after_the_provider_poll_cleaned_up(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ gate = asyncio.Event()
+ observation = _GatedCleanupObservation(gate)
+ source = _ActionDownloadSource(observation)
+ monitor = _FakeBrowserDownloadMonitor()
+
+ async def _loop(kwargs: dict[str, Any], captured: dict[str, Any]) -> LoopOutcome:
+ monitor.browser_download_event_count += 1
+ asyncio.get_running_loop().call_later(2.0, gate.set)
+ captured["probe_reason"] = await kwargs["completion_probe"](frozenset())
+ captured["cleanup_finished_at_return"] = observation.cleanup_finished
+ return LoopOutcome(status="budget_exhausted", reason="no download detected", billable_actions=[])
+
+ captured = await _run_execute_task_v3_download(
+ monkeypatch,
+ tmp_path,
+ drop_file=False,
+ provider_source=source,
+ download_monitor=monitor,
+ cancel_after_checks=0,
+ loop_fn=_loop,
+ )
+
+ assert captured["probe_reason"] is None
+ assert captured["cleanup_finished_at_return"] is True
diff --git a/tests/unit/test_aws_client_ttl.py b/tests/unit/test_aws_client_ttl.py
index 819dbb486..064dee799 100644
--- a/tests/unit/test_aws_client_ttl.py
+++ b/tests/unit/test_aws_client_ttl.py
@@ -582,7 +582,9 @@ async def test_upload_file_from_path_cancelled_mid_multipart_aborts_upload_and_s
@contextlib.asynccontextmanager
-async def _local_s3(barrier: asyncio.Barrier | None = None, port: int = 0): # type: ignore[no-untyped-def]
+async def _local_s3( # type: ignore[no-untyped-def]
+ barrier: asyncio.Barrier | None = None, port: int = 0, body: bytes = b"payload"
+):
"""A plain-HTTP stand-in for S3 that records which connection and credential served each request."""
seen: list[tuple[int, str, asyncio.BaseTransport]] = []
@@ -593,7 +595,7 @@ async def _local_s3(barrier: asyncio.Barrier | None = None, port: int = 0): # t
seen.append((transport.get_extra_info("peername")[1], request.headers.get("Authorization", ""), transport))
if barrier is not None:
await barrier.wait()
- return web.Response(body=b"payload" if request.method == "GET" else b"")
+ return web.Response(body=body if request.method == "GET" else b"")
app = web.Application()
app.router.add_route("*", "/{tail:.*}", handler)
@@ -699,3 +701,47 @@ def test_s3_client_keeps_working_across_event_loops() -> None:
assert asyncio.run(download_once()) == b"payload"
assert asyncio.run(download_once()) == b"payload"
+
+
+async def test_open_download_stream_keeps_its_client_open_until_closed_across_a_session_replacement() -> None:
+ """A streamed body outlives get_object, so the shared client it came from must not close under it."""
+ payload = bytes(range(256)) * 16 * 1024
+ async with _local_s3(body=payload) as (endpoint_url, _):
+ client = _local_client(endpoint_url)
+ stream = await client.open_download_stream("s3://bucket/large.zip")
+ replaced_client = client._shared_s3_clients[asyncio.get_running_loop()]
+ chunks = stream.iter_chunks(64 * 1024)
+ received = [await anext(chunks)]
+
+ client.session = aws.aioboto3.Session(aws_access_key_id="AKIAROTATED", aws_secret_access_key="rotated")
+ assert await client.download_file("s3://bucket/k") == payload
+ received.extend([chunk async for chunk in chunks])
+ assert not replaced_client._owner.done()
+ await stream.aclose()
+
+ async with asyncio.timeout(5):
+ await replaced_client._owner
+ assert stream.content_length == len(payload)
+ assert b"".join(received) == payload
+
+
+async def test_open_download_stream_releases_its_client_when_closing_the_body_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async with _local_s3() as (endpoint_url, _):
+ client = _local_client(endpoint_url)
+ stream = await client.open_download_stream("s3://bucket/k")
+ shared = client._shared_s3_clients[asyncio.get_running_loop()]
+ close_body = stream._body.close
+
+ def close_then_fail() -> None:
+ close_body()
+ raise RuntimeError("body close failed")
+
+ monkeypatch.setattr(stream._body, "close", close_then_fail)
+ with pytest.raises(RuntimeError):
+ await stream.aclose()
+
+ assert shared.leases == 0
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index 78504cbed..80b09c250 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -16,7 +16,6 @@ from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patch
import pytest
import structlog
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
-from pydantic import BaseModel, ValidationError
from structlog.testing import capture_logs
from skyvern.config import settings
@@ -61,6 +60,13 @@ from skyvern.webeye.browser_factory import (
from skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor
from skyvern.webeye.scraper.scraped_page import ScrapedPage
from skyvern.webeye.utils.page import BlobActionFreshness
+from tests.unit._download_provider_fakes import (
+ _ActionDownloadSource,
+ _FakeActionDownloadObservation,
+ _RaisingActionDownloadObservation,
+ _secret_bearing_validation_error,
+ _StallingActionDownloadObservation,
+)
from tests.unit.helpers import make_organization, make_step, make_task
# Five seconds is only a test-side runaway guard. The behavior under test is
@@ -7082,87 +7088,6 @@ async def test_retention_torn_down_on_cancelled() -> None:
result["teardown"].assert_awaited_once()
-class _FakeActionDownloadObservation:
- """Neutral provider observation: drops completed remote files into the run dir on its first poll,
- honoring the ``deadline`` (monotonic) contract strictly (a handler passing per-call ``timeout_seconds``
- materializes nothing -- the anti-terminal-cleanup RED)."""
-
- def __init__(self, files: list[tuple[str, bytes]], *, empty_polls: int = 0) -> None:
- self._pending = list(files)
- self._empty_polls = empty_polls
- self.poll_deadlines: list[float] = []
-
- async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:
- self.poll_deadlines.append(deadline)
- if len(self.poll_deadlines) <= self._empty_polls:
- return
- destination_dir.mkdir(parents=True, exist_ok=True)
- for name, data in self._pending:
- (destination_dir / name).write_bytes(data)
- self._pending = []
-
-
-class _RaisingActionDownloadObservation:
- """Provider observation whose poll raises a schema/validation error embedding a presigned URL, mirroring
- the real leak surface (the vendor list body is parsed into typed rows). When ``materialize_first`` is
- set, the first poll drops a completed file (so the handler reaches its final poll) and a later poll raises."""
-
- def __init__(self, error: BaseException, *, materialize_first: tuple[str, bytes] | None = None) -> None:
- self._error = error
- self._materialize_first = materialize_first
- self.calls = 0
-
- async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:
- self.calls += 1
- if self._materialize_first is not None and self.calls == 1:
- destination_dir.mkdir(parents=True, exist_ok=True)
- name, data = self._materialize_first
- (destination_dir / name).write_bytes(data)
- return
- raise self._error
-
-
-class _StallingActionDownloadObservation:
- """Provider observation whose poll never returns on its own -- only the handler's deadline can end it, so
- if the handler polls it once a local artifact already accounts for the action the stalled provider would
- consume the whole download deadline; the fix must never poll it in that case."""
-
- def __init__(self) -> None:
- self.poll_calls = 0
-
- async def poll_and_materialize(self, *, destination_dir: Path, deadline: float) -> None:
- self.poll_calls += 1
- await asyncio.Event().wait()
-
-
-class _ActionDownloadSource:
- """Neutral provider source: hands back a fixed observation and records begin bookkeeping."""
-
- def __init__(self, observation: object, *, begin_error: Exception | None = None) -> None:
- self._observation = observation
- self._begin_error = begin_error
- self.begin_calls = 0
- self.begin_deadline: float | None = None
-
- async def begin_observation(self, *, deadline: float) -> object:
- self.begin_calls += 1
- self.begin_deadline = deadline
- if self._begin_error is not None:
- raise self._begin_error
- return self._observation
-
-
-def _secret_bearing_validation_error(presigned_url: str) -> ValidationError:
- class _Row(BaseModel):
- size: int
-
- try:
- _Row.model_validate({"size": presigned_url})
- except ValidationError as exc:
- return exc
- raise AssertionError("expected a ValidationError")
-
-
@contextlib.contextmanager
def _capture_handler_logs() -> Iterator[io.StringIO]:
import skyvern.webeye.actions.handler as handler_module
diff --git a/tests/unit/test_taskv3_provider_downloads.py b/tests/unit/test_taskv3_provider_downloads.py
new file mode 100644
index 000000000..aee161e27
--- /dev/null
+++ b/tests/unit/test_taskv3_provider_downloads.py
@@ -0,0 +1,90 @@
+"""ProviderDownloadRecovery owns each provider poll it starts: a recover() call never returns, normally or by
+cancellation, while that poll is still running its own cleanup."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from pathlib import Path
+
+import pytest
+
+from skyvern.forge.taskv3 import provider_downloads
+from skyvern.forge.taskv3.provider_downloads import ProviderDownloadRecovery
+from tests.unit._download_provider_fakes import _FakeBrowserDownloadMonitor, _GatedCleanupObservation
+from tests.unit.scoped_asyncio import ScopedAsyncio
+
+
+async def _never_cancel() -> bool:
+ return False
+
+
+async def _always_cancel() -> bool:
+ return True
+
+
+def _recovery(tmp_path: Path, observation: object) -> tuple[ProviderDownloadRecovery, _FakeBrowserDownloadMonitor]:
+ monitor = _FakeBrowserDownloadMonitor()
+ recovery = ProviderDownloadRecovery(observation, tmp_path, monitor) # type: ignore[arg-type]
+ monitor.browser_download_event_count += 1
+ return recovery, monitor
+
+
+def _release_later(gate: asyncio.Event, delay: float) -> None:
+ asyncio.get_running_loop().call_later(delay, gate.set)
+
+
[email protected]("should_cancel", [_never_cancel, _always_cancel], ids=["deadline", "should_cancel"])
+async def test_recover_returns_only_after_its_cancelled_poll_finished_cleanup(
+ tmp_path: Path, should_cancel: Callable[[], Awaitable[bool]]
+) -> None:
+ gate = asyncio.Event()
+ observation = _GatedCleanupObservation(gate)
+ recovery, _monitor = _recovery(tmp_path, observation)
+ _release_later(gate, 1.5)
+
+ await recovery.recover(staged=(), timeout=0.6, should_cancel=should_cancel)
+
+ assert observation.started.is_set()
+ assert observation.cleanup_finished
+
+
+async def test_outer_cancellation_joins_the_poll_even_when_cancelled_again_during_cleanup(tmp_path: Path) -> None:
+ gate = asyncio.Event()
+ observation = _GatedCleanupObservation(gate)
+ recovery, _monitor = _recovery(tmp_path, observation)
+ task = asyncio.create_task(recovery.recover(staged=(), timeout=30.0, should_cancel=_never_cancel))
+ await asyncio.wait_for(observation.started.wait(), 5)
+
+ task.cancel()
+ await asyncio.wait_for(observation.cleanup_started.wait(), 5)
+ task.cancel()
+ await asyncio.sleep(0.05)
+ assert not task.done(), "recover returned while its cancelled poll was still cleaning up"
+ gate.set()
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert observation.cleanup_finished
+
+
+async def test_admission_grace_never_outlasts_the_probe_budget(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ sleeps: list[float] = []
+
+ async def _record(delay: float) -> None:
+ sleeps.append(delay)
+
+ monkeypatch.setattr(provider_downloads, "asyncio", ScopedAsyncio(sleep=_record))
+ gate = asyncio.Event()
+ gate.set()
+ observation = _GatedCleanupObservation(gate)
+ recovery, _monitor = _recovery(tmp_path, observation)
+
+ await recovery.recover(staged=(), timeout=0.05, should_cancel=_never_cancel)
+
+ assert sleeps
+ assert all(delay <= 0.05 for delay in sleeps)
+ assert observation.cleanup_finished or not observation.started.is_set()