Skip to content

Automate browser based workflows with AI

perf(SKY-9203): parallelize browser session file metadata + presign (#5876)

Aaron Perez6760da531796
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
index 4249701e2..9b6c347a7 100644
--- a/skyvern/forge/sdk/artifact/storage/s3.py
+++ b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -1,3 +1,4 @@
+import asyncio
import io
import os
import shutil
@@ -42,10 +43,21 @@ S3_ZSTD_COMPRESSED_SUFFIX = ".zst"
class S3Storage(BaseStorage):
_PATH_VERSION = "v1"
+ # Cap concurrent head_object fan-out in the legacy listing fallback.
+ # /browser_sessions/history gathers across up to page_size=100 sessions
+ # in parallel, so a per-call semaphore would still allow sessions x cap
+ # in-flight requests. The semaphore is held on the instance so every
+ # legacy listing call sharing this S3Storage shares the same cap.
+ _LEGACY_LISTING_HEAD_CONCURRENCY = 32
def __init__(self, bucket: str | None = None, endpoint_url: str | None = None) -> None:
self.async_client = AsyncAWSClient(endpoint_url=endpoint_url)
self.bucket = bucket or settings.AWS_S3_BUCKET_ARTIFACTS
+ # asyncio.Semaphore is event-loop-bound; constructing it eagerly is
+ # safe in 3.10+ (binds lazily on first acquire). The forge/cloud apps
+ # run a single asyncio loop, so one instance-level semaphore is
+ # enough to enforce a request-wide cap across the outer fanout.
+ self._head_object_semaphore = asyncio.Semaphore(self._LEGACY_LISTING_HEAD_CONCURRENCY)
def build_uri(self, *, organization_id: str, artifact_id: str, step: Step, artifact_type: ArtifactType) -> str:
file_ext = FILE_EXTENTSION_MAP[artifact_type]
@@ -369,6 +381,54 @@ class S3Storage(BaseStorage):
organization_id=organization_id, browser_session_id=browser_session_id
)
+ async def _safe_get_object_info(self, uri: str) -> dict | None:
+ try:
+ return await self.async_client.get_object_info(uri)
+ except Exception:
+ LOG.exception("Object info retrieval failed", uri=uri)
+ return None
+
+ async def _bounded_get_object_infos(self, keys: list[str]) -> list[dict | None]:
+ """Fan out head_object across keys under the instance-wide cap.
+
+ Uses ``self._head_object_semaphore`` so concurrent callers on the same
+ ``S3Storage`` (e.g. the per-session fanout inside
+ ``/browser_sessions/history``) share one limit instead of each getting
+ its own — otherwise page_size x cap could blow past S3 client limits.
+ """
+ if not keys:
+ return []
+
+ async def _bounded(uri: str) -> dict | None:
+ async with self._head_object_semaphore:
+ return await self._safe_get_object_info(uri)
+
+ return await asyncio.gather(*[_bounded(key) for key in keys])
+
+ async def _resilient_presign(self, keys: list[str]) -> list[str | None]:
+ """Batch presign with per-key fallback so one failure does not drop every URL."""
+ if not keys:
+ return []
+ batched = await self.async_client.create_presigned_urls(keys)
+ if batched is not None and len(batched) == len(keys):
+ return list(batched)
+
+ LOG.warning(
+ "Batch presign failed or returned wrong length; falling back to per-key signing",
+ requested=len(keys),
+ returned=(len(batched) if batched is not None else None),
+ )
+
+ async def _sign_one(uri: str) -> str | None:
+ try:
+ signed = await self.async_client.create_presigned_urls([uri])
+ return signed[0] if signed else None
+ except Exception:
+ LOG.exception("Per-key presign failed", uri=uri)
+ return None
+
+ return await asyncio.gather(*[_sign_one(key) for key in keys])
+
async def _get_shared_downloaded_files_in_browser_session_via_listing(
self, *, organization_id: str, browser_session_id: str
) -> list[FileInfo]:
@@ -386,34 +446,29 @@ class S3Storage(BaseStorage):
if len(object_keys) == 0:
return []
+ object_infos, presigned_urls = await asyncio.gather(
+ self._bounded_get_object_infos(object_keys),
+ self._resilient_presign(object_keys),
+ )
+
file_infos: list[FileInfo] = []
- for key in object_keys:
- metadata = {}
- modified_at: datetime | None = None
- # Get metadata (including checksum)
- try:
- object_info = await self.async_client.get_object_info(key)
- metadata = object_info.get("Metadata", {})
- modified_at = object_info.get("LastModified")
- except Exception:
- LOG.exception("Object info retrieval failed", uri=key)
+ for key, object_info, url in zip(object_keys, object_infos, presigned_urls):
+ if url is None:
+ continue
+ metadata = (object_info or {}).get("Metadata") or {}
+ modified_at: datetime | None = (object_info or {}).get("LastModified")
- # Create FileInfo object
filename = os.path.basename(key)
checksum = metadata.get("sha256_checksum") if metadata else None
- # Get presigned URL
- presigned_urls = await self.async_client.create_presigned_urls([key])
- if not presigned_urls:
- continue
-
- file_info = FileInfo(
- url=presigned_urls[0],
- checksum=checksum,
- filename=metadata.get("original_filename", filename) if metadata else filename,
- modified_at=modified_at,
+ file_infos.append(
+ FileInfo(
+ url=url,
+ checksum=checksum,
+ filename=metadata.get("original_filename", filename) if metadata else filename,
+ modified_at=modified_at,
+ )
)
- file_infos.append(file_info)
return file_infos
@@ -488,56 +543,58 @@ class S3Storage(BaseStorage):
# rows — every call here is a billable ListObjects request.
bucket = settings.AWS_S3_BUCKET_ARTIFACTS
listing_uri = f"s3://{bucket}/{self._PATH_VERSION}/{settings.ENV}/{organization_id}/browser_sessions/{browser_session_id}/videos"
- object_keys = [f"s3://{bucket}/{file}" for file in await self.async_client.list_files(uri=listing_uri)]
- if len(object_keys) == 0:
+ all_keys = [f"s3://{bucket}/{file}" for file in await self.async_client.list_files(uri=listing_uri)]
+ if len(all_keys) == 0:
return []
- file_infos: list[FileInfo] = []
- for key in object_keys:
- # Playwright's record_video_dir should only contain .webm files.
- # Filter defensively in case of unexpected files.
+ # Playwright's record_video_dir should only contain .webm files; filter defensively.
+ candidate_keys: list[str] = []
+ for key in all_keys:
key_lower = key.lower()
- if not (key_lower.endswith(".webm") or key_lower.endswith(".mp4")):
+ if key_lower.endswith(".webm") or key_lower.endswith(".mp4"):
+ candidate_keys.append(key)
+ else:
LOG.warning(
"Skipping recording file with unsupported extension",
uri=key,
organization_id=organization_id,
browser_session_id=browser_session_id,
)
+ if not candidate_keys:
+ return []
+
+ # Defer presign until zero-byte uploads are filtered out so we never sign discards.
+ candidate_infos = await self._bounded_get_object_infos(candidate_keys)
+ kept: list[tuple[str, dict | None]] = []
+ for key, info in zip(candidate_keys, candidate_infos):
+ if info is not None and info.get("ContentLength") == 0:
continue
+ kept.append((key, info))
+ if not kept:
+ return []
- metadata = {}
- modified_at: datetime | None = None
- content_length: int | None = None
- # Get metadata (including checksum)
- try:
- object_info = await self.async_client.get_object_info(key)
- metadata = object_info.get("Metadata", {})
- modified_at = object_info.get("LastModified")
- content_length = object_info.get("ContentLength")
- except Exception:
- LOG.exception("Recording object info retrieval failed", uri=key)
+ object_keys = [key for key, _ in kept]
+ object_infos = [info for _, info in kept]
+ presigned_urls = await self._resilient_presign(object_keys)
- # Skip zero-byte objects (if any incompleted uploads)
- if content_length == 0:
+ file_infos: list[FileInfo] = []
+ for key, object_info, url in zip(object_keys, object_infos, presigned_urls):
+ if url is None:
continue
+ metadata = (object_info or {}).get("Metadata") or {}
+ modified_at: datetime | None = (object_info or {}).get("LastModified")
- # Create FileInfo object
filename = os.path.basename(key)
checksum = metadata.get("sha256_checksum") if metadata else None
- # Get presigned URL
- presigned_urls = await self.async_client.create_presigned_urls([key])
- if not presigned_urls:
- continue
-
- file_info = FileInfo(
- url=presigned_urls[0],
- checksum=checksum,
- filename=metadata.get("original_filename", filename) if metadata else filename,
- modified_at=modified_at,
+ file_infos.append(
+ FileInfo(
+ url=url,
+ checksum=checksum,
+ filename=metadata.get("original_filename", filename) if metadata else filename,
+ modified_at=modified_at,
+ )
)
- file_infos.append(file_info)
# Prefer the newest recording first (S3 list order is not guaranteed).
# Treat None as "oldest".
diff --git a/skyvern/forge/sdk/artifact/storage/test_s3_storage.py b/skyvern/forge/sdk/artifact/storage/test_s3_storage.py
index 370986863..eb03963d1 100644
--- a/skyvern/forge/sdk/artifact/storage/test_s3_storage.py
+++ b/skyvern/forge/sdk/artifact/storage/test_s3_storage.py
@@ -1,4 +1,6 @@
+import asyncio
import io
+import os
from datetime import datetime
from pathlib import Path
from typing import Generator
@@ -462,6 +464,334 @@ class TestS3StorageBrowserSessionFiles:
"""Test storage_type returns 's3'."""
assert s3_storage.storage_type == "s3"
+ async def test_get_shared_downloaded_files_returns_all(
+ self,
+ s3_storage: S3Storage,
+ boto3_test_client: S3Client,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """Listing many downloaded files returns one FileInfo per object with a presigned URL."""
+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)
+ session_id = "bs_returns_all_test"
+ for i in range(5):
+ test_file = tmp_path / f"invoice_{i}.csv"
+ test_file.write_bytes(f"row,{i}\n".encode())
+ await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ artifact_type="downloads",
+ local_file_path=str(test_file),
+ remote_path=f"invoice_{i}.csv",
+ date=None,
+ )
+
+ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ )
+ assert len(file_infos) == 5
+ assert {fi.filename for fi in file_infos} == {f"invoice_{i}.csv" for i in range(5)}
+ for fi in file_infos:
+ assert fi.url and "Signature=" in fi.url
+
+ async def test_get_shared_downloaded_files_empty(
+ self, s3_storage: S3Storage, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """Empty directory returns an empty list, no head_object/presign work."""
+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)
+ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id="bs_no_files",
+ )
+ assert file_infos == []
+
+ async def test_get_shared_downloaded_files_runs_concurrently(
+ self,
+ s3_storage: S3Storage,
+ boto3_test_client: S3Client,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """head_object calls overlap rather than running sequentially."""
+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)
+ session_id = "bs_concurrent_test"
+ for i in range(8):
+ test_file = tmp_path / f"f_{i}.csv"
+ test_file.write_bytes(b"x")
+ await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ artifact_type="downloads",
+ local_file_path=str(test_file),
+ remote_path=f"f_{i}.csv",
+ date=None,
+ )
+
+ original = s3_storage.async_client.get_object_info
+ per_call_delay = 0.1
+ in_flight = 0
+ max_in_flight = 0
+
+ async def slow_get_object_info(uri: str) -> dict:
+ nonlocal in_flight, max_in_flight
+ in_flight += 1
+ max_in_flight = max(max_in_flight, in_flight)
+ try:
+ await asyncio.sleep(per_call_delay)
+ return await original(uri)
+ finally:
+ in_flight -= 1
+
+ monkeypatch.setattr(s3_storage.async_client, "get_object_info", slow_get_object_info)
+
+ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ )
+
+ assert len(file_infos) == 8
+ # max_in_flight > 1 directly proves head_object overlapped; wall-clock bounds flake under CI load.
+ assert max_in_flight > 1, "expected overlapping head_object calls"
+
+ async def test_get_shared_downloaded_files_caps_concurrency(
+ self,
+ s3_storage: S3Storage,
+ boto3_test_client: S3Client,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """head_object fan-out is bounded by the instance-level semaphore."""
+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)
+ # Replace the instance semaphore with a tighter one so the cap is
+ # hit with a small file count.
+ monkeypatch.setattr(s3_storage, "_head_object_semaphore", asyncio.Semaphore(3))
+ session_id = "bs_cap_test"
+ file_count = 12
+ for i in range(file_count):
+ test_file = tmp_path / f"f_{i}.csv"
+ test_file.write_bytes(b"x")
+ await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ artifact_type="downloads",
+ local_file_path=str(test_file),
+ remote_path=f"f_{i}.csv",
+ date=None,
+ )
+
+ original = s3_storage.async_client.get_object_info
+ in_flight = 0
+ max_in_flight = 0
+
+ async def tracked_get_object_info(uri: str) -> dict:
+ nonlocal in_flight, max_in_flight
+ in_flight += 1
+ max_in_flight = max(max_in_flight, in_flight)
+ try:
+ await asyncio.sleep(0.05)
+ return await original(uri)
+ finally:
+ in_flight -= 1
+
+ monkeypatch.setattr(s3_storage.async_client, "get_object_info", tracked_get_object_info)
+
+ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ )
+
+ assert len(file_infos) == file_count
+ assert max_in_flight <= 3, f"head_object exceeded the cap (max_in_flight={max_in_flight})"
+
+ async def test_head_concurrency_cap_is_shared_across_concurrent_calls(
+ self,
+ s3_storage: S3Storage,
+ boto3_test_client: S3Client,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """Concurrent listing calls (the /browser_sessions/history fanout) share one cap."""
+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)
+ # Tight cap so the test can prove the limit holds across callers.
+ monkeypatch.setattr(s3_storage, "_head_object_semaphore", asyncio.Semaphore(3))
+
+ session_ids = ["bs_share_a", "bs_share_b", "bs_share_c"]
+ for session_id in session_ids:
+ for i in range(4):
+ test_file = tmp_path / f"{session_id}_f_{i}.csv"
+ test_file.write_bytes(b"x")
+ await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ artifact_type="downloads",
+ local_file_path=str(test_file),
+ remote_path=f"f_{i}.csv",
+ date=None,
+ )
+
+ original = s3_storage.async_client.get_object_info
+ in_flight = 0
+ max_in_flight = 0
+
+ async def tracked_get_object_info(uri: str) -> dict:
+ nonlocal in_flight, max_in_flight
+ in_flight += 1
+ max_in_flight = max(max_in_flight, in_flight)
+ try:
+ await asyncio.sleep(0.05)
+ return await original(uri)
+ finally:
+ in_flight -= 1
+
+ monkeypatch.setattr(s3_storage.async_client, "get_object_info", tracked_get_object_info)
+
+ results = await asyncio.gather(
+ *[
+ s3_storage.get_shared_downloaded_files_in_browser_session(
+ organization_id=TEST_ORGANIZATION_ID, browser_session_id=session_id
+ )
+ for session_id in session_ids
+ ]
+ )
+
+ for file_infos in results:
+ assert len(file_infos) == 4
+ # Without a shared semaphore, 3 sessions x 4 files = 12 head_objects could overlap.
+ assert max_in_flight <= 3, f"shared semaphore breached the cap (max_in_flight={max_in_flight})"
+
+ async def test_get_shared_downloaded_files_falls_back_when_batch_presign_fails(
+ self,
+ s3_storage: S3Storage,
+ boto3_test_client: S3Client,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """When batch presign returns None, fall back to per-key signing rather than dropping all files."""
+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)
+ session_id = "bs_batch_fail_test"
+ for i in range(3):
+ test_file = tmp_path / f"f_{i}.csv"
+ test_file.write_bytes(b"x")
+ await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ artifact_type="downloads",
+ local_file_path=str(test_file),
+ remote_path=f"f_{i}.csv",
+ date=None,
+ )
+
+ original_create = s3_storage.async_client.create_presigned_urls
+
+ async def flaky_create(uris: list[str]) -> list[str] | None:
+ if len(uris) > 1:
+ return None # simulate batch failure
+ return await original_create(uris)
+
+ monkeypatch.setattr(s3_storage.async_client, "create_presigned_urls", flaky_create)
+
+ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ )
+ # Per-key fallback should preserve all 3 URLs.
+ assert len(file_infos) == 3
+ assert all(fi.url and "Signature=" in fi.url for fi in file_infos)
+
+ async def test_get_shared_downloaded_files_per_key_partial_failure(
+ self,
+ s3_storage: S3Storage,
+ boto3_test_client: S3Client,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """Per-key fallback skips only the failing key, preserving the rest."""
+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)
+ session_id = "bs_partial_fail_test"
+ for i in range(3):
+ test_file = tmp_path / f"f_{i}.csv"
+ test_file.write_bytes(b"x")
+ await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ artifact_type="downloads",
+ local_file_path=str(test_file),
+ remote_path=f"f_{i}.csv",
+ date=None,
+ )
+
+ original_create = s3_storage.async_client.create_presigned_urls
+
+ async def flaky_create(uris: list[str]) -> list[str] | None:
+ if len(uris) > 1:
+ return None # force per-key fallback
+ if uris[0].endswith("f_1.csv"):
+ return None # one key fails
+ return await original_create(uris)
+
+ monkeypatch.setattr(s3_storage.async_client, "create_presigned_urls", flaky_create)
+
+ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ )
+ names = {fi.filename for fi in file_infos}
+ assert names == {"f_0.csv", "f_2.csv"}
+
+ async def test_get_shared_recordings_skips_zero_byte_before_presign(
+ self,
+ s3_storage: S3Storage,
+ boto3_test_client: S3Client,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """Zero-byte recordings are filtered before they consume a presign slot."""
+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)
+ # Force the legacy listing path (DB-first path is exercised in test_browser_session_recording_artifacts).
+ monkeypatch.setattr(settings, "ARTIFACT_CONTENT_HMAC_KEYRING", None)
+ session_id = "bs_zero_byte_test"
+
+ good_file = tmp_path / "good.webm"
+ good_file.write_bytes(b"\x1a\x45\xdf\xa3" + b"\x00" * 32)
+ await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ artifact_type="videos",
+ local_file_path=str(good_file),
+ remote_path="good.webm",
+ date="2025-01-15",
+ )
+
+ empty_file = tmp_path / "empty.webm"
+ empty_file.write_bytes(b"")
+ await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ artifact_type="videos",
+ local_file_path=str(empty_file),
+ remote_path="empty.webm",
+ date="2025-01-15",
+ )
+
+ original_create = s3_storage.async_client.create_presigned_urls
+ seen_keys: list[list[str]] = []
+
+ async def tracking_create(uris: list[str]) -> list[str] | None:
+ seen_keys.append(list(uris))
+ return await original_create(uris)
+
+ monkeypatch.setattr(s3_storage.async_client, "create_presigned_urls", tracking_create)
+
+ file_infos = await s3_storage.get_shared_recordings_in_browser_session(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=session_id,
+ )
+ assert {fi.filename for fi in file_infos} == {"good.webm"}
+ # The empty.webm key must never be passed to create_presigned_urls.
+ passed_basenames = [os.path.basename(uri) for batch in seen_keys for uri in batch]
+ assert "empty.webm" not in passed_basenames
+
CONTENT_TYPE_TEST_CASES = [
# (filename, expected_content_type, artifact_type, date)