Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
perf(SKY-9203): parallelize browser session file metadata + presign (#5876)
Aaron Perez6760da531796
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.pyindex 4249701e2..9b6c347a7 100644--- a/skyvern/forge/sdk/artifact/storage/s3.py+++ b/skyvern/forge/sdk/artifact/storage/s3.py@@ -1,3 +1,4 @@+import asyncioimport ioimport osimport shutil@@ -42,10 +43,21 @@ S3_ZSTD_COMPRESSED_SUFFIX = ".zst"class S3Storage(BaseStorage):_PATH_VERSION = "v1"+ # Cap concurrent head_object fan-out in the legacy listing fallback.+ # /browser_sessions/history gathers across up to page_size=100 sessions+ # in parallel, so a per-call semaphore would still allow sessions x cap+ # in-flight requests. The semaphore is held on the instance so every+ # legacy listing call sharing this S3Storage shares the same cap.+ _LEGACY_LISTING_HEAD_CONCURRENCY = 32def __init__(self, bucket: str | None = None, endpoint_url: str | None = None) -> None:self.async_client = AsyncAWSClient(endpoint_url=endpoint_url)self.bucket = bucket or settings.AWS_S3_BUCKET_ARTIFACTS+ # asyncio.Semaphore is event-loop-bound; constructing it eagerly is+ # safe in 3.10+ (binds lazily on first acquire). The forge/cloud apps+ # run a single asyncio loop, so one instance-level semaphore is+ # enough to enforce a request-wide cap across the outer fanout.+ self._head_object_semaphore = asyncio.Semaphore(self._LEGACY_LISTING_HEAD_CONCURRENCY)def build_uri(self, *, organization_id: str, artifact_id: str, step: Step, artifact_type: ArtifactType) -> str:file_ext = FILE_EXTENTSION_MAP[artifact_type]@@ -369,6 +381,54 @@ class S3Storage(BaseStorage):organization_id=organization_id, browser_session_id=browser_session_id)+ async def _safe_get_object_info(self, uri: str) -> dict | None:+ try:+ return await self.async_client.get_object_info(uri)+ except Exception:+ LOG.exception("Object info retrieval failed", uri=uri)+ return None++ async def _bounded_get_object_infos(self, keys: list[str]) -> list[dict | None]:+ """Fan out head_object across keys under the instance-wide cap.++ Uses ``self._head_object_semaphore`` so concurrent callers on the same+ ``S3Storage`` (e.g. the per-session fanout inside+ ``/browser_sessions/history``) share one limit instead of each getting+ its own — otherwise page_size x cap could blow past S3 client limits.+ """+ if not keys:+ return []++ async def _bounded(uri: str) -> dict | None:+ async with self._head_object_semaphore:+ return await self._safe_get_object_info(uri)++ return await asyncio.gather(*[_bounded(key) for key in keys])++ async def _resilient_presign(self, keys: list[str]) -> list[str | None]:+ """Batch presign with per-key fallback so one failure does not drop every URL."""+ if not keys:+ return []+ batched = await self.async_client.create_presigned_urls(keys)+ if batched is not None and len(batched) == len(keys):+ return list(batched)++ LOG.warning(+ "Batch presign failed or returned wrong length; falling back to per-key signing",+ requested=len(keys),+ returned=(len(batched) if batched is not None else None),+ )++ async def _sign_one(uri: str) -> str | None:+ try:+ signed = await self.async_client.create_presigned_urls([uri])+ return signed[0] if signed else None+ except Exception:+ LOG.exception("Per-key presign failed", uri=uri)+ return None++ return await asyncio.gather(*[_sign_one(key) for key in keys])+async def _get_shared_downloaded_files_in_browser_session_via_listing(self, *, organization_id: str, browser_session_id: str) -> list[FileInfo]:@@ -386,34 +446,29 @@ class S3Storage(BaseStorage):if len(object_keys) == 0:return []+ object_infos, presigned_urls = await asyncio.gather(+ self._bounded_get_object_infos(object_keys),+ self._resilient_presign(object_keys),+ )+file_infos: list[FileInfo] = []- for key in object_keys:- metadata = {}- modified_at: datetime | None = None- # Get metadata (including checksum)- try:- object_info = await self.async_client.get_object_info(key)- metadata = object_info.get("Metadata", {})- modified_at = object_info.get("LastModified")- except Exception:- LOG.exception("Object info retrieval failed", uri=key)+ for key, object_info, url in zip(object_keys, object_infos, presigned_urls):+ if url is None:+ continue+ metadata = (object_info or {}).get("Metadata") or {}+ modified_at: datetime | None = (object_info or {}).get("LastModified")- # Create FileInfo objectfilename = os.path.basename(key)checksum = metadata.get("sha256_checksum") if metadata else None- # Get presigned URL- presigned_urls = await self.async_client.create_presigned_urls([key])- if not presigned_urls:- continue-- file_info = FileInfo(- url=presigned_urls[0],- checksum=checksum,- filename=metadata.get("original_filename", filename) if metadata else filename,- modified_at=modified_at,+ file_infos.append(+ FileInfo(+ url=url,+ checksum=checksum,+ filename=metadata.get("original_filename", filename) if metadata else filename,+ modified_at=modified_at,+ ))- file_infos.append(file_info)return file_infos@@ -488,56 +543,58 @@ class S3Storage(BaseStorage):# rows — every call here is a billable ListObjects request.bucket = settings.AWS_S3_BUCKET_ARTIFACTSlisting_uri = f"s3://{bucket}/{self._PATH_VERSION}/{settings.ENV}/{organization_id}/browser_sessions/{browser_session_id}/videos"- object_keys = [f"s3://{bucket}/{file}" for file in await self.async_client.list_files(uri=listing_uri)]- if len(object_keys) == 0:+ all_keys = [f"s3://{bucket}/{file}" for file in await self.async_client.list_files(uri=listing_uri)]+ if len(all_keys) == 0:return []- file_infos: list[FileInfo] = []- for key in object_keys:- # Playwright's record_video_dir should only contain .webm files.- # Filter defensively in case of unexpected files.+ # Playwright's record_video_dir should only contain .webm files; filter defensively.+ candidate_keys: list[str] = []+ for key in all_keys:key_lower = key.lower()- if not (key_lower.endswith(".webm") or key_lower.endswith(".mp4")):+ if key_lower.endswith(".webm") or key_lower.endswith(".mp4"):+ candidate_keys.append(key)+ else:LOG.warning("Skipping recording file with unsupported extension",uri=key,organization_id=organization_id,browser_session_id=browser_session_id,)+ if not candidate_keys:+ return []++ # Defer presign until zero-byte uploads are filtered out so we never sign discards.+ candidate_infos = await self._bounded_get_object_infos(candidate_keys)+ kept: list[tuple[str, dict | None]] = []+ for key, info in zip(candidate_keys, candidate_infos):+ if info is not None and info.get("ContentLength") == 0:continue+ kept.append((key, info))+ if not kept:+ return []- metadata = {}- modified_at: datetime | None = None- content_length: int | None = None- # Get metadata (including checksum)- try:- object_info = await self.async_client.get_object_info(key)- metadata = object_info.get("Metadata", {})- modified_at = object_info.get("LastModified")- content_length = object_info.get("ContentLength")- except Exception:- LOG.exception("Recording object info retrieval failed", uri=key)+ object_keys = [key for key, _ in kept]+ object_infos = [info for _, info in kept]+ presigned_urls = await self._resilient_presign(object_keys)- # Skip zero-byte objects (if any incompleted uploads)- if content_length == 0:+ file_infos: list[FileInfo] = []+ for key, object_info, url in zip(object_keys, object_infos, presigned_urls):+ if url is None:continue+ metadata = (object_info or {}).get("Metadata") or {}+ modified_at: datetime | None = (object_info or {}).get("LastModified")- # Create FileInfo objectfilename = os.path.basename(key)checksum = metadata.get("sha256_checksum") if metadata else None- # Get presigned URL- presigned_urls = await self.async_client.create_presigned_urls([key])- if not presigned_urls:- continue-- file_info = FileInfo(- url=presigned_urls[0],- checksum=checksum,- filename=metadata.get("original_filename", filename) if metadata else filename,- modified_at=modified_at,+ file_infos.append(+ FileInfo(+ url=url,+ checksum=checksum,+ filename=metadata.get("original_filename", filename) if metadata else filename,+ modified_at=modified_at,+ ))- file_infos.append(file_info)# Prefer the newest recording first (S3 list order is not guaranteed).# Treat None as "oldest".diff --git a/skyvern/forge/sdk/artifact/storage/test_s3_storage.py b/skyvern/forge/sdk/artifact/storage/test_s3_storage.pyindex 370986863..eb03963d1 100644--- a/skyvern/forge/sdk/artifact/storage/test_s3_storage.py+++ b/skyvern/forge/sdk/artifact/storage/test_s3_storage.py@@ -1,4 +1,6 @@+import asyncioimport io+import osfrom datetime import datetimefrom pathlib import Pathfrom typing import Generator@@ -462,6 +464,334 @@ class TestS3StorageBrowserSessionFiles:"""Test storage_type returns 's3'."""assert s3_storage.storage_type == "s3"+ async def test_get_shared_downloaded_files_returns_all(+ self,+ s3_storage: S3Storage,+ boto3_test_client: S3Client,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """Listing many downloaded files returns one FileInfo per object with a presigned URL."""+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)+ session_id = "bs_returns_all_test"+ for i in range(5):+ test_file = tmp_path / f"invoice_{i}.csv"+ test_file.write_bytes(f"row,{i}\n".encode())+ await s3_storage.sync_browser_session_file(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ artifact_type="downloads",+ local_file_path=str(test_file),+ remote_path=f"invoice_{i}.csv",+ date=None,+ )++ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ )+ assert len(file_infos) == 5+ assert {fi.filename for fi in file_infos} == {f"invoice_{i}.csv" for i in range(5)}+ for fi in file_infos:+ assert fi.url and "Signature=" in fi.url++ async def test_get_shared_downloaded_files_empty(+ self, s3_storage: S3Storage, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """Empty directory returns an empty list, no head_object/presign work."""+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)+ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id="bs_no_files",+ )+ assert file_infos == []++ async def test_get_shared_downloaded_files_runs_concurrently(+ self,+ s3_storage: S3Storage,+ boto3_test_client: S3Client,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """head_object calls overlap rather than running sequentially."""+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)+ session_id = "bs_concurrent_test"+ for i in range(8):+ test_file = tmp_path / f"f_{i}.csv"+ test_file.write_bytes(b"x")+ await s3_storage.sync_browser_session_file(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ artifact_type="downloads",+ local_file_path=str(test_file),+ remote_path=f"f_{i}.csv",+ date=None,+ )++ original = s3_storage.async_client.get_object_info+ per_call_delay = 0.1+ in_flight = 0+ max_in_flight = 0++ async def slow_get_object_info(uri: str) -> dict:+ nonlocal in_flight, max_in_flight+ in_flight += 1+ max_in_flight = max(max_in_flight, in_flight)+ try:+ await asyncio.sleep(per_call_delay)+ return await original(uri)+ finally:+ in_flight -= 1++ monkeypatch.setattr(s3_storage.async_client, "get_object_info", slow_get_object_info)++ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ )++ assert len(file_infos) == 8+ # max_in_flight > 1 directly proves head_object overlapped; wall-clock bounds flake under CI load.+ assert max_in_flight > 1, "expected overlapping head_object calls"++ async def test_get_shared_downloaded_files_caps_concurrency(+ self,+ s3_storage: S3Storage,+ boto3_test_client: S3Client,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """head_object fan-out is bounded by the instance-level semaphore."""+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)+ # Replace the instance semaphore with a tighter one so the cap is+ # hit with a small file count.+ monkeypatch.setattr(s3_storage, "_head_object_semaphore", asyncio.Semaphore(3))+ session_id = "bs_cap_test"+ file_count = 12+ for i in range(file_count):+ test_file = tmp_path / f"f_{i}.csv"+ test_file.write_bytes(b"x")+ await s3_storage.sync_browser_session_file(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ artifact_type="downloads",+ local_file_path=str(test_file),+ remote_path=f"f_{i}.csv",+ date=None,+ )++ original = s3_storage.async_client.get_object_info+ in_flight = 0+ max_in_flight = 0++ async def tracked_get_object_info(uri: str) -> dict:+ nonlocal in_flight, max_in_flight+ in_flight += 1+ max_in_flight = max(max_in_flight, in_flight)+ try:+ await asyncio.sleep(0.05)+ return await original(uri)+ finally:+ in_flight -= 1++ monkeypatch.setattr(s3_storage.async_client, "get_object_info", tracked_get_object_info)++ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ )++ assert len(file_infos) == file_count+ assert max_in_flight <= 3, f"head_object exceeded the cap (max_in_flight={max_in_flight})"++ async def test_head_concurrency_cap_is_shared_across_concurrent_calls(+ self,+ s3_storage: S3Storage,+ boto3_test_client: S3Client,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """Concurrent listing calls (the /browser_sessions/history fanout) share one cap."""+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)+ # Tight cap so the test can prove the limit holds across callers.+ monkeypatch.setattr(s3_storage, "_head_object_semaphore", asyncio.Semaphore(3))++ session_ids = ["bs_share_a", "bs_share_b", "bs_share_c"]+ for session_id in session_ids:+ for i in range(4):+ test_file = tmp_path / f"{session_id}_f_{i}.csv"+ test_file.write_bytes(b"x")+ await s3_storage.sync_browser_session_file(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ artifact_type="downloads",+ local_file_path=str(test_file),+ remote_path=f"f_{i}.csv",+ date=None,+ )++ original = s3_storage.async_client.get_object_info+ in_flight = 0+ max_in_flight = 0++ async def tracked_get_object_info(uri: str) -> dict:+ nonlocal in_flight, max_in_flight+ in_flight += 1+ max_in_flight = max(max_in_flight, in_flight)+ try:+ await asyncio.sleep(0.05)+ return await original(uri)+ finally:+ in_flight -= 1++ monkeypatch.setattr(s3_storage.async_client, "get_object_info", tracked_get_object_info)++ results = await asyncio.gather(+ *[+ s3_storage.get_shared_downloaded_files_in_browser_session(+ organization_id=TEST_ORGANIZATION_ID, browser_session_id=session_id+ )+ for session_id in session_ids+ ]+ )++ for file_infos in results:+ assert len(file_infos) == 4+ # Without a shared semaphore, 3 sessions x 4 files = 12 head_objects could overlap.+ assert max_in_flight <= 3, f"shared semaphore breached the cap (max_in_flight={max_in_flight})"++ async def test_get_shared_downloaded_files_falls_back_when_batch_presign_fails(+ self,+ s3_storage: S3Storage,+ boto3_test_client: S3Client,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """When batch presign returns None, fall back to per-key signing rather than dropping all files."""+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)+ session_id = "bs_batch_fail_test"+ for i in range(3):+ test_file = tmp_path / f"f_{i}.csv"+ test_file.write_bytes(b"x")+ await s3_storage.sync_browser_session_file(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ artifact_type="downloads",+ local_file_path=str(test_file),+ remote_path=f"f_{i}.csv",+ date=None,+ )++ original_create = s3_storage.async_client.create_presigned_urls++ async def flaky_create(uris: list[str]) -> list[str] | None:+ if len(uris) > 1:+ return None # simulate batch failure+ return await original_create(uris)++ monkeypatch.setattr(s3_storage.async_client, "create_presigned_urls", flaky_create)++ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ )+ # Per-key fallback should preserve all 3 URLs.+ assert len(file_infos) == 3+ assert all(fi.url and "Signature=" in fi.url for fi in file_infos)++ async def test_get_shared_downloaded_files_per_key_partial_failure(+ self,+ s3_storage: S3Storage,+ boto3_test_client: S3Client,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """Per-key fallback skips only the failing key, preserving the rest."""+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)+ session_id = "bs_partial_fail_test"+ for i in range(3):+ test_file = tmp_path / f"f_{i}.csv"+ test_file.write_bytes(b"x")+ await s3_storage.sync_browser_session_file(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ artifact_type="downloads",+ local_file_path=str(test_file),+ remote_path=f"f_{i}.csv",+ date=None,+ )++ original_create = s3_storage.async_client.create_presigned_urls++ async def flaky_create(uris: list[str]) -> list[str] | None:+ if len(uris) > 1:+ return None # force per-key fallback+ if uris[0].endswith("f_1.csv"):+ return None # one key fails+ return await original_create(uris)++ monkeypatch.setattr(s3_storage.async_client, "create_presigned_urls", flaky_create)++ file_infos = await s3_storage.get_shared_downloaded_files_in_browser_session(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ )+ names = {fi.filename for fi in file_infos}+ assert names == {"f_0.csv", "f_2.csv"}++ async def test_get_shared_recordings_skips_zero_byte_before_presign(+ self,+ s3_storage: S3Storage,+ boto3_test_client: S3Client,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """Zero-byte recordings are filtered before they consume a presign slot."""+ monkeypatch.setattr(settings, "AWS_S3_BUCKET_ARTIFACTS", TEST_BUCKET)+ # Force the legacy listing path (DB-first path is exercised in test_browser_session_recording_artifacts).+ monkeypatch.setattr(settings, "ARTIFACT_CONTENT_HMAC_KEYRING", None)+ session_id = "bs_zero_byte_test"++ good_file = tmp_path / "good.webm"+ good_file.write_bytes(b"\x1a\x45\xdf\xa3" + b"\x00" * 32)+ await s3_storage.sync_browser_session_file(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ artifact_type="videos",+ local_file_path=str(good_file),+ remote_path="good.webm",+ date="2025-01-15",+ )++ empty_file = tmp_path / "empty.webm"+ empty_file.write_bytes(b"")+ await s3_storage.sync_browser_session_file(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ artifact_type="videos",+ local_file_path=str(empty_file),+ remote_path="empty.webm",+ date="2025-01-15",+ )++ original_create = s3_storage.async_client.create_presigned_urls+ seen_keys: list[list[str]] = []++ async def tracking_create(uris: list[str]) -> list[str] | None:+ seen_keys.append(list(uris))+ return await original_create(uris)++ monkeypatch.setattr(s3_storage.async_client, "create_presigned_urls", tracking_create)++ file_infos = await s3_storage.get_shared_recordings_in_browser_session(+ organization_id=TEST_ORGANIZATION_ID,+ browser_session_id=session_id,+ )+ assert {fi.filename for fi in file_infos} == {"good.webm"}+ # The empty.webm key must never be passed to create_presigned_urls.+ passed_basenames = [os.path.basename(uri) for batch in seen_keys for uri in batch]+ assert "empty.webm" not in passed_basenames+CONTENT_TYPE_TEST_CASES = [# (filename, expected_content_type, artifact_type, date)