Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Flush the downloaded profile archive before unzipping it (#8063)
Celal Zamanoğlue17cf26c7ae4
diff --git a/skyvern/forge/sdk/api/files.py b/skyvern/forge/sdk/api/files.pyindex 893ec12fe..357a9b4d9 100644--- a/skyvern/forge/sdk/api/files.py+++ b/skyvern/forge/sdk/api/files.py@@ -386,6 +386,24 @@ def unzip_files(zip_file_path: str, output_dir: str) -> None:zip_ref.extractall(output_dir)+def unzip_bytes_to_temp_directory(zip_bytes: bytes, prefix: str) -> str:+ """Extract a downloaded archive into a fresh temp directory and return its path.++ The archive must be CLOSED before ZipFile reopens it by path: a write smaller than the io buffer+ (~8KB) is otherwise still unflushed, so ZipFile sees an empty file and raises BadZipFile.+ """+ temp_dir = make_temp_directory(prefix=prefix)+ with create_named_temporary_file(delete=False) as temp_zip_file:+ temp_zip_file.write(zip_bytes)+ temp_zip_file_path = temp_zip_file.name+ try:+ unzip_files(temp_zip_file_path, temp_dir)+ finally:+ # Cookie-bearing archives must not accumulate in TEMP_PATH on every retrieve.+ os.unlink(temp_zip_file_path)+ return temp_dir++_REMOTE_URL_PREFIXES = ("http://", "https://", "s3://", "gs://", "azure://", "www.")diff --git a/skyvern/forge/sdk/artifact/storage/azure.py b/skyvern/forge/sdk/artifact/storage/azure.pyindex 3d2a7cd1c..f454f2eef 100644--- a/skyvern/forge/sdk/artifact/storage/azure.py+++ b/skyvern/forge/sdk/artifact/storage/azure.py@@ -15,8 +15,7 @@ from skyvern.forge.sdk.api.files import (create_named_temporary_file,get_download_dir,get_skyvern_temp_dir,- make_temp_directory,- unzip_files,+ unzip_bytes_to_temp_directory,wait_for_pending_extension_rename,)from skyvern.forge.sdk.api.real_azure import RealAsyncAzureStorageClient@@ -223,14 +222,7 @@ class AzureStorage(BaseStorage):downloaded_zip_bytes = await self.async_client.download_file(browser_session_uri, log_exception=True)if not downloaded_zip_bytes:return None- temp_zip_file = create_named_temporary_file(delete=False)- temp_zip_file.write(downloaded_zip_bytes)- temp_zip_file_path = temp_zip_file.name-- temp_dir = make_temp_directory(prefix="skyvern_browser_session_")- unzip_files(temp_zip_file_path, temp_dir)- temp_zip_file.close()- return temp_dir+ return unzip_bytes_to_temp_directory(downloaded_zip_bytes, prefix="skyvern_browser_session_")async def delete_browser_session(self, organization_id: str, workflow_permanent_id: str) -> None:browser_session_uri = f"azure://{settings.AZURE_STORAGE_CONTAINER_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/{workflow_permanent_id}.zip"@@ -272,19 +264,7 @@ class AzureStorage(BaseStorage):downloaded_zip_bytes = await self.async_client.download_file(profile_uri, log_exception=True)if not downloaded_zip_bytes:return None- temp_zip_file = create_named_temporary_file(delete=False)- temp_zip_file.write(downloaded_zip_bytes)- temp_zip_file_path = temp_zip_file.name-- temp_dir = make_temp_directory(prefix="skyvern_browser_profile_")- try:- unzip_files(temp_zip_file_path, temp_dir)- finally:- # The downloaded archive is a delete=False temp file; remove it so cookie-bearing zips- # don't accumulate in TEMP_PATH on every profile retrieve (e.g. each cookie-only bank).- temp_zip_file.close()- os.unlink(temp_zip_file_path)- return temp_dir+ return unzip_bytes_to_temp_directory(downloaded_zip_bytes, prefix="skyvern_browser_profile_")async def browser_profile_exists(self, organization_id: str, profile_id: str) -> bool:"""Non-destructive existence check via blob metadata — avoids downloading the whole archive."""diff --git a/skyvern/forge/sdk/artifact/storage/gcs.py b/skyvern/forge/sdk/artifact/storage/gcs.pyindex 95c59ec76..719f41aa0 100644--- a/skyvern/forge/sdk/artifact/storage/gcs.py+++ b/skyvern/forge/sdk/artifact/storage/gcs.py@@ -15,8 +15,7 @@ from skyvern.forge.sdk.api.files import (create_named_temporary_file,get_download_dir,get_skyvern_temp_dir,- make_temp_directory,- unzip_files,+ unzip_bytes_to_temp_directory,wait_for_pending_extension_rename,)from skyvern.forge.sdk.api.gcp import STORAGE_CLASS_STANDARD, GcsUri@@ -233,14 +232,7 @@ class GcsStorage(BaseStorage):downloaded_zip_bytes = await self.async_client.download_file(browser_session_uri, log_exception=True)if not downloaded_zip_bytes:return None- temp_zip_file = create_named_temporary_file(delete=False)- temp_zip_file.write(downloaded_zip_bytes)- temp_zip_file_path = temp_zip_file.name-- temp_dir = make_temp_directory(prefix="skyvern_browser_session_")- unzip_files(temp_zip_file_path, temp_dir)- temp_zip_file.close()- return temp_dir+ return unzip_bytes_to_temp_directory(downloaded_zip_bytes, prefix="skyvern_browser_session_")async def delete_browser_session(self, organization_id: str, workflow_permanent_id: str) -> None:browser_session_uri = (@@ -290,19 +282,7 @@ class GcsStorage(BaseStorage):downloaded_zip_bytes = await self.async_client.download_file(profile_uri, log_exception=True)if not downloaded_zip_bytes:return None- temp_zip_file = create_named_temporary_file(delete=False)- temp_zip_file.write(downloaded_zip_bytes)- temp_zip_file_path = temp_zip_file.name-- temp_dir = make_temp_directory(prefix="skyvern_browser_profile_")- try:- unzip_files(temp_zip_file_path, temp_dir)- finally:- # The downloaded archive is a delete=False temp file; remove it so cookie-bearing zips- # don't accumulate in TEMP_PATH on every profile retrieve (e.g. each cookie-only bank).- temp_zip_file.close()- os.unlink(temp_zip_file_path)- return temp_dir+ return unzip_bytes_to_temp_directory(downloaded_zip_bytes, prefix="skyvern_browser_profile_")async def browser_profile_exists(self, organization_id: str, profile_id: str) -> bool:"""Non-destructive existence check via object metadata — avoids downloading the whole archive."""diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.pyindex 1b33cde9e..7fd09c70e 100644--- a/skyvern/forge/sdk/artifact/storage/s3.py+++ b/skyvern/forge/sdk/artifact/storage/s3.py@@ -20,8 +20,7 @@ from skyvern.forge.sdk.api.files import (create_named_temporary_file,get_download_dir,get_skyvern_temp_dir,- make_temp_directory,- unzip_files,+ unzip_bytes_to_temp_directory,wait_for_pending_extension_rename,)from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType@@ -285,14 +284,7 @@ class S3Storage(BaseStorage):downloaded_zip_bytes = await self.async_client.download_file(browser_session_uri, log_exception=True)if not downloaded_zip_bytes:return None- temp_zip_file = create_named_temporary_file(delete=False)- temp_zip_file.write(downloaded_zip_bytes)- temp_zip_file_path = temp_zip_file.name-- temp_dir = make_temp_directory(prefix="skyvern_browser_session_")- unzip_files(temp_zip_file_path, temp_dir)- temp_zip_file.close()- return temp_dir+ return unzip_bytes_to_temp_directory(downloaded_zip_bytes, prefix="skyvern_browser_session_")async def delete_browser_session(self, organization_id: str, workflow_permanent_id: str) -> None:browser_session_uri = f"s3://{settings.AWS_S3_BUCKET_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/{workflow_permanent_id}.zip"@@ -342,19 +334,7 @@ class S3Storage(BaseStorage):downloaded_zip_bytes = await self.async_client.download_file(profile_uri, log_exception=True)if not downloaded_zip_bytes:return None- temp_zip_file = create_named_temporary_file(delete=False)- temp_zip_file.write(downloaded_zip_bytes)- temp_zip_file_path = temp_zip_file.name-- temp_dir = make_temp_directory(prefix="skyvern_browser_profile_")- try:- unzip_files(temp_zip_file_path, temp_dir)- finally:- # The downloaded archive is a delete=False temp file; remove it so cookie-bearing zips- # don't accumulate in TEMP_PATH on every profile retrieve (e.g. each cookie-only bank).- temp_zip_file.close()- os.unlink(temp_zip_file_path)- return temp_dir+ return unzip_bytes_to_temp_directory(downloaded_zip_bytes, prefix="skyvern_browser_profile_")async def browser_profile_exists(self, organization_id: str, profile_id: str) -> bool:"""Cheap existence check (head_object) — avoids downloading the archive just to know it exists."""diff --git a/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py b/tests/unit/forge/sdk/artifact/storage/test_s3_storage.pyindex 65ae370c0..4ad9ad74d 100644--- a/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py+++ b/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py@@ -1248,6 +1248,29 @@ async def test_browser_profile_exists_propagates_non_not_found_errors() -> None:await storage.browser_profile_exists("o", "bp")+async def test_retrieve_browser_profile_extracts_sub_buffer_size_archive(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ # Regression: the archive used to be closed only AFTER unzipping, so anything smaller than the io+ # buffer (~8KB) — a cookie-only first bank — was still unflushed and ZipFile hit an empty file.+ zip_bytes = _build_zip({".skyvern_banked_cookies.json": b'[{"name":"session"}]'})+ assert len(zip_bytes) < 1024++ storage = S3Storage()+ storage.async_client = MagicMock()+ storage.async_client.download_file = AsyncMock(return_value=zip_bytes)++ monkeypatch.setattr(settings, "TEMP_PATH", str(tmp_path))+ profile_dir = await storage.retrieve_browser_profile("o", "bp")++ assert profile_dir is not None+ assert (Path(profile_dir) / ".skyvern_banked_cookies.json").read_bytes() == b'[{"name":"session"}]'+ # The extraction directory must be the ONLY thing left in TEMP_PATH — the downloaded archive is+ # written there under a suffixless temp name, so a leak would show up as a second entry.+ assert [entry.name for entry in tmp_path.iterdir()] == [Path(profile_dir).name]++@pytest.mark.asyncioasync def test_delete_browser_profile_hard_raises_soft_swallows() -> None:# hard_delete must PROPAGATE an S3 failure (raise_on_error=True) so the reap can't falsely report a