Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-10702): cap browser-session S3 growth via deterministic reaps (#6615)
Celal Zamanoğlu974dbc4a3b72
diff --git a/skyvern/forge/sdk/artifact/storage/azure.py b/skyvern/forge/sdk/artifact/storage/azure.pyindex 696598e74..0bdaf0e04 100644--- a/skyvern/forge/sdk/artifact/storage/azure.py+++ b/skyvern/forge/sdk/artifact/storage/azure.py@@ -257,6 +257,17 @@ class AzureStorage(BaseStorage):temp_zip_file.close()return temp_dir+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:+ """Delete a browser profile from Azure."""+ profile_uri = f"azure://{settings.AZURE_STORAGE_CONTAINER_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/profiles/{profile_id}.zip"+ LOG.info(+ "Deleting browser profile",+ organization_id=organization_id,+ profile_id=profile_id,+ profile_uri=profile_uri,+ )+ await self.async_client.delete_file(profile_uri)+async def list_downloaded_files_in_browser_session(self, organization_id: str, browser_session_id: str) -> list[str]:diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.pyindex 194767a49..df115f2ea 100644--- a/skyvern/forge/sdk/artifact/storage/base.py+++ b/skyvern/forge/sdk/artifact/storage/base.py@@ -200,6 +200,10 @@ class BaseStorage(ABC):async def retrieve_browser_profile(self, organization_id: str, profile_id: str) -> str | None:"""Retrieve a browser profile to a temporary directory."""+ @abstractmethod+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:+ """Delete a stored browser profile. Best-effort: a missing object is not an error."""+@abstractmethodasync def list_downloaded_files_in_browser_session(self, organization_id: str, browser_session_id: strdiff --git a/skyvern/forge/sdk/artifact/storage/gcs.py b/skyvern/forge/sdk/artifact/storage/gcs.pyindex 8ddf3ceba..4294e965d 100644--- a/skyvern/forge/sdk/artifact/storage/gcs.py+++ b/skyvern/forge/sdk/artifact/storage/gcs.py@@ -275,6 +275,19 @@ class GcsStorage(BaseStorage):temp_zip_file.close()return temp_dir+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:+ """Delete a browser profile from GCS."""+ profile_uri = (+ f"gs://{settings.GCS_BUCKET_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/profiles/{profile_id}.zip"+ )+ LOG.info(+ "Deleting browser profile",+ organization_id=organization_id,+ profile_id=profile_id,+ profile_uri=profile_uri,+ )+ await self.async_client.delete_file(profile_uri)+async def list_downloaded_files_in_browser_session(self, organization_id: str, browser_session_id: str) -> list[str]:diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.pyindex c7630986d..aa0db2ec1 100644--- a/skyvern/forge/sdk/artifact/storage/local.py+++ b/skyvern/forge/sdk/artifact/storage/local.py@@ -323,6 +323,29 @@ class LocalStorage(BaseStorage):return Nonereturn str(stored_folder_path)+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:+ """Delete a browser profile from local storage. Best-effort: a missing profile is a no-op."""+ stored_folder_path = self._resolve_browser_storage_path(organization_id, "profiles", profile_id)+ if stored_folder_path is None:+ LOG.warning(+ "Refused to delete browser profile outside storage base path",+ organization_id=organization_id,+ profile_id=profile_id,+ base_path=settings.BROWSER_SESSION_BASE_PATH,+ )+ return+ if not stored_folder_path.exists():+ return+ try:+ shutil.rmtree(stored_folder_path)+ except Exception:+ LOG.exception(+ "Failed to delete local browser profile",+ organization_id=organization_id,+ profile_id=profile_id,+ path=str(stored_folder_path),+ )+async def save_downloaded_files(self, organization_id: str, run_id: str | None) -> None:passdiff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.pyindex 25db431cf..2c0f8ca7e 100644--- a/skyvern/forge/sdk/artifact/storage/s3.py+++ b/skyvern/forge/sdk/artifact/storage/s3.py@@ -330,6 +330,21 @@ class S3Storage(BaseStorage):temp_zip_file.close()return temp_dir+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:+ """Delete a browser profile from S3."""+ profile_uri = (+ f"s3://{settings.AWS_S3_BUCKET_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/profiles/{profile_id}.zip"+ )+ LOG.info(+ "Deleting browser profile",+ organization_id=organization_id,+ profile_id=profile_id,+ profile_uri=profile_uri,+ )+ # DeleteObject is idempotent: deleting a missing key is a no-op. Best-effort — log and+ # swallow so a reap failure never breaks the promote or the soft-delete that triggered it.+ await self.async_client.delete_file(profile_uri, log_exception=True)+async def list_downloaded_files_in_browser_session(self, organization_id: str, browser_session_id: str) -> list[str]:diff --git a/skyvern/forge/sdk/routes/browser_profiles.py b/skyvern/forge/sdk/routes/browser_profiles.pyindex dca19fda7..34928ed14 100644--- a/skyvern/forge/sdk/routes/browser_profiles.py+++ b/skyvern/forge/sdk/routes/browser_profiles.py@@ -372,6 +372,20 @@ async def delete_browser_profile()raise+ # Reap the stored blob so soft-deleted profiles don't leave orphaned S3 objects behind.+ # Best-effort: the soft-delete already succeeded, so a reap failure must not fail the request.+ try:+ await app.STORAGE.delete_browser_profile(+ organization_id=organization_id,+ profile_id=profile_id,+ )+ except Exception:+ LOG.exception(+ "Failed to delete browser profile blob after soft-delete",+ organization_id=organization_id,+ browser_profile_id=profile_id,+ )+LOG.info("Deleted browser profile",organization_id=organization_id,@@ -488,6 +502,22 @@ async def _create_profile_from_session()raise+ # The promote copied the session's own export (profiles/{pbs_session}.zip) into the new bp_, so+ # reap that source now that it's redundant. Keyed on the session id, never a reused bp_ profile.+ # Best-effort: only after a successful promote, and a reap failure must not fail the request.+ try:+ await app.STORAGE.delete_browser_profile(+ organization_id=organization_id,+ profile_id=browser_session_id,+ )+ except Exception:+ LOG.exception(+ "Failed to delete source session profile blob after promote",+ organization_id=organization_id,+ browser_session_id=browser_session_id,+ browser_profile_id=profile.browser_profile_id,+ )+LOG.info("Created browser profile from session",organization_id=organization_id,diff --git a/tests/unit/forge/sdk/test_create_browser_profile_from_session_route.py b/tests/unit/forge/sdk/test_create_browser_profile_from_session_route.pynew file mode 100644index 000000000..44ad73faa--- /dev/null+++ b/tests/unit/forge/sdk/test_create_browser_profile_from_session_route.py@@ -0,0 +1,122 @@+"""Tests for POST /browser_profiles with a browser_session_id (create-from-session + source reap)."""++from __future__ import annotations++from datetime import datetime+from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest+from fastapi import FastAPI, Request+from fastapi.responses import JSONResponse+from fastapi.testclient import TestClient++from skyvern.exceptions import SkyvernHTTPException+from skyvern.forge import app as forge_app+from skyvern.forge.sdk.routes import routers as routers_module+from skyvern.forge.sdk.schemas.browser_profiles import BrowserProfile+from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession+from skyvern.forge.sdk.services import org_auth_service+++def _session(**kwargs: object) -> PersistentBrowserSession:+ base: dict[str, object] = {+ "persistent_browser_session_id": "pbs_1",+ "organization_id": "org_oss",+ "created_at": datetime(2026, 1, 1),+ "modified_at": datetime(2026, 1, 1),+ }+ base.update(kwargs)+ return PersistentBrowserSession(**base)+++def _profile() -> BrowserProfile:+ return BrowserProfile(+ browser_profile_id="bp_new",+ organization_id="org_oss",+ name="my profile",+ created_at=datetime(2026, 1, 1),+ modified_at=datetime(2026, 1, 1),+ )+++def _build_client(monkeypatch: pytest.MonkeyPatch) -> tuple[TestClient, SimpleNamespace]:+ async def _fake_org() -> SimpleNamespace:+ return SimpleNamespace(organization_id="org_oss")++ mocks = SimpleNamespace(+ get_session=AsyncMock(),+ create_profile=AsyncMock(return_value=_profile()),+ delete_db_profile=AsyncMock(),+ retrieve_profile=AsyncMock(),+ store_profile=AsyncMock(),+ delete_profile_blob=AsyncMock(),+ )+ monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "get_persistent_browser_session", mocks.get_session)+ monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "create_browser_profile", mocks.create_profile)+ monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "delete_browser_profile", mocks.delete_db_profile)+ monkeypatch.setattr(forge_app.STORAGE, "retrieve_browser_profile", mocks.retrieve_profile)+ monkeypatch.setattr(forge_app.STORAGE, "store_browser_profile", mocks.store_profile)+ monkeypatch.setattr(forge_app.STORAGE, "delete_browser_profile", mocks.delete_profile_blob)++ fastapi_app = FastAPI()+ fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org+ fastapi_app.include_router(routers_module.base_router, prefix="/v1")++ @fastapi_app.exception_handler(SkyvernHTTPException)+ async def _handle(request: Request, exc: SkyvernHTTPException) -> JSONResponse:+ return JSONResponse(status_code=exc.status_code, content={"detail": exc.message})++ # raise_server_exceptions=False so an un-mapped error surfaces as a 500 response (as in prod)+ # rather than propagating out of the TestClient — the failed-promote path re-raises after rollback.+ return TestClient(fastapi_app, raise_server_exceptions=False), mocks+++def test_promote_deletes_source_session_blob(monkeypatch: pytest.MonkeyPatch) -> None:+ client, mocks = _build_client(monkeypatch)+ mocks.get_session.return_value = _session(generate_browser_profile=True)+ mocks.retrieve_profile.return_value = "/tmp/session_dir"++ response = client.post("/v1/browser_profiles/", json={"name": "my profile", "browser_session_id": "pbs_1"})++ assert response.status_code == 200+ mocks.store_profile.assert_awaited_once()+ mocks.delete_profile_blob.assert_awaited_once_with(organization_id="org_oss", profile_id="pbs_1")+++def test_failed_promote_does_not_delete_source_blob(monkeypatch: pytest.MonkeyPatch) -> None:+ client, mocks = _build_client(monkeypatch)+ mocks.get_session.return_value = _session(generate_browser_profile=True)+ mocks.retrieve_profile.return_value = "/tmp/session_dir"+ mocks.store_profile.side_effect = RuntimeError("s3 upload failed")++ response = client.post("/v1/browser_profiles/", json={"name": "my profile", "browser_session_id": "pbs_1"})++ assert response.status_code == 500+ mocks.delete_db_profile.assert_awaited_once() # rolled back the half-created profile+ mocks.delete_profile_blob.assert_not_awaited() # never reap the source on a failed promote+++def test_promote_succeeds_even_if_source_reap_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ client, mocks = _build_client(monkeypatch)+ mocks.get_session.return_value = _session(generate_browser_profile=True)+ mocks.retrieve_profile.return_value = "/tmp/session_dir"+ mocks.delete_profile_blob.side_effect = RuntimeError("s3 AccessDenied")++ response = client.post("/v1/browser_profiles/", json={"name": "my profile", "browser_session_id": "pbs_1"})++ assert response.status_code == 200+ mocks.delete_profile_blob.assert_awaited_once()+++def test_non_opted_in_session_returns_400(monkeypatch: pytest.MonkeyPatch) -> None:+ client, mocks = _build_client(monkeypatch)+ mocks.get_session.return_value = _session(generate_browser_profile=False)+ mocks.retrieve_profile.return_value = None++ response = client.post("/v1/browser_profiles/", json={"name": "my profile", "browser_session_id": "pbs_1"})++ assert response.status_code == 400+ assert "not configured to generate a browser profile" in response.json()["detail"]+ mocks.store_profile.assert_not_awaited()+ mocks.delete_profile_blob.assert_not_awaited()diff --git a/tests/unit/forge/sdk/test_delete_browser_profile_route.py b/tests/unit/forge/sdk/test_delete_browser_profile_route.pynew file mode 100644index 000000000..aef68256b--- /dev/null+++ b/tests/unit/forge/sdk/test_delete_browser_profile_route.py@@ -0,0 +1,68 @@+"""Tests for DELETE /browser_profiles/{profile_id} (soft-delete + S3 blob reap)."""++from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest+from fastapi import FastAPI, Request+from fastapi.responses import JSONResponse+from fastapi.testclient import TestClient++from skyvern.exceptions import BrowserProfileNotFound, SkyvernHTTPException+from skyvern.forge import app as forge_app+from skyvern.forge.sdk.routes import routers as routers_module+from skyvern.forge.sdk.services import org_auth_service+++def _build_client(monkeypatch: pytest.MonkeyPatch) -> tuple[TestClient, SimpleNamespace]:+ async def _fake_org() -> SimpleNamespace:+ return SimpleNamespace(organization_id="org_oss")++ mocks = SimpleNamespace(+ delete_db_profile=AsyncMock(),+ delete_profile_blob=AsyncMock(),+ )+ monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "delete_browser_profile", mocks.delete_db_profile)+ monkeypatch.setattr(forge_app.STORAGE, "delete_browser_profile", mocks.delete_profile_blob)++ fastapi_app = FastAPI()+ fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org+ fastapi_app.include_router(routers_module.base_router, prefix="/v1")++ @fastapi_app.exception_handler(SkyvernHTTPException)+ async def _handle(request: Request, exc: SkyvernHTTPException) -> JSONResponse:+ return JSONResponse(status_code=exc.status_code, content={"detail": exc.message})++ return TestClient(fastapi_app), mocks+++def test_delete_reaps_profile_blob(monkeypatch: pytest.MonkeyPatch) -> None:+ client, mocks = _build_client(monkeypatch)++ response = client.delete("/v1/browser_profiles/bp_123/")++ assert response.status_code == 204+ mocks.delete_db_profile.assert_awaited_once_with(profile_id="bp_123", organization_id="org_oss")+ mocks.delete_profile_blob.assert_awaited_once_with(organization_id="org_oss", profile_id="bp_123")+++def test_delete_succeeds_when_blob_already_gone(monkeypatch: pytest.MonkeyPatch) -> None:+ client, mocks = _build_client(monkeypatch)+ mocks.delete_profile_blob.side_effect = RuntimeError("object not found / s3 error")++ response = client.delete("/v1/browser_profiles/bp_123/")++ assert response.status_code == 204+ mocks.delete_profile_blob.assert_awaited_once()+++def test_delete_missing_profile_does_not_reap_blob(monkeypatch: pytest.MonkeyPatch) -> None:+ client, mocks = _build_client(monkeypatch)+ mocks.delete_db_profile.side_effect = BrowserProfileNotFound(profile_id="bp_missing")++ response = client.delete("/v1/browser_profiles/bp_missing/")++ assert response.status_code == 404+ mocks.delete_profile_blob.assert_not_awaited()