Skip to content

Automate browser based workflows with AI

feat(SKY-10702): cap browser-session S3 growth via deterministic reaps (#6615)

Celal Zamanoğlu974dbc4a3b72
diff --git a/skyvern/forge/sdk/artifact/storage/azure.py b/skyvern/forge/sdk/artifact/storage/azure.py
index 696598e74..0bdaf0e04 100644
--- a/skyvern/forge/sdk/artifact/storage/azure.py
+++ b/skyvern/forge/sdk/artifact/storage/azure.py
@@ -257,6 +257,17 @@ class AzureStorage(BaseStorage):
temp_zip_file.close()
return temp_dir
+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:
+ """Delete a browser profile from Azure."""
+ profile_uri = f"azure://{settings.AZURE_STORAGE_CONTAINER_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/profiles/{profile_id}.zip"
+ LOG.info(
+ "Deleting browser profile",
+ organization_id=organization_id,
+ profile_id=profile_id,
+ profile_uri=profile_uri,
+ )
+ await self.async_client.delete_file(profile_uri)
+
async def list_downloaded_files_in_browser_session(
self, organization_id: str, browser_session_id: str
) -> list[str]:
diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.py
index 194767a49..df115f2ea 100644
--- a/skyvern/forge/sdk/artifact/storage/base.py
+++ b/skyvern/forge/sdk/artifact/storage/base.py
@@ -200,6 +200,10 @@ class BaseStorage(ABC):
async def retrieve_browser_profile(self, organization_id: str, profile_id: str) -> str | None:
"""Retrieve a browser profile to a temporary directory."""
+ @abstractmethod
+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:
+ """Delete a stored browser profile. Best-effort: a missing object is not an error."""
+
@abstractmethod
async def list_downloaded_files_in_browser_session(
self, organization_id: str, browser_session_id: str
diff --git a/skyvern/forge/sdk/artifact/storage/gcs.py b/skyvern/forge/sdk/artifact/storage/gcs.py
index 8ddf3ceba..4294e965d 100644
--- a/skyvern/forge/sdk/artifact/storage/gcs.py
+++ b/skyvern/forge/sdk/artifact/storage/gcs.py
@@ -275,6 +275,19 @@ class GcsStorage(BaseStorage):
temp_zip_file.close()
return temp_dir
+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:
+ """Delete a browser profile from GCS."""
+ profile_uri = (
+ f"gs://{settings.GCS_BUCKET_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/profiles/{profile_id}.zip"
+ )
+ LOG.info(
+ "Deleting browser profile",
+ organization_id=organization_id,
+ profile_id=profile_id,
+ profile_uri=profile_uri,
+ )
+ await self.async_client.delete_file(profile_uri)
+
async def list_downloaded_files_in_browser_session(
self, organization_id: str, browser_session_id: str
) -> list[str]:
diff --git a/skyvern/forge/sdk/artifact/storage/local.py b/skyvern/forge/sdk/artifact/storage/local.py
index c7630986d..aa0db2ec1 100644
--- a/skyvern/forge/sdk/artifact/storage/local.py
+++ b/skyvern/forge/sdk/artifact/storage/local.py
@@ -323,6 +323,29 @@ class LocalStorage(BaseStorage):
return None
return str(stored_folder_path)
+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:
+ """Delete a browser profile from local storage. Best-effort: a missing profile is a no-op."""
+ stored_folder_path = self._resolve_browser_storage_path(organization_id, "profiles", profile_id)
+ if stored_folder_path is None:
+ LOG.warning(
+ "Refused to delete browser profile outside storage base path",
+ organization_id=organization_id,
+ profile_id=profile_id,
+ base_path=settings.BROWSER_SESSION_BASE_PATH,
+ )
+ return
+ if not stored_folder_path.exists():
+ return
+ try:
+ shutil.rmtree(stored_folder_path)
+ except Exception:
+ LOG.exception(
+ "Failed to delete local browser profile",
+ organization_id=organization_id,
+ profile_id=profile_id,
+ path=str(stored_folder_path),
+ )
+
async def save_downloaded_files(self, organization_id: str, run_id: str | None) -> None:
pass
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
index 25db431cf..2c0f8ca7e 100644
--- a/skyvern/forge/sdk/artifact/storage/s3.py
+++ b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -330,6 +330,21 @@ class S3Storage(BaseStorage):
temp_zip_file.close()
return temp_dir
+ async def delete_browser_profile(self, organization_id: str, profile_id: str) -> None:
+ """Delete a browser profile from S3."""
+ profile_uri = (
+ f"s3://{settings.AWS_S3_BUCKET_BROWSER_SESSIONS}/{settings.ENV}/{organization_id}/profiles/{profile_id}.zip"
+ )
+ LOG.info(
+ "Deleting browser profile",
+ organization_id=organization_id,
+ profile_id=profile_id,
+ profile_uri=profile_uri,
+ )
+ # DeleteObject is idempotent: deleting a missing key is a no-op. Best-effort — log and
+ # swallow so a reap failure never breaks the promote or the soft-delete that triggered it.
+ await self.async_client.delete_file(profile_uri, log_exception=True)
+
async def list_downloaded_files_in_browser_session(
self, organization_id: str, browser_session_id: str
) -> list[str]:
diff --git a/skyvern/forge/sdk/routes/browser_profiles.py b/skyvern/forge/sdk/routes/browser_profiles.py
index dca19fda7..34928ed14 100644
--- a/skyvern/forge/sdk/routes/browser_profiles.py
+++ b/skyvern/forge/sdk/routes/browser_profiles.py
@@ -372,6 +372,20 @@ async def delete_browser_profile(
)
raise
+ # Reap the stored blob so soft-deleted profiles don't leave orphaned S3 objects behind.
+ # Best-effort: the soft-delete already succeeded, so a reap failure must not fail the request.
+ try:
+ await app.STORAGE.delete_browser_profile(
+ organization_id=organization_id,
+ profile_id=profile_id,
+ )
+ except Exception:
+ LOG.exception(
+ "Failed to delete browser profile blob after soft-delete",
+ organization_id=organization_id,
+ browser_profile_id=profile_id,
+ )
+
LOG.info(
"Deleted browser profile",
organization_id=organization_id,
@@ -488,6 +502,22 @@ async def _create_profile_from_session(
)
raise
+ # The promote copied the session's own export (profiles/{pbs_session}.zip) into the new bp_, so
+ # reap that source now that it's redundant. Keyed on the session id, never a reused bp_ profile.
+ # Best-effort: only after a successful promote, and a reap failure must not fail the request.
+ try:
+ await app.STORAGE.delete_browser_profile(
+ organization_id=organization_id,
+ profile_id=browser_session_id,
+ )
+ except Exception:
+ LOG.exception(
+ "Failed to delete source session profile blob after promote",
+ organization_id=organization_id,
+ browser_session_id=browser_session_id,
+ browser_profile_id=profile.browser_profile_id,
+ )
+
LOG.info(
"Created browser profile from session",
organization_id=organization_id,
diff --git a/tests/unit/forge/sdk/test_create_browser_profile_from_session_route.py b/tests/unit/forge/sdk/test_create_browser_profile_from_session_route.py
new file mode 100644
index 000000000..44ad73faa
--- /dev/null
+++ b/tests/unit/forge/sdk/test_create_browser_profile_from_session_route.py
@@ -0,0 +1,122 @@
+"""Tests for POST /browser_profiles with a browser_session_id (create-from-session + source reap)."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from fastapi.testclient import TestClient
+
+from skyvern.exceptions import SkyvernHTTPException
+from skyvern.forge import app as forge_app
+from skyvern.forge.sdk.routes import routers as routers_module
+from skyvern.forge.sdk.schemas.browser_profiles import BrowserProfile
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
+from skyvern.forge.sdk.services import org_auth_service
+
+
+def _session(**kwargs: object) -> PersistentBrowserSession:
+ base: dict[str, object] = {
+ "persistent_browser_session_id": "pbs_1",
+ "organization_id": "org_oss",
+ "created_at": datetime(2026, 1, 1),
+ "modified_at": datetime(2026, 1, 1),
+ }
+ base.update(kwargs)
+ return PersistentBrowserSession(**base)
+
+
+def _profile() -> BrowserProfile:
+ return BrowserProfile(
+ browser_profile_id="bp_new",
+ organization_id="org_oss",
+ name="my profile",
+ created_at=datetime(2026, 1, 1),
+ modified_at=datetime(2026, 1, 1),
+ )
+
+
+def _build_client(monkeypatch: pytest.MonkeyPatch) -> tuple[TestClient, SimpleNamespace]:
+ async def _fake_org() -> SimpleNamespace:
+ return SimpleNamespace(organization_id="org_oss")
+
+ mocks = SimpleNamespace(
+ get_session=AsyncMock(),
+ create_profile=AsyncMock(return_value=_profile()),
+ delete_db_profile=AsyncMock(),
+ retrieve_profile=AsyncMock(),
+ store_profile=AsyncMock(),
+ delete_profile_blob=AsyncMock(),
+ )
+ monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "get_persistent_browser_session", mocks.get_session)
+ monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "create_browser_profile", mocks.create_profile)
+ monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "delete_browser_profile", mocks.delete_db_profile)
+ monkeypatch.setattr(forge_app.STORAGE, "retrieve_browser_profile", mocks.retrieve_profile)
+ monkeypatch.setattr(forge_app.STORAGE, "store_browser_profile", mocks.store_profile)
+ monkeypatch.setattr(forge_app.STORAGE, "delete_browser_profile", mocks.delete_profile_blob)
+
+ fastapi_app = FastAPI()
+ fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org
+ fastapi_app.include_router(routers_module.base_router, prefix="/v1")
+
+ @fastapi_app.exception_handler(SkyvernHTTPException)
+ async def _handle(request: Request, exc: SkyvernHTTPException) -> JSONResponse:
+ return JSONResponse(status_code=exc.status_code, content={"detail": exc.message})
+
+ # raise_server_exceptions=False so an un-mapped error surfaces as a 500 response (as in prod)
+ # rather than propagating out of the TestClient — the failed-promote path re-raises after rollback.
+ return TestClient(fastapi_app, raise_server_exceptions=False), mocks
+
+
+def test_promote_deletes_source_session_blob(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, mocks = _build_client(monkeypatch)
+ mocks.get_session.return_value = _session(generate_browser_profile=True)
+ mocks.retrieve_profile.return_value = "/tmp/session_dir"
+
+ response = client.post("/v1/browser_profiles/", json={"name": "my profile", "browser_session_id": "pbs_1"})
+
+ assert response.status_code == 200
+ mocks.store_profile.assert_awaited_once()
+ mocks.delete_profile_blob.assert_awaited_once_with(organization_id="org_oss", profile_id="pbs_1")
+
+
+def test_failed_promote_does_not_delete_source_blob(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, mocks = _build_client(monkeypatch)
+ mocks.get_session.return_value = _session(generate_browser_profile=True)
+ mocks.retrieve_profile.return_value = "/tmp/session_dir"
+ mocks.store_profile.side_effect = RuntimeError("s3 upload failed")
+
+ response = client.post("/v1/browser_profiles/", json={"name": "my profile", "browser_session_id": "pbs_1"})
+
+ assert response.status_code == 500
+ mocks.delete_db_profile.assert_awaited_once() # rolled back the half-created profile
+ mocks.delete_profile_blob.assert_not_awaited() # never reap the source on a failed promote
+
+
+def test_promote_succeeds_even_if_source_reap_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, mocks = _build_client(monkeypatch)
+ mocks.get_session.return_value = _session(generate_browser_profile=True)
+ mocks.retrieve_profile.return_value = "/tmp/session_dir"
+ mocks.delete_profile_blob.side_effect = RuntimeError("s3 AccessDenied")
+
+ response = client.post("/v1/browser_profiles/", json={"name": "my profile", "browser_session_id": "pbs_1"})
+
+ assert response.status_code == 200
+ mocks.delete_profile_blob.assert_awaited_once()
+
+
+def test_non_opted_in_session_returns_400(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, mocks = _build_client(monkeypatch)
+ mocks.get_session.return_value = _session(generate_browser_profile=False)
+ mocks.retrieve_profile.return_value = None
+
+ response = client.post("/v1/browser_profiles/", json={"name": "my profile", "browser_session_id": "pbs_1"})
+
+ assert response.status_code == 400
+ assert "not configured to generate a browser profile" in response.json()["detail"]
+ mocks.store_profile.assert_not_awaited()
+ mocks.delete_profile_blob.assert_not_awaited()
diff --git a/tests/unit/forge/sdk/test_delete_browser_profile_route.py b/tests/unit/forge/sdk/test_delete_browser_profile_route.py
new file mode 100644
index 000000000..aef68256b
--- /dev/null
+++ b/tests/unit/forge/sdk/test_delete_browser_profile_route.py
@@ -0,0 +1,68 @@
+"""Tests for DELETE /browser_profiles/{profile_id} (soft-delete + S3 blob reap)."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from fastapi.testclient import TestClient
+
+from skyvern.exceptions import BrowserProfileNotFound, SkyvernHTTPException
+from skyvern.forge import app as forge_app
+from skyvern.forge.sdk.routes import routers as routers_module
+from skyvern.forge.sdk.services import org_auth_service
+
+
+def _build_client(monkeypatch: pytest.MonkeyPatch) -> tuple[TestClient, SimpleNamespace]:
+ async def _fake_org() -> SimpleNamespace:
+ return SimpleNamespace(organization_id="org_oss")
+
+ mocks = SimpleNamespace(
+ delete_db_profile=AsyncMock(),
+ delete_profile_blob=AsyncMock(),
+ )
+ monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "delete_browser_profile", mocks.delete_db_profile)
+ monkeypatch.setattr(forge_app.STORAGE, "delete_browser_profile", mocks.delete_profile_blob)
+
+ fastapi_app = FastAPI()
+ fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org
+ fastapi_app.include_router(routers_module.base_router, prefix="/v1")
+
+ @fastapi_app.exception_handler(SkyvernHTTPException)
+ async def _handle(request: Request, exc: SkyvernHTTPException) -> JSONResponse:
+ return JSONResponse(status_code=exc.status_code, content={"detail": exc.message})
+
+ return TestClient(fastapi_app), mocks
+
+
+def test_delete_reaps_profile_blob(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, mocks = _build_client(monkeypatch)
+
+ response = client.delete("/v1/browser_profiles/bp_123/")
+
+ assert response.status_code == 204
+ mocks.delete_db_profile.assert_awaited_once_with(profile_id="bp_123", organization_id="org_oss")
+ mocks.delete_profile_blob.assert_awaited_once_with(organization_id="org_oss", profile_id="bp_123")
+
+
+def test_delete_succeeds_when_blob_already_gone(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, mocks = _build_client(monkeypatch)
+ mocks.delete_profile_blob.side_effect = RuntimeError("object not found / s3 error")
+
+ response = client.delete("/v1/browser_profiles/bp_123/")
+
+ assert response.status_code == 204
+ mocks.delete_profile_blob.assert_awaited_once()
+
+
+def test_delete_missing_profile_does_not_reap_blob(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, mocks = _build_client(monkeypatch)
+ mocks.delete_db_profile.side_effect = BrowserProfileNotFound(profile_id="bp_missing")
+
+ response = client.delete("/v1/browser_profiles/bp_missing/")
+
+ assert response.status_code == 404
+ mocks.delete_profile_blob.assert_not_awaited()