Skip to content

Automate browser based workflows with AI

SKY-13097 Apply billing + rate-limit guards to task endpoints (#7984)

Aaron Perezb600f4a4db5d
diff --git a/skyvern/config.py b/skyvern/config.py
index 7cc34a969..1d530497c 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -287,6 +287,11 @@ class Settings(BaseSettings):
AWS_S3_BUCKET_SCREENSHOTS: str = "skyvern-screenshots"
AWS_S3_BUCKET_BROWSER_SESSIONS: str = "skyvern-browser-sessions"
AWS_S3_BUCKET_UPLOADS: str = "skyvern-uploads"
+ # ISO-8601 UTC timestamp. Runs created at/after it that have zero DOWNLOAD artifact
+ # rows skip the legacy S3 LIST fallback in get_downloaded_files — such runs register
+ # every download as a row at save time (SKY-8861), so the LIST can only return empty.
+ # None keeps the LIST fallback for every run.
+ DOWNLOADS_EMPTY_S3_LISTING_CUTOVER: str | None = None
# Azure Blob Storage settings
AZURE_STORAGE_ACCOUNT_NAME: str | None = None
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
index 930422c20..873656b85 100644
--- a/skyvern/forge/sdk/artifact/storage/s3.py
+++ b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -826,11 +826,43 @@ class S3Storage(BaseStorage):
artifacts = await self._list_download_artifacts_safe(organization_id=organization_id, run_id=run_id)
if artifacts:
return await _file_infos_from_download_artifacts(artifacts)
+ if await self._skip_empty_downloads_listing(organization_id=organization_id, run_id=run_id):
+ return []
# Legacy fallback — runs predating SKY-8861 (no artifact rows) and
# OSS-default deployments without HMAC signing both arrive here.
return await self._get_downloaded_files_via_s3_listing(organization_id=organization_id, run_id=run_id)
+ async def _skip_empty_downloads_listing(self, *, organization_id: str, run_id: str) -> bool:
+ """True when a run with zero DOWNLOAD rows may skip the legacy S3 LIST.
+
+ Only runs created at/after DOWNLOADS_EMPTY_S3_LISTING_CUTOVER qualify: they
+ register every download as an artifact row at save time, so an empty row set
+ means an empty S3 prefix. Anything unresolvable (cutover unset or unparseable,
+ run row missing, DB error) keeps the LIST fallback.
+ """
+ cutover_raw = settings.DOWNLOADS_EMPTY_S3_LISTING_CUTOVER
+ if not cutover_raw:
+ return False
+ try:
+ cutover = datetime.fromisoformat(cutover_raw)
+ run = await app.DATABASE.tasks.get_run(run_id=run_id, organization_id=organization_id)
+ except Exception:
+ LOG.warning(
+ "Failed to resolve run for empty-downloads listing skip; using S3 LIST",
+ run_id=run_id,
+ exc_info=True,
+ )
+ return False
+ if run is None:
+ return False
+ created_at = run.created_at
+ if created_at.tzinfo is None:
+ created_at = created_at.replace(tzinfo=timezone.utc)
+ if cutover.tzinfo is None:
+ cutover = cutover.replace(tzinfo=timezone.utc)
+ return created_at >= cutover
+
async def _list_download_artifacts_safe(self, *, organization_id: str, run_id: str) -> list[Artifact]:
try:
return await app.DATABASE.artifacts.list_artifacts_for_run_by_type(
diff --git a/skyvern/forge/sdk/routes/run_blocks.py b/skyvern/forge/sdk/routes/run_blocks.py
index c86e9ce2e..464a7c84b 100644
--- a/skyvern/forge/sdk/routes/run_blocks.py
+++ b/skyvern/forge/sdk/routes/run_blocks.py
@@ -8,6 +8,7 @@ from skyvern.constants import DEFAULT_LOGIN_PROMPT
from skyvern.exceptions import MissingBrowserAddressError
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory
from skyvern.forge.sdk.routes.code_samples import (
DOWNLOAD_FILES_CODE_SAMPLE_PYTHON,
DOWNLOAD_FILES_CODE_SAMPLE_TS,
@@ -156,6 +157,11 @@ async def login(
x_api_key: Annotated[str | None, Header()] = None,
x_user_agent: Annotated[str | None, Header()] = None,
) -> WorkflowRunResponse:
+ await PermissionCheckerFactory.get_instance().check(
+ organization, browser_session_id=login_request.browser_session_id
+ )
+ await app.RATE_LIMITER.rate_limit_submit_run(organization.organization_id)
+
url = _validate_url(login_request.url)
totp_verification_url = _validate_url(login_request.totp_url)
webhook_url = _validate_url(login_request.webhook_url)
@@ -320,6 +326,11 @@ async def download_files(
x_api_key: Annotated[str | None, Header()] = None,
x_user_agent: Annotated[str | None, Header()] = None,
) -> WorkflowRunResponse:
+ await PermissionCheckerFactory.get_instance().check(
+ organization, browser_session_id=download_files_request.browser_session_id
+ )
+ await app.RATE_LIMITER.rate_limit_submit_run(organization.organization_id)
+
url = _validate_url(download_files_request.url)
totp_verification_url = _validate_url(download_files_request.totp_url)
webhook_url = _validate_url(download_files_request.webhook_url)
diff --git a/tests/unit/forge/sdk/routes/test_run_blocks_authorization.py b/tests/unit/forge/sdk/routes/test_run_blocks_authorization.py
new file mode 100644
index 000000000..a4870c4ff
--- /dev/null
+++ b/tests/unit/forge/sdk/routes/test_run_blocks_authorization.py
@@ -0,0 +1,115 @@
+from types import SimpleNamespace
+from typing import Any, Callable
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from fastapi import BackgroundTasks, HTTPException, status
+
+from skyvern.exceptions import RateLimitExceeded
+from skyvern.forge.sdk.routes import run_blocks as run_blocks_mod
+from skyvern.schemas.credential_type import CredentialType
+from skyvern.schemas.run_blocks import DownloadFilesRequest, LoginRequest
+
+
+def _workflow_stub() -> SimpleNamespace:
+ return SimpleNamespace(
+ title="Generated workflow",
+ description=None,
+ status="auto_generated",
+ workflow_permanent_id="wpid_test",
+ workflow_id="wf_test",
+ )
+
+
+def _app_mock() -> MagicMock:
+ app_mock = MagicMock()
+ app_mock.WORKFLOW_SERVICE.create_empty_workflow = AsyncMock(return_value=_workflow_stub())
+ app_mock.WORKFLOW_SERVICE.create_workflow_from_request = AsyncMock(return_value=_workflow_stub())
+ app_mock.DATABASE.credentials.get_credential = AsyncMock(return_value=SimpleNamespace(totp_identifier=None))
+ app_mock.RATE_LIMITER.rate_limit_submit_run = AsyncMock()
+ return app_mock
+
+
+def _login_request() -> LoginRequest:
+ return LoginRequest(
+ url="https://example.com",
+ credential_type=CredentialType.skyvern,
+ credential_id="cred_test",
+ browser_session_id="pbs_123",
+ )
+
+
+def _download_files_request() -> DownloadFilesRequest:
+ return DownloadFilesRequest(
+ url="https://example.com",
+ navigation_goal="Download the statement.",
+ browser_session_id="pbs_123",
+ )
+
+
+ENDPOINTS = [
+ ("login", "login_request", _login_request),
+ ("download_files", "download_files_request", _download_files_request),
+]
+
+
+async def _invoke(handler_name: str, request_kwarg: str, request_obj: Any, organization: SimpleNamespace) -> Any:
+ handler = getattr(run_blocks_mod, handler_name)
+ return await handler(
+ request=MagicMock(),
+ background_tasks=BackgroundTasks(),
+ organization=organization,
+ **{request_kwarg: request_obj},
+ )
+
+
[email protected](("handler_name", "request_kwarg", "request_factory"), ENDPOINTS)
+async def test_task_endpoint_refuses_credit_exhausted_org(
+ handler_name: str, request_kwarg: str, request_factory: Callable[[], Any]
+) -> None:
+ app_mock = _app_mock()
+ organization = SimpleNamespace(organization_id="org_test")
+ permission_checker = SimpleNamespace(
+ check=AsyncMock(
+ side_effect=HTTPException(
+ status_code=status.HTTP_402_PAYMENT_REQUIRED,
+ detail="Credits exhausted. Enable overage or upgrade your plan.",
+ )
+ )
+ )
+
+ with (
+ patch.object(run_blocks_mod, "app", app_mock),
+ patch.object(run_blocks_mod.PermissionCheckerFactory, "get_instance", lambda: permission_checker),
+ ):
+ with pytest.raises(HTTPException) as exc_info:
+ await _invoke(handler_name, request_kwarg, request_factory(), organization)
+
+ assert exc_info.value.status_code == status.HTTP_402_PAYMENT_REQUIRED
+ permission_checker.check.assert_awaited_once_with(organization, browser_session_id="pbs_123")
+ app_mock.RATE_LIMITER.rate_limit_submit_run.assert_not_awaited()
+ app_mock.WORKFLOW_SERVICE.create_empty_workflow.assert_not_awaited()
+
+
[email protected](("handler_name", "request_kwarg", "request_factory"), ENDPOINTS)
+async def test_task_endpoint_enforces_rate_limit(
+ handler_name: str, request_kwarg: str, request_factory: Callable[[], Any]
+) -> None:
+ app_mock = _app_mock()
+ app_mock.RATE_LIMITER.rate_limit_submit_run = AsyncMock(
+ side_effect=RateLimitExceeded(organization_id="org_test", max_requests=5, window_seconds=60)
+ )
+ organization = SimpleNamespace(organization_id="org_test")
+ permission_checker = SimpleNamespace(check=AsyncMock())
+
+ with (
+ patch.object(run_blocks_mod, "app", app_mock),
+ patch.object(run_blocks_mod.PermissionCheckerFactory, "get_instance", lambda: permission_checker),
+ ):
+ with pytest.raises(RateLimitExceeded):
+ await _invoke(handler_name, request_kwarg, request_factory(), organization)
+
+ app_mock.RATE_LIMITER.rate_limit_submit_run.assert_awaited_once_with("org_test")
+ app_mock.WORKFLOW_SERVICE.create_empty_workflow.assert_not_awaited()
diff --git a/tests/unit/forge/sdk/routes/test_run_blocks_proxy_location.py b/tests/unit/forge/sdk/routes/test_run_blocks_proxy_location.py
index 1af37ad3b..810e37652 100644
--- a/tests/unit/forge/sdk/routes/test_run_blocks_proxy_location.py
+++ b/tests/unit/forge/sdk/routes/test_run_blocks_proxy_location.py
@@ -26,6 +26,7 @@ def _app_mock() -> MagicMock:
app_mock.WORKFLOW_SERVICE.create_empty_workflow = AsyncMock(return_value=_workflow_stub())
app_mock.WORKFLOW_SERVICE.create_workflow_from_request = AsyncMock(return_value=_workflow_stub())
app_mock.DATABASE.credentials.get_credential = AsyncMock(return_value=SimpleNamespace(totp_identifier=None))
+ app_mock.RATE_LIMITER.rate_limit_submit_run = AsyncMock()
return app_mock
diff --git a/tests/unit/test_downloaded_files_artifact_urls.py b/tests/unit/test_downloaded_files_artifact_urls.py
index f7df77bdb..378d30a49 100644
--- a/tests/unit/test_downloaded_files_artifact_urls.py
+++ b/tests/unit/test_downloaded_files_artifact_urls.py
@@ -562,3 +562,92 @@ def test_content_endpoint_download_filename_strips_header_injection():
assert "\r" not in disposition
assert "\n" not in disposition
assert disposition.count('"') == 2 # only the pair around filename
+
+
+def _run_with_created_at(created_at):
+ run = MagicMock()
+ run.created_at = created_at
+ return run
+
+
+async def test_get_downloaded_files_skips_listing_for_empty_post_cutover_run(keyring_configured):
+ """A post-cutover run with zero DOWNLOAD rows returns [] without the legacy S3 LIST:
+ every download registers a row at save time, so the LIST could only confirm emptiness."""
+ from datetime import datetime
+
+ from skyvern.config import settings
+
+ storage = S3Storage()
+ storage.async_client = MagicMock()
+ storage.async_client.list_files = AsyncMock() # must NOT be called
+
+ mock_list = AsyncMock(return_value=[])
+ mock_get_run = AsyncMock(return_value=_run_with_created_at(datetime(2026, 8, 2, 12, 0, 0)))
+
+ with (
+ patch.object(settings, "DOWNLOADS_EMPTY_S3_LISTING_CUTOVER", "2026-08-01T00:00:00"),
+ patch("skyvern.forge.sdk.artifact.storage.s3.app") as s3_app,
+ ):
+ s3_app.DATABASE.artifacts.list_artifacts_for_run_by_type = mock_list
+ s3_app.DATABASE.tasks.get_run = mock_get_run
+ result = await storage.get_downloaded_files(organization_id="o_1", run_id="wr_new")
+
+ assert result == []
+ storage.async_client.list_files.assert_not_awaited()
+ mock_get_run.assert_awaited_once_with(run_id="wr_new", organization_id="o_1")
+
+
+async def test_get_downloaded_files_lists_for_empty_pre_cutover_run(keyring_configured):
+ """A run created before the cutover keeps the legacy S3 LIST — its downloads may
+ predate row registration."""
+ from datetime import datetime
+
+ from skyvern.config import settings
+
+ storage = S3Storage()
+ storage.async_client = MagicMock()
+ storage.async_client.list_files = AsyncMock(return_value=[])
+
+ mock_list = AsyncMock(return_value=[])
+ mock_get_run = AsyncMock(return_value=_run_with_created_at(datetime(2026, 7, 1, 0, 0, 0)))
+
+ with (
+ patch.object(settings, "DOWNLOADS_EMPTY_S3_LISTING_CUTOVER", "2026-08-01T00:00:00"),
+ patch("skyvern.forge.sdk.artifact.storage.s3.app") as s3_app,
+ ):
+ s3_app.DATABASE.artifacts.list_artifacts_for_run_by_type = mock_list
+ s3_app.DATABASE.tasks.get_run = mock_get_run
+ result = await storage.get_downloaded_files(organization_id="o_1", run_id="wr_old")
+
+ assert result == []
+ storage.async_client.list_files.assert_awaited_once()
+
+
[email protected]("get_run_behavior", ["raises", "returns_none"])
+async def test_get_downloaded_files_lists_when_run_unresolvable(keyring_configured, get_run_behavior):
+ """DB errors or a missing run row fail open to the legacy S3 LIST."""
+ from skyvern.config import settings
+
+ storage = S3Storage()
+ storage.async_client = MagicMock()
+ storage.async_client.list_files = AsyncMock(return_value=[])
+
+ mock_list = AsyncMock(return_value=[])
+ if get_run_behavior == "raises":
+ mock_get_run = AsyncMock(side_effect=RuntimeError("db down"))
+ else:
+ mock_get_run = AsyncMock(return_value=None)
+
+ with (
+ patch.object(settings, "DOWNLOADS_EMPTY_S3_LISTING_CUTOVER", "2026-08-01T00:00:00"),
+ patch("skyvern.forge.sdk.artifact.storage.s3.app") as s3_app,
+ ):
+ s3_app.DATABASE.artifacts.list_artifacts_for_run_by_type = mock_list
+ s3_app.DATABASE.tasks.get_run = mock_get_run
+ result = await storage.get_downloaded_files(organization_id="o_1", run_id="wr_x")
+
+ assert result == []
+ storage.async_client.list_files.assert_awaited_once()