Skip to content

Automate browser based workflows with AI

test(SKY-12171): eliminate measured slow-test hotspots (adversarially reviewed) (#7282)

Suchintan081ad6adbe40
diff --git a/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py b/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py
index 3d1742342..e65a72a7e 100644
--- a/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py
+++ b/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import asyncio
import io
import os
@@ -6,15 +8,12 @@ import subprocess
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
-from typing import Generator
+from typing import TYPE_CHECKING
from unittest.mock import AsyncMock, MagicMock
-import boto3
import pytest
import zstandard as zstd
from freezegun import freeze_time
-from moto.server import ThreadedMotoServer
-from types_boto3_s3.client import S3Client
from skyvern.config import settings
from skyvern.forge.sdk.api.aws import S3StorageClass, S3Uri
@@ -31,6 +30,9 @@ from tests.unit.forge.sdk.artifact.storage.test_helpers import (
create_fake_workflow_run_block,
)
+if TYPE_CHECKING:
+ from types_boto3_s3.client import S3Client
+
# Test constants
TEST_BUCKET = "test-skyvern-bucket"
TEST_ORGANIZATION_ID = "test-org-123"
@@ -52,8 +54,9 @@ class S3StorageForTests(S3Storage):
@pytest.fixture
-def s3_storage(moto_server: str) -> S3Storage:
- return S3StorageForTests(bucket=TEST_BUCKET, endpoint_url=moto_server)
+def s3_storage() -> S3Storage:
+ """Construct storage for pure tests without starting an S3 server."""
+ return S3StorageForTests(bucket=TEST_BUCKET, endpoint_url="http://127.0.0.1:1")
@pytest.fixture(autouse=True)
@@ -63,54 +66,6 @@ def aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing")
[email protected](autouse=True)
-def mock_browser_session_artifact_create(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]:
- """Stub out the DB-side artifact-row inserts for browser-session files.
-
- ``S3Storage.sync_browser_session_file`` now awaits
- ``app.ARTIFACT_MANAGER.create_browser_session_download_artifact`` (for
- ``artifact_type="downloads"``) and
- ``app.ARTIFACT_MANAGER.create_browser_session_recording_artifact`` (for
- ``artifact_type="videos"``). These storage tests run against a moto S3
- with no forge app initialized, so we monkey-patch the module-level
- ``app`` reference in ``s3.py`` — patching ``app.ARTIFACT_MANAGER``
- directly would trip the lazy-init guard on AppHolder.
- """
- from unittest.mock import MagicMock
-
- import skyvern.forge.sdk.artifact.storage.s3 as s3_module
-
- fake_app = MagicMock()
- fake_app.ARTIFACT_MANAGER.create_browser_session_download_artifact = AsyncMock(return_value="a_test")
- fake_app.ARTIFACT_MANAGER.create_browser_session_recording_artifact = AsyncMock(return_value="a_test")
- monkeypatch.setattr(s3_module, "app", fake_app)
- yield
-
-
[email protected](scope="module")
-def moto_server() -> Generator[str, None, None]:
- # Note: pass `port=0` to get a random free port.
- server = ThreadedMotoServer(port=0)
- server.start()
- host, port = server.get_host_and_port()
- yield f"http://{host}:{port}"
- server.stop()
-
-
[email protected](scope="module", autouse=True)
-def boto3_test_client(moto_server: str) -> Generator[S3Client, None, None]:
- client = boto3.client(
- "s3",
- aws_access_key_id="testing",
- aws_secret_access_key="testing",
- region_name=settings.AWS_REGION,
- endpoint_url=moto_server,
- )
- client.create_bucket(Bucket=TEST_BUCKET) # Ensure the bucket exists for the test
- client.create_bucket(Bucket=settings.AWS_S3_BUCKET_UPLOADS)
- yield client
-
-
@freeze_time("2025-06-09T12:00:00")
class TestS3StorageBuildURIs:
def test_build_uri(self, s3_storage: S3Storage) -> None:
@@ -209,6 +164,8 @@ def _assert_object_content(boto3_test_client: S3Client, uri: str, expected_conte
class TestS3StorageStore:
"""Test S3Storage store methods."""
+ __test__ = False # Collected with moto fixtures in test_s3_storage_moto.py.
+
def _create_artifact_for_ai_suggestion(
self,
s3_storage: S3Storage,
@@ -264,6 +221,8 @@ TEST_BROWSER_SESSION_ID = "bs_test_123"
class TestS3StorageBrowserSessionFiles:
"""Test S3Storage browser session file methods."""
+ __test__ = False # Moto cases are collected in test_s3_storage_moto.py.
+
async def test_sync_browser_session_file_with_date(
self, s3_storage: S3Storage, boto3_test_client: S3Client, tmp_path: Path
) -> None:
@@ -811,6 +770,8 @@ CONTENT_TYPE_TEST_CASES = [
class TestS3StorageContentType:
"""Test S3Storage content type guessing."""
+ __test__ = False # Collected with moto fixtures in test_s3_storage_moto.py.
+
@pytest.mark.parametrize("filename,expected_content_type,artifact_type,date", CONTENT_TYPE_TEST_CASES)
async def test_content_type_guessing(
self,
@@ -844,6 +805,8 @@ class TestS3StorageContentType:
class TestS3StorageHARCompression:
"""Test S3Storage HAR file compression with zstd."""
+ __test__ = False # Collected with moto fixtures in test_s3_storage_moto.py.
+
def _create_har_artifact(self, s3_storage: S3Storage, step_id: str) -> Artifact:
"""Helper method to create a HAR Artifact."""
artifact_id_val = generate_artifact_id()
@@ -946,6 +909,8 @@ _build_zip = ArtifactManager._build_zip
class TestS3StorageZIPArchiveRetrieve:
"""Test retrieve_artifact with STEP_ARCHIVE / TASK_ARCHIVE bundle_key extraction."""
+ __test__ = False # Moto cases are collected in test_s3_storage_moto.py.
+
def _make_archive_artifact(
self,
s3_storage: S3Storage,
@@ -1151,6 +1116,8 @@ class TestS3StoragePerRunRecordingClips:
``S3Storage.sync_browser_session_file(videos)`` against moto S3 with a real
ffmpeg-generated recording, asserting a run-scoped clip is cut and uploaded."""
+ __test__ = False # Collected with moto fixtures in test_s3_storage_moto.py.
+
async def test_session_close_cuts_and_uploads_run_scoped_clip(
self,
s3_storage: S3Storage,
@@ -1223,3 +1190,37 @@ class TestS3StoragePerRunRecordingClips:
clip_key = S3Uri(clip_uri).key
head = boto3_test_client.head_object(Bucket=TEST_BUCKET, Key=clip_key)
assert head["ContentLength"] > 0
+
+
+class TestS3StorageBrowserSessionPure:
+ """URI authorization checks that do not perform S3 I/O."""
+
+ test_assert_managed_file_access_accepts_org_scoped_uploads = (
+ TestS3StorageBrowserSessionFiles.test_assert_managed_file_access_accepts_org_scoped_uploads
+ )
+ test_assert_managed_file_access_accepts_artifact_bucket = (
+ TestS3StorageBrowserSessionFiles.test_assert_managed_file_access_accepts_artifact_bucket
+ )
+ test_assert_managed_file_access_rejects_other_org = (
+ TestS3StorageBrowserSessionFiles.test_assert_managed_file_access_rejects_other_org
+ )
+ test_assert_managed_file_access_rejects_other_org_artifact_bucket = (
+ TestS3StorageBrowserSessionFiles.test_assert_managed_file_access_rejects_other_org_artifact_bucket
+ )
+ test_download_managed_file_rejects_other_org = (
+ TestS3StorageBrowserSessionFiles.test_download_managed_file_rejects_other_org
+ )
+ test_storage_type_property = TestS3StorageBrowserSessionFiles.test_storage_type_property
+
+
+class TestS3StorageZIPArchivePure:
+ """Archive URI checks that do not perform S3 I/O."""
+
+ test_build_uri_step_archive_has_zip_extension = (
+ TestS3StorageZIPArchiveRetrieve.test_build_uri_step_archive_has_zip_extension
+ )
+ test_build_uri_task_archive_has_zip_extension = (
+ TestS3StorageZIPArchiveRetrieve.test_build_uri_task_archive_has_zip_extension
+ )
diff --git a/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py b/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py
new file mode 100644
index 000000000..0c725190c
--- /dev/null
+++ b/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py
@@ -0,0 +1,112 @@
+"""Moto-backed S3 storage contracts separated from pure storage tests."""
+
+from __future__ import annotations
+
+from typing import Generator
+from unittest.mock import AsyncMock, MagicMock
+
+import boto3
+import pytest
+from moto.server import ThreadedMotoServer
+from types_boto3_s3.client import S3Client
+
+from skyvern.config import settings
+from skyvern.forge.sdk.artifact.storage.s3 import S3Storage
+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (
+ TEST_BUCKET,
+ S3StorageForTests,
+)
+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (
+ TestS3StorageBrowserSessionFiles as _BrowserSessionContracts,
+)
+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import TestS3StorageContentType as _ContentTypeContracts
+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (
+ TestS3StorageHARCompression as _HARCompressionContracts,
+)
+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (
+ TestS3StoragePerRunRecordingClips as _PerRunRecordingContracts,
+)
+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import TestS3StorageStore as _StoreContracts
+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (
+ TestS3StorageZIPArchiveRetrieve as _ZIPArchiveContracts,
+)
+
+
+def s3_storage(moto_server: str) -> S3Storage:
+ return S3StorageForTests(bucket=TEST_BUCKET, endpoint_url=moto_server)
+
+
[email protected](autouse=True)
+def aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing")
+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing")
+
+
[email protected](autouse=True)
+def mock_browser_session_artifact_create(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Stub DB artifact-row inserts while exercising real moto object storage."""
+ import skyvern.forge.sdk.artifact.storage.s3 as s3_module
+
+ fake_app = MagicMock()
+ fake_app.ARTIFACT_MANAGER.create_browser_session_download_artifact = AsyncMock(return_value="a_test")
+ fake_app.ARTIFACT_MANAGER.create_browser_session_recording_artifact = AsyncMock(return_value="a_test")
+ monkeypatch.setattr(s3_module, "app", fake_app)
+
+
[email protected](scope="module")
+def moto_server() -> Generator[str]:
+ server = ThreadedMotoServer(port=0)
+ server.start()
+ host, port = server.get_host_and_port()
+ yield f"http://{host}:{port}"
+ server.stop()
+
+
[email protected](scope="module", autouse=True)
+def boto3_test_client(moto_server: str) -> Generator[S3Client]:
+ client = boto3.client(
+ "s3",
+ aws_access_key_id="testing",
+ aws_secret_access_key="testing",
+ region_name=settings.AWS_REGION,
+ endpoint_url=moto_server,
+ )
+ client.create_bucket(Bucket=TEST_BUCKET)
+ client.create_bucket(Bucket=settings.AWS_S3_BUCKET_UPLOADS)
+ yield client
+
+
+class TestS3StorageStoreMoto(_StoreContracts):
+ __test__ = True
+
+
+class TestS3StorageBrowserSessionFilesMoto(_BrowserSessionContracts):
+ __test__ = True
+
+ # These checks are pure URI/guard behavior and stay in the fast module below.
+ test_assert_managed_file_access_accepts_org_scoped_uploads = None
+ test_assert_managed_file_access_accepts_artifact_bucket = None
+ test_assert_managed_file_access_rejects_other_org = None
+ test_assert_managed_file_access_rejects_other_org_artifact_bucket = None
+ test_download_managed_file_rejects_other_org = None
+ test_storage_type_property = None
+
+
+class TestS3StorageContentTypeMoto(_ContentTypeContracts):
+ __test__ = True
+
+
+class TestS3StorageHARCompressionMoto(_HARCompressionContracts):
+ __test__ = True
+
+
+class TestS3StorageZIPArchiveRetrieveMoto(_ZIPArchiveContracts):
+ __test__ = True
+
+ test_build_uri_step_archive_has_zip_extension = None
+ test_build_uri_task_archive_has_zip_extension = None
+
+
+class TestS3StoragePerRunRecordingClipsMoto(_PerRunRecordingContracts):
+ __test__ = True
diff --git a/tests/unit/forge/sdk/test_reset_browser_profile_route.py b/tests/unit/forge/sdk/test_reset_browser_profile_route.py
index caf57fd03..5fc594596 100644
--- a/tests/unit/forge/sdk/test_reset_browser_profile_route.py
+++ b/tests/unit/forge/sdk/test_reset_browser_profile_route.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from collections.abc import Generator
from types import SimpleNamespace
from unittest.mock import AsyncMock, call
@@ -16,7 +17,8 @@ from skyvern.forge.sdk.routes import routers as routers_module
from skyvern.forge.sdk.services import org_auth_service
-def _build_client(monkeypatch: pytest.MonkeyPatch) -> tuple[TestClient, SimpleNamespace]:
[email protected](scope="module")
+def app_client_and_mocks() -> Generator[tuple[TestClient, SimpleNamespace]]:
async def _fake_org() -> SimpleNamespace:
return SimpleNamespace(organization_id="org_oss")
@@ -27,29 +29,44 @@ def _build_client(monkeypatch: pytest.MonkeyPatch) -> tuple[TestClient, SimpleNa
delete_profile_blob=AsyncMock(),
delete_db_profile=AsyncMock(),
)
- monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", mocks.get_workflow)
- monkeypatch.setattr(forge_app.STORAGE, "delete_browser_session", mocks.delete_browser_session)
- monkeypatch.setattr(forge_app.STORAGE, "delete_browser_profile", mocks.delete_profile_blob)
- monkeypatch.setattr(
- forge_app.DATABASE.browser_sessions,
- "list_managed_browser_profiles_for_workflow",
- mocks.list_managed_profiles,
- )
- monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "delete_browser_profile", mocks.delete_db_profile)
-
- fastapi_app = FastAPI()
- fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org
- fastapi_app.include_router(routers_module.base_router, prefix="/v1")
-
- @fastapi_app.exception_handler(SkyvernHTTPException)
- async def _handle_skyvern_http_exception(request: Request, exc: SkyvernHTTPException) -> JSONResponse:
- return JSONResponse(status_code=exc.status_code, content={"detail": exc.message})
-
- return TestClient(fastapi_app), mocks
-
-
-def test_reset_browser_profile_clears_storage(monkeypatch: pytest.MonkeyPatch) -> None:
- client, mocks = _build_client(monkeypatch)
+ module_patches = pytest.MonkeyPatch()
+ try:
+ module_patches.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", mocks.get_workflow)
+ module_patches.setattr(forge_app.STORAGE, "delete_browser_session", mocks.delete_browser_session)
+ module_patches.setattr(forge_app.STORAGE, "delete_browser_profile", mocks.delete_profile_blob)
+ module_patches.setattr(
+ forge_app.DATABASE.browser_sessions,
+ "list_managed_browser_profiles_for_workflow",
+ mocks.list_managed_profiles,
+ )
+ module_patches.setattr(forge_app.DATABASE.browser_sessions, "delete_browser_profile", mocks.delete_db_profile)
+
+ fastapi_app = FastAPI()
+ fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org
+ fastapi_app.include_router(routers_module.base_router, prefix="/v1")
+
+ @fastapi_app.exception_handler(SkyvernHTTPException)
+ async def _handle_skyvern_http_exception(request: Request, exc: SkyvernHTTPException) -> JSONResponse:
+ return JSONResponse(status_code=exc.status_code, content={"detail": exc.message})
+
+ with TestClient(fastapi_app) as client:
+ yield client, mocks
+ finally:
+ module_patches.undo()
+
+
+def client_and_mocks(app_client_and_mocks: tuple[TestClient, SimpleNamespace]) -> tuple[TestClient, SimpleNamespace]:
+ """Reuse the app/router while returning pristine mock state to every test."""
+ client, mocks = app_client_and_mocks
+ for mock in vars(mocks).values():
+ mock.reset_mock(return_value=True, side_effect=True)
+ mocks.list_managed_profiles.return_value = []
+ return client, mocks
+
+
+def test_reset_browser_profile_clears_storage(client_and_mocks: tuple[TestClient, SimpleNamespace]) -> None:
+ client, mocks = client_and_mocks
mocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")
response = client.post("/v1/workflows/wpid_123/browser_session/reset_profile")
@@ -64,8 +81,8 @@ def test_reset_browser_profile_clears_storage(monkeypatch: pytest.MonkeyPatch) -
mocks.delete_db_profile.assert_not_awaited()
-def test_reset_browser_profile_workflow_not_found(monkeypatch: pytest.MonkeyPatch) -> None:
- client, mocks = _build_client(monkeypatch)
+def test_reset_browser_profile_workflow_not_found(client_and_mocks: tuple[TestClient, SimpleNamespace]) -> None:
+ client, mocks = client_and_mocks
mocks.get_workflow.side_effect = WorkflowNotFound(workflow_permanent_id="wpid_missing")
response = client.post("/v1/workflows/wpid_missing/browser_session/reset_profile")
@@ -75,8 +92,10 @@ def test_reset_browser_profile_workflow_not_found(monkeypatch: pytest.MonkeyPatc
mocks.list_managed_profiles.assert_not_awaited()
-def test_reset_browser_profile_storage_failure_returns_500(monkeypatch: pytest.MonkeyPatch) -> None:
- client, mocks = _build_client(monkeypatch)
+def test_reset_browser_profile_storage_failure_returns_500(
+ client_and_mocks: tuple[TestClient, SimpleNamespace],
+) -> None:
+ client, mocks = client_and_mocks
mocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")
mocks.delete_browser_session.side_effect = RuntimeError("s3 AccessDenied")
@@ -86,8 +105,10 @@ def test_reset_browser_profile_storage_failure_returns_500(monkeypatch: pytest.M
assert "retry" in response.json()["detail"].lower()
-def test_reset_browser_profile_clears_managed_profile_blobs_and_rows(monkeypatch: pytest.MonkeyPatch) -> None:
- client, mocks = _build_client(monkeypatch)
+def test_reset_browser_profile_clears_managed_profile_blobs_and_rows(
+ client_and_mocks: tuple[TestClient, SimpleNamespace],
+) -> None:
+ client, mocks = client_and_mocks
mocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")
mocks.list_managed_profiles.return_value = [
SimpleNamespace(browser_profile_id="bp_one", browser_profile_key_digest="", deleted_at=None),
@@ -112,8 +133,10 @@ def test_reset_browser_profile_clears_managed_profile_blobs_and_rows(monkeypatch
]
-def test_reset_browser_profile_clears_segmented_legacy_archives(monkeypatch: pytest.MonkeyPatch) -> None:
- client, mocks = _build_client(monkeypatch)
+def test_reset_browser_profile_clears_segmented_legacy_archives(
+ client_and_mocks: tuple[TestClient, SimpleNamespace],
+) -> None:
+ client, mocks = client_and_mocks
mocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")
mocks.list_managed_profiles.return_value = [
SimpleNamespace(browser_profile_id="bp_seg", browser_profile_key_digest="abc123digest", deleted_at=None),
@@ -144,8 +167,8 @@ def test_reset_browser_profile_clears_segmented_legacy_archives(monkeypatch: pyt
"/v1/workflows/wpid_123/browser_session/reset_profile/",
],
)
-def test_alias_paths_still_work(monkeypatch: pytest.MonkeyPatch, path: str) -> None:
- client, mocks = _build_client(monkeypatch)
+def test_alias_paths_still_work(client_and_mocks: tuple[TestClient, SimpleNamespace], path: str) -> None:
+ client, mocks = client_and_mocks
mocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")
response = client.post(path)
diff --git a/tests/unit/test_code_block_lint_mcp_tool.py b/tests/unit/test_code_block_lint_mcp_tool.py
index b5d7acc71..bed44ea30 100644
--- a/tests/unit/test_code_block_lint_mcp_tool.py
+++ b/tests/unit/test_code_block_lint_mcp_tool.py
@@ -2,11 +2,21 @@
from __future__ import annotations
+import sys
+from types import ModuleType, SimpleNamespace
+
import pytest
from skyvern.cli.mcp_tools.code_block import skyvern_code_block_lint
[email protected](autouse=True)
+def _stub_mypy_for_non_mypy_lint_contracts(monkeypatch: pytest.MonkeyPatch) -> None:
+ fake_mypy = ModuleType("mypy")
+ fake_mypy.__dict__["api"] = SimpleNamespace(run=lambda _args: ("", "", 0))
+ monkeypatch.setitem(sys.modules, "mypy", fake_mypy)
+
+
def _has_security_error(result: dict, *, reason_code: str, surface: str | None = None) -> bool:
return any(
error["reason_code"] == reason_code and (surface is None or error["surface"] == surface)
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 0c929e2dd..1a4c8c708 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -10,6 +10,7 @@ import json
import keyword
import sys
import textwrap
+from types import ModuleType, SimpleNamespace
from typing import Any
import pytest
@@ -45,6 +46,23 @@ from skyvern.forge.sdk.copilot.tools.workflow_update import _code_block_safety_e
from skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockStep
[email protected](autouse=True)
+def _stub_mypy_for_static_policy_cases(
+ request: pytest.FixtureRequest,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ if "real_mypy" in request.fixturenames:
+ return
+ fake_mypy = ModuleType("mypy")
+ fake_mypy.__dict__["api"] = SimpleNamespace(run=lambda _args: ("", "", 0))
+ monkeypatch.setitem(sys.modules, "mypy", fake_mypy)
+
+
+def real_mypy() -> None:
+ return None
+
+
def _interaction(tool_name: str, **fields: Any) -> dict[str, Any]:
return {"tool_name": tool_name, **fields}
@@ -2069,7 +2087,32 @@ class TestCredentialFillSynthesis:
assert _code_block_safety_errors(workflow_yaml, None) == []
-def test_code_block_preflight_restores_recursion_limit() -> None:
+ ("code", "expected_codes"),
+ [
+ pytest.param(
+ "await page.locator('button[type=submit]').first.click(timeout=5000)\n",
+ (),
+ id="valid-locator-click",
+ ),
+ pytest.param(
+ "await page.locator('button[type=submit]').first().click(timeout=5000)\n",
+ ("PLAYWRIGHT_API_MISMATCH",),
+ id="locator-property-called-as-method",
+ ),
+ ],
+)
+def test_code_block_preflight_real_mypy_contract_matrix(
+ real_mypy: None,
+ code: str,
+ expected_codes: tuple[str, ...],
+) -> None:
+ diagnostics = preflight_code_block(code)
+
+ assert [diagnostic.code for diagnostic in diagnostics] == list(expected_codes)
+
+
+def test_code_block_preflight_restores_recursion_limit(real_mypy: None) -> None:
before = sys.getrecursionlimit()
preflight_code_block("await page.locator('button[type=submit]').first.click(timeout=5000)\n")
diff --git a/tests/unit/test_copilot_composition_scrape.py b/tests/unit/test_copilot_composition_scrape.py
index 3a2ee64ef..4250e023f 100644
--- a/tests/unit/test_copilot_composition_scrape.py
+++ b/tests/unit/test_copilot_composition_scrape.py
@@ -5,7 +5,9 @@ served in a real scout because the agent acts between inspects.)
from __future__ import annotations
+import asyncio
from types import SimpleNamespace
+from unittest.mock import AsyncMock
import pytest
@@ -13,6 +15,14 @@ from skyvern.forge.sdk.copilot import tools
from skyvern.forge.sdk.copilot.tools import _normalized_inspect_url, _same_inspect_target
+class _AsyncioSleepProxy:
+ def __init__(self, sleep: AsyncMock) -> None:
+ self.sleep = sleep
+
+ def __getattr__(self, name: str):
+ return getattr(asyncio, name)
+
+
def test_normalized_inspect_url_preserves_distinguishing_parts() -> None:
assert _normalized_inspect_url("https://Example.com/Search?q=a#frag") == "https://example.com/Search?q=a#frag"
# query distinguishes search states; scheme and trailing slash are significant
@@ -61,6 +71,8 @@ async def test_recapture_skips_raw_get_html_after_cap_drop(monkeypatch: pytest.M
monkeypatch.setattr(
tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity
)
+ settle_sleep = AsyncMock()
+ monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(settle_sleep))
evidence, html_error = await tools._capture_composition_evidence(
SimpleNamespace(), inspected_url="https://example.com/s", current_url="https://example.com/s"
@@ -71,3 +83,4 @@ async def test_recapture_skips_raw_get_html_after_cap_drop(monkeypatch: pytest.M
assert tools.has_bounded_page_schema(evidence)
# First iteration's raw read is cap-dropped; the settle retry skips it entirely.
assert raw_calls["n"] == 1
+ settle_sleep.assert_awaited_once_with(tools.composition_capture._COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index b722d7efb..f77a3ac57 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -1367,7 +1367,6 @@ async def test_handle_action_discards_xhr_staging_when_native_file_present(
task, step, page, browser_state, scraped_page, action = _make_download_click_context(
now=now, organization=organization, page_url="https://example.com/download"
)
-
callbacks: dict[str, object] = {}
page.on.side_effect = lambda event, cb: callbacks.__setitem__(event, cb)
@@ -1430,6 +1429,7 @@ async def test_handle_action_uses_xhr_staging_fallback_when_no_native_file(
task, step, page, browser_state, scraped_page, action = _make_download_click_context(
now=now, organization=organization, page_url="https://example.com/download"
)
+ task.download_timeout = 0.01
callbacks: dict[str, object] = {}
page.on.side_effect = lambda event, cb: callbacks.__setitem__(event, cb)
@@ -1490,6 +1490,7 @@ async def test_handle_action_moves_multiple_staged_xhr_files_as_fallback(
task, step, page, browser_state, scraped_page, action = _make_download_click_context(
now=now, organization=organization, page_url="https://example.com/download"
)
+ task.download_timeout = 0.01
callbacks: dict[str, object] = {}
page.on.side_effect = lambda event, cb: callbacks.__setitem__(event, cb)
@@ -2021,7 +2022,7 @@ async def test_handle_action_adopted_session_helper_failure_does_not_short_circu
organization,
workflow_run_id="wr-1",
browser_session_id="bs-1",
- download_timeout=2.0,
+ download_timeout=0.01,
)
step = make_step(now, task, step_id="step-1", status=StepStatus.created, order=0, output=None)
diff --git a/tests/unit/test_load_prompt_with_elements_ceiling.py b/tests/unit/test_load_prompt_with_elements_ceiling.py
index aae067476..d58e0a394 100644
--- a/tests/unit/test_load_prompt_with_elements_ceiling.py
+++ b/tests/unit/test_load_prompt_with_elements_ceiling.py
@@ -4,6 +4,17 @@ from __future__ import annotations
from unittest.mock import MagicMock
+import pytest
+
+
+def small_prompt_ceiling(monkeypatch: pytest.MonkeyPatch) -> int:
+ from skyvern.utils import prompt_engine
+
+ ceiling = 500
+ monkeypatch.setattr(prompt_engine, "PROMPT_HARD_CEILING_TOKENS", ceiling)
+ return ceiling
+
def test_prompt_hard_ceiling_is_below_gpt5_mini_cap() -> None:
from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS
@@ -41,8 +52,7 @@ def test_load_prompt_with_elements_drops_previous_info_when_over_ceiling() -> No
from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, load_prompt_with_elements
from skyvern.utils.token_counter import count_tokens
- # List of distinct English-ish words well over the 180k token ceiling.
- oversized_prev = [{"iter": i, "marker": f"UNIQUE_BLOCK_{i}_" + ("lorem ipsum " * 200)} for i in range(3000)]
+ oversized_prev = [{"iter": 0, "marker": "UNIQUE_BLOCK_0_" + ("lorem " * 185_000)}]
rendered = load_prompt_with_elements(
element_tree_builder=_make_element_tree_builder(),
@@ -62,12 +72,12 @@ def test_load_prompt_with_elements_drops_previous_info_when_over_ceiling() -> No
assert "UNIQUE_BLOCK_0_" not in rendered
-def test_enforce_prompt_ceiling_drops_fallback_keys_without_elements() -> None:
+def test_enforce_prompt_ceiling_drops_fallback_keys_without_elements(small_prompt_ceiling: int) -> None:
from skyvern.forge.prompts import prompt_engine as engine_module
from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, enforce_prompt_ceiling
from skyvern.utils.token_counter import count_tokens
- giant_schema = {"type": "object", "_blob": "lorem " * 300_000}
+ giant_schema = {"type": "object", "_blob": "lorem " * (small_prompt_ceiling + 100)}
kwargs = {
"data_extraction_goal": "Extract",
"data_extraction_schema": giant_schema,
@@ -109,12 +119,14 @@ def test_load_prompt_with_elements_respects_ceiling_for_small_prompts() -> None:
assert count_tokens(rendered) <= PROMPT_HARD_CEILING_TOKENS
-def test_load_prompt_with_elements_tracked_drops_extracted_text_as_last_resort() -> None:
+def test_load_prompt_with_elements_tracked_drops_extracted_text_as_last_resort(
+ small_prompt_ceiling: int,
+) -> None:
from skyvern.forge.prompts import prompt_engine as engine_module
from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, load_prompt_with_elements_tracked
from skyvern.utils.token_counter import count_tokens
- oversized_extracted_text = "UNIQUE_EXTRACTED_TEXT " * 300_000
+ oversized_extracted_text = "UNIQUE_EXTRACTED_TEXT " + ("lorem " * (small_prompt_ceiling + 100))
rendered, post_kwargs = load_prompt_with_elements_tracked(
element_tree_builder=_make_element_tree_builder(),
@@ -135,12 +147,12 @@ def test_load_prompt_with_elements_tracked_drops_extracted_text_as_last_resort()
assert post_kwargs["extracted_text"] is None
-def test_enforce_prompt_ceiling_tracked_reports_dropped_keys() -> None:
+def test_enforce_prompt_ceiling_tracked_reports_dropped_keys(small_prompt_ceiling: int) -> None:
from skyvern.forge.prompts import prompt_engine as engine_module
from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, enforce_prompt_ceiling_tracked
from skyvern.utils.token_counter import count_tokens
- giant_schema = {"type": "object", "_blob": "lorem " * 300_000}
+ giant_schema = {"type": "object", "_blob": "lorem " * (small_prompt_ceiling + 100)}
kwargs = {
"data_extraction_goal": "Extract",
"data_extraction_schema": giant_schema,
@@ -184,12 +196,12 @@ def test_enforce_prompt_ceiling_tracked_noop_under_ceiling() -> None:
assert post_kwargs["data_extraction_schema"] == {"type": "object"}
-def test_load_prompt_with_elements_tracked_reports_dropped_keys() -> None:
+def test_load_prompt_with_elements_tracked_reports_dropped_keys(small_prompt_ceiling: int) -> None:
from skyvern.forge.prompts import prompt_engine as engine_module
from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, load_prompt_with_elements_tracked
from skyvern.utils.token_counter import count_tokens
- oversized_prev = [{"iter": i, "marker": f"UNIQUE_BLOCK_{i}_" + ("lorem ipsum " * 200)} for i in range(3000)]
+ oversized_prev = [{"iter": 0, "marker": "UNIQUE_BLOCK_0_" + ("lorem " * (small_prompt_ceiling + 100))}]
rendered, post_kwargs = load_prompt_with_elements_tracked(
element_tree_builder=_make_element_tree_builder(),
diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.py
index 3394ab247..dd4add53a 100644
--- a/tests/unit/test_real_browser_state_navigate.py
+++ b/tests/unit/test_real_browser_state_navigate.py
@@ -4,19 +4,25 @@ Covers SKY-8818: pages whose subresources never finish loading must still
succeed if the DOM has parsed, via progressive wait_until degradation.
"""
+from functools import partial
from unittest.mock import AsyncMock, MagicMock
import pytest
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from skyvern.exceptions import FailedToNavigateToUrl
+from skyvern.webeye import real_browser_state
+from skyvern.webeye.navigation import navigate_with_retry
from skyvern.webeye.real_browser_state import RealBrowserState, _same_page_ignoring_fragment
@pytest.fixture
-def browser_state() -> RealBrowserState:
+def browser_state(monkeypatch: pytest.MonkeyPatch) -> RealBrowserState:
+ retry_sleep = AsyncMock()
+ monkeypatch.setattr(real_browser_state, "navigate_with_retry", partial(navigate_with_retry, sleep=retry_sleep))
# Bypass __init__; navigate_to_url only uses `self` for LOG context and _wait_for_settle.
state = RealBrowserState.__new__(RealBrowserState)
+ monkeypatch.setattr(state, "_wait_for_settle", AsyncMock())
return state
diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.py
index fc183a669..f87921151 100644
--- a/tests/unit/test_script_skyvern_page.py
+++ b/tests/unit/test_script_skyvern_page.py
@@ -10,6 +10,7 @@ and wait() (accepts both seconds= and timeout_ms= parameter styles).
import inspect
import re
import time
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -21,28 +22,46 @@ from skyvern.core.script_generations.skyvern_page import SkyvernPage
from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException
-def create_mock_page():
- """Create a mock Playwright Page object with required attributes."""
- page = MagicMock()
- page.url = "https://example.com"
- # Required for Playwright Page base class
- page._loop = MagicMock()
- page._impl_obj = page
- return page
+class _KeyboardStub:
+ async def press(self, *_args: object, **_kwargs: object) -> None:
+ return None
+
+
+class _PageStub:
+ def __init__(self) -> None:
+ self.url = "https://example.com"
+ self.keyboard = _KeyboardStub()
+
+ async def evaluate(self, *_args: object, **_kwargs: object) -> None:
+ return None
+
+
+def create_mock_page() -> _PageStub:
+ return _PageStub()
+
+
+class _AiStub:
+ pass
+
+
+class _ScrapedPageStub:
+ def __init__(self) -> None:
+ self._browser_state = SimpleNamespace()
+
+
[email protected](autouse=True)
+def no_cached_action_delay(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "CACHED_ACTION_DELAY_SECONDS", 0)
@pytest.fixture
def mock_scraped_page():
- """Create a mock ScrapedPage object."""
- scraped_page = MagicMock()
- scraped_page._browser_state = MagicMock()
- return scraped_page
+ return _ScrapedPageStub()
@pytest.fixture
def mock_ai():
- """Create a mock SkyvernPageAi object."""
- return MagicMock()
+ return _AiStub()
@pytest.mark.asyncio
@@ -243,8 +262,7 @@ async def test_ensure_element_ids_skips_when_ids_exist(mock_scraped_page, mock_a
should NOT be called (fast path).
"""
mock_page = create_mock_page()
- # SkyvernPage.__getattribute__ delegates self.page to mock_page.page
- mock_page.page.evaluate = AsyncMock(return_value=True) # unique_ids exist
+ mock_page.evaluate = AsyncMock(return_value=True) # unique_ids exist
with patch(
"skyvern.core.script_generations.skyvern_page.Page.__init__",
@@ -273,9 +291,7 @@ async def test_ensure_element_ids_injects_when_ids_missing(mock_scraped_page, mo
domUtils.js and call buildTreeFromBody to set them.
"""
mock_page = create_mock_page()
- # SkyvernPage.__getattribute__ delegates self.page to mock_page.page,
- # so set evaluate on the delegated object
- mock_page.page.evaluate = AsyncMock(return_value=False) # no unique_ids
+ mock_page.evaluate = AsyncMock(return_value=False) # no unique_ids
with patch(
"skyvern.core.script_generations.skyvern_page.Page.__init__",
@@ -341,8 +357,7 @@ async def test_ensure_element_ids_catches_exceptions(mock_scraped_page, mock_ai)
action execution.
"""
mock_page = create_mock_page()
- # SkyvernPage.__getattribute__ delegates self.page to mock_page.page
- mock_page.page.evaluate = AsyncMock(side_effect=Exception("Page crashed"))
+ mock_page.evaluate = AsyncMock(side_effect=Exception("Page crashed"))
with patch(
"skyvern.core.script_generations.skyvern_page.Page.__init__",
@@ -1648,8 +1663,6 @@ def _skyvern_page_with_locator(mock_ai, locator: _RecordingLocator) -> SkyvernPa
return_value=None,
):
raw_page = create_mock_page()
- raw_page.page = raw_page
- raw_page.keyboard.press = AsyncMock()
skyvern_page = SkyvernPage(
page=raw_page,
ai=mock_ai,