Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
test(SKY-12171): eliminate measured slow-test hotspots (adversarially reviewed) (#7282)
Suchintan081ad6adbe40
diff --git a/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py b/tests/unit/forge/sdk/artifact/storage/test_s3_storage.pyindex 3d1742342..e65a72a7e 100644--- a/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py+++ b/tests/unit/forge/sdk/artifact/storage/test_s3_storage.py@@ -1,3 +1,5 @@+from __future__ import annotations+import asyncioimport ioimport os@@ -6,15 +8,12 @@ import subprocessfrom datetime import UTC, datetime, timedeltafrom pathlib import Pathfrom types import SimpleNamespace-from typing import Generator+from typing import TYPE_CHECKINGfrom unittest.mock import AsyncMock, MagicMock-import boto3import pytestimport zstandard as zstdfrom freezegun import freeze_time-from moto.server import ThreadedMotoServer-from types_boto3_s3.client import S3Clientfrom skyvern.config import settingsfrom skyvern.forge.sdk.api.aws import S3StorageClass, S3Uri@@ -31,6 +30,9 @@ from tests.unit.forge.sdk.artifact.storage.test_helpers import (create_fake_workflow_run_block,)+if TYPE_CHECKING:+ from types_boto3_s3.client import S3Client+# Test constantsTEST_BUCKET = "test-skyvern-bucket"TEST_ORGANIZATION_ID = "test-org-123"@@ -52,8 +54,9 @@ class S3StorageForTests(S3Storage):@pytest.fixture-def s3_storage(moto_server: str) -> S3Storage:- return S3StorageForTests(bucket=TEST_BUCKET, endpoint_url=moto_server)+def s3_storage() -> S3Storage:+ """Construct storage for pure tests without starting an S3 server."""+ return S3StorageForTests(bucket=TEST_BUCKET, endpoint_url="http://127.0.0.1:1")@pytest.fixture(autouse=True)@@ -63,54 +66,6 @@ def aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing")[email protected](autouse=True)-def mock_browser_session_artifact_create(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]:- """Stub out the DB-side artifact-row inserts for browser-session files.-- ``S3Storage.sync_browser_session_file`` now awaits- ``app.ARTIFACT_MANAGER.create_browser_session_download_artifact`` (for- ``artifact_type="downloads"``) and- ``app.ARTIFACT_MANAGER.create_browser_session_recording_artifact`` (for- ``artifact_type="videos"``). These storage tests run against a moto S3- with no forge app initialized, so we monkey-patch the module-level- ``app`` reference in ``s3.py`` — patching ``app.ARTIFACT_MANAGER``- directly would trip the lazy-init guard on AppHolder.- """- from unittest.mock import MagicMock-- import skyvern.forge.sdk.artifact.storage.s3 as s3_module-- fake_app = MagicMock()- fake_app.ARTIFACT_MANAGER.create_browser_session_download_artifact = AsyncMock(return_value="a_test")- fake_app.ARTIFACT_MANAGER.create_browser_session_recording_artifact = AsyncMock(return_value="a_test")- monkeypatch.setattr(s3_module, "app", fake_app)- yield--[email protected](scope="module")-def moto_server() -> Generator[str, None, None]:- # Note: pass `port=0` to get a random free port.- server = ThreadedMotoServer(port=0)- server.start()- host, port = server.get_host_and_port()- yield f"http://{host}:{port}"- server.stop()--[email protected](scope="module", autouse=True)-def boto3_test_client(moto_server: str) -> Generator[S3Client, None, None]:- client = boto3.client(- "s3",- aws_access_key_id="testing",- aws_secret_access_key="testing",- region_name=settings.AWS_REGION,- endpoint_url=moto_server,- )- client.create_bucket(Bucket=TEST_BUCKET) # Ensure the bucket exists for the test- client.create_bucket(Bucket=settings.AWS_S3_BUCKET_UPLOADS)- yield client--@freeze_time("2025-06-09T12:00:00")class TestS3StorageBuildURIs:def test_build_uri(self, s3_storage: S3Storage) -> None:@@ -209,6 +164,8 @@ def _assert_object_content(boto3_test_client: S3Client, uri: str, expected_conteclass TestS3StorageStore:"""Test S3Storage store methods."""+ __test__ = False # Collected with moto fixtures in test_s3_storage_moto.py.+def _create_artifact_for_ai_suggestion(self,s3_storage: S3Storage,@@ -264,6 +221,8 @@ TEST_BROWSER_SESSION_ID = "bs_test_123"class TestS3StorageBrowserSessionFiles:"""Test S3Storage browser session file methods."""+ __test__ = False # Moto cases are collected in test_s3_storage_moto.py.+async def test_sync_browser_session_file_with_date(self, s3_storage: S3Storage, boto3_test_client: S3Client, tmp_path: Path) -> None:@@ -811,6 +770,8 @@ CONTENT_TYPE_TEST_CASES = [class TestS3StorageContentType:"""Test S3Storage content type guessing."""+ __test__ = False # Collected with moto fixtures in test_s3_storage_moto.py.+@pytest.mark.parametrize("filename,expected_content_type,artifact_type,date", CONTENT_TYPE_TEST_CASES)async def test_content_type_guessing(self,@@ -844,6 +805,8 @@ class TestS3StorageContentType:class TestS3StorageHARCompression:"""Test S3Storage HAR file compression with zstd."""+ __test__ = False # Collected with moto fixtures in test_s3_storage_moto.py.+def _create_har_artifact(self, s3_storage: S3Storage, step_id: str) -> Artifact:"""Helper method to create a HAR Artifact."""artifact_id_val = generate_artifact_id()@@ -946,6 +909,8 @@ _build_zip = ArtifactManager._build_zipclass TestS3StorageZIPArchiveRetrieve:"""Test retrieve_artifact with STEP_ARCHIVE / TASK_ARCHIVE bundle_key extraction."""+ __test__ = False # Moto cases are collected in test_s3_storage_moto.py.+def _make_archive_artifact(self,s3_storage: S3Storage,@@ -1151,6 +1116,8 @@ class TestS3StoragePerRunRecordingClips:``S3Storage.sync_browser_session_file(videos)`` against moto S3 with a realffmpeg-generated recording, asserting a run-scoped clip is cut and uploaded."""+ __test__ = False # Collected with moto fixtures in test_s3_storage_moto.py.+async def test_session_close_cuts_and_uploads_run_scoped_clip(self,s3_storage: S3Storage,@@ -1223,3 +1190,37 @@ class TestS3StoragePerRunRecordingClips:clip_key = S3Uri(clip_uri).keyhead = boto3_test_client.head_object(Bucket=TEST_BUCKET, Key=clip_key)assert head["ContentLength"] > 0+++class TestS3StorageBrowserSessionPure:+ """URI authorization checks that do not perform S3 I/O."""++ test_assert_managed_file_access_accepts_org_scoped_uploads = (+ TestS3StorageBrowserSessionFiles.test_assert_managed_file_access_accepts_org_scoped_uploads+ )+ test_assert_managed_file_access_accepts_artifact_bucket = (+ TestS3StorageBrowserSessionFiles.test_assert_managed_file_access_accepts_artifact_bucket+ )+ test_assert_managed_file_access_rejects_other_org = (+ TestS3StorageBrowserSessionFiles.test_assert_managed_file_access_rejects_other_org+ )+ test_assert_managed_file_access_rejects_other_org_artifact_bucket = (+ TestS3StorageBrowserSessionFiles.test_assert_managed_file_access_rejects_other_org_artifact_bucket+ )+ test_download_managed_file_rejects_other_org = (+ TestS3StorageBrowserSessionFiles.test_download_managed_file_rejects_other_org+ )+ test_storage_type_property = TestS3StorageBrowserSessionFiles.test_storage_type_property+++class TestS3StorageZIPArchivePure:+ """Archive URI checks that do not perform S3 I/O."""++ test_build_uri_step_archive_has_zip_extension = (+ TestS3StorageZIPArchiveRetrieve.test_build_uri_step_archive_has_zip_extension+ )+ test_build_uri_task_archive_has_zip_extension = (+ TestS3StorageZIPArchiveRetrieve.test_build_uri_task_archive_has_zip_extension+ )diff --git a/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py b/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.pynew file mode 100644index 000000000..0c725190c--- /dev/null+++ b/tests/unit/forge/sdk/artifact/storage/test_s3_storage_moto.py@@ -0,0 +1,112 @@+"""Moto-backed S3 storage contracts separated from pure storage tests."""++from __future__ import annotations++from typing import Generator+from unittest.mock import AsyncMock, MagicMock++import boto3+import pytest+from moto.server import ThreadedMotoServer+from types_boto3_s3.client import S3Client++from skyvern.config import settings+from skyvern.forge.sdk.artifact.storage.s3 import S3Storage+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (+ TEST_BUCKET,+ S3StorageForTests,+)+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (+ TestS3StorageBrowserSessionFiles as _BrowserSessionContracts,+)+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import TestS3StorageContentType as _ContentTypeContracts+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (+ TestS3StorageHARCompression as _HARCompressionContracts,+)+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (+ TestS3StoragePerRunRecordingClips as _PerRunRecordingContracts,+)+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import TestS3StorageStore as _StoreContracts+from tests.unit.forge.sdk.artifact.storage.test_s3_storage import (+ TestS3StorageZIPArchiveRetrieve as _ZIPArchiveContracts,+)+++def s3_storage(moto_server: str) -> S3Storage:+ return S3StorageForTests(bucket=TEST_BUCKET, endpoint_url=moto_server)++[email protected](autouse=True)+def aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing")+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing")++[email protected](autouse=True)+def mock_browser_session_artifact_create(monkeypatch: pytest.MonkeyPatch) -> None:+ """Stub DB artifact-row inserts while exercising real moto object storage."""+ import skyvern.forge.sdk.artifact.storage.s3 as s3_module++ fake_app = MagicMock()+ fake_app.ARTIFACT_MANAGER.create_browser_session_download_artifact = AsyncMock(return_value="a_test")+ fake_app.ARTIFACT_MANAGER.create_browser_session_recording_artifact = AsyncMock(return_value="a_test")+ monkeypatch.setattr(s3_module, "app", fake_app)++[email protected](scope="module")+def moto_server() -> Generator[str]:+ server = ThreadedMotoServer(port=0)+ server.start()+ host, port = server.get_host_and_port()+ yield f"http://{host}:{port}"+ server.stop()++[email protected](scope="module", autouse=True)+def boto3_test_client(moto_server: str) -> Generator[S3Client]:+ client = boto3.client(+ "s3",+ aws_access_key_id="testing",+ aws_secret_access_key="testing",+ region_name=settings.AWS_REGION,+ endpoint_url=moto_server,+ )+ client.create_bucket(Bucket=TEST_BUCKET)+ client.create_bucket(Bucket=settings.AWS_S3_BUCKET_UPLOADS)+ yield client+++class TestS3StorageStoreMoto(_StoreContracts):+ __test__ = True+++class TestS3StorageBrowserSessionFilesMoto(_BrowserSessionContracts):+ __test__ = True++ # These checks are pure URI/guard behavior and stay in the fast module below.+ test_assert_managed_file_access_accepts_org_scoped_uploads = None+ test_assert_managed_file_access_accepts_artifact_bucket = None+ test_assert_managed_file_access_rejects_other_org = None+ test_assert_managed_file_access_rejects_other_org_artifact_bucket = None+ test_download_managed_file_rejects_other_org = None+ test_storage_type_property = None+++class TestS3StorageContentTypeMoto(_ContentTypeContracts):+ __test__ = True+++class TestS3StorageHARCompressionMoto(_HARCompressionContracts):+ __test__ = True+++class TestS3StorageZIPArchiveRetrieveMoto(_ZIPArchiveContracts):+ __test__ = True++ test_build_uri_step_archive_has_zip_extension = None+ test_build_uri_task_archive_has_zip_extension = None+++class TestS3StoragePerRunRecordingClipsMoto(_PerRunRecordingContracts):+ __test__ = Truediff --git a/tests/unit/forge/sdk/test_reset_browser_profile_route.py b/tests/unit/forge/sdk/test_reset_browser_profile_route.pyindex caf57fd03..5fc594596 100644--- a/tests/unit/forge/sdk/test_reset_browser_profile_route.py+++ b/tests/unit/forge/sdk/test_reset_browser_profile_route.py@@ -2,6 +2,7 @@from __future__ import annotations+from collections.abc import Generatorfrom types import SimpleNamespacefrom unittest.mock import AsyncMock, call@@ -16,7 +17,8 @@ from skyvern.forge.sdk.routes import routers as routers_modulefrom skyvern.forge.sdk.services import org_auth_service-def _build_client(monkeypatch: pytest.MonkeyPatch) -> tuple[TestClient, SimpleNamespace]:[email protected](scope="module")+def app_client_and_mocks() -> Generator[tuple[TestClient, SimpleNamespace]]:async def _fake_org() -> SimpleNamespace:return SimpleNamespace(organization_id="org_oss")@@ -27,29 +29,44 @@ def _build_client(monkeypatch: pytest.MonkeyPatch) -> tuple[TestClient, SimpleNadelete_profile_blob=AsyncMock(),delete_db_profile=AsyncMock(),)- monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", mocks.get_workflow)- monkeypatch.setattr(forge_app.STORAGE, "delete_browser_session", mocks.delete_browser_session)- monkeypatch.setattr(forge_app.STORAGE, "delete_browser_profile", mocks.delete_profile_blob)- monkeypatch.setattr(- forge_app.DATABASE.browser_sessions,- "list_managed_browser_profiles_for_workflow",- mocks.list_managed_profiles,- )- monkeypatch.setattr(forge_app.DATABASE.browser_sessions, "delete_browser_profile", mocks.delete_db_profile)-- fastapi_app = FastAPI()- fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org- fastapi_app.include_router(routers_module.base_router, prefix="/v1")-- @fastapi_app.exception_handler(SkyvernHTTPException)- async def _handle_skyvern_http_exception(request: Request, exc: SkyvernHTTPException) -> JSONResponse:- return JSONResponse(status_code=exc.status_code, content={"detail": exc.message})-- return TestClient(fastapi_app), mocks---def test_reset_browser_profile_clears_storage(monkeypatch: pytest.MonkeyPatch) -> None:- client, mocks = _build_client(monkeypatch)+ module_patches = pytest.MonkeyPatch()+ try:+ module_patches.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", mocks.get_workflow)+ module_patches.setattr(forge_app.STORAGE, "delete_browser_session", mocks.delete_browser_session)+ module_patches.setattr(forge_app.STORAGE, "delete_browser_profile", mocks.delete_profile_blob)+ module_patches.setattr(+ forge_app.DATABASE.browser_sessions,+ "list_managed_browser_profiles_for_workflow",+ mocks.list_managed_profiles,+ )+ module_patches.setattr(forge_app.DATABASE.browser_sessions, "delete_browser_profile", mocks.delete_db_profile)++ fastapi_app = FastAPI()+ fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org+ fastapi_app.include_router(routers_module.base_router, prefix="/v1")++ @fastapi_app.exception_handler(SkyvernHTTPException)+ async def _handle_skyvern_http_exception(request: Request, exc: SkyvernHTTPException) -> JSONResponse:+ return JSONResponse(status_code=exc.status_code, content={"detail": exc.message})++ with TestClient(fastapi_app) as client:+ yield client, mocks+ finally:+ module_patches.undo()+++def client_and_mocks(app_client_and_mocks: tuple[TestClient, SimpleNamespace]) -> tuple[TestClient, SimpleNamespace]:+ """Reuse the app/router while returning pristine mock state to every test."""+ client, mocks = app_client_and_mocks+ for mock in vars(mocks).values():+ mock.reset_mock(return_value=True, side_effect=True)+ mocks.list_managed_profiles.return_value = []+ return client, mocks+++def test_reset_browser_profile_clears_storage(client_and_mocks: tuple[TestClient, SimpleNamespace]) -> None:+ client, mocks = client_and_mocksmocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")response = client.post("/v1/workflows/wpid_123/browser_session/reset_profile")@@ -64,8 +81,8 @@ def test_reset_browser_profile_clears_storage(monkeypatch: pytest.MonkeyPatch) -mocks.delete_db_profile.assert_not_awaited()-def test_reset_browser_profile_workflow_not_found(monkeypatch: pytest.MonkeyPatch) -> None:- client, mocks = _build_client(monkeypatch)+def test_reset_browser_profile_workflow_not_found(client_and_mocks: tuple[TestClient, SimpleNamespace]) -> None:+ client, mocks = client_and_mocksmocks.get_workflow.side_effect = WorkflowNotFound(workflow_permanent_id="wpid_missing")response = client.post("/v1/workflows/wpid_missing/browser_session/reset_profile")@@ -75,8 +92,10 @@ def test_reset_browser_profile_workflow_not_found(monkeypatch: pytest.MonkeyPatcmocks.list_managed_profiles.assert_not_awaited()-def test_reset_browser_profile_storage_failure_returns_500(monkeypatch: pytest.MonkeyPatch) -> None:- client, mocks = _build_client(monkeypatch)+def test_reset_browser_profile_storage_failure_returns_500(+ client_and_mocks: tuple[TestClient, SimpleNamespace],+) -> None:+ client, mocks = client_and_mocksmocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")mocks.delete_browser_session.side_effect = RuntimeError("s3 AccessDenied")@@ -86,8 +105,10 @@ def test_reset_browser_profile_storage_failure_returns_500(monkeypatch: pytest.Massert "retry" in response.json()["detail"].lower()-def test_reset_browser_profile_clears_managed_profile_blobs_and_rows(monkeypatch: pytest.MonkeyPatch) -> None:- client, mocks = _build_client(monkeypatch)+def test_reset_browser_profile_clears_managed_profile_blobs_and_rows(+ client_and_mocks: tuple[TestClient, SimpleNamespace],+) -> None:+ client, mocks = client_and_mocksmocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")mocks.list_managed_profiles.return_value = [SimpleNamespace(browser_profile_id="bp_one", browser_profile_key_digest="", deleted_at=None),@@ -112,8 +133,10 @@ def test_reset_browser_profile_clears_managed_profile_blobs_and_rows(monkeypatch]-def test_reset_browser_profile_clears_segmented_legacy_archives(monkeypatch: pytest.MonkeyPatch) -> None:- client, mocks = _build_client(monkeypatch)+def test_reset_browser_profile_clears_segmented_legacy_archives(+ client_and_mocks: tuple[TestClient, SimpleNamespace],+) -> None:+ client, mocks = client_and_mocksmocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")mocks.list_managed_profiles.return_value = [SimpleNamespace(browser_profile_id="bp_seg", browser_profile_key_digest="abc123digest", deleted_at=None),@@ -144,8 +167,8 @@ def test_reset_browser_profile_clears_segmented_legacy_archives(monkeypatch: pyt"/v1/workflows/wpid_123/browser_session/reset_profile/",],)-def test_alias_paths_still_work(monkeypatch: pytest.MonkeyPatch, path: str) -> None:- client, mocks = _build_client(monkeypatch)+def test_alias_paths_still_work(client_and_mocks: tuple[TestClient, SimpleNamespace], path: str) -> None:+ client, mocks = client_and_mocksmocks.get_workflow.return_value = SimpleNamespace(workflow_permanent_id="wpid_123")response = client.post(path)diff --git a/tests/unit/test_code_block_lint_mcp_tool.py b/tests/unit/test_code_block_lint_mcp_tool.pyindex b5d7acc71..bed44ea30 100644--- a/tests/unit/test_code_block_lint_mcp_tool.py+++ b/tests/unit/test_code_block_lint_mcp_tool.py@@ -2,11 +2,21 @@from __future__ import annotations+import sys+from types import ModuleType, SimpleNamespace+import pytestfrom skyvern.cli.mcp_tools.code_block import skyvern_code_block_lint[email protected](autouse=True)+def _stub_mypy_for_non_mypy_lint_contracts(monkeypatch: pytest.MonkeyPatch) -> None:+ fake_mypy = ModuleType("mypy")+ fake_mypy.__dict__["api"] = SimpleNamespace(run=lambda _args: ("", "", 0))+ monkeypatch.setitem(sys.modules, "mypy", fake_mypy)++def _has_security_error(result: dict, *, reason_code: str, surface: str | None = None) -> bool:return any(error["reason_code"] == reason_code and (surface is None or error["surface"] == surface)diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.pyindex 0c929e2dd..1a4c8c708 100644--- a/tests/unit/test_copilot_code_block_synthesis.py+++ b/tests/unit/test_copilot_code_block_synthesis.py@@ -10,6 +10,7 @@ import jsonimport keywordimport sysimport textwrap+from types import ModuleType, SimpleNamespacefrom typing import Anyimport pytest@@ -45,6 +46,23 @@ from skyvern.forge.sdk.copilot.tools.workflow_update import _code_block_safety_efrom skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockStep[email protected](autouse=True)+def _stub_mypy_for_static_policy_cases(+ request: pytest.FixtureRequest,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ if "real_mypy" in request.fixturenames:+ return+ fake_mypy = ModuleType("mypy")+ fake_mypy.__dict__["api"] = SimpleNamespace(run=lambda _args: ("", "", 0))+ monkeypatch.setitem(sys.modules, "mypy", fake_mypy)+++def real_mypy() -> None:+ return None++def _interaction(tool_name: str, **fields: Any) -> dict[str, Any]:return {"tool_name": tool_name, **fields}@@ -2069,7 +2087,32 @@ class TestCredentialFillSynthesis:assert _code_block_safety_errors(workflow_yaml, None) == []-def test_code_block_preflight_restores_recursion_limit() -> None:+ ("code", "expected_codes"),+ [+ pytest.param(+ "await page.locator('button[type=submit]').first.click(timeout=5000)\n",+ (),+ id="valid-locator-click",+ ),+ pytest.param(+ "await page.locator('button[type=submit]').first().click(timeout=5000)\n",+ ("PLAYWRIGHT_API_MISMATCH",),+ id="locator-property-called-as-method",+ ),+ ],+)+def test_code_block_preflight_real_mypy_contract_matrix(+ real_mypy: None,+ code: str,+ expected_codes: tuple[str, ...],+) -> None:+ diagnostics = preflight_code_block(code)++ assert [diagnostic.code for diagnostic in diagnostics] == list(expected_codes)+++def test_code_block_preflight_restores_recursion_limit(real_mypy: None) -> None:before = sys.getrecursionlimit()preflight_code_block("await page.locator('button[type=submit]').first.click(timeout=5000)\n")diff --git a/tests/unit/test_copilot_composition_scrape.py b/tests/unit/test_copilot_composition_scrape.pyindex 3a2ee64ef..4250e023f 100644--- a/tests/unit/test_copilot_composition_scrape.py+++ b/tests/unit/test_copilot_composition_scrape.py@@ -5,7 +5,9 @@ served in a real scout because the agent acts between inspects.)from __future__ import annotations+import asynciofrom types import SimpleNamespace+from unittest.mock import AsyncMockimport pytest@@ -13,6 +15,14 @@ from skyvern.forge.sdk.copilot import toolsfrom skyvern.forge.sdk.copilot.tools import _normalized_inspect_url, _same_inspect_target+class _AsyncioSleepProxy:+ def __init__(self, sleep: AsyncMock) -> None:+ self.sleep = sleep++ def __getattr__(self, name: str):+ return getattr(asyncio, name)++def test_normalized_inspect_url_preserves_distinguishing_parts() -> None:assert _normalized_inspect_url("https://Example.com/Search?q=a#frag") == "https://example.com/Search?q=a#frag"# query distinguishes search states; scheme and trailing slash are significant@@ -61,6 +71,8 @@ async def test_recapture_skips_raw_get_html_after_cap_drop(monkeypatch: pytest.Mmonkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity)+ settle_sleep = AsyncMock()+ monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(settle_sleep))evidence, html_error = await tools._capture_composition_evidence(SimpleNamespace(), inspected_url="https://example.com/s", current_url="https://example.com/s"@@ -71,3 +83,4 @@ async def test_recapture_skips_raw_get_html_after_cap_drop(monkeypatch: pytest.Massert tools.has_bounded_page_schema(evidence)# First iteration's raw read is cap-dropped; the settle retry skips it entirely.assert raw_calls["n"] == 1+ settle_sleep.assert_awaited_once_with(tools.composition_capture._COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.pyindex b722d7efb..f77a3ac57 100644--- a/tests/unit/test_download_file_action_handler.py+++ b/tests/unit/test_download_file_action_handler.py@@ -1367,7 +1367,6 @@ async def test_handle_action_discards_xhr_staging_when_native_file_present(task, step, page, browser_state, scraped_page, action = _make_download_click_context(now=now, organization=organization, page_url="https://example.com/download")-callbacks: dict[str, object] = {}page.on.side_effect = lambda event, cb: callbacks.__setitem__(event, cb)@@ -1430,6 +1429,7 @@ async def test_handle_action_uses_xhr_staging_fallback_when_no_native_file(task, step, page, browser_state, scraped_page, action = _make_download_click_context(now=now, organization=organization, page_url="https://example.com/download")+ task.download_timeout = 0.01callbacks: dict[str, object] = {}page.on.side_effect = lambda event, cb: callbacks.__setitem__(event, cb)@@ -1490,6 +1490,7 @@ async def test_handle_action_moves_multiple_staged_xhr_files_as_fallback(task, step, page, browser_state, scraped_page, action = _make_download_click_context(now=now, organization=organization, page_url="https://example.com/download")+ task.download_timeout = 0.01callbacks: dict[str, object] = {}page.on.side_effect = lambda event, cb: callbacks.__setitem__(event, cb)@@ -2021,7 +2022,7 @@ async def test_handle_action_adopted_session_helper_failure_does_not_short_circuorganization,workflow_run_id="wr-1",browser_session_id="bs-1",- download_timeout=2.0,+ download_timeout=0.01,)step = make_step(now, task, step_id="step-1", status=StepStatus.created, order=0, output=None)diff --git a/tests/unit/test_load_prompt_with_elements_ceiling.py b/tests/unit/test_load_prompt_with_elements_ceiling.pyindex aae067476..d58e0a394 100644--- a/tests/unit/test_load_prompt_with_elements_ceiling.py+++ b/tests/unit/test_load_prompt_with_elements_ceiling.py@@ -4,6 +4,17 @@ from __future__ import annotationsfrom unittest.mock import MagicMock+import pytest+++def small_prompt_ceiling(monkeypatch: pytest.MonkeyPatch) -> int:+ from skyvern.utils import prompt_engine++ ceiling = 500+ monkeypatch.setattr(prompt_engine, "PROMPT_HARD_CEILING_TOKENS", ceiling)+ return ceiling+def test_prompt_hard_ceiling_is_below_gpt5_mini_cap() -> None:from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS@@ -41,8 +52,7 @@ def test_load_prompt_with_elements_drops_previous_info_when_over_ceiling() -> Nofrom skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, load_prompt_with_elementsfrom skyvern.utils.token_counter import count_tokens- # List of distinct English-ish words well over the 180k token ceiling.- oversized_prev = [{"iter": i, "marker": f"UNIQUE_BLOCK_{i}_" + ("lorem ipsum " * 200)} for i in range(3000)]+ oversized_prev = [{"iter": 0, "marker": "UNIQUE_BLOCK_0_" + ("lorem " * 185_000)}]rendered = load_prompt_with_elements(element_tree_builder=_make_element_tree_builder(),@@ -62,12 +72,12 @@ def test_load_prompt_with_elements_drops_previous_info_when_over_ceiling() -> Noassert "UNIQUE_BLOCK_0_" not in rendered-def test_enforce_prompt_ceiling_drops_fallback_keys_without_elements() -> None:+def test_enforce_prompt_ceiling_drops_fallback_keys_without_elements(small_prompt_ceiling: int) -> None:from skyvern.forge.prompts import prompt_engine as engine_modulefrom skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, enforce_prompt_ceilingfrom skyvern.utils.token_counter import count_tokens- giant_schema = {"type": "object", "_blob": "lorem " * 300_000}+ giant_schema = {"type": "object", "_blob": "lorem " * (small_prompt_ceiling + 100)}kwargs = {"data_extraction_goal": "Extract","data_extraction_schema": giant_schema,@@ -109,12 +119,14 @@ def test_load_prompt_with_elements_respects_ceiling_for_small_prompts() -> None:assert count_tokens(rendered) <= PROMPT_HARD_CEILING_TOKENS-def test_load_prompt_with_elements_tracked_drops_extracted_text_as_last_resort() -> None:+def test_load_prompt_with_elements_tracked_drops_extracted_text_as_last_resort(+ small_prompt_ceiling: int,+) -> None:from skyvern.forge.prompts import prompt_engine as engine_modulefrom skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, load_prompt_with_elements_trackedfrom skyvern.utils.token_counter import count_tokens- oversized_extracted_text = "UNIQUE_EXTRACTED_TEXT " * 300_000+ oversized_extracted_text = "UNIQUE_EXTRACTED_TEXT " + ("lorem " * (small_prompt_ceiling + 100))rendered, post_kwargs = load_prompt_with_elements_tracked(element_tree_builder=_make_element_tree_builder(),@@ -135,12 +147,12 @@ def test_load_prompt_with_elements_tracked_drops_extracted_text_as_last_resort()assert post_kwargs["extracted_text"] is None-def test_enforce_prompt_ceiling_tracked_reports_dropped_keys() -> None:+def test_enforce_prompt_ceiling_tracked_reports_dropped_keys(small_prompt_ceiling: int) -> None:from skyvern.forge.prompts import prompt_engine as engine_modulefrom skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, enforce_prompt_ceiling_trackedfrom skyvern.utils.token_counter import count_tokens- giant_schema = {"type": "object", "_blob": "lorem " * 300_000}+ giant_schema = {"type": "object", "_blob": "lorem " * (small_prompt_ceiling + 100)}kwargs = {"data_extraction_goal": "Extract","data_extraction_schema": giant_schema,@@ -184,12 +196,12 @@ def test_enforce_prompt_ceiling_tracked_noop_under_ceiling() -> None:assert post_kwargs["data_extraction_schema"] == {"type": "object"}-def test_load_prompt_with_elements_tracked_reports_dropped_keys() -> None:+def test_load_prompt_with_elements_tracked_reports_dropped_keys(small_prompt_ceiling: int) -> None:from skyvern.forge.prompts import prompt_engine as engine_modulefrom skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, load_prompt_with_elements_trackedfrom skyvern.utils.token_counter import count_tokens- oversized_prev = [{"iter": i, "marker": f"UNIQUE_BLOCK_{i}_" + ("lorem ipsum " * 200)} for i in range(3000)]+ oversized_prev = [{"iter": 0, "marker": "UNIQUE_BLOCK_0_" + ("lorem " * (small_prompt_ceiling + 100))}]rendered, post_kwargs = load_prompt_with_elements_tracked(element_tree_builder=_make_element_tree_builder(),diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.pyindex 3394ab247..dd4add53a 100644--- a/tests/unit/test_real_browser_state_navigate.py+++ b/tests/unit/test_real_browser_state_navigate.py@@ -4,19 +4,25 @@ Covers SKY-8818: pages whose subresources never finish loading must stillsucceed if the DOM has parsed, via progressive wait_until degradation."""+from functools import partialfrom unittest.mock import AsyncMock, MagicMockimport pytestfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrorfrom skyvern.exceptions import FailedToNavigateToUrl+from skyvern.webeye import real_browser_state+from skyvern.webeye.navigation import navigate_with_retryfrom skyvern.webeye.real_browser_state import RealBrowserState, _same_page_ignoring_fragment@pytest.fixture-def browser_state() -> RealBrowserState:+def browser_state(monkeypatch: pytest.MonkeyPatch) -> RealBrowserState:+ retry_sleep = AsyncMock()+ monkeypatch.setattr(real_browser_state, "navigate_with_retry", partial(navigate_with_retry, sleep=retry_sleep))# Bypass __init__; navigate_to_url only uses `self` for LOG context and _wait_for_settle.state = RealBrowserState.__new__(RealBrowserState)+ monkeypatch.setattr(state, "_wait_for_settle", AsyncMock())return statediff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.pyindex fc183a669..f87921151 100644--- a/tests/unit/test_script_skyvern_page.py+++ b/tests/unit/test_script_skyvern_page.py@@ -10,6 +10,7 @@ and wait() (accepts both seconds= and timeout_ms= parameter styles).import inspectimport reimport time+from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -21,28 +22,46 @@ from skyvern.core.script_generations.skyvern_page import SkyvernPagefrom skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException-def create_mock_page():- """Create a mock Playwright Page object with required attributes."""- page = MagicMock()- page.url = "https://example.com"- # Required for Playwright Page base class- page._loop = MagicMock()- page._impl_obj = page- return page+class _KeyboardStub:+ async def press(self, *_args: object, **_kwargs: object) -> None:+ return None+++class _PageStub:+ def __init__(self) -> None:+ self.url = "https://example.com"+ self.keyboard = _KeyboardStub()++ async def evaluate(self, *_args: object, **_kwargs: object) -> None:+ return None+++def create_mock_page() -> _PageStub:+ return _PageStub()+++class _AiStub:+ pass+++class _ScrapedPageStub:+ def __init__(self) -> None:+ self._browser_state = SimpleNamespace()++[email protected](autouse=True)+def no_cached_action_delay(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(settings, "CACHED_ACTION_DELAY_SECONDS", 0)@pytest.fixturedef mock_scraped_page():- """Create a mock ScrapedPage object."""- scraped_page = MagicMock()- scraped_page._browser_state = MagicMock()- return scraped_page+ return _ScrapedPageStub()@pytest.fixturedef mock_ai():- """Create a mock SkyvernPageAi object."""- return MagicMock()+ return _AiStub()@pytest.mark.asyncio@@ -243,8 +262,7 @@ async def test_ensure_element_ids_skips_when_ids_exist(mock_scraped_page, mock_ashould NOT be called (fast path)."""mock_page = create_mock_page()- # SkyvernPage.__getattribute__ delegates self.page to mock_page.page- mock_page.page.evaluate = AsyncMock(return_value=True) # unique_ids exist+ mock_page.evaluate = AsyncMock(return_value=True) # unique_ids existwith patch("skyvern.core.script_generations.skyvern_page.Page.__init__",@@ -273,9 +291,7 @@ async def test_ensure_element_ids_injects_when_ids_missing(mock_scraped_page, modomUtils.js and call buildTreeFromBody to set them."""mock_page = create_mock_page()- # SkyvernPage.__getattribute__ delegates self.page to mock_page.page,- # so set evaluate on the delegated object- mock_page.page.evaluate = AsyncMock(return_value=False) # no unique_ids+ mock_page.evaluate = AsyncMock(return_value=False) # no unique_idswith patch("skyvern.core.script_generations.skyvern_page.Page.__init__",@@ -341,8 +357,7 @@ async def test_ensure_element_ids_catches_exceptions(mock_scraped_page, mock_ai)action execution."""mock_page = create_mock_page()- # SkyvernPage.__getattribute__ delegates self.page to mock_page.page- mock_page.page.evaluate = AsyncMock(side_effect=Exception("Page crashed"))+ mock_page.evaluate = AsyncMock(side_effect=Exception("Page crashed"))with patch("skyvern.core.script_generations.skyvern_page.Page.__init__",@@ -1648,8 +1663,6 @@ def _skyvern_page_with_locator(mock_ai, locator: _RecordingLocator) -> SkyvernPareturn_value=None,):raw_page = create_mock_page()- raw_page.page = raw_page- raw_page.keyboard.press = AsyncMock()skyvern_page = SkyvernPage(page=raw_page,ai=mock_ai,