Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Drive the registered-browser deadline tests with a frozen test clock (SKY-18881) (#9034)
Shuchang Zhengdd1f938e7e45
diff --git a/tests/unit/_external_cdp_fakes.py b/tests/unit/_external_cdp_fakes.pyindex d9cdc20ac..63a32e59b 100644--- a/tests/unit/_external_cdp_fakes.py+++ b/tests/unit/_external_cdp_fakes.py@@ -1,11 +1,12 @@import asynciofrom collections.abc import AsyncIteratorfrom contextlib import asynccontextmanager-from datetime import datetime, timedelta, timezone+from datetime import UTC, datetime, timedelta, timezone, tzinfofrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport httpx+import pytestfrom sqlalchemy import updatefrom sqlalchemy.ext.asyncio import async_sessionmaker, create_async_enginefrom sqlalchemy.pool import StaticPool@@ -19,13 +20,16 @@ from skyvern.forge.sdk.db.repositories.browser_sessions import BrowserSessionsRefrom skyvern.forge.sdk.encrypt import encryptorfrom skyvern.forge.sdk.encrypt.bootstrap import register_aes_encryptorfrom skyvern.forge.sdk.routes import browser_sessions as routes_mod+from skyvern.forge.sdk.schemas import persistent_browser_sessions as persistent_browser_sessions_modfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import EXTERNAL_CDP_BROWSER_VENDOR, PersistentBrowserSessionfrom skyvern.forge.sdk.services import org_auth_service+from skyvern.webeye import external_cdp_sessions as external_modfrom skyvern.webeye.browser_artifacts import BrowserArtifactsfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.default_persistent_sessions_manager import DefaultPersistentSessionsManagerfrom skyvern.webeye.persistent_sessions_manager import PersistentSessionsManagerfrom skyvern.webeye.real_browser_state import RealBrowserState+from tests.unit.scoped_asyncio import ScopedAsyncioTOKEN = "tok-9f3c1d7e5b"CDP_URL = f"wss://127.0.0.1:9333/devtools/browser/b1?token={TOKEN}"@@ -125,12 +129,41 @@ async def external_cdp_stack() -> AsyncIterator[SimpleNamespace]:await engine.dispose()-async def backdate_registration(stack: SimpleNamespace, session_id: str, minutes: float) -> None:+class DeadlineClock:+ """The clock registration deadlines are read against and the deadline watcher's sleep: frozen until a test sets+ ``ticking``, after which each watcher poll moves it by exactly what the poll slept. Concurrent watchers share it,+ so it moves by the sum of their sleeps."""++ def __init__(self) -> None:+ self._now = datetime.now(UTC)+ self.ticking = asyncio.Event()+ self.polls: list[float] = []++ def now(self, tz: tzinfo) -> datetime:+ return self._now.astimezone(tz)++ async def sleep(self, seconds: float) -> None:+ self.polls.append(seconds)+ await self.ticking.wait()+ self._now += timedelta(seconds=seconds)+++def install_deadline_clock(monkeypatch: pytest.MonkeyPatch) -> DeadlineClock:+ clock = DeadlineClock()+ monkeypatch.setattr(persistent_browser_sessions_mod, "datetime", clock)+ monkeypatch.setattr(external_mod, "asyncio", ScopedAsyncio(sleep=clock.sleep))+ return clock+++async def backdate_registration(+ stack: SimpleNamespace, session_id: str, minutes: float, *, clock: DeadlineClock | None = None+) -> None:+ now = naive_utc_now() if clock is None else clock.now(UTC).replace(tzinfo=None)async with stack.session_factory() as session:await session.execute(update(PersistentBrowserSessionModel).where(PersistentBrowserSessionModel.persistent_browser_session_id == session_id)- .values(started_at=naive_utc_now() - timedelta(minutes=minutes))+ .values(started_at=now - timedelta(minutes=minutes)))await session.commit()diff --git a/tests/unit/test_external_cdp_sessions.py b/tests/unit/test_external_cdp_sessions.pyindex 93e0bf01f..53bed7860 100644--- a/tests/unit/test_external_cdp_sessions.py+++ b/tests/unit/test_external_cdp_sessions.py@@ -1,6 +1,7 @@import asyncioimport jsonfrom collections.abc import AsyncIterator, Callable+from datetime import UTC, timedeltafrom pathlib import Pathfrom types import SimpleNamespacefrom typing import Any, cast@@ -61,6 +62,7 @@ from tests.unit._external_cdp_fakes import (begin_lease,external_browser_state,external_cdp_stack,+ install_deadline_clock,register_external,release_lease,release_lease_by,@@ -724,34 +726,28 @@ async def test_a_run_that_waited_behind_another_attach_while_its_lease_moved_getasync def test_the_lease_holder_detaches_when_its_registration_expires_or_another_process_closes_it(stack: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, end: str) -> None:+ clock = install_deadline_clock(monkeypatch)session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]attached, context, pw = external_browser_state(pages=2)- polls: list[float] = []- next_poll = asyncio.Event()-- async def poll_sleep(seconds: float) -> None:- polls.append(seconds)- await next_poll.wait()if end == "deadline":- await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60)- else:- monkeypatch.setattr(external_mod, "asyncio", ScopedAsyncio(sleep=poll_sleep))+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)with patch.object(manager_mod, "attach_registered_cdp_browser", AsyncMock(return_value=attached)):generation = await begin_lease(stack.manager, session_id, "tsk_a")assert await acquire_lease(stack.manager, session_id, "tsk_a", generation) is attachedif end == "closed_elsewhere":await stack.repo.close_persistent_browser_session(session_id, ORG_A)- next_poll.set()+ clock.ticking.set()await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=5)assert await acquire_lease(stack.manager, session_id, "tsk_a", generation) is Noneclosed = await stack.repo.get_persistent_browser_session(session_id, ORG_A)assert closed is not None and closed.completed_at is not None and closed.upstream_cdp_url is Noneif end == "deadline":- assert closed.close_reason == "expired"+ assert closed.close_reason == "expired" and closed.started_at is not None+ assert clock.now(UTC) == closed.started_at.replace(tzinfo=UTC) + timedelta(minutes=5)else:- assert polls == [external_mod.EXTERNAL_CDP_CLOSE_POLL_SECONDS]+ assert clock.polls == [external_mod.EXTERNAL_CDP_CLOSE_POLL_SECONDS]assert session_id not in stack.manager._browser_sessions and not stack.manager._external_deadlinespw.stop.assert_awaited_once()context.close.assert_not_awaited()@@ -786,14 +782,18 @@ async def test_closing_a_registration_whose_driver_hangs_on_detach_still_closes_@pytest.mark.asyncio@pytest.mark.parametrize("stale", [False, True], ids=["owner_release", "stale_release"])-async def test_only_the_owners_release_cancels_the_deadline(stack: SimpleNamespace, stale: bool) -> None:+async def test_only_the_owners_release_cancels_the_deadline(+ stack: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, stale: bool+) -> None:+ clock = install_deadline_clock(monkeypatch)session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]- await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60)+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)generation = await begin_lease(stack.manager, session_id, "tsk_a")deadline = stack.manager._external_deadlines[session_id]released = await release_lease(stack.manager, session_id, "tsk_a", "gen_stale" if stale else generation)assert released is not stale+ clock.ticking.set()if stale:await asyncio.wait_for(deadline, timeout=5)else:@@ -950,8 +950,9 @@ async def test_the_deadline_watcher_keeps_its_own_lease_and_detaches_one_anotherasync def test_a_failed_close_at_the_deadline_is_retried_until_the_row_is_final(stack: SimpleNamespace, monkeypatch: pytest.MonkeyPatch) -> None:+ clock = install_deadline_clock(monkeypatch)session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]- await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60)+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)real_close = stack.repo.close_persistent_browser_sessionclose_attempts: list[str] = []@@ -961,12 +962,9 @@ async def test_a_failed_close_at_the_deadline_is_retried_until_the_row_is_final(raise RuntimeError("transient database error")return await real_close(session_id, organization_id)- async def short_sleep(seconds: float) -> None:- await asyncio.sleep(min(seconds, 0.1))-- monkeypatch.setattr(external_mod, "asyncio", ScopedAsyncio(sleep=short_sleep))with patch.object(stack.repo, "close_persistent_browser_session", close_fails_once):await begin_lease(stack.manager, session_id, "tsk_a")+ clock.ticking.set()await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=5)row = await stack.repo.get_persistent_browser_session(session_id, ORG_A)@@ -988,11 +986,12 @@ async def test_a_failed_close_at_the_deadline_is_retried_until_the_row_is_final(],)async def test_the_next_step_after_losing_a_registration_fails_with_that_fact_and_launches_nothing(- stack: SimpleNamespace, run_type: str, loss: str+ stack: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, run_type: str, loss: str) -> None:+ clock = install_deadline_clock(monkeypatch)session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]if loss == "expired":- await backdate_registration(stack, session_id, minutes=5 - 10 / 60)+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)run_id = "tsk_1" if run_type == "task" else "wr_1"generation = await stack.manager.begin_session(browser_session_id=session_id, runnable_type=run_type, runnable_id=run_id, organization_id=ORG_A@@ -1001,7 +1000,8 @@ async def test_the_next_step_after_losing_a_registration_fails_with_that_fact_anwith patch.object(manager_mod, "attach_registered_cdp_browser", AsyncMock(return_value=lost)):assert await acquire_lease(stack.manager, session_id, run_id, generation) is lostif loss == "expired":- await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=15)+ clock.ticking.set()+ await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=5)if loss in ("closed", "vanished"):await stack.repo.close_persistent_browser_session(session_id, ORG_A)context.browser.is_connected.return_value = Falsediff --git a/tests/unit/test_workflow_runtime_completion.py b/tests/unit/test_workflow_runtime_completion.pyindex 0a1ab1484..4d5469602 100644--- a/tests/unit/test_workflow_runtime_completion.py+++ b/tests/unit/test_workflow_runtime_completion.py@@ -36,6 +36,7 @@ from tests.unit._external_cdp_fakes import (begin_lease,external_browser_state,external_cdp_stack,+ install_deadline_clock,register_external,)from tests.unit.conftest import FakeWorkflowRunAttemptsRepository@@ -805,14 +806,16 @@ def test_apply_proposed_workflow_route_is_bound_to_the_route_handler() -> None:ids=["open", "expired_mid_run", "browser_died_mid_run", "browser_died_and_row_unreadable"],)async def test_a_workflow_whose_registered_browser_ended_mid_run_fails_at_finalization(- registration: str, browser_connected: bool, expected_reason: str | None+ monkeypatch: pytest.MonkeyPatch, registration: str, browser_connected: bool, expected_reason: str | None) -> None:+ clock = install_deadline_clock(monkeypatch)async with external_cdp_stack() as stack:session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]if registration == "expired":- await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60)+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)await begin_lease(stack.manager, session_id, "wr_a")if registration == "expired":+ clock.ticking.set()await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=5)run = SimpleNamespace(workflow_run_id="wr_a",