Skip to content

Automate browser based workflows with AI

Drive the registered-browser deadline tests with a frozen test clock (SKY-18881) (#9034)

Shuchang Zhengdd1f938e7e45
diff --git a/tests/unit/_external_cdp_fakes.py b/tests/unit/_external_cdp_fakes.py
index d9cdc20ac..63a32e59b 100644
--- a/tests/unit/_external_cdp_fakes.py
+++ b/tests/unit/_external_cdp_fakes.py
@@ -1,11 +1,12 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
-from datetime import datetime, timedelta, timezone
+from datetime import UTC, datetime, timedelta, timezone, tzinfo
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
+import pytest
from sqlalchemy import update
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import StaticPool
@@ -19,13 +20,16 @@ from skyvern.forge.sdk.db.repositories.browser_sessions import BrowserSessionsRe
from skyvern.forge.sdk.encrypt import encryptor
from skyvern.forge.sdk.encrypt.bootstrap import register_aes_encryptor
from skyvern.forge.sdk.routes import browser_sessions as routes_mod
+from skyvern.forge.sdk.schemas import persistent_browser_sessions as persistent_browser_sessions_mod
from skyvern.forge.sdk.schemas.persistent_browser_sessions import EXTERNAL_CDP_BROWSER_VENDOR, PersistentBrowserSession
from skyvern.forge.sdk.services import org_auth_service
+from skyvern.webeye import external_cdp_sessions as external_mod
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.default_persistent_sessions_manager import DefaultPersistentSessionsManager
from skyvern.webeye.persistent_sessions_manager import PersistentSessionsManager
from skyvern.webeye.real_browser_state import RealBrowserState
+from tests.unit.scoped_asyncio import ScopedAsyncio
TOKEN = "tok-9f3c1d7e5b"
CDP_URL = f"wss://127.0.0.1:9333/devtools/browser/b1?token={TOKEN}"
@@ -125,12 +129,41 @@ async def external_cdp_stack() -> AsyncIterator[SimpleNamespace]:
await engine.dispose()
-async def backdate_registration(stack: SimpleNamespace, session_id: str, minutes: float) -> None:
+class DeadlineClock:
+ """The clock registration deadlines are read against and the deadline watcher's sleep: frozen until a test sets
+ ``ticking``, after which each watcher poll moves it by exactly what the poll slept. Concurrent watchers share it,
+ so it moves by the sum of their sleeps."""
+
+ def __init__(self) -> None:
+ self._now = datetime.now(UTC)
+ self.ticking = asyncio.Event()
+ self.polls: list[float] = []
+
+ def now(self, tz: tzinfo) -> datetime:
+ return self._now.astimezone(tz)
+
+ async def sleep(self, seconds: float) -> None:
+ self.polls.append(seconds)
+ await self.ticking.wait()
+ self._now += timedelta(seconds=seconds)
+
+
+def install_deadline_clock(monkeypatch: pytest.MonkeyPatch) -> DeadlineClock:
+ clock = DeadlineClock()
+ monkeypatch.setattr(persistent_browser_sessions_mod, "datetime", clock)
+ monkeypatch.setattr(external_mod, "asyncio", ScopedAsyncio(sleep=clock.sleep))
+ return clock
+
+
+async def backdate_registration(
+ stack: SimpleNamespace, session_id: str, minutes: float, *, clock: DeadlineClock | None = None
+) -> None:
+ now = naive_utc_now() if clock is None else clock.now(UTC).replace(tzinfo=None)
async with stack.session_factory() as session:
await session.execute(
update(PersistentBrowserSessionModel)
.where(PersistentBrowserSessionModel.persistent_browser_session_id == session_id)
- .values(started_at=naive_utc_now() - timedelta(minutes=minutes))
+ .values(started_at=now - timedelta(minutes=minutes))
)
await session.commit()
diff --git a/tests/unit/test_external_cdp_sessions.py b/tests/unit/test_external_cdp_sessions.py
index 93e0bf01f..53bed7860 100644
--- a/tests/unit/test_external_cdp_sessions.py
+++ b/tests/unit/test_external_cdp_sessions.py
@@ -1,6 +1,7 @@
import asyncio
import json
from collections.abc import AsyncIterator, Callable
+from datetime import UTC, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any, cast
@@ -61,6 +62,7 @@ from tests.unit._external_cdp_fakes import (
begin_lease,
external_browser_state,
external_cdp_stack,
+ install_deadline_clock,
register_external,
release_lease,
release_lease_by,
@@ -724,34 +726,28 @@ async def test_a_run_that_waited_behind_another_attach_while_its_lease_moved_get
async def test_the_lease_holder_detaches_when_its_registration_expires_or_another_process_closes_it(
stack: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, end: str
) -> None:
+ clock = install_deadline_clock(monkeypatch)
session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]
attached, context, pw = external_browser_state(pages=2)
- polls: list[float] = []
- next_poll = asyncio.Event()
-
- async def poll_sleep(seconds: float) -> None:
- polls.append(seconds)
- await next_poll.wait()
if end == "deadline":
- await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60)
- else:
- monkeypatch.setattr(external_mod, "asyncio", ScopedAsyncio(sleep=poll_sleep))
+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)
with patch.object(manager_mod, "attach_registered_cdp_browser", AsyncMock(return_value=attached)):
generation = await begin_lease(stack.manager, session_id, "tsk_a")
assert await acquire_lease(stack.manager, session_id, "tsk_a", generation) is attached
if end == "closed_elsewhere":
await stack.repo.close_persistent_browser_session(session_id, ORG_A)
- next_poll.set()
+ clock.ticking.set()
await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=5)
assert await acquire_lease(stack.manager, session_id, "tsk_a", generation) is None
closed = await stack.repo.get_persistent_browser_session(session_id, ORG_A)
assert closed is not None and closed.completed_at is not None and closed.upstream_cdp_url is None
if end == "deadline":
- assert closed.close_reason == "expired"
+ assert closed.close_reason == "expired" and closed.started_at is not None
+ assert clock.now(UTC) == closed.started_at.replace(tzinfo=UTC) + timedelta(minutes=5)
else:
- assert polls == [external_mod.EXTERNAL_CDP_CLOSE_POLL_SECONDS]
+ assert clock.polls == [external_mod.EXTERNAL_CDP_CLOSE_POLL_SECONDS]
assert session_id not in stack.manager._browser_sessions and not stack.manager._external_deadlines
pw.stop.assert_awaited_once()
context.close.assert_not_awaited()
@@ -786,14 +782,18 @@ async def test_closing_a_registration_whose_driver_hangs_on_detach_still_closes_
@pytest.mark.asyncio
@pytest.mark.parametrize("stale", [False, True], ids=["owner_release", "stale_release"])
-async def test_only_the_owners_release_cancels_the_deadline(stack: SimpleNamespace, stale: bool) -> None:
+async def test_only_the_owners_release_cancels_the_deadline(
+ stack: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, stale: bool
+) -> None:
+ clock = install_deadline_clock(monkeypatch)
session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]
- await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60)
+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)
generation = await begin_lease(stack.manager, session_id, "tsk_a")
deadline = stack.manager._external_deadlines[session_id]
released = await release_lease(stack.manager, session_id, "tsk_a", "gen_stale" if stale else generation)
assert released is not stale
+ clock.ticking.set()
if stale:
await asyncio.wait_for(deadline, timeout=5)
else:
@@ -950,8 +950,9 @@ async def test_the_deadline_watcher_keeps_its_own_lease_and_detaches_one_another
async def test_a_failed_close_at_the_deadline_is_retried_until_the_row_is_final(
stack: SimpleNamespace, monkeypatch: pytest.MonkeyPatch
) -> None:
+ clock = install_deadline_clock(monkeypatch)
session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]
- await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60)
+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)
real_close = stack.repo.close_persistent_browser_session
close_attempts: list[str] = []
@@ -961,12 +962,9 @@ async def test_a_failed_close_at_the_deadline_is_retried_until_the_row_is_final(
raise RuntimeError("transient database error")
return await real_close(session_id, organization_id)
- async def short_sleep(seconds: float) -> None:
- await asyncio.sleep(min(seconds, 0.1))
-
- monkeypatch.setattr(external_mod, "asyncio", ScopedAsyncio(sleep=short_sleep))
with patch.object(stack.repo, "close_persistent_browser_session", close_fails_once):
await begin_lease(stack.manager, session_id, "tsk_a")
+ clock.ticking.set()
await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=5)
row = await stack.repo.get_persistent_browser_session(session_id, ORG_A)
@@ -988,11 +986,12 @@ async def test_a_failed_close_at_the_deadline_is_retried_until_the_row_is_final(
],
)
async def test_the_next_step_after_losing_a_registration_fails_with_that_fact_and_launches_nothing(
- stack: SimpleNamespace, run_type: str, loss: str
+ stack: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, run_type: str, loss: str
) -> None:
+ clock = install_deadline_clock(monkeypatch)
session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]
if loss == "expired":
- await backdate_registration(stack, session_id, minutes=5 - 10 / 60)
+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)
run_id = "tsk_1" if run_type == "task" else "wr_1"
generation = await stack.manager.begin_session(
browser_session_id=session_id, runnable_type=run_type, runnable_id=run_id, organization_id=ORG_A
@@ -1001,7 +1000,8 @@ async def test_the_next_step_after_losing_a_registration_fails_with_that_fact_an
with patch.object(manager_mod, "attach_registered_cdp_browser", AsyncMock(return_value=lost)):
assert await acquire_lease(stack.manager, session_id, run_id, generation) is lost
if loss == "expired":
- await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=15)
+ clock.ticking.set()
+ await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=5)
if loss in ("closed", "vanished"):
await stack.repo.close_persistent_browser_session(session_id, ORG_A)
context.browser.is_connected.return_value = False
diff --git a/tests/unit/test_workflow_runtime_completion.py b/tests/unit/test_workflow_runtime_completion.py
index 0a1ab1484..4d5469602 100644
--- a/tests/unit/test_workflow_runtime_completion.py
+++ b/tests/unit/test_workflow_runtime_completion.py
@@ -36,6 +36,7 @@ from tests.unit._external_cdp_fakes import (
begin_lease,
external_browser_state,
external_cdp_stack,
+ install_deadline_clock,
register_external,
)
from tests.unit.conftest import FakeWorkflowRunAttemptsRepository
@@ -805,14 +806,16 @@ def test_apply_proposed_workflow_route_is_bound_to_the_route_handler() -> None:
ids=["open", "expired_mid_run", "browser_died_mid_run", "browser_died_and_row_unreadable"],
)
async def test_a_workflow_whose_registered_browser_ended_mid_run_fails_at_finalization(
- registration: str, browser_connected: bool, expected_reason: str | None
+ monkeypatch: pytest.MonkeyPatch, registration: str, browser_connected: bool, expected_reason: str | None
) -> None:
+ clock = install_deadline_clock(monkeypatch)
async with external_cdp_stack() as stack:
session_id = (await register_external(stack.client, timeout=5)).json()["browser_session_id"]
if registration == "expired":
- await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60)
+ await backdate_registration(stack, session_id, minutes=5 - 0.5 / 60, clock=clock)
await begin_lease(stack.manager, session_id, "wr_a")
if registration == "expired":
+ clock.ticking.set()
await asyncio.wait_for(stack.manager._external_deadlines[session_id], timeout=5)
run = SimpleNamespace(
workflow_run_id="wr_a",