Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(browser): pin one browser engine per logical run (SKY-12007) [2/5] (#7571)
LawyZheng82e0212b68fd
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex b865aac86..0d856e624 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -103,7 +103,10 @@ class ScriptSkyvernPage(SkyvernPage):else:raise WorkflowRunNotFound(workflow_run_id=context.workflow_run_id)else:- browser_state = await app.BROWSER_MANAGER.get_or_create_for_script(browser_session_id=browser_session_id)+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_script(+ script_id=context.script_id if context else None,+ browser_session_id=browser_session_id,+ )return browser_state@classmethod@@ -139,7 +142,6 @@ class ScriptSkyvernPage(SkyvernPage):url: str | None = None,) -> ScrapedPage:# initialize browser state- # TODO: add workflow_run_id or eventually script_id/script_run_idbrowser_state = await cls._get_or_create_browser_state(browser_session_id=browser_session_id, url=url)return await browser_state.scrape_website(url="",diff --git a/skyvern/webeye/browser_engine.py b/skyvern/webeye/browser_engine.pyindex f870d2eca..4f300e0d4 100644--- a/skyvern/webeye/browser_engine.py+++ b/skyvern/webeye/browser_engine.py@@ -254,6 +254,7 @@ class BrowserEngineContext:workflow_run_id: str | None = Noneworkflow_permanent_id: str | None = Nonetask_id: str | None = None+ script_id: str | None = Nonebrowser_source: str | None = Nonediff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.pyindex 627c8bb71..79d1e95a6 100644--- a/skyvern/webeye/default_persistent_sessions_manager.py+++ b/skyvern/webeye/default_persistent_sessions_manager.py@@ -5,6 +5,7 @@ from dataclasses import dataclassfrom datetime import datetime, timedelta, timezonefrom math import floorfrom pathlib import Path+from typing import castimport structlogfrom playwright._impl._errors import TargetClosedError@@ -409,7 +410,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):launch_proxy_location = session.proxy_location if session.proxy_location is not None else proxy_locationextra_http_headers = app.AGENT_FUNCTION.build_proxy_session_extra_http_headers(session.proxy_session_id)- browser_state = await RealBrowserManager._create_browser_state(+ browser_state = await cast(RealBrowserManager, app.BROWSER_MANAGER)._create_browser_state(proxy_location=launch_proxy_location,url=url,organization_id=organization_id,diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.pyindex 3ddbb712f..e47652647 100644--- a/skyvern/webeye/real_browser_manager.py+++ b/skyvern/webeye/real_browser_manager.py@@ -1,11 +1,12 @@from __future__ import annotationsimport asyncio+import functoolsimport osimport structlog-from playwright.async_api import async_playwright+from skyvern.config import settingsfrom skyvern.constants import BROWSER_CLOSE_TIMEOUTfrom skyvern.exceptions import FailedToNavigateToUrl, MissingBrowserStatefrom skyvern.forge import app@@ -16,6 +17,11 @@ from skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRunfrom skyvern.schemas.runs import ProxyLocation, ProxyLocationInputfrom skyvern.webeye.browser_artifacts import VideoArtifact+from skyvern.webeye.browser_engine import (+ BrowserEngineContext,+ BrowserEngineSelection,+ resolve_browser_engine,+)from skyvern.webeye.browser_factory import BrowserContextFactory, rebind_download_dirfrom skyvern.webeye.browser_manager import BrowserManagerfrom skyvern.webeye.browser_state import BrowserState@@ -87,9 +93,49 @@ def _resolve_stream_key(*, workflow_run_id: str | None, task_id: str | None) ->return None+def canonical_run_key(+ *,+ workflow_run_id: str | None = None,+ task_id: str | None = None,+ script_id: str | None = None,+) -> str | None:+ """The one stable key a logical run's engine selection is pinned under. ``workflow_run_id`` wins+ so a workflow-owned task and its workflow share a single selection owner (never two). Returns+ None when the run has no durable identity (e.g. a standalone script with no id), in which case+ the resource is ephemeral and its engine is not pinned/cached."""+ return workflow_run_id or task_id or script_id+++class _EngineSelectionOwner:+ """Per-run single-flight owner of the pinned engine selection.++ The resolution runs inside a shared ``asyncio.Task``: concurrent first acquisitions for one run+ await the same task and receive the same frozen selection. Waiters await it through+ ``asyncio.shield`` (see ``get_or_resolve_engine_selection``); the shield is what keeps one waiter's+ cancellation from aborting the shared resolution — awaiting a task WITHOUT shielding would propagate+ the waiter's cancellation to it. The resolved value lives on THIS owner object, not a bare per-key+ dict, so a resolver whose owner was already dropped by terminal cleanup cannot resurrect the run's+ selection — its result is simply unreferenced. ``terminal`` is set by ``_drop_engine_owner`` before it+ cancels the resolver: it marks the owner as being torn down so a same-key acquisition waits it out+ instead of starting a second resolver, and so the done-callback evicts it whatever the outcome.+ """++ __slots__ = ("task", "terminal")++ def __init__(self, task: asyncio.Task[BrowserEngineSelection]) -> None:+ self.task = task+ self.terminal = False++class RealBrowserManager(BrowserManager):def __init__(self) -> None:self.pages: dict[str, BrowserState] = {}+ # Engine pinned per logical run, keyed by run id (workflow_run_id / task_id / script_id) via a+ # per-key single-flight owner. Resolved once at the first browser-resource creation for a run+ # and reused for every later resource/recreation in that run, so recreation can never+ # re-resolve to a different engine (e.g. after a flag change). Dropped — with its in-flight+ # resolution cancelled — when the run's browser state is cleaned up.+ self._engine_owners: dict[str, _EngineSelectionOwner] = {}# CDP frame publishers keyed by stream key (``{wr}.png`` / ``{task}.png``).self._frame_publishers: dict[str, CDPFramePublisher] = {}# Serializes the check/create/start/store/register sequence in@@ -183,8 +229,97 @@ class RealBrowserManager(BrowserManager):exc_info=True,)- @staticmethod+ async def get_or_resolve_engine_selection(+ self,+ *,+ run_key: str | None,+ context: BrowserEngineContext,+ ) -> BrowserEngineSelection:+ """Single owner of a logical run's engine selection: resolve it once under ``run_key`` and+ reuse it for EVERY browser resource in the run — this manager's states and the persistent+ session attach alike — so all paths for one run share one pinned engine and recreation never+ re-resolves to a different one (e.g. after a flag flip). Resolution per key is single-flighted+ via a per-key owner task, so concurrent first acquisitions await one resolution and receive+ the same frozen selection while different keys resolve concurrently. ``run_key`` None means an+ ephemeral resource with no durable run identity: it is not pinned or cached (the resolver still+ fails closed on capability). The source-capability check runs on every return, including cache+ hits, so a capability-restricted run fails closed the moment it reaches an unsupported source.++ Failure-bounded: if the shared resolution finishes exceptionally (resolver raised, or terminal+ cleanup cancelled it) the owner is dropped so nothing is stored and a later acquisition+ re-resolves cleanly — no orphan owner accumulates on failed run keys."""+ if run_key is None:+ return await resolve_browser_engine(context)+ while True:+ owner = self._engine_owners.get(run_key)+ if owner is None:+ # No await between the miss and the store, so concurrent first acquisitions for one key+ # observe the same owner and share its single resolution task (single-flight).+ owner = _EngineSelectionOwner(asyncio.ensure_future(resolve_browser_engine(context)))+ self._engine_owners[run_key] = owner+ # Reap the owner if its resolution ends with nothing selectable (failed/cancelled, or+ # marked dropping) and no live waiter, so it neither lingers nor leaks an exception.+ owner.task.add_done_callback(functools.partial(self._reap_failed_owner, run_key, owner))+ break+ if not owner.terminal:+ break+ # Terminal owner mid-teardown: wait it out, evict once it ends, then loop for a fresh owner.+ try:+ await asyncio.shield(owner.task)+ except asyncio.CancelledError:+ if (t := asyncio.current_task()) is not None and t.cancelling() > 0:+ raise # this acquirer itself was cancelled — leave the still-running owner untouched+ except Exception:+ pass+ if owner.task.done() and self._engine_owners.get(run_key) is owner:+ del self._engine_owners[run_key]+ # shield so cancelling THIS waiter cannot cancel the shared resolution task (asyncio otherwise+ # propagates it); only terminal cleanup cancels the task. Evicting a failed/cancelled/terminal owner+ # is the done-callback's job, so a waiter cancel never drops a healthy owner here (even after success).+ selection = await asyncio.shield(owner.task)+ selection.ensure_supports(context.browser_source)+ return selection++ def _reap_failed_owner(self, run_key: str, owner: _EngineSelectionOwner, task: asyncio.Task) -> None:+ """Done-callback for an owner's resolution task. Consumes the outcome so a failed/cancelled+ resolution never surfaces an unretrieved-exception warning, and evicts the owner when it should+ no longer be selectable — finished exceptionally/cancelled OR marked terminal by cleanup —+ provided it is still the current owner for ``run_key`` (``is owner`` guards a reused key). A+ successful, non-terminal selection is kept."""+ failed = task.cancelled() or task.exception() is not None+ if (failed or owner.terminal) and self._engine_owners.get(run_key) is owner:+ del self._engine_owners[run_key]++ async def _drop_engine_owner(self, run_key: str | None) -> None:+ """Terminally remove a run's pinned engine owner. Idempotent. Marks the owner terminal, cancels+ the in-flight resolution, and AWAITS its termination (keeping the owner registered as it unwinds)+ so no second same-key resolver starts until the first definitively ends — even one that+ suppresses/delays cancellation. If THIS cleanup coroutine is itself cancelled while the resolver+ still runs, the cancellation propagates and the terminal owner stays registered. Removes by id."""+ if run_key is None:+ return+ owner = self._engine_owners.get(run_key)+ if owner is None:+ return+ owner.terminal = True+ if not owner.task.done():+ owner.task.cancel()+ try:+ # shield keeps the resolver alive if this cleanup coroutine is cancelled mid-await. Propagate+ # only OUR cancellation (via the current task's cancellation count) — never the resolver+ # task's own cancellation surfacing through shield — so an external cancel is never swallowed+ # (even if the resolver finishes in the same tick); the owner stays for its done-callback.+ await asyncio.shield(owner.task)+ except asyncio.CancelledError:+ if (t := asyncio.current_task()) is not None and t.cancelling() > 0:+ raise+ except Exception:+ pass+ if self._engine_owners.get(run_key) is owner:+ del self._engine_owners[run_key]+async def _create_browser_state(+ self,proxy_location: ProxyLocationInput = None,url: str | None = None,task_id: str | None = None,@@ -196,8 +331,28 @@ class RealBrowserManager(BrowserManager):cdp_connect_headers: dict[str, str] | None = None,browser_address: str | None = None,browser_profile_id: str | None = None,+ engine_run_key: str | None = None,) -> BrowserState:- pw = await async_playwright().start()+ engine_selection = await self.get_or_resolve_engine_selection(+ run_key=engine_run_key+ or canonical_run_key(workflow_run_id=workflow_run_id, task_id=task_id, script_id=script_id),+ context=BrowserEngineContext(+ organization_id=organization_id,+ workflow_run_id=workflow_run_id,+ workflow_permanent_id=workflow_permanent_id,+ task_id=task_id,+ script_id=script_id,+ browser_source=settings.BROWSER_TYPE,+ ),+ )+ LOG.info(+ "Creating browser state",+ task_id=task_id,+ workflow_run_id=workflow_run_id,+ browser_source=settings.BROWSER_TYPE,+ **engine_selection.attribution(),+ )+ pw = await engine_selection.start_driver()try:(browser_context,@@ -241,6 +396,7 @@ class RealBrowserManager(BrowserManager):browser_artifacts=browser_artifacts,browser_cleanup=browser_cleanup,release_driver_on_close=browser_address is not None,+ engine_selection=engine_selection,)def evict_page(self, page_id: str) -> None:@@ -309,6 +465,11 @@ class RealBrowserManager(BrowserManager):proxy_location=proxy_location,url=task.url,task_id=task.task_id,+ # Pin the engine under the workflow_run_id for a workflow-owned task so it shares one+ # selection owner with the workflow path (not a second task_id owner). Passed only as+ # the engine key — workflow_run_id is deliberately kept out of browser-context creation+ # here to preserve the task path's existing download-dir / artifact behavior.+ engine_run_key=canonical_run_key(workflow_run_id=task.workflow_run_id, task_id=task.task_id),workflow_permanent_id=task.workflow_permanent_id,organization_id=task.organization_id,extra_http_headers=extra_http_headers,@@ -651,6 +812,8 @@ class RealBrowserManager(BrowserManager):for browser_state in self.pages.values():await browser_state.close()self.pages = dict()+ for run_key in list(self._engine_owners):+ await self._drop_engine_owner(run_key)LOG.info("BrowserManger is closed")async def cleanup_for_task(@@ -665,6 +828,7 @@ class RealBrowserManager(BrowserManager):If error occurs, log it and address the cleanup error."""LOG.info("Cleaning up for task")+ await self._drop_engine_owner(task_id)browser_state_to_close = self.pages.pop(task_id, None)if browser_state_to_close:# Stop tracing before closing the browser if tracing is enabled@@ -709,6 +873,11 @@ class RealBrowserManager(BrowserManager):LOG.info("Cleaning up for workflow run", sampling=True)browser_state_to_close = self.pages.get(workflow_run_id)+ # Drop the run's pinned engine — the run is ending, so no further browser resource will be+ # created for it. Covers the run, its inherited children, and its tasks.+ for run_key in (workflow_run_id, *(child_workflow_run_ids or ()), *task_ids):+ await self._drop_engine_owner(run_key)+# Pop child workflow_run entries first — these are orphaned because child# workflows skip clean_up_workflow. Must happen before the shared check# so the task loop can correctly detect when the browser is no longer shared.diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex fac3c3684..dee26e46e 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -23,6 +23,7 @@ from skyvern.forge import appfrom skyvern.forge.sdk.trace import tracedfrom skyvern.schemas.runs import ProxyLocationInputfrom skyvern.webeye.browser_artifacts import BrowserArtifacts+from skyvern.webeye.browser_engine import BrowserEngineSelectionfrom skyvern.webeye.browser_factory import BrowserCleanupFunc, BrowserContextFactoryfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.navigation import is_permanent_navigation_error, navigate_with_retry@@ -60,6 +61,7 @@ class RealBrowserState(BrowserState):browser_artifacts: BrowserArtifacts = BrowserArtifacts(),browser_cleanup: BrowserCleanupFunc = None,release_driver_on_close: bool = False,+ engine_selection: BrowserEngineSelection | None = None,):self.__page = page# An explicitly selected tab (set by NEW_TAB/SWITCH_TAB). When set, it overrides the@@ -78,6 +80,10 @@ class RealBrowserState(BrowserState):# this state and must be released on close even when the remote# browser is left running.self.release_driver_on_close = release_driver_on_close+ # The engine this state's driver was created with, pinned for the state's lifetime so+ # reconnect starts the same engine and error classification stays this run's identity. None+ # for states built outside the per-run seam (legacy/direct construction).+ self.engine_selection = engine_selection# One-shot callbacks fired first inside ``close()``. Cleared after# firing so re-entry into ``close()`` is safe.self._on_close_callbacks: list[Callable[[], Awaitable[None]]] = []@@ -461,7 +467,12 @@ class RealBrowserState(BrowserState):stale_pw = self.pwself.browser_context = Noneawait self.set_working_page(None)- self.pw = await async_playwright().start()+ # Reconnect on the SAME engine this state was pinned to at creation; never silently switch+ # engines underneath a live run. States built outside the per-run seam keep the stock driver.+ if self.engine_selection is not None:+ self.pw = await self.engine_selection.start_driver()+ else:+ self.pw = await async_playwright().start()try:await self.check_and_fix_state(proxy_location=proxy_location,diff --git a/tests/unit/test_remote_cdp_driver_release.py b/tests/unit/test_remote_cdp_driver_release.pyindex 54b903515..967fd6f86 100644--- a/tests/unit/test_remote_cdp_driver_release.py+++ b/tests/unit/test_remote_cdp_driver_release.py@@ -17,8 +17,15 @@ from unittest.mock import AsyncMock, MagicMockimport pytest+from skyvern.webeye import browser_enginefrom skyvern.webeye.browser_artifacts import BrowserArtifacts-from skyvern.webeye.real_browser_manager import RealBrowserManager+from skyvern.webeye.browser_engine import (+ BrowserEngineContext,+ BrowserEngineMetadata,+ BrowserEngineSelection,+ BrowserSourceNotSupportedByEngine,+)+from skyvern.webeye.real_browser_manager import RealBrowserManager, canonical_run_keyfrom skyvern.webeye.real_browser_state import RealBrowserState@@ -94,7 +101,7 @@ def _patch_create_browser_state(either returns a context tuple or raises (e.g. a connect_over_cdp failure)."""playwright_launcher = MagicMock()playwright_launcher.return_value.start = AsyncMock(return_value=pw)- monkeypatch.setattr("skyvern.webeye.real_browser_manager.async_playwright", playwright_launcher)+ monkeypatch.setattr("skyvern.webeye.browser_engine.async_playwright", playwright_launcher)if raises is not None:create_browser_context = AsyncMock(side_effect=raises)@@ -117,7 +124,7 @@ async def test_create_browser_state_stops_driver_when_context_creation_fails(_patch_create_browser_state(monkeypatch, pw, raises=RuntimeError("connect_over_cdp failed"))with pytest.raises(RuntimeError, match="connect_over_cdp failed"):- await RealBrowserManager._create_browser_state(browser_address="http://192.0.2.10:9222")+ await RealBrowserManager()._create_browser_state(browser_address="http://192.0.2.10:9222")pw.stop.assert_awaited_once()@@ -132,7 +139,7 @@ async def test_create_browser_state_cleanup_failure_does_not_mask_original(_patch_create_browser_state(monkeypatch, pw, raises=ValueError("original create failure"))with pytest.raises(ValueError, match="original create failure"):- await RealBrowserManager._create_browser_state()+ await RealBrowserManager()._create_browser_state()pw.stop.assert_awaited_once()@@ -144,7 +151,7 @@ async def test_create_browser_state_success_retains_driver(monkeypatch: pytest.Mcontext = _context_stub()_patch_create_browser_state(monkeypatch, pw, context_result=(context, BrowserArtifacts(), None))- created = await RealBrowserManager._create_browser_state()+ created = await RealBrowserManager()._create_browser_state()pw.stop.assert_not_awaited()assert created.pw is pw@@ -158,7 +165,7 @@ async def test_create_browser_state_stops_driver_on_cancellation(monkeypatch: py_patch_create_browser_state(monkeypatch, pw, raises=asyncio.CancelledError())with pytest.raises(asyncio.CancelledError):- await RealBrowserManager._create_browser_state()+ await RealBrowserManager()._create_browser_state()pw.stop.assert_awaited_once()@@ -179,7 +186,7 @@ async def test_create_browser_state_hung_stop_does_not_block_original_error(_patch_create_browser_state(monkeypatch, pw, raises=ValueError("original create failure"))with pytest.raises(ValueError, match="original create failure"):- await asyncio.wait_for(RealBrowserManager._create_browser_state(), timeout=2)+ await asyncio.wait_for(RealBrowserManager()._create_browser_state(), timeout=2)pw.stop.assert_awaited_once()@@ -189,7 +196,7 @@ async def test_create_browser_state_marks_remote_cdp_states(monkeypatch: pytest.pw = _pw_stub()playwright_launcher = MagicMock()playwright_launcher.return_value.start = AsyncMock(return_value=pw)- monkeypatch.setattr("skyvern.webeye.real_browser_manager.async_playwright", playwright_launcher)+ monkeypatch.setattr("skyvern.webeye.browser_engine.async_playwright", playwright_launcher)create_browser_context = AsyncMock(return_value=(_context_stub(), BrowserArtifacts(), None))monkeypatch.setattr(@@ -197,8 +204,8 @@ async def test_create_browser_state_marks_remote_cdp_states(monkeypatch: pytest.create_browser_context,)- remote_state = await RealBrowserManager._create_browser_state(browser_address="http://192.0.2.10:9222")- local_state = await RealBrowserManager._create_browser_state()+ remote_state = await RealBrowserManager()._create_browser_state(browser_address="http://192.0.2.10:9222")+ local_state = await RealBrowserManager()._create_browser_state()assert isinstance(remote_state, RealBrowserState)assert isinstance(local_state, RealBrowserState)@@ -279,3 +286,285 @@ async def test_cleanup_for_workflow_run_final_close_lets_state_decide() -> None:assert final_call.kwargs["release_driver"] is Noneassert "wr_1" not in manager.pagesassert "tsk_1" not in manager.pages+++class _FakeEngineError(Exception):+ pass+++def _fake_selection(name: str, *, allowed_sources: frozenset[str] | None = None) -> BrowserEngineSelection:+ async def _ok_start() -> object:+ return object()++ return BrowserEngineSelection(+ name=name,+ start_driver=_ok_start,+ error_type=_FakeEngineError,+ timeout_error_type=_FakeEngineError,+ metadata=BrowserEngineMetadata(name=name, version="0.0.0", allowed_browser_sources=allowed_sources),+ selection_reason="test",+ )+++def _ctx(run_id: str | None, source: str | None = "local-browser") -> BrowserEngineContext:+ return BrowserEngineContext(workflow_run_id=run_id, browser_source=source)+++async def _seed_engine_owner(manager: RealBrowserManager, run_key: str, name: str) -> None:+ """Install a completed engine owner for ``run_key`` (mirrors a resolved run)."""+ selection = _fake_selection(name)+ browser_engine.set_browser_engine_resolver(lambda ctx: _coro(selection))+ await manager.get_or_resolve_engine_selection(run_key=run_key, context=_ctx(run_key, None))+++def _restore_resolver():+ yield+ browser_engine.reset_browser_engine_resolver()+++async def test_manager_pins_one_engine_per_run_and_reuses_on_recreation(_restore_resolver: None) -> None:+ calls = {"n": 0}++ async def counting_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:+ calls["n"] += 1+ return _fake_selection(f"engine-{calls['n']}")++ browser_engine.set_browser_engine_resolver(counting_resolver)+ manager = RealBrowserManager()+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source=None) # unrestricted engine allows None++ first = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)+ second = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)+ assert first is second+ assert calls["n"] == 1++ other = await manager.get_or_resolve_engine_selection(run_key="wr_2", context=_ctx("wr_2"))+ assert other is not first+ assert calls["n"] == 2++ await manager._drop_engine_owner("wr_1")+ third = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)+ assert third is not first+ assert calls["n"] == 3+++async def test_concurrent_first_acquisitions_resolve_once_and_share_selection(_restore_resolver: None) -> None:+ # check -> await resolver -> store must be single-flighted per run key.+ calls = {"n": 0}++ async def slow_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:+ calls["n"] += 1+ await asyncio.sleep(0) # force the two coroutines to interleave inside the resolver window+ return _fake_selection(f"engine-{calls['n']}")++ browser_engine.set_browser_engine_resolver(slow_resolver)+ manager = RealBrowserManager()+ same = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")++ a, b = await asyncio.gather(+ manager.get_or_resolve_engine_selection(run_key="wr_1", context=same),+ manager.get_or_resolve_engine_selection(run_key="wr_1", context=same),+ )+ assert a is b+ assert calls["n"] == 1 # resolver invoked exactly once for the shared key++ # A different key resolves independently, concurrently, without blocking on the first key.+ other = await manager.get_or_resolve_engine_selection(run_key="wr_2", context=_ctx("wr_2"))+ assert other is not a+ assert calls["n"] == 2+++async def test_failed_first_resolution_leaves_no_orphan_owner(_restore_resolver: None) -> None:+ # A resolver failure must leave no owner (not even an unlocked entry), so retries resolve and failed keys can't pile up.+ attempts = {"n": 0}++ async def flaky_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:+ attempts["n"] += 1+ if attempts["n"] == 1:+ raise RuntimeError("resolver boom")+ return _fake_selection("engine-ok")++ browser_engine.set_browser_engine_resolver(flaky_resolver)+ manager = RealBrowserManager()+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")++ with pytest.raises(RuntimeError, match="resolver boom"):+ await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)+ assert "wr_1" not in manager._engine_owners++ recovered = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)+ assert recovered.name == "engine-ok"+++async def test_concurrent_failed_first_resolution_leaves_no_orphan_owner(_restore_resolver: None) -> None:+ # Two waiters on the same key whose shared resolution fails: both observe the failure and neither orphans an owner.+ async def boom_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:+ await asyncio.sleep(0)+ raise RuntimeError("resolver boom")++ browser_engine.set_browser_engine_resolver(boom_resolver)+ manager = RealBrowserManager()+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")++ results = await asyncio.gather(+ manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx),+ manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx),+ return_exceptions=True,+ )+ assert all(isinstance(r, RuntimeError) for r in results)+ assert manager._engine_owners == {}+++async def test_waiter_cancellation_does_not_abort_shared_resolution(_restore_resolver: None) -> None:+ # Cancelling one waiter must not abort the shared resolution nor drop the owner the survivor still needs.+ calls = {"n": 0}+ gate = asyncio.Event()++ async def gated_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:+ calls["n"] += 1+ await gate.wait()+ return _fake_selection(f"engine-{calls['n']}")++ browser_engine.set_browser_engine_resolver(gated_resolver)+ manager = RealBrowserManager()+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")++ survivor = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx))+ victim = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx))+ await asyncio.sleep(0) # let both attach to the shared owner task+ victim.cancel()+ with pytest.raises(asyncio.CancelledError):+ await victim+ gate.set()+ selection = await survivor+ assert selection.name == "engine-1"+ assert calls["n"] == 1 # the shared resolution ran exactly once despite the cancelled waiter++ # A waiter cancelled AFTER the resolution succeeds must not evict the healthy owner; that is the done-callback's job.+ gate.clear()+ ctx2 = BrowserEngineContext(workflow_run_id="wr_2", browser_source="local-browser")+ keeper = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_2", context=ctx2))+ late = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_2", context=ctx2))+ await asyncio.sleep(0) # both attach to the shared owner task+ manager._engine_owners["wr_2"].task.add_done_callback(lambda _t: late.cancel()) # cancel in success tick+ gate.set()+ with pytest.raises(asyncio.CancelledError):+ await late+ assert (await keeper).name == "engine-2"+ assert "wr_2" in manager._engine_owners # successful owner survived the late waiter cancel+ assert (await manager.get_or_resolve_engine_selection(run_key="wr_2", context=ctx2)) is keeper.result()+ assert calls["n"] == 2 # wr_1 once, wr_2 once; the late cancel forced no re-resolution+++async def test_terminal_cleanup_racing_resolver_prevents_resurrection(_restore_resolver: None) -> None:+ # Cleanup while a resolver is in flight must cancel it, leave no owner, not resurrect state; a later acquire resolves fresh.+ calls = {"n": 0}+ gate = asyncio.Event()++ async def gated_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:+ calls["n"] += 1+ await gate.wait()+ return _fake_selection(f"engine-{calls['n']}")++ browser_engine.set_browser_engine_resolver(gated_resolver)+ manager = RealBrowserManager()+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")++ inflight = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx))+ await asyncio.sleep(0) # resolver is now attached and blocked on the gate+ await manager._drop_engine_owner("wr_1") # terminal cleanup awaits the racing resolver to termination+ with pytest.raises(asyncio.CancelledError):+ await inflight+ # The cancelled resolver stored nothing and did not reinstall an owner: no resurrection.+ assert "wr_1" not in manager._engine_owners++ gate.set() # a fresh resolution for the reused key resolves cleanly under a brand-new owner+ fresh = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)+ assert manager._engine_owners["wr_1"].task.result() is fresh+++async def test_null_run_key_is_ephemeral_and_not_pinned(_restore_resolver: None) -> None:+ calls = {"n": 0}++ async def counting_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:+ calls["n"] += 1+ return _fake_selection(f"engine-{calls['n']}")++ browser_engine.set_browser_engine_resolver(counting_resolver)+ manager = RealBrowserManager()+ ctx = BrowserEngineContext(browser_source="local-browser")++ first = await manager.get_or_resolve_engine_selection(run_key=None, context=ctx)+ second = await manager.get_or_resolve_engine_selection(run_key=None, context=ctx)+ assert first is not second # no durable identity => resolved each time, never cached+ assert manager._engine_owners == {}+++async def test_workflow_owned_task_pins_engine_under_workflow_run_id(+ monkeypatch: pytest.MonkeyPatch, _restore_resolver: None+) -> None:+ # The task path passes engine_run_key=canonical_run_key(...); a workflow-owned task pins under workflow_run_id only.+ assert canonical_run_key(workflow_run_id="wr_1", task_id="tsk_1") == "wr_1"+ assert canonical_run_key(task_id="tsk_1") == "tsk_1"++ _patch_create_browser_state(monkeypatch, _pw_stub(), context_result=(_context_stub(), BrowserArtifacts(), None))+ manager = RealBrowserManager()+ await manager._create_browser_state(+ task_id="tsk_1", engine_run_key=canonical_run_key(workflow_run_id="wr_1", task_id="tsk_1")+ )+ assert "wr_1" in manager._engine_owners+ assert "tsk_1" not in manager._engine_owners++[email protected]("unsupported_source", ["local-browser", None])+async def test_manager_capability_gate_applies_even_on_cache_hit(+ _restore_resolver: None, unsupported_source: str | None+) -> None:+ # A restricted engine rejects an unsupported cached source (incl. None, failed closed); the pinned owner survives.+ restricted = _fake_selection("restricted-engine", allowed_sources=frozenset({"cdp-connection-browser"}))+ browser_engine.set_browser_engine_resolver(lambda ctx: _coro(restricted))+ manager = RealBrowserManager()++ ok = await manager.get_or_resolve_engine_selection(+ run_key="wr_x",+ context=BrowserEngineContext(workflow_run_id="wr_x", browser_source="cdp-connection-browser"),+ )+ assert ok is restricted+ owner = manager._engine_owners["wr_x"]++ with pytest.raises(BrowserSourceNotSupportedByEngine):+ await manager.get_or_resolve_engine_selection(+ run_key="wr_x",+ context=BrowserEngineContext(workflow_run_id="wr_x", browser_source=unsupported_source),+ )+ assert manager._engine_owners.get("wr_x") is owner+++async def test_cleanup_drops_pinned_engine_owner_for_run(_restore_resolver: None) -> None:+ manager = RealBrowserManager()+ await _seed_engine_owner(manager, "wr_1", "engine-1")+ await _seed_engine_owner(manager, "tsk_1", "engine-1")+ manager.pages["wr_1"] = _fake_browser_state()++ await manager.cleanup_for_workflow_run("wr_1", ["tsk_1"], close_browser_on_completion=False)++ assert "wr_1" not in manager._engine_owners+ assert "tsk_1" not in manager._engine_owners+++def _coro(value: BrowserEngineSelection):+ async def _c(ctx: BrowserEngineContext | None = None) -> BrowserEngineSelection:+ return value++ return _c()