Skip to content

Automate browser based workflows with AI

feat(browser): pin one browser engine per logical run (SKY-12007) [2/5] (#7571)

LawyZheng82e0212b68fd
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py
index b865aac86..0d856e624 100644
--- a/skyvern/core/script_generations/script_skyvern_page.py
+++ b/skyvern/core/script_generations/script_skyvern_page.py
@@ -103,7 +103,10 @@ class ScriptSkyvernPage(SkyvernPage):
else:
raise WorkflowRunNotFound(workflow_run_id=context.workflow_run_id)
else:
- browser_state = await app.BROWSER_MANAGER.get_or_create_for_script(browser_session_id=browser_session_id)
+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_script(
+ script_id=context.script_id if context else None,
+ browser_session_id=browser_session_id,
+ )
return browser_state
@classmethod
@@ -139,7 +142,6 @@ class ScriptSkyvernPage(SkyvernPage):
url: str | None = None,
) -> ScrapedPage:
# initialize browser state
- # TODO: add workflow_run_id or eventually script_id/script_run_id
browser_state = await cls._get_or_create_browser_state(browser_session_id=browser_session_id, url=url)
return await browser_state.scrape_website(
url="",
diff --git a/skyvern/webeye/browser_engine.py b/skyvern/webeye/browser_engine.py
index f870d2eca..4f300e0d4 100644
--- a/skyvern/webeye/browser_engine.py
+++ b/skyvern/webeye/browser_engine.py
@@ -254,6 +254,7 @@ class BrowserEngineContext:
workflow_run_id: str | None = None
workflow_permanent_id: str | None = None
task_id: str | None = None
+ script_id: str | None = None
browser_source: str | None = None
diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.py
index 627c8bb71..79d1e95a6 100644
--- a/skyvern/webeye/default_persistent_sessions_manager.py
+++ b/skyvern/webeye/default_persistent_sessions_manager.py
@@ -5,6 +5,7 @@ from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from math import floor
from pathlib import Path
+from typing import cast
import structlog
from playwright._impl._errors import TargetClosedError
@@ -409,7 +410,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
launch_proxy_location = session.proxy_location if session.proxy_location is not None else proxy_location
extra_http_headers = app.AGENT_FUNCTION.build_proxy_session_extra_http_headers(session.proxy_session_id)
- browser_state = await RealBrowserManager._create_browser_state(
+ browser_state = await cast(RealBrowserManager, app.BROWSER_MANAGER)._create_browser_state(
proxy_location=launch_proxy_location,
url=url,
organization_id=organization_id,
diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.py
index 3ddbb712f..e47652647 100644
--- a/skyvern/webeye/real_browser_manager.py
+++ b/skyvern/webeye/real_browser_manager.py
@@ -1,11 +1,12 @@
from __future__ import annotations
import asyncio
+import functools
import os
import structlog
-from playwright.async_api import async_playwright
+from skyvern.config import settings
from skyvern.constants import BROWSER_CLOSE_TIMEOUT
from skyvern.exceptions import FailedToNavigateToUrl, MissingBrowserState
from skyvern.forge import app
@@ -16,6 +17,11 @@ from skyvern.forge.sdk.schemas.tasks import Task
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun
from skyvern.schemas.runs import ProxyLocation, ProxyLocationInput
from skyvern.webeye.browser_artifacts import VideoArtifact
+from skyvern.webeye.browser_engine import (
+ BrowserEngineContext,
+ BrowserEngineSelection,
+ resolve_browser_engine,
+)
from skyvern.webeye.browser_factory import BrowserContextFactory, rebind_download_dir
from skyvern.webeye.browser_manager import BrowserManager
from skyvern.webeye.browser_state import BrowserState
@@ -87,9 +93,49 @@ def _resolve_stream_key(*, workflow_run_id: str | None, task_id: str | None) ->
return None
+def canonical_run_key(
+ *,
+ workflow_run_id: str | None = None,
+ task_id: str | None = None,
+ script_id: str | None = None,
+) -> str | None:
+ """The one stable key a logical run's engine selection is pinned under. ``workflow_run_id`` wins
+ so a workflow-owned task and its workflow share a single selection owner (never two). Returns
+ None when the run has no durable identity (e.g. a standalone script with no id), in which case
+ the resource is ephemeral and its engine is not pinned/cached."""
+ return workflow_run_id or task_id or script_id
+
+
+class _EngineSelectionOwner:
+ """Per-run single-flight owner of the pinned engine selection.
+
+ The resolution runs inside a shared ``asyncio.Task``: concurrent first acquisitions for one run
+ await the same task and receive the same frozen selection. Waiters await it through
+ ``asyncio.shield`` (see ``get_or_resolve_engine_selection``); the shield is what keeps one waiter's
+ cancellation from aborting the shared resolution — awaiting a task WITHOUT shielding would propagate
+ the waiter's cancellation to it. The resolved value lives on THIS owner object, not a bare per-key
+ dict, so a resolver whose owner was already dropped by terminal cleanup cannot resurrect the run's
+ selection — its result is simply unreferenced. ``terminal`` is set by ``_drop_engine_owner`` before it
+ cancels the resolver: it marks the owner as being torn down so a same-key acquisition waits it out
+ instead of starting a second resolver, and so the done-callback evicts it whatever the outcome.
+ """
+
+ __slots__ = ("task", "terminal")
+
+ def __init__(self, task: asyncio.Task[BrowserEngineSelection]) -> None:
+ self.task = task
+ self.terminal = False
+
+
class RealBrowserManager(BrowserManager):
def __init__(self) -> None:
self.pages: dict[str, BrowserState] = {}
+ # Engine pinned per logical run, keyed by run id (workflow_run_id / task_id / script_id) via a
+ # per-key single-flight owner. Resolved once at the first browser-resource creation for a run
+ # and reused for every later resource/recreation in that run, so recreation can never
+ # re-resolve to a different engine (e.g. after a flag change). Dropped — with its in-flight
+ # resolution cancelled — when the run's browser state is cleaned up.
+ self._engine_owners: dict[str, _EngineSelectionOwner] = {}
# CDP frame publishers keyed by stream key (``{wr}.png`` / ``{task}.png``).
self._frame_publishers: dict[str, CDPFramePublisher] = {}
# Serializes the check/create/start/store/register sequence in
@@ -183,8 +229,97 @@ class RealBrowserManager(BrowserManager):
exc_info=True,
)
- @staticmethod
+ async def get_or_resolve_engine_selection(
+ self,
+ *,
+ run_key: str | None,
+ context: BrowserEngineContext,
+ ) -> BrowserEngineSelection:
+ """Single owner of a logical run's engine selection: resolve it once under ``run_key`` and
+ reuse it for EVERY browser resource in the run — this manager's states and the persistent
+ session attach alike — so all paths for one run share one pinned engine and recreation never
+ re-resolves to a different one (e.g. after a flag flip). Resolution per key is single-flighted
+ via a per-key owner task, so concurrent first acquisitions await one resolution and receive
+ the same frozen selection while different keys resolve concurrently. ``run_key`` None means an
+ ephemeral resource with no durable run identity: it is not pinned or cached (the resolver still
+ fails closed on capability). The source-capability check runs on every return, including cache
+ hits, so a capability-restricted run fails closed the moment it reaches an unsupported source.
+
+ Failure-bounded: if the shared resolution finishes exceptionally (resolver raised, or terminal
+ cleanup cancelled it) the owner is dropped so nothing is stored and a later acquisition
+ re-resolves cleanly — no orphan owner accumulates on failed run keys."""
+ if run_key is None:
+ return await resolve_browser_engine(context)
+ while True:
+ owner = self._engine_owners.get(run_key)
+ if owner is None:
+ # No await between the miss and the store, so concurrent first acquisitions for one key
+ # observe the same owner and share its single resolution task (single-flight).
+ owner = _EngineSelectionOwner(asyncio.ensure_future(resolve_browser_engine(context)))
+ self._engine_owners[run_key] = owner
+ # Reap the owner if its resolution ends with nothing selectable (failed/cancelled, or
+ # marked dropping) and no live waiter, so it neither lingers nor leaks an exception.
+ owner.task.add_done_callback(functools.partial(self._reap_failed_owner, run_key, owner))
+ break
+ if not owner.terminal:
+ break
+ # Terminal owner mid-teardown: wait it out, evict once it ends, then loop for a fresh owner.
+ try:
+ await asyncio.shield(owner.task)
+ except asyncio.CancelledError:
+ if (t := asyncio.current_task()) is not None and t.cancelling() > 0:
+ raise # this acquirer itself was cancelled — leave the still-running owner untouched
+ except Exception:
+ pass
+ if owner.task.done() and self._engine_owners.get(run_key) is owner:
+ del self._engine_owners[run_key]
+ # shield so cancelling THIS waiter cannot cancel the shared resolution task (asyncio otherwise
+ # propagates it); only terminal cleanup cancels the task. Evicting a failed/cancelled/terminal owner
+ # is the done-callback's job, so a waiter cancel never drops a healthy owner here (even after success).
+ selection = await asyncio.shield(owner.task)
+ selection.ensure_supports(context.browser_source)
+ return selection
+
+ def _reap_failed_owner(self, run_key: str, owner: _EngineSelectionOwner, task: asyncio.Task) -> None:
+ """Done-callback for an owner's resolution task. Consumes the outcome so a failed/cancelled
+ resolution never surfaces an unretrieved-exception warning, and evicts the owner when it should
+ no longer be selectable — finished exceptionally/cancelled OR marked terminal by cleanup —
+ provided it is still the current owner for ``run_key`` (``is owner`` guards a reused key). A
+ successful, non-terminal selection is kept."""
+ failed = task.cancelled() or task.exception() is not None
+ if (failed or owner.terminal) and self._engine_owners.get(run_key) is owner:
+ del self._engine_owners[run_key]
+
+ async def _drop_engine_owner(self, run_key: str | None) -> None:
+ """Terminally remove a run's pinned engine owner. Idempotent. Marks the owner terminal, cancels
+ the in-flight resolution, and AWAITS its termination (keeping the owner registered as it unwinds)
+ so no second same-key resolver starts until the first definitively ends — even one that
+ suppresses/delays cancellation. If THIS cleanup coroutine is itself cancelled while the resolver
+ still runs, the cancellation propagates and the terminal owner stays registered. Removes by id."""
+ if run_key is None:
+ return
+ owner = self._engine_owners.get(run_key)
+ if owner is None:
+ return
+ owner.terminal = True
+ if not owner.task.done():
+ owner.task.cancel()
+ try:
+ # shield keeps the resolver alive if this cleanup coroutine is cancelled mid-await. Propagate
+ # only OUR cancellation (via the current task's cancellation count) — never the resolver
+ # task's own cancellation surfacing through shield — so an external cancel is never swallowed
+ # (even if the resolver finishes in the same tick); the owner stays for its done-callback.
+ await asyncio.shield(owner.task)
+ except asyncio.CancelledError:
+ if (t := asyncio.current_task()) is not None and t.cancelling() > 0:
+ raise
+ except Exception:
+ pass
+ if self._engine_owners.get(run_key) is owner:
+ del self._engine_owners[run_key]
+
async def _create_browser_state(
+ self,
proxy_location: ProxyLocationInput = None,
url: str | None = None,
task_id: str | None = None,
@@ -196,8 +331,28 @@ class RealBrowserManager(BrowserManager):
cdp_connect_headers: dict[str, str] | None = None,
browser_address: str | None = None,
browser_profile_id: str | None = None,
+ engine_run_key: str | None = None,
) -> BrowserState:
- pw = await async_playwright().start()
+ engine_selection = await self.get_or_resolve_engine_selection(
+ run_key=engine_run_key
+ or canonical_run_key(workflow_run_id=workflow_run_id, task_id=task_id, script_id=script_id),
+ context=BrowserEngineContext(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ task_id=task_id,
+ script_id=script_id,
+ browser_source=settings.BROWSER_TYPE,
+ ),
+ )
+ LOG.info(
+ "Creating browser state",
+ task_id=task_id,
+ workflow_run_id=workflow_run_id,
+ browser_source=settings.BROWSER_TYPE,
+ **engine_selection.attribution(),
+ )
+ pw = await engine_selection.start_driver()
try:
(
browser_context,
@@ -241,6 +396,7 @@ class RealBrowserManager(BrowserManager):
browser_artifacts=browser_artifacts,
browser_cleanup=browser_cleanup,
release_driver_on_close=browser_address is not None,
+ engine_selection=engine_selection,
)
def evict_page(self, page_id: str) -> None:
@@ -309,6 +465,11 @@ class RealBrowserManager(BrowserManager):
proxy_location=proxy_location,
url=task.url,
task_id=task.task_id,
+ # Pin the engine under the workflow_run_id for a workflow-owned task so it shares one
+ # selection owner with the workflow path (not a second task_id owner). Passed only as
+ # the engine key — workflow_run_id is deliberately kept out of browser-context creation
+ # here to preserve the task path's existing download-dir / artifact behavior.
+ engine_run_key=canonical_run_key(workflow_run_id=task.workflow_run_id, task_id=task.task_id),
workflow_permanent_id=task.workflow_permanent_id,
organization_id=task.organization_id,
extra_http_headers=extra_http_headers,
@@ -651,6 +812,8 @@ class RealBrowserManager(BrowserManager):
for browser_state in self.pages.values():
await browser_state.close()
self.pages = dict()
+ for run_key in list(self._engine_owners):
+ await self._drop_engine_owner(run_key)
LOG.info("BrowserManger is closed")
async def cleanup_for_task(
@@ -665,6 +828,7 @@ class RealBrowserManager(BrowserManager):
If error occurs, log it and address the cleanup error.
"""
LOG.info("Cleaning up for task")
+ await self._drop_engine_owner(task_id)
browser_state_to_close = self.pages.pop(task_id, None)
if browser_state_to_close:
# Stop tracing before closing the browser if tracing is enabled
@@ -709,6 +873,11 @@ class RealBrowserManager(BrowserManager):
LOG.info("Cleaning up for workflow run", sampling=True)
browser_state_to_close = self.pages.get(workflow_run_id)
+ # Drop the run's pinned engine — the run is ending, so no further browser resource will be
+ # created for it. Covers the run, its inherited children, and its tasks.
+ for run_key in (workflow_run_id, *(child_workflow_run_ids or ()), *task_ids):
+ await self._drop_engine_owner(run_key)
+
# Pop child workflow_run entries first — these are orphaned because child
# workflows skip clean_up_workflow. Must happen before the shared check
# so the task loop can correctly detect when the browser is no longer shared.
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index fac3c3684..dee26e46e 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -23,6 +23,7 @@ from skyvern.forge import app
from skyvern.forge.sdk.trace import traced
from skyvern.schemas.runs import ProxyLocationInput
from skyvern.webeye.browser_artifacts import BrowserArtifacts
+from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.browser_factory import BrowserCleanupFunc, BrowserContextFactory
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.navigation import is_permanent_navigation_error, navigate_with_retry
@@ -60,6 +61,7 @@ class RealBrowserState(BrowserState):
browser_artifacts: BrowserArtifacts = BrowserArtifacts(),
browser_cleanup: BrowserCleanupFunc = None,
release_driver_on_close: bool = False,
+ engine_selection: BrowserEngineSelection | None = None,
):
self.__page = page
# An explicitly selected tab (set by NEW_TAB/SWITCH_TAB). When set, it overrides the
@@ -78,6 +80,10 @@ class RealBrowserState(BrowserState):
# this state and must be released on close even when the remote
# browser is left running.
self.release_driver_on_close = release_driver_on_close
+ # The engine this state's driver was created with, pinned for the state's lifetime so
+ # reconnect starts the same engine and error classification stays this run's identity. None
+ # for states built outside the per-run seam (legacy/direct construction).
+ self.engine_selection = engine_selection
# One-shot callbacks fired first inside ``close()``. Cleared after
# firing so re-entry into ``close()`` is safe.
self._on_close_callbacks: list[Callable[[], Awaitable[None]]] = []
@@ -461,7 +467,12 @@ class RealBrowserState(BrowserState):
stale_pw = self.pw
self.browser_context = None
await self.set_working_page(None)
- self.pw = await async_playwright().start()
+ # Reconnect on the SAME engine this state was pinned to at creation; never silently switch
+ # engines underneath a live run. States built outside the per-run seam keep the stock driver.
+ if self.engine_selection is not None:
+ self.pw = await self.engine_selection.start_driver()
+ else:
+ self.pw = await async_playwright().start()
try:
await self.check_and_fix_state(
proxy_location=proxy_location,
diff --git a/tests/unit/test_remote_cdp_driver_release.py b/tests/unit/test_remote_cdp_driver_release.py
index 54b903515..967fd6f86 100644
--- a/tests/unit/test_remote_cdp_driver_release.py
+++ b/tests/unit/test_remote_cdp_driver_release.py
@@ -17,8 +17,15 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
+from skyvern.webeye import browser_engine
from skyvern.webeye.browser_artifacts import BrowserArtifacts
-from skyvern.webeye.real_browser_manager import RealBrowserManager
+from skyvern.webeye.browser_engine import (
+ BrowserEngineContext,
+ BrowserEngineMetadata,
+ BrowserEngineSelection,
+ BrowserSourceNotSupportedByEngine,
+)
+from skyvern.webeye.real_browser_manager import RealBrowserManager, canonical_run_key
from skyvern.webeye.real_browser_state import RealBrowserState
@@ -94,7 +101,7 @@ def _patch_create_browser_state(
either returns a context tuple or raises (e.g. a connect_over_cdp failure)."""
playwright_launcher = MagicMock()
playwright_launcher.return_value.start = AsyncMock(return_value=pw)
- monkeypatch.setattr("skyvern.webeye.real_browser_manager.async_playwright", playwright_launcher)
+ monkeypatch.setattr("skyvern.webeye.browser_engine.async_playwright", playwright_launcher)
if raises is not None:
create_browser_context = AsyncMock(side_effect=raises)
@@ -117,7 +124,7 @@ async def test_create_browser_state_stops_driver_when_context_creation_fails(
_patch_create_browser_state(monkeypatch, pw, raises=RuntimeError("connect_over_cdp failed"))
with pytest.raises(RuntimeError, match="connect_over_cdp failed"):
- await RealBrowserManager._create_browser_state(browser_address="http://192.0.2.10:9222")
+ await RealBrowserManager()._create_browser_state(browser_address="http://192.0.2.10:9222")
pw.stop.assert_awaited_once()
@@ -132,7 +139,7 @@ async def test_create_browser_state_cleanup_failure_does_not_mask_original(
_patch_create_browser_state(monkeypatch, pw, raises=ValueError("original create failure"))
with pytest.raises(ValueError, match="original create failure"):
- await RealBrowserManager._create_browser_state()
+ await RealBrowserManager()._create_browser_state()
pw.stop.assert_awaited_once()
@@ -144,7 +151,7 @@ async def test_create_browser_state_success_retains_driver(monkeypatch: pytest.M
context = _context_stub()
_patch_create_browser_state(monkeypatch, pw, context_result=(context, BrowserArtifacts(), None))
- created = await RealBrowserManager._create_browser_state()
+ created = await RealBrowserManager()._create_browser_state()
pw.stop.assert_not_awaited()
assert created.pw is pw
@@ -158,7 +165,7 @@ async def test_create_browser_state_stops_driver_on_cancellation(monkeypatch: py
_patch_create_browser_state(monkeypatch, pw, raises=asyncio.CancelledError())
with pytest.raises(asyncio.CancelledError):
- await RealBrowserManager._create_browser_state()
+ await RealBrowserManager()._create_browser_state()
pw.stop.assert_awaited_once()
@@ -179,7 +186,7 @@ async def test_create_browser_state_hung_stop_does_not_block_original_error(
_patch_create_browser_state(monkeypatch, pw, raises=ValueError("original create failure"))
with pytest.raises(ValueError, match="original create failure"):
- await asyncio.wait_for(RealBrowserManager._create_browser_state(), timeout=2)
+ await asyncio.wait_for(RealBrowserManager()._create_browser_state(), timeout=2)
pw.stop.assert_awaited_once()
@@ -189,7 +196,7 @@ async def test_create_browser_state_marks_remote_cdp_states(monkeypatch: pytest.
pw = _pw_stub()
playwright_launcher = MagicMock()
playwright_launcher.return_value.start = AsyncMock(return_value=pw)
- monkeypatch.setattr("skyvern.webeye.real_browser_manager.async_playwright", playwright_launcher)
+ monkeypatch.setattr("skyvern.webeye.browser_engine.async_playwright", playwright_launcher)
create_browser_context = AsyncMock(return_value=(_context_stub(), BrowserArtifacts(), None))
monkeypatch.setattr(
@@ -197,8 +204,8 @@ async def test_create_browser_state_marks_remote_cdp_states(monkeypatch: pytest.
create_browser_context,
)
- remote_state = await RealBrowserManager._create_browser_state(browser_address="http://192.0.2.10:9222")
- local_state = await RealBrowserManager._create_browser_state()
+ remote_state = await RealBrowserManager()._create_browser_state(browser_address="http://192.0.2.10:9222")
+ local_state = await RealBrowserManager()._create_browser_state()
assert isinstance(remote_state, RealBrowserState)
assert isinstance(local_state, RealBrowserState)
@@ -279,3 +286,285 @@ async def test_cleanup_for_workflow_run_final_close_lets_state_decide() -> None:
assert final_call.kwargs["release_driver"] is None
assert "wr_1" not in manager.pages
assert "tsk_1" not in manager.pages
+
+
+class _FakeEngineError(Exception):
+ pass
+
+
+def _fake_selection(name: str, *, allowed_sources: frozenset[str] | None = None) -> BrowserEngineSelection:
+ async def _ok_start() -> object:
+ return object()
+
+ return BrowserEngineSelection(
+ name=name,
+ start_driver=_ok_start,
+ error_type=_FakeEngineError,
+ timeout_error_type=_FakeEngineError,
+ metadata=BrowserEngineMetadata(name=name, version="0.0.0", allowed_browser_sources=allowed_sources),
+ selection_reason="test",
+ )
+
+
+def _ctx(run_id: str | None, source: str | None = "local-browser") -> BrowserEngineContext:
+ return BrowserEngineContext(workflow_run_id=run_id, browser_source=source)
+
+
+async def _seed_engine_owner(manager: RealBrowserManager, run_key: str, name: str) -> None:
+ """Install a completed engine owner for ``run_key`` (mirrors a resolved run)."""
+ selection = _fake_selection(name)
+ browser_engine.set_browser_engine_resolver(lambda ctx: _coro(selection))
+ await manager.get_or_resolve_engine_selection(run_key=run_key, context=_ctx(run_key, None))
+
+
+def _restore_resolver():
+ yield
+ browser_engine.reset_browser_engine_resolver()
+
+
+async def test_manager_pins_one_engine_per_run_and_reuses_on_recreation(_restore_resolver: None) -> None:
+ calls = {"n": 0}
+
+ async def counting_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:
+ calls["n"] += 1
+ return _fake_selection(f"engine-{calls['n']}")
+
+ browser_engine.set_browser_engine_resolver(counting_resolver)
+ manager = RealBrowserManager()
+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source=None) # unrestricted engine allows None
+
+ first = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)
+ second = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)
+ assert first is second
+ assert calls["n"] == 1
+
+ other = await manager.get_or_resolve_engine_selection(run_key="wr_2", context=_ctx("wr_2"))
+ assert other is not first
+ assert calls["n"] == 2
+
+ await manager._drop_engine_owner("wr_1")
+ third = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)
+ assert third is not first
+ assert calls["n"] == 3
+
+
+async def test_concurrent_first_acquisitions_resolve_once_and_share_selection(_restore_resolver: None) -> None:
+ # check -> await resolver -> store must be single-flighted per run key.
+ calls = {"n": 0}
+
+ async def slow_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:
+ calls["n"] += 1
+ await asyncio.sleep(0) # force the two coroutines to interleave inside the resolver window
+ return _fake_selection(f"engine-{calls['n']}")
+
+ browser_engine.set_browser_engine_resolver(slow_resolver)
+ manager = RealBrowserManager()
+ same = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")
+
+ a, b = await asyncio.gather(
+ manager.get_or_resolve_engine_selection(run_key="wr_1", context=same),
+ manager.get_or_resolve_engine_selection(run_key="wr_1", context=same),
+ )
+ assert a is b
+ assert calls["n"] == 1 # resolver invoked exactly once for the shared key
+
+ # A different key resolves independently, concurrently, without blocking on the first key.
+ other = await manager.get_or_resolve_engine_selection(run_key="wr_2", context=_ctx("wr_2"))
+ assert other is not a
+ assert calls["n"] == 2
+
+
+async def test_failed_first_resolution_leaves_no_orphan_owner(_restore_resolver: None) -> None:
+ # A resolver failure must leave no owner (not even an unlocked entry), so retries resolve and failed keys can't pile up.
+ attempts = {"n": 0}
+
+ async def flaky_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:
+ attempts["n"] += 1
+ if attempts["n"] == 1:
+ raise RuntimeError("resolver boom")
+ return _fake_selection("engine-ok")
+
+ browser_engine.set_browser_engine_resolver(flaky_resolver)
+ manager = RealBrowserManager()
+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")
+
+ with pytest.raises(RuntimeError, match="resolver boom"):
+ await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)
+ assert "wr_1" not in manager._engine_owners
+
+ recovered = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)
+ assert recovered.name == "engine-ok"
+
+
+async def test_concurrent_failed_first_resolution_leaves_no_orphan_owner(_restore_resolver: None) -> None:
+ # Two waiters on the same key whose shared resolution fails: both observe the failure and neither orphans an owner.
+ async def boom_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:
+ await asyncio.sleep(0)
+ raise RuntimeError("resolver boom")
+
+ browser_engine.set_browser_engine_resolver(boom_resolver)
+ manager = RealBrowserManager()
+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")
+
+ results = await asyncio.gather(
+ manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx),
+ manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx),
+ return_exceptions=True,
+ )
+ assert all(isinstance(r, RuntimeError) for r in results)
+ assert manager._engine_owners == {}
+
+
+async def test_waiter_cancellation_does_not_abort_shared_resolution(_restore_resolver: None) -> None:
+ # Cancelling one waiter must not abort the shared resolution nor drop the owner the survivor still needs.
+ calls = {"n": 0}
+ gate = asyncio.Event()
+
+ async def gated_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:
+ calls["n"] += 1
+ await gate.wait()
+ return _fake_selection(f"engine-{calls['n']}")
+
+ browser_engine.set_browser_engine_resolver(gated_resolver)
+ manager = RealBrowserManager()
+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")
+
+ survivor = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx))
+ victim = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx))
+ await asyncio.sleep(0) # let both attach to the shared owner task
+ victim.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await victim
+ gate.set()
+ selection = await survivor
+ assert selection.name == "engine-1"
+ assert calls["n"] == 1 # the shared resolution ran exactly once despite the cancelled waiter
+
+ # A waiter cancelled AFTER the resolution succeeds must not evict the healthy owner; that is the done-callback's job.
+ gate.clear()
+ ctx2 = BrowserEngineContext(workflow_run_id="wr_2", browser_source="local-browser")
+ keeper = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_2", context=ctx2))
+ late = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_2", context=ctx2))
+ await asyncio.sleep(0) # both attach to the shared owner task
+ manager._engine_owners["wr_2"].task.add_done_callback(lambda _t: late.cancel()) # cancel in success tick
+ gate.set()
+ with pytest.raises(asyncio.CancelledError):
+ await late
+ assert (await keeper).name == "engine-2"
+ assert "wr_2" in manager._engine_owners # successful owner survived the late waiter cancel
+ assert (await manager.get_or_resolve_engine_selection(run_key="wr_2", context=ctx2)) is keeper.result()
+ assert calls["n"] == 2 # wr_1 once, wr_2 once; the late cancel forced no re-resolution
+
+
+async def test_terminal_cleanup_racing_resolver_prevents_resurrection(_restore_resolver: None) -> None:
+ # Cleanup while a resolver is in flight must cancel it, leave no owner, not resurrect state; a later acquire resolves fresh.
+ calls = {"n": 0}
+ gate = asyncio.Event()
+
+ async def gated_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:
+ calls["n"] += 1
+ await gate.wait()
+ return _fake_selection(f"engine-{calls['n']}")
+
+ browser_engine.set_browser_engine_resolver(gated_resolver)
+ manager = RealBrowserManager()
+ ctx = BrowserEngineContext(workflow_run_id="wr_1", browser_source="local-browser")
+
+ inflight = asyncio.ensure_future(manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx))
+ await asyncio.sleep(0) # resolver is now attached and blocked on the gate
+ await manager._drop_engine_owner("wr_1") # terminal cleanup awaits the racing resolver to termination
+ with pytest.raises(asyncio.CancelledError):
+ await inflight
+ # The cancelled resolver stored nothing and did not reinstall an owner: no resurrection.
+ assert "wr_1" not in manager._engine_owners
+
+ gate.set() # a fresh resolution for the reused key resolves cleanly under a brand-new owner
+ fresh = await manager.get_or_resolve_engine_selection(run_key="wr_1", context=ctx)
+ assert manager._engine_owners["wr_1"].task.result() is fresh
+
+
+async def test_null_run_key_is_ephemeral_and_not_pinned(_restore_resolver: None) -> None:
+ calls = {"n": 0}
+
+ async def counting_resolver(ctx: BrowserEngineContext) -> BrowserEngineSelection:
+ calls["n"] += 1
+ return _fake_selection(f"engine-{calls['n']}")
+
+ browser_engine.set_browser_engine_resolver(counting_resolver)
+ manager = RealBrowserManager()
+ ctx = BrowserEngineContext(browser_source="local-browser")
+
+ first = await manager.get_or_resolve_engine_selection(run_key=None, context=ctx)
+ second = await manager.get_or_resolve_engine_selection(run_key=None, context=ctx)
+ assert first is not second # no durable identity => resolved each time, never cached
+ assert manager._engine_owners == {}
+
+
+async def test_workflow_owned_task_pins_engine_under_workflow_run_id(
+ monkeypatch: pytest.MonkeyPatch, _restore_resolver: None
+) -> None:
+ # The task path passes engine_run_key=canonical_run_key(...); a workflow-owned task pins under workflow_run_id only.
+ assert canonical_run_key(workflow_run_id="wr_1", task_id="tsk_1") == "wr_1"
+ assert canonical_run_key(task_id="tsk_1") == "tsk_1"
+
+ _patch_create_browser_state(monkeypatch, _pw_stub(), context_result=(_context_stub(), BrowserArtifacts(), None))
+ manager = RealBrowserManager()
+ await manager._create_browser_state(
+ task_id="tsk_1", engine_run_key=canonical_run_key(workflow_run_id="wr_1", task_id="tsk_1")
+ )
+ assert "wr_1" in manager._engine_owners
+ assert "tsk_1" not in manager._engine_owners
+
+
[email protected]("unsupported_source", ["local-browser", None])
+async def test_manager_capability_gate_applies_even_on_cache_hit(
+ _restore_resolver: None, unsupported_source: str | None
+) -> None:
+ # A restricted engine rejects an unsupported cached source (incl. None, failed closed); the pinned owner survives.
+ restricted = _fake_selection("restricted-engine", allowed_sources=frozenset({"cdp-connection-browser"}))
+ browser_engine.set_browser_engine_resolver(lambda ctx: _coro(restricted))
+ manager = RealBrowserManager()
+
+ ok = await manager.get_or_resolve_engine_selection(
+ run_key="wr_x",
+ context=BrowserEngineContext(workflow_run_id="wr_x", browser_source="cdp-connection-browser"),
+ )
+ assert ok is restricted
+ owner = manager._engine_owners["wr_x"]
+
+ with pytest.raises(BrowserSourceNotSupportedByEngine):
+ await manager.get_or_resolve_engine_selection(
+ run_key="wr_x",
+ context=BrowserEngineContext(workflow_run_id="wr_x", browser_source=unsupported_source),
+ )
+ assert manager._engine_owners.get("wr_x") is owner
+
+
+async def test_cleanup_drops_pinned_engine_owner_for_run(_restore_resolver: None) -> None:
+ manager = RealBrowserManager()
+ await _seed_engine_owner(manager, "wr_1", "engine-1")
+ await _seed_engine_owner(manager, "tsk_1", "engine-1")
+ manager.pages["wr_1"] = _fake_browser_state()
+
+ await manager.cleanup_for_workflow_run("wr_1", ["tsk_1"], close_browser_on_completion=False)
+
+ assert "wr_1" not in manager._engine_owners
+ assert "tsk_1" not in manager._engine_owners
+
+
+def _coro(value: BrowserEngineSelection):
+ async def _c(ctx: BrowserEngineContext | None = None) -> BrowserEngineSelection:
+ return value
+
+ return _c()