Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-8803): unblock concurrent browser launches in parallel for-loop engine (#5436)
Aaron Perezeb9ad1025a58
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 5479633a4..68f55e817 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -1916,7 +1916,7 @@ class ForLoopBlock(Block):# parallel iterations — every iteration would race on the same# live page. Fall back to sequential execution rather than# corrupt the session.- LOG.info(+ LOG.warning("Persistent browser session set; forcing sequential execution for for-loop",workflow_run_id=workflow_run_id,browser_session_id=browser_session_id,@@ -1945,6 +1945,14 @@ class ForLoopBlock(Block):# Fall through to sequential path belowif effective_concurrency > 1:+ if self.max_concurrency and effective_concurrency < self.max_concurrency:+ LOG.warning(+ "ForLoopBlock parallel granted concurrency below requested",+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ requested_concurrency=self.max_concurrency,+ granted_concurrency=effective_concurrency,+ )return await self._execute_loop_parallel(workflow_run_id=workflow_run_id,workflow_run_block_id=workflow_run_block_id,diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.pyindex 0b6151523..3f21b5718 100644--- a/skyvern/webeye/real_browser_manager.py+++ b/skyvern/webeye/real_browser_manager.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport os+import timeimport structlogfrom playwright.async_api import async_playwright@@ -22,6 +23,30 @@ from skyvern.webeye.real_browser_state import RealBrowserStateLOG = structlog.get_logger()+def _consume_future_exception(future: asyncio.Future[BrowserState]) -> None:+ """Done-callback that retrieves a future's exception so asyncio does not+ emit an "exception was never retrieved" warning when the owning coroutine+ re-raises and no other waiter attached to the in-flight future. The+ owner always re-raises the real exception on its own task, so dropping+ it here is safe — this callback only exists to silence asyncio's+ unhandled-exception warning when the in-flight group happened to have+ zero waiters at the time of failure.++ Logs at INFO so launch failures and teardown-race exceptions are+ correlatable with incidents even when the owner's re-raise lands in a+ separate log entry (cleanup/close/owner-cancellation paths intentionally+ surface a RuntimeError here; real Playwright failures surface the real+ exception type)."""+ if future.cancelled():+ return+ exc = future.exception()+ if exc is not None:+ LOG.info(+ "Consumed in-flight loop iteration launch exception (re-raised on owner)",+ exc_type=type(exc).__name__,+ )++class RealBrowserManager(BrowserManager):def __init__(self) -> None:self.pages: dict[str, BrowserState] = {}@@ -29,6 +54,10 @@ class RealBrowserManager(BrowserManager):# to the wrong event loop when RealBrowserManager is instantiated# during module import or test fixtures that haven't started a loop.self._loop_iteration_lock_instance: asyncio.Lock | None = None+ # Single-flight registry for in-progress loop-iteration browser+ # launches. The lock is held only across check-then-insert, not+ # across the Playwright launch itself.+ self._loop_iteration_inflight: dict[str, asyncio.Future[BrowserState]] = {}@propertydef _loop_iteration_lock(self) -> asyncio.Lock:@@ -396,9 +425,33 @@ class RealBrowserManager(BrowserManager):async def close(self) -> None:LOG.info("Closing BrowserManager")- for browser_state in self.pages.values():- await browser_state.close()- self.pages = dict()+ # Hold the loop-iteration lock across the whole shutdown so a+ # concurrent Phase 1 registration cannot land a fresh in-flight+ # future after we've already cancelled + cleared the dict, and+ # so cleanup_loop_iterations cannot race with us for the same+ # keys. cleanup_loop_iterations already takes this lock; close()+ # must do the same for the invariant to hold.+ async with self._loop_iteration_lock:+ # Fail in-flight launches with a regular Exception (not cancel())+ # so shielded same-key waiters see a RuntimeError they can route+ # through _execute_single_iteration_parallel's Exception handler,+ # instead of a CancelledError that would bypass `isinstance(result,+ # Exception)` in _execute_loop_parallel and crash the batch unpack.+ # The owner of each launch still discards its new browser state+ # via the identity-guarded check below (stored_future mismatch) —+ # future.cancelled() is not required for discard.+ for key, future in list(self._loop_iteration_inflight.items()):+ if not future.done():+ future.set_exception(RuntimeError(f"loop iteration {key} was cleaned up during launch"))+ self._loop_iteration_inflight.clear()+ # Intentionally sequential here (unlike cleanup_loop_iterations,+ # which parallelizes via asyncio.gather). close() runs under+ # _loop_iteration_lock so that Phase 1 registration cannot+ # race the teardown; a future refactor to gather() must+ # preserve that lock-held invariant.+ for browser_state in self.pages.values():+ await browser_state.close()+ self.pages = dict()LOG.info("BrowserManger is closed")async def cleanup_for_task(@@ -589,44 +642,188 @@ class RealBrowserManager(BrowserManager):Each iteration gets its own browser context so cookies, auth state, andstorage are fully isolated between concurrent iterations.- Uses _loop_iteration_lock to prevent concurrent create_task coroutines- from racing through the check-then-act on self.pages.+ Uses _loop_iteration_lock to guard the check-then-register on+ self.pages / self._loop_iteration_inflight, but releases it before+ doing the Playwright launch so concurrent iterations with distinct+ keys can cold-start in parallel. Single-flight semantics are+ preserved via an in-flight futures map so concurrent callers for+ the same key share a single launch."""key = loop_iteration_key(workflow_run_id, loop_idx)+ # Phase 1: short critical section — either return cached, attach to+ # an in-flight launch, or register a new future and proceed to launch.async with self._loop_iteration_lock:if key in self.pages:- return self.pages[key]-- # Persistent sessions cannot be aliased under per-iteration keys —- # multiple iterations would race on the same live page. The caller- # (execute_loop_helper) is expected to force sequential execution- # when a persistent session is in use; this branch only runs if- # that contract is bypassed, in which case we still create a fresh- # isolated context to preserve correctness over the persistence.- if browser_session_id:+ cached_state = self.pages[key]+ inflight_future: asyncio.Future[BrowserState] | None = None+ owns_launch = False+ elif key in self._loop_iteration_inflight:+ cached_state = None+ inflight_future = self._loop_iteration_inflight[key]+ owns_launch = False+ else:+ cached_state = None+ inflight_future = asyncio.get_running_loop().create_future()+ # Eagerly observe any exception the future may carry so that+ # a launch failure with no attached waiters doesn't emit an+ # "exception was never retrieved" warning from asyncio. The+ # owning coroutine always re-raises separately.+ inflight_future.add_done_callback(_consume_future_exception)+ self._loop_iteration_inflight[key] = inflight_future+ owns_launch = True++ # Persistent sessions cannot be aliased under per-iteration keys —+ # multiple iterations would race on the same live page. The caller+ # (execute_loop_helper) is expected to force sequential execution+ # when a persistent session is in use; this branch only runs if+ # that contract is bypassed, in which case we still create a fresh+ # isolated context to preserve correctness over the persistence.+ if browser_session_id:+ LOG.warning(+ "Persistent browser session not used for parallel loop iteration — "+ "creating isolated context to avoid cross-iteration races",+ workflow_run_id=workflow_run_id,+ loop_idx=loop_idx,+ browser_session_id=browser_session_id,+ )++ LOG.info(+ "Creating isolated browser state for loop iteration",+ workflow_run_id=workflow_run_id,+ loop_idx=loop_idx,+ key=key,+ )++ # Phase 2: fulfil the request outside the lock.+ if cached_state is not None:+ browser_state = cached_state+ elif not owns_launch:+ # Attached to another coroutine's in-flight launch. Shield the+ # shared future from this waiter's cancellation so one waiter+ # being cancelled does not cancel the shared launch and fan out+ # CancelledError to the entire single-flight group.+ assert inflight_future is not None+ browser_state = await asyncio.shield(inflight_future)+ else:+ # This coroutine owns the launch — do it unlocked so other keys+ # can launch in parallel. The owner also performs the first+ # get_or_create_page before publishing the future so same-key+ # waiters don't race inside RealBrowserState.get_or_create_page+ # (which is not internally synchronized).+ assert inflight_future is not None+ owner_state: BrowserState | None = None+ launch_start = time.perf_counter()+ try:+ owner_state = await self._create_browser_state(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ )+ await owner_state.get_or_create_page(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ )+ except BaseException as exc:+ # The pop + future-resolution pair below must be synchronous+ # (no awaits between them) so cancellation cannot split the+ # pop from the future resolution and orphan the in-flight+ # entry. Any attached waiters see the exception; subsequent+ # callers get a fresh launch.+ #+ # Identity-guarded pop: a concurrent cleanup may have+ # cancelled+removed our future and a replacement owner may+ # have registered a new one for the same key. Only remove+ # the entry if it is still ours — otherwise leave the+ # replacement alone.+ if self._loop_iteration_inflight.get(key) is inflight_future:+ self._loop_iteration_inflight.pop(key, None)+ if not inflight_future.done():+ # Normalize owner-task cancellation to a RuntimeError for+ # shielded waiters. A raw CancelledError on the shared+ # future would bypass `isinstance(result, Exception)` in+ # _execute_loop_parallel (CancelledError is a BaseException)+ # and crash the batch success-unpack. The owner task+ # itself still re-raises the original CancelledError via+ # the `raise` below to let its own cancellation propagate.+ if isinstance(exc, asyncio.CancelledError):+ inflight_future.set_exception(RuntimeError(f"loop iteration {key} launch cancelled"))+ else:+ inflight_future.set_exception(exc)+ # Teardown below runs AFTER the future is already settled,+ # so the await does not affect waiters or reopen a race on+ # the in-flight entry. Best effort — a failing close should+ # not mask the original exception.+ if owner_state is not None:+ try:+ await asyncio.shield(owner_state.close())+ except BaseException:+ LOG.warning(+ "Failed to close half-initialized loop iteration browser state",+ workflow_run_id=workflow_run_id,+ loop_idx=loop_idx,+ exc_info=True,+ )+ raise++ assert owner_state is not None+ browser_state = owner_state+ # Success path is synchronous from here on — no awaits between+ # page init completing and the future being resolved, so+ # cancellation of this owner task cannot orphan the in-flight+ # entry.+ #+ # cleanup_loop_iterations and close() can run concurrently with+ # the launch (launch is no longer under the lock). Both teardown+ # paths pop our entry from _loop_iteration_inflight and then set+ # a RuntimeError on the future, so our identity check below+ # detects the race via `stored_future is not inflight_future`+ # alone. No `inflight_future.cancelled()` check is needed+ # because neither teardown path calls `.cancel()` — both use+ # set_exception so shielded waiters receive a regular Exception+ # that _execute_single_iteration_parallel handles normally+ # instead of a BaseException-derived CancelledError that would+ # bypass the batch-result dispatch.+ stored_future = self._loop_iteration_inflight.get(key)+ if stored_future is inflight_future:+ self._loop_iteration_inflight.pop(key, None)+ if stored_future is not inflight_future:LOG.warning(- "Persistent browser session not used for parallel loop iteration — "- "creating isolated context to avoid cross-iteration races",+ "Loop iteration cleanup ran during launch, discarding browser state",workflow_run_id=workflow_run_id,loop_idx=loop_idx,- browser_session_id=browser_session_id,+ key=key,)+ try:+ await asyncio.shield(browser_state.close())+ except BaseException:+ LOG.warning(+ "Failed to close loop iteration browser state after concurrent cleanup",+ workflow_run_id=workflow_run_id,+ loop_idx=loop_idx,+ exc_info=True,+ )+ # Use a plain Exception subclass rather than CancelledError:+ # the parallel for-loop driver routes gather() results via+ # isinstance(result, Exception), and BaseException-derived+ # CancelledError would bypass that check and crash the+ # success-unpack path with a TypeError.+ raise RuntimeError(f"loop iteration {key} was cleaned up during launch")+ self.pages[key] = browser_state+ if not inflight_future.done():+ inflight_future.set_result(browser_state)LOG.info(- "Creating isolated browser state for loop iteration",+ "Loop iteration browser launch completed",workflow_run_id=workflow_run_id,loop_idx=loop_idx,key=key,+ elapsed_s=round(time.perf_counter() - launch_start, 3),)- browser_state = await self._create_browser_state(- workflow_run_id=workflow_run_id,- organization_id=organization_id,- )- self.pages[key] = browser_state+ return browser_state- # Page creation can happen outside the lock — the key is already- # reserved in self.pages so no other coroutine will race on it.+ # Waiters/cache-hit path: the owner has already completed the+ # initial get_or_create_page before publishing the future, so this+ # call will cache-hit and not race with any concurrent setup.await browser_state.get_or_create_page(workflow_run_id=workflow_run_id,organization_id=organization_id,@@ -646,10 +843,24 @@ class RealBrowserManager(BrowserManager):"""# Collect entries to close under the lock, then close outside it# to avoid holding the lock during potentially slow browser teardown.+ # Also fail any in-flight launches for these keys so a launch still+ # running in its unlocked section discards its new browser state+ # instead of publishing it after cleanup has already run.to_close: list[tuple[int, BrowserState]] = []async with self._loop_iteration_lock:for loop_idx in loop_indices:key = loop_iteration_key(workflow_run_id, loop_idx)+ inflight_future = self._loop_iteration_inflight.pop(key, None)+ if inflight_future is not None and not inflight_future.done():+ # set_exception (not cancel()) so shielded same-key+ # waiters receive a RuntimeError that+ # _execute_single_iteration_parallel can catch via its+ # `except Exception` handler, instead of a CancelledError+ # that would bypass _execute_loop_parallel's+ # `isinstance(result, Exception)` dispatch and crash the+ # batch success-unpack path (CancelledError is a+ # BaseException, not an Exception).+ inflight_future.set_exception(RuntimeError(f"loop iteration {key} was cleaned up during launch"))browser_state = self.pages.pop(key, None)if browser_state is None:continue@@ -658,13 +869,34 @@ class RealBrowserManager(BrowserManager):if not shared:to_close.append((loop_idx, browser_state))- for loop_idx, browser_state in to_close:- try:- await browser_state.close()- except Exception:+ if not to_close:+ return++ # Close all entries in parallel — Playwright context teardown is+ # IO-bound and would otherwise dominate the batch gap.+ close_start = time.perf_counter()+ results = await asyncio.gather(+ *(browser_state.close() for _, browser_state in to_close),+ return_exceptions=True,+ )+ close_elapsed_s = time.perf_counter() - close_start+ failure_count = sum(1 for r in results if isinstance(r, BaseException))+ LOG.info(+ "cleanup_loop_iterations closed entries",+ workflow_run_id=workflow_run_id,+ count=len(to_close),+ failures=failure_count,+ elapsed_s=round(close_elapsed_s, 3),+ )+ for (loop_idx, _), result in zip(to_close, results):+ if isinstance(result, BaseException):LOG.warning("Failed to close loop iteration browser state",workflow_run_id=workflow_run_id,loop_idx=loop_idx,- exc_info=True,+ # return_exceptions=True means there is no active+ # exception context, so exc_info=True won't pick+ # anything up. Pass an explicit (type, value, tb)+ # tuple — structlog's documented signature.+ exc_info=(type(result), result, result.__traceback__),)diff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.pyindex 2fd02c117..251878923 100644--- a/tests/unit/test_real_browser_manager.py+++ b/tests/unit/test_real_browser_manager.py@@ -6,10 +6,13 @@ behind `if not browser_session_id:`, causing PBS workflow runs to skip the cacheon every call and re-invoke navigate_to_url() on every step."""+import asyncio+import timefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from skyvern.constants import loop_iteration_keyfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.webeye.real_browser_manager import RealBrowserManager@@ -222,3 +225,552 @@ async def test_parallel_iteration_different_iterations_get_different_browsers()assert result0 is iter0_stateassert result1 is iter1_stateassert result0 is not result1+++# ---------------------------------------------------------------------------+# Parallel-loop browser launch must not be serialized on a lock.+# ---------------------------------------------------------------------------+++def _make_fake_browser_state() -> MagicMock:+ state = MagicMock()+ state.get_or_create_page = AsyncMock(return_value=None)+ state.close = AsyncMock(return_value=None)+ return state+++async def test_loop_iteration_launches_overlap() -> None:+ """Concurrent get_or_create_for_loop_iteration calls with distinct keys+ must overlap in wall-clock time. If launches serialize under a lock,+ N * per_launch is observed; this asserts < 2 * per_launch for 4 calls.+ """+ manager = RealBrowserManager()+ per_launch = 0.3+ call_count = 0++ async def fake_create(**_kwargs: object) -> MagicMock:+ nonlocal call_count+ call_count += 1+ await asyncio.sleep(per_launch)+ return _make_fake_browser_state()++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=fake_create):+ start = time.perf_counter()+ results = await asyncio.gather(+ *(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_overlap",+ loop_idx=i,+ organization_id="org_test",+ )+ for i in range(4)+ )+ )+ elapsed = time.perf_counter() - start++ assert call_count == 4, f"expected 4 launches, got {call_count}"+ assert len(manager.pages) == 4+ # Serialized would be ~1.2s; parallel ~0.3s. 3x gives jitter slack for+ # slow CI runners (GitHub Actions can stall 500ms+ on GC). call_count+ # already proves correctness; this bound is a secondary signal.+ assert elapsed < per_launch * 3, (+ f"concurrent launches did not overlap: elapsed={elapsed:.3f}s "+ f"(per_launch={per_launch}s, expected < {per_launch * 3}s)"+ )+ # Every returned state must be a distinct object.+ assert len({id(r) for r in results}) == 4+++async def test_loop_iteration_single_flight_same_key() -> None:+ """Concurrent callers for the same key must collapse to a single+ underlying browser launch (single-flight)."""+ manager = RealBrowserManager()+ call_count = 0+ created_state = _make_fake_browser_state()++ async def fake_create(**_kwargs: object) -> MagicMock:+ nonlocal call_count+ call_count += 1+ await asyncio.sleep(0.1)+ return created_state++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=fake_create):+ results = await asyncio.gather(+ *(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_single_flight",+ loop_idx=7,+ organization_id="org_test",+ )+ for _ in range(4)+ )+ )++ assert call_count == 1+ assert len(manager.pages) == 1+ assert all(r is created_state for r in results)+++async def test_loop_iteration_owner_cancellation_does_not_orphan_inflight() -> None:+ """If the launch-owning task is cancelled mid-launch, subsequent callers+ for the same key must not await a stale in-flight future forever — the+ in-flight entry must be cleared so the next caller starts a fresh+ launch."""+ manager = RealBrowserManager()+ launch_started = asyncio.Event()+ call_count = 0++ async def hanging_create(**_kwargs: object) -> MagicMock:+ nonlocal call_count+ call_count += 1+ launch_started.set()+ # Hang until cancelled.+ await asyncio.sleep(10)+ return _make_fake_browser_state() # pragma: no cover++ async def fast_create(**_kwargs: object) -> MagicMock:+ nonlocal call_count+ call_count += 1+ return _make_fake_browser_state()++ # First call: owner, gets cancelled while _create_browser_state hangs.+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=hanging_create):+ owner_task = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_cancel",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await launch_started.wait()+ owner_task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await owner_task++ # In-flight entry must be cleared so a retry is not stuck waiting on it.+ assert "wr_cancel__iter_0" not in manager._loop_iteration_inflight+ assert "wr_cancel__iter_0" not in manager.pages++ # Retry succeeds with a fresh launch.+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=fast_create):+ result = await manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_cancel",+ loop_idx=0,+ organization_id="org_test",+ )++ assert result is not None+ assert call_count == 2+ assert "wr_cancel__iter_0" in manager.pages+++async def test_loop_iteration_waiter_cancellation_does_not_cancel_owner() -> None:+ """Cancelling one waiter on a single-flight key must not cancel the+ shared launch future — otherwise the owner and every other waiter+ would fan out into CancelledError."""+ manager = RealBrowserManager()+ launch_started = asyncio.Event()+ call_count = 0+ created_state = _make_fake_browser_state()++ async def slow_create(**_kwargs: object) -> MagicMock:+ nonlocal call_count+ call_count += 1+ launch_started.set()+ await asyncio.sleep(0.2)+ return created_state++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=slow_create):+ owner = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_waiter_cancel",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await launch_started.wait()++ waiter_to_cancel = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_waiter_cancel",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ surviving_waiter = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_waiter_cancel",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ # Let the waiters reach their shielded await.+ await asyncio.sleep(0)+ waiter_to_cancel.cancel()+ with pytest.raises(asyncio.CancelledError):+ await waiter_to_cancel++ owner_result = await owner+ surviving_result = await surviving_waiter++ assert call_count == 1+ assert owner_result is created_state+ assert surviving_result is created_state+++async def test_cleanup_during_launch_discards_orphan_state() -> None:+ """If cleanup_loop_iterations runs while a launch is still in-flight+ for the same key, the owner must close the browser state it created+ instead of publishing it into self.pages — otherwise an orphaned+ Playwright context is leaked."""+ manager = RealBrowserManager()+ launch_started = asyncio.Event()+ cleanup_done = asyncio.Event()+ created_state = _make_fake_browser_state()++ async def gated_create(**_kwargs: object) -> MagicMock:+ launch_started.set()+ await cleanup_done.wait()+ return created_state++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):+ owner = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_race",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await launch_started.wait()+ # Cleanup runs mid-launch.+ await manager.cleanup_loop_iterations(+ workflow_run_id="wr_race",+ loop_indices=[0],+ organization_id="org_test",+ )+ cleanup_done.set()+ with pytest.raises(RuntimeError, match="cleaned up during launch"):+ await owner++ assert "wr_race__iter_0" not in manager.pages+ assert "wr_race__iter_0" not in manager._loop_iteration_inflight+ created_state.close.assert_awaited()+++async def test_cleanup_during_launch_does_not_disturb_replacement_owner() -> None:+ """After cleanup cancels owner A's in-flight future, a replacement+ owner B may register a new future for the same key before owner A+ finishes unwinding. Owner A must not pop B's future (Codex P1)."""+ manager = RealBrowserManager()+ owner_a_started = asyncio.Event()+ owner_a_can_finish = asyncio.Event()+ states: list[MagicMock] = []++ async def gated_create(**_kwargs: object) -> MagicMock:+ state = _make_fake_browser_state()+ states.append(state)+ if len(states) == 1:+ owner_a_started.set()+ await owner_a_can_finish.wait()+ return state++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):+ owner_a = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_replace",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await owner_a_started.wait()++ # Cleanup cancels owner A's in-flight future.+ await manager.cleanup_loop_iterations(+ workflow_run_id="wr_replace",+ loop_indices=[0],+ organization_id="org_test",+ )++ # Owner B registers a brand-new future for the same key.+ owner_b = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_replace",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ # Let owner B reach its await on _create_browser_state (second call).+ await asyncio.sleep(0)++ # Now let owner A finish unwinding. It must not disturb owner B's+ # inflight entry.+ owner_a_can_finish.set()+ with pytest.raises(RuntimeError, match="cleaned up during launch"):+ await owner_a++ result_b = await owner_b++ assert result_b is states[1]+ assert manager.pages["wr_replace__iter_0"] is states[1]+ assert "wr_replace__iter_0" not in manager._loop_iteration_inflight+ # Owner A's orphaned state was closed; owner B's state is intact.+ states[0].close.assert_awaited()+ states[1].close.assert_not_awaited()+++async def test_owner_failure_does_not_disturb_replacement_inflight() -> None:+ """Owner A's failure path must identity-check before popping inflight.+ If cleanup cancelled A and owner B registered a replacement future,+ A's failure must not disturb B's entry (Codex P1)."""+ manager = RealBrowserManager()+ owner_a_started = asyncio.Event()+ owner_a_can_finish = asyncio.Event()+ states: list[MagicMock] = []++ class LaunchFailure(RuntimeError):+ pass++ async def gated_create(**_kwargs: object) -> MagicMock:+ if not states:+ states.append(object()) # sentinel for "A ran"+ owner_a_started.set()+ await owner_a_can_finish.wait()+ raise LaunchFailure("A failed")+ state = _make_fake_browser_state()+ states.append(state)+ return state++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):+ owner_a = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_fail_race",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await owner_a_started.wait()++ await manager.cleanup_loop_iterations(+ workflow_run_id="wr_fail_race",+ loop_indices=[0],+ organization_id="org_test",+ )++ owner_b = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_fail_race",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await asyncio.sleep(0)++ owner_a_can_finish.set()+ # Owner A raises — it was cancelled by cleanup, so CancelledError+ # propagates first (cancellation takes precedence over the+ # subsequent LaunchFailure); either way it must not touch B.+ with pytest.raises(BaseException):+ await owner_a++ result_b = await owner_b++ assert result_b is states[1]+ assert manager.pages["wr_fail_race__iter_0"] is states[1]+ assert "wr_fail_race__iter_0" not in manager._loop_iteration_inflight+ states[1].close.assert_not_awaited()+++async def test_close_during_launch_does_not_orphan_state() -> None:+ """RealBrowserManager.close() while an owner is mid-launch must+ cancel the owner's in-flight future and cause the owner to close+ its own newly-created state instead of publishing it into+ self.pages after close() has already reset the dict."""+ manager = RealBrowserManager()+ launch_started = asyncio.Event()+ close_done = asyncio.Event()+ created_state = _make_fake_browser_state()++ async def gated_create(**_kwargs: object) -> MagicMock:+ launch_started.set()+ await close_done.wait()+ return created_state++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):+ owner = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_close_race",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await launch_started.wait()++ await manager.close()+ close_done.set()+ with pytest.raises(RuntimeError, match="cleaned up during launch"):+ await owner++ assert "wr_close_race__iter_0" not in manager.pages+ assert "wr_close_race__iter_0" not in manager._loop_iteration_inflight+ created_state.close.assert_awaited()+++async def test_cleanup_during_launch_surfaces_exception_to_waiters() -> None:+ """Codex P2 regression: cleanup_loop_iterations must surface a regular+ Exception (not CancelledError) to same-key waiters. CancelledError is a+ BaseException and would bypass `_execute_single_iteration_parallel`'s+ `except Exception` handler and `_execute_loop_parallel`'s+ `isinstance(result, Exception)` dispatch, crashing the batch unpack."""+ manager = RealBrowserManager()+ owner_started = asyncio.Event()+ cleanup_done = asyncio.Event()+ created_state = _make_fake_browser_state()++ async def gated_create(**_kwargs: object) -> MagicMock:+ owner_started.set()+ await cleanup_done.wait()+ return created_state++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):+ owner = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_waiter_exc",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await owner_started.wait()++ # Attach waiters AFTER the in-flight future is registered but+ # BEFORE cleanup runs, so they have a shielded reference that+ # cleanup's set_exception must reach.+ waiters = [+ asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_waiter_exc",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ for _ in range(2)+ ]+ # Yield so waiters reach `await asyncio.shield(inflight_future)`.+ await asyncio.sleep(0)++ await manager.cleanup_loop_iterations(+ workflow_run_id="wr_waiter_exc",+ loop_indices=[0],+ organization_id="org_test",+ )+ cleanup_done.set()++ for waiter in waiters:+ with pytest.raises(Exception) as excinfo:+ await waiter+ # Critically, not a CancelledError — that would bypass the+ # batch handler and crash the success-unpack path.+ assert not isinstance(excinfo.value, asyncio.CancelledError), (+ "waiter must receive a regular Exception, not CancelledError"+ )+ assert "cleaned up during launch" in str(excinfo.value)++ with pytest.raises(RuntimeError, match="cleaned up during launch"):+ await owner++ assert "wr_waiter_exc__iter_0" not in manager.pages+ assert "wr_waiter_exc__iter_0" not in manager._loop_iteration_inflight+++async def test_owner_cancellation_surfaces_exception_to_waiters() -> None:+ """If the launch-owning task is cancelled while same-key waiters are+ attached, waiters must receive a regular Exception (not CancelledError)+ so the batch driver's `isinstance(result, Exception)` dispatch catches+ them instead of the success-unpack path crashing on a BaseException."""+ manager = RealBrowserManager()+ owner_started = asyncio.Event()++ async def hanging_create(**_kwargs: object) -> MagicMock:+ owner_started.set()+ await asyncio.sleep(10)+ return _make_fake_browser_state() # pragma: no cover++ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=hanging_create):+ owner = asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_owner_cancel_waiters",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ await owner_started.wait()++ waiters = [+ asyncio.create_task(+ manager.get_or_create_for_loop_iteration(+ workflow_run_id="wr_owner_cancel_waiters",+ loop_idx=0,+ organization_id="org_test",+ )+ )+ for _ in range(2)+ ]+ # Let waiters reach their shielded await.+ await asyncio.sleep(0)++ owner.cancel()+ with pytest.raises(asyncio.CancelledError):+ await owner++ for waiter in waiters:+ with pytest.raises(Exception) as excinfo:+ await waiter+ assert not isinstance(excinfo.value, asyncio.CancelledError), (+ "waiter must receive a regular Exception when owner is cancelled"+ )+ assert "launch cancelled" in str(excinfo.value)++ assert "wr_owner_cancel_waiters__iter_0" not in manager._loop_iteration_inflight+ assert "wr_owner_cancel_waiters__iter_0" not in manager.pages+++async def test_cleanup_loop_iterations_runs_close_in_parallel() -> None:+ """cleanup_loop_iterations must close browser contexts in parallel.+ Sequential close of 4 entries at 0.2s each would take ~0.8s; parallel+ should finish in ~0.2s. 0.4s ceiling gives CI slack."""+ manager = RealBrowserManager()+ per_close = 0.2++ async def slow_close() -> None:+ await asyncio.sleep(per_close)++ loop_indices = [0, 1, 2, 3]+ for idx in loop_indices:+ state = _make_fake_browser_state()+ state.close = AsyncMock(side_effect=slow_close)+ manager.pages[loop_iteration_key("wr_cleanup", idx)] = state++ start = time.perf_counter()+ await manager.cleanup_loop_iterations(+ workflow_run_id="wr_cleanup",+ loop_indices=loop_indices,+ organization_id="org_test",+ )+ elapsed = time.perf_counter() - start++ assert len(manager.pages) == 0+ # 3x for CI jitter slack; len(manager.pages) == 0 is the real correctness check.+ assert elapsed < per_close * 3, (+ f"cleanup did not run closes in parallel: elapsed={elapsed:.3f}s "+ f"(per_close={per_close}s, expected < {per_close * 3}s)"+ )