Skip to content

Automate browser based workflows with AI

fix(SKY-8803): unblock concurrent browser launches in parallel for-loop engine (#5436)

Aaron Perezeb9ad1025a58
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 5479633a4..68f55e817 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -1916,7 +1916,7 @@ class ForLoopBlock(Block):
# parallel iterations — every iteration would race on the same
# live page. Fall back to sequential execution rather than
# corrupt the session.
- LOG.info(
+ LOG.warning(
"Persistent browser session set; forcing sequential execution for for-loop",
workflow_run_id=workflow_run_id,
browser_session_id=browser_session_id,
@@ -1945,6 +1945,14 @@ class ForLoopBlock(Block):
# Fall through to sequential path below
if effective_concurrency > 1:
+ if self.max_concurrency and effective_concurrency < self.max_concurrency:
+ LOG.warning(
+ "ForLoopBlock parallel granted concurrency below requested",
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ requested_concurrency=self.max_concurrency,
+ granted_concurrency=effective_concurrency,
+ )
return await self._execute_loop_parallel(
workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.py
index 0b6151523..3f21b5718 100644
--- a/skyvern/webeye/real_browser_manager.py
+++ b/skyvern/webeye/real_browser_manager.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import os
+import time
import structlog
from playwright.async_api import async_playwright
@@ -22,6 +23,30 @@ from skyvern.webeye.real_browser_state import RealBrowserState
LOG = structlog.get_logger()
+def _consume_future_exception(future: asyncio.Future[BrowserState]) -> None:
+ """Done-callback that retrieves a future's exception so asyncio does not
+ emit an "exception was never retrieved" warning when the owning coroutine
+ re-raises and no other waiter attached to the in-flight future. The
+ owner always re-raises the real exception on its own task, so dropping
+ it here is safe — this callback only exists to silence asyncio's
+ unhandled-exception warning when the in-flight group happened to have
+ zero waiters at the time of failure.
+
+ Logs at INFO so launch failures and teardown-race exceptions are
+ correlatable with incidents even when the owner's re-raise lands in a
+ separate log entry (cleanup/close/owner-cancellation paths intentionally
+ surface a RuntimeError here; real Playwright failures surface the real
+ exception type)."""
+ if future.cancelled():
+ return
+ exc = future.exception()
+ if exc is not None:
+ LOG.info(
+ "Consumed in-flight loop iteration launch exception (re-raised on owner)",
+ exc_type=type(exc).__name__,
+ )
+
+
class RealBrowserManager(BrowserManager):
def __init__(self) -> None:
self.pages: dict[str, BrowserState] = {}
@@ -29,6 +54,10 @@ class RealBrowserManager(BrowserManager):
# to the wrong event loop when RealBrowserManager is instantiated
# during module import or test fixtures that haven't started a loop.
self._loop_iteration_lock_instance: asyncio.Lock | None = None
+ # Single-flight registry for in-progress loop-iteration browser
+ # launches. The lock is held only across check-then-insert, not
+ # across the Playwright launch itself.
+ self._loop_iteration_inflight: dict[str, asyncio.Future[BrowserState]] = {}
@property
def _loop_iteration_lock(self) -> asyncio.Lock:
@@ -396,9 +425,33 @@ class RealBrowserManager(BrowserManager):
async def close(self) -> None:
LOG.info("Closing BrowserManager")
- for browser_state in self.pages.values():
- await browser_state.close()
- self.pages = dict()
+ # Hold the loop-iteration lock across the whole shutdown so a
+ # concurrent Phase 1 registration cannot land a fresh in-flight
+ # future after we've already cancelled + cleared the dict, and
+ # so cleanup_loop_iterations cannot race with us for the same
+ # keys. cleanup_loop_iterations already takes this lock; close()
+ # must do the same for the invariant to hold.
+ async with self._loop_iteration_lock:
+ # Fail in-flight launches with a regular Exception (not cancel())
+ # so shielded same-key waiters see a RuntimeError they can route
+ # through _execute_single_iteration_parallel's Exception handler,
+ # instead of a CancelledError that would bypass `isinstance(result,
+ # Exception)` in _execute_loop_parallel and crash the batch unpack.
+ # The owner of each launch still discards its new browser state
+ # via the identity-guarded check below (stored_future mismatch) —
+ # future.cancelled() is not required for discard.
+ for key, future in list(self._loop_iteration_inflight.items()):
+ if not future.done():
+ future.set_exception(RuntimeError(f"loop iteration {key} was cleaned up during launch"))
+ self._loop_iteration_inflight.clear()
+ # Intentionally sequential here (unlike cleanup_loop_iterations,
+ # which parallelizes via asyncio.gather). close() runs under
+ # _loop_iteration_lock so that Phase 1 registration cannot
+ # race the teardown; a future refactor to gather() must
+ # preserve that lock-held invariant.
+ for browser_state in self.pages.values():
+ await browser_state.close()
+ self.pages = dict()
LOG.info("BrowserManger is closed")
async def cleanup_for_task(
@@ -589,44 +642,188 @@ class RealBrowserManager(BrowserManager):
Each iteration gets its own browser context so cookies, auth state, and
storage are fully isolated between concurrent iterations.
- Uses _loop_iteration_lock to prevent concurrent create_task coroutines
- from racing through the check-then-act on self.pages.
+ Uses _loop_iteration_lock to guard the check-then-register on
+ self.pages / self._loop_iteration_inflight, but releases it before
+ doing the Playwright launch so concurrent iterations with distinct
+ keys can cold-start in parallel. Single-flight semantics are
+ preserved via an in-flight futures map so concurrent callers for
+ the same key share a single launch.
"""
key = loop_iteration_key(workflow_run_id, loop_idx)
+ # Phase 1: short critical section — either return cached, attach to
+ # an in-flight launch, or register a new future and proceed to launch.
async with self._loop_iteration_lock:
if key in self.pages:
- return self.pages[key]
-
- # Persistent sessions cannot be aliased under per-iteration keys —
- # multiple iterations would race on the same live page. The caller
- # (execute_loop_helper) is expected to force sequential execution
- # when a persistent session is in use; this branch only runs if
- # that contract is bypassed, in which case we still create a fresh
- # isolated context to preserve correctness over the persistence.
- if browser_session_id:
+ cached_state = self.pages[key]
+ inflight_future: asyncio.Future[BrowserState] | None = None
+ owns_launch = False
+ elif key in self._loop_iteration_inflight:
+ cached_state = None
+ inflight_future = self._loop_iteration_inflight[key]
+ owns_launch = False
+ else:
+ cached_state = None
+ inflight_future = asyncio.get_running_loop().create_future()
+ # Eagerly observe any exception the future may carry so that
+ # a launch failure with no attached waiters doesn't emit an
+ # "exception was never retrieved" warning from asyncio. The
+ # owning coroutine always re-raises separately.
+ inflight_future.add_done_callback(_consume_future_exception)
+ self._loop_iteration_inflight[key] = inflight_future
+ owns_launch = True
+
+ # Persistent sessions cannot be aliased under per-iteration keys —
+ # multiple iterations would race on the same live page. The caller
+ # (execute_loop_helper) is expected to force sequential execution
+ # when a persistent session is in use; this branch only runs if
+ # that contract is bypassed, in which case we still create a fresh
+ # isolated context to preserve correctness over the persistence.
+ if browser_session_id:
+ LOG.warning(
+ "Persistent browser session not used for parallel loop iteration — "
+ "creating isolated context to avoid cross-iteration races",
+ workflow_run_id=workflow_run_id,
+ loop_idx=loop_idx,
+ browser_session_id=browser_session_id,
+ )
+
+ LOG.info(
+ "Creating isolated browser state for loop iteration",
+ workflow_run_id=workflow_run_id,
+ loop_idx=loop_idx,
+ key=key,
+ )
+
+ # Phase 2: fulfil the request outside the lock.
+ if cached_state is not None:
+ browser_state = cached_state
+ elif not owns_launch:
+ # Attached to another coroutine's in-flight launch. Shield the
+ # shared future from this waiter's cancellation so one waiter
+ # being cancelled does not cancel the shared launch and fan out
+ # CancelledError to the entire single-flight group.
+ assert inflight_future is not None
+ browser_state = await asyncio.shield(inflight_future)
+ else:
+ # This coroutine owns the launch — do it unlocked so other keys
+ # can launch in parallel. The owner also performs the first
+ # get_or_create_page before publishing the future so same-key
+ # waiters don't race inside RealBrowserState.get_or_create_page
+ # (which is not internally synchronized).
+ assert inflight_future is not None
+ owner_state: BrowserState | None = None
+ launch_start = time.perf_counter()
+ try:
+ owner_state = await self._create_browser_state(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ )
+ await owner_state.get_or_create_page(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ )
+ except BaseException as exc:
+ # The pop + future-resolution pair below must be synchronous
+ # (no awaits between them) so cancellation cannot split the
+ # pop from the future resolution and orphan the in-flight
+ # entry. Any attached waiters see the exception; subsequent
+ # callers get a fresh launch.
+ #
+ # Identity-guarded pop: a concurrent cleanup may have
+ # cancelled+removed our future and a replacement owner may
+ # have registered a new one for the same key. Only remove
+ # the entry if it is still ours — otherwise leave the
+ # replacement alone.
+ if self._loop_iteration_inflight.get(key) is inflight_future:
+ self._loop_iteration_inflight.pop(key, None)
+ if not inflight_future.done():
+ # Normalize owner-task cancellation to a RuntimeError for
+ # shielded waiters. A raw CancelledError on the shared
+ # future would bypass `isinstance(result, Exception)` in
+ # _execute_loop_parallel (CancelledError is a BaseException)
+ # and crash the batch success-unpack. The owner task
+ # itself still re-raises the original CancelledError via
+ # the `raise` below to let its own cancellation propagate.
+ if isinstance(exc, asyncio.CancelledError):
+ inflight_future.set_exception(RuntimeError(f"loop iteration {key} launch cancelled"))
+ else:
+ inflight_future.set_exception(exc)
+ # Teardown below runs AFTER the future is already settled,
+ # so the await does not affect waiters or reopen a race on
+ # the in-flight entry. Best effort — a failing close should
+ # not mask the original exception.
+ if owner_state is not None:
+ try:
+ await asyncio.shield(owner_state.close())
+ except BaseException:
+ LOG.warning(
+ "Failed to close half-initialized loop iteration browser state",
+ workflow_run_id=workflow_run_id,
+ loop_idx=loop_idx,
+ exc_info=True,
+ )
+ raise
+
+ assert owner_state is not None
+ browser_state = owner_state
+ # Success path is synchronous from here on — no awaits between
+ # page init completing and the future being resolved, so
+ # cancellation of this owner task cannot orphan the in-flight
+ # entry.
+ #
+ # cleanup_loop_iterations and close() can run concurrently with
+ # the launch (launch is no longer under the lock). Both teardown
+ # paths pop our entry from _loop_iteration_inflight and then set
+ # a RuntimeError on the future, so our identity check below
+ # detects the race via `stored_future is not inflight_future`
+ # alone. No `inflight_future.cancelled()` check is needed
+ # because neither teardown path calls `.cancel()` — both use
+ # set_exception so shielded waiters receive a regular Exception
+ # that _execute_single_iteration_parallel handles normally
+ # instead of a BaseException-derived CancelledError that would
+ # bypass the batch-result dispatch.
+ stored_future = self._loop_iteration_inflight.get(key)
+ if stored_future is inflight_future:
+ self._loop_iteration_inflight.pop(key, None)
+ if stored_future is not inflight_future:
LOG.warning(
- "Persistent browser session not used for parallel loop iteration — "
- "creating isolated context to avoid cross-iteration races",
+ "Loop iteration cleanup ran during launch, discarding browser state",
workflow_run_id=workflow_run_id,
loop_idx=loop_idx,
- browser_session_id=browser_session_id,
+ key=key,
)
+ try:
+ await asyncio.shield(browser_state.close())
+ except BaseException:
+ LOG.warning(
+ "Failed to close loop iteration browser state after concurrent cleanup",
+ workflow_run_id=workflow_run_id,
+ loop_idx=loop_idx,
+ exc_info=True,
+ )
+ # Use a plain Exception subclass rather than CancelledError:
+ # the parallel for-loop driver routes gather() results via
+ # isinstance(result, Exception), and BaseException-derived
+ # CancelledError would bypass that check and crash the
+ # success-unpack path with a TypeError.
+ raise RuntimeError(f"loop iteration {key} was cleaned up during launch")
+ self.pages[key] = browser_state
+ if not inflight_future.done():
+ inflight_future.set_result(browser_state)
LOG.info(
- "Creating isolated browser state for loop iteration",
+ "Loop iteration browser launch completed",
workflow_run_id=workflow_run_id,
loop_idx=loop_idx,
key=key,
+ elapsed_s=round(time.perf_counter() - launch_start, 3),
)
- browser_state = await self._create_browser_state(
- workflow_run_id=workflow_run_id,
- organization_id=organization_id,
- )
- self.pages[key] = browser_state
+ return browser_state
- # Page creation can happen outside the lock — the key is already
- # reserved in self.pages so no other coroutine will race on it.
+ # Waiters/cache-hit path: the owner has already completed the
+ # initial get_or_create_page before publishing the future, so this
+ # call will cache-hit and not race with any concurrent setup.
await browser_state.get_or_create_page(
workflow_run_id=workflow_run_id,
organization_id=organization_id,
@@ -646,10 +843,24 @@ class RealBrowserManager(BrowserManager):
"""
# Collect entries to close under the lock, then close outside it
# to avoid holding the lock during potentially slow browser teardown.
+ # Also fail any in-flight launches for these keys so a launch still
+ # running in its unlocked section discards its new browser state
+ # instead of publishing it after cleanup has already run.
to_close: list[tuple[int, BrowserState]] = []
async with self._loop_iteration_lock:
for loop_idx in loop_indices:
key = loop_iteration_key(workflow_run_id, loop_idx)
+ inflight_future = self._loop_iteration_inflight.pop(key, None)
+ if inflight_future is not None and not inflight_future.done():
+ # set_exception (not cancel()) so shielded same-key
+ # waiters receive a RuntimeError that
+ # _execute_single_iteration_parallel can catch via its
+ # `except Exception` handler, instead of a CancelledError
+ # that would bypass _execute_loop_parallel's
+ # `isinstance(result, Exception)` dispatch and crash the
+ # batch success-unpack path (CancelledError is a
+ # BaseException, not an Exception).
+ inflight_future.set_exception(RuntimeError(f"loop iteration {key} was cleaned up during launch"))
browser_state = self.pages.pop(key, None)
if browser_state is None:
continue
@@ -658,13 +869,34 @@ class RealBrowserManager(BrowserManager):
if not shared:
to_close.append((loop_idx, browser_state))
- for loop_idx, browser_state in to_close:
- try:
- await browser_state.close()
- except Exception:
+ if not to_close:
+ return
+
+ # Close all entries in parallel — Playwright context teardown is
+ # IO-bound and would otherwise dominate the batch gap.
+ close_start = time.perf_counter()
+ results = await asyncio.gather(
+ *(browser_state.close() for _, browser_state in to_close),
+ return_exceptions=True,
+ )
+ close_elapsed_s = time.perf_counter() - close_start
+ failure_count = sum(1 for r in results if isinstance(r, BaseException))
+ LOG.info(
+ "cleanup_loop_iterations closed entries",
+ workflow_run_id=workflow_run_id,
+ count=len(to_close),
+ failures=failure_count,
+ elapsed_s=round(close_elapsed_s, 3),
+ )
+ for (loop_idx, _), result in zip(to_close, results):
+ if isinstance(result, BaseException):
LOG.warning(
"Failed to close loop iteration browser state",
workflow_run_id=workflow_run_id,
loop_idx=loop_idx,
- exc_info=True,
+ # return_exceptions=True means there is no active
+ # exception context, so exc_info=True won't pick
+ # anything up. Pass an explicit (type, value, tb)
+ # tuple — structlog's documented signature.
+ exc_info=(type(result), result, result.__traceback__),
)
diff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.py
index 2fd02c117..251878923 100644
--- a/tests/unit/test_real_browser_manager.py
+++ b/tests/unit/test_real_browser_manager.py
@@ -6,10 +6,13 @@ behind `if not browser_session_id:`, causing PBS workflow runs to skip the cache
on every call and re-invoke navigate_to_url() on every step.
"""
+import asyncio
+import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from skyvern.constants import loop_iteration_key
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.webeye.real_browser_manager import RealBrowserManager
@@ -222,3 +225,552 @@ async def test_parallel_iteration_different_iterations_get_different_browsers()
assert result0 is iter0_state
assert result1 is iter1_state
assert result0 is not result1
+
+
+# ---------------------------------------------------------------------------
+# Parallel-loop browser launch must not be serialized on a lock.
+# ---------------------------------------------------------------------------
+
+
+def _make_fake_browser_state() -> MagicMock:
+ state = MagicMock()
+ state.get_or_create_page = AsyncMock(return_value=None)
+ state.close = AsyncMock(return_value=None)
+ return state
+
+
+async def test_loop_iteration_launches_overlap() -> None:
+ """Concurrent get_or_create_for_loop_iteration calls with distinct keys
+ must overlap in wall-clock time. If launches serialize under a lock,
+ N * per_launch is observed; this asserts < 2 * per_launch for 4 calls.
+ """
+ manager = RealBrowserManager()
+ per_launch = 0.3
+ call_count = 0
+
+ async def fake_create(**_kwargs: object) -> MagicMock:
+ nonlocal call_count
+ call_count += 1
+ await asyncio.sleep(per_launch)
+ return _make_fake_browser_state()
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=fake_create):
+ start = time.perf_counter()
+ results = await asyncio.gather(
+ *(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_overlap",
+ loop_idx=i,
+ organization_id="org_test",
+ )
+ for i in range(4)
+ )
+ )
+ elapsed = time.perf_counter() - start
+
+ assert call_count == 4, f"expected 4 launches, got {call_count}"
+ assert len(manager.pages) == 4
+ # Serialized would be ~1.2s; parallel ~0.3s. 3x gives jitter slack for
+ # slow CI runners (GitHub Actions can stall 500ms+ on GC). call_count
+ # already proves correctness; this bound is a secondary signal.
+ assert elapsed < per_launch * 3, (
+ f"concurrent launches did not overlap: elapsed={elapsed:.3f}s "
+ f"(per_launch={per_launch}s, expected < {per_launch * 3}s)"
+ )
+ # Every returned state must be a distinct object.
+ assert len({id(r) for r in results}) == 4
+
+
+async def test_loop_iteration_single_flight_same_key() -> None:
+ """Concurrent callers for the same key must collapse to a single
+ underlying browser launch (single-flight)."""
+ manager = RealBrowserManager()
+ call_count = 0
+ created_state = _make_fake_browser_state()
+
+ async def fake_create(**_kwargs: object) -> MagicMock:
+ nonlocal call_count
+ call_count += 1
+ await asyncio.sleep(0.1)
+ return created_state
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=fake_create):
+ results = await asyncio.gather(
+ *(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_single_flight",
+ loop_idx=7,
+ organization_id="org_test",
+ )
+ for _ in range(4)
+ )
+ )
+
+ assert call_count == 1
+ assert len(manager.pages) == 1
+ assert all(r is created_state for r in results)
+
+
+async def test_loop_iteration_owner_cancellation_does_not_orphan_inflight() -> None:
+ """If the launch-owning task is cancelled mid-launch, subsequent callers
+ for the same key must not await a stale in-flight future forever — the
+ in-flight entry must be cleared so the next caller starts a fresh
+ launch."""
+ manager = RealBrowserManager()
+ launch_started = asyncio.Event()
+ call_count = 0
+
+ async def hanging_create(**_kwargs: object) -> MagicMock:
+ nonlocal call_count
+ call_count += 1
+ launch_started.set()
+ # Hang until cancelled.
+ await asyncio.sleep(10)
+ return _make_fake_browser_state() # pragma: no cover
+
+ async def fast_create(**_kwargs: object) -> MagicMock:
+ nonlocal call_count
+ call_count += 1
+ return _make_fake_browser_state()
+
+ # First call: owner, gets cancelled while _create_browser_state hangs.
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=hanging_create):
+ owner_task = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_cancel",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await launch_started.wait()
+ owner_task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await owner_task
+
+ # In-flight entry must be cleared so a retry is not stuck waiting on it.
+ assert "wr_cancel__iter_0" not in manager._loop_iteration_inflight
+ assert "wr_cancel__iter_0" not in manager.pages
+
+ # Retry succeeds with a fresh launch.
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=fast_create):
+ result = await manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_cancel",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+
+ assert result is not None
+ assert call_count == 2
+ assert "wr_cancel__iter_0" in manager.pages
+
+
+async def test_loop_iteration_waiter_cancellation_does_not_cancel_owner() -> None:
+ """Cancelling one waiter on a single-flight key must not cancel the
+ shared launch future — otherwise the owner and every other waiter
+ would fan out into CancelledError."""
+ manager = RealBrowserManager()
+ launch_started = asyncio.Event()
+ call_count = 0
+ created_state = _make_fake_browser_state()
+
+ async def slow_create(**_kwargs: object) -> MagicMock:
+ nonlocal call_count
+ call_count += 1
+ launch_started.set()
+ await asyncio.sleep(0.2)
+ return created_state
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=slow_create):
+ owner = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_waiter_cancel",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await launch_started.wait()
+
+ waiter_to_cancel = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_waiter_cancel",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ surviving_waiter = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_waiter_cancel",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ # Let the waiters reach their shielded await.
+ await asyncio.sleep(0)
+ waiter_to_cancel.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await waiter_to_cancel
+
+ owner_result = await owner
+ surviving_result = await surviving_waiter
+
+ assert call_count == 1
+ assert owner_result is created_state
+ assert surviving_result is created_state
+
+
+async def test_cleanup_during_launch_discards_orphan_state() -> None:
+ """If cleanup_loop_iterations runs while a launch is still in-flight
+ for the same key, the owner must close the browser state it created
+ instead of publishing it into self.pages — otherwise an orphaned
+ Playwright context is leaked."""
+ manager = RealBrowserManager()
+ launch_started = asyncio.Event()
+ cleanup_done = asyncio.Event()
+ created_state = _make_fake_browser_state()
+
+ async def gated_create(**_kwargs: object) -> MagicMock:
+ launch_started.set()
+ await cleanup_done.wait()
+ return created_state
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):
+ owner = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_race",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await launch_started.wait()
+ # Cleanup runs mid-launch.
+ await manager.cleanup_loop_iterations(
+ workflow_run_id="wr_race",
+ loop_indices=[0],
+ organization_id="org_test",
+ )
+ cleanup_done.set()
+ with pytest.raises(RuntimeError, match="cleaned up during launch"):
+ await owner
+
+ assert "wr_race__iter_0" not in manager.pages
+ assert "wr_race__iter_0" not in manager._loop_iteration_inflight
+ created_state.close.assert_awaited()
+
+
+async def test_cleanup_during_launch_does_not_disturb_replacement_owner() -> None:
+ """After cleanup cancels owner A's in-flight future, a replacement
+ owner B may register a new future for the same key before owner A
+ finishes unwinding. Owner A must not pop B's future (Codex P1)."""
+ manager = RealBrowserManager()
+ owner_a_started = asyncio.Event()
+ owner_a_can_finish = asyncio.Event()
+ states: list[MagicMock] = []
+
+ async def gated_create(**_kwargs: object) -> MagicMock:
+ state = _make_fake_browser_state()
+ states.append(state)
+ if len(states) == 1:
+ owner_a_started.set()
+ await owner_a_can_finish.wait()
+ return state
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):
+ owner_a = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_replace",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await owner_a_started.wait()
+
+ # Cleanup cancels owner A's in-flight future.
+ await manager.cleanup_loop_iterations(
+ workflow_run_id="wr_replace",
+ loop_indices=[0],
+ organization_id="org_test",
+ )
+
+ # Owner B registers a brand-new future for the same key.
+ owner_b = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_replace",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ # Let owner B reach its await on _create_browser_state (second call).
+ await asyncio.sleep(0)
+
+ # Now let owner A finish unwinding. It must not disturb owner B's
+ # inflight entry.
+ owner_a_can_finish.set()
+ with pytest.raises(RuntimeError, match="cleaned up during launch"):
+ await owner_a
+
+ result_b = await owner_b
+
+ assert result_b is states[1]
+ assert manager.pages["wr_replace__iter_0"] is states[1]
+ assert "wr_replace__iter_0" not in manager._loop_iteration_inflight
+ # Owner A's orphaned state was closed; owner B's state is intact.
+ states[0].close.assert_awaited()
+ states[1].close.assert_not_awaited()
+
+
+async def test_owner_failure_does_not_disturb_replacement_inflight() -> None:
+ """Owner A's failure path must identity-check before popping inflight.
+ If cleanup cancelled A and owner B registered a replacement future,
+ A's failure must not disturb B's entry (Codex P1)."""
+ manager = RealBrowserManager()
+ owner_a_started = asyncio.Event()
+ owner_a_can_finish = asyncio.Event()
+ states: list[MagicMock] = []
+
+ class LaunchFailure(RuntimeError):
+ pass
+
+ async def gated_create(**_kwargs: object) -> MagicMock:
+ if not states:
+ states.append(object()) # sentinel for "A ran"
+ owner_a_started.set()
+ await owner_a_can_finish.wait()
+ raise LaunchFailure("A failed")
+ state = _make_fake_browser_state()
+ states.append(state)
+ return state
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):
+ owner_a = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_fail_race",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await owner_a_started.wait()
+
+ await manager.cleanup_loop_iterations(
+ workflow_run_id="wr_fail_race",
+ loop_indices=[0],
+ organization_id="org_test",
+ )
+
+ owner_b = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_fail_race",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await asyncio.sleep(0)
+
+ owner_a_can_finish.set()
+ # Owner A raises — it was cancelled by cleanup, so CancelledError
+ # propagates first (cancellation takes precedence over the
+ # subsequent LaunchFailure); either way it must not touch B.
+ with pytest.raises(BaseException):
+ await owner_a
+
+ result_b = await owner_b
+
+ assert result_b is states[1]
+ assert manager.pages["wr_fail_race__iter_0"] is states[1]
+ assert "wr_fail_race__iter_0" not in manager._loop_iteration_inflight
+ states[1].close.assert_not_awaited()
+
+
+async def test_close_during_launch_does_not_orphan_state() -> None:
+ """RealBrowserManager.close() while an owner is mid-launch must
+ cancel the owner's in-flight future and cause the owner to close
+ its own newly-created state instead of publishing it into
+ self.pages after close() has already reset the dict."""
+ manager = RealBrowserManager()
+ launch_started = asyncio.Event()
+ close_done = asyncio.Event()
+ created_state = _make_fake_browser_state()
+
+ async def gated_create(**_kwargs: object) -> MagicMock:
+ launch_started.set()
+ await close_done.wait()
+ return created_state
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):
+ owner = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_close_race",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await launch_started.wait()
+
+ await manager.close()
+ close_done.set()
+ with pytest.raises(RuntimeError, match="cleaned up during launch"):
+ await owner
+
+ assert "wr_close_race__iter_0" not in manager.pages
+ assert "wr_close_race__iter_0" not in manager._loop_iteration_inflight
+ created_state.close.assert_awaited()
+
+
+async def test_cleanup_during_launch_surfaces_exception_to_waiters() -> None:
+ """Codex P2 regression: cleanup_loop_iterations must surface a regular
+ Exception (not CancelledError) to same-key waiters. CancelledError is a
+ BaseException and would bypass `_execute_single_iteration_parallel`'s
+ `except Exception` handler and `_execute_loop_parallel`'s
+ `isinstance(result, Exception)` dispatch, crashing the batch unpack."""
+ manager = RealBrowserManager()
+ owner_started = asyncio.Event()
+ cleanup_done = asyncio.Event()
+ created_state = _make_fake_browser_state()
+
+ async def gated_create(**_kwargs: object) -> MagicMock:
+ owner_started.set()
+ await cleanup_done.wait()
+ return created_state
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=gated_create):
+ owner = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_waiter_exc",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await owner_started.wait()
+
+ # Attach waiters AFTER the in-flight future is registered but
+ # BEFORE cleanup runs, so they have a shielded reference that
+ # cleanup's set_exception must reach.
+ waiters = [
+ asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_waiter_exc",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ for _ in range(2)
+ ]
+ # Yield so waiters reach `await asyncio.shield(inflight_future)`.
+ await asyncio.sleep(0)
+
+ await manager.cleanup_loop_iterations(
+ workflow_run_id="wr_waiter_exc",
+ loop_indices=[0],
+ organization_id="org_test",
+ )
+ cleanup_done.set()
+
+ for waiter in waiters:
+ with pytest.raises(Exception) as excinfo:
+ await waiter
+ # Critically, not a CancelledError — that would bypass the
+ # batch handler and crash the success-unpack path.
+ assert not isinstance(excinfo.value, asyncio.CancelledError), (
+ "waiter must receive a regular Exception, not CancelledError"
+ )
+ assert "cleaned up during launch" in str(excinfo.value)
+
+ with pytest.raises(RuntimeError, match="cleaned up during launch"):
+ await owner
+
+ assert "wr_waiter_exc__iter_0" not in manager.pages
+ assert "wr_waiter_exc__iter_0" not in manager._loop_iteration_inflight
+
+
+async def test_owner_cancellation_surfaces_exception_to_waiters() -> None:
+ """If the launch-owning task is cancelled while same-key waiters are
+ attached, waiters must receive a regular Exception (not CancelledError)
+ so the batch driver's `isinstance(result, Exception)` dispatch catches
+ them instead of the success-unpack path crashing on a BaseException."""
+ manager = RealBrowserManager()
+ owner_started = asyncio.Event()
+
+ async def hanging_create(**_kwargs: object) -> MagicMock:
+ owner_started.set()
+ await asyncio.sleep(10)
+ return _make_fake_browser_state() # pragma: no cover
+
+ with patch.object(RealBrowserManager, "_create_browser_state", side_effect=hanging_create):
+ owner = asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_owner_cancel_waiters",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ await owner_started.wait()
+
+ waiters = [
+ asyncio.create_task(
+ manager.get_or_create_for_loop_iteration(
+ workflow_run_id="wr_owner_cancel_waiters",
+ loop_idx=0,
+ organization_id="org_test",
+ )
+ )
+ for _ in range(2)
+ ]
+ # Let waiters reach their shielded await.
+ await asyncio.sleep(0)
+
+ owner.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await owner
+
+ for waiter in waiters:
+ with pytest.raises(Exception) as excinfo:
+ await waiter
+ assert not isinstance(excinfo.value, asyncio.CancelledError), (
+ "waiter must receive a regular Exception when owner is cancelled"
+ )
+ assert "launch cancelled" in str(excinfo.value)
+
+ assert "wr_owner_cancel_waiters__iter_0" not in manager._loop_iteration_inflight
+ assert "wr_owner_cancel_waiters__iter_0" not in manager.pages
+
+
+async def test_cleanup_loop_iterations_runs_close_in_parallel() -> None:
+ """cleanup_loop_iterations must close browser contexts in parallel.
+ Sequential close of 4 entries at 0.2s each would take ~0.8s; parallel
+ should finish in ~0.2s. 0.4s ceiling gives CI slack."""
+ manager = RealBrowserManager()
+ per_close = 0.2
+
+ async def slow_close() -> None:
+ await asyncio.sleep(per_close)
+
+ loop_indices = [0, 1, 2, 3]
+ for idx in loop_indices:
+ state = _make_fake_browser_state()
+ state.close = AsyncMock(side_effect=slow_close)
+ manager.pages[loop_iteration_key("wr_cleanup", idx)] = state
+
+ start = time.perf_counter()
+ await manager.cleanup_loop_iterations(
+ workflow_run_id="wr_cleanup",
+ loop_indices=loop_indices,
+ organization_id="org_test",
+ )
+ elapsed = time.perf_counter() - start
+
+ assert len(manager.pages) == 0
+ # 3x for CI jitter slack; len(manager.pages) == 0 is the real correctness check.
+ assert elapsed < per_close * 3, (
+ f"cleanup did not run closes in parallel: elapsed={elapsed:.3f}s "
+ f"(per_close={per_close}s, expected < {per_close * 3}s)"
+ )