Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: qualify shared browser ownership with live run context (#8121)
LawyZhengc2c2633f8da8
diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.pyindex 57b16dea9..671c14c47 100644--- a/skyvern/forge/sdk/routes/sdk.py+++ b/skyvern/forge/sdk/routes/sdk.py@@ -24,6 +24,27 @@ from skyvern.schemas.workflows import BlockType, WorkflowStatusLOG = structlog.get_logger()+# Concurrent run_action calls can share a client-provided workflow_run_id on this process-global+# manager, so the run-context must survive until the LAST sibling finishes. Reference-count it here+# rather than only tearing down self-created runs — a client-provided run's context was otherwise+# never removed, leaving a permanent liveness ghost that would veto a real run's terminal browser+# close. No await separates an inc from its matching dec, so a plain dict is race-free.+_sdk_action_context_refcounts: dict[str, int] = {}+++def _acquire_sdk_action_context(workflow_run_id: str) -> None:+ _sdk_action_context_refcounts[workflow_run_id] = _sdk_action_context_refcounts.get(workflow_run_id, 0) + 1+++def _release_sdk_action_context(workflow_run_id: str) -> bool:+ """Decrement the refcount; return True when this was the last live SDK action for the run."""+ remaining = _sdk_action_context_refcounts.get(workflow_run_id, 0) - 1+ if remaining <= 0:+ _sdk_action_context_refcounts.pop(workflow_run_id, None)+ return True+ _sdk_action_context_refcounts[workflow_run_id] = remaining+ return False+@base_router.post("/sdk/run_action",@@ -121,169 +142,179 @@ async def run_sdk_action(task_id=task.task_id,)- await app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context(- organization,- workflow_run.workflow_run_id,- workflow.title,- workflow.workflow_id,- workflow.workflow_permanent_id,- [],- [],- [],- [],- None,- workflow,- mask_secrets=getattr(workflow, "mask_secrets", False),- )-- context = skyvern_context.ensure_context()- skyvern_context.replace(- SkyvernContext(- request_id=context.request_id,- organization_id=task.organization_id,- task_id=task.task_id,- step_id=step.step_id,- browser_session_id=browser_session_id,- max_screenshot_scrolls=task.max_screenshot_scrolls,- workflow_id=workflow.workflow_id,- workflow_run_id=workflow_run.workflow_run_id,- workflow_run_is_synthetic=created_workflow_run,- )- )- result: Any | None = None+ _acquire_sdk_action_context(workflow_run.workflow_run_id)+ # Nested finally: the OUTERMOST finally releases the ownership refcount with no await before+ # it, so it runs even when initialize_workflow_run_context fails/is cancelled or the+ # upload-drain await below is cancelled. Otherwise a client-provided run id could leak its+ # refcount and leave a permanent liveness ghost that vetoes a real run's terminal browser+ # close.try:- scraped_page = await ScriptSkyvernPage.create_scraped_page(browser_session_id=browser_session_id)- page = await scraped_page._browser_state.must_get_working_page()- page_ai = RealSkyvernPageAi(scraped_page, page)+ await app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context(+ organization,+ workflow_run.workflow_run_id,+ workflow.title,+ workflow.workflow_id,+ workflow.workflow_permanent_id,+ [],+ [],+ [],+ [],+ None,+ workflow,+ mask_secrets=getattr(workflow, "mask_secrets", False),+ )- if action.type == "ai_click":- result = await page_ai.ai_click(- selector=action.selector,- intention=action.intention,- data=action.data,- timeout=action.timeout,- )- elif action.type == "ai_input_text":- result = await page_ai.ai_input_text(- selector=action.selector,- value=action.value,- intention=action.intention,- data=action.data,- totp_identifier=action.totp_identifier,- totp_url=action.totp_url,- timeout=action.timeout,+ context = skyvern_context.ensure_context()+ skyvern_context.replace(+ SkyvernContext(+ request_id=context.request_id,+ organization_id=task.organization_id,+ task_id=task.task_id,+ step_id=step.step_id,+ browser_session_id=browser_session_id,+ max_screenshot_scrolls=task.max_screenshot_scrolls,+ workflow_id=workflow.workflow_id,+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_run_is_synthetic=created_workflow_run,)- elif action.type == "ai_select_option":- result = await page_ai.ai_select_option(- selector=action.selector,- value=action.value,- intention=action.intention,- data=action.data,- timeout=action.timeout,+ )+ result: Any | None = None+ try:+ scraped_page = await ScriptSkyvernPage.create_scraped_page(browser_session_id=browser_session_id)+ page = await scraped_page._browser_state.must_get_working_page()+ page_ai = RealSkyvernPageAi(scraped_page, page)++ if action.type == "ai_click":+ result = await page_ai.ai_click(+ selector=action.selector,+ intention=action.intention,+ data=action.data,+ timeout=action.timeout,+ )+ elif action.type == "ai_input_text":+ result = await page_ai.ai_input_text(+ selector=action.selector,+ value=action.value,+ intention=action.intention,+ data=action.data,+ totp_identifier=action.totp_identifier,+ totp_url=action.totp_url,+ timeout=action.timeout,+ )+ elif action.type == "ai_select_option":+ result = await page_ai.ai_select_option(+ selector=action.selector,+ value=action.value,+ intention=action.intention,+ data=action.data,+ timeout=action.timeout,+ )+ elif action.type == "ai_upload_file":+ if action.file_url and not validate_download_url(action.file_url, organization_id=organization_id):+ raise HTTPException(status_code=400, detail="Unsupported file url")+ result = await page_ai.ai_upload_file(+ selector=action.selector,+ files=action.file_url,+ intention=action.intention,+ data=action.data,+ timeout=action.timeout,+ )+ elif action.type == "ai_act":+ await page_ai.ai_act(+ prompt=action.intention,+ )+ result = None+ elif action.type == "extract":+ extract_result = await page_ai.ai_extract(+ prompt=action.prompt,+ schema=action.extract_schema,+ error_code_mapping=action.error_code_mapping,+ intention=action.intention,+ data=action.data,+ )+ result = extract_result+ elif action.type == "locate_element":+ xpath_result = await page_ai.ai_locate_element(+ prompt=action.prompt,+ )+ result = xpath_result+ elif action.type == "validate":+ validation_result = await page_ai.ai_validate(+ prompt=action.prompt,+ model=action.model,+ )+ result = validation_result+ elif action.type == "prompt":+ prompt_result = await page_ai.ai_prompt(+ prompt=action.prompt,+ schema=action.response_schema,+ model=action.model,+ )+ result = prompt_result+ await app.DATABASE.tasks.update_task(+ task_id=task.task_id,+ organization_id=organization_id,+ status=TaskStatus.completed,)- elif action.type == "ai_upload_file":- if action.file_url and not validate_download_url(action.file_url, organization_id=organization_id):- raise HTTPException(status_code=400, detail="Unsupported file url")- result = await page_ai.ai_upload_file(- selector=action.selector,- files=action.file_url,- intention=action.intention,- data=action.data,- timeout=action.timeout,+ except ScrapingFailed as e:+ await app.DATABASE.tasks.update_task(+ task_id=task.task_id,+ organization_id=organization_id,+ status=TaskStatus.failed,+ failure_reason=str(e.reason) if e.reason else str(e),)- elif action.type == "ai_act":- await page_ai.ai_act(- prompt=action.intention,+ LOG.warning(+ "SDK action failed due to scraping error",+ action_type=action.type,+ error=str(e),)- result = None- elif action.type == "extract":- extract_result = await page_ai.ai_extract(- prompt=action.prompt,- schema=action.extract_schema,- error_code_mapping=action.error_code_mapping,- intention=action.intention,- data=action.data,+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.reason or str(e))+ except SkyvernActionFailed as e:+ await app.DATABASE.tasks.update_task(+ task_id=task.task_id,+ organization_id=organization_id,+ status=TaskStatus.failed,+ failure_reason=e.reason,)- result = extract_result- elif action.type == "locate_element":- xpath_result = await page_ai.ai_locate_element(- prompt=action.prompt,+ LOG.warning(+ "SDK action failed",+ action_type=action.type,+ error=e.reason,)- result = xpath_result- elif action.type == "validate":- validation_result = await page_ai.ai_validate(- prompt=action.prompt,- model=action.model,+ raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=e.reason)+ except Exception as e:+ await app.DATABASE.tasks.update_task(+ task_id=task.task_id,+ organization_id=organization_id,+ status=TaskStatus.failed,+ failure_reason=str(e),)- result = validation_result- elif action.type == "prompt":- prompt_result = await page_ai.ai_prompt(- prompt=action.prompt,- schema=action.response_schema,- model=action.model,+ LOG.error(+ "SDK action failed",+ action_type=action.type,+ error=str(e),+ exc_info=True,)- result = prompt_result- await app.DATABASE.tasks.update_task(- task_id=task.task_id,- organization_id=organization_id,- status=TaskStatus.completed,- )- except ScrapingFailed as e:- await app.DATABASE.tasks.update_task(- task_id=task.task_id,- organization_id=organization_id,- status=TaskStatus.failed,- failure_reason=str(e.reason) if e.reason else str(e),- )- LOG.warning(- "SDK action failed due to scraping error",- action_type=action.type,- error=str(e),- )- raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.reason or str(e))- except SkyvernActionFailed as e:- await app.DATABASE.tasks.update_task(- task_id=task.task_id,- organization_id=organization_id,- status=TaskStatus.failed,- failure_reason=e.reason,- )- LOG.warning(- "SDK action failed",- action_type=action.type,- error=e.reason,- )- raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=e.reason)- except Exception as e:- await app.DATABASE.tasks.update_task(- task_id=task.task_id,- organization_id=organization_id,- status=TaskStatus.failed,- failure_reason=str(e),- )- LOG.error(- "SDK action failed",- action_type=action.type,- error=str(e),- exc_info=True,+ raise+ finally:+ # This route runs the whole mini-task in-process, so nothing in the worker lifecycle+ # ever drains its artifact uploads / step archives (SKY-12524). Best-effort: this drain+ # await may itself be cancelled — the run-context release lives in the OUTER finally so it+ # still runs in that case.+ try:+ await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([task.task_id])+ except Exception:+ LOG.warning("Failed to drain artifact uploads for SDK action", task_id=task.task_id, exc_info=True)+ skyvern_context.reset()++ return RunSdkActionResponse(+ workflow_run_id=workflow_run.workflow_run_id,+ result=result,)- raisefinally:- # This route runs the whole mini-task in-process, so nothing in the worker- # lifecycle ever drains its artifact uploads / step archives or removes its- # workflow-run context — without this, every call leaks them (SKY-12524).- try:- await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([task.task_id])- except Exception:- LOG.warning("Failed to drain artifact uploads for SDK action", task_id=task.task_id, exc_info=True)- # Only tear down a run we created. A client-provided workflow_run_id is shared across- # concurrent run_action calls on this process-global manager, so removing it here yanks the- # context out from under an in-flight sibling call.- if created_workflow_run:+ # Remove the run-context only once the last concurrent run_action for this run finishes.+ # A client-provided workflow_run_id is shared across sibling calls, so an unconditional+ # removal would yank the context out from under an in-flight sibling; never removing it+ # left a permanent liveness ghost. The refcount resolves both.+ if _release_sdk_action_context(workflow_run.workflow_run_id):app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context(workflow_run.workflow_run_id)- skyvern_context.reset()-- return RunSdkActionResponse(- workflow_run_id=workflow_run.workflow_run_id,- result=result,- )diff --git a/skyvern/forge/sdk/workflow/context_manager.py b/skyvern/forge/sdk/workflow/context_manager.pyindex cd12841fc..59550cdbe 100644--- a/skyvern/forge/sdk/workflow/context_manager.py+++ b/skyvern/forge/sdk/workflow/context_manager.py@@ -1813,6 +1813,13 @@ class WorkflowContextManager:def remove_workflow_run_context(self, workflow_run_id: str) -> None:self.workflow_run_contexts.pop(workflow_run_id, None)+ def has_workflow_run_context(self, workflow_run_id: str) -> bool:+ """Whether a run is live in THIS process. Initialized before any browser is acquired and+ removed in clean_up_workflow, so it is a faithful per-process run-liveness signal — used to+ decide whether a non-PBS shared-browser alias may veto a terminal close. This is+ process-local and must never be used to reason about PBS lifetime, which is distributed."""+ return workflow_run_id in self.workflow_run_contexts+def mask_secrets_enabled_for_run(self, workflow_run_id: str | None) -> bool:if workflow_run_id is None:return Falsediff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.pyindex fb785c9d1..359b11ca6 100644--- a/skyvern/webeye/real_browser_manager.py+++ b/skyvern/webeye/real_browser_manager.py@@ -1184,13 +1184,24 @@ class RealBrowserManager(BrowserManager):return browser_state_to_closedef _shared_with_another_workflow_run(self, workflow_run_id: str, browser_state_to_close: BrowserState) -> bool:- # Cross-run sharing is only ever registered under another run's workflow_run_id- # (parent/child inheritance via use_parent_browser_session). Same-run task, script, or- # ghost aliases must not suppress the terminal close — the browser would outlive the run.+ # NON-PBS ONLY. Python-object sharing of an ephemeral BrowserState is process-local, so an+ # alias may veto the terminal close only when it denotes ANOTHER workflow run that is+ # currently live in THIS process and legitimately shares this exact state (parent/child+ # use_parent_browser_session). Three qualifications, all required:+ # 1. another run's key (wr_ prefix, not our own) — same-run task/script aliases never share;+ # 2. object identity — it points at the exact state we are about to close;+ # 3. that run is live here — it still has a workflow-run context. A remote/finished/ghost+ # parent whose key was forward-synced into this process, or a never-cleaned+ # synthetic-run key, has no context and must not veto the close, or the browser+ # would outlive every run in this process.+ # PBS lifetime is distributed (runnable_id + generation CAS across processes) and is governed+ # elsewhere: close is force-gated off (close_browser_on_completion=False) and release goes+ # through the expected-owner CAS. This local-liveness predicate must never decide PBS close.return any(page_id != workflow_run_idand page_id.startswith(_WORKFLOW_RUN_KEY_PREFIX)and browser_state is browser_state_to_close+ and app.WORKFLOW_CONTEXT_MANAGER.has_workflow_run_context(page_id)for page_id, browser_state in self.pages.items())@@ -1325,8 +1336,10 @@ class RealBrowserManager(BrowserManager):continueif task_browser_state is browser_state_to_close and finalization_attempted:continue- # Same shared-state check for task-level entries- shared = any(bs is task_browser_state for bs in self.pages.values())+ # Same liveness-qualified ownership predicate as the run-level close: a distinct+ # task-level state must not be held open by a ghost alias, and it must still yield to a+ # genuinely live cross-run sharer.+ shared = self._shared_with_another_workflow_run(task_id, task_browser_state)effective_close = close_browser_on_completion and not sharedif shared:LOG.info(diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.pyindex cfabe7531..5fdae1ead 100644--- a/tests/unit/force_stub_app.py+++ b/tests/unit/force_stub_app.py@@ -17,6 +17,10 @@ def create_forge_stub_app() -> ForgeApp:fake_app_module.WORKFLOW_CONTEXT_MANAGER = _LazyNamespace()fake_app_module.WORKFLOW_CONTEXT_MANAGER.mask_secrets_enabled_for_run = MagicMock(return_value=False)fake_app_module.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run = MagicMock(return_value=set())+ # Sync liveness predicate — _LazyNamespace would auto-mock it as a truthy (never-awaited) AsyncMock,+ # making every wr_ alias read as a live sharer. Default to "no run is live" so tests must opt a run+ # into liveness explicitly (non-PBS ownership signal).+ fake_app_module.WORKFLOW_CONTEXT_MANAGER.has_workflow_run_context = MagicMock(return_value=False)fake_app_module.WORKFLOW_SERVICE = _LazyNamespace()fake_app_module.BROWSER_MANAGER = _LazyNamespace()# get_for_task is a sync lookup returning None when no browser state is registered; _LazyNamespacediff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.pyindex 30eb4098d..199d983eb 100644--- a/tests/unit/test_real_browser_manager.py+++ b/tests/unit/test_real_browser_manager.py@@ -7,12 +7,15 @@ on every call and re-invoke navigate_to_url() on every step."""import asyncio+from contextlib import contextmanagerfrom types import SimpleNamespace+from typing import Iteratorfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest-from skyvern.exceptions import MissingBrowserStateForBrowserSession+from skyvern.exceptions import BrowserSessionAlreadyOccupiedError, MissingBrowserStateForBrowserSession+from skyvern.forge import app as forge_appfrom skyvern.forge.sdk.artifact.storage.recording_test_helpers import fake_prepared_recordingfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext@@ -1049,11 +1052,13 @@ async def test_cleanup_persists_session_cookies_when_close_deferred_for_streams(monkeypatch.setattr("skyvern.webeye.real_browser_manager.stream_ref_active", lambda wrid: True)monkeypatch.setattr("skyvern.webeye.real_browser_manager.set_deferred_close_params", defer_mock)- result = await manager.cleanup_for_workflow_run(- "wfr_streamed",- task_ids=["tsk_streamed"],- close_browser_on_completion=True,- )+ # The parent alias only suppresses the close when it is a genuinely live run in this process.+ with _live_workflow_run_contexts("wr_parent"):+ result = await manager.cleanup_for_workflow_run(+ "wfr_streamed",+ task_ids=["tsk_streamed"],+ close_browser_on_completion=True,+ )persist_mock.assert_awaited_once_with(browser_state.browser_context, "/tmp/fake_profile")defer_mock.assert_called_once_with(@@ -1953,6 +1958,21 @@ def _close_true_count(state: MagicMock) -> int:return sum(1 for call in state.close.await_args_list if call.kwargs.get("close_browser_on_completion") is True)+@contextmanager+def _live_workflow_run_contexts(*run_ids: str) -> Iterator[None]:+ """Control the process-local run-liveness signal the non-PBS sharing predicate consults: only+ the given run ids read as live, every other wr_ alias is treated as a ghost. Backs the C1 fix —+ a forward-synced parent key whose run is not live here must not veto the terminal close."""+ wcm = forge_app.WORKFLOW_CONTEXT_MANAGER+ live = set(run_ids)+ original = wcm.has_workflow_run_context+ wcm.has_workflow_run_context = lambda workflow_run_id: workflow_run_id in live+ try:+ yield+ finally:+ wcm.has_workflow_run_context = original++@pytest.mark.asyncioasync def test_cleanup_same_run_alias_and_ghost_alias_closes_browser_once() -> None:manager = RealBrowserManager()@@ -1981,20 +2001,79 @@ async def test_cleanup_ghost_only_alias_absent_from_task_list_still_closes() ->@pytest.mark.asyncio-async def test_cleanup_genuine_parent_sharing_does_not_close_early() -> None:+async def test_cleanup_genuine_live_parent_sharing_does_not_close_early() -> None:+ # A use_parent_browser_session parent that is genuinely live in THIS process shares the exact+ # BrowserState; the child's terminal cleanup must not close it out from under the live parent.manager = RealBrowserManager()state = _fake_cleanup_state()manager.pages["wr_child"] = statemanager.pages["tsk_c"] = statemanager.pages["wr_parent"] = state- result = await manager.cleanup_for_workflow_run("wr_child", ["tsk_c"], close_browser_on_completion=True)+ with _live_workflow_run_contexts("wr_child", "wr_parent"):+ result = await manager.cleanup_for_workflow_run("wr_child", ["tsk_c"], close_browser_on_completion=True)assert _close_true_count(state) == 0assert "wr_parent" in manager.pagesassert result.recording_finalized is False+async def test_cleanup_ghost_parent_alias_without_live_context_closes_once() -> None:+ # A ghost parent: a child whose parent ran in another process creates a fresh+ # state and forward-syncs pages[wr_parent] = state. The parent is NOT live in this process, so+ # its ghost alias must not veto the child's terminal close — otherwise the browser leaks.+ manager = RealBrowserManager()+ state = _fake_cleanup_state()+ manager.pages["wr_child"] = state+ manager.pages["tsk_c"] = state+ manager.pages["wr_parent"] = state # forward-synced ghost; parent lives in another process++ with _live_workflow_run_contexts("wr_child"): # only the child is live here+ result = await manager.cleanup_for_workflow_run("wr_child", ["tsk_c"], close_browser_on_completion=True)+ # The ghost parent key is not a live run, so it does not veto the close.+ assert manager._shared_with_another_workflow_run("wr_child", state) is False++ # The browser is closed exactly once instead of leaking behind the ghost parent veto.+ assert _close_true_count(state) == 1+ assert result.recording_finalized is True+++async def test_cleanup_synthetic_ghost_wr_alias_does_not_suppress_close() -> None:+ # A never-cleaned synthetic-run wr_ key (SDK action ghost) aliasing this state has no live+ # context and must not suppress the terminal close.+ manager = RealBrowserManager()+ state = _fake_cleanup_state()+ manager.pages["wr_1"] = state+ manager.pages["wr_synthetic_ghost"] = state++ with _live_workflow_run_contexts("wr_1"):+ result = await manager.cleanup_for_workflow_run("wr_1", [], close_browser_on_completion=True)++ assert _close_true_count(state) == 1+ assert result.recording_finalized is True+++async def test_cleanup_task_state_ghost_alias_still_closes_uniformly() -> None:+ # Task-loop uniformity: a distinct task-level state aliased only by a ghost tsk_ key (not a live+ # wr_ run) must still close exactly once — the task-loop predicate is the same liveness-qualified+ # ownership check as the run-level close.+ manager = RealBrowserManager()+ run_state = _fake_cleanup_state()+ task_state = _fake_cleanup_state()+ manager.pages["wr_1"] = run_state+ manager.pages["tsk_a"] = task_state+ manager.pages["tsk_ghost"] = task_state # ghost alias of the distinct task state++ with _live_workflow_run_contexts("wr_1"):+ await manager.cleanup_for_workflow_run("wr_1", ["tsk_a"], close_browser_on_completion=True)++ assert _close_true_count(run_state) == 1+ assert _close_true_count(task_state) == 1++@pytest.mark.asyncioasync def test_cleanup_separate_task_browser_states_each_close_once() -> None:manager = RealBrowserManager()@@ -2059,3 +2138,186 @@ async def test_cleanup_task_close_error_is_contained() -> None:assert _close_true_count(run_state) == 1assert _close_true_count(ok_state) == 1assert result.browser_state is run_state+++async def test_pbs_cleanup_release_only_even_with_zero_local_contexts(monkeypatch: pytest.MonkeyPatch) -> None:+ # PBS/non-PBS boundary: with NO live workflow context anywhere and a ghost wr_ alias present, a+ # PBS cleanup must still be release-only. The local-liveness predicate can never drive a PBS+ # terminal close — PBS lifetime is distributed, not decided by this process.+ workflow_run_id = "wr_pbs_zero_ctx"+ manager = RealBrowserManager()+ state = MagicMock()+ state.browser_artifacts.traces_dir = None+ state.close = AsyncMock(return_value=False)+ manager.pages[workflow_run_id] = state+ manager.pages["wr_ghost_local"] = state # a wr_ alias; irrelevant under the PBS regime+ manager._persistent_session_leases[workflow_run_id] = _PersistentSessionLease(+ session_id="pbs_zero", organization_id="org_test", runnable_id=workflow_run_id, browser_state=state+ )+ sessions = MagicMock()+ sessions.release_browser_session = AsyncMock(return_value=True)+ monkeypatch.setattr(+ real_browser_manager,+ "app",+ MagicMock(+ PERSISTENT_SESSIONS_MANAGER=sessions,+ WORKFLOW_CONTEXT_MANAGER=MagicMock(has_workflow_run_context=MagicMock(return_value=False)),+ ),+ )++ await manager.cleanup_for_workflow_run(+ workflow_run_id,+ task_ids=[],+ close_browser_on_completion=False,+ browser_session_id="pbs_zero",+ organization_id="org_test",+ )++ state.close.assert_awaited_once_with(close_browser_on_completion=False, release_driver=False)+ sessions.release_browser_session.assert_awaited_once()+++async def test_pbs_cross_process_stale_cleanup_is_release_cas_noop_and_never_closes(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Two processes, one PBS. Ownership has moved to process B (generation gen2, wrapper_b); process A+ # is stale (gen1, wrapper_a). Process A's cleanup must (a) never terminal-close the remote browser+ # and (b) release only through the expected-owner CAS, which rejects its stale identity — so+ # process B's live ownership is untouched (PBS distributed ownership).+ session_id = "pbs_shared"+ wrapper_b = object()+ current_owner = {"runnable_id": "wr_podB", "generation": "gen2", "wrapper": wrapper_b}++ async def release_cas(+ *,+ session_id: str,+ organization_id: str,+ expected_runnable_id: str | None,+ expected_runnable_generation_id: str | None,+ expected_browser_state: object,+ ) -> bool:+ return (+ expected_runnable_id == current_owner["runnable_id"]+ and expected_runnable_generation_id == current_owner["generation"]+ and expected_browser_state is current_owner["wrapper"]+ )++ sessions = MagicMock()+ sessions.release_browser_session = AsyncMock(side_effect=release_cas)+ monkeypatch.setattr(real_browser_manager, "app", MagicMock(PERSISTENT_SESSIONS_MANAGER=sessions))++ mgr_a = RealBrowserManager()+ wrapper_a = MagicMock()+ wrapper_a.browser_artifacts.traces_dir = None+ wrapper_a.close = AsyncMock(return_value=False)+ mgr_a.pages["wr_podA"] = wrapper_a+ mgr_a._persistent_session_leases["wr_podA"] = _PersistentSessionLease(+ session_id=session_id,+ organization_id="org_test",+ runnable_id="wr_podA",+ runnable_generation_id="gen1",+ browser_state=wrapper_a,+ )++ await mgr_a.cleanup_for_workflow_run(+ "wr_podA",+ task_ids=[],+ close_browser_on_completion=False,+ browser_session_id=session_id,+ organization_id="org_test",+ )++ # The stale Pod never terminal-closes the remote browser.+ wrapper_a.close.assert_awaited_once_with(close_browser_on_completion=False, release_driver=False)+ # It releases with its OWN stale expected-owner tuple, which the CAS rejects.+ release_call = sessions.release_browser_session.await_args+ assert release_call.kwargs["expected_runnable_id"] == "wr_podA"+ assert release_call.kwargs["expected_runnable_generation_id"] == "gen1"+ assert release_call.kwargs["expected_browser_state"] is wrapper_a+ assert (+ await release_cas(+ session_id=session_id,+ organization_id="org_test",+ expected_runnable_id="wr_podA",+ expected_runnable_generation_id="gen1",+ expected_browser_state=wrapper_a,+ )+ is False+ )+ # The live owner (Pod B, gen2, wrapper_b) is still accepted — ownership genuinely moved and holds.+ assert (+ await release_cas(+ session_id=session_id,+ organization_id="org_test",+ expected_runnable_id="wr_podB",+ expected_runnable_generation_id="gen2",+ expected_browser_state=wrapper_b,+ )+ is True+ )+++async def test_pbs_second_occupancy_rejected_leaves_no_local_lease() -> None:+ # A second concurrent runnable attempting to occupy a PBS already held by another runnable is+ # rejected by begin_session's occupancy CAS; the manager propagates and leaves no local lease.+ manager = RealBrowserManager()+ task = make_task("tsk_second", workflow_run_id=None)++ with patch("skyvern.webeye.real_browser_manager.app") as mock_app:+ configure_browser_context_acquired_hook(mock_app)+ mock_app.PERSISTENT_SESSIONS_MANAGER.begin_session = AsyncMock(+ side_effect=BrowserSessionAlreadyOccupiedError("pbs_busy", "wr_first_owner")+ )+ with pytest.raises(BrowserSessionAlreadyOccupiedError):+ await manager.get_or_create_for_task(task=task, browser_session_id="pbs_busy")++ assert manager.live_session_runnable_ids() == set()+++async def test_address_only_cleanup_preserves_remote_and_defers_local_driver_with_zero_contexts(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Broad PBS, remote-address form: a caller-provided remote browser has no DB session lease, so the+ # service gate hands cleanup close_browser_on_completion=False. Even with a ghost wr_ alias and NO+ # live workflow context anywhere, the manager must keep the remote browser alive — close(False) —+ # and pass release_driver=None so the state stops only its own per-run local driver via+ # release_driver_on_close. Local liveness must never terminal-close the durable remote browser.+ manager = RealBrowserManager()+ state = _fake_cleanup_state()+ manager.pages["wr_addr"] = state+ manager.pages["wr_ghost_local"] = state # ghost wr_ alias; not a live run+ monkeypatch.setattr(+ real_browser_manager,+ "app",+ MagicMock(WORKFLOW_CONTEXT_MANAGER=MagicMock(has_workflow_run_context=MagicMock(return_value=False))),+ )++ await manager.cleanup_for_workflow_run("wr_addr", [], close_browser_on_completion=False)++ state.close.assert_awaited_once_with(close_browser_on_completion=False, release_driver=None)+ assert _close_true_count(state) == 0+++async def test_address_only_cross_pod_stale_cleanup_never_terminal_closes_durable_browser() -> None:+ # Two Pods hold different local wrappers for the same durable remote browser reached by+ # browser_address (no session id, no lease). Pod A's cleanup must never terminal-close the durable+ # browser and must not touch Pod B's wrapper — the remote browser is owned by neither process.+ mgr_a = RealBrowserManager()+ wrapper_a = _fake_cleanup_state()+ mgr_a.pages["wr_podA"] = wrapper_a++ wrapper_b = _fake_cleanup_state() # Pod B's independent wrapper for the same remote browser++ with _live_workflow_run_contexts(): # nothing is live in Pod A's process+ await mgr_a.cleanup_for_workflow_run("wr_podA", [], close_browser_on_completion=False)++ # The durable remote browser is preserved: close(False), never a terminal close.+ wrapper_a.close.assert_awaited_once_with(close_browser_on_completion=False, release_driver=None)+ assert _close_true_count(wrapper_a) == 0+ # Pod B's wrapper is entirely untouched by Pod A's cleanup.+ wrapper_b.close.assert_not_awaited()diff --git a/tests/unit/test_remote_cdp_driver_release.py b/tests/unit/test_remote_cdp_driver_release.pyindex dcac5d547..b616f88a8 100644--- a/tests/unit/test_remote_cdp_driver_release.py+++ b/tests/unit/test_remote_cdp_driver_release.py@@ -14,11 +14,14 @@ from __future__ import annotationsimport asyncioimport time+from contextlib import contextmanager+from typing import Iteratorfrom unittest.mock import AsyncMock, MagicMock, callimport pytestfrom skyvern.exceptions import MissingBrowserStateForBrowserSession, MissingOrganizationForBrowserSession+from skyvern.forge import app as forge_appfrom skyvern.webeye import browser_enginefrom skyvern.webeye.browser_artifacts import BrowserArtifactsfrom skyvern.webeye.browser_engine import (@@ -198,10 +201,10 @@ async def test_close_true_still_stops_driver_and_context() -> None:@pytest.mark.asyncio-async def test_close_true_cancellation_resistant_interceptor_disable_still_runs_provider_cleanup(+async def test_close_true_cancellation_resistant_interceptor_disable_still_runs_remote_cleanup(monkeypatch: pytest.MonkeyPatch,) -> None:- # The paid-provider cleanup must still be attempted exactly once even when disabling the+ # The remote browser cleanup must still be attempted exactly once even when disabling the# download interceptor is cancellation-resistant (ignores the cancel it receives when its# budget is exhausted). close() must return within an outer watchdog, and the stuck drain# must stay owned (its eventual exception retrieved) rather than orphaned.@@ -237,7 +240,7 @@ async def test_close_true_cancellation_resistant_interceptor_disable_still_runs_elapsed = time.monotonic() - startassert elapsed < 1.0- cleanup.assert_awaited_once() # provider cleanup ran despite the stuck drain+ cleanup.assert_awaited_once() # remote cleanup ran despite the stuck draincontext.close.assert_awaited_once()pw.stop.assert_awaited_once()@@ -302,10 +305,10 @@ async def test_close_true_detached_teardown_task_is_strongly_held_until_completi@pytest.mark.asyncio-async def test_close_true_context_close_hang_does_not_suppress_provider_cleanup(+async def test_close_true_context_close_hang_does_not_suppress_remote_cleanup(monkeypatch: pytest.MonkeyPatch,) -> None:- # A hung context close must not starve the paid-provider cleanup: the context-teardown phase is+ # A hung context close must not starve the remote browser cleanup: the context-teardown phase is# bounded independently, so cleanup still runs exactly once after the teardown budget elapses.monkeypatch.setattr("skyvern.webeye.real_browser_state.BROWSER_CLOSE_TIMEOUT", 0.05)pw = _pw_stub()@@ -327,7 +330,7 @@ async def test_close_true_context_close_hang_does_not_suppress_provider_cleanup(@pytest.mark.asyncio-async def test_close_true_persist_cookies_failure_does_not_suppress_provider_cleanup(+async def test_close_true_persist_cookies_failure_does_not_suppress_remote_cleanup(monkeypatch: pytest.MonkeyPatch,) -> None:# A raising cookie-persistence step (an "earlier teardown callback raises") must not suppress cleanup.@@ -353,7 +356,7 @@ async def test_close_true_persist_cookies_failure_does_not_suppress_provider_cle@pytest.mark.asyncioasync def test_close_true_browser_cleanup_exception_is_handled_not_leaked() -> None:- # A raising provider cleanup must be caught (best-effort teardown) and must not leak or block driver stop.+ # A raising remote cleanup must be caught (best-effort teardown) and must not leak or block driver stop.pw = _pw_stub()context = _context_stub()context.cookies = AsyncMock(return_value=[])@@ -408,8 +411,8 @@ async def test_close_false_preserves_reuse_and_does_not_invoke_cleanup() -> None@pytest.mark.asyncio-async def test_close_true_runs_provider_cleanup_exactly_once_across_reentry() -> None:- # Re-entrant close() must not stop/delete the paid provider a second time (avoid double-cleanup).+async def test_close_true_runs_remote_cleanup_exactly_once_across_reentry() -> None:+ # Re-entrant close() must not stop/delete the remote browser a second time (avoid double-cleanup).pw = _pw_stub()context = _context_stub()context.cookies = AsyncMock(return_value=[])@@ -553,6 +556,20 @@ def _fake_browser_state() -> MagicMock:return state+@contextmanager+def _live_workflow_run_contexts(*run_ids: str) -> Iterator[None]:+ """Mark the given workflow runs as live in this process so a shared wr_ alias legitimately+ suppresses the terminal close (non-PBS ownership signal)."""+ wcm = forge_app.WORKFLOW_CONTEXT_MANAGER+ live = set(run_ids)+ original = wcm.has_workflow_run_context+ wcm.has_workflow_run_context = lambda workflow_run_id: workflow_run_id in live+ try:+ yield+ finally:+ wcm.has_workflow_run_context = original++@pytest.mark.asyncioasync def test_cleanup_for_task_keeps_driver_for_persistent_session() -> None:manager = RealBrowserManager()@@ -588,11 +605,12 @@ async def test_cleanup_for_workflow_run_keeps_driver_while_shared() -> None:manager.pages["tsk_1"] = statemanager.pages["wr_parent"] = state- await manager.cleanup_for_workflow_run(- "wr_child",- ["tsk_1"],- close_browser_on_completion=True,- )+ with _live_workflow_run_contexts("wr_child", "wr_parent"):+ await manager.cleanup_for_workflow_run(+ "wr_child",+ ["tsk_1"],+ close_browser_on_completion=True,+ )# Both the workflow-run-level close and the task-level close observe the# parent's surviving reference and must not release the shared driver.@@ -608,11 +626,12 @@ async def test_cleanup_for_workflow_run_reports_recording_not_finalized_while_shmanager.pages["wr_child"] = statemanager.pages["wr_parent"] = state- result = await manager.cleanup_for_workflow_run(- "wr_child",- [],- close_browser_on_completion=True,- )+ with _live_workflow_run_contexts("wr_child", "wr_parent"):+ result = await manager.cleanup_for_workflow_run(+ "wr_child",+ [],+ close_browser_on_completion=True,+ )assert result.browser_state is stateassert result.recording_finalized is Falsediff --git a/tests/unit/test_run_sdk_action_failures.py b/tests/unit/test_run_sdk_action_failures.pyindex 7052251b1..3a8f3c062 100644--- a/tests/unit/test_run_sdk_action_failures.py+++ b/tests/unit/test_run_sdk_action_failures.py@@ -1,5 +1,6 @@from __future__ import annotations+import asynciofrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock, patch@@ -7,7 +8,7 @@ import pytestfrom fastapi import HTTPExceptionfrom skyvern.exceptions import ScrapingFailed, SkyvernActionFailed-from skyvern.forge.sdk.routes.sdk import run_sdk_action+from skyvern.forge.sdk.routes.sdk import _sdk_action_context_refcounts, run_sdk_action@pytest.fixture@@ -134,13 +135,13 @@ async def test_handler_propagates_unknown_exception(mock_request: Any, mock_orga@pytest.mark.asyncio-async def test_client_provided_workflow_run_context_is_not_removed_on_cleanup(+async def test_client_provided_context_removed_when_last_sibling_finishes(mock_request: Any, mock_organization: Any, mock_app: Any) -> None:- # A client-provided workflow_run_id is shared across concurrent run_action calls; the- # process-global context manager is keyed by that id, so tearing it down in this request's- # cleanup yanks the context out from under an in-flight sibling call, which then raises- # WorkflowRunContextNotInitialized in _render_template_with_label.+ # A client-provided workflow_run_id is reference-counted. With no concurrent sibling holding it,+ # this lone call is the last one, so its run-context IS removed on cleanup — otherwise it would+ # persist as a permanent liveness ghost that vetoes a real run's terminal browser close.+ # The refcount must return to empty for the run.mock_request.workflow_run_id = "wr_shared"mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=MagicMock(workflow_run_id="wr_shared", workflow_id="w_test")@@ -158,7 +159,40 @@ async def test_client_provided_workflow_run_context_is_not_removed_on_cleanup(with pytest.raises(RuntimeError, match="stop inside try"):await run_sdk_action(mock_request, organization=mock_organization)- mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_not_called()+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_called_once_with("wr_shared")+ assert "wr_shared" not in _sdk_action_context_refcounts+++async def test_client_provided_context_retained_while_sibling_in_flight(+ mock_request: Any, mock_organization: Any, mock_app: Any+) -> None:+ # A concurrent sibling run_action already holds the shared context (refcount pre-seeded). This+ # call must NOT remove it on cleanup — tearing it down would yank the context out from under the+ # in-flight sibling. The refcount drops back to the sibling's hold, not to zero.+ mock_request.workflow_run_id = "wr_shared"+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(+ return_value=MagicMock(workflow_run_id="wr_shared", workflow_id="w_test")+ )+ _sdk_action_context_refcounts["wr_shared"] = 1 # a sibling call is in flight+ try:+ with (+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,+ patch(+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",+ new_callable=AsyncMock,+ side_effect=RuntimeError("stop inside try"),+ ),+ ):+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)+ with pytest.raises(RuntimeError, match="stop inside try"):+ await run_sdk_action(mock_request, organization=mock_organization)++ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_not_called()+ assert _sdk_action_context_refcounts["wr_shared"] == 1+ finally:+ _sdk_action_context_refcounts.pop("wr_shared", None)@pytest.mark.asyncio@@ -239,3 +273,78 @@ async def test_client_provided_workflow_run_is_not_marked_synthetic(replaced_context = mock_ctx.replace.call_args.args[0]assert replaced_context.workflow_run_is_synthetic is False+++async def test_context_refcount_released_when_initialization_fails(+ mock_request: Any, mock_organization: Any, mock_app: Any+) -> None:+ # F1: the refcount acquire sits before initialize_workflow_run_context. If init RAISES, the+ # release must still run (outer finally), or the client-provided run id leaks its refcount and+ # becomes a permanent liveness ghost that vetoes a real run's terminal browser close.+ mock_request.workflow_run_id = "wr_test"+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(+ return_value=MagicMock(workflow_run_id="wr_test", workflow_id="w_test")+ )+ mock_app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context = AsyncMock(side_effect=RuntimeError("init boom"))+ with (+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context"),+ ):+ with pytest.raises(RuntimeError, match="init boom"):+ await run_sdk_action(mock_request, organization=mock_organization)++ assert "wr_test" not in _sdk_action_context_refcounts+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_called_once_with("wr_test")+++async def test_context_refcount_released_when_initialization_is_cancelled(+ mock_request: Any, mock_organization: Any, mock_app: Any+) -> None:+ # F1 (cancellation): a CancelledError during init must not leak the refcount either — the outer+ # finally releases it before the cancellation propagates.+ mock_request.workflow_run_id = "wr_test"+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(+ return_value=MagicMock(workflow_run_id="wr_test", workflow_id="w_test")+ )+ mock_app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context = AsyncMock(side_effect=asyncio.CancelledError())+ with (+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context"),+ ):+ with pytest.raises(asyncio.CancelledError):+ await run_sdk_action(mock_request, organization=mock_organization)++ assert "wr_test" not in _sdk_action_context_refcounts+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_called_once_with("wr_test")+++async def test_context_refcount_released_when_upload_drain_is_cancelled(+ mock_request: Any, mock_organization: Any, mock_app: Any+) -> None:+ # F1 (drain cancellation): the release used to sit AFTER the upload-drain await inside the same+ # finally, so a CancelledError during the drain skipped it — recreating the ghost. With the+ # release in the OUTER finally it still runs when the drain await is cancelled.+ mock_request.workflow_run_id = "wr_test"+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(+ return_value=MagicMock(workflow_run_id="wr_test", workflow_id="w_test")+ )+ mock_app.ARTIFACT_MANAGER.wait_for_upload_aiotasks = AsyncMock(side_effect=asyncio.CancelledError())+ with (+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,+ patch(+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",+ new_callable=AsyncMock,+ side_effect=RuntimeError("stop inside try"),+ ),+ ):+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)+ # The drain's CancelledError (raised in the inner finally) supersedes the body error.+ with pytest.raises(asyncio.CancelledError):+ await run_sdk_action(mock_request, organization=mock_organization)++ assert "wr_test" not in _sdk_action_context_refcounts+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_called_once_with("wr_test")diff --git a/tests/unit/test_workflow_run_context_eviction.py b/tests/unit/test_workflow_run_context_eviction.pyindex 05cc2f3b2..6dc1a8999 100644--- a/tests/unit/test_workflow_run_context_eviction.py+++ b/tests/unit/test_workflow_run_context_eviction.py@@ -33,6 +33,18 @@ def test_remove_workflow_run_context_evicts_and_is_idempotent() -> None:manager.remove_workflow_run_context("wr_never_seen")+def test_has_workflow_run_context_tracks_process_local_liveness() -> None:+ # The non-PBS sharing predicate reads this as its run-liveness signal: True only while the run's+ # context is registered in THIS process, flipping to False once cleanup removes it.+ manager = _context_manager_with(["wr_live"])++ assert manager.has_workflow_run_context("wr_live") is True+ assert manager.has_workflow_run_context("wr_absent") is False++ manager.remove_workflow_run_context("wr_live")+ assert manager.has_workflow_run_context("wr_live") is False++@pytest.mark.asyncioasync def test_clean_up_workflow_evicts_run_and_child_contexts(monkeypatch: pytest.MonkeyPatch) -> None:service = WorkflowService()