Skip to content

Automate browser based workflows with AI

fix: qualify shared browser ownership with live run context (#8121)

LawyZhengc2c2633f8da8
diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.py
index 57b16dea9..671c14c47 100644
--- a/skyvern/forge/sdk/routes/sdk.py
+++ b/skyvern/forge/sdk/routes/sdk.py
@@ -24,6 +24,27 @@ from skyvern.schemas.workflows import BlockType, WorkflowStatus
LOG = structlog.get_logger()
+# Concurrent run_action calls can share a client-provided workflow_run_id on this process-global
+# manager, so the run-context must survive until the LAST sibling finishes. Reference-count it here
+# rather than only tearing down self-created runs — a client-provided run's context was otherwise
+# never removed, leaving a permanent liveness ghost that would veto a real run's terminal browser
+# close. No await separates an inc from its matching dec, so a plain dict is race-free.
+_sdk_action_context_refcounts: dict[str, int] = {}
+
+
+def _acquire_sdk_action_context(workflow_run_id: str) -> None:
+ _sdk_action_context_refcounts[workflow_run_id] = _sdk_action_context_refcounts.get(workflow_run_id, 0) + 1
+
+
+def _release_sdk_action_context(workflow_run_id: str) -> bool:
+ """Decrement the refcount; return True when this was the last live SDK action for the run."""
+ remaining = _sdk_action_context_refcounts.get(workflow_run_id, 0) - 1
+ if remaining <= 0:
+ _sdk_action_context_refcounts.pop(workflow_run_id, None)
+ return True
+ _sdk_action_context_refcounts[workflow_run_id] = remaining
+ return False
+
@base_router.post(
"/sdk/run_action",
@@ -121,169 +142,179 @@ async def run_sdk_action(
task_id=task.task_id,
)
- await app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context(
- organization,
- workflow_run.workflow_run_id,
- workflow.title,
- workflow.workflow_id,
- workflow.workflow_permanent_id,
- [],
- [],
- [],
- [],
- None,
- workflow,
- mask_secrets=getattr(workflow, "mask_secrets", False),
- )
-
- context = skyvern_context.ensure_context()
- skyvern_context.replace(
- SkyvernContext(
- request_id=context.request_id,
- organization_id=task.organization_id,
- task_id=task.task_id,
- step_id=step.step_id,
- browser_session_id=browser_session_id,
- max_screenshot_scrolls=task.max_screenshot_scrolls,
- workflow_id=workflow.workflow_id,
- workflow_run_id=workflow_run.workflow_run_id,
- workflow_run_is_synthetic=created_workflow_run,
- )
- )
- result: Any | None = None
+ _acquire_sdk_action_context(workflow_run.workflow_run_id)
+ # Nested finally: the OUTERMOST finally releases the ownership refcount with no await before
+ # it, so it runs even when initialize_workflow_run_context fails/is cancelled or the
+ # upload-drain await below is cancelled. Otherwise a client-provided run id could leak its
+ # refcount and leave a permanent liveness ghost that vetoes a real run's terminal browser
+ # close.
try:
- scraped_page = await ScriptSkyvernPage.create_scraped_page(browser_session_id=browser_session_id)
- page = await scraped_page._browser_state.must_get_working_page()
- page_ai = RealSkyvernPageAi(scraped_page, page)
+ await app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context(
+ organization,
+ workflow_run.workflow_run_id,
+ workflow.title,
+ workflow.workflow_id,
+ workflow.workflow_permanent_id,
+ [],
+ [],
+ [],
+ [],
+ None,
+ workflow,
+ mask_secrets=getattr(workflow, "mask_secrets", False),
+ )
- if action.type == "ai_click":
- result = await page_ai.ai_click(
- selector=action.selector,
- intention=action.intention,
- data=action.data,
- timeout=action.timeout,
- )
- elif action.type == "ai_input_text":
- result = await page_ai.ai_input_text(
- selector=action.selector,
- value=action.value,
- intention=action.intention,
- data=action.data,
- totp_identifier=action.totp_identifier,
- totp_url=action.totp_url,
- timeout=action.timeout,
+ context = skyvern_context.ensure_context()
+ skyvern_context.replace(
+ SkyvernContext(
+ request_id=context.request_id,
+ organization_id=task.organization_id,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ browser_session_id=browser_session_id,
+ max_screenshot_scrolls=task.max_screenshot_scrolls,
+ workflow_id=workflow.workflow_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_run_is_synthetic=created_workflow_run,
)
- elif action.type == "ai_select_option":
- result = await page_ai.ai_select_option(
- selector=action.selector,
- value=action.value,
- intention=action.intention,
- data=action.data,
- timeout=action.timeout,
+ )
+ result: Any | None = None
+ try:
+ scraped_page = await ScriptSkyvernPage.create_scraped_page(browser_session_id=browser_session_id)
+ page = await scraped_page._browser_state.must_get_working_page()
+ page_ai = RealSkyvernPageAi(scraped_page, page)
+
+ if action.type == "ai_click":
+ result = await page_ai.ai_click(
+ selector=action.selector,
+ intention=action.intention,
+ data=action.data,
+ timeout=action.timeout,
+ )
+ elif action.type == "ai_input_text":
+ result = await page_ai.ai_input_text(
+ selector=action.selector,
+ value=action.value,
+ intention=action.intention,
+ data=action.data,
+ totp_identifier=action.totp_identifier,
+ totp_url=action.totp_url,
+ timeout=action.timeout,
+ )
+ elif action.type == "ai_select_option":
+ result = await page_ai.ai_select_option(
+ selector=action.selector,
+ value=action.value,
+ intention=action.intention,
+ data=action.data,
+ timeout=action.timeout,
+ )
+ elif action.type == "ai_upload_file":
+ if action.file_url and not validate_download_url(action.file_url, organization_id=organization_id):
+ raise HTTPException(status_code=400, detail="Unsupported file url")
+ result = await page_ai.ai_upload_file(
+ selector=action.selector,
+ files=action.file_url,
+ intention=action.intention,
+ data=action.data,
+ timeout=action.timeout,
+ )
+ elif action.type == "ai_act":
+ await page_ai.ai_act(
+ prompt=action.intention,
+ )
+ result = None
+ elif action.type == "extract":
+ extract_result = await page_ai.ai_extract(
+ prompt=action.prompt,
+ schema=action.extract_schema,
+ error_code_mapping=action.error_code_mapping,
+ intention=action.intention,
+ data=action.data,
+ )
+ result = extract_result
+ elif action.type == "locate_element":
+ xpath_result = await page_ai.ai_locate_element(
+ prompt=action.prompt,
+ )
+ result = xpath_result
+ elif action.type == "validate":
+ validation_result = await page_ai.ai_validate(
+ prompt=action.prompt,
+ model=action.model,
+ )
+ result = validation_result
+ elif action.type == "prompt":
+ prompt_result = await page_ai.ai_prompt(
+ prompt=action.prompt,
+ schema=action.response_schema,
+ model=action.model,
+ )
+ result = prompt_result
+ await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.completed,
)
- elif action.type == "ai_upload_file":
- if action.file_url and not validate_download_url(action.file_url, organization_id=organization_id):
- raise HTTPException(status_code=400, detail="Unsupported file url")
- result = await page_ai.ai_upload_file(
- selector=action.selector,
- files=action.file_url,
- intention=action.intention,
- data=action.data,
- timeout=action.timeout,
+ except ScrapingFailed as e:
+ await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.failed,
+ failure_reason=str(e.reason) if e.reason else str(e),
)
- elif action.type == "ai_act":
- await page_ai.ai_act(
- prompt=action.intention,
+ LOG.warning(
+ "SDK action failed due to scraping error",
+ action_type=action.type,
+ error=str(e),
)
- result = None
- elif action.type == "extract":
- extract_result = await page_ai.ai_extract(
- prompt=action.prompt,
- schema=action.extract_schema,
- error_code_mapping=action.error_code_mapping,
- intention=action.intention,
- data=action.data,
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.reason or str(e))
+ except SkyvernActionFailed as e:
+ await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.failed,
+ failure_reason=e.reason,
)
- result = extract_result
- elif action.type == "locate_element":
- xpath_result = await page_ai.ai_locate_element(
- prompt=action.prompt,
+ LOG.warning(
+ "SDK action failed",
+ action_type=action.type,
+ error=e.reason,
)
- result = xpath_result
- elif action.type == "validate":
- validation_result = await page_ai.ai_validate(
- prompt=action.prompt,
- model=action.model,
+ raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=e.reason)
+ except Exception as e:
+ await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.failed,
+ failure_reason=str(e),
)
- result = validation_result
- elif action.type == "prompt":
- prompt_result = await page_ai.ai_prompt(
- prompt=action.prompt,
- schema=action.response_schema,
- model=action.model,
+ LOG.error(
+ "SDK action failed",
+ action_type=action.type,
+ error=str(e),
+ exc_info=True,
)
- result = prompt_result
- await app.DATABASE.tasks.update_task(
- task_id=task.task_id,
- organization_id=organization_id,
- status=TaskStatus.completed,
- )
- except ScrapingFailed as e:
- await app.DATABASE.tasks.update_task(
- task_id=task.task_id,
- organization_id=organization_id,
- status=TaskStatus.failed,
- failure_reason=str(e.reason) if e.reason else str(e),
- )
- LOG.warning(
- "SDK action failed due to scraping error",
- action_type=action.type,
- error=str(e),
- )
- raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.reason or str(e))
- except SkyvernActionFailed as e:
- await app.DATABASE.tasks.update_task(
- task_id=task.task_id,
- organization_id=organization_id,
- status=TaskStatus.failed,
- failure_reason=e.reason,
- )
- LOG.warning(
- "SDK action failed",
- action_type=action.type,
- error=e.reason,
- )
- raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=e.reason)
- except Exception as e:
- await app.DATABASE.tasks.update_task(
- task_id=task.task_id,
- organization_id=organization_id,
- status=TaskStatus.failed,
- failure_reason=str(e),
- )
- LOG.error(
- "SDK action failed",
- action_type=action.type,
- error=str(e),
- exc_info=True,
+ raise
+ finally:
+ # This route runs the whole mini-task in-process, so nothing in the worker lifecycle
+ # ever drains its artifact uploads / step archives (SKY-12524). Best-effort: this drain
+ # await may itself be cancelled — the run-context release lives in the OUTER finally so it
+ # still runs in that case.
+ try:
+ await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([task.task_id])
+ except Exception:
+ LOG.warning("Failed to drain artifact uploads for SDK action", task_id=task.task_id, exc_info=True)
+ skyvern_context.reset()
+
+ return RunSdkActionResponse(
+ workflow_run_id=workflow_run.workflow_run_id,
+ result=result,
)
- raise
finally:
- # This route runs the whole mini-task in-process, so nothing in the worker
- # lifecycle ever drains its artifact uploads / step archives or removes its
- # workflow-run context — without this, every call leaks them (SKY-12524).
- try:
- await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([task.task_id])
- except Exception:
- LOG.warning("Failed to drain artifact uploads for SDK action", task_id=task.task_id, exc_info=True)
- # Only tear down a run we created. A client-provided workflow_run_id is shared across
- # concurrent run_action calls on this process-global manager, so removing it here yanks the
- # context out from under an in-flight sibling call.
- if created_workflow_run:
+ # Remove the run-context only once the last concurrent run_action for this run finishes.
+ # A client-provided workflow_run_id is shared across sibling calls, so an unconditional
+ # removal would yank the context out from under an in-flight sibling; never removing it
+ # left a permanent liveness ghost. The refcount resolves both.
+ if _release_sdk_action_context(workflow_run.workflow_run_id):
app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context(workflow_run.workflow_run_id)
- skyvern_context.reset()
-
- return RunSdkActionResponse(
- workflow_run_id=workflow_run.workflow_run_id,
- result=result,
- )
diff --git a/skyvern/forge/sdk/workflow/context_manager.py b/skyvern/forge/sdk/workflow/context_manager.py
index cd12841fc..59550cdbe 100644
--- a/skyvern/forge/sdk/workflow/context_manager.py
+++ b/skyvern/forge/sdk/workflow/context_manager.py
@@ -1813,6 +1813,13 @@ class WorkflowContextManager:
def remove_workflow_run_context(self, workflow_run_id: str) -> None:
self.workflow_run_contexts.pop(workflow_run_id, None)
+ def has_workflow_run_context(self, workflow_run_id: str) -> bool:
+ """Whether a run is live in THIS process. Initialized before any browser is acquired and
+ removed in clean_up_workflow, so it is a faithful per-process run-liveness signal — used to
+ decide whether a non-PBS shared-browser alias may veto a terminal close. This is
+ process-local and must never be used to reason about PBS lifetime, which is distributed."""
+ return workflow_run_id in self.workflow_run_contexts
+
def mask_secrets_enabled_for_run(self, workflow_run_id: str | None) -> bool:
if workflow_run_id is None:
return False
diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.py
index fb785c9d1..359b11ca6 100644
--- a/skyvern/webeye/real_browser_manager.py
+++ b/skyvern/webeye/real_browser_manager.py
@@ -1184,13 +1184,24 @@ class RealBrowserManager(BrowserManager):
return browser_state_to_close
def _shared_with_another_workflow_run(self, workflow_run_id: str, browser_state_to_close: BrowserState) -> bool:
- # Cross-run sharing is only ever registered under another run's workflow_run_id
- # (parent/child inheritance via use_parent_browser_session). Same-run task, script, or
- # ghost aliases must not suppress the terminal close — the browser would outlive the run.
+ # NON-PBS ONLY. Python-object sharing of an ephemeral BrowserState is process-local, so an
+ # alias may veto the terminal close only when it denotes ANOTHER workflow run that is
+ # currently live in THIS process and legitimately shares this exact state (parent/child
+ # use_parent_browser_session). Three qualifications, all required:
+ # 1. another run's key (wr_ prefix, not our own) — same-run task/script aliases never share;
+ # 2. object identity — it points at the exact state we are about to close;
+ # 3. that run is live here — it still has a workflow-run context. A remote/finished/ghost
+ # parent whose key was forward-synced into this process, or a never-cleaned
+ # synthetic-run key, has no context and must not veto the close, or the browser
+ # would outlive every run in this process.
+ # PBS lifetime is distributed (runnable_id + generation CAS across processes) and is governed
+ # elsewhere: close is force-gated off (close_browser_on_completion=False) and release goes
+ # through the expected-owner CAS. This local-liveness predicate must never decide PBS close.
return any(
page_id != workflow_run_id
and page_id.startswith(_WORKFLOW_RUN_KEY_PREFIX)
and browser_state is browser_state_to_close
+ and app.WORKFLOW_CONTEXT_MANAGER.has_workflow_run_context(page_id)
for page_id, browser_state in self.pages.items()
)
@@ -1325,8 +1336,10 @@ class RealBrowserManager(BrowserManager):
continue
if task_browser_state is browser_state_to_close and finalization_attempted:
continue
- # Same shared-state check for task-level entries
- shared = any(bs is task_browser_state for bs in self.pages.values())
+ # Same liveness-qualified ownership predicate as the run-level close: a distinct
+ # task-level state must not be held open by a ghost alias, and it must still yield to a
+ # genuinely live cross-run sharer.
+ shared = self._shared_with_another_workflow_run(task_id, task_browser_state)
effective_close = close_browser_on_completion and not shared
if shared:
LOG.info(
diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.py
index cfabe7531..5fdae1ead 100644
--- a/tests/unit/force_stub_app.py
+++ b/tests/unit/force_stub_app.py
@@ -17,6 +17,10 @@ def create_forge_stub_app() -> ForgeApp:
fake_app_module.WORKFLOW_CONTEXT_MANAGER = _LazyNamespace()
fake_app_module.WORKFLOW_CONTEXT_MANAGER.mask_secrets_enabled_for_run = MagicMock(return_value=False)
fake_app_module.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run = MagicMock(return_value=set())
+ # Sync liveness predicate — _LazyNamespace would auto-mock it as a truthy (never-awaited) AsyncMock,
+ # making every wr_ alias read as a live sharer. Default to "no run is live" so tests must opt a run
+ # into liveness explicitly (non-PBS ownership signal).
+ fake_app_module.WORKFLOW_CONTEXT_MANAGER.has_workflow_run_context = MagicMock(return_value=False)
fake_app_module.WORKFLOW_SERVICE = _LazyNamespace()
fake_app_module.BROWSER_MANAGER = _LazyNamespace()
# get_for_task is a sync lookup returning None when no browser state is registered; _LazyNamespace
diff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.py
index 30eb4098d..199d983eb 100644
--- a/tests/unit/test_real_browser_manager.py
+++ b/tests/unit/test_real_browser_manager.py
@@ -7,12 +7,15 @@ on every call and re-invoke navigate_to_url() on every step.
"""
import asyncio
+from contextlib import contextmanager
from types import SimpleNamespace
+from typing import Iterator
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from skyvern.exceptions import MissingBrowserStateForBrowserSession
+from skyvern.exceptions import BrowserSessionAlreadyOccupiedError, MissingBrowserStateForBrowserSession
+from skyvern.forge import app as forge_app
from skyvern.forge.sdk.artifact.storage.recording_test_helpers import fake_prepared_recording
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
@@ -1049,11 +1052,13 @@ async def test_cleanup_persists_session_cookies_when_close_deferred_for_streams(
monkeypatch.setattr("skyvern.webeye.real_browser_manager.stream_ref_active", lambda wrid: True)
monkeypatch.setattr("skyvern.webeye.real_browser_manager.set_deferred_close_params", defer_mock)
- result = await manager.cleanup_for_workflow_run(
- "wfr_streamed",
- task_ids=["tsk_streamed"],
- close_browser_on_completion=True,
- )
+ # The parent alias only suppresses the close when it is a genuinely live run in this process.
+ with _live_workflow_run_contexts("wr_parent"):
+ result = await manager.cleanup_for_workflow_run(
+ "wfr_streamed",
+ task_ids=["tsk_streamed"],
+ close_browser_on_completion=True,
+ )
persist_mock.assert_awaited_once_with(browser_state.browser_context, "/tmp/fake_profile")
defer_mock.assert_called_once_with(
@@ -1953,6 +1958,21 @@ def _close_true_count(state: MagicMock) -> int:
return sum(1 for call in state.close.await_args_list if call.kwargs.get("close_browser_on_completion") is True)
+@contextmanager
+def _live_workflow_run_contexts(*run_ids: str) -> Iterator[None]:
+ """Control the process-local run-liveness signal the non-PBS sharing predicate consults: only
+ the given run ids read as live, every other wr_ alias is treated as a ghost. Backs the C1 fix —
+ a forward-synced parent key whose run is not live here must not veto the terminal close."""
+ wcm = forge_app.WORKFLOW_CONTEXT_MANAGER
+ live = set(run_ids)
+ original = wcm.has_workflow_run_context
+ wcm.has_workflow_run_context = lambda workflow_run_id: workflow_run_id in live
+ try:
+ yield
+ finally:
+ wcm.has_workflow_run_context = original
+
+
@pytest.mark.asyncio
async def test_cleanup_same_run_alias_and_ghost_alias_closes_browser_once() -> None:
manager = RealBrowserManager()
@@ -1981,20 +2001,79 @@ async def test_cleanup_ghost_only_alias_absent_from_task_list_still_closes() ->
@pytest.mark.asyncio
-async def test_cleanup_genuine_parent_sharing_does_not_close_early() -> None:
+async def test_cleanup_genuine_live_parent_sharing_does_not_close_early() -> None:
+ # A use_parent_browser_session parent that is genuinely live in THIS process shares the exact
+ # BrowserState; the child's terminal cleanup must not close it out from under the live parent.
manager = RealBrowserManager()
state = _fake_cleanup_state()
manager.pages["wr_child"] = state
manager.pages["tsk_c"] = state
manager.pages["wr_parent"] = state
- result = await manager.cleanup_for_workflow_run("wr_child", ["tsk_c"], close_browser_on_completion=True)
+ with _live_workflow_run_contexts("wr_child", "wr_parent"):
+ result = await manager.cleanup_for_workflow_run("wr_child", ["tsk_c"], close_browser_on_completion=True)
assert _close_true_count(state) == 0
assert "wr_parent" in manager.pages
assert result.recording_finalized is False
+async def test_cleanup_ghost_parent_alias_without_live_context_closes_once() -> None:
+ # A ghost parent: a child whose parent ran in another process creates a fresh
+ # state and forward-syncs pages[wr_parent] = state. The parent is NOT live in this process, so
+ # its ghost alias must not veto the child's terminal close — otherwise the browser leaks.
+ manager = RealBrowserManager()
+ state = _fake_cleanup_state()
+ manager.pages["wr_child"] = state
+ manager.pages["tsk_c"] = state
+ manager.pages["wr_parent"] = state # forward-synced ghost; parent lives in another process
+
+ with _live_workflow_run_contexts("wr_child"): # only the child is live here
+ result = await manager.cleanup_for_workflow_run("wr_child", ["tsk_c"], close_browser_on_completion=True)
+ # The ghost parent key is not a live run, so it does not veto the close.
+ assert manager._shared_with_another_workflow_run("wr_child", state) is False
+
+ # The browser is closed exactly once instead of leaking behind the ghost parent veto.
+ assert _close_true_count(state) == 1
+ assert result.recording_finalized is True
+
+
+async def test_cleanup_synthetic_ghost_wr_alias_does_not_suppress_close() -> None:
+ # A never-cleaned synthetic-run wr_ key (SDK action ghost) aliasing this state has no live
+ # context and must not suppress the terminal close.
+ manager = RealBrowserManager()
+ state = _fake_cleanup_state()
+ manager.pages["wr_1"] = state
+ manager.pages["wr_synthetic_ghost"] = state
+
+ with _live_workflow_run_contexts("wr_1"):
+ result = await manager.cleanup_for_workflow_run("wr_1", [], close_browser_on_completion=True)
+
+ assert _close_true_count(state) == 1
+ assert result.recording_finalized is True
+
+
+async def test_cleanup_task_state_ghost_alias_still_closes_uniformly() -> None:
+ # Task-loop uniformity: a distinct task-level state aliased only by a ghost tsk_ key (not a live
+ # wr_ run) must still close exactly once — the task-loop predicate is the same liveness-qualified
+ # ownership check as the run-level close.
+ manager = RealBrowserManager()
+ run_state = _fake_cleanup_state()
+ task_state = _fake_cleanup_state()
+ manager.pages["wr_1"] = run_state
+ manager.pages["tsk_a"] = task_state
+ manager.pages["tsk_ghost"] = task_state # ghost alias of the distinct task state
+
+ with _live_workflow_run_contexts("wr_1"):
+ await manager.cleanup_for_workflow_run("wr_1", ["tsk_a"], close_browser_on_completion=True)
+
+ assert _close_true_count(run_state) == 1
+ assert _close_true_count(task_state) == 1
+
+
@pytest.mark.asyncio
async def test_cleanup_separate_task_browser_states_each_close_once() -> None:
manager = RealBrowserManager()
@@ -2059,3 +2138,186 @@ async def test_cleanup_task_close_error_is_contained() -> None:
assert _close_true_count(run_state) == 1
assert _close_true_count(ok_state) == 1
assert result.browser_state is run_state
+
+
+async def test_pbs_cleanup_release_only_even_with_zero_local_contexts(monkeypatch: pytest.MonkeyPatch) -> None:
+ # PBS/non-PBS boundary: with NO live workflow context anywhere and a ghost wr_ alias present, a
+ # PBS cleanup must still be release-only. The local-liveness predicate can never drive a PBS
+ # terminal close — PBS lifetime is distributed, not decided by this process.
+ workflow_run_id = "wr_pbs_zero_ctx"
+ manager = RealBrowserManager()
+ state = MagicMock()
+ state.browser_artifacts.traces_dir = None
+ state.close = AsyncMock(return_value=False)
+ manager.pages[workflow_run_id] = state
+ manager.pages["wr_ghost_local"] = state # a wr_ alias; irrelevant under the PBS regime
+ manager._persistent_session_leases[workflow_run_id] = _PersistentSessionLease(
+ session_id="pbs_zero", organization_id="org_test", runnable_id=workflow_run_id, browser_state=state
+ )
+ sessions = MagicMock()
+ sessions.release_browser_session = AsyncMock(return_value=True)
+ monkeypatch.setattr(
+ real_browser_manager,
+ "app",
+ MagicMock(
+ PERSISTENT_SESSIONS_MANAGER=sessions,
+ WORKFLOW_CONTEXT_MANAGER=MagicMock(has_workflow_run_context=MagicMock(return_value=False)),
+ ),
+ )
+
+ await manager.cleanup_for_workflow_run(
+ workflow_run_id,
+ task_ids=[],
+ close_browser_on_completion=False,
+ browser_session_id="pbs_zero",
+ organization_id="org_test",
+ )
+
+ state.close.assert_awaited_once_with(close_browser_on_completion=False, release_driver=False)
+ sessions.release_browser_session.assert_awaited_once()
+
+
+async def test_pbs_cross_process_stale_cleanup_is_release_cas_noop_and_never_closes(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Two processes, one PBS. Ownership has moved to process B (generation gen2, wrapper_b); process A
+ # is stale (gen1, wrapper_a). Process A's cleanup must (a) never terminal-close the remote browser
+ # and (b) release only through the expected-owner CAS, which rejects its stale identity — so
+ # process B's live ownership is untouched (PBS distributed ownership).
+ session_id = "pbs_shared"
+ wrapper_b = object()
+ current_owner = {"runnable_id": "wr_podB", "generation": "gen2", "wrapper": wrapper_b}
+
+ async def release_cas(
+ *,
+ session_id: str,
+ organization_id: str,
+ expected_runnable_id: str | None,
+ expected_runnable_generation_id: str | None,
+ expected_browser_state: object,
+ ) -> bool:
+ return (
+ expected_runnable_id == current_owner["runnable_id"]
+ and expected_runnable_generation_id == current_owner["generation"]
+ and expected_browser_state is current_owner["wrapper"]
+ )
+
+ sessions = MagicMock()
+ sessions.release_browser_session = AsyncMock(side_effect=release_cas)
+ monkeypatch.setattr(real_browser_manager, "app", MagicMock(PERSISTENT_SESSIONS_MANAGER=sessions))
+
+ mgr_a = RealBrowserManager()
+ wrapper_a = MagicMock()
+ wrapper_a.browser_artifacts.traces_dir = None
+ wrapper_a.close = AsyncMock(return_value=False)
+ mgr_a.pages["wr_podA"] = wrapper_a
+ mgr_a._persistent_session_leases["wr_podA"] = _PersistentSessionLease(
+ session_id=session_id,
+ organization_id="org_test",
+ runnable_id="wr_podA",
+ runnable_generation_id="gen1",
+ browser_state=wrapper_a,
+ )
+
+ await mgr_a.cleanup_for_workflow_run(
+ "wr_podA",
+ task_ids=[],
+ close_browser_on_completion=False,
+ browser_session_id=session_id,
+ organization_id="org_test",
+ )
+
+ # The stale Pod never terminal-closes the remote browser.
+ wrapper_a.close.assert_awaited_once_with(close_browser_on_completion=False, release_driver=False)
+ # It releases with its OWN stale expected-owner tuple, which the CAS rejects.
+ release_call = sessions.release_browser_session.await_args
+ assert release_call.kwargs["expected_runnable_id"] == "wr_podA"
+ assert release_call.kwargs["expected_runnable_generation_id"] == "gen1"
+ assert release_call.kwargs["expected_browser_state"] is wrapper_a
+ assert (
+ await release_cas(
+ session_id=session_id,
+ organization_id="org_test",
+ expected_runnable_id="wr_podA",
+ expected_runnable_generation_id="gen1",
+ expected_browser_state=wrapper_a,
+ )
+ is False
+ )
+ # The live owner (Pod B, gen2, wrapper_b) is still accepted — ownership genuinely moved and holds.
+ assert (
+ await release_cas(
+ session_id=session_id,
+ organization_id="org_test",
+ expected_runnable_id="wr_podB",
+ expected_runnable_generation_id="gen2",
+ expected_browser_state=wrapper_b,
+ )
+ is True
+ )
+
+
+async def test_pbs_second_occupancy_rejected_leaves_no_local_lease() -> None:
+ # A second concurrent runnable attempting to occupy a PBS already held by another runnable is
+ # rejected by begin_session's occupancy CAS; the manager propagates and leaves no local lease.
+ manager = RealBrowserManager()
+ task = make_task("tsk_second", workflow_run_id=None)
+
+ with patch("skyvern.webeye.real_browser_manager.app") as mock_app:
+ configure_browser_context_acquired_hook(mock_app)
+ mock_app.PERSISTENT_SESSIONS_MANAGER.begin_session = AsyncMock(
+ side_effect=BrowserSessionAlreadyOccupiedError("pbs_busy", "wr_first_owner")
+ )
+ with pytest.raises(BrowserSessionAlreadyOccupiedError):
+ await manager.get_or_create_for_task(task=task, browser_session_id="pbs_busy")
+
+ assert manager.live_session_runnable_ids() == set()
+
+
+async def test_address_only_cleanup_preserves_remote_and_defers_local_driver_with_zero_contexts(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Broad PBS, remote-address form: a caller-provided remote browser has no DB session lease, so the
+ # service gate hands cleanup close_browser_on_completion=False. Even with a ghost wr_ alias and NO
+ # live workflow context anywhere, the manager must keep the remote browser alive — close(False) —
+ # and pass release_driver=None so the state stops only its own per-run local driver via
+ # release_driver_on_close. Local liveness must never terminal-close the durable remote browser.
+ manager = RealBrowserManager()
+ state = _fake_cleanup_state()
+ manager.pages["wr_addr"] = state
+ manager.pages["wr_ghost_local"] = state # ghost wr_ alias; not a live run
+ monkeypatch.setattr(
+ real_browser_manager,
+ "app",
+ MagicMock(WORKFLOW_CONTEXT_MANAGER=MagicMock(has_workflow_run_context=MagicMock(return_value=False))),
+ )
+
+ await manager.cleanup_for_workflow_run("wr_addr", [], close_browser_on_completion=False)
+
+ state.close.assert_awaited_once_with(close_browser_on_completion=False, release_driver=None)
+ assert _close_true_count(state) == 0
+
+
+async def test_address_only_cross_pod_stale_cleanup_never_terminal_closes_durable_browser() -> None:
+ # Two Pods hold different local wrappers for the same durable remote browser reached by
+ # browser_address (no session id, no lease). Pod A's cleanup must never terminal-close the durable
+ # browser and must not touch Pod B's wrapper — the remote browser is owned by neither process.
+ mgr_a = RealBrowserManager()
+ wrapper_a = _fake_cleanup_state()
+ mgr_a.pages["wr_podA"] = wrapper_a
+
+ wrapper_b = _fake_cleanup_state() # Pod B's independent wrapper for the same remote browser
+
+ with _live_workflow_run_contexts(): # nothing is live in Pod A's process
+ await mgr_a.cleanup_for_workflow_run("wr_podA", [], close_browser_on_completion=False)
+
+ # The durable remote browser is preserved: close(False), never a terminal close.
+ wrapper_a.close.assert_awaited_once_with(close_browser_on_completion=False, release_driver=None)
+ assert _close_true_count(wrapper_a) == 0
+ # Pod B's wrapper is entirely untouched by Pod A's cleanup.
+ wrapper_b.close.assert_not_awaited()
diff --git a/tests/unit/test_remote_cdp_driver_release.py b/tests/unit/test_remote_cdp_driver_release.py
index dcac5d547..b616f88a8 100644
--- a/tests/unit/test_remote_cdp_driver_release.py
+++ b/tests/unit/test_remote_cdp_driver_release.py
@@ -14,11 +14,14 @@ from __future__ import annotations
import asyncio
import time
+from contextlib import contextmanager
+from typing import Iterator
from unittest.mock import AsyncMock, MagicMock, call
import pytest
from skyvern.exceptions import MissingBrowserStateForBrowserSession, MissingOrganizationForBrowserSession
+from skyvern.forge import app as forge_app
from skyvern.webeye import browser_engine
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from skyvern.webeye.browser_engine import (
@@ -198,10 +201,10 @@ async def test_close_true_still_stops_driver_and_context() -> None:
@pytest.mark.asyncio
-async def test_close_true_cancellation_resistant_interceptor_disable_still_runs_provider_cleanup(
+async def test_close_true_cancellation_resistant_interceptor_disable_still_runs_remote_cleanup(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- # The paid-provider cleanup must still be attempted exactly once even when disabling the
+ # The remote browser cleanup must still be attempted exactly once even when disabling the
# download interceptor is cancellation-resistant (ignores the cancel it receives when its
# budget is exhausted). close() must return within an outer watchdog, and the stuck drain
# must stay owned (its eventual exception retrieved) rather than orphaned.
@@ -237,7 +240,7 @@ async def test_close_true_cancellation_resistant_interceptor_disable_still_runs_
elapsed = time.monotonic() - start
assert elapsed < 1.0
- cleanup.assert_awaited_once() # provider cleanup ran despite the stuck drain
+ cleanup.assert_awaited_once() # remote cleanup ran despite the stuck drain
context.close.assert_awaited_once()
pw.stop.assert_awaited_once()
@@ -302,10 +305,10 @@ async def test_close_true_detached_teardown_task_is_strongly_held_until_completi
@pytest.mark.asyncio
-async def test_close_true_context_close_hang_does_not_suppress_provider_cleanup(
+async def test_close_true_context_close_hang_does_not_suppress_remote_cleanup(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- # A hung context close must not starve the paid-provider cleanup: the context-teardown phase is
+ # A hung context close must not starve the remote browser cleanup: the context-teardown phase is
# bounded independently, so cleanup still runs exactly once after the teardown budget elapses.
monkeypatch.setattr("skyvern.webeye.real_browser_state.BROWSER_CLOSE_TIMEOUT", 0.05)
pw = _pw_stub()
@@ -327,7 +330,7 @@ async def test_close_true_context_close_hang_does_not_suppress_provider_cleanup(
@pytest.mark.asyncio
-async def test_close_true_persist_cookies_failure_does_not_suppress_provider_cleanup(
+async def test_close_true_persist_cookies_failure_does_not_suppress_remote_cleanup(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# A raising cookie-persistence step (an "earlier teardown callback raises") must not suppress cleanup.
@@ -353,7 +356,7 @@ async def test_close_true_persist_cookies_failure_does_not_suppress_provider_cle
@pytest.mark.asyncio
async def test_close_true_browser_cleanup_exception_is_handled_not_leaked() -> None:
- # A raising provider cleanup must be caught (best-effort teardown) and must not leak or block driver stop.
+ # A raising remote cleanup must be caught (best-effort teardown) and must not leak or block driver stop.
pw = _pw_stub()
context = _context_stub()
context.cookies = AsyncMock(return_value=[])
@@ -408,8 +411,8 @@ async def test_close_false_preserves_reuse_and_does_not_invoke_cleanup() -> None
@pytest.mark.asyncio
-async def test_close_true_runs_provider_cleanup_exactly_once_across_reentry() -> None:
- # Re-entrant close() must not stop/delete the paid provider a second time (avoid double-cleanup).
+async def test_close_true_runs_remote_cleanup_exactly_once_across_reentry() -> None:
+ # Re-entrant close() must not stop/delete the remote browser a second time (avoid double-cleanup).
pw = _pw_stub()
context = _context_stub()
context.cookies = AsyncMock(return_value=[])
@@ -553,6 +556,20 @@ def _fake_browser_state() -> MagicMock:
return state
+@contextmanager
+def _live_workflow_run_contexts(*run_ids: str) -> Iterator[None]:
+ """Mark the given workflow runs as live in this process so a shared wr_ alias legitimately
+ suppresses the terminal close (non-PBS ownership signal)."""
+ wcm = forge_app.WORKFLOW_CONTEXT_MANAGER
+ live = set(run_ids)
+ original = wcm.has_workflow_run_context
+ wcm.has_workflow_run_context = lambda workflow_run_id: workflow_run_id in live
+ try:
+ yield
+ finally:
+ wcm.has_workflow_run_context = original
+
+
@pytest.mark.asyncio
async def test_cleanup_for_task_keeps_driver_for_persistent_session() -> None:
manager = RealBrowserManager()
@@ -588,11 +605,12 @@ async def test_cleanup_for_workflow_run_keeps_driver_while_shared() -> None:
manager.pages["tsk_1"] = state
manager.pages["wr_parent"] = state
- await manager.cleanup_for_workflow_run(
- "wr_child",
- ["tsk_1"],
- close_browser_on_completion=True,
- )
+ with _live_workflow_run_contexts("wr_child", "wr_parent"):
+ await manager.cleanup_for_workflow_run(
+ "wr_child",
+ ["tsk_1"],
+ close_browser_on_completion=True,
+ )
# Both the workflow-run-level close and the task-level close observe the
# parent's surviving reference and must not release the shared driver.
@@ -608,11 +626,12 @@ async def test_cleanup_for_workflow_run_reports_recording_not_finalized_while_sh
manager.pages["wr_child"] = state
manager.pages["wr_parent"] = state
- result = await manager.cleanup_for_workflow_run(
- "wr_child",
- [],
- close_browser_on_completion=True,
- )
+ with _live_workflow_run_contexts("wr_child", "wr_parent"):
+ result = await manager.cleanup_for_workflow_run(
+ "wr_child",
+ [],
+ close_browser_on_completion=True,
+ )
assert result.browser_state is state
assert result.recording_finalized is False
diff --git a/tests/unit/test_run_sdk_action_failures.py b/tests/unit/test_run_sdk_action_failures.py
index 7052251b1..3a8f3c062 100644
--- a/tests/unit/test_run_sdk_action_failures.py
+++ b/tests/unit/test_run_sdk_action_failures.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
@@ -7,7 +8,7 @@ import pytest
from fastapi import HTTPException
from skyvern.exceptions import ScrapingFailed, SkyvernActionFailed
-from skyvern.forge.sdk.routes.sdk import run_sdk_action
+from skyvern.forge.sdk.routes.sdk import _sdk_action_context_refcounts, run_sdk_action
@pytest.fixture
@@ -134,13 +135,13 @@ async def test_handler_propagates_unknown_exception(mock_request: Any, mock_orga
@pytest.mark.asyncio
-async def test_client_provided_workflow_run_context_is_not_removed_on_cleanup(
+async def test_client_provided_context_removed_when_last_sibling_finishes(
mock_request: Any, mock_organization: Any, mock_app: Any
) -> None:
- # A client-provided workflow_run_id is shared across concurrent run_action calls; the
- # process-global context manager is keyed by that id, so tearing it down in this request's
- # cleanup yanks the context out from under an in-flight sibling call, which then raises
- # WorkflowRunContextNotInitialized in _render_template_with_label.
+ # A client-provided workflow_run_id is reference-counted. With no concurrent sibling holding it,
+ # this lone call is the last one, so its run-context IS removed on cleanup — otherwise it would
+ # persist as a permanent liveness ghost that vetoes a real run's terminal browser close.
+ # The refcount must return to empty for the run.
mock_request.workflow_run_id = "wr_shared"
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
return_value=MagicMock(workflow_run_id="wr_shared", workflow_id="w_test")
@@ -158,7 +159,40 @@ async def test_client_provided_workflow_run_context_is_not_removed_on_cleanup(
with pytest.raises(RuntimeError, match="stop inside try"):
await run_sdk_action(mock_request, organization=mock_organization)
- mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_not_called()
+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_called_once_with("wr_shared")
+ assert "wr_shared" not in _sdk_action_context_refcounts
+
+
+async def test_client_provided_context_retained_while_sibling_in_flight(
+ mock_request: Any, mock_organization: Any, mock_app: Any
+) -> None:
+ # A concurrent sibling run_action already holds the shared context (refcount pre-seeded). This
+ # call must NOT remove it on cleanup — tearing it down would yank the context out from under the
+ # in-flight sibling. The refcount drops back to the sibling's hold, not to zero.
+ mock_request.workflow_run_id = "wr_shared"
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
+ return_value=MagicMock(workflow_run_id="wr_shared", workflow_id="w_test")
+ )
+ _sdk_action_context_refcounts["wr_shared"] = 1 # a sibling call is in flight
+ try:
+ with (
+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),
+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",
+ new_callable=AsyncMock,
+ side_effect=RuntimeError("stop inside try"),
+ ),
+ ):
+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)
+ with pytest.raises(RuntimeError, match="stop inside try"):
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_not_called()
+ assert _sdk_action_context_refcounts["wr_shared"] == 1
+ finally:
+ _sdk_action_context_refcounts.pop("wr_shared", None)
@pytest.mark.asyncio
@@ -239,3 +273,78 @@ async def test_client_provided_workflow_run_is_not_marked_synthetic(
replaced_context = mock_ctx.replace.call_args.args[0]
assert replaced_context.workflow_run_is_synthetic is False
+
+
+async def test_context_refcount_released_when_initialization_fails(
+ mock_request: Any, mock_organization: Any, mock_app: Any
+) -> None:
+ # F1: the refcount acquire sits before initialize_workflow_run_context. If init RAISES, the
+ # release must still run (outer finally), or the client-provided run id leaks its refcount and
+ # becomes a permanent liveness ghost that vetoes a real run's terminal browser close.
+ mock_request.workflow_run_id = "wr_test"
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
+ return_value=MagicMock(workflow_run_id="wr_test", workflow_id="w_test")
+ )
+ mock_app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context = AsyncMock(side_effect=RuntimeError("init boom"))
+ with (
+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),
+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context"),
+ ):
+ with pytest.raises(RuntimeError, match="init boom"):
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ assert "wr_test" not in _sdk_action_context_refcounts
+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_called_once_with("wr_test")
+
+
+async def test_context_refcount_released_when_initialization_is_cancelled(
+ mock_request: Any, mock_organization: Any, mock_app: Any
+) -> None:
+ # F1 (cancellation): a CancelledError during init must not leak the refcount either — the outer
+ # finally releases it before the cancellation propagates.
+ mock_request.workflow_run_id = "wr_test"
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
+ return_value=MagicMock(workflow_run_id="wr_test", workflow_id="w_test")
+ )
+ mock_app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context = AsyncMock(side_effect=asyncio.CancelledError())
+ with (
+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),
+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context"),
+ ):
+ with pytest.raises(asyncio.CancelledError):
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ assert "wr_test" not in _sdk_action_context_refcounts
+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_called_once_with("wr_test")
+
+
+async def test_context_refcount_released_when_upload_drain_is_cancelled(
+ mock_request: Any, mock_organization: Any, mock_app: Any
+) -> None:
+ # F1 (drain cancellation): the release used to sit AFTER the upload-drain await inside the same
+ # finally, so a CancelledError during the drain skipped it — recreating the ghost. With the
+ # release in the OUTER finally it still runs when the drain await is cancelled.
+ mock_request.workflow_run_id = "wr_test"
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
+ return_value=MagicMock(workflow_run_id="wr_test", workflow_id="w_test")
+ )
+ mock_app.ARTIFACT_MANAGER.wait_for_upload_aiotasks = AsyncMock(side_effect=asyncio.CancelledError())
+ with (
+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),
+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",
+ new_callable=AsyncMock,
+ side_effect=RuntimeError("stop inside try"),
+ ),
+ ):
+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)
+ # The drain's CancelledError (raised in the inner finally) supersedes the body error.
+ with pytest.raises(asyncio.CancelledError):
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ assert "wr_test" not in _sdk_action_context_refcounts
+ mock_app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context.assert_called_once_with("wr_test")
diff --git a/tests/unit/test_workflow_run_context_eviction.py b/tests/unit/test_workflow_run_context_eviction.py
index 05cc2f3b2..6dc1a8999 100644
--- a/tests/unit/test_workflow_run_context_eviction.py
+++ b/tests/unit/test_workflow_run_context_eviction.py
@@ -33,6 +33,18 @@ def test_remove_workflow_run_context_evicts_and_is_idempotent() -> None:
manager.remove_workflow_run_context("wr_never_seen")
+def test_has_workflow_run_context_tracks_process_local_liveness() -> None:
+ # The non-PBS sharing predicate reads this as its run-liveness signal: True only while the run's
+ # context is registered in THIS process, flipping to False once cleanup removes it.
+ manager = _context_manager_with(["wr_live"])
+
+ assert manager.has_workflow_run_context("wr_live") is True
+ assert manager.has_workflow_run_context("wr_absent") is False
+
+ manager.remove_workflow_run_context("wr_live")
+ assert manager.has_workflow_run_context("wr_live") is False
+
+
@pytest.mark.asyncio
async def test_clean_up_workflow_evicts_run_and_child_contexts(monkeypatch: pytest.MonkeyPatch) -> None:
service = WorkflowService()