Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12367): release upload aiotask tracking on every wait exit path (#7394)
LawyZhengaf95cb7e1f16
diff --git a/skyvern/forge/sdk/artifact/manager.py b/skyvern/forge/sdk/artifact/manager.pyindex 6addb5e61..95bbee60d 100644--- a/skyvern/forge/sdk/artifact/manager.py+++ b/skyvern/forge/sdk/artifact/manager.py@@ -1728,9 +1728,13 @@ class ArtifactManager:f"Timeout (30s) while waiting for upload aio tasks for primary_keys={primary_keys}",primary_keys=primary_keys,)-- for primary_key in primary_keys:- del self.upload_aiotasks_map[primary_key]+ finally:+ # Release tracking refs on every exit path — including caller cancellation and+ # non-timeout gather errors — so an orphaned entry can't pin a completed upload Task+ # and the artifact bytes its traceback retains. pop() keeps overlapping waits on the+ # same key idempotent (no KeyError).+ for primary_key in primary_keys:+ self.upload_aiotasks_map.pop(primary_key, None)# Flush any accumulated step archives for the given task IDsprimary_key_set = set(primary_keys)diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.pyindex 6a13353d2..4bb083c05 100644--- a/skyvern/webeye/default_persistent_sessions_manager.py+++ b/skyvern/webeye/default_persistent_sessions_manager.py@@ -22,6 +22,7 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import (PersistentBrowserType,is_final_status,)+from skyvern.schemas.run_enums import RunTypefrom skyvern.schemas.runs import ProxyLocation, ProxyLocationInputfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.cdp_ports import _release_cdp_port@@ -687,6 +688,33 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):except Exception:LOG.exception("Browser session reaper pass failed")+ async def _owning_run_is_active(self, runnable_id: str, runnable_type: str | None, organization_id: str) -> bool:+ """Whether the run occupying a session is still live, so the reaper must leave teardown to it.++ A run that dies before release_browser_session leaves runnable_id set forever; treating a+ terminal or missing owner as inactive lets the normal timeout+grace gate reclaim the session+ instead of skipping it forever. Owners we can't authoritatively resolve — an unrecognized+ runnable type or a lookup failure — are treated as active, so a reap never races a run we+ can't prove is gone."""+ if runnable_type != RunType.workflow_run:+ return True+ try:+ workflow_run = await self.database.workflow_runs.get_workflow_run(+ workflow_run_id=runnable_id,+ organization_id=organization_id,+ )+ except Exception:+ LOG.warning(+ "Could not resolve owning run for persistent browser session; leaving it protected",+ runnable_id=runnable_id,+ organization_id=organization_id,+ exc_info=True,+ )+ return True+ if workflow_run is None:+ return False+ return not workflow_run.status.is_final()+async def reap_expired_sessions(self) -> None:"""Close the sessions this process holds whose timeout (plus grace) has elapsed viaclose_session, which tears down the context (stopping its ffmpeg recorder), syncs the@@ -698,8 +726,12 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):# Only reap browsers this process holds; completing another process's row would hide its leak.if db_session.persistent_browser_session_id not in self._browser_sessions:continue- # Leave sessions occupied by a running task/workflow to that run's own teardown.- if db_session.runnable_id is not None:+ # Leave sessions occupied by a still-live run to that run's own teardown. A run that died+ # before releasing occupancy leaves runnable_id set forever, so resolve the owner and let+ # an expired session with a terminal/missing owner fall through to the expiry gate below.+ if db_session.runnable_id is not None and await self._owning_run_is_active(+ db_session.runnable_id, db_session.runnable_type, db_session.organization_id+ ):continue# Leave sessions an active copilot turn is driving (no runnable_id and not renewed).if db_session.persistent_browser_session_id in copilot_session_ids:diff --git a/tests/unit/test_persistent_session_reaper.py b/tests/unit/test_persistent_session_reaper.pyindex 4f729f300..3156d7a12 100644--- a/tests/unit/test_persistent_session_reaper.py+++ b/tests/unit/test_persistent_session_reaper.py@@ -2,11 +2,15 @@Chromium + record_video ffmpeg encoders don't leak."""from datetime import datetime, timedelta, timezone+from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest-from skyvern.webeye.default_persistent_sessions_manager import DefaultPersistentSessionsManager+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus+from skyvern.schemas.run_enums import RunType+from skyvern.webeye import default_persistent_sessions_manager as manager_mod+from skyvern.webeye.default_persistent_sessions_manager import BrowserSession, DefaultPersistentSessionsManagerMODULE = "skyvern.webeye.default_persistent_sessions_manager"@@ -19,6 +23,9 @@ def _make_manager(uncompleted_sessions: list, owned_ids: list[str] | None = Nonedb = MagicMock()db.browser_sessions = MagicMock()db.browser_sessions.get_uncompleted_persistent_browser_sessions = AsyncMock(return_value=uncompleted_sessions)+ db.workflow_runs = MagicMock()+ # Default: the owning run row is gone (stale). Tests that need a live/terminal owner override this.+ db.workflow_runs.get_workflow_run = AsyncMock(return_value=None)manager = DefaultPersistentSessionsManager(database=db)# Register the browsers this process "holds" — the reaper only touches these.held = owned_ids if owned_ids is not None else [s.persistent_browser_session_id for s in uncompleted_sessions]@@ -32,6 +39,7 @@ def _session(started_minutes_ago: float | None,timeout_minutes: int | None,runnable_id: str | None = None,+ runnable_type: str | None = None,) -> MagicMock:started_at = Noneif started_minutes_ago is not None:@@ -42,9 +50,15 @@ def _session(started_at=started_at,timeout_minutes=timeout_minutes,runnable_id=runnable_id,+ runnable_type=runnable_type,)+def _workflow_run(status: WorkflowRunStatus) -> MagicMock:+ # Real WorkflowRunStatus so .is_final() runs the production logic, not a mocked truth value.+ return MagicMock(status=status)++@pytest.mark.asyncioasync def test_reaps_only_sessions_past_timeout_and_grace() -> None:sessions = [@@ -89,8 +103,161 @@ async def test_skips_expired_session_not_held_by_this_process() -> None:async def test_skips_session_occupied_by_running_runnable() -> None:# Past timeout and held here, but still occupied by a running workflow (renewal caps at 2h# while runs can go longer). Its run owns teardown, so the reaper must not close it.- sessions = [_session("pbs_in_run", started_minutes_ago=180, timeout_minutes=60, runnable_id="wr_active")]+ sessions = [+ _session(+ "pbs_in_run",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_active",+ runnable_type=RunType.workflow_run,+ )+ ]+ manager = _make_manager(sessions)+ manager.database.workflow_runs.get_workflow_run = AsyncMock(return_value=_workflow_run(WorkflowRunStatus.running))+ manager.close_session = AsyncMock()++ await manager.reap_expired_sessions()++ manager.close_session.assert_not_awaited()+++async def test_skips_session_occupied_by_paused_runnable() -> None:+ # A paused workflow run still owns the session; paused is non-final, so the reaper must not close it.+ sessions = [+ _session(+ "pbs_paused",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_paused",+ runnable_type=RunType.workflow_run,+ )+ ]+ manager = _make_manager(sessions)+ manager.database.workflow_runs.get_workflow_run = AsyncMock(return_value=_workflow_run(WorkflowRunStatus.paused))+ manager.close_session = AsyncMock()++ await manager.reap_expired_sessions()++ manager.close_session.assert_not_awaited()+++ "terminal_status",+ [+ WorkflowRunStatus.completed,+ WorkflowRunStatus.failed,+ WorkflowRunStatus.terminated,+ WorkflowRunStatus.canceled,+ WorkflowRunStatus.timed_out,+ ],+)+async def test_reaps_expired_session_whose_owning_run_is_terminal(terminal_status: WorkflowRunStatus) -> None:+ # The owning run finished but died before release_browser_session cleared runnable_id. Occupancy+ # is stale, the session is past timeout+grace — the reaper must reclaim it instead of skipping forever.+ sessions = [+ _session(+ "pbs_stuck",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_dead",+ runnable_type=RunType.workflow_run,+ )+ ]+ manager = _make_manager(sessions)+ manager.database.workflow_runs.get_workflow_run = AsyncMock(return_value=_workflow_run(terminal_status))+ manager.close_session = AsyncMock()++ await manager.reap_expired_sessions()++ manager.close_session.assert_awaited_once_with("org_test", "pbs_stuck")+ manager.database.workflow_runs.get_workflow_run.assert_awaited_once_with(+ workflow_run_id="wr_dead",+ organization_id="org_test",+ )+++async def test_reaps_expired_session_whose_owning_run_is_missing() -> None:+ # The owning run row is gone entirely (deleted/never findable). No live owner — reclaim the session.+ sessions = [+ _session(+ "pbs_orphan",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_gone",+ runnable_type=RunType.workflow_run,+ )+ ]+ manager = _make_manager(sessions)+ manager.database.workflow_runs.get_workflow_run = AsyncMock(return_value=None)+ manager.close_session = AsyncMock()++ await manager.reap_expired_sessions()++ manager.close_session.assert_awaited_once_with("org_test", "pbs_orphan")+++async def test_does_not_reap_terminal_owned_session_before_expiry() -> None:+ # Owner is terminal, but the session is still inside its timeout window. Stale ownership alone+ # must not trigger a reap — the timeout+grace gate still governs, so this is NOT reaped yet.+ sessions = [+ _session(+ "pbs_recent",+ started_minutes_ago=1,+ timeout_minutes=60,+ runnable_id="wr_dead",+ runnable_type=RunType.workflow_run,+ )+ ]+ manager = _make_manager(sessions)+ manager.database.workflow_runs.get_workflow_run = AsyncMock(return_value=_workflow_run(WorkflowRunStatus.completed))+ manager.close_session = AsyncMock()++ await manager.reap_expired_sessions()++ manager.close_session.assert_not_awaited()+++async def test_protects_expired_session_with_unknown_runnable_type() -> None:+ # A runnable_id set with a runnable_type the reaper can't authoritatively resolve stays protected:+ # no broad age-only fallback, so this is left to its owner's teardown rather than reaped.+ sessions = [+ _session(+ "pbs_unknown",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="tsk_unknown",+ runnable_type="task_v2",+ )+ ]+ manager = _make_manager(sessions)+ manager.close_session = AsyncMock()++ await manager.reap_expired_sessions()++ manager.close_session.assert_not_awaited()+ manager.database.workflow_runs.get_workflow_run.assert_not_awaited()+++async def test_protects_expired_session_when_owner_lookup_fails() -> None:+ # If the owner lookup errors we cannot prove the run is dead, so fail safe: never reap a session+ # whose liveness is unknown (would otherwise risk killing an active run's browser).+ sessions = [+ _session(+ "pbs_dberr",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_dberr",+ runnable_type=RunType.workflow_run,+ )+ ]manager = _make_manager(sessions)+ manager.database.workflow_runs.get_workflow_run = AsyncMock(side_effect=RuntimeError("db down"))manager.close_session = AsyncMock()await manager.reap_expired_sessions()@@ -112,6 +279,121 @@ async def test_skips_session_held_by_active_copilot_turn() -> None:manager.close_session.assert_not_awaited()+async def test_reaping_stale_owned_session_closes_local_browser_state() -> None:+ # End-to-end: a stale-owned expired session must be torn down, not just marked in the DB — the+ # in-process BrowserState (Chromium + driver) is closed and dropped from the local registry.+ session = _session(+ "pbs_teardown",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_dead",+ runnable_type=RunType.workflow_run,+ )+ manager = _make_manager([session])+ manager.database.workflow_runs.get_workflow_run = AsyncMock(return_value=_workflow_run(WorkflowRunStatus.failed))+ manager.database.browser_sessions.close_persistent_browser_session = AsyncMock()++ browser_state = MagicMock()+ browser_state.close = AsyncMock()+ browser_state.browser_artifacts = SimpleNamespace(browser_session_dir=None, video_artifacts=[])+ manager._browser_sessions["pbs_teardown"] = BrowserSession(browser_state=browser_state, organization_id="org_test")++ with patch.object(manager_mod.settings, "BROWSER_STREAMING_MODE", "vnc"):+ await manager.reap_expired_sessions()++ browser_state.close.assert_awaited_once()+ assert "pbs_teardown" not in manager._browser_sessions+ manager.database.browser_sessions.close_persistent_browser_session.assert_awaited_once_with(+ "pbs_teardown",+ "org_test",+ )+++async def test_reap_pass_survives_close_failure_without_dropping_session() -> None:+ # A teardown/DB failure while reaping one session must not abort the pass or silently drop the+ # session: the exception is contained, and the row stays uncompleted so a later pass retries it.+ session = _session(+ "pbs_flaky_close",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_dead",+ runnable_type=RunType.workflow_run,+ )+ manager = _make_manager([session])+ manager.database.workflow_runs.get_workflow_run = AsyncMock(return_value=_workflow_run(WorkflowRunStatus.completed))+ manager.close_session = AsyncMock(side_effect=RuntimeError("close failed"))++ await manager.reap_expired_sessions() # must not raise++ manager.close_session.assert_awaited_once_with("org_test", "pbs_flaky_close")+++async def test_reap_is_idempotent_after_session_reclaimed() -> None:+ # After a stale-owned session is reaped its BrowserState is popped from the local registry, so a+ # later pass (even if the DB row is still returned) hits the not-held guard and does not re-tear+ # it down. Repeated reaps never double-close a session.+ session = _session(+ "pbs_idem",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_dead",+ runnable_type=RunType.workflow_run,+ )+ manager = _make_manager([session])+ manager.database.workflow_runs.get_workflow_run = AsyncMock(return_value=_workflow_run(WorkflowRunStatus.completed))+ manager.database.browser_sessions.close_persistent_browser_session = AsyncMock()++ browser_state = MagicMock()+ browser_state.close = AsyncMock()+ browser_state.browser_artifacts = SimpleNamespace(browser_session_dir=None, video_artifacts=[])+ manager._browser_sessions["pbs_idem"] = BrowserSession(browser_state=browser_state, organization_id="org_test")++ with patch.object(manager_mod.settings, "BROWSER_STREAMING_MODE", "vnc"):+ await manager.reap_expired_sessions()+ await manager.reap_expired_sessions()++ browser_state.close.assert_awaited_once()+ manager.database.browser_sessions.close_persistent_browser_session.assert_awaited_once_with(+ "pbs_idem",+ "org_test",+ )+++async def test_reclaims_cdp_connect_request_level_session_after_run_dies() -> None:+ # Production wiring: a run submitted with browser_session_id under cdp-connect occupies the session+ # via begin_session(runnable_type="workflow_run", runnable_id=workflow_run_id). If that run dies+ # without releasing, the reaper resolves the same workflow_run and reclaims the expired session.+ assert RunType.workflow_run == "workflow_run" # the exact literal begin_session writes+ session = _session(+ "pbs_request_level",+ started_minutes_ago=180,+ timeout_minutes=60,+ runnable_id="wr_request_level",+ runnable_type=RunType.workflow_run,+ )+ manager = _make_manager([session])+ manager.database.workflow_runs.get_workflow_run = AsyncMock(+ return_value=_workflow_run(WorkflowRunStatus.terminated)+ )+ manager.close_session = AsyncMock()++ with (+ patch.object(manager_mod.settings, "BROWSER_TYPE", "cdp-connect"),+ patch.object(manager_mod.settings, "BROWSER_STREAMING_MODE", "vnc"),+ ):+ await manager.reap_expired_sessions()++ manager.close_session.assert_awaited_once_with("org_test", "pbs_request_level")+ manager.database.workflow_runs.get_workflow_run.assert_awaited_once_with(+ workflow_run_id="wr_request_level",+ organization_id="org_test",+ )++@pytest.mark.asyncioasync def test_start_reaper_is_noop_when_no_in_process_browsers() -> None:# Neither trigger for an in-process browser launch: nothing to reap, so don't start the loop.diff --git a/tests/unit/test_step_archive_accumulator.py b/tests/unit/test_step_archive_accumulator.pyindex d626a2a17..f79dec03f 100644--- a/tests/unit/test_step_archive_accumulator.py+++ b/tests/unit/test_step_archive_accumulator.py@@ -4,7 +4,10 @@ These tests exercise the in-memory accumulation helpers and the ZIP-buildingutility without touching S3 or the database."""+import asyncio+import gcimport io+import weakrefimport zipfilefrom unittest.mock import AsyncMock, MagicMock, patch@@ -701,3 +704,183 @@ class TestFlushStepArchive:# The fallback should find nothing to flush — no extra uploadsmock_storage.store_artifact.assert_not_awaited()mock_database.artifacts.bulk_create_artifacts.assert_not_awaited()+++class TestWaitForUploadAiotasksCleanup:+ """wait_for_upload_aiotasks must release its upload_aiotasks_map entry on every exit path.++ upload_aiotasks_map is the sole strong reference to the fire-and-forget upload Tasks. An+ orphaned entry pins a completed-with-exception Task, whose traceback pins the store_artifact+ coroutine frame and its artifact byte payload. Cleanup must therefore run on success, timeout,+ caller cancellation, and non-timeout gather exceptions — not only the first two.+ """++ @pytest.mark.asyncio+ async def test_success_removes_map_entry(self) -> None:+ manager = ArtifactManager()++ async def quick() -> None:+ return None++ task = asyncio.create_task(quick())+ manager.upload_aiotasks_map["k"].append(task)++ await manager.wait_for_upload_aiotasks(["k"])++ assert task.done()+ assert "k" not in manager.upload_aiotasks_map++ @pytest.mark.asyncio+ async def test_timeout_removes_map_entry_and_swallows_timeout(self) -> None:+ manager = ArtifactManager()++ async def hang() -> None:+ await asyncio.Event().wait()++ task = asyncio.create_task(hang())+ manager.upload_aiotasks_map["k"].append(task)++ real_timeout = asyncio.timeout+ # Fire the 30s guard fast without altering the production semantics under test.+ with patch(+ "skyvern.forge.sdk.artifact.manager.asyncio.timeout",+ lambda _seconds: real_timeout(0.05),+ ):+ # Established behavior: the timeout is logged and swallowed, not re-raised.+ await manager.wait_for_upload_aiotasks(["k"])++ assert "k" not in manager.upload_aiotasks_map+ assert task.done()++ @pytest.mark.asyncio+ async def test_caller_cancellation_removes_map_entry_and_reraises(self) -> None:+ manager = ArtifactManager()++ async def hang() -> None:+ await asyncio.Event().wait()++ task = asyncio.create_task(hang())+ manager.upload_aiotasks_map["k"].append(task)++ wait_task = asyncio.create_task(manager.wait_for_upload_aiotasks(["k"]))+ await asyncio.sleep(0.01) # let wait_task suspend inside the gather await+ wait_task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await wait_task++ assert "k" not in manager.upload_aiotasks_map+ assert task.done()++ @pytest.mark.asyncio+ async def test_non_timeout_gather_exception_removes_map_entry_and_reraises(self) -> None:+ manager = ArtifactManager()++ async def raise_soon() -> None:+ await asyncio.sleep(0)+ raise ValueError("upload boom")++ task = asyncio.create_task(raise_soon())+ manager.upload_aiotasks_map["k"].append(task)++ with pytest.raises(ValueError, match="upload boom"):+ await manager.wait_for_upload_aiotasks(["k"])++ assert "k" not in manager.upload_aiotasks_map++ @pytest.mark.asyncio+ async def test_failed_completed_task_released_on_cancellation(self) -> None:+ manager = ArtifactManager()++ async def fail_now() -> None:+ raise ValueError("pins byte payload via traceback")++ failed = asyncio.create_task(fail_now())+ await asyncio.sleep(0) # let it complete with an exception+ assert failed.done() and failed.exception() is not None # retrieve → no "never retrieved" warning++ async def hang() -> None:+ await asyncio.Event().wait()++ hanging = asyncio.create_task(hang())+ manager.upload_aiotasks_map["k"].extend([failed, hanging])+ ref = weakref.ref(failed)++ wait_task = asyncio.create_task(manager.wait_for_upload_aiotasks(["k"]))+ await asyncio.sleep(0.01)+ wait_task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await wait_task++ assert "k" not in manager.upload_aiotasks_map+ assert all(failed is not t for lst in manager.upload_aiotasks_map.values() for t in lst)++ del failed+ gc.collect()+ assert ref() is None # traceback + byte payload no longer reachable through the map++ await asyncio.gather(hanging, return_exceptions=True) # settle the gather-cancelled task++ @pytest.mark.asyncio+ async def test_absent_key_is_safe(self) -> None:+ manager = ArtifactManager()++ await manager.wait_for_upload_aiotasks(["never-added"])++ assert "never-added" not in manager.upload_aiotasks_map++ @pytest.mark.asyncio+ async def test_duplicate_wait_same_key_is_idempotent(self) -> None:+ manager = ArtifactManager()++ async def quick() -> None:+ return None++ manager.upload_aiotasks_map["k"].append(asyncio.create_task(quick()))++ await manager.wait_for_upload_aiotasks(["k"])+ assert "k" not in manager.upload_aiotasks_map++ await manager.wait_for_upload_aiotasks(["k"]) # second wait on the now-absent key is a no-op+ assert "k" not in manager.upload_aiotasks_map++ @pytest.mark.asyncio+ async def test_concurrent_wait_same_key_is_safe(self) -> None:+ manager = ArtifactManager()++ async def settle() -> None:+ await asyncio.sleep(0)++ manager.upload_aiotasks_map["k"].append(asyncio.create_task(settle()))++ results = await asyncio.gather(+ manager.wait_for_upload_aiotasks(["k"]),+ manager.wait_for_upload_aiotasks(["k"]),+ return_exceptions=True,+ )++ assert results == [None, None] # neither concurrent waiter raised KeyError+ assert "k" not in manager.upload_aiotasks_map++ @pytest.mark.asyncio+ async def test_tasks_appended_during_wait_are_dropped_from_map(self) -> None:+ # Documented decision: a task appended to the same key *during* the wait is cleared with+ # the rest of the entry and is NOT awaited by the in-flight call. This preserves the+ # pre-existing whole-entry cleanup semantics rather than draining late arrivals.+ manager = ArtifactManager()+ appended_holder: list[asyncio.Task[None]] = []++ async def gate() -> None:+ late = asyncio.create_task(asyncio.sleep(3600))+ appended_holder.append(late)+ manager.upload_aiotasks_map["k"].append(late)++ manager.upload_aiotasks_map["k"].append(asyncio.create_task(gate()))++ await manager.wait_for_upload_aiotasks(["k"])++ assert "k" not in manager.upload_aiotasks_map+ late = appended_holder[0]+ assert not late.done() # the late arrival was not awaited to completion by this call+ late.cancel()+ await asyncio.gather(late, return_exceptions=True) # settle the cancelled task