Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Self-discard artifact upload tasks and stop resurrecting the execution-channel driver (#7456)
Aaron Perez457eb4f34675
diff --git a/skyvern/forge/sdk/artifact/manager.py b/skyvern/forge/sdk/artifact/manager.pyindex 826d2b213..2608b3480 100644--- a/skyvern/forge/sdk/artifact/manager.py+++ b/skyvern/forge/sdk/artifact/manager.py@@ -136,6 +136,36 @@ class ArtifactManager:# step_id -> accumulator for step archive artifactsself._step_archives: dict[str, StepArchiveAccumulator] = {}+ def _track_upload_aiotask(self, primary_key: str, aio_task: asyncio.Task[None]) -> None:+ """Track a fire-and-forget upload so wait_for_upload_aiotasks can barrier on it.++ Tasks self-discard on completion: writers key this map by ids no lifecycle ever+ drains (script deploys, ai suggestions, SDK actions), so relying on+ wait_for_upload_aiotasks as the sole reclaim path pins completed tasks — and, for+ failed uploads, the artifact bytes their tracebacks retain — forever (SKY-12524).+ """+ self.upload_aiotasks_map[primary_key].append(aio_task)++ def _discard(task: asyncio.Task[None]) -> None:+ if not task.cancelled() and (exc := task.exception()) is not None:+ LOG.warning(+ "Artifact upload task failed",+ primary_key=primary_key,+ error_type=type(exc).__name__,+ error=str(exc)[:200],+ )+ tasks = self.upload_aiotasks_map.get(primary_key)+ if tasks is None:+ return+ try:+ tasks.remove(task)+ except ValueError:+ return+ if not tasks:+ self.upload_aiotasks_map.pop(primary_key, None)++ aio_task.add_done_callback(_discard)+@staticmethoddef _build_artifact_model(artifact_id: str,@@ -247,11 +277,11 @@ class ArtifactManager:if data:# Fire and forgetaio_task = asyncio.create_task(app.STORAGE.store_artifact(artifact, data))- self.upload_aiotasks_map[aio_task_primary_key].append(aio_task)+ self._track_upload_aiotask(aio_task_primary_key, aio_task)elif path:# Fire and forgetaio_task = asyncio.create_task(app.STORAGE.store_artifact_from_path(artifact, path))- self.upload_aiotasks_map[aio_task_primary_key].append(aio_task)+ self._track_upload_aiotask(aio_task_primary_key, aio_task)return artifact_id@@ -818,10 +848,10 @@ class ArtifactManager:for artifact, artifact_data in zip(artifacts, request.artifacts):if artifact_data.data is not None:aio_task = asyncio.create_task(app.STORAGE.store_artifact(artifact, artifact_data.data))- self.upload_aiotasks_map[request.primary_key].append(aio_task)+ self._track_upload_aiotask(request.primary_key, aio_task)elif artifact_data.path is not None:aio_task = asyncio.create_task(app.STORAGE.store_artifact_from_path(artifact, artifact_data.path))- self.upload_aiotasks_map[request.primary_key].append(aio_task)+ self._track_upload_aiotask(request.primary_key, aio_task)return [model.artifact_id for model in artifact_models]@@ -1125,7 +1155,7 @@ class ArtifactManager:aio_task_key = artifact[primary_key] or artifact["workflow_run_block_id"] or artifact["run_id"]if not aio_task_key:raise ValueError("artifact must have a task_id, workflow_run_block_id, or run_id to track its upload.")- self.upload_aiotasks_map[aio_task_key].append(aio_task)+ self._track_upload_aiotask(aio_task_key, aio_task)return aio_task_keyasync def retrieve_artifact(self, artifact: Artifact) -> bytes | None:@@ -1760,7 +1790,7 @@ class ArtifactManager:*[aio_taskfor primary_key in primary_keys- for aio_task in self.upload_aiotasks_map[primary_key]+ for aio_task in self.upload_aiotasks_map.get(primary_key, ())if not aio_task.done()])diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.pyindex dd3c7cb11..9dce33147 100644--- a/skyvern/forge/sdk/routes/sdk.py+++ b/skyvern/forge/sdk/routes/sdk.py@@ -266,6 +266,14 @@ async def run_sdk_action()raisefinally:+ # This route runs the whole mini-task in-process, so nothing in the worker+ # lifecycle ever drains its artifact uploads / step archives or removes its+ # workflow-run context — without this, every call leaks them (SKY-12524).+ try:+ await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([task.task_id])+ except Exception:+ LOG.warning("Failed to drain artifact uploads for SDK action", task_id=task.task_id, exc_info=True)+ app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context(workflow_run.workflow_run_id)skyvern_context.reset()return RunSdkActionResponse(diff --git a/skyvern/forge/sdk/routes/streaming/channels/cdp.py b/skyvern/forge/sdk/routes/streaming/channels/cdp.pyindex 9d534e505..da668658a 100644--- a/skyvern/forge/sdk/routes/streaming/channels/cdp.py+++ b/skyvern/forge/sdk/routes/streaming/channels/cdp.py@@ -185,22 +185,36 @@ class CdpChannel:async def close(self) -> None:LOG.info(f"{self.class_name} closing connection", **self.identity)- if self.browser:- try:- await self.browser.close()- except Exception:- pass- self.browser = None-- if self.pw:- await self.pw.stop()- self.pw = None-- self.browser_context = None- self.page = None+ try:+ if self.browser:+ try:+ await self.browser.close()+ except Exception:+ pass+ self.browser = None+ finally:+ # Release the driver even when browser.close() raised on a dead target;+ # a skipped stop() orphans the node subprocess for the process lifetime.+ if self.pw:+ try:+ await self.pw.stop()+ except Exception:+ LOG.warning(f"{self.class_name} failed to stop playwright driver", **self.identity, exc_info=True)+ self.pw = None++ self.browser_context = None+ self.page = NoneLOG.info(f"{self.class_name} closed", **self.identity)+ async def stop(self) -> t.Self:+ """Terminal close: unlike close() (which connect() reuses to recycle a dropped+ connection), stop() marks the channel closing so the browser "disconnected"+ callback registered in connect() cannot resurrect a fresh driver."""+ self._closing = True+ await self.close()+ return self+async def evaluate_js(self,expression: str,diff --git a/skyvern/forge/sdk/routes/streaming/channels/execution.py b/skyvern/forge/sdk/routes/streaming/channels/execution.pyindex 40763730a..8f3cc2714 100644--- a/skyvern/forge/sdk/routes/streaming/channels/execution.py+++ b/skyvern/forge/sdk/routes/streaming/channels/execution.py@@ -239,21 +239,6 @@ class ExecutionChannel(CdpChannel):return ""return f"{parsed.scheme}://{parsed.netloc}"- async def close(self) -> None:- LOG.info(f"{self.class_name} closing connection", **self.identity)-- if self.browser:- await self.browser.close()- self.browser = None- self.browser_context = None- self.page = None-- if self.pw:- await self.pw.stop()- self.pw = None-- LOG.info(f"{self.class_name} closed", **self.identity)-class LocalExecutionChannel(ExecutionChannel):def __init__(self, *, page: Page) -> None: # type: ignore[override]@@ -360,4 +345,6 @@ async def execution_channel(vnc_channel: VncChannel) -> t.AsyncIterator[Executioyield channelfinally:- await channel.close()+ # stop(), not close(): a bare close() fires the browser "disconnected" event and+ # the reconnect callback resurrects a driver nothing will ever release (SKY-12524).+ await channel.stop()diff --git a/tests/unit/forge/sdk/routes/streaming/test_execution_channel.py b/tests/unit/forge/sdk/routes/streaming/test_execution_channel.pynew file mode 100644index 000000000..137d5605c--- /dev/null+++ b/tests/unit/forge/sdk/routes/streaming/test_execution_channel.py@@ -0,0 +1,74 @@+"""Regression tests for the ExecutionChannel driver lifecycle (SKY-12524)."""++from __future__ import annotations++import asyncio+from unittest.mock import MagicMock++import pytest+from playwright._impl._errors import TargetClosedError++from skyvern.forge.sdk.routes.streaming.channels.execution import ExecutionChannel, execution_channel+from tests.unit.forge.sdk.routes.streaming.test_exfiltration_channel import (+ _FakePw,+ _make_vnc_channel,+ _patch_pw_stack,+)+++async def _drain_loop() -> None:+ for _ in range(30):+ await asyncio.sleep(0)+++class _RaisingBrowser:+ def __init__(self) -> None:+ self.close_calls = 0++ async def close(self) -> None:+ self.close_calls += 1+ raise TargetClosedError("target already dead")+++async def test_execution_channel_cm_does_not_resurrect_driver(monkeypatch: pytest.MonkeyPatch) -> None:+ # Playwright fires "disconnected" during an intentional browser.close(); the+ # on_close handler in CdpChannel.connect must not chain a fresh driver spawn.+ state = _patch_pw_stack(monkeypatch, fire_disconnect_on_close=True)++ async with execution_channel(_make_vnc_channel()):+ pass+ await _drain_loop()++ assert state.start_calls == 1+ assert state.pws[0].stopped is True+++async def test_stop_releases_driver_when_browser_close_raises() -> None:+ channel = ExecutionChannel(vnc_channel=_make_vnc_channel())+ browser = _RaisingBrowser()+ pw = _FakePw()+ channel.browser = browser # type: ignore[assignment]+ channel.pw = pw # type: ignore[assignment]++ await channel.stop()++ assert browser.close_calls == 1+ assert pw.stopped is True+ assert channel.browser is None+ assert channel.pw is None+ assert channel._closing is True+++async def test_local_execution_channel_stop_is_safe() -> None:+ from skyvern.forge.sdk.routes.streaming.channels.execution import LocalExecutionChannel++ page = MagicMock()+ page.context = MagicMock()+ channel = LocalExecutionChannel(page=page)++ await channel.stop()++ assert channel._closing is Truediff --git a/tests/unit/forge/sdk/test_artifact_manager.py b/tests/unit/forge/sdk/test_artifact_manager.pynew file mode 100644index 000000000..802fca975--- /dev/null+++ b/tests/unit/forge/sdk/test_artifact_manager.py@@ -0,0 +1,100 @@+import asyncio++import pytest++from skyvern.forge.sdk.artifact.manager import ArtifactManager+++async def _noop() -> None:+ return None+++async def _boom() -> None:+ raise RuntimeError("upload failed")+++async def _drain_loop() -> None:+ for _ in range(10):+ await asyncio.sleep(0)+++async def test_upload_task_self_discards_on_success() -> None:+ manager = ArtifactManager()+ task = asyncio.create_task(_noop())++ manager._track_upload_aiotask("tsk_1", task)+ assert manager.upload_aiotasks_map["tsk_1"] == [task]++ await task+ await _drain_loop()++ assert "tsk_1" not in manager.upload_aiotasks_map+++async def test_upload_task_self_discards_on_failure_and_retrieves_exception() -> None:+ manager = ArtifactManager()+ task = asyncio.create_task(_boom())++ manager._track_upload_aiotask("tsk_1", task)+ with pytest.raises(RuntimeError):+ await task+ await _drain_loop()++ # The failed task must not stay pinned in the map (its traceback retains the artifact bytes).+ assert "tsk_1" not in manager.upload_aiotasks_map+++async def test_upload_task_discard_keeps_other_tasks_for_same_key() -> None:+ manager = ArtifactManager()+ done_task = asyncio.create_task(_noop())+ pending_task = asyncio.create_task(asyncio.sleep(30))++ manager._track_upload_aiotask("tsk_1", done_task)+ manager._track_upload_aiotask("tsk_1", pending_task)+ await done_task+ await _drain_loop()++ assert manager.upload_aiotasks_map["tsk_1"] == [pending_task]++ pending_task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await pending_task+ await _drain_loop()+ assert "tsk_1" not in manager.upload_aiotasks_map+++async def test_wait_for_upload_aiotasks_still_barriers_pending_uploads() -> None:+ manager = ArtifactManager()+ started = asyncio.Event()+ release = asyncio.Event()++ async def _slow_upload() -> None:+ started.set()+ await release.wait()++ task = asyncio.create_task(_slow_upload())+ manager._track_upload_aiotask("tsk_1", task)+ await started.wait()++ waiter = asyncio.create_task(manager.wait_for_upload_aiotasks(["tsk_1"]))+ await _drain_loop()+ assert not waiter.done()++ release.set()+ await waiter++ assert task.done()+ assert "tsk_1" not in manager.upload_aiotasks_map+++async def test_wait_for_upload_aiotasks_does_not_autovivify_keys() -> None:+ manager = ArtifactManager()++ await manager.wait_for_upload_aiotasks(["tsk_never_seen"])++ assert "tsk_never_seen" not in manager.upload_aiotasks_mapdiff --git a/tests/unit/test_code_block_recording_artifact.py b/tests/unit/test_code_block_recording_artifact.pyindex ade8437a6..c977ebf50 100644--- a/tests/unit/test_code_block_recording_artifact.py+++ b/tests/unit/test_code_block_recording_artifact.py@@ -308,9 +308,15 @@ class TestUpdateArtifactDataScopeFallback:patch(f"{_MANAGER_PATH}.STORAGE.store_artifact", AsyncMock()),):await manager.update_artifact_data(artifact_id="a_recording", organization_id="o_1", data=b"video")- await asyncio.gather(*manager.upload_aiotasks_map["wrb_1"])- assert list(manager.upload_aiotasks_map.keys()) == ["wrb_1"]+ # Observe the keying while the upload is still pending: completed tasks+ # self-discard from the map (SKY-12524).+ assert list(manager.upload_aiotasks_map.keys()) == ["wrb_1"]++ await asyncio.gather(*list(manager.upload_aiotasks_map["wrb_1"]))+ await asyncio.sleep(0)++ assert "wrb_1" not in manager.upload_aiotasks_map@pytest.mark.asyncioasync def test_prefers_task_id_when_present(self) -> None:@@ -321,10 +327,14 @@ class TestUpdateArtifactDataScopeFallback:patch(f"{_MANAGER_PATH}.STORAGE.store_artifact", AsyncMock()),):await manager.update_artifact_data(artifact_id="a_recording", organization_id="o_1", data=b"video")- await asyncio.gather(*manager.upload_aiotasks_map["tsk_1"])- assert "tsk_1" in manager.upload_aiotasks_map- assert "wrb_1" not in manager.upload_aiotasks_map+ assert "tsk_1" in manager.upload_aiotasks_map+ assert "wrb_1" not in manager.upload_aiotasks_map++ await asyncio.gather(*list(manager.upload_aiotasks_map["tsk_1"]))+ await asyncio.sleep(0)++ assert "tsk_1" not in manager.upload_aiotasks_map@pytest.mark.asyncioasync def test_raises_when_no_scope_id_available(self) -> None: