Skip to content

Automate browser based workflows with AI

Self-discard artifact upload tasks and stop resurrecting the execution-channel driver (#7456)

Aaron Perez457eb4f34675
diff --git a/skyvern/forge/sdk/artifact/manager.py b/skyvern/forge/sdk/artifact/manager.py
index 826d2b213..2608b3480 100644
--- a/skyvern/forge/sdk/artifact/manager.py
+++ b/skyvern/forge/sdk/artifact/manager.py
@@ -136,6 +136,36 @@ class ArtifactManager:
# step_id -> accumulator for step archive artifacts
self._step_archives: dict[str, StepArchiveAccumulator] = {}
+ def _track_upload_aiotask(self, primary_key: str, aio_task: asyncio.Task[None]) -> None:
+ """Track a fire-and-forget upload so wait_for_upload_aiotasks can barrier on it.
+
+ Tasks self-discard on completion: writers key this map by ids no lifecycle ever
+ drains (script deploys, ai suggestions, SDK actions), so relying on
+ wait_for_upload_aiotasks as the sole reclaim path pins completed tasks — and, for
+ failed uploads, the artifact bytes their tracebacks retain — forever (SKY-12524).
+ """
+ self.upload_aiotasks_map[primary_key].append(aio_task)
+
+ def _discard(task: asyncio.Task[None]) -> None:
+ if not task.cancelled() and (exc := task.exception()) is not None:
+ LOG.warning(
+ "Artifact upload task failed",
+ primary_key=primary_key,
+ error_type=type(exc).__name__,
+ error=str(exc)[:200],
+ )
+ tasks = self.upload_aiotasks_map.get(primary_key)
+ if tasks is None:
+ return
+ try:
+ tasks.remove(task)
+ except ValueError:
+ return
+ if not tasks:
+ self.upload_aiotasks_map.pop(primary_key, None)
+
+ aio_task.add_done_callback(_discard)
+
@staticmethod
def _build_artifact_model(
artifact_id: str,
@@ -247,11 +277,11 @@ class ArtifactManager:
if data:
# Fire and forget
aio_task = asyncio.create_task(app.STORAGE.store_artifact(artifact, data))
- self.upload_aiotasks_map[aio_task_primary_key].append(aio_task)
+ self._track_upload_aiotask(aio_task_primary_key, aio_task)
elif path:
# Fire and forget
aio_task = asyncio.create_task(app.STORAGE.store_artifact_from_path(artifact, path))
- self.upload_aiotasks_map[aio_task_primary_key].append(aio_task)
+ self._track_upload_aiotask(aio_task_primary_key, aio_task)
return artifact_id
@@ -818,10 +848,10 @@ class ArtifactManager:
for artifact, artifact_data in zip(artifacts, request.artifacts):
if artifact_data.data is not None:
aio_task = asyncio.create_task(app.STORAGE.store_artifact(artifact, artifact_data.data))
- self.upload_aiotasks_map[request.primary_key].append(aio_task)
+ self._track_upload_aiotask(request.primary_key, aio_task)
elif artifact_data.path is not None:
aio_task = asyncio.create_task(app.STORAGE.store_artifact_from_path(artifact, artifact_data.path))
- self.upload_aiotasks_map[request.primary_key].append(aio_task)
+ self._track_upload_aiotask(request.primary_key, aio_task)
return [model.artifact_id for model in artifact_models]
@@ -1125,7 +1155,7 @@ class ArtifactManager:
aio_task_key = artifact[primary_key] or artifact["workflow_run_block_id"] or artifact["run_id"]
if not aio_task_key:
raise ValueError("artifact must have a task_id, workflow_run_block_id, or run_id to track its upload.")
- self.upload_aiotasks_map[aio_task_key].append(aio_task)
+ self._track_upload_aiotask(aio_task_key, aio_task)
return aio_task_key
async def retrieve_artifact(self, artifact: Artifact) -> bytes | None:
@@ -1760,7 +1790,7 @@ class ArtifactManager:
*[
aio_task
for primary_key in primary_keys
- for aio_task in self.upload_aiotasks_map[primary_key]
+ for aio_task in self.upload_aiotasks_map.get(primary_key, ())
if not aio_task.done()
]
)
diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.py
index dd3c7cb11..9dce33147 100644
--- a/skyvern/forge/sdk/routes/sdk.py
+++ b/skyvern/forge/sdk/routes/sdk.py
@@ -266,6 +266,14 @@ async def run_sdk_action(
)
raise
finally:
+ # This route runs the whole mini-task in-process, so nothing in the worker
+ # lifecycle ever drains its artifact uploads / step archives or removes its
+ # workflow-run context — without this, every call leaks them (SKY-12524).
+ try:
+ await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([task.task_id])
+ except Exception:
+ LOG.warning("Failed to drain artifact uploads for SDK action", task_id=task.task_id, exc_info=True)
+ app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context(workflow_run.workflow_run_id)
skyvern_context.reset()
return RunSdkActionResponse(
diff --git a/skyvern/forge/sdk/routes/streaming/channels/cdp.py b/skyvern/forge/sdk/routes/streaming/channels/cdp.py
index 9d534e505..da668658a 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/cdp.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/cdp.py
@@ -185,22 +185,36 @@ class CdpChannel:
async def close(self) -> None:
LOG.info(f"{self.class_name} closing connection", **self.identity)
- if self.browser:
- try:
- await self.browser.close()
- except Exception:
- pass
- self.browser = None
-
- if self.pw:
- await self.pw.stop()
- self.pw = None
-
- self.browser_context = None
- self.page = None
+ try:
+ if self.browser:
+ try:
+ await self.browser.close()
+ except Exception:
+ pass
+ self.browser = None
+ finally:
+ # Release the driver even when browser.close() raised on a dead target;
+ # a skipped stop() orphans the node subprocess for the process lifetime.
+ if self.pw:
+ try:
+ await self.pw.stop()
+ except Exception:
+ LOG.warning(f"{self.class_name} failed to stop playwright driver", **self.identity, exc_info=True)
+ self.pw = None
+
+ self.browser_context = None
+ self.page = None
LOG.info(f"{self.class_name} closed", **self.identity)
+ async def stop(self) -> t.Self:
+ """Terminal close: unlike close() (which connect() reuses to recycle a dropped
+ connection), stop() marks the channel closing so the browser "disconnected"
+ callback registered in connect() cannot resurrect a fresh driver."""
+ self._closing = True
+ await self.close()
+ return self
+
async def evaluate_js(
self,
expression: str,
diff --git a/skyvern/forge/sdk/routes/streaming/channels/execution.py b/skyvern/forge/sdk/routes/streaming/channels/execution.py
index 40763730a..8f3cc2714 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/execution.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/execution.py
@@ -239,21 +239,6 @@ class ExecutionChannel(CdpChannel):
return ""
return f"{parsed.scheme}://{parsed.netloc}"
- async def close(self) -> None:
- LOG.info(f"{self.class_name} closing connection", **self.identity)
-
- if self.browser:
- await self.browser.close()
- self.browser = None
- self.browser_context = None
- self.page = None
-
- if self.pw:
- await self.pw.stop()
- self.pw = None
-
- LOG.info(f"{self.class_name} closed", **self.identity)
-
class LocalExecutionChannel(ExecutionChannel):
def __init__(self, *, page: Page) -> None: # type: ignore[override]
@@ -360,4 +345,6 @@ async def execution_channel(vnc_channel: VncChannel) -> t.AsyncIterator[Executio
yield channel
finally:
- await channel.close()
+ # stop(), not close(): a bare close() fires the browser "disconnected" event and
+ # the reconnect callback resurrects a driver nothing will ever release (SKY-12524).
+ await channel.stop()
diff --git a/tests/unit/forge/sdk/routes/streaming/test_execution_channel.py b/tests/unit/forge/sdk/routes/streaming/test_execution_channel.py
new file mode 100644
index 000000000..137d5605c
--- /dev/null
+++ b/tests/unit/forge/sdk/routes/streaming/test_execution_channel.py
@@ -0,0 +1,74 @@
+"""Regression tests for the ExecutionChannel driver lifecycle (SKY-12524)."""
+
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import MagicMock
+
+import pytest
+from playwright._impl._errors import TargetClosedError
+
+from skyvern.forge.sdk.routes.streaming.channels.execution import ExecutionChannel, execution_channel
+from tests.unit.forge.sdk.routes.streaming.test_exfiltration_channel import (
+ _FakePw,
+ _make_vnc_channel,
+ _patch_pw_stack,
+)
+
+
+async def _drain_loop() -> None:
+ for _ in range(30):
+ await asyncio.sleep(0)
+
+
+class _RaisingBrowser:
+ def __init__(self) -> None:
+ self.close_calls = 0
+
+ async def close(self) -> None:
+ self.close_calls += 1
+ raise TargetClosedError("target already dead")
+
+
+async def test_execution_channel_cm_does_not_resurrect_driver(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Playwright fires "disconnected" during an intentional browser.close(); the
+ # on_close handler in CdpChannel.connect must not chain a fresh driver spawn.
+ state = _patch_pw_stack(monkeypatch, fire_disconnect_on_close=True)
+
+ async with execution_channel(_make_vnc_channel()):
+ pass
+ await _drain_loop()
+
+ assert state.start_calls == 1
+ assert state.pws[0].stopped is True
+
+
+async def test_stop_releases_driver_when_browser_close_raises() -> None:
+ channel = ExecutionChannel(vnc_channel=_make_vnc_channel())
+ browser = _RaisingBrowser()
+ pw = _FakePw()
+ channel.browser = browser # type: ignore[assignment]
+ channel.pw = pw # type: ignore[assignment]
+
+ await channel.stop()
+
+ assert browser.close_calls == 1
+ assert pw.stopped is True
+ assert channel.browser is None
+ assert channel.pw is None
+ assert channel._closing is True
+
+
+async def test_local_execution_channel_stop_is_safe() -> None:
+ from skyvern.forge.sdk.routes.streaming.channels.execution import LocalExecutionChannel
+
+ page = MagicMock()
+ page.context = MagicMock()
+ channel = LocalExecutionChannel(page=page)
+
+ await channel.stop()
+
+ assert channel._closing is True
diff --git a/tests/unit/forge/sdk/test_artifact_manager.py b/tests/unit/forge/sdk/test_artifact_manager.py
new file mode 100644
index 000000000..802fca975
--- /dev/null
+++ b/tests/unit/forge/sdk/test_artifact_manager.py
@@ -0,0 +1,100 @@
+import asyncio
+
+import pytest
+
+from skyvern.forge.sdk.artifact.manager import ArtifactManager
+
+
+async def _noop() -> None:
+ return None
+
+
+async def _boom() -> None:
+ raise RuntimeError("upload failed")
+
+
+async def _drain_loop() -> None:
+ for _ in range(10):
+ await asyncio.sleep(0)
+
+
+async def test_upload_task_self_discards_on_success() -> None:
+ manager = ArtifactManager()
+ task = asyncio.create_task(_noop())
+
+ manager._track_upload_aiotask("tsk_1", task)
+ assert manager.upload_aiotasks_map["tsk_1"] == [task]
+
+ await task
+ await _drain_loop()
+
+ assert "tsk_1" not in manager.upload_aiotasks_map
+
+
+async def test_upload_task_self_discards_on_failure_and_retrieves_exception() -> None:
+ manager = ArtifactManager()
+ task = asyncio.create_task(_boom())
+
+ manager._track_upload_aiotask("tsk_1", task)
+ with pytest.raises(RuntimeError):
+ await task
+ await _drain_loop()
+
+ # The failed task must not stay pinned in the map (its traceback retains the artifact bytes).
+ assert "tsk_1" not in manager.upload_aiotasks_map
+
+
+async def test_upload_task_discard_keeps_other_tasks_for_same_key() -> None:
+ manager = ArtifactManager()
+ done_task = asyncio.create_task(_noop())
+ pending_task = asyncio.create_task(asyncio.sleep(30))
+
+ manager._track_upload_aiotask("tsk_1", done_task)
+ manager._track_upload_aiotask("tsk_1", pending_task)
+ await done_task
+ await _drain_loop()
+
+ assert manager.upload_aiotasks_map["tsk_1"] == [pending_task]
+
+ pending_task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await pending_task
+ await _drain_loop()
+ assert "tsk_1" not in manager.upload_aiotasks_map
+
+
+async def test_wait_for_upload_aiotasks_still_barriers_pending_uploads() -> None:
+ manager = ArtifactManager()
+ started = asyncio.Event()
+ release = asyncio.Event()
+
+ async def _slow_upload() -> None:
+ started.set()
+ await release.wait()
+
+ task = asyncio.create_task(_slow_upload())
+ manager._track_upload_aiotask("tsk_1", task)
+ await started.wait()
+
+ waiter = asyncio.create_task(manager.wait_for_upload_aiotasks(["tsk_1"]))
+ await _drain_loop()
+ assert not waiter.done()
+
+ release.set()
+ await waiter
+
+ assert task.done()
+ assert "tsk_1" not in manager.upload_aiotasks_map
+
+
+async def test_wait_for_upload_aiotasks_does_not_autovivify_keys() -> None:
+ manager = ArtifactManager()
+
+ await manager.wait_for_upload_aiotasks(["tsk_never_seen"])
+
+ assert "tsk_never_seen" not in manager.upload_aiotasks_map
diff --git a/tests/unit/test_code_block_recording_artifact.py b/tests/unit/test_code_block_recording_artifact.py
index ade8437a6..c977ebf50 100644
--- a/tests/unit/test_code_block_recording_artifact.py
+++ b/tests/unit/test_code_block_recording_artifact.py
@@ -308,9 +308,15 @@ class TestUpdateArtifactDataScopeFallback:
patch(f"{_MANAGER_PATH}.STORAGE.store_artifact", AsyncMock()),
):
await manager.update_artifact_data(artifact_id="a_recording", organization_id="o_1", data=b"video")
- await asyncio.gather(*manager.upload_aiotasks_map["wrb_1"])
- assert list(manager.upload_aiotasks_map.keys()) == ["wrb_1"]
+ # Observe the keying while the upload is still pending: completed tasks
+ # self-discard from the map (SKY-12524).
+ assert list(manager.upload_aiotasks_map.keys()) == ["wrb_1"]
+
+ await asyncio.gather(*list(manager.upload_aiotasks_map["wrb_1"]))
+ await asyncio.sleep(0)
+
+ assert "wrb_1" not in manager.upload_aiotasks_map
@pytest.mark.asyncio
async def test_prefers_task_id_when_present(self) -> None:
@@ -321,10 +327,14 @@ class TestUpdateArtifactDataScopeFallback:
patch(f"{_MANAGER_PATH}.STORAGE.store_artifact", AsyncMock()),
):
await manager.update_artifact_data(artifact_id="a_recording", organization_id="o_1", data=b"video")
- await asyncio.gather(*manager.upload_aiotasks_map["tsk_1"])
- assert "tsk_1" in manager.upload_aiotasks_map
- assert "wrb_1" not in manager.upload_aiotasks_map
+ assert "tsk_1" in manager.upload_aiotasks_map
+ assert "wrb_1" not in manager.upload_aiotasks_map
+
+ await asyncio.gather(*list(manager.upload_aiotasks_map["tsk_1"]))
+ await asyncio.sleep(0)
+
+ assert "tsk_1" not in manager.upload_aiotasks_map
@pytest.mark.asyncio
async def test_raises_when_no_scope_id_available(self) -> None: