Skip to content

Automate browser based workflows with AI

Turn off skyvern.cdp_proxy.* metric export to SigNoz (#8251)

Shuchang Zheng64efcfd6f994
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 0e37a791a..ca3d107a6 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -3512,6 +3512,13 @@ class ForgeAgent:
screenshot_base64 = base64.b64encode(scraped_page.screenshots[0]).decode("utf-8")
if last_call_id is None:
current_context = skyvern_context.ensure_context()
+ reasoning = reasonings[0].summary[0].text if reasonings and reasonings[0].summary else None
+ assistant_message: str | None = None
+ if assistant_messages:
+ assistant_message = "\n".join(
+ part.text for part in assistant_messages[-1].content if getattr(part, "text", None)
+ )
+ cua_question = assistant_message
resp_content = None
if task.task_id in current_context.totp_codes:
verification_code = current_context.totp_codes[task.task_id]
@@ -3524,8 +3531,6 @@ class ForgeAgent:
resp_content = f"Here is the verification code: {verification_code}"
else:
# try address the conversation with the context we have
- reasoning = reasonings[0].summary[0].text if reasonings and reasonings[0].summary else None
- assistant_message = assistant_messages[0].content[0].text if assistant_messages else None
skyvern_repsonse_prompt = load_prompt_with_elements(
element_tree_builder=scraped_page,
prompt_engine=prompt_engine,
@@ -3542,14 +3547,23 @@ class ForgeAgent:
system_prompt=task.workflow_system_prompt,
)
LOG.info("Skyvern response to CUA question", skyvern_response=skyvern_response)
+ cua_question = cua_question or skyvern_response.get("question_or_decision")
resp_content = skyvern_response.get("answer")
if not resp_content:
resp_content = "I don't know. Can you help me make the best decision to achieve the goal?"
else:
resp_content = f"{resp_content}\n\n{CUA_EXECUTE_ACTIONS_DIRECTIVE}"
+ # The Computer tool rejects a user-supplied input_image when the request also threads a
+ # previous response, so this visual turn must be a fresh, self-contained request. Restate
+ # the navigation goal and the question being answered so a context-dependent answer (e.g.
+ # "yes", "the second option") is not detached from what it responds to.
+ continuation_sections = [f"Task: {task.navigation_goal}"]
+ if cua_question:
+ continuation_sections.append(f"Question or decision to address: {cua_question}")
+ continuation_sections.append(resp_content)
+ continuation_content = "\n\n".join(continuation_sections)
current_response = await app.OPENAI_CLIENT.responses.create(
model=cua_model,
- previous_response_id=previous_response.id,
tools=[
{
"type": "computer_use_preview",
@@ -3562,7 +3576,7 @@ class ForgeAgent:
{
"role": "user",
"content": [
- {"type": "input_text", "text": resp_content},
+ {"type": "input_text", "text": continuation_content},
{
"type": "input_image",
"image_url": f"data:image/png;base64,{screenshot_base64}",
diff --git a/skyvern/forge/sdk/routes/streaming/channels/cdp.py b/skyvern/forge/sdk/routes/streaming/channels/cdp.py
index 50eb53f1b..dee210ae2 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/cdp.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/cdp.py
@@ -27,6 +27,7 @@ from skyvern.webeye.cdp_connection import (
redact_cdp_url,
resolve_local_pbs_cdp_url,
)
+from skyvern.webeye.driver_connection import close_driver_connection_on_transport_loss
from skyvern.webeye.main_world_eval import evaluate_in_main_world
if t.TYPE_CHECKING:
@@ -117,7 +118,10 @@ class CdpChannel:
LOG.info(f"{self.class_name} connecting to CDP", **self.identity)
- pw = self.pw or await async_playwright().start()
+ pw = self.pw
+ if pw is None:
+ pw = await async_playwright().start()
+ close_driver_connection_on_transport_loss(pw)
self.pw = pw
diff --git a/skyvern/forge/sdk/routes/streaming/client_disconnect.py b/skyvern/forge/sdk/routes/streaming/client_disconnect.py
new file mode 100644
index 000000000..0c1feb9a1
--- /dev/null
+++ b/skyvern/forge/sdk/routes/streaming/client_disconnect.py
@@ -0,0 +1,29 @@
+"""Make a client disconnect observable to a send-only streaming websocket."""
+
+from __future__ import annotations
+
+import asyncio
+
+from fastapi import WebSocket
+
+
+async def _wait_for_client_disconnect(websocket: WebSocket) -> None:
+ while True:
+ try:
+ message = await websocket.receive()
+ except Exception:
+ # A receive that cannot complete means the socket is unusable, which is the same signal.
+ return
+ if message.get("type") == "websocket.disconnect":
+ return
+
+
+def watch_for_client_disconnect(websocket: WebSocket) -> asyncio.Task[None]:
+ """Task that completes once the client is gone. Callers must cancel it when they finish.
+
+ A handler that only sends never reads its socket, so Starlette never processes the
+ ``websocket.disconnect`` message ASGI already delivered: ``client_state`` stays CONNECTED and
+ ``send`` keeps writing into a transport the peer dropped, which asyncio warns about once per
+ write. Draining the receive side is what turns that into an observable event.
+ """
+ return asyncio.create_task(_wait_for_client_disconnect(websocket))
diff --git a/skyvern/forge/sdk/routes/streaming/screencast.py b/skyvern/forge/sdk/routes/streaming/screencast.py
index aa4ab6ba9..958a2b91e 100644
--- a/skyvern/forge/sdk/routes/streaming/screencast.py
+++ b/skyvern/forge/sdk/routes/streaming/screencast.py
@@ -10,11 +10,12 @@ import time
from collections.abc import Awaitable, Callable
import structlog
-from fastapi import WebSocket
+from fastapi import WebSocket, WebSocketDisconnect
from opentelemetry import metrics
from playwright.async_api import CDPSession
from skyvern.forge import app
+from skyvern.forge.sdk.routes.streaming.client_disconnect import watch_for_client_disconnect
from skyvern.webeye.browser_state import BrowserState
LOG = structlog.get_logger()
@@ -445,6 +446,7 @@ async def start_screencast_loop(
max_poll_interval_seconds=ACTIVE_PAGE_MAX_POLL_INTERVAL,
)
+ client_disconnected = False
try:
page = await _resolve_working_page(browser_state, entity_id, entity_type, workflow_run_id, organization_id)
if page is None:
@@ -455,11 +457,13 @@ async def start_screencast_loop(
forward_task = asyncio.create_task(_frame_forwarding_loop())
poll_task = asyncio.create_task(_completion_polling_loop())
page_monitor_task = asyncio.create_task(_active_page_monitor_loop())
+ disconnect_task = watch_for_client_disconnect(websocket)
done, pending = await asyncio.wait(
- [forward_task, poll_task, page_monitor_task],
+ [forward_task, poll_task, page_monitor_task, disconnect_task],
return_when=asyncio.FIRST_COMPLETED,
)
+ client_disconnected = disconnect_task in done
for task in pending:
task.cancel()
try:
@@ -482,3 +486,6 @@ async def start_screencast_loop(
finally:
await _stop_current_screencast()
LOG.info("CDP screencast cleaned up", entity_id=entity_id, entity_type=entity_type)
+ # Raised after cleanup so the caller's disconnect branch runs instead of its final-status write.
+ if client_disconnected:
+ raise WebSocketDisconnect()
diff --git a/skyvern/forge/sdk/routes/streaming/screenshot.py b/skyvern/forge/sdk/routes/streaming/screenshot.py
index 1c098b128..3f43fc8cc 100644
--- a/skyvern/forge/sdk/routes/streaming/screenshot.py
+++ b/skyvern/forge/sdk/routes/streaming/screenshot.py
@@ -24,6 +24,7 @@ from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router
+from skyvern.forge.sdk.routes.streaming.client_disconnect import watch_for_client_disconnect
from skyvern.forge.sdk.routes.streaming.screencast import (
release_browser_state,
start_screencast_loop,
@@ -80,8 +81,12 @@ async def task_stream(
# timestamp last time when streaming activity happens
last_activity_timestamp = datetime.utcnow()
+ disconnected = watch_for_client_disconnect(websocket)
try:
while True:
+ if disconnected.done():
+ LOG.info("Client disconnected. Closing connection", task_id=task_id, organization_id=organization_id)
+ return
# if no activity for 5 minutes, close the connection
if (datetime.utcnow() - last_activity_timestamp).total_seconds() > STREAMING_TIMEOUT:
LOG.info(
@@ -155,6 +160,8 @@ async def task_stream(
except Exception:
LOG.warning("Error while streaming", task_id=task_id, organization_id=organization_id, exc_info=True)
return
+ finally:
+ disconnected.cancel()
LOG.info("WebSocket connection closed successfully", task_id=task_id, organization_id=organization_id)
return
@@ -203,8 +210,16 @@ async def workflow_run_streaming(
# timestamp last time when streaming activity happens
last_activity_timestamp = datetime.utcnow()
+ disconnected = watch_for_client_disconnect(websocket)
try:
while True:
+ if disconnected.done():
+ LOG.info(
+ "WofklowRun Streaming: Client disconnected. Closing connection",
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ )
+ return
# if no activity for 5 minutes, close the connection
if (datetime.utcnow() - last_activity_timestamp).total_seconds() > STREAMING_TIMEOUT:
LOG.info(
@@ -302,6 +317,8 @@ async def workflow_run_streaming(
exc_info=True,
)
return
+ finally:
+ disconnected.cancel()
LOG.info(
"WofklowRun Streaming: WebSocket connection closed successfully",
workflow_run_id=workflow_run_id,
diff --git a/skyvern/webeye/browser_engine.py b/skyvern/webeye/browser_engine.py
index faeae0b97..6791c32b4 100644
--- a/skyvern/webeye/browser_engine.py
+++ b/skyvern/webeye/browser_engine.py
@@ -39,6 +39,7 @@ from skyvern.webeye.browser_errors import (
BrowserErrorFamiliesConfigError,
classify_browser_error,
)
+from skyvern.webeye.driver_connection import close_driver_connection_on_transport_loss
if TYPE_CHECKING:
from skyvern.forge.sdk.schemas.tasks import Task
@@ -345,7 +346,9 @@ class BrowserEngineRegistry:
async def _start_stock_driver() -> Playwright:
- return await async_playwright().start()
+ driver = await async_playwright().start()
+ close_driver_connection_on_transport_loss(driver)
+ return driver
def _stock_error_types() -> tuple[type[BaseException], type[BaseException]]:
@@ -384,7 +387,9 @@ def _stock_rich_error_types() -> BrowserEngineRichErrorTypes:
async def _start_rustwright_driver() -> Playwright:
from rustwright.async_api import async_playwright as rustwright_async_playwright
- return await rustwright_async_playwright().start()
+ driver = await rustwright_async_playwright().start()
+ close_driver_connection_on_transport_loss(driver)
+ return driver
def _rustwright_error_types() -> tuple[type[BaseException], type[BaseException]]:
diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py
index 75e558069..d4b358d5f 100644
--- a/skyvern/webeye/browser_factory.py
+++ b/skyvern/webeye/browser_factory.py
@@ -15,7 +15,7 @@ from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
-from typing import Any, Awaitable, Callable, Protocol, cast
+from typing import Any, Awaitable, Callable, Literal, Protocol, cast
from urllib.parse import parse_qsl, urlparse
import psutil
@@ -75,6 +75,11 @@ LOG = structlog.get_logger()
BrowserCleanupFunc = Callable[[], Awaitable[None]] | None
+# Playwright defaults a ".har" path to content="embed", which holds every response body in the driver's
+# JS heap for the life of the context and serializes them all in one JSON.stringify at close, so long runs
+# blow the ~3 GB V8 heap there and the driver dies with every task on it. "omit" keeps the HAR bounded by
+# request count instead.
+HAR_CONTENT_POLICY: Literal["omit"] = "omit"
# Header to signal fresh browser context creation (stripped before sending to websites)
# When set to "true", creates a new incognito-like context instead of reusing existing ones
FRESH_CONTEXT_HEADER = "X-Skyvern-Fresh-Context"
@@ -697,6 +702,7 @@ class BrowserContextFactory:
"--enable-automation",
],
"record_har_path": har_dir,
+ "record_har_content": HAR_CONTENT_POLICY,
"record_video_dir": video_dir,
"viewport": {
"width": settings.BROWSER_WIDTH,
diff --git a/skyvern/webeye/driver_connection.py b/skyvern/webeye/driver_connection.py
new file mode 100644
index 000000000..d41a84c79
--- /dev/null
+++ b/skyvern/webeye/driver_connection.py
@@ -0,0 +1,51 @@
+"""Make a Playwright driver whose transport died report itself closed.
+
+Playwright sets ``Connection._closed_error`` only from an explicit ``stop()``. When the driver
+subprocess dies on its own, its read loop completes ``transport.on_error_future`` and nothing else
+happens, so every retained page/context keeps passing the send guard in ``_send_message_to_server``:
+each call writes into the dead pipe (asyncio warns once per write past its fifth) and then awaits a
+reply that can never arrive. Completing the same latch ``stop()`` uses turns those calls into an
+immediate ``TargetClosedError``, which the callers already handle.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import structlog
+
+LOG = structlog.get_logger()
+
+_ARMED_FLAG = "_skyvern_transport_loss_armed"
+
+
+def close_driver_connection_on_transport_loss(driver: Any) -> None:
+ """Arm ``driver`` so a dead transport marks its connection closed. Idempotent, and a no-op on an
+ engine that is a Playwright driver by duck type only, with no connection/transport underneath."""
+ connection = getattr(driver, "_connection", None)
+ transport = getattr(connection, "_transport", None)
+ on_error_future = getattr(transport, "on_error_future", None)
+ cleanup = getattr(connection, "cleanup", None)
+ if not isinstance(on_error_future, asyncio.Future) or not callable(cleanup):
+ return
+ if getattr(connection, _ARMED_FLAG, False):
+ return
+ setattr(connection, _ARMED_FLAG, True)
+
+ def _mark_closed(finished: asyncio.Future) -> None:
+ if finished.cancelled():
+ return
+ # Retrieving the exception here is also what keeps a lost driver from surfacing as an
+ # orphaned "Future exception was never retrieved" record.
+ error = finished.exception()
+ if getattr(connection, "_closed_error", None) is not None:
+ return
+ cleanup(f"Playwright driver transport closed: {error}")
+ LOG.warning(
+ "Playwright driver transport closed; failing further calls on this driver",
+ error_type=type(error).__name__ if error is not None else None,
+ error_message=str(error)[:200] if error is not None else None,
+ )
+
+ on_error_future.add_done_callback(_mark_closed)
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index b718c27e1..a4b144140 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -36,6 +36,7 @@ from skyvern.webeye.browser_factory import BrowserCleanupFunc, BrowserContextFac
from skyvern.webeye.browser_health import BrowserOperation
from skyvern.webeye.browser_state import BLANK_PAGE_URLS, BrowserState
from skyvern.webeye.cdp_download_interceptor import disable_download_interceptor_for_context
+from skyvern.webeye.driver_connection import close_driver_connection_on_transport_loss
from skyvern.webeye.navigation import is_permanent_navigation_error, navigate_with_retry
from skyvern.webeye.scraper import scraper
from skyvern.webeye.scraper.scraped_page import CleanupElementTreeFunc, ScrapedPage, ScrapeExcludeFunc
@@ -659,6 +660,7 @@ class RealBrowserState(BrowserState):
self.pw = await self.engine_selection.start_driver()
else:
self.pw = await async_playwright().start()
+ close_driver_connection_on_transport_loss(self.pw)
try:
await self.check_and_fix_state(
proxy_location=proxy_location,
diff --git a/tests/unit/forge/sdk/routes/streaming/test_screenshot_stream_disconnect.py b/tests/unit/forge/sdk/routes/streaming/test_screenshot_stream_disconnect.py
new file mode 100644
index 000000000..183e55fdd
--- /dev/null
+++ b/tests/unit/forge/sdk/routes/streaming/test_screenshot_stream_disconnect.py
@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+from typing import cast
+from unittest.mock import AsyncMock
+
+import pytest
+from fastapi import WebSocket
+
+from skyvern.forge import app
+from skyvern.forge.sdk.routes.streaming import screenshot
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+
+
+class _DisconnectedWebSocket:
+ """A client that is already gone but whose sends still succeed.
+
+ That is the production shape: ASGI delivered ``websocket.disconnect``, but a send-only handler
+ never reads it, so Starlette keeps accepting sends and asyncio writes them into a dead socket.
+ """
+
+ def __init__(self) -> None:
+ self.sent: list[dict] = []
+ self._disconnect_delivered = False
+
+ async def accept(self) -> None:
+ return None
+
+ async def send_text(self, text: str) -> None:
+ return None
+
+ async def send_json(self, data: dict) -> None:
+ self.sent.append(data)
+
+ async def receive(self) -> dict:
+ if not self._disconnect_delivered:
+ self._disconnect_delivered = True
+ return {"type": "websocket.disconnect", "code": 1006}
+ await asyncio.Event().wait()
+ raise AssertionError("unreachable")
+
+
+async def test_workflow_run_streaming_stops_sending_once_the_client_is_gone(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # SKY-14645: this loop used to keep send_json-ing every 2s until the run finalized, emitting one
+ # stdlib "socket.send() raised exception." per write for the whole remaining run.
+ websocket = _DisconnectedWebSocket()
+ workflow_run = SimpleNamespace(
+ status=WorkflowRunStatus.running,
+ organization_id="o_1",
+ )
+ database = SimpleNamespace(
+ workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(return_value=workflow_run)),
+ )
+
+ monkeypatch.setattr(screenshot.settings, "BROWSER_STREAMING_MODE", "vnc")
+ monkeypatch.setattr(screenshot, "get_current_org", AsyncMock(return_value=SimpleNamespace(organization_id="o_1")))
+ monkeypatch.setattr(app, "DATABASE", database)
+ monkeypatch.setattr(app, "STORAGE", SimpleNamespace(get_streaming_file=AsyncMock(return_value=b"jpeg")))
+ monkeypatch.setattr(app, "AGENT_FUNCTION", SimpleNamespace(mark_streaming_viewer_active=AsyncMock()))
+
+ await asyncio.wait_for(
+ screenshot.workflow_run_streaming(
+ websocket=cast(WebSocket, websocket),
+ workflow_run_id="wr_1",
+ apikey="key",
+ ),
+ timeout=10,
+ )
+
+ assert len(websocket.sent) <= 1
diff --git a/tests/unit/test_driver_connection.py b/tests/unit/test_driver_connection.py
new file mode 100644
index 000000000..86a69cd26
--- /dev/null
+++ b/tests/unit/test_driver_connection.py
@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+from typing import Any, cast
+
+import pytest
+from playwright._impl._connection import Connection
+from playwright._impl._errors import TargetClosedError
+from playwright._impl._transport import Transport
+
+from skyvern.webeye.driver_connection import close_driver_connection_on_transport_loss
+
+
+class _RecordingTransport(Transport):
+ def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+ super().__init__(loop)
+ self.sent: list[dict] = []
+
+ def request_stop(self) -> None:
+ return None
+
+ async def wait_until_stopped(self) -> None:
+ return None
+
+ async def connect(self) -> None:
+ return None
+
+ async def run(self) -> None:
+ return None
+
+ def send(self, message: dict) -> None:
+ self.sent.append(message)
+
+
+def _connection_with(transport: Transport, loop: asyncio.AbstractEventLoop) -> Connection:
+ connection = Connection(None, lambda *args: None, transport, loop)
+ connection._api_zone.set(cast(Any, {"apiName": "test", "frames": [], "title": None}))
+ return connection
+
+
+async def test_a_lost_driver_transport_stops_further_writes_and_unblocks_pending_calls() -> None:
+ # SKY-14645: Playwright sets `_closed_error` only from an explicit stop(), so a driver that dies
+ # on its own leaves every retained handle writing into the closed pipe (asyncio warns once per
+ # write) while its reply never arrives.
+ loop = asyncio.get_running_loop()
+ transport = _RecordingTransport(loop)
+ connection = _connection_with(transport, loop)
+ owner = cast(Any, SimpleNamespace(_guid="root", _was_collected=False))
+
+ close_driver_connection_on_transport_loss(SimpleNamespace(_connection=connection))
+
+ in_flight = connection._send_message_to_server(owner, "Browser.newContext", {})
+ assert len(transport.sent) == 1
+
+ transport.on_error_future.set_exception(Exception("Connection closed while reading from the driver"))
+ await asyncio.sleep(0)
+
+ assert isinstance(in_flight.future.exception(), TargetClosedError)
+ with pytest.raises(TargetClosedError):
+ connection._send_message_to_server(owner, "Browser.newContext", {})
+ assert len(transport.sent) == 1
+
+
+async def test_arming_a_driver_without_playwright_internals_is_a_no_op() -> None:
+ # An attach-only engine is a driver by duck type only; it exposes no connection/transport to arm.
+ close_driver_connection_on_transport_loss(SimpleNamespace())
+ close_driver_connection_on_transport_loss(SimpleNamespace(_connection=SimpleNamespace()))
diff --git a/tests/unit/test_generate_cua_actions_visual_continuation.py b/tests/unit/test_generate_cua_actions_visual_continuation.py
index 85c1f7099..147100d11 100644
--- a/tests/unit/test_generate_cua_actions_visual_continuation.py
+++ b/tests/unit/test_generate_cua_actions_visual_continuation.py
@@ -24,6 +24,7 @@ from tests.unit.helpers import make_organization, make_step, make_task
SCREENSHOT_BYTES = b"\x89PNG\r\n\x1a\nfake-screenshot-bytes"
HELPER_ANSWER = "I'll scroll down to see the rest of the page."
+HELPER_QUESTION = "Which download icon should I click to export the report?"
def _fake_usage() -> SimpleNamespace:
@@ -55,15 +56,22 @@ def _flatten_input_message(create_input: list[dict[str, Any]]) -> tuple[list[str
async def _invoke_no_call_continuation(
- monkeypatch: pytest.MonkeyPatch, helper_answer: str | None = HELPER_ANSWER
-) -> AsyncMock:
+ monkeypatch: pytest.MonkeyPatch,
+ helper_answer: str | None = HELPER_ANSWER,
+ totp_code: str | None = None,
+ assistant_texts: list[str] | None = None,
+) -> tuple[AsyncMock, Any]:
agent = ForgeAgent()
now = datetime.now(UTC)
organization = make_organization(now)
task = make_task(now, organization)
step = make_step(now, task, step_id="cua-step", status=StepStatus.created, order=0, output=None)
- previous_response = SimpleNamespace(id="resp_prev", output=[])
+ output = [
+ SimpleNamespace(type="message", role="assistant", content=[SimpleNamespace(type="text", text=text)])
+ for text in (assistant_texts or [])
+ ]
+ previous_response = SimpleNamespace(id="resp_prev", output=output)
scraped_page = SimpleNamespace(screenshots=[SCREENSHOT_BYTES], window_dimension=None)
create_mock = AsyncMock(return_value=SimpleNamespace(id="resp_new", output=[], usage=_fake_usage()))
@@ -81,29 +89,26 @@ async def _invoke_no_call_continuation(
)
monkeypatch.setattr(agent_module, "load_prompt_with_elements", MagicMock(return_value="prompt"))
- monkeypatch.setattr(
- agent_module.skyvern_context,
- "ensure_context",
- lambda: SimpleNamespace(totp_codes={}),
- )
+ context = SimpleNamespace(totp_codes={task.task_id: totp_code} if totp_code else {})
+ monkeypatch.setattr(agent_module.skyvern_context, "ensure_context", lambda: context)
monkeypatch.setattr(
agent_module,
"get_org_aware_primary_llm_api_handler",
- lambda default=None: AsyncMock(return_value={"answer": helper_answer}),
+ lambda default=None: AsyncMock(return_value={"answer": helper_answer, "question_or_decision": HELPER_QUESTION}),
)
monkeypatch.setattr(agent_module, "parse_cua_actions", AsyncMock(return_value=[]))
await agent._generate_cua_actions(
task=task, step=step, scraped_page=scraped_page, previous_response=previous_response
)
- return create_mock
+ return create_mock, task
@pytest.mark.asyncio
async def test_no_computer_call_continuation_includes_answer_and_screenshot(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- create_mock = await _invoke_no_call_continuation(monkeypatch)
+ create_mock, _ = await _invoke_no_call_continuation(monkeypatch)
create_mock.assert_awaited_once()
create_input = create_mock.await_args.kwargs["input"]
@@ -115,6 +120,53 @@ async def test_no_computer_call_continuation_includes_answer_and_screenshot(
assert expected_data_url in image_urls
+async def test_no_computer_call_continuation_omits_previous_response_id_and_includes_goal(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The no-call visual turn is a fresh request: it must not thread `previous_response_id`
+ (the Computer tool rejects a user-supplied input_image alongside a previous response), and it
+ must carry the original navigation goal so action generation keeps its context."""
+ create_mock, task = await _invoke_no_call_continuation(monkeypatch)
+
+ kwargs = create_mock.await_args.kwargs
+ assert "previous_response_id" not in kwargs
+
+ texts, _ = _flatten_input_message(kwargs["input"])
+ assert any(task.navigation_goal in text for text in texts)
+ assert any(HELPER_ANSWER in text for text in texts)
+ # The answered question must travel with the answer so a context-dependent reply stays anchored.
+ assert any(HELPER_QUESTION in text for text in texts)
+
+
+async def test_fresh_continuation_anchors_on_latest_assistant_question(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A multi-message no-call response must anchor on the model's latest question, not the first,
+ so the helper's answer is attached to what CUA actually asked last."""
+ create_mock, _ = await _invoke_no_call_continuation(
+ monkeypatch, assistant_texts=["An earlier aside.", "The final question to answer?"]
+ )
+
+ texts, _ = _flatten_input_message(create_mock.await_args.kwargs["input"])
+ assert any("The final question to answer?" in text for text in texts)
+
+
+async def test_totp_continuation_is_self_contained(monkeypatch: pytest.MonkeyPatch) -> None:
+ create_mock, task = await _invoke_no_call_continuation(monkeypatch, totp_code="123456")
+
+ kwargs = create_mock.await_args.kwargs
+ assert "previous_response_id" not in kwargs
+
+ texts, image_urls = _flatten_input_message(kwargs["input"])
+ assert any(task.navigation_goal in text for text in texts)
+ assert any("123456" in text for text in texts)
+ assert all(agent_module.CUA_EXECUTE_ACTIONS_DIRECTIVE not in text for text in texts)
+
+ expected_data_url = f"data:image/png;base64,{base64.b64encode(SCREENSHOT_BYTES).decode('utf-8')}"
+ assert expected_data_url in image_urls
+
+
async def _capture_cua_tools(
monkeypatch: pytest.MonkeyPatch, window_dimension: dict[str, int] | None
) -> list[dict[str, Any]]:
@@ -175,7 +227,7 @@ async def test_cua_tool_falls_back_to_settings_without_a_live_viewport(monkeypat
@pytest.mark.asyncio
async def test_truthy_answer_appends_execute_directive(monkeypatch: pytest.MonkeyPatch) -> None:
- create_mock = await _invoke_no_call_continuation(monkeypatch)
+ create_mock, _ = await _invoke_no_call_continuation(monkeypatch)
create_input = create_mock.await_args.kwargs["input"]
texts, _ = _flatten_input_message(create_input)
@@ -186,7 +238,7 @@ async def test_truthy_answer_appends_execute_directive(monkeypatch: pytest.Monke
@pytest.mark.asyncio
async def test_no_answer_default_question_omits_directive(monkeypatch: pytest.MonkeyPatch) -> None:
- create_mock = await _invoke_no_call_continuation(monkeypatch, helper_answer="")
+ create_mock, _ = await _invoke_no_call_continuation(monkeypatch, helper_answer="")
create_input = create_mock.await_args.kwargs["input"]
texts, _ = _flatten_input_message(create_input)
@@ -231,9 +283,10 @@ async def test_computer_call_output_continuation_shape_unchanged(monkeypatch: py
task=task, step=step, scraped_page=scraped_page, previous_response=previous_response
)
- create_input = create_mock.await_args.kwargs["input"]
+ kwargs = create_mock.await_args.kwargs
+ assert kwargs["previous_response_id"] == "resp_prev"
expected_data_url = f"data:image/png;base64,{base64.b64encode(SCREENSHOT_BYTES).decode('utf-8')}"
- assert create_input == [
+ assert kwargs["input"] == [
{
"call_id": "call_123",
"type": "computer_call_output",
diff --git a/tests/unit_tests/test_streaming_screencast.py b/tests/unit_tests/test_streaming_screencast.py
index 430fd9e15..a34096a21 100644
--- a/tests/unit_tests/test_streaming_screencast.py
+++ b/tests/unit_tests/test_streaming_screencast.py
@@ -7,6 +7,7 @@ from types import SimpleNamespace
from unittest.mock import ANY, AsyncMock, Mock
import pytest
+from fastapi import WebSocketDisconnect
from tests.unit_tests._stub_streaming import import_with_stubs
@@ -246,6 +247,16 @@ class _FakePage:
self.viewport_size = {"width": 800, "height": 600}
+def _connected_websocket(send_json: object) -> SimpleNamespace:
+ """A real WebSocket is readable, and the screencast loop reads it to notice a client disconnect."""
+
+ async def _receive() -> dict:
+ await asyncio.Event().wait()
+ raise AssertionError("unreachable")
+
+ return SimpleNamespace(send_json=send_json, receive=_receive)
+
+
async def _wait_for(predicate: Callable[[], bool]) -> bool:
for _ in range(500):
if predicate():
@@ -271,7 +282,7 @@ async def test_forwarded_frame_is_acked_and_records_forward_latency(monkeypatch:
if len(sent) == 2:
raise RuntimeError("client gone")
- websocket = SimpleNamespace(send_json=_send_json)
+ websocket = _connected_websocket(_send_json)
async def _never_finalized() -> bool:
return False
@@ -297,6 +308,28 @@ async def test_forwarded_frame_is_acked_and_records_forward_latency(monkeypatch:
assert value >= 0 and attributes == {"entity_type": "browser_session"}
+async def test_client_disconnect_is_raised_so_the_caller_does_not_write_a_final_status(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ session = _FakeCdpSession()
+ monkeypatch.setattr(screencast, "_resolve_working_page", AsyncMock(return_value=_FakePage(session)))
+ websocket = SimpleNamespace(
+ send_json=AsyncMock(),
+ receive=AsyncMock(return_value={"type": "websocket.disconnect", "code": 1006}),
+ )
+
+ async def _never_finalized() -> bool:
+ return False
+
+ with pytest.raises(WebSocketDisconnect):
+ await asyncio.wait_for(
+ screencast.start_screencast_loop(websocket, object(), "pbs_1", "browser_session", _never_finalized),
+ timeout=5,
+ )
+ assert session.detached
+
+
@pytest.mark.asyncio
async def test_evicted_frame_records_queue_dwell(monkeypatch: pytest.MonkeyPatch) -> None:
session = _FakeCdpSession()
@@ -318,7 +351,7 @@ async def test_evicted_frame_records_queue_dwell(monkeypatch: pytest.MonkeyPatch
elif payload["screenshot"] == "frame-3":
raise RuntimeError("client gone")
- websocket = SimpleNamespace(send_json=_send_json)
+ websocket = _connected_websocket(_send_json)
async def _never_finalized() -> bool:
return False
@@ -390,7 +423,7 @@ async def test_a_page_that_cannot_be_attached_does_not_kill_the_running_stream(
if payload["screenshot"] == "frame-from-next-page":
raise RuntimeError("client gone")
- websocket = SimpleNamespace(send_json=_send_json)
+ websocket = _connected_websocket(_send_json)
async def _never_finalized() -> bool:
return False
@@ -449,7 +482,7 @@ async def test_a_frame_that_lands_mid_swap_is_still_acked(monkeypatch: pytest.Mo
await asyncio.wait_for(
screencast.start_screencast_loop(
- SimpleNamespace(send_json=_send_json), object(), "pbs_1", "browser_session", _never_finalized
+ _connected_websocket(_send_json), object(), "pbs_1", "browser_session", _never_finalized
),
timeout=5,
)
@@ -520,7 +553,7 @@ async def _drive_page_monitor(
task = asyncio.create_task(
screencast.start_screencast_loop(
- SimpleNamespace(send_json=AsyncMock()),
+ _connected_websocket(AsyncMock()),
object(),
entity_id,
entity_type,