Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9454 fix copilot timeout WIP rescue (#5789)
Andrew Neilsonf5535c93de86
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 3a94aa802..cefa0611c 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -325,6 +325,13 @@ def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | No)+def _build_cancelled_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:+ if ctx.copilot_total_timeout_exceeded:+ LOG.info("Copilot cancellation resolved as total timeout")+ return _build_timeout_exit_result(ctx, global_llm_context)+ return _build_cancel_exit_result(ctx, global_llm_context)++def _build_max_turns_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:return _build_wip_exit_result(ctx,@@ -686,8 +693,8 @@ async def run_copilot_agent(except asyncio.CancelledError:# Re-raising would leave the route with ``agent_result is None``# and skip its ``workflow_was_persisted`` rollback decision.- LOG.info("Copilot run cancelled by user")- return _build_cancel_exit_result(ctx, global_llm_context)+ LOG.info("Copilot run cancelled")+ return _build_cancelled_exit_result(ctx, global_llm_context)except MaxTurnsExceeded:return _build_max_turns_exit_result(ctx, global_llm_context)except CopilotTotalTimeoutError:diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 2b815a5b7..a2e17b3aa 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -160,6 +160,7 @@ class CopilotContext(AgentContext):post_update_nudge_count: int = 0coverage_nudge_count: int = 0format_nudge_count: int = 0+ copilot_total_timeout_exceeded: bool = Falseuser_message: str = ""# Tool trackingdiff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 3b22fda80..cd349a16d 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -280,6 +280,15 @@ class CopilotTotalTimeoutError(Exception):"""Raised when the copilot agent exceeds the total allowed runtime."""+def _mark_copilot_total_timeout(ctx: Any) -> None:+ ctx.copilot_total_timeout_exceeded = True+++def _mark_copilot_total_timeout_if_elapsed(ctx: Any, start_time: float) -> None:+ if time.monotonic() - start_time >= TOTAL_TIMEOUT_SECONDS:+ _mark_copilot_total_timeout(ctx)++class CopilotNonRetriableNavError(Exception):"""Raised from run_with_enforcement when the copilot's most recent runhit a permanent navigation error (DNS / cert / SSL / invalid URL) and@@ -1017,6 +1026,7 @@ async def _run_streamed_with_deadline(finally:_accumulate_usage(result, ctx)except asyncio.TimeoutError:+ _mark_copilot_total_timeout(ctx)LOG.warning("Copilot total timeout exceeded mid-iteration",elapsed_seconds=round(time.monotonic() - start_time, 3),@@ -1047,6 +1057,7 @@ async def run_with_enforcement(# chat history on the server side (see SKY-8986).elapsed = time.monotonic() - start_timeif elapsed > TOTAL_TIMEOUT_SECONDS:+ _mark_copilot_total_timeout(ctx)raise CopilotTotalTimeoutError()if iteration >= MAX_ITERATIONS:@@ -1079,6 +1090,9 @@ async def run_with_enforcement(start_time,iteration,)+ except asyncio.CancelledError:+ _mark_copilot_total_timeout_if_elapsed(ctx, start_time)+ raiseexcept Exception as e:if not _is_context_window_error(e):raise@@ -1098,7 +1112,11 @@ async def run_with_enforcement(iteration=iteration,has_session=session is not None,)- current_input, images_stripped = await _recover_from_context_overflow(session, current_input)+ try:+ current_input, images_stripped = await _recover_from_context_overflow(session, current_input)+ except asyncio.CancelledError:+ _mark_copilot_total_timeout_if_elapsed(ctx, start_time)+ raiseif images_stripped:# The agent could otherwise reason about the page from# memory on the next turn; warn it explicitly.@@ -1115,6 +1133,9 @@ async def run_with_enforcement(start_time,iteration,)+ except asyncio.CancelledError:+ _mark_copilot_total_timeout_if_elapsed(ctx, start_time)+ raiseexcept Exception as retry_err:# Never retry twice; even a second overflow surfaces as a# real failure rather than spinning.diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 31fefac76..2b0bb021d 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -15,6 +15,8 @@ from skyvern.forge.sdk.copilot.loop_detection import LOOP_DETECTED_MARKERif TYPE_CHECKING:from agents.result import RunResultStreaming+_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY = "_copilot_internal_run_cancelled_by_watchdog"+_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"_JPEG_PREFIX = b"\xff\xd8\xff"@@ -211,6 +213,7 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict]sanitized["data"] = datasanitized.pop("_workflow", None)+ sanitized.pop(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY, None)return sanitizeddiff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 3c5e03a95..fe53fbca5 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -75,6 +75,7 @@ class AgentContext:post_update_nudge_count: int = 0coverage_nudge_count: int = 0format_nudge_count: int = 0+ copilot_total_timeout_exceeded: bool = Falsefailed_test_nudge_count: int = 0explore_without_workflow_nudge_count: int = 0null_data_streak_count: int = 0diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 31ac42f9b..0b40c4a60 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -34,6 +34,7 @@ from skyvern.forge.sdk.copilot.narration import NarratorStatefrom skyvern.forge.sdk.copilot.narration import handler_available as narration_handler_availablefrom skyvern.forge.sdk.copilot.narration import narrator_poll_tickfrom skyvern.forge.sdk.copilot.output_utils import (+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,build_run_blocks_response,iter_failure_reasons,sanitize_tool_result_for_llm,@@ -1170,6 +1171,7 @@ async def _run_blocks_and_collect_debug(final_status: str | None = Nonerun: Any = initial_runexit_reason: WatchdogExitReason | None = None+ run_cancelled_by_watchdog = False# Quiet blocks (WAIT/TEXT_PROMPT/HUMAN_INTERACTION) legitimately have# DB-silent periods; disable stagnation for any invocation that includes# one. Safety ceiling still applies.@@ -1257,6 +1259,7 @@ async def _run_blocks_and_collect_debug(exit_reason = "success"else:await _cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id)+ run_cancelled_by_watchdog = Truerun = await _safe_read_workflow_run(workflow_run.workflow_run_id, ctx.organization_id, context="post-drain")@@ -1273,7 +1276,10 @@ async def _run_blocks_and_collect_debug(ctx.pending_reconciliation_run_id = workflow_run.workflow_run_idassert exit_reason is not None # narrows for mypy; outer check excludes "success" but not Noneerror_msg = await _watchdog_error_message(exit_reason, ctx, workflow_run.workflow_run_id, run)- return {"ok": False, "error": error_msg}+ result = {"ok": False, "error": error_msg}+ if run_cancelled_by_watchdog:+ result[_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY] = True+ return resultexcept asyncio.CancelledError:# The SDK's @function_tool(timeout=...) cancelled us mid-poll. Shield# the cleanup so the parent cancellation can't interrupt it mid-await.@@ -2097,7 +2103,11 @@ def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:run_ok = bool(result.get("ok", False))- copilot_ctx.last_test_ok = run_ok+ # Watchdog cancels normally count as ok=False; only a coincident total+ # timeout softens to ``None`` to keep the unvalidated WIP rescue open.+ cancelled_by_watchdog = result.get(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY) is True+ timeout_latched = bool(copilot_ctx.copilot_total_timeout_exceeded)+ copilot_ctx.last_test_ok = None if (cancelled_by_watchdog and timeout_latched) else run_okcopilot_ctx.last_test_failure_reason = Nonecopilot_ctx.last_test_suspicious_success = Falsecopilot_ctx.last_test_anti_bot = Nonediff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex a0344184d..2d65f98c2 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -26,6 +26,7 @@ from skyvern.forge.sdk.copilot.enforcement import (_summarize_tool_output,)from skyvern.forge.sdk.copilot.tools import (+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,_analyze_run_blocks,_is_meaningful_extracted_data,_record_run_blocks_result,@@ -258,6 +259,7 @@ def _fresh_ctx_for_record() -> Any:pending_action_sequence_fingerprint=None,last_action_sequence_fingerprint=None,repeated_action_fingerprint_streak_count=0,+ copilot_total_timeout_exceeded=False,)@@ -274,6 +276,38 @@ def test_record_run_blocks_result_flips_last_test_ok_on_empty_extraction_envelopassert ctx.last_test_failure_reason is not None+def test_record_run_blocks_result_keeps_failure_when_watchdog_cancel_without_timeout() -> None:+ """Stagnation/ceiling cancels mid-session must still set last_test_ok=False+ so the failed-test nudge can fire — only a coincident total timeout softens+ to ``None`` for the unvalidated WIP rescue path."""+ ctx = _fresh_ctx_for_record()+ result = {+ "ok": False,+ "error": "Run ID: wr_stagnation. Stuck.",+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY: True,+ }++ _record_run_blocks_result(ctx, result)++ assert ctx.last_test_ok is False+ assert ctx.last_test_failure_reason == "Run ID: wr_stagnation. Stuck."+++def test_record_run_blocks_result_sets_last_test_ok_none_on_watchdog_cancel_at_timeout() -> None:+ ctx = _fresh_ctx_for_record()+ ctx.copilot_total_timeout_exceeded = True+ result = {+ "ok": False,+ "error": "Run ID: wr_timeout. Outcome is uncertain.",+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY: True,+ }++ _record_run_blocks_result(ctx, result)++ assert ctx.last_test_ok is None+ assert ctx.last_test_failure_reason == "Run ID: wr_timeout. Outcome is uncertain."++# ---------------------------------------------------------------------------# Repeated null-data escalation# ---------------------------------------------------------------------------diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex b740bfb9f..a8d95b151 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -5,6 +5,7 @@ from __future__ import annotationsfrom unittest.mock import MagicMockfrom skyvern.forge.sdk.copilot.output_utils import (+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,format_tool_result_for_user,parse_final_response,sanitize_tool_result_for_llm,@@ -127,6 +128,19 @@ def test_sanitize_other_tools_do_not_touch_block_screenshot_b64() -> None:assert sanitized["data"]["blocks"][0]["screenshot_b64"] == "stays_here"+def test_sanitize_strips_internal_watchdog_cancel_marker() -> None:+ result = {+ "ok": False,+ "error": "Run ID: wr_timeout. Outcome is uncertain.",+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY: True,+ }++ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)++ assert _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY not in sanitized+ assert sanitized["error"] == "Run ID: wr_timeout. Outcome is uncertain."++class TestSanitization:def test_screenshot_sanitization(self) -> None:from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llmdiff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.pyindex d71cec5eb..88d108767 100644--- a/tests/unit/test_copilot_runner_deadline.py+++ b/tests/unit/test_copilot_runner_deadline.py@@ -46,6 +46,7 @@ async def test_runner_deadline_raises_total_timeout_when_tool_exceeds_budget()ctx = MagicMock()+ ctx.copilot_total_timeout_exceeded = Falsewith pytest.raises(CopilotTotalTimeoutError):await run_with_enforcement(agent=MagicMock(),@@ -53,6 +54,7 @@ async def test_runner_deadline_raises_total_timeout_when_tool_exceeds_budget(ctx=ctx,stream=stream,)+ assert ctx.copilot_total_timeout_exceeded is True@pytest.mark.asyncio@@ -94,6 +96,7 @@ async def test_runner_deadline_protects_context_overflow_recovery_path()ctx = MagicMock()+ ctx.copilot_total_timeout_exceeded = Falsewith pytest.raises(CopilotTotalTimeoutError):await run_with_enforcement(agent=MagicMock(),@@ -102,6 +105,7 @@ async def test_runner_deadline_protects_context_overflow_recovery_path(stream=stream,)assert call_count["n"] == 2, "recovery path should have triggered a second Runner call"+ assert ctx.copilot_total_timeout_exceeded is True@pytest.mark.asyncio@@ -129,6 +133,7 @@ async def test_runner_deadline_does_not_fire_when_tool_completes_in_time()ctx = MagicMock()+ ctx.copilot_total_timeout_exceeded = Falsereturned = await run_with_enforcement(agent=MagicMock(),initial_input="hello",@@ -136,3 +141,4 @@ async def test_runner_deadline_does_not_fire_when_tool_completes_in_time(stream=stream,)assert returned is fake+ assert ctx.copilot_total_timeout_exceeded is Falsediff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex 030bad801..10250e679 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.agent import (_UNEXPECTED_ERROR_REPLY_TESTED,_UNEXPECTED_ERROR_REPLY_UNVALIDATED,_build_cancel_exit_result,+ _build_cancelled_exit_result,_build_max_turns_exit_result,_build_timeout_exit_result,_build_unexpected_error_exit_result,@@ -36,6 +37,7 @@ def _ctx(ctx.last_workflow_yaml = last_workflow_yamlctx.last_test_ok = last_test_okctx.last_test_suspicious_success = last_test_suspicious_success+ ctx.copilot_total_timeout_exceeded = Falsectx.workflow_persisted = last_workflow is not Nonectx.total_tokens_used = Nonereturn ctx@@ -180,6 +182,33 @@ class TestBuildMaxTurnsExitResult:assert result.user_response == _MAX_TURNS_REPLY_DEFAULT+class TestBuildCancelledExitResult:+ def test_total_timeout_latch_routes_cancel_to_timeout_wip(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx.copilot_total_timeout_exceeded = True++ result = _build_cancelled_exit_result(ctx, global_llm_context=None)++ assert result.cancelled is False+ assert result.updated_workflow is wf+ assert result.workflow_yaml == "version: '1.0'"+ assert result.unvalidated is True+ assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED++ def test_regular_cancel_uses_cancel_wip_path(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)++ result = _build_cancelled_exit_result(ctx, global_llm_context=None)++ assert result.cancelled is True+ assert result.updated_workflow is wf+ assert result.workflow_yaml == "version: '1.0'"+ assert result.unvalidated is True+ assert result.user_response == _CANCEL_REPLY_UNVALIDATED++class TestBuildUnexpectedErrorExitResult:def test_no_workflow_falls_back_to_default_reply(self) -> None:ctx = _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None)diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.pyindex 4e0185e9b..c3b396f3f 100644--- a/tests/unit/test_workflow_copilot_cancel.py+++ b/tests/unit/test_workflow_copilot_cancel.py@@ -524,6 +524,83 @@ async def test_pre_agent_cancel_clears_stale_proposed_workflow() -> None:assert workflow_params.update_workflow_copilot_chat.await_args.kwargs["proposed_workflow"] is None+async def test_timeout_wip_result_streams_normal_response_frame(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Timeout WIP rescue must use normal finalisation, not the cancel ERROR path."""+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)+ captured = _install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )+ updated_workflow = MagicMock()+ updated_workflow.model_dump.side_effect = lambda mode="json": {"workflow_id": "wf-draft", "title": "Draft"}+ agent_result = SimpleNamespace(+ user_response="I ran out of time before I could finish testing. I have a draft workflow you can keep.",+ updated_workflow=updated_workflow,+ global_llm_context=None,+ workflow_yaml="version: '1.0'",+ workflow_was_persisted=True,+ clear_proposed_workflow=False,+ cancelled=False,+ total_tokens=123,+ response_type="REPLY",+ unvalidated=True,+ )+ restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)++ request = MagicMock()+ request.headers = {"x-api-key": "sk-test"}+ organization = SimpleNamespace(organization_id="org-1")++ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)+ assert response is captured["sentinel"]++ sent_payloads: list[Any] = []+ stream = MagicMock()++ async def _send(payload: Any) -> bool:+ sent_payloads.append(payload)+ return True++ stream.send = _send+ stream.is_disconnected = AsyncMock(return_value=False)++ handler = captured["handler"]+ assert callable(handler)+ await handler(stream)++ restore_mock.assert_awaited_once()+ proposal = workflow_params.update_workflow_copilot_chat.await_args.kwargs["proposed_workflow"]+ assert proposal["_copilot_yaml"] == "version: '1.0'"+ assert proposal["_copilot_unvalidated"] is True++ response_frames = [+ p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.RESPONSE+ ]+ assert len(response_frames) == 1+ assert response_frames[0].updated_workflow == {"workflow_id": "wf-draft", "title": "Draft"}+ assert response_frames[0].unvalidated is True+ assert response_frames[0].total_tokens == 123++ error_frames = [p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.ERROR]+ assert error_frames == []+ contents = [c.kwargs.get("content") for c in workflow_params.create_workflow_copilot_chat_message.await_args_list]+ assert "Cancelled by user." not in contents++# ---------------------------------------------------------------------------# /workflow/copilot/cancel endpoint# ---------------------------------------------------------------------------