Skip to content

Automate browser based workflows with AI

SKY-9454 fix copilot timeout WIP rescue (#5789)

Andrew Neilsonf5535c93de86
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 3a94aa802..cefa0611c 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -325,6 +325,13 @@ def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | No
)
+def _build_cancelled_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
+ if ctx.copilot_total_timeout_exceeded:
+ LOG.info("Copilot cancellation resolved as total timeout")
+ return _build_timeout_exit_result(ctx, global_llm_context)
+ return _build_cancel_exit_result(ctx, global_llm_context)
+
+
def _build_max_turns_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
return _build_wip_exit_result(
ctx,
@@ -686,8 +693,8 @@ async def run_copilot_agent(
except asyncio.CancelledError:
# Re-raising would leave the route with ``agent_result is None``
# and skip its ``workflow_was_persisted`` rollback decision.
- LOG.info("Copilot run cancelled by user")
- return _build_cancel_exit_result(ctx, global_llm_context)
+ LOG.info("Copilot run cancelled")
+ return _build_cancelled_exit_result(ctx, global_llm_context)
except MaxTurnsExceeded:
return _build_max_turns_exit_result(ctx, global_llm_context)
except CopilotTotalTimeoutError:
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 2b815a5b7..a2e17b3aa 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -160,6 +160,7 @@ class CopilotContext(AgentContext):
post_update_nudge_count: int = 0
coverage_nudge_count: int = 0
format_nudge_count: int = 0
+ copilot_total_timeout_exceeded: bool = False
user_message: str = ""
# Tool tracking
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 3b22fda80..cd349a16d 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -280,6 +280,15 @@ class CopilotTotalTimeoutError(Exception):
"""Raised when the copilot agent exceeds the total allowed runtime."""
+def _mark_copilot_total_timeout(ctx: Any) -> None:
+ ctx.copilot_total_timeout_exceeded = True
+
+
+def _mark_copilot_total_timeout_if_elapsed(ctx: Any, start_time: float) -> None:
+ if time.monotonic() - start_time >= TOTAL_TIMEOUT_SECONDS:
+ _mark_copilot_total_timeout(ctx)
+
+
class CopilotNonRetriableNavError(Exception):
"""Raised from run_with_enforcement when the copilot's most recent run
hit a permanent navigation error (DNS / cert / SSL / invalid URL) and
@@ -1017,6 +1026,7 @@ async def _run_streamed_with_deadline(
finally:
_accumulate_usage(result, ctx)
except asyncio.TimeoutError:
+ _mark_copilot_total_timeout(ctx)
LOG.warning(
"Copilot total timeout exceeded mid-iteration",
elapsed_seconds=round(time.monotonic() - start_time, 3),
@@ -1047,6 +1057,7 @@ async def run_with_enforcement(
# chat history on the server side (see SKY-8986).
elapsed = time.monotonic() - start_time
if elapsed > TOTAL_TIMEOUT_SECONDS:
+ _mark_copilot_total_timeout(ctx)
raise CopilotTotalTimeoutError()
if iteration >= MAX_ITERATIONS:
@@ -1079,6 +1090,9 @@ async def run_with_enforcement(
start_time,
iteration,
)
+ except asyncio.CancelledError:
+ _mark_copilot_total_timeout_if_elapsed(ctx, start_time)
+ raise
except Exception as e:
if not _is_context_window_error(e):
raise
@@ -1098,7 +1112,11 @@ async def run_with_enforcement(
iteration=iteration,
has_session=session is not None,
)
- current_input, images_stripped = await _recover_from_context_overflow(session, current_input)
+ try:
+ current_input, images_stripped = await _recover_from_context_overflow(session, current_input)
+ except asyncio.CancelledError:
+ _mark_copilot_total_timeout_if_elapsed(ctx, start_time)
+ raise
if images_stripped:
# The agent could otherwise reason about the page from
# memory on the next turn; warn it explicitly.
@@ -1115,6 +1133,9 @@ async def run_with_enforcement(
start_time,
iteration,
)
+ except asyncio.CancelledError:
+ _mark_copilot_total_timeout_if_elapsed(ctx, start_time)
+ raise
except Exception as retry_err:
# Never retry twice; even a second overflow surfaces as a
# real failure rather than spinning.
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 31fefac76..2b0bb021d 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -15,6 +15,8 @@ from skyvern.forge.sdk.copilot.loop_detection import LOOP_DETECTED_MARKER
if TYPE_CHECKING:
from agents.result import RunResultStreaming
+_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY = "_copilot_internal_run_cancelled_by_watchdog"
+
_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
_JPEG_PREFIX = b"\xff\xd8\xff"
@@ -211,6 +213,7 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict
]
sanitized["data"] = data
sanitized.pop("_workflow", None)
+ sanitized.pop(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY, None)
return sanitized
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 3c5e03a95..fe53fbca5 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -75,6 +75,7 @@ class AgentContext:
post_update_nudge_count: int = 0
coverage_nudge_count: int = 0
format_nudge_count: int = 0
+ copilot_total_timeout_exceeded: bool = False
failed_test_nudge_count: int = 0
explore_without_workflow_nudge_count: int = 0
null_data_streak_count: int = 0
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 31ac42f9b..0b40c4a60 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -34,6 +34,7 @@ from skyvern.forge.sdk.copilot.narration import NarratorState
from skyvern.forge.sdk.copilot.narration import handler_available as narration_handler_available
from skyvern.forge.sdk.copilot.narration import narrator_poll_tick
from skyvern.forge.sdk.copilot.output_utils import (
+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
build_run_blocks_response,
iter_failure_reasons,
sanitize_tool_result_for_llm,
@@ -1170,6 +1171,7 @@ async def _run_blocks_and_collect_debug(
final_status: str | None = None
run: Any = initial_run
exit_reason: WatchdogExitReason | None = None
+ run_cancelled_by_watchdog = False
# Quiet blocks (WAIT/TEXT_PROMPT/HUMAN_INTERACTION) legitimately have
# DB-silent periods; disable stagnation for any invocation that includes
# one. Safety ceiling still applies.
@@ -1257,6 +1259,7 @@ async def _run_blocks_and_collect_debug(
exit_reason = "success"
else:
await _cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id)
+ run_cancelled_by_watchdog = True
run = await _safe_read_workflow_run(
workflow_run.workflow_run_id, ctx.organization_id, context="post-drain"
)
@@ -1273,7 +1276,10 @@ async def _run_blocks_and_collect_debug(
ctx.pending_reconciliation_run_id = workflow_run.workflow_run_id
assert exit_reason is not None # narrows for mypy; outer check excludes "success" but not None
error_msg = await _watchdog_error_message(exit_reason, ctx, workflow_run.workflow_run_id, run)
- return {"ok": False, "error": error_msg}
+ result = {"ok": False, "error": error_msg}
+ if run_cancelled_by_watchdog:
+ result[_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY] = True
+ return result
except asyncio.CancelledError:
# The SDK's @function_tool(timeout=...) cancelled us mid-poll. Shield
# the cleanup so the parent cancellation can't interrupt it mid-await.
@@ -2097,7 +2103,11 @@ def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:
def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:
run_ok = bool(result.get("ok", False))
- copilot_ctx.last_test_ok = run_ok
+ # Watchdog cancels normally count as ok=False; only a coincident total
+ # timeout softens to ``None`` to keep the unvalidated WIP rescue open.
+ cancelled_by_watchdog = result.get(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY) is True
+ timeout_latched = bool(copilot_ctx.copilot_total_timeout_exceeded)
+ copilot_ctx.last_test_ok = None if (cancelled_by_watchdog and timeout_latched) else run_ok
copilot_ctx.last_test_failure_reason = None
copilot_ctx.last_test_suspicious_success = False
copilot_ctx.last_test_anti_bot = None
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index a0344184d..2d65f98c2 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -26,6 +26,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
_summarize_tool_output,
)
from skyvern.forge.sdk.copilot.tools import (
+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
_analyze_run_blocks,
_is_meaningful_extracted_data,
_record_run_blocks_result,
@@ -258,6 +259,7 @@ def _fresh_ctx_for_record() -> Any:
pending_action_sequence_fingerprint=None,
last_action_sequence_fingerprint=None,
repeated_action_fingerprint_streak_count=0,
+ copilot_total_timeout_exceeded=False,
)
@@ -274,6 +276,38 @@ def test_record_run_blocks_result_flips_last_test_ok_on_empty_extraction_envelop
assert ctx.last_test_failure_reason is not None
+def test_record_run_blocks_result_keeps_failure_when_watchdog_cancel_without_timeout() -> None:
+ """Stagnation/ceiling cancels mid-session must still set last_test_ok=False
+ so the failed-test nudge can fire — only a coincident total timeout softens
+ to ``None`` for the unvalidated WIP rescue path."""
+ ctx = _fresh_ctx_for_record()
+ result = {
+ "ok": False,
+ "error": "Run ID: wr_stagnation. Stuck.",
+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY: True,
+ }
+
+ _record_run_blocks_result(ctx, result)
+
+ assert ctx.last_test_ok is False
+ assert ctx.last_test_failure_reason == "Run ID: wr_stagnation. Stuck."
+
+
+def test_record_run_blocks_result_sets_last_test_ok_none_on_watchdog_cancel_at_timeout() -> None:
+ ctx = _fresh_ctx_for_record()
+ ctx.copilot_total_timeout_exceeded = True
+ result = {
+ "ok": False,
+ "error": "Run ID: wr_timeout. Outcome is uncertain.",
+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY: True,
+ }
+
+ _record_run_blocks_result(ctx, result)
+
+ assert ctx.last_test_ok is None
+ assert ctx.last_test_failure_reason == "Run ID: wr_timeout. Outcome is uncertain."
+
+
# ---------------------------------------------------------------------------
# Repeated null-data escalation
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index b740bfb9f..a8d95b151 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -5,6 +5,7 @@ from __future__ import annotations
from unittest.mock import MagicMock
from skyvern.forge.sdk.copilot.output_utils import (
+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
format_tool_result_for_user,
parse_final_response,
sanitize_tool_result_for_llm,
@@ -127,6 +128,19 @@ def test_sanitize_other_tools_do_not_touch_block_screenshot_b64() -> None:
assert sanitized["data"]["blocks"][0]["screenshot_b64"] == "stays_here"
+def test_sanitize_strips_internal_watchdog_cancel_marker() -> None:
+ result = {
+ "ok": False,
+ "error": "Run ID: wr_timeout. Outcome is uncertain.",
+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY: True,
+ }
+
+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)
+
+ assert _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY not in sanitized
+ assert sanitized["error"] == "Run ID: wr_timeout. Outcome is uncertain."
+
+
class TestSanitization:
def test_screenshot_sanitization(self) -> None:
from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm
diff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.py
index d71cec5eb..88d108767 100644
--- a/tests/unit/test_copilot_runner_deadline.py
+++ b/tests/unit/test_copilot_runner_deadline.py
@@ -46,6 +46,7 @@ async def test_runner_deadline_raises_total_timeout_when_tool_exceeds_budget(
)
ctx = MagicMock()
+ ctx.copilot_total_timeout_exceeded = False
with pytest.raises(CopilotTotalTimeoutError):
await run_with_enforcement(
agent=MagicMock(),
@@ -53,6 +54,7 @@ async def test_runner_deadline_raises_total_timeout_when_tool_exceeds_budget(
ctx=ctx,
stream=stream,
)
+ assert ctx.copilot_total_timeout_exceeded is True
@pytest.mark.asyncio
@@ -94,6 +96,7 @@ async def test_runner_deadline_protects_context_overflow_recovery_path(
)
ctx = MagicMock()
+ ctx.copilot_total_timeout_exceeded = False
with pytest.raises(CopilotTotalTimeoutError):
await run_with_enforcement(
agent=MagicMock(),
@@ -102,6 +105,7 @@ async def test_runner_deadline_protects_context_overflow_recovery_path(
stream=stream,
)
assert call_count["n"] == 2, "recovery path should have triggered a second Runner call"
+ assert ctx.copilot_total_timeout_exceeded is True
@pytest.mark.asyncio
@@ -129,6 +133,7 @@ async def test_runner_deadline_does_not_fire_when_tool_completes_in_time(
)
ctx = MagicMock()
+ ctx.copilot_total_timeout_exceeded = False
returned = await run_with_enforcement(
agent=MagicMock(),
initial_input="hello",
@@ -136,3 +141,4 @@ async def test_runner_deadline_does_not_fire_when_tool_completes_in_time(
stream=stream,
)
assert returned is fake
+ assert ctx.copilot_total_timeout_exceeded is False
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
index 030bad801..10250e679 100644
--- a/tests/unit/test_copilot_timeout_wip.py
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.agent import (
_UNEXPECTED_ERROR_REPLY_TESTED,
_UNEXPECTED_ERROR_REPLY_UNVALIDATED,
_build_cancel_exit_result,
+ _build_cancelled_exit_result,
_build_max_turns_exit_result,
_build_timeout_exit_result,
_build_unexpected_error_exit_result,
@@ -36,6 +37,7 @@ def _ctx(
ctx.last_workflow_yaml = last_workflow_yaml
ctx.last_test_ok = last_test_ok
ctx.last_test_suspicious_success = last_test_suspicious_success
+ ctx.copilot_total_timeout_exceeded = False
ctx.workflow_persisted = last_workflow is not None
ctx.total_tokens_used = None
return ctx
@@ -180,6 +182,33 @@ class TestBuildMaxTurnsExitResult:
assert result.user_response == _MAX_TURNS_REPLY_DEFAULT
+class TestBuildCancelledExitResult:
+ def test_total_timeout_latch_routes_cancel_to_timeout_wip(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx.copilot_total_timeout_exceeded = True
+
+ result = _build_cancelled_exit_result(ctx, global_llm_context=None)
+
+ assert result.cancelled is False
+ assert result.updated_workflow is wf
+ assert result.workflow_yaml == "version: '1.0'"
+ assert result.unvalidated is True
+ assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED
+
+ def test_regular_cancel_uses_cancel_wip_path(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+
+ result = _build_cancelled_exit_result(ctx, global_llm_context=None)
+
+ assert result.cancelled is True
+ assert result.updated_workflow is wf
+ assert result.workflow_yaml == "version: '1.0'"
+ assert result.unvalidated is True
+ assert result.user_response == _CANCEL_REPLY_UNVALIDATED
+
+
class TestBuildUnexpectedErrorExitResult:
def test_no_workflow_falls_back_to_default_reply(self) -> None:
ctx = _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None)
diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.py
index 4e0185e9b..c3b396f3f 100644
--- a/tests/unit/test_workflow_copilot_cancel.py
+++ b/tests/unit/test_workflow_copilot_cancel.py
@@ -524,6 +524,83 @@ async def test_pre_agent_cancel_clears_stale_proposed_workflow() -> None:
assert workflow_params.update_workflow_copilot_chat.await_args.kwargs["proposed_workflow"] is None
+async def test_timeout_wip_result_streams_normal_response_frame(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Timeout WIP rescue must use normal finalisation, not the cancel ERROR path."""
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ captured = _install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ updated_workflow = MagicMock()
+ updated_workflow.model_dump.side_effect = lambda mode="json": {"workflow_id": "wf-draft", "title": "Draft"}
+ agent_result = SimpleNamespace(
+ user_response="I ran out of time before I could finish testing. I have a draft workflow you can keep.",
+ updated_workflow=updated_workflow,
+ global_llm_context=None,
+ workflow_yaml="version: '1.0'",
+ workflow_was_persisted=True,
+ clear_proposed_workflow=False,
+ cancelled=False,
+ total_tokens=123,
+ response_type="REPLY",
+ unvalidated=True,
+ )
+ restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)
+
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test"}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+ assert response is captured["sentinel"]
+
+ sent_payloads: list[Any] = []
+ stream = MagicMock()
+
+ async def _send(payload: Any) -> bool:
+ sent_payloads.append(payload)
+ return True
+
+ stream.send = _send
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(stream)
+
+ restore_mock.assert_awaited_once()
+ proposal = workflow_params.update_workflow_copilot_chat.await_args.kwargs["proposed_workflow"]
+ assert proposal["_copilot_yaml"] == "version: '1.0'"
+ assert proposal["_copilot_unvalidated"] is True
+
+ response_frames = [
+ p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.RESPONSE
+ ]
+ assert len(response_frames) == 1
+ assert response_frames[0].updated_workflow == {"workflow_id": "wf-draft", "title": "Draft"}
+ assert response_frames[0].unvalidated is True
+ assert response_frames[0].total_tokens == 123
+
+ error_frames = [p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.ERROR]
+ assert error_frames == []
+ contents = [c.kwargs.get("content") for c in workflow_params.create_workflow_copilot_chat_message.await_args_list]
+ assert "Cancelled by user." not in contents
+
+
# ---------------------------------------------------------------------------
# /workflow/copilot/cancel endpoint
# ---------------------------------------------------------------------------