Skip to content

Automate browser based workflows with AI

copilot: bypass name-only tool-loop guard for block-running tools (#5647)

Andrew Neilsonf0652692742e
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index fd52080a2..08d344fc4 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import asyncio
import copy
import json
import re
@@ -982,6 +983,48 @@ def _accumulate_usage(result: RunResultStreaming, ctx: Any) -> None:
ctx.output_tokens_used += getattr(usage, "output_tokens", 0) or 0
+async def _run_streamed_with_deadline(
+ agent: Agent,
+ current_input: str | list,
+ ctx: Any,
+ session: Any,
+ tracked_stream: _SendTrackingStream,
+ runner_kwargs: dict[str, Any],
+ start_time: float,
+ iteration: int,
+) -> Any:
+ """Run ``Runner.run_streamed`` + ``stream_to_sse`` with a deadline
+ against ``TOTAL_TIMEOUT_SECONDS``.
+
+ The top-of-loop elapsed check only fires between iterations; a
+ long-running tool inside ``Runner.run_streamed`` needs ``wait_for``
+ to raise ``CopilotTotalTimeoutError`` mid-tool so the caller's
+ ``_build_exit_result`` path emits a non-empty REPLY before the
+ client's own transport timeout closes the stream.
+
+ ``max(1.0, ...)`` floors ``remaining`` so ``wait_for(timeout=0)``
+ never panics on an already-spent budget.
+ """
+ from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse
+
+ elapsed = time.monotonic() - start_time
+ remaining = max(1.0, TOTAL_TIMEOUT_SECONDS - elapsed)
+ result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)
+ try:
+ try:
+ await asyncio.wait_for(stream_to_sse(result, tracked_stream, ctx), timeout=remaining)
+ finally:
+ _accumulate_usage(result, ctx)
+ except asyncio.TimeoutError:
+ LOG.warning(
+ "Copilot total timeout exceeded mid-iteration",
+ elapsed_seconds=round(time.monotonic() - start_time, 3),
+ iteration=iteration,
+ )
+ raise CopilotTotalTimeoutError() from None
+ return result
+
+
async def run_with_enforcement(
agent: Agent,
initial_input: str | list,
@@ -990,9 +1033,6 @@ async def run_with_enforcement(
**runner_kwargs: Any,
) -> RunResultStreaming:
"""Run agent with enforcement nudges, preserving conversation history."""
- # Lazy import: streaming_adapter lives in a sibling PR in the stack.
- from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse
-
session = runner_kwargs.pop("session", None)
current_input: str | list = initial_input
start_time = time.monotonic()
@@ -1028,11 +1068,16 @@ async def run_with_enforcement(
data={"iteration": iteration, "elapsed_seconds": round(elapsed, 3)},
):
try:
- result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)
- try:
- await stream_to_sse(result, tracked_stream, ctx)
- finally:
- _accumulate_usage(result, ctx)
+ result = await _run_streamed_with_deadline(
+ agent,
+ current_input,
+ ctx,
+ session,
+ tracked_stream,
+ runner_kwargs,
+ start_time,
+ iteration,
+ )
except Exception as e:
if not _is_context_window_error(e):
raise
@@ -1059,13 +1104,16 @@ async def run_with_enforcement(
pending_recovery_nudge = SCREENSHOT_DROPPED_NUDGE
tracked_stream = _SendTrackingStream(stream)
try:
- result = Runner.run_streamed(
- agent, input=current_input, context=ctx, session=session, **runner_kwargs
+ result = await _run_streamed_with_deadline(
+ agent,
+ current_input,
+ ctx,
+ session,
+ tracked_stream,
+ runner_kwargs,
+ start_time,
+ iteration,
)
- try:
- await stream_to_sse(result, tracked_stream, ctx)
- finally:
- _accumulate_usage(result, ctx)
except Exception as retry_err:
# Never retry twice; even a second overflow surfaces as a
# real failure rather than spinning.
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 7f1612650..13fe7cd74 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -366,8 +366,11 @@ BLOCK_RUNNING_TOOLS = frozenset({"run_blocks_and_collect_debug", "update_and_run
def _tool_loop_error(ctx: AgentContext, tool_name: str) -> str | None:
+ # The name-only guard false-positives on the intended iterative build
+ # (one new block per update_and_run_blocks). Block-running tools rely
+ # on the progress-aware checks below instead.
tracker = getattr(ctx, "consecutive_tool_tracker", None)
- if isinstance(tracker, list):
+ if isinstance(tracker, list) and tool_name not in BLOCK_RUNNING_TOOLS:
detected = detect_tool_loop(tracker, tool_name)
if detected is not None:
return detected
diff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.py
new file mode 100644
index 000000000..d71cec5eb
--- /dev/null
+++ b/tests/unit/test_copilot_runner_deadline.py
@@ -0,0 +1,138 @@
+"""Tests for the per-iteration Runner deadline (SKY-9243)."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.forge.sdk.copilot.enforcement import (
+ CopilotTotalTimeoutError,
+ run_with_enforcement,
+)
+
+
+def _fake_result() -> MagicMock:
+ r = MagicMock()
+ r.final_output = None
+ r.new_items = []
+ r.to_input_list.return_value = []
+ r.raw_responses = []
+ return r
+
+
+async def test_runner_deadline_raises_total_timeout_when_tool_exceeds_budget(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.05)
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed",
+ lambda *a, **kw: _fake_result(),
+ )
+
+ async def hanging_stream(result: Any, s: Any, c: Any) -> None:
+ await asyncio.sleep(5.0)
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse",
+ hanging_stream,
+ )
+
+ ctx = MagicMock()
+ with pytest.raises(CopilotTotalTimeoutError):
+ await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ )
+
+
+async def test_runner_deadline_protects_context_overflow_recovery_path(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.05)
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ call_count = {"n": 0}
+
+ def fake_run_streamed(*a: Any, **kw: Any) -> Any:
+ call_count["n"] += 1
+ return _fake_result()
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed",
+ fake_run_streamed,
+ )
+
+ async def stream_impl(result: Any, s: Any, c: Any) -> None:
+ if call_count["n"] == 1:
+ raise Exception("context_length_exceeded: message too long")
+ await asyncio.sleep(5.0)
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse",
+ stream_impl,
+ )
+
+ async def fake_recover(session: Any, current_input: Any) -> Any:
+ return current_input, False
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.enforcement._recover_from_context_overflow",
+ fake_recover,
+ )
+
+ ctx = MagicMock()
+ with pytest.raises(CopilotTotalTimeoutError):
+ await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ )
+ assert call_count["n"] == 2, "recovery path should have triggered a second Runner call"
+
+
+async def test_runner_deadline_does_not_fire_when_tool_completes_in_time(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 5.0)
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ fake = _fake_result()
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed",
+ lambda *a, **kw: fake,
+ )
+
+ async def quick_stream(result: Any, s: Any, c: Any) -> None:
+ await asyncio.sleep(0.01)
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse",
+ quick_stream,
+ )
+
+ ctx = MagicMock()
+ returned = await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ )
+ assert returned is fake
diff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.py
new file mode 100644
index 000000000..0d8819569
--- /dev/null
+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py
@@ -0,0 +1,57 @@
+"""Tests for `_tool_loop_error`'s BLOCK_RUNNING_TOOLS bypass (SKY-9249)."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+from skyvern.forge.sdk.copilot.tools import BLOCK_RUNNING_TOOLS, _tool_loop_error
+
+
+def _ctx(**overrides: Any) -> Any:
+ base: dict[str, Any] = {
+ "consecutive_tool_tracker": [],
+ "pending_reconciliation_run_id": None,
+ "repeated_action_fingerprint_streak_count": 0,
+ "last_test_non_retriable_nav_error": None,
+ "non_retriable_nav_error_last_emitted_signature": None,
+ }
+ base.update(overrides)
+ return SimpleNamespace(**base)
+
+
+def test_block_running_tool_is_not_blocked_by_name_only_streak() -> None:
+ ctx = _ctx()
+ for _ in range(5):
+ assert _tool_loop_error(ctx, "update_and_run_blocks") is None
+
+
+def test_planning_tool_still_trips_name_only_streak() -> None:
+ ctx = _ctx()
+ assert _tool_loop_error(ctx, "update_workflow") is None
+ assert _tool_loop_error(ctx, "update_workflow") is None
+ msg = _tool_loop_error(ctx, "update_workflow")
+ assert msg is not None
+ assert "LOOP DETECTED" in msg
+
+
+def test_block_running_tool_is_blocked_by_pending_reconciliation() -> None:
+ ctx = _ctx(pending_reconciliation_run_id="wr_123")
+ msg = _tool_loop_error(ctx, "update_and_run_blocks")
+ assert msg is not None
+ assert "wr_123" in msg
+
+
+def test_block_running_tool_is_blocked_by_repeated_action_streak() -> None:
+ ctx = _ctx(repeated_action_fingerprint_streak_count=5)
+ msg = _tool_loop_error(ctx, "update_and_run_blocks")
+ assert msg is not None
+ assert "Repeated-action abort" in msg
+
+
+def test_bypass_applies_to_both_block_running_tool_names() -> None:
+ assert "update_and_run_blocks" in BLOCK_RUNNING_TOOLS
+ assert "run_blocks_and_collect_debug" in BLOCK_RUNNING_TOOLS
+ ctx = _ctx()
+ for _ in range(5):
+ assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is None