Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
copilot: bypass name-only tool-loop guard for block-running tools (#5647)
Andrew Neilsonf0652692742e
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex fd52080a2..08d344fc4 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -2,6 +2,7 @@from __future__ import annotations+import asyncioimport copyimport jsonimport re@@ -982,6 +983,48 @@ def _accumulate_usage(result: RunResultStreaming, ctx: Any) -> None:ctx.output_tokens_used += getattr(usage, "output_tokens", 0) or 0+async def _run_streamed_with_deadline(+ agent: Agent,+ current_input: str | list,+ ctx: Any,+ session: Any,+ tracked_stream: _SendTrackingStream,+ runner_kwargs: dict[str, Any],+ start_time: float,+ iteration: int,+) -> Any:+ """Run ``Runner.run_streamed`` + ``stream_to_sse`` with a deadline+ against ``TOTAL_TIMEOUT_SECONDS``.++ The top-of-loop elapsed check only fires between iterations; a+ long-running tool inside ``Runner.run_streamed`` needs ``wait_for``+ to raise ``CopilotTotalTimeoutError`` mid-tool so the caller's+ ``_build_exit_result`` path emits a non-empty REPLY before the+ client's own transport timeout closes the stream.++ ``max(1.0, ...)`` floors ``remaining`` so ``wait_for(timeout=0)``+ never panics on an already-spent budget.+ """+ from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse++ elapsed = time.monotonic() - start_time+ remaining = max(1.0, TOTAL_TIMEOUT_SECONDS - elapsed)+ result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)+ try:+ try:+ await asyncio.wait_for(stream_to_sse(result, tracked_stream, ctx), timeout=remaining)+ finally:+ _accumulate_usage(result, ctx)+ except asyncio.TimeoutError:+ LOG.warning(+ "Copilot total timeout exceeded mid-iteration",+ elapsed_seconds=round(time.monotonic() - start_time, 3),+ iteration=iteration,+ )+ raise CopilotTotalTimeoutError() from None+ return result++async def run_with_enforcement(agent: Agent,initial_input: str | list,@@ -990,9 +1033,6 @@ async def run_with_enforcement(**runner_kwargs: Any,) -> RunResultStreaming:"""Run agent with enforcement nudges, preserving conversation history."""- # Lazy import: streaming_adapter lives in a sibling PR in the stack.- from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse-session = runner_kwargs.pop("session", None)current_input: str | list = initial_inputstart_time = time.monotonic()@@ -1028,11 +1068,16 @@ async def run_with_enforcement(data={"iteration": iteration, "elapsed_seconds": round(elapsed, 3)},):try:- result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)- try:- await stream_to_sse(result, tracked_stream, ctx)- finally:- _accumulate_usage(result, ctx)+ result = await _run_streamed_with_deadline(+ agent,+ current_input,+ ctx,+ session,+ tracked_stream,+ runner_kwargs,+ start_time,+ iteration,+ )except Exception as e:if not _is_context_window_error(e):raise@@ -1059,13 +1104,16 @@ async def run_with_enforcement(pending_recovery_nudge = SCREENSHOT_DROPPED_NUDGEtracked_stream = _SendTrackingStream(stream)try:- result = Runner.run_streamed(- agent, input=current_input, context=ctx, session=session, **runner_kwargs+ result = await _run_streamed_with_deadline(+ agent,+ current_input,+ ctx,+ session,+ tracked_stream,+ runner_kwargs,+ start_time,+ iteration,)- try:- await stream_to_sse(result, tracked_stream, ctx)- finally:- _accumulate_usage(result, ctx)except Exception as retry_err:# Never retry twice; even a second overflow surfaces as a# real failure rather than spinning.diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 7f1612650..13fe7cd74 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -366,8 +366,11 @@ BLOCK_RUNNING_TOOLS = frozenset({"run_blocks_and_collect_debug", "update_and_rundef _tool_loop_error(ctx: AgentContext, tool_name: str) -> str | None:+ # The name-only guard false-positives on the intended iterative build+ # (one new block per update_and_run_blocks). Block-running tools rely+ # on the progress-aware checks below instead.tracker = getattr(ctx, "consecutive_tool_tracker", None)- if isinstance(tracker, list):+ if isinstance(tracker, list) and tool_name not in BLOCK_RUNNING_TOOLS:detected = detect_tool_loop(tracker, tool_name)if detected is not None:return detecteddiff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.pynew file mode 100644index 000000000..d71cec5eb--- /dev/null+++ b/tests/unit/test_copilot_runner_deadline.py@@ -0,0 +1,138 @@+"""Tests for the per-iteration Runner deadline (SKY-9243)."""++from __future__ import annotations++import asyncio+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.forge.sdk.copilot.enforcement import (+ CopilotTotalTimeoutError,+ run_with_enforcement,+)+++def _fake_result() -> MagicMock:+ r = MagicMock()+ r.final_output = None+ r.new_items = []+ r.to_input_list.return_value = []+ r.raw_responses = []+ return r+++async def test_runner_deadline_raises_total_timeout_when_tool_exceeds_budget(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.05)++ stream = MagicMock()+ stream.is_disconnected = AsyncMock(return_value=False)++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed",+ lambda *a, **kw: _fake_result(),+ )++ async def hanging_stream(result: Any, s: Any, c: Any) -> None:+ await asyncio.sleep(5.0)++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse",+ hanging_stream,+ )++ ctx = MagicMock()+ with pytest.raises(CopilotTotalTimeoutError):+ await run_with_enforcement(+ agent=MagicMock(),+ initial_input="hello",+ ctx=ctx,+ stream=stream,+ )+++async def test_runner_deadline_protects_context_overflow_recovery_path(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.05)++ stream = MagicMock()+ stream.is_disconnected = AsyncMock(return_value=False)++ call_count = {"n": 0}++ def fake_run_streamed(*a: Any, **kw: Any) -> Any:+ call_count["n"] += 1+ return _fake_result()++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed",+ fake_run_streamed,+ )++ async def stream_impl(result: Any, s: Any, c: Any) -> None:+ if call_count["n"] == 1:+ raise Exception("context_length_exceeded: message too long")+ await asyncio.sleep(5.0)++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse",+ stream_impl,+ )++ async def fake_recover(session: Any, current_input: Any) -> Any:+ return current_input, False++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.enforcement._recover_from_context_overflow",+ fake_recover,+ )++ ctx = MagicMock()+ with pytest.raises(CopilotTotalTimeoutError):+ await run_with_enforcement(+ agent=MagicMock(),+ initial_input="hello",+ ctx=ctx,+ stream=stream,+ )+ assert call_count["n"] == 2, "recovery path should have triggered a second Runner call"+++async def test_runner_deadline_does_not_fire_when_tool_completes_in_time(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 5.0)++ stream = MagicMock()+ stream.is_disconnected = AsyncMock(return_value=False)++ fake = _fake_result()++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed",+ lambda *a, **kw: fake,+ )++ async def quick_stream(result: Any, s: Any, c: Any) -> None:+ await asyncio.sleep(0.01)++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse",+ quick_stream,+ )++ ctx = MagicMock()+ returned = await run_with_enforcement(+ agent=MagicMock(),+ initial_input="hello",+ ctx=ctx,+ stream=stream,+ )+ assert returned is fakediff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.pynew file mode 100644index 000000000..0d8819569--- /dev/null+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py@@ -0,0 +1,57 @@+"""Tests for `_tool_loop_error`'s BLOCK_RUNNING_TOOLS bypass (SKY-9249)."""++from __future__ import annotations++from types import SimpleNamespace+from typing import Any++from skyvern.forge.sdk.copilot.tools import BLOCK_RUNNING_TOOLS, _tool_loop_error+++def _ctx(**overrides: Any) -> Any:+ base: dict[str, Any] = {+ "consecutive_tool_tracker": [],+ "pending_reconciliation_run_id": None,+ "repeated_action_fingerprint_streak_count": 0,+ "last_test_non_retriable_nav_error": None,+ "non_retriable_nav_error_last_emitted_signature": None,+ }+ base.update(overrides)+ return SimpleNamespace(**base)+++def test_block_running_tool_is_not_blocked_by_name_only_streak() -> None:+ ctx = _ctx()+ for _ in range(5):+ assert _tool_loop_error(ctx, "update_and_run_blocks") is None+++def test_planning_tool_still_trips_name_only_streak() -> None:+ ctx = _ctx()+ assert _tool_loop_error(ctx, "update_workflow") is None+ assert _tool_loop_error(ctx, "update_workflow") is None+ msg = _tool_loop_error(ctx, "update_workflow")+ assert msg is not None+ assert "LOOP DETECTED" in msg+++def test_block_running_tool_is_blocked_by_pending_reconciliation() -> None:+ ctx = _ctx(pending_reconciliation_run_id="wr_123")+ msg = _tool_loop_error(ctx, "update_and_run_blocks")+ assert msg is not None+ assert "wr_123" in msg+++def test_block_running_tool_is_blocked_by_repeated_action_streak() -> None:+ ctx = _ctx(repeated_action_fingerprint_streak_count=5)+ msg = _tool_loop_error(ctx, "update_and_run_blocks")+ assert msg is not None+ assert "Repeated-action abort" in msg+++def test_bypass_applies_to_both_block_running_tool_names() -> None:+ assert "update_and_run_blocks" in BLOCK_RUNNING_TOOLS+ assert "run_blocks_and_collect_debug" in BLOCK_RUNNING_TOOLS+ ctx = _ctx()+ for _ in range(5):+ assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is None