Skip to content

Automate browser based workflows with AI

SKY-12357: Make Workflow Copilot loop guard argument-aware so multi-field builds are not halted as loops (#7355)

Andrew Neilson6ff4c6b0b1df
diff --git a/skyvern/forge/sdk/copilot/loop_detection.py b/skyvern/forge/sdk/copilot/loop_detection.py
index 2843c399c..9fdfcb8e2 100644
--- a/skyvern/forge/sdk/copilot/loop_detection.py
+++ b/skyvern/forge/sdk/copilot/loop_detection.py
@@ -2,9 +2,9 @@
Two independent guards:
-* ``detect_tool_loop`` fires on strictly consecutive same-tool streaks
- (A-A-A). Resets the moment the tool name changes, so oscillating
- patterns (A-B-A-B) bypass it by design.
+* ``detect_tool_loop`` fires on strictly consecutive same-identity streaks
+ (A-A-A over one tool_step_identity). Resets the moment the identity
+ changes, so oscillating and argument-distinct patterns bypass it by design.
* ``detect_failed_tool_step_loop`` fires on N repeated failures of the
same (tool, args) pair, even when other tools dispatch in between.
Block-running credential/config failures are keyed by failure category
@@ -53,10 +53,12 @@ WORKFLOW_RUN_CREATION_TOOL_NAMES = frozenset({"run_blocks_and_collect_debug", "u
def detect_tool_loop(
tracker: list[str],
tool_name: str,
+ arguments: Mapping[str, Any] | None = None,
threshold: int = MAX_CONSECUTIVE_SAME_TOOL,
) -> str | None:
- """Track tool invocation order and return a loop error message when threshold is hit."""
- tracker.append(tool_name)
+ """Track tool step identities and return a loop error message when threshold is hit."""
+ identity = tool_step_identity(tool_name, arguments)
+ tracker.append(identity)
if len(tracker) >= threshold and len(set(tracker[-threshold:])) == 1:
tracker.clear()
@@ -69,16 +71,23 @@ def detect_tool_loop(
if len(tracker) >= 2 and tracker[-1] != tracker[-2]:
tracker.clear()
- tracker.append(tool_name)
+ tracker.append(identity)
return None
-def record_consecutive_tool_result_boundary(tracker: list[str], tool_name: str) -> None:
- if tracker and tracker[-1] == tool_name:
+def record_consecutive_tool_result_boundary(
+ tracker: list[str],
+ tool_name: str,
+ arguments: Mapping[str, Any] | None = None,
+) -> None:
+ # No-op on a same-name boundary regardless of arguments: streak identity is owned by
+ # admission-time detect_tool_loop, so re-seeding on completion order could clobber a
+ # live argument-distinct streak.
+ if tracker and tracker[-1].startswith(f"{tool_name}:"):
return
tracker.clear()
- tracker.append(tool_name)
+ tracker.append(tool_step_identity(tool_name, arguments))
def _result_records_workflow_progress(tool_name: str, result: Mapping[str, Any]) -> bool:
@@ -100,6 +109,7 @@ def record_consecutive_tool_result_boundary_for_ctx(
ctx: Any,
tool_name: str,
result: Mapping[str, Any],
+ arguments: Mapping[str, Any] | None = None,
) -> None:
tracker = getattr(ctx, "consecutive_tool_tracker", None)
if not isinstance(tracker, list):
@@ -107,7 +117,7 @@ def record_consecutive_tool_result_boundary_for_ctx(
if _result_records_workflow_progress(tool_name, result):
tracker.clear()
return
- record_consecutive_tool_result_boundary(tracker, tool_name)
+ record_consecutive_tool_result_boundary(tracker, tool_name, arguments)
def _normalize_step_argument(value: Any) -> Any:
@@ -285,7 +295,7 @@ def record_tool_step_result_for_ctx(
tracker = _ctx_failed_step_tracker(ctx)
if tracker is not None:
record_tool_step_result(tracker, tool_name, arguments, result)
- record_consecutive_tool_result_boundary_for_ctx(ctx, tool_name, result)
+ record_consecutive_tool_result_boundary_for_ctx(ctx, tool_name, result, arguments)
# Strict ``is True`` check: a malformed result dict missing ``ok`` entirely
# must not be treated as success and accidentally clear a blocker signal.
if result.get("ok") is True:
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index f67b3f381..dba02cf08 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -351,7 +351,7 @@ class SkyvernOverlayMCPServer(MCPServer):
return _copilot_to_call_tool_result({"ok": False, "error": payload})
tracker = getattr(copilot_ctx, "consecutive_tool_tracker", None)
- loop_error = detect_tool_loop(tracker, tool_name) if isinstance(tracker, list) else None
+ loop_error = detect_tool_loop(tracker, tool_name, arguments) if isinstance(tracker, list) else None
if loop_error:
LOG.warning(
"Tool loop detected, skipping execution",
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index 21d35d6a7..5e57355f0 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -1568,7 +1568,7 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
and tool_name not in _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS
and not output_contract_owns_turn
):
- detected = detect_tool_loop(tracker, tool_name)
+ detected = detect_tool_loop(tracker, tool_name, arguments)
if detected is not None:
return _emit_tool_blocker_signal(
ctx,
diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.py
index 8e1de1433..ef8503f74 100644
--- a/skyvern/forge/sdk/copilot/tools/guardrails.py
+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py
@@ -91,6 +91,7 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Tool
getattr(tool_context, "context", None),
tool_name,
{"ok": False, "error": error},
+ arguments=tool_arguments,
)
return ToolGuardrailFunctionOutput.reject_content(error, output_info=trace_data)
return ToolGuardrailFunctionOutput.allow(output_info=trace_data)
diff --git a/tests/unit/test_copilot_loop_detection.py b/tests/unit/test_copilot_loop_detection.py
index b1f744d2d..68475cd3d 100644
--- a/tests/unit/test_copilot_loop_detection.py
+++ b/tests/unit/test_copilot_loop_detection.py
@@ -13,6 +13,7 @@ from skyvern.forge.sdk.copilot.loop_detection import (
record_consecutive_tool_result_boundary_for_ctx,
record_tool_step_result,
record_tool_step_result_for_ctx,
+ tool_step_identity,
)
from skyvern.forge.sdk.copilot.tools.run_execution import _diagnosis_repair_tool_error
@@ -21,7 +22,7 @@ def test_returns_none_below_threshold() -> None:
tracker: list[str] = []
assert detect_tool_loop(tracker, "click") is None
assert detect_tool_loop(tracker, "click") is None
- assert tracker == ["click", "click"]
+ assert tracker == [tool_step_identity("click"), tool_step_identity("click")]
def test_fires_at_threshold_and_clears_tracker() -> None:
@@ -41,7 +42,7 @@ def test_tool_switch_resets_tracker() -> None:
detect_tool_loop(tracker, "click")
detect_tool_loop(tracker, "click")
assert detect_tool_loop(tracker, "type_text") is None
- assert tracker == ["type_text"]
+ assert tracker == [tool_step_identity("type_text")]
def test_requires_full_fresh_threshold_after_warning() -> None:
@@ -55,24 +56,61 @@ def test_requires_full_fresh_threshold_after_warning() -> None:
assert detect_tool_loop(tracker, "click") is not None
+def test_distinct_argument_identities_never_fire_over_repeated_same_tool() -> None:
+ tracker: list[str] = []
+ for index in range(5):
+ assert detect_tool_loop(tracker, "type_text", {"selector": f"#field-{index}", "text": f"v{index}"}) is None
+ assert tracker == [tool_step_identity("type_text", {"selector": "#field-4", "text": "v4"})]
+
+
+def test_third_identical_identity_fires_and_clears() -> None:
+ tracker: list[str] = []
+ args = {"selector": "#name", "text": "Ada"}
+ assert detect_tool_loop(tracker, "type_text", args) is None
+ assert detect_tool_loop(tracker, "type_text", args) is None
+ msg = detect_tool_loop(tracker, "type_text", args)
+ assert msg is not None
+ assert "LOOP DETECTED" in msg
+ assert "type_text" in msg
+ assert tracker == []
+
+
def test_same_tool_result_boundary_does_not_double_count_dispatch() -> None:
- tracker = ["update_workflow"]
+ tracker = [tool_step_identity("update_workflow")]
record_consecutive_tool_result_boundary(tracker, "update_workflow")
- assert tracker == ["update_workflow"]
+ assert tracker == [tool_step_identity("update_workflow")]
+
+
+def test_same_tool_result_boundary_no_op_even_when_identity_differs() -> None:
+ tracker = [tool_step_identity("type_text", {"selector": "#a"})]
+
+ record_consecutive_tool_result_boundary(tracker, "type_text", {"selector": "#b"})
+
+ assert tracker == [tool_step_identity("type_text", {"selector": "#a"})]
def test_different_tool_result_boundary_resets_to_new_tool() -> None:
- tracker = ["update_workflow", "update_workflow"]
+ tracker = [tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]
record_consecutive_tool_result_boundary(tracker, "inspect_page_for_composition")
- assert tracker == ["inspect_page_for_composition"]
+ assert tracker == [tool_step_identity("inspect_page_for_composition")]
+
+
+def test_cross_name_result_boundary_seeds_full_argument_identity() -> None:
+ tracker = [tool_step_identity("update_workflow")]
+
+ record_consecutive_tool_result_boundary(tracker, "type_text", {"selector": "#c"})
+
+ assert tracker == [tool_step_identity("type_text", {"selector": "#c"})]
def test_result_recording_makes_normal_tool_body_boundaries_visible() -> None:
- ctx = SimpleNamespace(consecutive_tool_tracker=["update_workflow", "update_workflow"])
+ ctx = SimpleNamespace(
+ consecutive_tool_tracker=[tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]
+ )
record_tool_step_result_for_ctx(
ctx,
@@ -81,7 +119,37 @@ def test_result_recording_makes_normal_tool_body_boundaries_visible() -> None:
{"ok": True, "data": {"summary": "observed"}},
)
- assert ctx.consecutive_tool_tracker == ["inspect_page_for_composition"]
+ assert ctx.consecutive_tool_tracker == [
+ tool_step_identity("inspect_page_for_composition", {"target_url": "current_page"})
+ ]
+
+
+def test_adversarial_same_name_interleave_is_completion_order_insensitive() -> None:
+ tracker: list[str] = []
+ args_x = {"selector": "#x"}
+ args_y = {"selector": "#y"}
+
+ assert detect_tool_loop(tracker, "type_text", args_x) is None
+ assert detect_tool_loop(tracker, "type_text", args_y) is None
+ record_consecutive_tool_result_boundary(tracker, "type_text", args_x)
+ assert detect_tool_loop(tracker, "type_text", args_x) is None
+ assert detect_tool_loop(tracker, "type_text", args_x) is None
+
+ assert tracker == [tool_step_identity("type_text", args_x), tool_step_identity("type_text", args_x)]
+
+
+def test_interleaved_same_name_completions_still_fire_at_third_admission() -> None:
+ tracker: list[str] = []
+ args = {"selector": "#stuck"}
+
+ assert detect_tool_loop(tracker, "type_text", args) is None
+ record_consecutive_tool_result_boundary(tracker, "type_text", args)
+ assert detect_tool_loop(tracker, "type_text", args) is None
+ record_consecutive_tool_result_boundary(tracker, "type_text", args)
+ msg = detect_tool_loop(tracker, "type_text", args)
+
+ assert msg is not None
+ assert "LOOP DETECTED" in msg
def test_workflow_progress_result_boundary_clears_tracker() -> None:
@@ -100,7 +168,9 @@ def test_workflow_progress_result_boundary_clears_tracker() -> None:
def test_get_run_results_workflow_run_id_is_not_run_creation_progress() -> None:
- ctx = SimpleNamespace(consecutive_tool_tracker=["update_workflow", "update_workflow"])
+ ctx = SimpleNamespace(
+ consecutive_tool_tracker=[tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]
+ )
record_consecutive_tool_result_boundary_for_ctx(
ctx,
@@ -108,7 +178,7 @@ def test_get_run_results_workflow_run_id_is_not_run_creation_progress() -> None:
{"ok": True, "data": {"workflow_run_id": "wr_123"}},
)
- assert ctx.consecutive_tool_tracker == ["get_run_results"]
+ assert ctx.consecutive_tool_tracker == [tool_step_identity("get_run_results")]
def test_telco_mixed_dispatch_stream_does_not_trigger_consecutive_update_halt() -> None:
@@ -141,14 +211,14 @@ def test_telco_mixed_dispatch_stream_does_not_trigger_consecutive_update_halt()
"update_and_run_blocks",
{"ok": False, "error": "guardrail blocked the run"},
)
- assert ctx.consecutive_tool_tracker == ["update_and_run_blocks"]
+ assert ctx.consecutive_tool_tracker == [tool_step_identity("update_and_run_blocks")]
record_consecutive_tool_result_boundary_for_ctx(
ctx,
"inspect_page_for_composition",
{"ok": True, "data": {"observation_step": 4}},
)
- assert ctx.consecutive_tool_tracker == ["inspect_page_for_composition"]
+ assert ctx.consecutive_tool_tracker == [tool_step_identity("inspect_page_for_composition")]
assert detect_tool_loop(ctx.consecutive_tool_tracker, "update_workflow") is None
@@ -177,13 +247,13 @@ def test_diagnosis_repair_tool_error_records_consecutive_boundary_only() -> None
browser_session_id=None,
stream=MagicMock(),
)
- ctx.consecutive_tool_tracker = ["update_workflow", "update_workflow"]
+ ctx.consecutive_tool_tracker = [tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]
ctx.failed_tool_step_tracker = {"sentinel": 2}
payload = json.loads(_diagnosis_repair_tool_error(ctx, "run_blocks_and_collect_debug", "blocked"))
assert payload == {"ok": False, "error": "blocked"}
- assert ctx.consecutive_tool_tracker == ["run_blocks_and_collect_debug"]
+ assert ctx.consecutive_tool_tracker == [tool_step_identity("run_blocks_and_collect_debug")]
assert ctx.failed_tool_step_tracker == {"sentinel": 2}
diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.py
index 0af98aa18..6b350f842 100644
--- a/tests/unit/test_copilot_loop_detection_blocker.py
+++ b/tests/unit/test_copilot_loop_detection_blocker.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import json
from types import SimpleNamespace
@@ -17,6 +18,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+from skyvern.forge.sdk.copilot.loop_detection import tool_step_identity
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer, _stash_and_emit_loop_blocker
from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
@@ -49,6 +51,11 @@ def _ctx(
return ctx
+def _streak(tool_name: str, arguments: dict[str, object] | None = None) -> list[str]:
+ identity = tool_step_identity(tool_name, arguments)
+ return [identity, identity]
+
+
def _current_page_challenge_evidence(*, observed_after_workflow_run: bool = True) -> dict[str, object]:
evidence: dict[str, object] = {
"current_url": "https://example.test/search",
@@ -131,7 +138,7 @@ def test_native_dispatch_failed_step_loop_sets_signal_and_returns_payload() -> N
def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:
- ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))
payload = _tool_loop_error(ctx, "list_credentials", None)
assert payload is not None
assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
@@ -142,7 +149,7 @@ def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:
def test_consecutive_evaluate_loop_with_loaded_results_uses_goal_aware_copy() -> None:
- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate"))
ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(
result_container_count=1,
table_result_container_count=1,
@@ -161,7 +168,7 @@ def test_consecutive_evaluate_loop_with_loaded_results_uses_goal_aware_copy() ->
def test_consecutive_non_evaluate_loop_ignores_loaded_result_steer() -> None:
- ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))
ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(
result_container_count=1,
table_result_container_count=1,
@@ -179,7 +186,7 @@ def test_consecutive_non_evaluate_loop_ignores_loaded_result_steer() -> None:
def test_consecutive_evaluate_loop_without_composition_steer_uses_generic_copy() -> None:
- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate"))
payload = _tool_loop_error(ctx, "evaluate", None)
assert payload is not None
@@ -448,7 +455,7 @@ def test_fixed_tier_copy_is_clean_for_every_loop_prone_tool(blocked_tool: str, r
def test_native_and_mcp_paths_carry_equivalent_evidence_bearing_signals() -> None:
def _prepped_ctx() -> CopilotContext:
- ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))
ctx.last_outcome_gate_reason = (
"The run completed but did not demonstrate the goal outcome(s): the requested record is checked "
"on a public registry site with a search form and expandable result rows."
@@ -485,7 +492,7 @@ def test_native_and_mcp_paths_carry_equivalent_evidence_bearing_signals() -> Non
def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() -> None:
- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))
ctx.composition_page_evidence = _current_page_challenge_evidence()
payload = _tool_loop_error(ctx, "evaluate", {"expression": "document.body.innerText"})
@@ -501,7 +508,7 @@ def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() ->
def test_native_tool_loop_error_does_not_preempt_on_pre_attempt_challenge() -> None:
- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))
ctx.composition_page_evidence = _current_page_challenge_evidence(observed_after_workflow_run=False)
payload = _tool_loop_error(ctx, "evaluate", {"expression": "document.body.innerText"})
@@ -517,7 +524,7 @@ def test_native_tool_loop_error_does_not_preempt_on_pre_attempt_challenge() -> N
@pytest.mark.asyncio
async def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() -> None:
- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))
ctx.composition_page_evidence = _current_page_challenge_evidence()
class _UnexpectedClient:
@@ -549,7 +556,7 @@ async def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() ->
@pytest.mark.asyncio
async def test_mcp_synthesized_offer_gate_preempts_evaluate_loop_without_terminal_halt() -> None:
- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])
+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))
ctx.turn_intent = TurnIntent(
mode=TurnIntentMode.BUILD,
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
@@ -697,3 +704,109 @@ def test_composed_loop_reply_with_draft_affordance_passes_output_policy() -> Non
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
)
assert verdict.allowed, [code.value for code in verdict.reason_codes]
+
+
+def _dispatch_server(ctx: CopilotContext, client: object) -> SkyvernOverlayMCPServer:
+ server = SkyvernOverlayMCPServer(
+ transport=None,
+ overlays={},
+ alias_map={},
+ allowlist=frozenset({"evaluate"}),
+ context_provider=lambda: ctx,
+ )
+ server._client = client # type: ignore[assignment]
+ return server
+
+
+class _OkResult:
+ structured_content = {"ok": True, "data": {}}
+ is_error = False
+ content: list = []
+
+
+async def test_mcp_admission_blocks_third_identical_identity_without_dispatch() -> None:
+ args = {"expression": "document.title"}
+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", args))
+
+ class _UnexpectedClient:
+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:
+ raise AssertionError("consecutive loop blocker should skip MCP execution")
+
+ result = await _dispatch_server(ctx, _UnexpectedClient()).call_tool("evaluate", args)
+
+ parsed = json.loads(result.content[0].text)
+ assert result.isError is True
+ assert parsed["ok"] is False
+ signal = ctx.blocker_signal
+ assert isinstance(signal, CopilotToolBlockerSignal)
+ assert signal.internal_reason_code == "loop_detected_consecutive_same_tool"
+
+
+async def test_mcp_admission_admits_distinct_argument_identity() -> None:
+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "a"}))
+ calls: list[dict] = []
+
+ class _RecordingClient:
+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:
+ calls.append(a)
+ return _OkResult()
+
+ result = await _dispatch_server(ctx, _RecordingClient()).call_tool("evaluate", {"expression": "b"})
+
+ parsed = json.loads(result.content[0].text)
+ assert parsed["ok"] is True
+ assert len(calls) == 1
+ assert ctx.blocker_signal is None
+
+
+async def test_mcp_exception_completion_leaves_same_consecutive_state_as_success() -> None:
+ args = {"expression": "document.title"}
+
+ success_ctx = _ctx(consecutive_tool_tracker=[])
+
+ class _OkClient:
+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:
+ return _OkResult()
+
+ await _dispatch_server(success_ctx, _OkClient()).call_tool("evaluate", args)
+
+ error_ctx = _ctx(consecutive_tool_tracker=[])
+
+ class _RaisingClient:
+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:
+ raise RuntimeError("boom")
+
+ result = await _dispatch_server(error_ctx, _RaisingClient()).call_tool("evaluate", args)
+ assert json.loads(result.content[0].text)["ok"] is False
+
+ assert error_ctx.consecutive_tool_tracker == success_ctx.consecutive_tool_tracker
+ assert success_ctx.consecutive_tool_tracker == [tool_step_identity("evaluate", args)]
+
+
+async def test_mcp_cancelled_admission_matches_error_completion_state() -> None:
+ args = {"expression": "document.title"}
+
+ error_ctx = _ctx(consecutive_tool_tracker=[], failed_tool_step_tracker={})
+
+ class _RaisingClient:
+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:
+ raise RuntimeError("boom")
+
+ await _dispatch_server(error_ctx, _RaisingClient()).call_tool("evaluate", args)
+
+ cancel_ctx = _ctx(consecutive_tool_tracker=[], failed_tool_step_tracker={})
+
+ class _CancellingClient:
+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:
+ raise asyncio.CancelledError
+
+ with pytest.raises(asyncio.CancelledError):
+ await _dispatch_server(cancel_ctx, _CancellingClient()).call_tool("evaluate", args)
+
+ assert cancel_ctx.consecutive_tool_tracker == error_ctx.consecutive_tool_tracker
+ assert cancel_ctx.consecutive_tool_tracker == [tool_step_identity("evaluate", args)]
+ assert cancel_ctx.failed_tool_step_tracker == {}
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 898ea3e3a..141cf3d25 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -21,6 +21,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
PRESENT_COMPLETION_CONTRACT_ASK_RETRY,
_response_coverage_nudge,
)
+from skyvern.forge.sdk.copilot.loop_detection import tool_step_identity
from skyvern.forge.sdk.copilot.output_contracts import OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE
from skyvern.forge.sdk.copilot.output_policy import (
ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
@@ -2077,26 +2078,23 @@ async def test_workflow_mutation_tools_have_sdk_input_guardrails_and_reject_raw_
assert guarded_tools["update_and_run_blocks"].tool_input_guardrails == [_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL]
ctx = _ctx()
- ctx.consecutive_tool_tracker = ["update_workflow", "update_workflow"]
+ ctx.consecutive_tool_tracker = [tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]
ctx.failed_tool_step_tracker = {"sentinel": 2}
- result = await _WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL.run(
- ToolInputGuardrailData(
- context=ToolContext(
- context=ctx,
- tool_name="update_and_run_blocks",
- tool_call_id="call-1",
- tool_arguments=json.dumps(
- {
- "workflow_yaml": """
+ rejected_yaml = """
workflow_definition:
blocks:
- block_type: navigation
label: login
navigation_goal: Type password: hunter2 into the password field.
"""
- }
- ),
+ result = await _WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL.run(
+ ToolInputGuardrailData(
+ context=ToolContext(
+ context=ctx,
+ tool_name="update_and_run_blocks",
+ tool_call_id="call-1",
+ tool_arguments=json.dumps({"workflow_yaml": rejected_yaml}),
),
agent=SimpleNamespace(),
)
@@ -2104,7 +2102,9 @@ workflow_definition:
assert result.behavior["type"] == "reject_content"
assert "raw_secret_leak" in result.behavior["message"]
- assert ctx.consecutive_tool_tracker == ["update_and_run_blocks"]
+ assert ctx.consecutive_tool_tracker == [
+ tool_step_identity("update_and_run_blocks", {"workflow_yaml": rejected_yaml})
+ ]
assert ctx.failed_tool_step_tracker == {"sentinel": 2}
diff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.py
index a592fba8c..8a97e4634 100644
--- a/tests/unit/test_copilot_tool_loop_error_block_running.py
+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py
@@ -15,6 +15,7 @@ from typing import Any
from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
from skyvern.forge.sdk.copilot.failure_tracking import compute_action_sequence_fingerprint
+from skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctx
from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState
from skyvern.forge.sdk.copilot.tools import (
BLOCK_RUNNING_TOOLS,
@@ -136,6 +137,22 @@ def test_planning_tool_still_trips_name_only_streak() -> None:
assert "LOOP DETECTED" in msg
+def test_native_site_distinct_arguments_do_not_trip_streak() -> None:
+ ctx = _ctx()
+ for value in ("a", "b", "c", "d"):
+ assert _tool_loop_error(ctx, "update_workflow", {"workflow_yaml": value}) is None
+
+
+def test_exempt_tool_completion_breaks_non_exempt_streak() -> None:
+ ctx = _ctx()
+ assert _tool_loop_error(ctx, "update_workflow") is None
+ assert _tool_loop_error(ctx, "update_workflow") is None
+ record_consecutive_tool_result_boundary_for_ctx(
+ ctx, "fill_credential_field", {"ok": True, "data": {}}, {"field": "username"}
+ )
+ assert _tool_loop_error(ctx, "update_workflow") is None
+
+
def test_unresolved_output_contract_ladder_skips_name_only_streak_for_authoring_tool() -> None:
ctx = _ctx(output_contract_actuation_count_by_signature={"sig_active": 1})
for _ in range(5):