Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-12357: Make Workflow Copilot loop guard argument-aware so multi-field builds are not halted as loops (#7355)
Andrew Neilson6ff4c6b0b1df
diff --git a/skyvern/forge/sdk/copilot/loop_detection.py b/skyvern/forge/sdk/copilot/loop_detection.pyindex 2843c399c..9fdfcb8e2 100644--- a/skyvern/forge/sdk/copilot/loop_detection.py+++ b/skyvern/forge/sdk/copilot/loop_detection.py@@ -2,9 +2,9 @@Two independent guards:-* ``detect_tool_loop`` fires on strictly consecutive same-tool streaks- (A-A-A). Resets the moment the tool name changes, so oscillating- patterns (A-B-A-B) bypass it by design.+* ``detect_tool_loop`` fires on strictly consecutive same-identity streaks+ (A-A-A over one tool_step_identity). Resets the moment the identity+ changes, so oscillating and argument-distinct patterns bypass it by design.* ``detect_failed_tool_step_loop`` fires on N repeated failures of thesame (tool, args) pair, even when other tools dispatch in between.Block-running credential/config failures are keyed by failure category@@ -53,10 +53,12 @@ WORKFLOW_RUN_CREATION_TOOL_NAMES = frozenset({"run_blocks_and_collect_debug", "udef detect_tool_loop(tracker: list[str],tool_name: str,+ arguments: Mapping[str, Any] | None = None,threshold: int = MAX_CONSECUTIVE_SAME_TOOL,) -> str | None:- """Track tool invocation order and return a loop error message when threshold is hit."""- tracker.append(tool_name)+ """Track tool step identities and return a loop error message when threshold is hit."""+ identity = tool_step_identity(tool_name, arguments)+ tracker.append(identity)if len(tracker) >= threshold and len(set(tracker[-threshold:])) == 1:tracker.clear()@@ -69,16 +71,23 @@ def detect_tool_loop(if len(tracker) >= 2 and tracker[-1] != tracker[-2]:tracker.clear()- tracker.append(tool_name)+ tracker.append(identity)return None-def record_consecutive_tool_result_boundary(tracker: list[str], tool_name: str) -> None:- if tracker and tracker[-1] == tool_name:+def record_consecutive_tool_result_boundary(+ tracker: list[str],+ tool_name: str,+ arguments: Mapping[str, Any] | None = None,+) -> None:+ # No-op on a same-name boundary regardless of arguments: streak identity is owned by+ # admission-time detect_tool_loop, so re-seeding on completion order could clobber a+ # live argument-distinct streak.+ if tracker and tracker[-1].startswith(f"{tool_name}:"):returntracker.clear()- tracker.append(tool_name)+ tracker.append(tool_step_identity(tool_name, arguments))def _result_records_workflow_progress(tool_name: str, result: Mapping[str, Any]) -> bool:@@ -100,6 +109,7 @@ def record_consecutive_tool_result_boundary_for_ctx(ctx: Any,tool_name: str,result: Mapping[str, Any],+ arguments: Mapping[str, Any] | None = None,) -> None:tracker = getattr(ctx, "consecutive_tool_tracker", None)if not isinstance(tracker, list):@@ -107,7 +117,7 @@ def record_consecutive_tool_result_boundary_for_ctx(if _result_records_workflow_progress(tool_name, result):tracker.clear()return- record_consecutive_tool_result_boundary(tracker, tool_name)+ record_consecutive_tool_result_boundary(tracker, tool_name, arguments)def _normalize_step_argument(value: Any) -> Any:@@ -285,7 +295,7 @@ def record_tool_step_result_for_ctx(tracker = _ctx_failed_step_tracker(ctx)if tracker is not None:record_tool_step_result(tracker, tool_name, arguments, result)- record_consecutive_tool_result_boundary_for_ctx(ctx, tool_name, result)+ record_consecutive_tool_result_boundary_for_ctx(ctx, tool_name, result, arguments)# Strict ``is True`` check: a malformed result dict missing ``ok`` entirely# must not be treated as success and accidentally clear a blocker signal.if result.get("ok") is True:diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex f67b3f381..dba02cf08 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -351,7 +351,7 @@ class SkyvernOverlayMCPServer(MCPServer):return _copilot_to_call_tool_result({"ok": False, "error": payload})tracker = getattr(copilot_ctx, "consecutive_tool_tracker", None)- loop_error = detect_tool_loop(tracker, tool_name) if isinstance(tracker, list) else None+ loop_error = detect_tool_loop(tracker, tool_name, arguments) if isinstance(tracker, list) else Noneif loop_error:LOG.warning("Tool loop detected, skipping execution",diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.pyindex 21d35d6a7..5e57355f0 100644--- a/skyvern/forge/sdk/copilot/tools/blockers.py+++ b/skyvern/forge/sdk/copilot/tools/blockers.py@@ -1568,7 +1568,7 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Anyand tool_name not in _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLSand not output_contract_owns_turn):- detected = detect_tool_loop(tracker, tool_name)+ detected = detect_tool_loop(tracker, tool_name, arguments)if detected is not None:return _emit_tool_blocker_signal(ctx,diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.pyindex 8e1de1433..ef8503f74 100644--- a/skyvern/forge/sdk/copilot/tools/guardrails.py+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py@@ -91,6 +91,7 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Toolgetattr(tool_context, "context", None),tool_name,{"ok": False, "error": error},+ arguments=tool_arguments,)return ToolGuardrailFunctionOutput.reject_content(error, output_info=trace_data)return ToolGuardrailFunctionOutput.allow(output_info=trace_data)diff --git a/tests/unit/test_copilot_loop_detection.py b/tests/unit/test_copilot_loop_detection.pyindex b1f744d2d..68475cd3d 100644--- a/tests/unit/test_copilot_loop_detection.py+++ b/tests/unit/test_copilot_loop_detection.py@@ -13,6 +13,7 @@ from skyvern.forge.sdk.copilot.loop_detection import (record_consecutive_tool_result_boundary_for_ctx,record_tool_step_result,record_tool_step_result_for_ctx,+ tool_step_identity,)from skyvern.forge.sdk.copilot.tools.run_execution import _diagnosis_repair_tool_error@@ -21,7 +22,7 @@ def test_returns_none_below_threshold() -> None:tracker: list[str] = []assert detect_tool_loop(tracker, "click") is Noneassert detect_tool_loop(tracker, "click") is None- assert tracker == ["click", "click"]+ assert tracker == [tool_step_identity("click"), tool_step_identity("click")]def test_fires_at_threshold_and_clears_tracker() -> None:@@ -41,7 +42,7 @@ def test_tool_switch_resets_tracker() -> None:detect_tool_loop(tracker, "click")detect_tool_loop(tracker, "click")assert detect_tool_loop(tracker, "type_text") is None- assert tracker == ["type_text"]+ assert tracker == [tool_step_identity("type_text")]def test_requires_full_fresh_threshold_after_warning() -> None:@@ -55,24 +56,61 @@ def test_requires_full_fresh_threshold_after_warning() -> None:assert detect_tool_loop(tracker, "click") is not None+def test_distinct_argument_identities_never_fire_over_repeated_same_tool() -> None:+ tracker: list[str] = []+ for index in range(5):+ assert detect_tool_loop(tracker, "type_text", {"selector": f"#field-{index}", "text": f"v{index}"}) is None+ assert tracker == [tool_step_identity("type_text", {"selector": "#field-4", "text": "v4"})]+++def test_third_identical_identity_fires_and_clears() -> None:+ tracker: list[str] = []+ args = {"selector": "#name", "text": "Ada"}+ assert detect_tool_loop(tracker, "type_text", args) is None+ assert detect_tool_loop(tracker, "type_text", args) is None+ msg = detect_tool_loop(tracker, "type_text", args)+ assert msg is not None+ assert "LOOP DETECTED" in msg+ assert "type_text" in msg+ assert tracker == []++def test_same_tool_result_boundary_does_not_double_count_dispatch() -> None:- tracker = ["update_workflow"]+ tracker = [tool_step_identity("update_workflow")]record_consecutive_tool_result_boundary(tracker, "update_workflow")- assert tracker == ["update_workflow"]+ assert tracker == [tool_step_identity("update_workflow")]+++def test_same_tool_result_boundary_no_op_even_when_identity_differs() -> None:+ tracker = [tool_step_identity("type_text", {"selector": "#a"})]++ record_consecutive_tool_result_boundary(tracker, "type_text", {"selector": "#b"})++ assert tracker == [tool_step_identity("type_text", {"selector": "#a"})]def test_different_tool_result_boundary_resets_to_new_tool() -> None:- tracker = ["update_workflow", "update_workflow"]+ tracker = [tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]record_consecutive_tool_result_boundary(tracker, "inspect_page_for_composition")- assert tracker == ["inspect_page_for_composition"]+ assert tracker == [tool_step_identity("inspect_page_for_composition")]+++def test_cross_name_result_boundary_seeds_full_argument_identity() -> None:+ tracker = [tool_step_identity("update_workflow")]++ record_consecutive_tool_result_boundary(tracker, "type_text", {"selector": "#c"})++ assert tracker == [tool_step_identity("type_text", {"selector": "#c"})]def test_result_recording_makes_normal_tool_body_boundaries_visible() -> None:- ctx = SimpleNamespace(consecutive_tool_tracker=["update_workflow", "update_workflow"])+ ctx = SimpleNamespace(+ consecutive_tool_tracker=[tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]+ )record_tool_step_result_for_ctx(ctx,@@ -81,7 +119,37 @@ def test_result_recording_makes_normal_tool_body_boundaries_visible() -> None:{"ok": True, "data": {"summary": "observed"}},)- assert ctx.consecutive_tool_tracker == ["inspect_page_for_composition"]+ assert ctx.consecutive_tool_tracker == [+ tool_step_identity("inspect_page_for_composition", {"target_url": "current_page"})+ ]+++def test_adversarial_same_name_interleave_is_completion_order_insensitive() -> None:+ tracker: list[str] = []+ args_x = {"selector": "#x"}+ args_y = {"selector": "#y"}++ assert detect_tool_loop(tracker, "type_text", args_x) is None+ assert detect_tool_loop(tracker, "type_text", args_y) is None+ record_consecutive_tool_result_boundary(tracker, "type_text", args_x)+ assert detect_tool_loop(tracker, "type_text", args_x) is None+ assert detect_tool_loop(tracker, "type_text", args_x) is None++ assert tracker == [tool_step_identity("type_text", args_x), tool_step_identity("type_text", args_x)]+++def test_interleaved_same_name_completions_still_fire_at_third_admission() -> None:+ tracker: list[str] = []+ args = {"selector": "#stuck"}++ assert detect_tool_loop(tracker, "type_text", args) is None+ record_consecutive_tool_result_boundary(tracker, "type_text", args)+ assert detect_tool_loop(tracker, "type_text", args) is None+ record_consecutive_tool_result_boundary(tracker, "type_text", args)+ msg = detect_tool_loop(tracker, "type_text", args)++ assert msg is not None+ assert "LOOP DETECTED" in msgdef test_workflow_progress_result_boundary_clears_tracker() -> None:@@ -100,7 +168,9 @@ def test_workflow_progress_result_boundary_clears_tracker() -> None:def test_get_run_results_workflow_run_id_is_not_run_creation_progress() -> None:- ctx = SimpleNamespace(consecutive_tool_tracker=["update_workflow", "update_workflow"])+ ctx = SimpleNamespace(+ consecutive_tool_tracker=[tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]+ )record_consecutive_tool_result_boundary_for_ctx(ctx,@@ -108,7 +178,7 @@ def test_get_run_results_workflow_run_id_is_not_run_creation_progress() -> None:{"ok": True, "data": {"workflow_run_id": "wr_123"}},)- assert ctx.consecutive_tool_tracker == ["get_run_results"]+ assert ctx.consecutive_tool_tracker == [tool_step_identity("get_run_results")]def test_telco_mixed_dispatch_stream_does_not_trigger_consecutive_update_halt() -> None:@@ -141,14 +211,14 @@ def test_telco_mixed_dispatch_stream_does_not_trigger_consecutive_update_halt()"update_and_run_blocks",{"ok": False, "error": "guardrail blocked the run"},)- assert ctx.consecutive_tool_tracker == ["update_and_run_blocks"]+ assert ctx.consecutive_tool_tracker == [tool_step_identity("update_and_run_blocks")]record_consecutive_tool_result_boundary_for_ctx(ctx,"inspect_page_for_composition",{"ok": True, "data": {"observation_step": 4}},)- assert ctx.consecutive_tool_tracker == ["inspect_page_for_composition"]+ assert ctx.consecutive_tool_tracker == [tool_step_identity("inspect_page_for_composition")]assert detect_tool_loop(ctx.consecutive_tool_tracker, "update_workflow") is None@@ -177,13 +247,13 @@ def test_diagnosis_repair_tool_error_records_consecutive_boundary_only() -> Nonebrowser_session_id=None,stream=MagicMock(),)- ctx.consecutive_tool_tracker = ["update_workflow", "update_workflow"]+ ctx.consecutive_tool_tracker = [tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]ctx.failed_tool_step_tracker = {"sentinel": 2}payload = json.loads(_diagnosis_repair_tool_error(ctx, "run_blocks_and_collect_debug", "blocked"))assert payload == {"ok": False, "error": "blocked"}- assert ctx.consecutive_tool_tracker == ["run_blocks_and_collect_debug"]+ assert ctx.consecutive_tool_tracker == [tool_step_identity("run_blocks_and_collect_debug")]assert ctx.failed_tool_step_tracker == {"sentinel": 2}diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.pyindex 0af98aa18..6b350f842 100644--- a/tests/unit/test_copilot_loop_detection_blocker.py+++ b/tests/unit/test_copilot_loop_detection_blocker.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport jsonfrom types import SimpleNamespace@@ -17,6 +18,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE+from skyvern.forge.sdk.copilot.loop_detection import tool_step_identityfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer, _stash_and_emit_loop_blockerfrom skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policyfrom skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence@@ -49,6 +51,11 @@ def _ctx(return ctx+def _streak(tool_name: str, arguments: dict[str, object] | None = None) -> list[str]:+ identity = tool_step_identity(tool_name, arguments)+ return [identity, identity]++def _current_page_challenge_evidence(*, observed_after_workflow_run: bool = True) -> dict[str, object]:evidence: dict[str, object] = {"current_url": "https://example.test/search",@@ -131,7 +138,7 @@ def test_native_dispatch_failed_step_loop_sets_signal_and_returns_payload() -> Ndef test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:- ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])+ ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))payload = _tool_loop_error(ctx, "list_credentials", None)assert payload is not Noneassert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)@@ -142,7 +149,7 @@ def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:def test_consecutive_evaluate_loop_with_loaded_results_uses_goal_aware_copy() -> None:- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate"))ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(result_container_count=1,table_result_container_count=1,@@ -161,7 +168,7 @@ def test_consecutive_evaluate_loop_with_loaded_results_uses_goal_aware_copy() ->def test_consecutive_non_evaluate_loop_ignores_loaded_result_steer() -> None:- ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])+ ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(result_container_count=1,table_result_container_count=1,@@ -179,7 +186,7 @@ def test_consecutive_non_evaluate_loop_ignores_loaded_result_steer() -> None:def test_consecutive_evaluate_loop_without_composition_steer_uses_generic_copy() -> None:- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate"))payload = _tool_loop_error(ctx, "evaluate", None)assert payload is not None@@ -448,7 +455,7 @@ def test_fixed_tier_copy_is_clean_for_every_loop_prone_tool(blocked_tool: str, rdef test_native_and_mcp_paths_carry_equivalent_evidence_bearing_signals() -> None:def _prepped_ctx() -> CopilotContext:- ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])+ ctx = _ctx(consecutive_tool_tracker=_streak("list_credentials"))ctx.last_outcome_gate_reason = ("The run completed but did not demonstrate the goal outcome(s): the requested record is checked ""on a public registry site with a search form and expandable result rows."@@ -485,7 +492,7 @@ def test_native_and_mcp_paths_carry_equivalent_evidence_bearing_signals() -> Nondef test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() -> None:- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))ctx.composition_page_evidence = _current_page_challenge_evidence()payload = _tool_loop_error(ctx, "evaluate", {"expression": "document.body.innerText"})@@ -501,7 +508,7 @@ def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() ->def test_native_tool_loop_error_does_not_preempt_on_pre_attempt_challenge() -> None:- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))ctx.composition_page_evidence = _current_page_challenge_evidence(observed_after_workflow_run=False)payload = _tool_loop_error(ctx, "evaluate", {"expression": "document.body.innerText"})@@ -517,7 +524,7 @@ def test_native_tool_loop_error_does_not_preempt_on_pre_attempt_challenge() -> N@pytest.mark.asyncioasync def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() -> None:- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))ctx.composition_page_evidence = _current_page_challenge_evidence()class _UnexpectedClient:@@ -549,7 +556,7 @@ async def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() ->@pytest.mark.asyncioasync def test_mcp_synthesized_offer_gate_preempts_evaluate_loop_without_terminal_halt() -> None:- ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))ctx.turn_intent = TurnIntent(mode=TurnIntentMode.BUILD,authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),@@ -697,3 +704,109 @@ def test_composed_loop_reply_with_draft_affordance_passes_output_policy() -> Nonoutput_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,)assert verdict.allowed, [code.value for code in verdict.reason_codes]+++def _dispatch_server(ctx: CopilotContext, client: object) -> SkyvernOverlayMCPServer:+ server = SkyvernOverlayMCPServer(+ transport=None,+ overlays={},+ alias_map={},+ allowlist=frozenset({"evaluate"}),+ context_provider=lambda: ctx,+ )+ server._client = client # type: ignore[assignment]+ return server+++class _OkResult:+ structured_content = {"ok": True, "data": {}}+ is_error = False+ content: list = []+++async def test_mcp_admission_blocks_third_identical_identity_without_dispatch() -> None:+ args = {"expression": "document.title"}+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", args))++ class _UnexpectedClient:+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:+ raise AssertionError("consecutive loop blocker should skip MCP execution")++ result = await _dispatch_server(ctx, _UnexpectedClient()).call_tool("evaluate", args)++ parsed = json.loads(result.content[0].text)+ assert result.isError is True+ assert parsed["ok"] is False+ signal = ctx.blocker_signal+ assert isinstance(signal, CopilotToolBlockerSignal)+ assert signal.internal_reason_code == "loop_detected_consecutive_same_tool"+++async def test_mcp_admission_admits_distinct_argument_identity() -> None:+ ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "a"}))+ calls: list[dict] = []++ class _RecordingClient:+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:+ calls.append(a)+ return _OkResult()++ result = await _dispatch_server(ctx, _RecordingClient()).call_tool("evaluate", {"expression": "b"})++ parsed = json.loads(result.content[0].text)+ assert parsed["ok"] is True+ assert len(calls) == 1+ assert ctx.blocker_signal is None+++async def test_mcp_exception_completion_leaves_same_consecutive_state_as_success() -> None:+ args = {"expression": "document.title"}++ success_ctx = _ctx(consecutive_tool_tracker=[])++ class _OkClient:+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:+ return _OkResult()++ await _dispatch_server(success_ctx, _OkClient()).call_tool("evaluate", args)++ error_ctx = _ctx(consecutive_tool_tracker=[])++ class _RaisingClient:+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:+ raise RuntimeError("boom")++ result = await _dispatch_server(error_ctx, _RaisingClient()).call_tool("evaluate", args)+ assert json.loads(result.content[0].text)["ok"] is False++ assert error_ctx.consecutive_tool_tracker == success_ctx.consecutive_tool_tracker+ assert success_ctx.consecutive_tool_tracker == [tool_step_identity("evaluate", args)]+++async def test_mcp_cancelled_admission_matches_error_completion_state() -> None:+ args = {"expression": "document.title"}++ error_ctx = _ctx(consecutive_tool_tracker=[], failed_tool_step_tracker={})++ class _RaisingClient:+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:+ raise RuntimeError("boom")++ await _dispatch_server(error_ctx, _RaisingClient()).call_tool("evaluate", args)++ cancel_ctx = _ctx(consecutive_tool_tracker=[], failed_tool_step_tracker={})++ class _CancellingClient:+ async def call_tool(self, name: str, a: dict, raise_on_error: bool = False) -> object:+ raise asyncio.CancelledError++ with pytest.raises(asyncio.CancelledError):+ await _dispatch_server(cancel_ctx, _CancellingClient()).call_tool("evaluate", args)++ assert cancel_ctx.consecutive_tool_tracker == error_ctx.consecutive_tool_tracker+ assert cancel_ctx.consecutive_tool_tracker == [tool_step_identity("evaluate", args)]+ assert cancel_ctx.failed_tool_step_tracker == {}diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 898ea3e3a..141cf3d25 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -21,6 +21,7 @@ from skyvern.forge.sdk.copilot.enforcement import (PRESENT_COMPLETION_CONTRACT_ASK_RETRY,_response_coverage_nudge,)+from skyvern.forge.sdk.copilot.loop_detection import tool_step_identityfrom skyvern.forge.sdk.copilot.output_contracts import OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODEfrom skyvern.forge.sdk.copilot.output_policy import (ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,@@ -2077,26 +2078,23 @@ async def test_workflow_mutation_tools_have_sdk_input_guardrails_and_reject_raw_assert guarded_tools["update_and_run_blocks"].tool_input_guardrails == [_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL]ctx = _ctx()- ctx.consecutive_tool_tracker = ["update_workflow", "update_workflow"]+ ctx.consecutive_tool_tracker = [tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]ctx.failed_tool_step_tracker = {"sentinel": 2}- result = await _WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL.run(- ToolInputGuardrailData(- context=ToolContext(- context=ctx,- tool_name="update_and_run_blocks",- tool_call_id="call-1",- tool_arguments=json.dumps(- {- "workflow_yaml": """+ rejected_yaml = """workflow_definition:blocks:- block_type: navigationlabel: loginnavigation_goal: Type password: hunter2 into the password field."""- }- ),+ result = await _WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL.run(+ ToolInputGuardrailData(+ context=ToolContext(+ context=ctx,+ tool_name="update_and_run_blocks",+ tool_call_id="call-1",+ tool_arguments=json.dumps({"workflow_yaml": rejected_yaml}),),agent=SimpleNamespace(),)@@ -2104,7 +2102,9 @@ workflow_definition:assert result.behavior["type"] == "reject_content"assert "raw_secret_leak" in result.behavior["message"]- assert ctx.consecutive_tool_tracker == ["update_and_run_blocks"]+ assert ctx.consecutive_tool_tracker == [+ tool_step_identity("update_and_run_blocks", {"workflow_yaml": rejected_yaml})+ ]assert ctx.failed_tool_step_tracker == {"sentinel": 2}diff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.pyindex a592fba8c..8a97e4634 100644--- a/tests/unit/test_copilot_tool_loop_error_block_running.py+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py@@ -15,6 +15,7 @@ from typing import Anyfrom skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDSfrom skyvern.forge.sdk.copilot.failure_tracking import compute_action_sequence_fingerprint+from skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctxfrom skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryStatefrom skyvern.forge.sdk.copilot.tools import (BLOCK_RUNNING_TOOLS,@@ -136,6 +137,22 @@ def test_planning_tool_still_trips_name_only_streak() -> None:assert "LOOP DETECTED" in msg+def test_native_site_distinct_arguments_do_not_trip_streak() -> None:+ ctx = _ctx()+ for value in ("a", "b", "c", "d"):+ assert _tool_loop_error(ctx, "update_workflow", {"workflow_yaml": value}) is None+++def test_exempt_tool_completion_breaks_non_exempt_streak() -> None:+ ctx = _ctx()+ assert _tool_loop_error(ctx, "update_workflow") is None+ assert _tool_loop_error(ctx, "update_workflow") is None+ record_consecutive_tool_result_boundary_for_ctx(+ ctx, "fill_credential_field", {"ok": True, "data": {}}, {"field": "username"}+ )+ assert _tool_loop_error(ctx, "update_workflow") is None++def test_unresolved_output_contract_ladder_skips_name_only_streak_for_authoring_tool() -> None:ctx = _ctx(output_contract_actuation_count_by_signature={"sig_active": 1})for _ in range(5):