Skip to content

Automate browser based workflows with AI

fix(SKY-10987): keep run authority on explicit re-test diagnose turns (#6570)

Andrew Neilson394bec9ff63a
diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.py
index 4edb4e46e..002abd2a6 100644
--- a/skyvern/forge/sdk/copilot/turn_intent.py
+++ b/skyvern/forge/sdk/copilot/turn_intent.py
@@ -100,6 +100,7 @@ class TurnIntentReasonCode(StrEnum):
REQUEST_POLICY_DERIVED = "request_policy_derived"
REQUEST_POLICY_CLARIFICATION = "request_policy_clarification"
TESTING_INTENT_SKIP_TEST = "testing_intent_skip_test"
+ TESTING_INTENT_RUN_OVERRIDES_DIAGNOSE = "testing_intent_run_overrides_diagnose"
WORKFLOW_CONTEXT_PRESENT = "workflow_context_present"
CHAT_HISTORY_PRESENT = "chat_history_present"
RUN_CONTEXT_PRESENT = "run_context_present"
@@ -800,13 +801,18 @@ def build_turn_intent(
authority.may_update_workflow = False
authority.may_run_blocks = False
authority.requires_user_input = True
- elif mode == TurnIntentMode.DIAGNOSE and RequiredContextKey.LATEST_RUN_RESULT not in required_context:
- authority.may_update_workflow = False
- authority.may_run_blocks = False
- required_context.append(RequiredContextKey.LATEST_RUN_RESULT)
elif mode == TurnIntentMode.DIAGNOSE:
authority.may_update_workflow = False
- authority.may_run_blocks = False
+ retest_mandated = request_policy.testing_intent == "require_test" and request_policy.allow_run_blocks
+ if retest_mandated:
+ # RequestPolicy owns testing policy; an explicit require_test re-run must not be inverted
+ # by the diagnose classification.
+ authority.may_run_blocks = True
+ reason_codes.append(TurnIntentReasonCode.TESTING_INTENT_RUN_OVERRIDES_DIAGNOSE)
+ else:
+ authority.may_run_blocks = False
+ if RequiredContextKey.LATEST_RUN_RESULT not in required_context:
+ required_context.append(RequiredContextKey.LATEST_RUN_RESULT)
authority.may_read_run_context = mode == TurnIntentMode.DIAGNOSE
diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.py
index 7d9b96090..612eb59a0 100644
--- a/tests/unit/test_copilot_turn_intent.py
+++ b/tests/unit/test_copilot_turn_intent.py
@@ -240,6 +240,43 @@ def test_build_turn_intent_llm_diagnose_outranks_skip_test_policy() -> None:
assert TurnIntentReasonCode.TESTING_INTENT_SKIP_TEST not in intent.reason_codes
+def test_build_turn_intent_diagnose_with_require_test_keeps_run_authority() -> None:
+ intent = build_turn_intent(
+ user_message="Test it again and confirm what it extracted.",
+ workflow_yaml="title: Existing\nworkflow_definition:\n blocks: []\n",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(testing_intent="require_test", allow_update_workflow=True, allow_run_blocks=True),
+ workflow_run_id="wr_123",
+ classification=_classification(TurnIntentMode.DIAGNOSE),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert intent.authority.may_run_blocks is True
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_read_run_context is True
+ assert RequiredContextKey.LATEST_RUN_RESULT in intent.required_context
+ assert TurnIntentReasonCode.TESTING_INTENT_RUN_OVERRIDES_DIAGNOSE in intent.reason_codes
+
+
+def test_build_turn_intent_diagnose_without_require_test_stays_answer_only() -> None:
+ intent = build_turn_intent(
+ user_message="What did the last run extract?",
+ workflow_yaml="title: Existing\nworkflow_definition:\n blocks: []\n",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(allow_update_workflow=False, allow_run_blocks=False),
+ workflow_run_id="wr_123",
+ classification=_classification(TurnIntentMode.DIAGNOSE),
+ )
+
+ assert intent.mode == TurnIntentMode.DIAGNOSE
+ assert intent.authority.may_run_blocks is False
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_read_run_context is True
+ assert TurnIntentReasonCode.TESTING_INTENT_RUN_OVERRIDES_DIAGNOSE not in intent.reason_codes
+
+
def test_build_turn_intent_uses_request_policy_clarification_over_llm_classification() -> None:
policy = RequestPolicy(
user_response_policy="ask_clarification",
diff --git a/tests/unit/test_copilot_turn_intent_tool_gate.py b/tests/unit/test_copilot_turn_intent_tool_gate.py
index 8aeb0f345..8ae8bf7da 100644
--- a/tests/unit/test_copilot_turn_intent_tool_gate.py
+++ b/tests/unit/test_copilot_turn_intent_tool_gate.py
@@ -368,6 +368,19 @@ def test_update_and_run_blocks_reports_both_blocked_authorities() -> None:
)
+def test_turn_intent_gate_allows_diagnose_run_with_retest_authority() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.DIAGNOSE,
+ authority=TurnIntentAuthority(
+ may_update_workflow=False,
+ may_run_blocks=True,
+ may_read_run_context=True,
+ ),
+ )
+
+ assert _turn_intent_tool_error(_ctx(intent), "run_blocks_and_collect_debug") is None
+
+
def test_get_run_results_routes_through_authority_dispatcher_but_request_policy_does_not_gate_it() -> None:
"""Pins the dispatcher routing: `get_run_results` flows through `_authority_tool_error` (not `_turn_intent_tool_error` directly), and current request-policy scope (`update_workflow` + `BLOCK_RUNNING_TOOLS`) does not include `get_run_results`, so the call passes through."""
intent = TurnIntent(