Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(copilot): keep native tools registered for authority blockers (#6291)
Andrew Neilsone12da142ef46
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 9c5f67453..0b145dfac 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -78,7 +78,6 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (merge_failure_into_context,)from skyvern.forge.sdk.copilot.request_policy import (- CREDENTIAL_DEFERRED_DRAFT_REASONS,RAW_SECRET_REFUSAL_SENTINEL,RequestPolicy,build_request_policy,@@ -93,7 +92,6 @@ from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_from skyvern.forge.sdk.copilot.turn_context import TurnContextAssembler, TurnContextInputs, TurnContextPacketfrom skyvern.forge.sdk.copilot.turn_intent import (NO_MUTATION_TURN_INTENT_MODES,- READ_CONTEXT_DENIED_MODES,RequiredContextKey,TurnIntent,TurnIntentMode,@@ -589,34 +587,15 @@ def _turn_intent_disables_tools(turn_intent: TurnIntent | None) -> bool:return not authority.may_update_workflow and not authority.may_run_blocks-def _request_policy_requires_update_and_run_skip_path(request_policy: RequestPolicy | None) -> bool:- return (- isinstance(request_policy, RequestPolicy)- and request_policy.allow_update_workflow- and not request_policy.allow_run_blocks- and request_policy.allow_missing_credentials_in_draft- and request_policy.clarification_reason in CREDENTIAL_DEFERRED_DRAFT_REASONS- )--def _native_tools_for_turn(native_tools: list[Any],turn_intent: TurnIntent | None,request_policy: RequestPolicy | None = None,) -> list[Any]:- # Deferred: matches the existing tools.py <-> routes.workflow_copilot <-> agent.py cycle workaround.- from skyvern.forge.sdk.copilot.tools import ANSWER_ONLY_CONTEXT_TOOLS-- if _turn_intent_disables_tools(turn_intent):- if (- turn_intent is not None- and turn_intent.authority.may_read_run_context- and turn_intent.mode not in READ_CONTEXT_DENIED_MODES- ):- return [tool for tool in native_tools if getattr(tool, "name", None) in ANSWER_ONLY_CONTEXT_TOOLS]- return []- if _request_policy_requires_update_and_run_skip_path(request_policy):- return [tool for tool in native_tools if getattr(tool, "name", None) != "update_workflow"]+ # Keep native tools registered even when the current turn is not allowed to+ # use them. The tool implementations enforce TurnIntent/RequestPolicy+ # authority and return structured blockers; removing a tool lets the model+ # hit an SDK-level ModelBehaviorError if static prompt text still names it.return list(native_tools)diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 8c45ad964..e8f86f8a0 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -1219,6 +1219,7 @@ async def _attach_failed_block_screenshots(BLOCK_RUNNING_TOOLS = frozenset({"run_blocks_and_collect_debug", "update_and_run_blocks"})WORKFLOW_MUTATION_TOOLS = frozenset({"update_workflow", "update_and_run_blocks"})ANSWER_ONLY_CONTEXT_TOOLS = frozenset({"get_run_results"})+CREDENTIAL_METADATA_TOOLS = frozenset({"list_credentials"})PAGE_INSPECTION_TOOLS = frozenset({"inspect_page_for_composition", "evaluate", "get_browser_screenshot"})PAGE_SCHEMA_CONTEXT_TOOLS = frozenset({"inspect_page_for_composition"})_CURRENT_PAGE_INSPECTION_TARGETS = frozenset({"", "current", "current_page", "__current_page__"})@@ -1757,6 +1758,9 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBloblocks_page_inspection = tool_name in PAGE_SCHEMA_CONTEXT_TOOLS and (intent.mode in NO_MUTATION_TURN_INTENT_MODES or not authority.may_update_workflow)+ blocks_credential_metadata = tool_name in CREDENTIAL_METADATA_TOOLS and not (+ authority.may_update_workflow or authority.may_run_blocks+ )# Two paths grant read access to ANSWER_ONLY_CONTEXT_TOOLS, both excluded for DOCS_ANSWER/REFUSE/CLARIFY:# (1) authority.may_read_run_context — classifier-derived (DIAGNOSE turns)# (2) pending_reconciliation_run_id — within-turn override anchored to the run-blocks watchdog@@ -1781,7 +1785,13 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBloif blocks_run and not blocks_update and _request_policy_allows_update_and_skip_run(ctx, tool_name):return None- if not blocks_update and not blocks_run and not blocks_context_read and not blocks_page_inspection:+ if (+ not blocks_update+ and not blocks_run+ and not blocks_context_read+ and not blocks_page_inspection+ and not blocks_credential_metadata+ ):if within_turn_read_override:LOG.info("copilot authority gate allowed tool via within-turn read override",@@ -1804,6 +1814,8 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBloreason_code = "turn_intent_page_inspection_blocked"elif blocks_context_read:reason_code = "turn_intent_context_read_blocked"+ elif blocks_credential_metadata:+ reason_code = "turn_intent_credential_metadata_blocked"else:reason_code = "turn_intent_run_blocked"LOG.info(@@ -5293,6 +5305,12 @@ async def list_credentials_tool(if loop_error:return json.dumps({"ok": False, "error": loop_error})+ authority_error = _authority_tool_error(copilot_ctx, "list_credentials")+ if authority_error:+ result = {"ok": False, "error": authority_error}+ record_tool_step_result_for_ctx(copilot_ctx, "list_credentials", arguments, result)+ return json.dumps(result)+result = await _list_credentials(arguments, copilot_ctx)record_tool_step_result_for_ctx(copilot_ctx, "list_credentials", arguments, result)sanitized = sanitize_tool_result_for_llm("list_credentials", result)diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 7128f39fa..8fb3cb18d 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -1910,7 +1910,7 @@ class TestNativeToolSurface:assert "inspect_page_for_composition" in names@pytest.mark.parametrize("reason", ["workflow_credential_inputs_unbound", "credential_name_unresolved"])- def test_credential_deferred_draft_removes_update_workflow_tool(self, reason: str) -> None:+ def test_credential_deferred_draft_keeps_all_native_tools_registered(self, reason: str) -> None:from skyvern.forge.sdk.copilot.request_policy import RequestPolicypolicy = RequestPolicy(@@ -1928,7 +1928,7 @@ class TestNativeToolSurface:filtered = agent_module._native_tools_for_turn(tools, turn_intent=None, request_policy=policy)- assert [tool.name for tool in filtered] == ["list_credentials", "update_and_run_blocks"]+ assert [tool.name for tool in filtered] == ["update_workflow", "list_credentials", "update_and_run_blocks"]def test_non_deferred_policy_keeps_update_workflow_tool(self) -> None:from skyvern.forge.sdk.copilot.request_policy import RequestPolicydiff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.pyindex 5cb4eb40e..1ae017280 100644--- a/tests/unit/test_copilot_turn_intent.py+++ b/tests/unit/test_copilot_turn_intent.py@@ -617,7 +617,7 @@ def test_store_request_policy_attaches_turn_intent_to_context() -> None:assert ctx.turn_intent.authority.may_run_blocks is False-def test_answer_only_turn_intent_hides_get_run_results_tool() -> None:+def test_answer_only_turn_intent_keeps_native_tools_registered() -> None:tools = [SimpleNamespace(name="update_workflow"),SimpleNamespace(name="get_run_results"),@@ -630,7 +630,7 @@ def test_answer_only_turn_intent_hides_get_run_results_tool() -> None:filtered = _native_tools_for_turn(tools, intent)- assert filtered == []+ assert filtered == toolsdef test_build_turn_intent_marks_declarative_step_request_as_edit() -> None:diff --git a/tests/unit/test_copilot_turn_intent_tool_gate.py b/tests/unit/test_copilot_turn_intent_tool_gate.pyindex d1e74f653..e74efa0b8 100644--- a/tests/unit/test_copilot_turn_intent_tool_gate.py+++ b/tests/unit/test_copilot_turn_intent_tool_gate.py@@ -82,6 +82,8 @@ def _assert_signal((TurnIntentMode.DOCS_ANSWER, "update_workflow", "turn_intent_no_mutation_update_blocked"),(TurnIntentMode.DIAGNOSE, "run_blocks_and_collect_debug", "turn_intent_no_mutation_run_blocked"),(TurnIntentMode.DOCS_ANSWER, "inspect_page_for_composition", "turn_intent_page_inspection_blocked"),+ (TurnIntentMode.DOCS_ANSWER, "list_credentials", "turn_intent_credential_metadata_blocked"),+ (TurnIntentMode.DIAGNOSE, "list_credentials", "turn_intent_credential_metadata_blocked"),(TurnIntentMode.CLARIFY, "inspect_page_for_composition", "turn_intent_page_inspection_blocked"),(TurnIntentMode.CLARIFY, "update_and_run_blocks", "turn_intent_no_mutation_run_blocked"),(TurnIntentMode.REFUSE, "update_and_run_blocks", "turn_intent_no_mutation_run_blocked"),@@ -123,6 +125,15 @@ def test_turn_intent_gate_allows_build_page_inspection_with_update_authority() -assert _turn_intent_tool_error(_ctx(intent), "inspect_page_for_composition") is None+def test_turn_intent_gate_allows_credential_metadata_with_update_authority() -> None:+ intent = TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=False),+ )++ assert _turn_intent_tool_error(_ctx(intent), "list_credentials") is None++def test_turn_intent_gate_blocks_draft_only_run_tools() -> None:intent = TurnIntent(mode=TurnIntentMode.DRAFT_ONLY,@@ -408,7 +419,8 @@ def test_docs_answer_blocks_get_run_results_even_with_read_flag() -> None:classifier_mode="docs_answer",blocked_tool="get_run_results",)- assert _native_tools_for_turn(list(NATIVE_TOOLS), intent) == []+ filtered = _native_tools_for_turn(list(NATIVE_TOOLS), intent)+ assert {getattr(tool, "name", None) for tool in filtered} == {tool.name for tool in NATIVE_TOOLS}def test_within_turn_override_pending_reconciliation_allows_read() -> None:@@ -518,7 +530,7 @@ async def test_get_run_results_defaults_to_latest_same_turn_run_when_no_success(mock_app.WORKFLOW_SERVICE.get_workflow_runs_for_workflow_permanent_id.assert_not_called()-def test_recovery_diagnose_keeps_get_run_results_in_native_tools() -> None:+def test_recovery_diagnose_keeps_all_native_tools_registered() -> None:intent = TurnIntent(mode=TurnIntentMode.DIAGNOSE,authority=TurnIntentAuthority(@@ -531,7 +543,7 @@ def test_recovery_diagnose_keeps_get_run_results_in_native_tools() -> None:filtered = _native_tools_for_turn(list(NATIVE_TOOLS), intent)names = {getattr(tool, "name", None) for tool in filtered}- assert names == {"get_run_results"}+ assert names == {tool.name for tool in NATIVE_TOOLS}@pytest.mark.asyncio