Skip to content

Automate browser based workflows with AI

fix(copilot): keep native tools registered for authority blockers (#6291)

Andrew Neilsone12da142ef46
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 9c5f67453..0b145dfac 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -78,7 +78,6 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (
merge_failure_into_context,
)
from skyvern.forge.sdk.copilot.request_policy import (
- CREDENTIAL_DEFERRED_DRAFT_REASONS,
RAW_SECRET_REFUSAL_SENTINEL,
RequestPolicy,
build_request_policy,
@@ -93,7 +92,6 @@ from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_
from skyvern.forge.sdk.copilot.turn_context import TurnContextAssembler, TurnContextInputs, TurnContextPacket
from skyvern.forge.sdk.copilot.turn_intent import (
NO_MUTATION_TURN_INTENT_MODES,
- READ_CONTEXT_DENIED_MODES,
RequiredContextKey,
TurnIntent,
TurnIntentMode,
@@ -589,34 +587,15 @@ def _turn_intent_disables_tools(turn_intent: TurnIntent | None) -> bool:
return not authority.may_update_workflow and not authority.may_run_blocks
-def _request_policy_requires_update_and_run_skip_path(request_policy: RequestPolicy | None) -> bool:
- return (
- isinstance(request_policy, RequestPolicy)
- and request_policy.allow_update_workflow
- and not request_policy.allow_run_blocks
- and request_policy.allow_missing_credentials_in_draft
- and request_policy.clarification_reason in CREDENTIAL_DEFERRED_DRAFT_REASONS
- )
-
-
def _native_tools_for_turn(
native_tools: list[Any],
turn_intent: TurnIntent | None,
request_policy: RequestPolicy | None = None,
) -> list[Any]:
- # Deferred: matches the existing tools.py <-> routes.workflow_copilot <-> agent.py cycle workaround.
- from skyvern.forge.sdk.copilot.tools import ANSWER_ONLY_CONTEXT_TOOLS
-
- if _turn_intent_disables_tools(turn_intent):
- if (
- turn_intent is not None
- and turn_intent.authority.may_read_run_context
- and turn_intent.mode not in READ_CONTEXT_DENIED_MODES
- ):
- return [tool for tool in native_tools if getattr(tool, "name", None) in ANSWER_ONLY_CONTEXT_TOOLS]
- return []
- if _request_policy_requires_update_and_run_skip_path(request_policy):
- return [tool for tool in native_tools if getattr(tool, "name", None) != "update_workflow"]
+ # Keep native tools registered even when the current turn is not allowed to
+ # use them. The tool implementations enforce TurnIntent/RequestPolicy
+ # authority and return structured blockers; removing a tool lets the model
+ # hit an SDK-level ModelBehaviorError if static prompt text still names it.
return list(native_tools)
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 8c45ad964..e8f86f8a0 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -1219,6 +1219,7 @@ async def _attach_failed_block_screenshots(
BLOCK_RUNNING_TOOLS = frozenset({"run_blocks_and_collect_debug", "update_and_run_blocks"})
WORKFLOW_MUTATION_TOOLS = frozenset({"update_workflow", "update_and_run_blocks"})
ANSWER_ONLY_CONTEXT_TOOLS = frozenset({"get_run_results"})
+CREDENTIAL_METADATA_TOOLS = frozenset({"list_credentials"})
PAGE_INSPECTION_TOOLS = frozenset({"inspect_page_for_composition", "evaluate", "get_browser_screenshot"})
PAGE_SCHEMA_CONTEXT_TOOLS = frozenset({"inspect_page_for_composition"})
_CURRENT_PAGE_INSPECTION_TARGETS = frozenset({"", "current", "current_page", "__current_page__"})
@@ -1757,6 +1758,9 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlo
blocks_page_inspection = tool_name in PAGE_SCHEMA_CONTEXT_TOOLS and (
intent.mode in NO_MUTATION_TURN_INTENT_MODES or not authority.may_update_workflow
)
+ blocks_credential_metadata = tool_name in CREDENTIAL_METADATA_TOOLS and not (
+ authority.may_update_workflow or authority.may_run_blocks
+ )
# Two paths grant read access to ANSWER_ONLY_CONTEXT_TOOLS, both excluded for DOCS_ANSWER/REFUSE/CLARIFY:
# (1) authority.may_read_run_context — classifier-derived (DIAGNOSE turns)
# (2) pending_reconciliation_run_id — within-turn override anchored to the run-blocks watchdog
@@ -1781,7 +1785,13 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlo
if blocks_run and not blocks_update and _request_policy_allows_update_and_skip_run(ctx, tool_name):
return None
- if not blocks_update and not blocks_run and not blocks_context_read and not blocks_page_inspection:
+ if (
+ not blocks_update
+ and not blocks_run
+ and not blocks_context_read
+ and not blocks_page_inspection
+ and not blocks_credential_metadata
+ ):
if within_turn_read_override:
LOG.info(
"copilot authority gate allowed tool via within-turn read override",
@@ -1804,6 +1814,8 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlo
reason_code = "turn_intent_page_inspection_blocked"
elif blocks_context_read:
reason_code = "turn_intent_context_read_blocked"
+ elif blocks_credential_metadata:
+ reason_code = "turn_intent_credential_metadata_blocked"
else:
reason_code = "turn_intent_run_blocked"
LOG.info(
@@ -5293,6 +5305,12 @@ async def list_credentials_tool(
if loop_error:
return json.dumps({"ok": False, "error": loop_error})
+ authority_error = _authority_tool_error(copilot_ctx, "list_credentials")
+ if authority_error:
+ result = {"ok": False, "error": authority_error}
+ record_tool_step_result_for_ctx(copilot_ctx, "list_credentials", arguments, result)
+ return json.dumps(result)
+
result = await _list_credentials(arguments, copilot_ctx)
record_tool_step_result_for_ctx(copilot_ctx, "list_credentials", arguments, result)
sanitized = sanitize_tool_result_for_llm("list_credentials", result)
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 7128f39fa..8fb3cb18d 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -1910,7 +1910,7 @@ class TestNativeToolSurface:
assert "inspect_page_for_composition" in names
@pytest.mark.parametrize("reason", ["workflow_credential_inputs_unbound", "credential_name_unresolved"])
- def test_credential_deferred_draft_removes_update_workflow_tool(self, reason: str) -> None:
+ def test_credential_deferred_draft_keeps_all_native_tools_registered(self, reason: str) -> None:
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
policy = RequestPolicy(
@@ -1928,7 +1928,7 @@ class TestNativeToolSurface:
filtered = agent_module._native_tools_for_turn(tools, turn_intent=None, request_policy=policy)
- assert [tool.name for tool in filtered] == ["list_credentials", "update_and_run_blocks"]
+ assert [tool.name for tool in filtered] == ["update_workflow", "list_credentials", "update_and_run_blocks"]
def test_non_deferred_policy_keeps_update_workflow_tool(self) -> None:
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.py
index 5cb4eb40e..1ae017280 100644
--- a/tests/unit/test_copilot_turn_intent.py
+++ b/tests/unit/test_copilot_turn_intent.py
@@ -617,7 +617,7 @@ def test_store_request_policy_attaches_turn_intent_to_context() -> None:
assert ctx.turn_intent.authority.may_run_blocks is False
-def test_answer_only_turn_intent_hides_get_run_results_tool() -> None:
+def test_answer_only_turn_intent_keeps_native_tools_registered() -> None:
tools = [
SimpleNamespace(name="update_workflow"),
SimpleNamespace(name="get_run_results"),
@@ -630,7 +630,7 @@ def test_answer_only_turn_intent_hides_get_run_results_tool() -> None:
filtered = _native_tools_for_turn(tools, intent)
- assert filtered == []
+ assert filtered == tools
def test_build_turn_intent_marks_declarative_step_request_as_edit() -> None:
diff --git a/tests/unit/test_copilot_turn_intent_tool_gate.py b/tests/unit/test_copilot_turn_intent_tool_gate.py
index d1e74f653..e74efa0b8 100644
--- a/tests/unit/test_copilot_turn_intent_tool_gate.py
+++ b/tests/unit/test_copilot_turn_intent_tool_gate.py
@@ -82,6 +82,8 @@ def _assert_signal(
(TurnIntentMode.DOCS_ANSWER, "update_workflow", "turn_intent_no_mutation_update_blocked"),
(TurnIntentMode.DIAGNOSE, "run_blocks_and_collect_debug", "turn_intent_no_mutation_run_blocked"),
(TurnIntentMode.DOCS_ANSWER, "inspect_page_for_composition", "turn_intent_page_inspection_blocked"),
+ (TurnIntentMode.DOCS_ANSWER, "list_credentials", "turn_intent_credential_metadata_blocked"),
+ (TurnIntentMode.DIAGNOSE, "list_credentials", "turn_intent_credential_metadata_blocked"),
(TurnIntentMode.CLARIFY, "inspect_page_for_composition", "turn_intent_page_inspection_blocked"),
(TurnIntentMode.CLARIFY, "update_and_run_blocks", "turn_intent_no_mutation_run_blocked"),
(TurnIntentMode.REFUSE, "update_and_run_blocks", "turn_intent_no_mutation_run_blocked"),
@@ -123,6 +125,15 @@ def test_turn_intent_gate_allows_build_page_inspection_with_update_authority() -
assert _turn_intent_tool_error(_ctx(intent), "inspect_page_for_composition") is None
+def test_turn_intent_gate_allows_credential_metadata_with_update_authority() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=False),
+ )
+
+ assert _turn_intent_tool_error(_ctx(intent), "list_credentials") is None
+
+
def test_turn_intent_gate_blocks_draft_only_run_tools() -> None:
intent = TurnIntent(
mode=TurnIntentMode.DRAFT_ONLY,
@@ -408,7 +419,8 @@ def test_docs_answer_blocks_get_run_results_even_with_read_flag() -> None:
classifier_mode="docs_answer",
blocked_tool="get_run_results",
)
- assert _native_tools_for_turn(list(NATIVE_TOOLS), intent) == []
+ filtered = _native_tools_for_turn(list(NATIVE_TOOLS), intent)
+ assert {getattr(tool, "name", None) for tool in filtered} == {tool.name for tool in NATIVE_TOOLS}
def test_within_turn_override_pending_reconciliation_allows_read() -> None:
@@ -518,7 +530,7 @@ async def test_get_run_results_defaults_to_latest_same_turn_run_when_no_success(
mock_app.WORKFLOW_SERVICE.get_workflow_runs_for_workflow_permanent_id.assert_not_called()
-def test_recovery_diagnose_keeps_get_run_results_in_native_tools() -> None:
+def test_recovery_diagnose_keeps_all_native_tools_registered() -> None:
intent = TurnIntent(
mode=TurnIntentMode.DIAGNOSE,
authority=TurnIntentAuthority(
@@ -531,7 +543,7 @@ def test_recovery_diagnose_keeps_get_run_results_in_native_tools() -> None:
filtered = _native_tools_for_turn(list(NATIVE_TOOLS), intent)
names = {getattr(tool, "name", None) for tool in filtered}
- assert names == {"get_run_results"}
+ assert names == {tool.name for tool in NATIVE_TOOLS}
@pytest.mark.asyncio