Skip to content

Automate browser based workflows with AI

copilot: structured tool-blocker signal contract + lifecycle (SKY-10314 1/2) (#6187)

Andrew Neilson2ed17ccc5818
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
new file mode 100644
index 000000000..4dbe8982c
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -0,0 +1,174 @@
+"""Structured tool-blocker signal."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Any, Literal, Protocol
+
+import structlog
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator
+
+BlockerKind = Literal[
+ "authority_denied",
+ "loop_detected",
+ "phase_gated",
+ "tool_error",
+ "missing_required_context",
+]
+
+RecoveryHint = Literal[
+ "retry_with_different_tool",
+ "ask_user_clarifying",
+ "report_blocker_to_user",
+ "stop",
+]
+
+LOG = structlog.get_logger()
+
+
+# Matched case-insensitively. Imperative variants are narrow ("do not run" etc.) so plain "do not worry" prose doesn't false-positive.
+_LEAK_DENY_TOKENS: tuple[str, ...] = (
+ "safe_reason_code",
+ "LOOP DETECTED:",
+ "recovery_hint=",
+ "do not run",
+ "do not call",
+ "do not retry",
+ "do not start",
+ "do not update",
+ "do not fetch",
+ "do not execute",
+ "do not attempt",
+ "don't run",
+ "don't call",
+ "don't retry",
+ "don't fetch",
+ "don't execute",
+ "don't attempt",
+ "never run",
+ "never call",
+ "never retry",
+ "must not run",
+ "must not call",
+ "send me",
+ "normal instruction",
+ "like 'continue",
+ 'like "continue',
+)
+
+
+def assert_clean_user_facing_text(value: str, *, blocked_tool: str | None = None) -> None:
+ lowered = value.lower()
+ for token in _LEAK_DENY_TOKENS:
+ if token.lower() in lowered:
+ raise ValueError(f"blocker user-facing text leaked token {token!r}: {value!r}")
+ if blocked_tool and blocked_tool.lower() in lowered:
+ raise ValueError(f"blocker user-facing text leaked tool name {blocked_tool!r}: {value!r}")
+
+
+class CopilotToolBlockerSignal(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ blocker_kind: BlockerKind
+ agent_steering_text: str
+ user_facing_reason: str
+ recovery_hint: RecoveryHint
+ cleared_by_tools: frozenset[str] = Field(default_factory=frozenset)
+ preserves_workflow_draft: bool = False
+
+ internal_reason_code: str | None = None
+ blocked_tool: str | None = None
+ classifier_mode: str | None = None
+ exception_type: str | None = None
+ # `Mapping` (not `dict`) signals the immutability contract; `frozen=True` does not freeze the container.
+ extra: Mapping[str, Any] = Field(default_factory=dict)
+
+ @model_validator(mode="after")
+ def _validate_user_facing_clean(self) -> CopilotToolBlockerSignal:
+ assert_clean_user_facing_text(self.user_facing_reason, blocked_tool=self.blocked_tool)
+ if not isinstance(self.extra, MappingProxyType):
+ object.__setattr__(self, "extra", MappingProxyType(dict(self.extra)))
+ return self
+
+ @field_serializer("extra")
+ def _serialize_extra(self, value: Mapping[str, Any]) -> dict[str, Any]:
+ return dict(value)
+
+
+def build_llm_tool_error_payload(signal: CopilotToolBlockerSignal) -> str:
+ return signal.agent_steering_text
+
+
+def to_trace_data(signal: CopilotToolBlockerSignal) -> dict[str, Any]:
+ return {
+ "blocker_kind": signal.blocker_kind,
+ "recovery_hint": signal.recovery_hint,
+ "cleared_by_tools": sorted(signal.cleared_by_tools),
+ "internal_reason_code": signal.internal_reason_code,
+ "blocked_tool": signal.blocked_tool,
+ "classifier_mode": signal.classifier_mode,
+ "exception_type": signal.exception_type,
+ "extra": dict(signal.extra),
+ }
+
+
+class _BlockerSignalCtx(Protocol):
+ blocker_signal: CopilotToolBlockerSignal | None
+
+
+def maybe_clear_blocker_signal_on_tool_success(ctx: _BlockerSignalCtx, succeeded_tool_name: str) -> None:
+ signal = getattr(ctx, "blocker_signal", None)
+ if isinstance(signal, CopilotToolBlockerSignal) and succeeded_tool_name in signal.cleared_by_tools:
+ ctx.blocker_signal = None
+
+
+def clear_blocker_signal_for_reason_codes(ctx: _BlockerSignalCtx, internal_reason_codes: frozenset[str]) -> None:
+ signal = getattr(ctx, "blocker_signal", None)
+ if isinstance(signal, CopilotToolBlockerSignal) and signal.internal_reason_code in internal_reason_codes:
+ ctx.blocker_signal = None
+
+
+def stash_blocker_signal(ctx: _BlockerSignalCtx, signal: CopilotToolBlockerSignal) -> str:
+ """First-wins stash + observability log; returns the LLM-visible payload."""
+ existing = getattr(ctx, "blocker_signal", None)
+ stashed = not isinstance(existing, CopilotToolBlockerSignal)
+ if stashed:
+ ctx.blocker_signal = signal
+ extra: dict[str, Any] = {"stashed": stashed}
+ if not stashed and isinstance(existing, CopilotToolBlockerSignal):
+ extra["existing_reason_code"] = existing.internal_reason_code
+ extra["existing_blocker_kind"] = existing.blocker_kind
+ LOG.info("copilot tool blocker signal", **extra, **to_trace_data(signal))
+ return build_llm_tool_error_payload(signal)
+
+
+def build_loop_blocker_signal(loop_message: str, *, tool_name: str) -> CopilotToolBlockerSignal:
+ # Category markers come first because credential/parameter messages also contain "has already failed".
+ if "with CREDENTIAL_ERROR" in loop_message or "with PARAMETER_BINDING_ERROR" in loop_message:
+ internal = "loop_detected_credential_or_parameter_misconfig"
+ user_facing = (
+ "I couldn't run this with the current credential or parameter setup. Update them and ask me to try again."
+ )
+ recovery_hint: RecoveryHint = "ask_user_clarifying"
+ elif "has already failed" in loop_message:
+ internal = "loop_detected_repeated_failed_step"
+ user_facing = "I retried without making progress. Tell me what to change and I'll try a different approach."
+ recovery_hint = "report_blocker_to_user"
+ elif "has been called" in loop_message:
+ internal = "loop_detected_consecutive_same_tool"
+ user_facing = "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."
+ recovery_hint = "report_blocker_to_user"
+ else:
+ internal = "loop_detected_generic"
+ user_facing = "I couldn't keep going on this turn. Tell me what to change and I'll try again."
+ recovery_hint = "report_blocker_to_user"
+ return CopilotToolBlockerSignal(
+ blocker_kind="loop_detected",
+ agent_steering_text=loop_message,
+ user_facing_reason=user_facing,
+ recovery_hint=recovery_hint,
+ cleared_by_tools=frozenset(),
+ internal_reason_code=internal,
+ blocked_tool=tool_name,
+ )
diff --git a/skyvern/forge/sdk/copilot/build_phase.py b/skyvern/forge/sdk/copilot/build_phase.py
index 4c041c49d..b3e5b813b 100644
--- a/skyvern/forge/sdk/copilot/build_phase.py
+++ b/skyvern/forge/sdk/copilot/build_phase.py
@@ -27,6 +27,7 @@ import yaml
from skyvern.utils.yaml_loader import safe_load_no_dates
if TYPE_CHECKING:
+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent
@@ -180,15 +181,10 @@ def advance_to_testing(ctx: CopilotContext) -> None:
_log_transition(ctx, prev=prev, new=ctx.build_phase, reason="update_workflow_succeeded")
-def _phase_tool_error(ctx: Any, tool_name: str) -> str | None:
- """Shared phase-aware authority error. Returns the steering error string
- when the (phase, tool) pair is forbidden; None otherwise.
+def _phase_blocker_signal(ctx: Any, tool_name: str) -> CopilotToolBlockerSignal | None:
+ """Phase-aware authority blocker, parallel to `_turn_intent_tool_error` / `_request_policy_tool_error`."""
+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
- Called from both the native gate (`_authority_tool_error` in tools.py)
- and the MCP-adapter gate (`SkyvernOverlayMCPServer.call_tool` pre-pre-hook).
- Errors flow back to the LLM as next-step steering, per the ai-agents
- 'errors as steering' guidance.
- """
phase = getattr(ctx, "build_phase", None)
if not isinstance(phase, BuildPhase):
return None
@@ -197,24 +193,54 @@ def _phase_tool_error(ctx: Any, tool_name: str) -> str | None:
in_mutation = phase in MUTATION_PERMITTED_PHASES
if tool_name in _DISCOVERY_TOOLS and in_mutation:
- return (
- "discover_workflow_entrypoint is only available before composition. "
- "The workflow already has a target URL — proceed with update_workflow or update_and_run_blocks. "
- "safe_reason_code=build_phase_discovery_disallowed_post_compose."
+ return CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text=(
+ "discover_workflow_entrypoint is only available before composition. "
+ "The workflow already has a target URL — proceed with update_workflow or update_and_run_blocks. "
+ "safe_reason_code=build_phase_discovery_disallowed_post_compose."
+ ),
+ user_facing_reason="I already have a target for this workflow — I'll keep editing it instead of starting over.",
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({"update_workflow", "update_and_run_blocks"}),
+ internal_reason_code="build_phase_discovery_disallowed_post_compose",
+ blocked_tool=tool_name,
)
if tool_name in _BROWSER_PRIMITIVE_TOOLS and in_discovery:
- return (
- "Direct browser tools are not callable before composition. "
- "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL. "
- "safe_reason_code=build_phase_browser_blocked_pre_compose."
+ return CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text=(
+ "Direct browser tools are not callable before composition. "
+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL. "
+ "safe_reason_code=build_phase_browser_blocked_pre_compose."
+ ),
+ user_facing_reason="I need to know what site to work on before I can browse there. What URL should I use?",
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="build_phase_browser_blocked_pre_compose",
+ blocked_tool=tool_name,
)
if tool_name in _MUTATION_TOOLS and in_discovery:
- return (
- "Workflow mutation is gated to composition. "
- "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL first. "
- "safe_reason_code=build_phase_mutation_blocked_pre_compose."
+ return CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text=(
+ "Workflow mutation is gated to composition. "
+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL first. "
+ "safe_reason_code=build_phase_mutation_blocked_pre_compose."
+ ),
+ user_facing_reason="I need to know what site to work on before I can build a workflow. What URL should I use?",
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="build_phase_mutation_blocked_pre_compose",
+ blocked_tool=tool_name,
)
return None
+
+
+def _phase_tool_error(ctx: Any, tool_name: str) -> str | None:
+ """Thin compatibility shim: returns the LLM-visible steering text from `_phase_blocker_signal`."""
+ signal = _phase_blocker_signal(ctx, tool_name)
+ return signal.agent_steering_text if signal is not None else None
diff --git a/skyvern/forge/sdk/copilot/loop_detection.py b/skyvern/forge/sdk/copilot/loop_detection.py
index e4744e478..d4091d032 100644
--- a/skyvern/forge/sdk/copilot/loop_detection.py
+++ b/skyvern/forge/sdk/copilot/loop_detection.py
@@ -20,6 +20,7 @@ from collections.abc import Iterable, Mapping, MutableMapping
from typing import Any
from skyvern.forge.failure_classifier import classify_from_failure_reason
+from skyvern.forge.sdk.copilot.blocker_signal import maybe_clear_blocker_signal_on_tool_success
MAX_CONSECUTIVE_SAME_TOOL = 3
MAX_REPEATED_FAILED_STEP = 3
@@ -225,9 +226,12 @@ def record_tool_step_result_for_ctx(
result: Mapping[str, Any],
) -> None:
tracker = _ctx_failed_step_tracker(ctx)
- if tracker is None:
- return
- record_tool_step_result(tracker, tool_name, arguments, result)
+ if tracker is not None:
+ record_tool_step_result(tracker, tool_name, arguments, result)
+ # Strict ``is True`` check: a malformed result dict missing ``ok`` entirely
+ # must not be treated as success and accidentally clear a blocker signal.
+ if result.get("ok") is True:
+ maybe_clear_blocker_signal_on_tool_success(ctx, tool_name)
def clear_failed_step_tracker_for_tools_in_ctx(ctx: Any, tool_names: Iterable[str]) -> None:
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 2371959f4..4e9d9de1a 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -20,7 +20,11 @@ from mcp.types import (
TextContent,
)
-from skyvern.forge.sdk.copilot.build_phase import _phase_tool_error
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ build_loop_blocker_signal,
+ stash_blocker_signal,
+)
+from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
from skyvern.forge.sdk.copilot.loop_detection import (
detect_failed_tool_step_loop_for_ctx,
detect_tool_loop,
@@ -58,6 +62,10 @@ LOG = structlog.get_logger()
_INTERNAL_TOOL_ARG_KEYS = frozenset({"_summarized"})
+def _stash_and_emit_loop_blocker(ctx: Any, loop_message: str, tool_name: str) -> str:
+ return stash_blocker_signal(ctx, build_loop_blocker_signal(loop_message, tool_name=tool_name))
+
+
def _apply_schema_overlay(
input_schema: dict[str, Any],
overlay: SchemaOverlay,
@@ -106,7 +114,7 @@ def _copilot_to_call_tool_result(
) -> CallToolResult:
sanitized = sanitize_tool_result_for_llm("", copilot_result)
content: list[TextContent] = [TextContent(type="text", text=json.dumps(sanitized))]
- is_error = not copilot_result.get("ok", True)
+ is_error = copilot_result.get("ok", True) is not True
return CallToolResult(content=content, isError=is_error)
@@ -201,21 +209,17 @@ class SkyvernOverlayMCPServer(MCPServer):
copilot_ctx = self._context_provider()
overlay = self._overlays.get(tool_name, SchemaOverlay())
- # Defense-in-depth phase gate for MCP-exposed browser primitives. The
- # native gate (`_authority_tool_error` in tools.py) covers native
- # function_tool calls; this branch covers tools dispatched only
- # through SkyvernOverlayMCPServer, which never pass through the
- # native gate. The two checks share `_phase_tool_error`, so a tool
- # gated at both layers reports the same error message.
- phase_error = _phase_tool_error(copilot_ctx, tool_name)
- if phase_error:
+ # MCP-side phase gate; mirror of `_authority_tool_error` in tools.py for MCP-only tools.
+ phase_signal = _phase_blocker_signal(copilot_ctx, tool_name)
+ if phase_signal is not None:
LOG.warning(
"Phase-gated MCP tool call rejected",
tool_name=tool_name,
build_phase=getattr(getattr(copilot_ctx, "build_phase", None), "value", None),
)
- record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, {"ok": False, "error": phase_error})
- return _copilot_to_call_tool_result({"ok": False, "error": phase_error})
+ payload = stash_blocker_signal(copilot_ctx, phase_signal)
+ record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, {"ok": False, "error": payload})
+ return _copilot_to_call_tool_result({"ok": False, "error": payload})
loop_error = detect_failed_tool_step_loop_for_ctx(copilot_ctx, tool_name, arguments)
if loop_error:
@@ -223,7 +227,8 @@ class SkyvernOverlayMCPServer(MCPServer):
"Failed tool step loop detected, skipping execution",
tool_name=tool_name,
)
- return _copilot_to_call_tool_result({"ok": False, "error": loop_error})
+ payload = _stash_and_emit_loop_blocker(copilot_ctx, loop_error, tool_name)
+ return _copilot_to_call_tool_result({"ok": False, "error": payload})
tracker = getattr(copilot_ctx, "consecutive_tool_tracker", None)
loop_error = detect_tool_loop(tracker, tool_name) if isinstance(tracker, list) else None
@@ -232,20 +237,8 @@ class SkyvernOverlayMCPServer(MCPServer):
"Tool loop detected, skipping execution",
tool_name=tool_name,
)
- return CallToolResult(
- content=[
- TextContent(
- type="text",
- text=json.dumps(
- {
- "ok": False,
- "error": loop_error,
- }
- ),
- )
- ],
- isError=True,
- )
+ payload = _stash_and_emit_loop_blocker(copilot_ctx, loop_error, tool_name)
+ return _copilot_to_call_tool_result({"ok": False, "error": payload})
if overlay.pre_hook:
hook_result = await overlay.pre_hook(arguments, copilot_ctx)
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 6f5e62d33..612eaf155 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -30,6 +30,7 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.library.skyvern_browser import SkyvernBrowser
if TYPE_CHECKING:
+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream
@@ -108,6 +109,12 @@ class AgentContext:
last_good_workflow: Any | None = None
last_good_workflow_yaml: str | None = None
+ # Set by tool gates / loop guards / tool-side error branches when a tool
+ # dispatch is blocked. The finalization shim in agent.py reads this at
+ # turn end and overrides the AgentResult with a deterministic
+ # product-language reply. See blocker_signal.py for the contract.
+ blocker_signal: CopilotToolBlockerSignal | None = None
+
def mcp_to_copilot(mcp_result: dict[str, Any]) -> dict[str, Any]:
"""Convert an MCP result dict to the copilot {ok, data, error} format."""
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 75ca6f1e7..624ef3c13 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -30,9 +30,16 @@ from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_workflow_block_goals
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ CopilotToolBlockerSignal,
+ RecoveryHint,
+ build_loop_blocker_signal,
+ clear_blocker_signal_for_reason_codes,
+ stash_blocker_signal,
+)
from skyvern.forge.sdk.copilot.build_phase import (
BuildPhase,
- _phase_tool_error,
+ _phase_blocker_signal,
advance_to_composing,
advance_to_discovering,
advance_to_testing,
@@ -550,9 +557,42 @@ def _maybe_clear_reconciliation_flag(copilot_ctx: Any, result: Any) -> None:
if is_trusted_final or was_per_tool_budget:
copilot_ctx.pending_reconciliation_run_id = None
copilot_ctx.pending_reconciliation_requires_user_input = False
+ clear_blocker_signal_for_reason_codes(
+ copilot_ctx,
+ frozenset(
+ {
+ "tool_error_pending_reconciliation_no_input",
+ "tool_error_pending_reconciliation_requires_input",
+ }
+ ),
+ )
return
if resolved_status == WorkflowRunStatus.canceled.value:
copilot_ctx.pending_reconciliation_requires_user_input = True
+ # Replace the no_input blocker with the requires-input one; unrelated
+ # blockers (e.g. `loop_detected`) survive the targeted clear.
+ existing_blocker = getattr(copilot_ctx, "blocker_signal", None)
+ # Preserve the original blocked_tool so trace queries filtering on it correlate the no_input → requires_input transition.
+ original_blocked_tool = (
+ existing_blocker.blocked_tool
+ if (
+ isinstance(existing_blocker, CopilotToolBlockerSignal)
+ and existing_blocker.internal_reason_code == "tool_error_pending_reconciliation_no_input"
+ and existing_blocker.blocked_tool
+ )
+ else "get_run_results"
+ )
+ clear_blocker_signal_for_reason_codes(
+ copilot_ctx,
+ frozenset({"tool_error_pending_reconciliation_no_input"}),
+ )
+ stash_blocker_signal(
+ copilot_ctx,
+ _pending_reconciliation_requires_input_signal(
+ pending_run_id=pending_run_id,
+ blocked_tool=original_blocked_tool,
+ ),
+ )
def _mark_pending_reconciliation_run(copilot_ctx: Any, workflow_run_id: str) -> None:
@@ -635,7 +675,9 @@ def _requested_block_label_set(arguments: dict[str, Any] | None) -> set[str] | N
return {label for label in block_labels if isinstance(label, str) and label}
-def _per_tool_budget_problem_rerun_error(ctx: Any, arguments: dict[str, Any] | None) -> str | None:
+def _per_tool_budget_problem_rerun_signal(
+ ctx: Any, arguments: dict[str, Any] | None, tool_name: str
+) -> CopilotToolBlockerSignal | None:
problem_labels = _per_tool_budget_problem_label_set(ctx)
if not problem_labels:
return None
@@ -646,7 +688,7 @@ def _per_tool_budget_problem_rerun_error(ctx: Any, arguments: dict[str, Any] | N
return None
labels = ", ".join(sorted(blocked_labels))
- return (
+ agent_steering = (
"The prior PER_TOOL_BUDGET run's get_run_results showed navigation "
f"block(s) [{labels}] were canceled or failed while applying page state. "
"Do NOT rerun those block label(s) unchanged with run_blocks_and_collect_debug. "
@@ -654,6 +696,62 @@ def _per_tool_budget_problem_rerun_error(ctx: Any, arguments: dict[str, Any] | N
"prefer a code or validation block for DOM/page-state verification, or run only "
"newly-created smaller block labels that apply one missing constraint at a time."
)
+ user_facing = "The previous run hit the per-step time budget. I need a smaller scope before I can rerun."
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({"update_workflow", "update_and_run_blocks"}),
+ internal_reason_code="tool_error_per_tool_budget_rerun",
+ blocked_tool=tool_name,
+ )
+
+
+_RECONCILIATION_REQUIRES_INPUT_USER_FACING = (
+ "The previous run was canceled. Tell me whether to retry, keep the draft as-is, or adjust the workflow first."
+)
+_RECONCILIATION_NO_INPUT_USER_FACING = (
+ "The previous run ended without a verified result. I'll check what happened before doing anything else this turn."
+)
+
+
+def _pending_reconciliation_requires_input_signal(
+ *, pending_run_id: str, blocked_tool: str
+) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=(
+ f"The canceled run {pending_run_id} has already been inspected. "
+ f"Do NOT run more blocks in this turn; ask the user whether to retry, "
+ f"accept the unverified draft, or adjust the workflow first. This guard "
+ f"prevents duplicate side effects on live sites."
+ ),
+ user_facing_reason=_RECONCILIATION_REQUIRES_INPUT_USER_FACING,
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="tool_error_pending_reconciliation_requires_input",
+ blocked_tool=blocked_tool,
+ )
+
+
+def _pending_reconciliation_no_input_signal(*, pending_run_id: str, blocked_tool: str) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=(
+ f"The previous block-running tool call for run {pending_run_id} "
+ f"ended without a trustworthy terminal status. "
+ f'Call `get_run_results(workflow_run_id="{pending_run_id}")` '
+ f"first, report the result to the user, then await user input "
+ f"before running more blocks. This guard prevents duplicate "
+ f"side effects on live sites."
+ ),
+ user_facing_reason=_RECONCILIATION_NO_INPUT_USER_FACING,
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="tool_error_pending_reconciliation_no_input",
+ blocked_tool=blocked_tool,
+ )
# Streak threshold at which the copilot hard-aborts a tool call because the
@@ -813,7 +911,7 @@ def _active_block_run_budget_seconds(ctx: AgentContext) -> int:
return max(1, min(PER_TOOL_CALL_BUDGET_SECONDS, int(remaining_after_reply_reserve)))
-def _late_block_running_call_error(ctx: AgentContext) -> str | None:
+def _late_block_running_call_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
remaining = _copilot_seconds_remaining(ctx)
if remaining is None or remaining > COPILOT_FINAL_REPLY_RESERVE_SECONDS:
return None
@@ -826,31 +924,45 @@ def _late_block_running_call_error(ctx: AgentContext) -> str | None:
and isinstance(last_good_workflow_yaml, str)
and last_good_workflow_yaml
):
- return (
+ agent_steering = (
f"Wall-clock budget too low to retry: about {int(max(0.0, remaining))}s remain of the "
f"{TOTAL_TIMEOUT_SECONDS}s session budget. A verified workflow exists from before the failure. "
"Do NOT call update_and_run_blocks or run_blocks_and_collect_debug again. REPLY now: summarize "
"what worked, name the block that failed, and tell the user they can keep the verified prefix or discard."
)
-
- if isinstance(last_failed_workflow_yaml, str) and last_failed_workflow_yaml:
- return (
+ elif isinstance(last_failed_workflow_yaml, str) and last_failed_workflow_yaml:
+ agent_steering = (
f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn "
"after the previous workflow run failed. Do NOT retry block-running tools; reply to the user "
"with the failure evidence gathered so far and make clear that any draft workflow is unverified."
)
+ else:
+ agent_steering = (
+ f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn. "
+ "Do NOT start another block-running tool call; reply to the user with the workflow draft and "
+ "progress gathered so far, and make clear which parts have not been verified end-to-end."
+ )
- return (
- f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn. "
- "Do NOT start another block-running tool call; reply to the user with the workflow draft and "
- "progress gathered so far, and make clear which parts have not been verified end-to-end."
+ user_facing = "I'm running out of time on this turn. I'll wrap up with what I have so far."
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="stop",
+ cleared_by_tools=frozenset(),
+ # The "wrap up with what we have" semantic means the draft saved
+ # earlier in the turn should still surface — only the chat reply
+ # is overridden by the renderer.
+ preserves_workflow_draft=True,
+ internal_reason_code="tool_error_late_block_running",
+ blocked_tool=tool_name,
)
def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any] | None = None) -> str | None:
detected = detect_failed_tool_step_loop_for_ctx(ctx, tool_name, arguments or {})
if detected is not None:
- return detected
+ return _emit_tool_blocker_signal(ctx, _build_loop_blocker_signal(detected, tool_name=tool_name))
# Consecutive same-name guard: false-positives on the intended iterative
# build (one new block per update_and_run_blocks). Block-running tools
@@ -859,7 +971,7 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
if isinstance(tracker, list) and tool_name not in BLOCK_RUNNING_TOOLS:
detected = detect_tool_loop(tracker, tool_name)
if detected is not None:
- return detected
+ return _emit_tool_blocker_signal(ctx, _build_loop_blocker_signal(detected, tool_name=tool_name))
# Hard-abort when the agent has re-fired the same action sequence against
# the page N times without intervening success. This is the signal that
@@ -879,30 +991,26 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
if isinstance(pending_run_id, str) and pending_run_id:
if getattr(ctx, "pending_reconciliation_requires_user_input", False) is True:
- return (
- f"The canceled run {pending_run_id} has already been inspected. "
- f"Do NOT run more blocks in this turn; ask the user whether to retry, "
- f"accept the unverified draft, or adjust the workflow first. This guard "
- f"prevents duplicate side effects on live sites."
+ return _emit_tool_blocker_signal(
+ ctx,
+ _pending_reconciliation_requires_input_signal(
+ pending_run_id=pending_run_id, blocked_tool=tool_name
+ ),
)
- return (
- f"The previous block-running tool call for run {pending_run_id} "
- f"ended without a trustworthy terminal status. "
- f'Call `get_run_results(workflow_run_id="{pending_run_id}")` '
- f"first, report the result to the user, then await user input "
- f"before running more blocks. This guard prevents duplicate "
- f"side effects on live sites."
+ return _emit_tool_blocker_signal(
+ ctx,
+ _pending_reconciliation_no_input_signal(pending_run_id=pending_run_id, blocked_tool=tool_name),
)
if tool_name == "run_blocks_and_collect_debug":
- budget_rerun_error = _per_tool_budget_problem_rerun_error(ctx, arguments)
- if budget_rerun_error is not None:
- return budget_rerun_error
+ budget_signal = _per_tool_budget_problem_rerun_signal(ctx, arguments, tool_name)
+ if budget_signal is not None:
+ return _emit_tool_blocker_signal(ctx, budget_signal)
streak_raw = getattr(ctx, "repeated_action_fingerprint_streak_count", 0)
streak = streak_raw if isinstance(streak_raw, int) else 0
if streak >= REPEATED_ACTION_STREAK_ABORT_AT:
- return (
+ agent_steering = (
f"Repeated-action abort: the last {streak} runs fired the same "
"action sequence against the page without making progress. "
"The site is likely blocked by a captcha, popup, anti-bot "
@@ -910,6 +1018,21 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
"detecting. Do NOT retry this tool — conclude the workflow is "
"not automatable as-is and report back to the user."
)
+ user_facing = (
+ "I tried the same actions a few times without making progress. The site looks blocked. I'll stop here."
+ )
+ return _emit_tool_blocker_signal(
+ ctx,
+ CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="stop",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="tool_error_repeated_action_abort",
+ blocked_tool=tool_name,
+ ),
+ )
# Within-turn fail-fast for permanent navigation errors (DNS / cert /
# SSL / invalid URL). The enforcement-loop stop nudge only runs
@@ -921,57 +1044,109 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
# body.
prior_nav_error = getattr(ctx, "last_test_non_retriable_nav_error", None)
if isinstance(prior_nav_error, str) and prior_nav_error:
- return (
+ agent_steering = (
f"Prior run in this turn hit a permanent navigation error "
f"({prior_nav_error[:200]}). Do NOT retry — the URL is unreachable "
"regardless of subdomain or path variations. Reply to the user "
"explaining the failure and asking them to verify the URL."
)
+ user_facing = "The URL I tried isn't reachable. Tell me the correct address and I'll try again."
+ return _emit_tool_blocker_signal(
+ ctx,
+ CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="tool_error_non_retriable_nav",
+ blocked_tool=tool_name,
+ ),
+ )
- late_block_running_error = _late_block_running_call_error(ctx)
- if late_block_running_error is not None:
- return late_block_running_error
+ late_signal = _late_block_running_call_signal(ctx, tool_name)
+ if late_signal is not None:
+ return _emit_tool_blocker_signal(ctx, late_signal)
return None
-def _request_policy_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
+_build_loop_blocker_signal = build_loop_blocker_signal
+
+
+def _request_policy_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
policy = getattr(ctx, "request_policy", None)
if not isinstance(policy, RequestPolicy):
return None
- error: str | None = None
+
+ agent_steering: str | None = None
+ user_facing: str | None = None
+ reason_code: str | None = None
+ recovery_hint: RecoveryHint = "report_blocker_to_user"
+ cleared_by: frozenset[str] = frozenset()
+
if tool_name == "update_workflow" and not policy.allow_update_workflow:
- error = (
+ reason_code = "request_policy_blocks_update_workflow"
+ agent_steering = (
"Request policy blocks workflow updates for the latest user message. "
"Ask the user for safe stored credential metadata instead."
)
+ user_facing = "I need stored credential metadata before I can update this workflow."
+ recovery_hint = "ask_user_clarifying"
elif tool_name in BLOCK_RUNNING_TOOLS and not policy.allow_run_blocks:
if policy.testing_intent == "skip_test":
- error = "Request policy says the latest user message asked for an untested draft. Use update_workflow only."
+ reason_code = "request_policy_blocks_run_blocks_skip_test"
+ agent_steering = (
+ "Request policy says the latest user message asked for an untested draft. Use update_workflow only."
+ )
+ user_facing = "I'll save this as an untested draft because the request asked me not to run it."
+ recovery_hint = "retry_with_different_tool"
+ cleared_by = frozenset({"update_workflow"})
elif policy.clarification_reason == "workflow_credential_inputs_unbound":
- error = (
+ reason_code = "request_policy_blocks_run_blocks_credential_unbound"
+ agent_steering = (
"Skipped test run: the existing workflow references credential parameters "
"whose keys point to workflow inputs that are not configured. REPLY to the user "
"with: 'I applied your requested change. I couldn't test the modified workflow "
"because I couldn't find the required credentials — please add them via the "
"Credentials UI, then I can try again.' Keep the unvalidated draft surfaced."
)
+ user_facing = (
+ "I applied your requested change. I couldn't test the modified workflow because "
+ "the required credentials aren't set up. Add them in the Credentials UI and ask "
+ "me to test it."
+ )
+ recovery_hint = "report_blocker_to_user"
else:
- error = (
+ reason_code = "request_policy_blocks_run_blocks_generic"
+ agent_steering = (
"Request policy blocks block-running tools for the latest user message. "
"Ask the user for the required safe credential or clarification before testing."
)
- if error is not None:
- LOG.info(
- "copilot authority gate blocked tool",
- authority_gate_layer="request_policy",
- blocked_tool=tool_name,
- request_policy_allow_update_workflow=policy.allow_update_workflow,
- request_policy_allow_run_blocks=policy.allow_run_blocks,
- request_policy_testing_intent=policy.testing_intent,
- request_policy_clarification_reason=policy.clarification_reason,
- )
- return error
- return None
+ user_facing = "I need a credential or clarification from you before I can test this workflow."
+ recovery_hint = "ask_user_clarifying"
+
+ if reason_code is None or agent_steering is None or user_facing is None:
+ return None
+
+ LOG.info(
+ "copilot authority gate evaluated tool",
+ authority_gate_layer="request_policy",
+ blocked_tool=tool_name,
+ request_policy_allow_update_workflow=policy.allow_update_workflow,
+ request_policy_allow_run_blocks=policy.allow_run_blocks,
+ request_policy_testing_intent=policy.testing_intent,
+ request_policy_clarification_reason=policy.clarification_reason,
+ safe_reason_code=reason_code,
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint=recovery_hint,
+ cleared_by_tools=cleared_by,
+ internal_reason_code=reason_code,
+ blocked_tool=tool_name,
+ )
def _request_policy_allows_credential_deferred_draft(ctx: AgentContext) -> bool:
@@ -1003,7 +1178,7 @@ def _turn_intent_has_edit_target(intent: TurnIntent) -> bool:
)
-def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
+def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
intent = getattr(ctx, "turn_intent", None)
if not isinstance(intent, TurnIntent):
return None
@@ -1015,7 +1190,7 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
labels = sorted(_workflow_yaml_blocks_by_label(getattr(ctx, "workflow_yaml", None)))
label_hint = ", ".join(labels[:8]) if labels else "no labeled blocks"
LOG.info(
- "copilot authority gate blocked tool",
+ "copilot authority gate evaluated tool",
authority_gate_layer="turn_intent",
turn_intent_mode=intent.mode.value,
turn_intent_target_entity_types=sorted(intent.target_entities),
@@ -1023,11 +1198,21 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
blocked_tool=tool_name,
safe_reason_code=reason_code,
)
- return (
- f"TurnIntent classified this turn as `edit`, but the latest user message references workflow/block "
- f"identifier(s) that are not present in the current workflow: {', '.join(unresolved_refs)}. "
- f"Current workflow labels include: {label_hint}. Ask the user which current block should change before "
- f"mutating or running blocks. safe_reason_code={reason_code}."
+ unresolved_str = ", ".join(unresolved_refs)
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ f"The latest user message references workflow/block identifier(s) that are not present in the "
+ f"current workflow: {unresolved_str}. Current workflow labels include: {label_hint}. Ask the user "
+ f"which current block should change before mutating or running blocks."
+ ),
+ user_facing_reason=(
+ f"I couldn't find the block(s) you mentioned ({unresolved_str}). "
+ f"Tell me which existing block to change."
+ ),
+ recovery_hint="ask_user_clarifying",
)
if (
@@ -1037,17 +1222,23 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
):
reason_code = "turn_intent_missing_edit_target"
LOG.info(
- "copilot authority gate blocked tool",
+ "copilot authority gate evaluated tool",
authority_gate_layer="turn_intent",
turn_intent_mode=intent.mode.value,
turn_intent_target_entity_types=sorted(intent.target_entities),
blocked_tool=tool_name,
safe_reason_code=reason_code,
)
- return (
- f"TurnIntent classified this turn as `edit`, but could not identify a specific workflow edit target, "
- f"so `{tool_name}` is not allowed. Ask the user which workflow/block should change before mutating. "
- f"safe_reason_code={reason_code}."
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ "Could not identify a specific workflow edit target. Ask the user which workflow/block should "
+ "change before mutating."
+ ),
+ user_facing_reason="Tell me which block or workflow you'd like me to change.",
+ recovery_hint="ask_user_clarifying",
)
blocks_update = tool_name in WORKFLOW_MUTATION_TOOLS and not authority.may_update_workflow
@@ -1091,7 +1282,7 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
else:
reason_code = "turn_intent_run_blocked"
LOG.info(
- "copilot authority gate blocked tool",
+ "copilot authority gate evaluated tool",
authority_gate_layer="turn_intent",
turn_intent_mode=intent.mode.value,
turn_intent_target_entity_types=sorted(intent.target_entities),
@@ -1103,54 +1294,104 @@ def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
)
action = "ask the user" if authority.requires_user_input else "answer the user"
detail = f" Ask: {intent.missing_context_question}" if intent.missing_context_question else ""
+
if blocks_run and not blocks_update and authority.may_update_workflow:
- return (
- f"TurnIntent classified this turn as `{intent.mode.value}`, so `{tool_name}` is not allowed "
- f"to run browser blocks for the latest user message. Use `update_workflow` only and keep the draft "
- f"unvalidated. safe_reason_code={reason_code}.{detail}"
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ "Browser blocks may not run for the latest user message. Use update_workflow only and keep the "
+ f"draft unvalidated.{detail}"
+ ),
+ user_facing_reason="I'll save the change as a draft without running it.",
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({"update_workflow"}),
)
if blocks_context_read and not blocks_update and not blocks_run:
- return (
- f"TurnIntent classified this turn as `{intent.mode.value}`, so `{tool_name}` is not allowed "
- f"to read run context for the latest user message. Answer using the context already provided. "
- f"safe_reason_code={reason_code}.{detail}"
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ "Run context may not be read for the latest user message. Answer using the context already "
+ f"provided.{detail}"
+ ),
+ user_facing_reason="I'll answer with the information I already have.",
+ recovery_hint="report_blocker_to_user",
)
- return (
- f"TurnIntent classified this turn as `{intent.mode.value}`, so `{tool_name}` is not allowed "
- "for the latest user message. Do not update workflow YAML or run browser blocks, and do not "
- f"fetch additional run context with tools; {action} using the available context instead. "
- f"safe_reason_code={reason_code}.{detail}"
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ "This tool is not allowed for the latest user message. Do not update workflow YAML or run browser "
+ "blocks, and do not fetch additional run context with tools; "
+ f"{action} using the available context instead.{detail}"
+ ),
+ user_facing_reason="I'll respond with the information I already have.",
+ recovery_hint="ask_user_clarifying" if authority.requires_user_input else "report_blocker_to_user",
+ )
+
+
+def _build_turn_intent_signal(
+ *,
+ tool_name: str,
+ classifier_mode: str,
+ reason_code: str,
+ agent_steering_text: str,
+ user_facing_reason: str,
+ recovery_hint: RecoveryHint,
+ cleared_by_tools: frozenset[str] = frozenset(),
+) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text=agent_steering_text,
+ user_facing_reason=user_facing_reason,
+ recovery_hint=recovery_hint,
+ cleared_by_tools=cleared_by_tools,
+ internal_reason_code=reason_code,
+ blocked_tool=tool_name,
+ classifier_mode=classifier_mode,
)
+def _emit_tool_blocker_signal(ctx: AgentContext, signal: CopilotToolBlockerSignal) -> str:
+ return stash_blocker_signal(ctx, signal)
+
+
def _authority_tool_error(
ctx: AgentContext,
tool_name: str,
*,
ignore_request_policy_error: bool = False,
) -> str | None:
- turn_intent_error = _turn_intent_tool_error(ctx, tool_name)
- request_policy_error = _request_policy_tool_error(ctx, tool_name)
- if turn_intent_error is not None and request_policy_error is not None:
+ # Request-policy precedes turn-intent unless explicitly ignored.
+ turn_intent_signal = _turn_intent_tool_error(ctx, tool_name)
+ request_policy_signal = _request_policy_tool_error(ctx, tool_name)
+ if turn_intent_signal is not None and request_policy_signal is not None:
LOG.info(
"copilot authority gate blocked tool",
authority_gate_layer="both",
blocked_tool=tool_name,
)
- if request_policy_error is not None and not ignore_request_policy_error:
- return request_policy_error
- if turn_intent_error is not None:
- return turn_intent_error
- phase_error = _phase_tool_error(ctx, tool_name)
- if phase_error is not None:
+ chosen = (
+ request_policy_signal
+ if (request_policy_signal is not None and not ignore_request_policy_error)
+ else turn_intent_signal
+ )
+ if chosen is not None:
+ return _emit_tool_blocker_signal(ctx, chosen)
+ phase_signal = _phase_blocker_signal(ctx, tool_name)
+ if phase_signal is not None:
LOG.info(
"copilot authority gate blocked tool",
authority_gate_layer="build_phase",
blocked_tool=tool_name,
build_phase=getattr(getattr(ctx, "build_phase", None), "value", None),
- safe_reason_code="build_phase_gate",
)
- return phase_error
+ return _emit_tool_blocker_signal(ctx, phase_signal)
+ return None
_PARAMETER_TYPE_PLACEHOLDERS: dict[WorkflowParameterType, Any] = {
@@ -4134,13 +4375,25 @@ async def update_workflow_tool(
if loop_error:
return json.dumps({"ok": False, "error": loop_error})
if _request_policy_allows_credential_deferred_draft(copilot_ctx):
- result = {
- "ok": False,
- "error": (
- "Use update_and_run_blocks for this credential-deferred draft. It will save the workflow draft "
- "and skip the browser run with the required credential setup message."
- ),
- }
+ agent_steering = (
+ "Use update_and_run_blocks for this credential-deferred draft. It will save the workflow draft "
+ "and skip the browser run with the required credential setup message."
+ )
+ user_facing = (
+ "I can save this as a draft without running it because the credentials aren't set up yet. "
+ "Add them in the Credentials UI and ask me to test the workflow."
+ )
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({"update_and_run_blocks"}),
+ internal_reason_code="request_policy_credential_deferred_redirect",
+ blocked_tool="update_workflow",
+ )
+ payload = _emit_tool_blocker_signal(copilot_ctx, signal)
+ result = {"ok": False, "error": payload}
record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
return json.dumps(result)
@@ -4308,9 +4561,9 @@ async def get_run_results_tool(
loop_error = _tool_loop_error(copilot_ctx, "get_run_results", params)
if loop_error:
return json.dumps({"ok": False, "error": loop_error})
- turn_intent_error = _turn_intent_tool_error(copilot_ctx, "get_run_results")
- if turn_intent_error:
- return json.dumps({"ok": False, "error": turn_intent_error})
+ authority_error = _authority_tool_error(copilot_ctx, "get_run_results")
+ if authority_error:
+ return json.dumps({"ok": False, "error": authority_error})
result = await _get_run_results(params, copilot_ctx)
_record_per_tool_budget_problem_blocks_from_results(copilot_ctx, result)
diff --git a/tests/unit/test_copilot_blocker_lifecycle.py b/tests/unit/test_copilot_blocker_lifecycle.py
new file mode 100644
index 000000000..b42fbae27
--- /dev/null
+++ b/tests/unit/test_copilot_blocker_lifecycle.py
@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from skyvern.forge.sdk.copilot.blocker_signal import BlockerKind, CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
+
+
+def _ctx() -> CopilotContext:
+ return CopilotContext(
+ organization_id="org",
+ workflow_id="wf",
+ workflow_permanent_id="wfp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+
+
+def _signal(
+ *,
+ kind: BlockerKind = "authority_denied",
+ cleared_by: frozenset[str] = frozenset(),
+ reason: str = "some_reason",
+) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind=kind,
+ agent_steering_text="steering",
+ user_facing_reason="I couldn't do that on this turn.",
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=cleared_by,
+ internal_reason_code=reason,
+ blocked_tool="update_and_run_blocks",
+ )
+
+
+def test_recoverable_blocker_clears_on_matching_tool_success() -> None:
+ ctx = _ctx()
+ ctx.blocker_signal = _signal(cleared_by=frozenset({"update_workflow"}))
+ record_tool_step_result_for_ctx(ctx, "update_workflow", {"workflow_yaml": "y"}, {"ok": True})
+ assert ctx.blocker_signal is None
+
+
+def test_recoverable_blocker_does_not_clear_on_unrelated_tool_success() -> None:
+ ctx = _ctx()
+ signal = _signal(cleared_by=frozenset({"update_workflow"}))
+ ctx.blocker_signal = signal
+ record_tool_step_result_for_ctx(ctx, "list_credentials", None, {"ok": True})
+ assert ctx.blocker_signal is signal
+
+
+def test_loop_blocker_stays_sticky_on_any_success() -> None:
+ ctx = _ctx()
+ signal = _signal(kind="loop_detected", cleared_by=frozenset(), reason="loop_detected_generic")
+ ctx.blocker_signal = signal
+ record_tool_step_result_for_ctx(ctx, "update_workflow", {"workflow_yaml": "y"}, {"ok": True})
+ assert ctx.blocker_signal is signal
+
+
+def test_terminal_tool_error_stays_sticky_on_any_success() -> None:
+ ctx = _ctx()
+ signal = _signal(kind="tool_error", cleared_by=frozenset(), reason="tool_error_repeated_action_abort")
+ ctx.blocker_signal = signal
+ record_tool_step_result_for_ctx(ctx, "update_workflow", {"workflow_yaml": "y"}, {"ok": True})
+ assert ctx.blocker_signal is signal
+
+
+def test_failed_dispatch_does_not_clear_signal() -> None:
+ ctx = _ctx()
+ signal = _signal(cleared_by=frozenset({"update_workflow"}))
+ ctx.blocker_signal = signal
+ record_tool_step_result_for_ctx(ctx, "update_workflow", {"workflow_yaml": "y"}, {"ok": False, "error": "x"})
+ # Failed dispatch must not satisfy a recovery hint.
+ assert ctx.blocker_signal is signal
+
+
+def test_per_tool_budget_blocker_clears_on_update_and_run_blocks_success() -> None:
+ """Per-tool-budget steering directs the agent to split blocks; the recovery
+ can land via either ``update_workflow`` or ``update_and_run_blocks``."""
+ from skyvern.forge.sdk.copilot.tools import _per_tool_budget_problem_rerun_signal
+
+ ctx = _ctx()
+ ctx.per_tool_budget_problem_block_labels = ["heavy_navigation"]
+ signal = _per_tool_budget_problem_rerun_signal(ctx, None, "run_blocks_and_collect_debug")
+ assert signal is not None
+ assert "update_and_run_blocks" in signal.cleared_by_tools
+ ctx.blocker_signal = signal
+ record_tool_step_result_for_ctx(ctx, "update_and_run_blocks", {"workflow_yaml": "y"}, {"ok": True})
+ assert ctx.blocker_signal is None
+
+
+def test_reconciliation_canceled_status_replaces_no_input_signal_with_requires_input() -> None:
+ """When the reconciliation read resolves the pending run as canceled, the
+ 'I'll check what happened' blocker is replaced with one that asks the
+ user to decide."""
+ from skyvern.forge.sdk.copilot.tools import _maybe_clear_reconciliation_flag
+ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+
+ ctx = _ctx()
+ ctx.pending_reconciliation_run_id = "wr_pending"
+ ctx.blocker_signal = _signal(
+ kind="tool_error",
+ cleared_by=frozenset(),
+ reason="tool_error_pending_reconciliation_no_input",
+ )
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_pending",
+ "overall_status": WorkflowRunStatus.canceled.value,
+ },
+ }
+ _maybe_clear_reconciliation_flag(ctx, result)
+ assert ctx.pending_reconciliation_requires_user_input is True
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == "tool_error_pending_reconciliation_requires_input"
+ assert ctx.blocker_signal.recovery_hint == "ask_user_clarifying"
+
+
+def test_reconciliation_canceled_does_not_overwrite_unrelated_blocker() -> None:
+ """If ctx already holds an unrelated blocker (e.g. ``loop_detected``), the
+ canceled-status transition must not silently replace it. The no_input
+ clear is a no-op against the unrelated reason code, and the subsequent
+ ``isinstance`` check sees the blocker still set and skips the set."""
+ from skyvern.forge.sdk.copilot.tools import _maybe_clear_reconciliation_flag
+ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+
+ ctx = _ctx()
+ ctx.pending_reconciliation_run_id = "wr_pending"
+ unrelated = _signal(
+ kind="loop_detected",
+ cleared_by=frozenset(),
+ reason="loop_detected_generic",
+ )
+ ctx.blocker_signal = unrelated
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_pending",
+ "overall_status": WorkflowRunStatus.canceled.value,
+ },
+ }
+ _maybe_clear_reconciliation_flag(ctx, result)
+ assert ctx.pending_reconciliation_requires_user_input is True
+ assert ctx.blocker_signal is unrelated
+
+
+def test_reconciliation_signal_steering_text_does_not_leak_through_user_facing_check() -> None:
+ """The reconciliation requires-input signal contains 'do not run' in its
+ ``agent_steering_text`` (legitimate agent imperative). If a future
+ refactor accidentally swaps the field used by the renderer, the user
+ would see leaky agent-control prose. Test that
+ ``assert_clean_user_facing_text`` correctly rejects the steering text so
+ such a swap would fail tests immediately."""
+ import pytest
+
+ from skyvern.forge.sdk.copilot.blocker_signal import assert_clean_user_facing_text
+ from skyvern.forge.sdk.copilot.tools import _maybe_clear_reconciliation_flag
+ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+
+ ctx = _ctx()
+ ctx.pending_reconciliation_run_id = "wr_pending"
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_pending",
+ "overall_status": WorkflowRunStatus.canceled.value,
+ },
+ }
+ _maybe_clear_reconciliation_flag(ctx, result)
+ assert ctx.blocker_signal is not None
+ # user_facing_reason must pass the deny list.
+ assert_clean_user_facing_text(ctx.blocker_signal.user_facing_reason)
+ # agent_steering_text must FAIL the deny list — that's the contract:
+ # steering text is for the LLM, not the user, and may carry imperatives.
+ # If a future refactor pipes steering text into the renderer, this fails.
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text(ctx.blocker_signal.agent_steering_text)
diff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.py
new file mode 100644
index 000000000..3ef635473
--- /dev/null
+++ b/tests/unit/test_copilot_blocker_signal.py
@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+import pytest
+
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ _LEAK_DENY_TOKENS,
+ BlockerKind,
+ CopilotToolBlockerSignal,
+ assert_clean_user_facing_text,
+ build_llm_tool_error_payload,
+ clear_blocker_signal_for_reason_codes,
+ maybe_clear_blocker_signal_on_tool_success,
+ stash_blocker_signal,
+ to_trace_data,
+)
+
+
+def _make(
+ *,
+ kind: BlockerKind = "authority_denied",
+ cleared_by_tools: frozenset[str] = frozenset(),
+ internal_reason_code: str = "some_reason",
+ blocked_tool: str = "update_workflow",
+) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind=kind,
+ agent_steering_text="Take this recovery step.",
+ user_facing_reason="I couldn't do that on this turn.",
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=cleared_by_tools,
+ internal_reason_code=internal_reason_code,
+ blocked_tool=blocked_tool,
+ classifier_mode="docs_answer",
+ )
+
+
+def test_model_round_trips_through_validate() -> None:
+ signal = _make()
+ restored = CopilotToolBlockerSignal.model_validate(signal.model_dump())
+ assert restored == signal
+
+
+def test_build_llm_payload_is_agent_steering_text_only() -> None:
+ signal = _make()
+ payload = build_llm_tool_error_payload(signal)
+ assert payload == signal.agent_steering_text
+ assert "recovery_hint" not in payload
+ assert signal.internal_reason_code is not None and signal.internal_reason_code not in payload
+ assert "docs_answer" not in payload
+
+
+def test_to_trace_data_surfaces_internal_fields() -> None:
+ signal = _make(internal_reason_code="r1", cleared_by_tools=frozenset({"a", "b"}))
+ trace = to_trace_data(signal)
+ assert trace["internal_reason_code"] == "r1"
+ assert trace["blocker_kind"] == "authority_denied"
+ assert trace["classifier_mode"] == "docs_answer"
+ assert trace["cleared_by_tools"] == ["a", "b"]
+ assert trace["extra"] == {}
+
+
+def test_to_trace_data_namespaces_extra_so_it_cannot_shadow_explicit_fields() -> None:
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text="x",
+ user_facing_reason="y",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code="r1",
+ # A producer that stuffs ``blocker_kind`` into ``extra`` must not
+ # silently shadow the canonical top-level field.
+ extra={"blocker_kind": "evil", "custom_metric": 7},
+ )
+ trace = to_trace_data(signal)
+ assert trace["blocker_kind"] == "authority_denied"
+ assert trace["extra"] == {"blocker_kind": "evil", "custom_metric": 7}
+
+
[email protected]("token", _LEAK_DENY_TOKENS)
+def test_assert_clean_raises_on_each_deny_token(token: str) -> None:
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text(f"prefix {token} suffix")
+
+
+def test_assert_clean_raises_on_blocked_tool_substring() -> None:
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text("calling get_run_results was wrong", blocked_tool="get_run_results")
+
+
+def test_assert_clean_passes_normal_product_language() -> None:
+ assert_clean_user_facing_text("I couldn't complete that on this turn.")
+
+
+class _Ctx:
+ blocker_signal: CopilotToolBlockerSignal | None = None
+
+
+def test_maybe_clear_on_tool_success_clears_when_in_cleared_by_tools() -> None:
+ ctx = _Ctx()
+ ctx.blocker_signal = _make(cleared_by_tools=frozenset({"update_workflow"}))
+ maybe_clear_blocker_signal_on_tool_success(ctx, "update_workflow")
+ assert ctx.blocker_signal is None
+
+
+def test_maybe_clear_on_tool_success_no_match_keeps_signal() -> None:
+ ctx = _Ctx()
+ signal = _make(cleared_by_tools=frozenset({"update_workflow"}))
+ ctx.blocker_signal = signal
+ maybe_clear_blocker_signal_on_tool_success(ctx, "list_credentials")
+ assert ctx.blocker_signal is signal
+
+
+def test_maybe_clear_on_tool_success_empty_set_keeps_signal() -> None:
+ ctx = _Ctx()
+ signal = _make()
+ ctx.blocker_signal = signal
+ maybe_clear_blocker_signal_on_tool_success(ctx, "update_workflow")
+ assert ctx.blocker_signal is signal
+
+
+def test_clear_for_reason_codes_matches() -> None:
+ ctx = _Ctx()
+ ctx.blocker_signal = _make(internal_reason_code="tool_error_pending_reconciliation_no_input")
+ clear_blocker_signal_for_reason_codes(ctx, frozenset({"tool_error_pending_reconciliation_no_input"}))
+ assert ctx.blocker_signal is None
+
+
+def test_clear_for_reason_codes_no_match() -> None:
+ ctx = _Ctx()
+ signal = _make(internal_reason_code="loop_detected_generic")
+ ctx.blocker_signal = signal
+ clear_blocker_signal_for_reason_codes(ctx, frozenset({"tool_error_pending_reconciliation_no_input"}))
+ assert ctx.blocker_signal is signal
+
+
+def test_clear_helpers_ignore_non_signal_values_on_ctx() -> None:
+ """Defensive: structurally satisfying the Protocol with a non-signal value
+ (e.g. a stray attribute set by another subsystem) must not be cleared."""
+ ctx = _Ctx()
+ ctx.blocker_signal = "not a signal" # type: ignore[assignment]
+ maybe_clear_blocker_signal_on_tool_success(ctx, "update_workflow")
+ assert ctx.blocker_signal == "not a signal"
+ clear_blocker_signal_for_reason_codes(ctx, frozenset({"some_reason"}))
+ assert ctx.blocker_signal == "not a signal"
+
+
+def test_deny_list_is_case_insensitive() -> None:
+ """A future template that drops or flips casing on a leak phrase must
+ still be caught — agent prompts have used both ``Do NOT`` and ``do not``."""
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text("Do not run that step again")
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text("DO NOT RUN that step again")
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text("safe_REASON_code=foo")
+
+
+def test_deny_list_narrow_imperatives_do_not_false_positive() -> None:
+ """Plain ``do not`` followed by non-imperative copy is legitimate
+ product language and must not trip the guard."""
+ assert_clean_user_facing_text("I'm sorry, do not worry — I'll try again.")
+ assert_clean_user_facing_text("Please do not hesitate to share more context.")
+
+
+def test_model_validator_rejects_leaky_user_facing_at_construction() -> None:
+ with pytest.raises(ValueError):
+ CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text="agent steering — anything goes here",
+ user_facing_reason="DO NOT RUN this — talk to user first",
+ recovery_hint="report_blocker_to_user",
+ )
+
+
+def test_model_validator_rejects_blocked_tool_name_in_user_facing() -> None:
+ with pytest.raises(ValueError):
+ CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="x",
+ user_facing_reason="couldn't call get_run_results on this turn",
+ recovery_hint="report_blocker_to_user",
+ blocked_tool="get_run_results",
+ )
+
+
+def test_model_validator_blocked_tool_check_is_case_insensitive() -> None:
+ with pytest.raises(ValueError):
+ CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="x",
+ user_facing_reason="I had to call Update_Workflow",
+ recovery_hint="report_blocker_to_user",
+ blocked_tool="update_workflow",
+ )
+
+
+def test_extra_is_mapping_proxy_after_construction() -> None:
+ from types import MappingProxyType
+
+ signal = _make()
+ assert isinstance(signal.extra, MappingProxyType)
+ with pytest.raises(TypeError):
+ signal.extra["k"] = "v" # type: ignore[index]
+
+
+def test_extra_default_does_not_alias_across_instances() -> None:
+ a = _make()
+ b = _make()
+ assert a.extra is not b.extra
+
+
+def test_stash_blocker_signal_first_wins_returns_llm_payload() -> None:
+ ctx = _Ctx()
+ first = _make(internal_reason_code="first")
+ payload = stash_blocker_signal(ctx, first)
+ assert payload == first.agent_steering_text
+ assert ctx.blocker_signal is first
+
+ second = _make(internal_reason_code="second")
+ payload2 = stash_blocker_signal(ctx, second)
+ assert payload2 == second.agent_steering_text # LLM payload is the current signal's
+ assert ctx.blocker_signal is first # stash is sticky
+
+
+def test_agent_context_and_copilot_context_blocker_signal_defaults_match() -> None:
+ """The field is declared on both AgentContext (parent) and CopilotContext
+ (child) per the field-shadowing convention. Default values must stay in
+ sync so callers reading via the AgentContext annotation see the same
+ initial state as callers reading via CopilotContext."""
+ from types import SimpleNamespace
+
+ from skyvern.forge.sdk.copilot.context import CopilotContext
+ from skyvern.forge.sdk.copilot.runtime import AgentContext
+
+ agent_ctx = AgentContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+ copilot_ctx = CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+ assert agent_ctx.blocker_signal is None
+ assert copilot_ctx.blocker_signal is None
+ assert agent_ctx.blocker_signal == copilot_ctx.blocker_signal
diff --git a/tests/unit/test_copilot_build_phase.py b/tests/unit/test_copilot_build_phase.py
index 8a39fc3c6..d01f27c35 100644
--- a/tests/unit/test_copilot_build_phase.py
+++ b/tests/unit/test_copilot_build_phase.py
@@ -253,3 +253,60 @@ def test_phase_tool_error_returns_none_when_phase_attr_missing() -> None:
pass
assert _phase_tool_error(_NoPhase(), "navigate_browser") is None
+
+
+ "tool_name,phase,expected_reason_code,expected_recovery_hint,cleared_by",
+ [
+ (
+ "discover_workflow_entrypoint",
+ BuildPhase.COMPOSING,
+ "build_phase_discovery_disallowed_post_compose",
+ "retry_with_different_tool",
+ frozenset({"update_workflow", "update_and_run_blocks"}),
+ ),
+ (
+ "navigate_browser",
+ BuildPhase.INITIAL,
+ "build_phase_browser_blocked_pre_compose",
+ "ask_user_clarifying",
+ frozenset(),
+ ),
+ (
+ "update_workflow",
+ BuildPhase.INITIAL,
+ "build_phase_mutation_blocked_pre_compose",
+ "ask_user_clarifying",
+ frozenset(),
+ ),
+ ],
+)
+def test_phase_blocker_signal_returns_structured_signal(
+ tool_name: str,
+ phase: BuildPhase,
+ expected_reason_code: str,
+ expected_recovery_hint: str,
+ cleared_by: frozenset[str],
+) -> None:
+ from skyvern.forge.sdk.copilot.blocker_signal import _LEAK_DENY_TOKENS
+ from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
+
+ ctx = _Ctx(phase)
+ signal = _phase_blocker_signal(ctx, tool_name)
+ assert signal is not None
+ assert signal.blocker_kind == "phase_gated"
+ assert signal.internal_reason_code == expected_reason_code
+ assert signal.recovery_hint == expected_recovery_hint
+ assert signal.cleared_by_tools == cleared_by
+ assert signal.blocked_tool == tool_name
+ for token in _LEAK_DENY_TOKENS:
+ assert token.lower() not in signal.user_facing_reason.lower()
+
+
+def test_phase_blocker_signal_returns_none_when_phase_attr_missing() -> None:
+ from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
+
+ class _NoPhase:
+ pass
+
+ assert _phase_blocker_signal(_NoPhase(), "navigate_browser") is None
diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.py
new file mode 100644
index 000000000..f80fcedb6
--- /dev/null
+++ b/tests/unit/test_copilot_loop_detection_blocker.py
@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_error
+
+_LEAK_TOKENS = ("safe_reason_code", "LOOP DETECTED:")
+
+
+def _ctx(
+ *,
+ failed_tool_step_tracker: dict[str, int] | None = None,
+ consecutive_tool_tracker: list[str] | None = None,
+) -> CopilotContext:
+ ctx = CopilotContext(
+ organization_id="org",
+ workflow_id="wf",
+ workflow_permanent_id="wfp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+ if failed_tool_step_tracker is not None:
+ ctx.failed_tool_step_tracker = failed_tool_step_tracker
+ if consecutive_tool_tracker is not None:
+ ctx.consecutive_tool_tracker = consecutive_tool_tracker
+ return ctx
+
+
+ ("loop_message", "expected_reason"),
+ [
+ (
+ "LOOP DETECTED: 'update_and_run_blocks' has already failed 3 times with CREDENTIAL_ERROR; blocking attempt #4.",
+ "loop_detected_credential_or_parameter_misconfig",
+ ),
+ (
+ "LOOP DETECTED: 'update_and_run_blocks' has already failed 3 times with PARAMETER_BINDING_ERROR; blocking attempt #4.",
+ "loop_detected_credential_or_parameter_misconfig",
+ ),
+ (
+ "LOOP DETECTED: 'update_workflow' has already failed 3 consecutive times with these arguments; blocking attempt #4.",
+ "loop_detected_repeated_failed_step",
+ ),
+ (
+ "LOOP DETECTED: 'list_credentials' has been called 3 times consecutively.",
+ "loop_detected_consecutive_same_tool",
+ ),
+ ],
+)
+def test_loop_blocker_dispatches_on_substring(loop_message: str, expected_reason: str) -> None:
+ signal = _build_loop_blocker_signal(loop_message, tool_name="update_and_run_blocks")
+ assert signal.blocker_kind == "loop_detected"
+ assert signal.internal_reason_code == expected_reason
+ # LLM-visible string keeps the raw marker so output_utils sanitization fires.
+ assert signal.agent_steering_text == loop_message
+ # User-facing string must NOT carry the marker.
+ for token in _LEAK_TOKENS:
+ assert token not in signal.user_facing_reason
+ assert signal.cleared_by_tools == frozenset()
+
+
+def test_loop_blocker_falls_back_to_generic_on_novel_message() -> None:
+ # Pins the catch-all branch so a new loop format from detect_*_loop doesn't
+ # silently slip into one of the substring-matched buckets.
+ signal = _build_loop_blocker_signal("something unfamiliar happened", tool_name="update_workflow")
+ assert signal.internal_reason_code == "loop_detected_generic"
+ assert signal.recovery_hint == "report_blocker_to_user"
+
+
+def test_native_dispatch_failed_step_loop_sets_signal_and_returns_payload() -> None:
+ # Tracker simulates 2 prior failures of (update_workflow, {workflow_yaml: 'y'}).
+ # detect_failed_tool_step_loop_for_ctx uses tool_step_identity to look up the
+ # threshold; mimicking the real identity is brittle, so call the dispatcher
+ # with a tracker that already has the same identity prepopulated at threshold-1.
+ from skyvern.forge.sdk.copilot.loop_detection import tool_step_identity
+
+ identity = tool_step_identity("update_workflow", {"workflow_yaml": "yaml-1"})
+ ctx = _ctx(failed_tool_step_tracker={identity: 3})
+ payload = _tool_loop_error(ctx, "update_workflow", {"workflow_yaml": "yaml-1"})
+ assert payload is not None
+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert ctx.blocker_signal.blocker_kind == "loop_detected"
+ assert ctx.blocker_signal.internal_reason_code == "loop_detected_repeated_failed_step"
+ # LLM payload is agent_steering_text only — raw LOOP DETECTED: marker is in it.
+ assert payload.startswith("LOOP DETECTED:")
+ # Renderer-side string is clean.
+ for token in _LEAK_TOKENS:
+ assert token not in ctx.blocker_signal.user_facing_reason
+
+
+def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:
+ ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])
+ payload = _tool_loop_error(ctx, "list_credentials", None)
+ assert payload is not None
+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"
+ assert ctx.blocker_signal.cleared_by_tools == frozenset()
+
+
+def test_native_and_mcp_paths_produce_equivalent_loop_signal() -> None:
+ """Native dispatch and the MCP adapter must produce the same signal shape
+ for the same loop-detection message so a regression in one path can't
+ diverge from the other.
+ """
+ from skyvern.forge.sdk.copilot.mcp_adapter import _stash_and_emit_loop_blocker
+
+ loop_message = (
+ "LOOP DETECTED: 'update_workflow' has already failed 3 consecutive times with these arguments; "
+ "blocking attempt #4."
+ )
+ native_ctx = _ctx()
+ mcp_ctx = _ctx()
+
+ # Native path: build the signal via _build_loop_blocker_signal directly
+ # (same as _tool_loop_error's first branch).
+ native_signal = _build_loop_blocker_signal(loop_message, tool_name="update_workflow")
+ native_ctx.blocker_signal = native_signal
+
+ # MCP path: _stash_and_emit_loop_blocker uses _build_loop_blocker_signal
+ # under the hood, stashes on ctx, returns LLM payload.
+ mcp_payload = _stash_and_emit_loop_blocker(mcp_ctx, loop_message, "update_workflow")
+
+ assert mcp_payload == native_signal.agent_steering_text
+ assert isinstance(mcp_ctx.blocker_signal, CopilotToolBlockerSignal)
+ # Signal fields match exactly across paths.
+ assert mcp_ctx.blocker_signal.blocker_kind == native_signal.blocker_kind
+ assert mcp_ctx.blocker_signal.internal_reason_code == native_signal.internal_reason_code
+ assert mcp_ctx.blocker_signal.user_facing_reason == native_signal.user_facing_reason
+ assert mcp_ctx.blocker_signal.agent_steering_text == native_signal.agent_steering_text
+ assert mcp_ctx.blocker_signal.recovery_hint == native_signal.recovery_hint
+ assert mcp_ctx.blocker_signal.cleared_by_tools == native_signal.cleared_by_tools
+ assert mcp_ctx.blocker_signal.blocked_tool == native_signal.blocked_tool
diff --git a/tests/unit/test_copilot_turn_intent_tool_gate.py b/tests/unit/test_copilot_turn_intent_tool_gate.py
index fd2d88b56..820ceb92c 100644
--- a/tests/unit/test_copilot_turn_intent_tool_gate.py
+++ b/tests/unit/test_copilot_turn_intent_tool_gate.py
@@ -6,9 +6,16 @@ from unittest.mock import AsyncMock, patch
import pytest
from skyvern.forge.sdk.copilot.agent import _native_tools_for_turn
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
-from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS, _get_run_results, _turn_intent_tool_error, _update_workflow
+from skyvern.forge.sdk.copilot.tools import (
+ NATIVE_TOOLS,
+ _authority_tool_error,
+ _get_run_results,
+ _turn_intent_tool_error,
+ _update_workflow,
+)
from skyvern.forge.sdk.copilot.turn_intent import (
UNRESOLVED_BLOCK_REF_TARGET_ENTITY,
TurnIntent,
@@ -16,6 +23,10 @@ from skyvern.forge.sdk.copilot.turn_intent import (
TurnIntentMode,
)
+# Tokens that must never appear in any LLM-visible string emitted by an
+# authority gate. The previous prose path leaked all of these.
+_LEAK_TOKENS = ("safe_reason_code", "`TurnIntent`", "TurnIntent classified")
+
def _ctx(
turn_intent: TurnIntent,
@@ -41,16 +52,36 @@ def _ctx(
return ctx
+def _assert_signal(
+ signal: CopilotToolBlockerSignal | None,
+ *,
+ internal_reason_code: str,
+ classifier_mode: str | None = None,
+ blocked_tool: str,
+) -> CopilotToolBlockerSignal:
+ assert signal is not None
+ assert signal.blocker_kind == "authority_denied"
+ assert signal.internal_reason_code == internal_reason_code
+ assert signal.blocked_tool == blocked_tool
+ if classifier_mode is not None:
+ assert signal.classifier_mode == classifier_mode
+ for token in _LEAK_TOKENS:
+ assert token not in signal.user_facing_reason
+ return signal
+
+
@pytest.mark.parametrize(
- ("mode", "tool_name"),
+ ("mode", "tool_name", "expected_reason"),
[
- (TurnIntentMode.DOCS_ANSWER, "update_workflow"),
- (TurnIntentMode.DIAGNOSE, "run_blocks_and_collect_debug"),
- (TurnIntentMode.CLARIFY, "update_and_run_blocks"),
- (TurnIntentMode.REFUSE, "update_and_run_blocks"),
+ (TurnIntentMode.DOCS_ANSWER, "update_workflow", "turn_intent_no_mutation_update_blocked"),
+ (TurnIntentMode.DIAGNOSE, "run_blocks_and_collect_debug", "turn_intent_no_mutation_run_blocked"),
+ (TurnIntentMode.CLARIFY, "update_and_run_blocks", "turn_intent_no_mutation_run_blocked"),
+ (TurnIntentMode.REFUSE, "update_and_run_blocks", "turn_intent_no_mutation_run_blocked"),
],
)
-def test_no_mutation_turn_intent_blocks_mutating_tools(mode: TurnIntentMode, tool_name: str) -> None:
+def test_no_mutation_turn_intent_blocks_mutating_tools(
+ mode: TurnIntentMode, tool_name: str, expected_reason: str
+) -> None:
intent = TurnIntent(
mode=mode,
authority=TurnIntentAuthority(
@@ -61,13 +92,9 @@ def test_no_mutation_turn_intent_blocks_mutating_tools(mode: TurnIntentMode, too
missing_context_question="Which target should I use?" if mode == TurnIntentMode.CLARIFY else None,
)
- error = _turn_intent_tool_error(_ctx(intent), tool_name)
-
- assert error is not None
- assert f"`{mode.value}`" in error
- assert f"`{tool_name}`" in error
- assert "Do not update workflow YAML or run browser blocks" in error
- assert "safe_reason_code=turn_intent_no_mutation" in error
+ ctx = _ctx(intent)
+ signal = _turn_intent_tool_error(ctx, tool_name)
+ _assert_signal(signal, internal_reason_code=expected_reason, classifier_mode=mode.value, blocked_tool=tool_name)
def test_turn_intent_gate_allows_draft_update_without_run_authority() -> None:
@@ -85,11 +112,16 @@ def test_turn_intent_gate_blocks_draft_only_run_tools() -> None:
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=False),
)
- error = _turn_intent_tool_error(_ctx(intent), "update_and_run_blocks")
-
- assert error is not None
- assert "Use `update_workflow` only" in error
- assert "safe_reason_code=turn_intent_run_blocked" in error
+ signal = _turn_intent_tool_error(_ctx(intent), "update_and_run_blocks")
+ _assert_signal(
+ signal,
+ internal_reason_code="turn_intent_run_blocked",
+ classifier_mode="draft_only",
+ blocked_tool="update_and_run_blocks",
+ )
+ assert signal is not None
+ assert signal.recovery_hint == "retry_with_different_tool"
+ assert "update_workflow" in signal.cleared_by_tools
def test_turn_intent_gate_allows_build_update_and_run_authority() -> None:
@@ -107,11 +139,15 @@ def test_turn_intent_gate_blocks_edit_without_target_context() -> None:
authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
)
- error = _turn_intent_tool_error(_ctx(intent), "update_workflow")
-
- assert error is not None
- assert "could not identify a specific workflow edit target" in error
- assert "safe_reason_code=turn_intent_missing_edit_target" in error
+ signal = _turn_intent_tool_error(_ctx(intent), "update_workflow")
+ _assert_signal(
+ signal,
+ internal_reason_code="turn_intent_missing_edit_target",
+ classifier_mode="edit",
+ blocked_tool="update_workflow",
+ )
+ assert signal is not None
+ assert signal.recovery_hint == "ask_user_clarifying"
def test_turn_intent_gate_allows_edit_with_target_context() -> None:
@@ -150,13 +186,20 @@ workflow_definition:
navigation_goal: Confirm success.
"""
- error = _turn_intent_tool_error(ctx, "update_and_run_blocks")
-
- assert error is not None
- assert "WF_trigger_SSO_login" in error
- assert "update_card" in error
- assert "navigate_to_SSO" in error
- assert "safe_reason_code=turn_intent_unresolved_edit_target" in error
+ signal = _turn_intent_tool_error(ctx, "update_and_run_blocks")
+ _assert_signal(
+ signal,
+ internal_reason_code="turn_intent_unresolved_edit_target",
+ classifier_mode="edit",
+ blocked_tool="update_and_run_blocks",
+ )
+ assert signal is not None
+ # The agent steering text still names the missing refs so the agent can ask
+ # specifically; the user_facing_reason names them too.
+ assert "WF_trigger_SSO_login" in signal.agent_steering_text
+ assert "update_card" in signal.agent_steering_text
+ assert "navigate_to_SSO" in signal.agent_steering_text
+ assert "WF_trigger_SSO_login" in signal.user_facing_reason
def test_turn_intent_gate_does_not_scan_raw_user_message_for_snake_case_refs() -> None:
@@ -222,7 +265,12 @@ async def test_update_workflow_stops_before_persisting_for_answer_only_intent()
result = await _update_workflow({"workflow_yaml": ctx.workflow_yaml}, ctx)
assert result["ok"] is False
- assert "`docs_answer`" in result["error"]
+ # LLM-visible error is agent_steering_text — product-language imperative,
+ # no internal vocabulary.
+ for token in _LEAK_TOKENS:
+ assert token not in result["error"]
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.classifier_mode == "docs_answer"
mock_app.WORKFLOW_SERVICE.update_workflow_definition.assert_not_called()
@@ -239,19 +287,52 @@ async def test_request_policy_refusal_wins_even_when_turn_intent_allows_update()
result = await _update_workflow({"workflow_yaml": ctx.workflow_yaml}, ctx)
assert result["ok"] is False
- assert result["error"].startswith("Request policy blocks workflow updates")
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == "request_policy_blocks_update_workflow"
mock_app.WORKFLOW_SERVICE.update_workflow_definition.assert_not_called()
+def test_authority_dispatcher_stashes_request_policy_signal_when_both_block() -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.DIAGNOSE,
+ authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ )
+ policy = RequestPolicy(allow_update_workflow=False, allow_run_blocks=False)
+ ctx = _ctx(intent, policy)
+
+ payload = _authority_tool_error(ctx, "update_workflow")
+ assert payload is not None
+ assert ctx.blocker_signal is not None
+ # Request policy precedence — turn-intent signal must not win the stash.
+ assert ctx.blocker_signal.internal_reason_code == "request_policy_blocks_update_workflow"
+ for token in _LEAK_TOKENS:
+ assert token not in payload
+
+
def test_update_and_run_blocks_reports_both_blocked_authorities() -> None:
intent = TurnIntent(
mode=TurnIntentMode.DIAGNOSE,
authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
)
- error = _turn_intent_tool_error(_ctx(intent), "update_and_run_blocks")
+ signal = _turn_intent_tool_error(_ctx(intent), "update_and_run_blocks")
+ _assert_signal(
+ signal,
+ internal_reason_code="turn_intent_no_mutation_run_blocked",
+ classifier_mode="diagnose",
+ blocked_tool="update_and_run_blocks",
+ )
+
- assert error is not None
- assert "turn_intent_no_mutation_run_blocked" in error
+def test_get_run_results_routes_through_authority_dispatcher_but_request_policy_does_not_gate_it() -> None:
+ """Pins the dispatcher routing: `get_run_results` flows through `_authority_tool_error` (not `_turn_intent_tool_error` directly), and current request-policy scope (`update_workflow` + `BLOCK_RUNNING_TOOLS`) does not include `get_run_results`, so the call passes through."""
+ intent = TurnIntent(
+ mode=TurnIntentMode.DIAGNOSE,
+ authority=TurnIntentAuthority(may_read_run_context=True),
+ )
+ policy = RequestPolicy(allow_update_workflow=False, allow_run_blocks=False)
+ ctx = _ctx(intent, policy)
+ assert _authority_tool_error(ctx, "get_run_results") is None
+ assert ctx.blocker_signal is None
def test_diagnose_allows_get_run_results_tool() -> None:
@@ -273,14 +354,13 @@ def test_unknown_without_run_context_blocks_get_run_results() -> None:
authority=TurnIntentAuthority(),
)
- error = _turn_intent_tool_error(_ctx(intent), "get_run_results")
-
- assert error is not None
- assert "`unknown`" in error
- assert "`get_run_results`" in error
- assert "turn_intent_context_read_blocked" in error
- assert "read run context" in error
- assert "Do not update workflow YAML or run browser blocks" not in error
+ signal = _turn_intent_tool_error(_ctx(intent), "get_run_results")
+ _assert_signal(
+ signal,
+ internal_reason_code="turn_intent_context_read_blocked",
+ classifier_mode="unknown",
+ blocked_tool="get_run_results",
+ )
def test_docs_answer_blocks_get_run_results() -> None:
@@ -289,11 +369,13 @@ def test_docs_answer_blocks_get_run_results() -> None:
authority=TurnIntentAuthority(),
)
- error = _turn_intent_tool_error(_ctx(intent), "get_run_results")
-
- assert error is not None
- assert "`docs_answer`" in error
- assert "turn_intent_context_read_blocked" in error
+ signal = _turn_intent_tool_error(_ctx(intent), "get_run_results")
+ _assert_signal(
+ signal,
+ internal_reason_code="turn_intent_context_read_blocked",
+ classifier_mode="docs_answer",
+ blocked_tool="get_run_results",
+ )
def test_docs_answer_blocks_get_run_results_even_with_read_flag() -> None:
@@ -302,11 +384,13 @@ def test_docs_answer_blocks_get_run_results_even_with_read_flag() -> None:
authority=TurnIntentAuthority(may_read_run_context=True),
)
- error = _turn_intent_tool_error(_ctx(intent), "get_run_results")
-
- assert error is not None
- assert "`docs_answer`" in error
- assert "turn_intent_context_read_blocked" in error
+ signal = _turn_intent_tool_error(_ctx(intent), "get_run_results")
+ _assert_signal(
+ signal,
+ internal_reason_code="turn_intent_context_read_blocked",
+ classifier_mode="docs_answer",
+ blocked_tool="get_run_results",
+ )
assert _native_tools_for_turn(list(NATIVE_TOOLS), intent) == []
@@ -328,10 +412,13 @@ def test_within_turn_override_excluded_for_docs_answer() -> None:
)
ctx = _ctx(intent, pending_reconciliation_run_id="wr_pending_test")
- error = _turn_intent_tool_error(ctx, "get_run_results")
-
- assert error is not None
- assert "turn_intent_context_read_blocked" in error
+ signal = _turn_intent_tool_error(ctx, "get_run_results")
+ _assert_signal(
+ signal,
+ internal_reason_code="turn_intent_context_read_blocked",
+ classifier_mode="docs_answer",
+ blocked_tool="get_run_results",
+ )
def test_tool_activity_is_not_a_substitute_for_pending_reconciliation_run_id() -> None:
@@ -416,3 +503,95 @@ async def test_get_run_results_rejects_different_run_while_reconciliation_pendin
"error": "Run inspection is pending for wr_pending; call get_run_results with that workflow_run_id first.",
}
mock_app.DATABASE.workflow_runs.get_workflow_run.assert_not_called()
+
+
+ ("policy", "expected_reason_code", "expected_recovery_hint", "expects_cleared_by_update"),
+ [
+ (
+ RequestPolicy(allow_update_workflow=True, allow_run_blocks=False, testing_intent="skip_test"),
+ "request_policy_blocks_run_blocks_skip_test",
+ "retry_with_different_tool",
+ True,
+ ),
+ (
+ RequestPolicy(
+ allow_update_workflow=True,
+ allow_run_blocks=False,
+ clarification_reason="workflow_credential_inputs_unbound",
+ ),
+ "request_policy_blocks_run_blocks_credential_unbound",
+ "report_blocker_to_user",
+ False,
+ ),
+ (
+ RequestPolicy(allow_update_workflow=True, allow_run_blocks=False),
+ "request_policy_blocks_run_blocks_generic",
+ "ask_user_clarifying",
+ False,
+ ),
+ ],
+)
+def test_request_policy_run_block_branches(
+ policy: RequestPolicy,
+ expected_reason_code: str,
+ expected_recovery_hint: str,
+ expects_cleared_by_update: bool,
+) -> None:
+ intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ )
+ ctx = _ctx(intent, policy)
+ payload = _authority_tool_error(ctx, "update_and_run_blocks")
+ assert payload is not None
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == expected_reason_code
+ assert ctx.blocker_signal.recovery_hint == expected_recovery_hint
+ if expects_cleared_by_update:
+ assert "update_workflow" in ctx.blocker_signal.cleared_by_tools
+ else:
+ assert ctx.blocker_signal.cleared_by_tools == frozenset()
+ for token in _LEAK_TOKENS:
+ assert token not in payload
+
+
+def test_cleared_by_tools_implies_retry_recovery_hint_convention() -> None:
+ """Convention: a signal is sticky unless cleared_by_tools is non-empty,
+ in which case the recovery_hint must be retry_with_different_tool."""
+ intents = [
+ TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ ),
+ TurnIntent(
+ mode=TurnIntentMode.DIAGNOSE,
+ authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ ),
+ ]
+ policies = [
+ RequestPolicy(allow_update_workflow=True, allow_run_blocks=False, testing_intent="skip_test"),
+ RequestPolicy(
+ allow_update_workflow=True,
+ allow_run_blocks=False,
+ clarification_reason="workflow_credential_inputs_unbound",
+ ),
+ RequestPolicy(allow_update_workflow=False, allow_run_blocks=False),
+ ]
+ tool_names = ("update_workflow", "update_and_run_blocks", "run_blocks_and_collect_debug", "get_run_results")
+ seen_any = False
+ for intent in intents:
+ for policy in policies:
+ for tool in tool_names:
+ ctx = _ctx(intent, policy)
+ _authority_tool_error(ctx, tool)
+ signal = ctx.blocker_signal
+ if signal is None:
+ continue
+ seen_any = True
+ if signal.cleared_by_tools:
+ assert signal.recovery_hint == "retry_with_different_tool", (
+ f"signal with non-empty cleared_by_tools must use retry_with_different_tool, "
+ f"got {signal.recovery_hint!r} for {signal.internal_reason_code}"
+ )
+ assert seen_any, "expected at least one signal in the convention sweep"