Skip to content

Automate browser based workflows with AI

fix(SKY-10933): enforce code-only block policy (#6536)

Andrew Neilsonb3a2affbafdc
diff --git a/skyvern/client/types/artifact_type.py b/skyvern/client/types/artifact_type.py
index 5e3106d98..c9e599864 100644
--- a/skyvern/client/types/artifact_type.py
+++ b/skyvern/client/types/artifact_type.py
@@ -5,7 +5,6 @@ import typing
ArtifactType = typing.Union[
typing.Literal[
"recording",
- "session_replay",
"browser_console_log",
"skyvern_log",
"skyvern_log_raw",
diff --git a/skyvern/client/types/for_loop_block_loop_blocks_item.py b/skyvern/client/types/for_loop_block_loop_blocks_item.py
index af4e94f33..6cb76c3fa 100644
--- a/skyvern/client/types/for_loop_block_loop_blocks_item.py
+++ b/skyvern/client/types/for_loop_block_loop_blocks_item.py
@@ -959,8 +959,6 @@ else:
ForLoopBlockLoopBlocksItem_WorkflowTrigger,
):
update_forward_refs(_cls)
- # Back-resolve everything that couldn't resolve mid-chain:
- # 1. while_loop_block_loop_blocks_item variants (need our ForLoopBlockLoopBlocksItem)
from . import while_loop_block_loop_blocks_item as _wllb
_wllb.ForLoopBlockLoopBlocksItem = ForLoopBlockLoopBlocksItem # type: ignore[attr-defined]
for _cls in (
@@ -985,12 +983,10 @@ else:
_wllb.WhileLoopBlockLoopBlocksItem_WorkflowTrigger,
):
update_forward_refs(_cls)
- # 2. for_loop_block.ForLoopBlock (needs our ForLoopBlockLoopBlocksItem in its globals)
from . import for_loop_block as _flb
_flb.ForLoopBlockLoopBlocksItem = ForLoopBlockLoopBlocksItem # type: ignore[attr-defined]
_flb.WhileLoopBlockLoopBlocksItem = WhileLoopBlockLoopBlocksItem # type: ignore[attr-defined]
update_forward_refs(_flb.ForLoopBlock)
- # 3. while_loop_block.WhileLoopBlock
from . import while_loop_block as _wlb
_wlb.ForLoopBlockLoopBlocksItem = ForLoopBlockLoopBlocksItem # type: ignore[attr-defined]
_wlb.WhileLoopBlockLoopBlocksItem = WhileLoopBlockLoopBlocksItem # type: ignore[attr-defined]
diff --git a/skyvern/client/types/while_loop_block_loop_blocks_item.py b/skyvern/client/types/while_loop_block_loop_blocks_item.py
index 59d2a0672..149915771 100644
--- a/skyvern/client/types/while_loop_block_loop_blocks_item.py
+++ b/skyvern/client/types/while_loop_block_loop_blocks_item.py
@@ -957,7 +957,6 @@ else:
WhileLoopBlockLoopBlocksItem_WorkflowTrigger,
):
update_forward_refs(_cls)
- # Back-resolve everything that couldn't resolve mid-chain.
from . import for_loop_block_loop_blocks_item as _fllb
_fllb.WhileLoopBlockLoopBlocksItem = WhileLoopBlockLoopBlocksItem # type: ignore[attr-defined]
for _cls in (
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 4da9cd476..38d23a5f1 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -99,6 +99,7 @@ from skyvern.forge.sdk.copilot.streaming_adapter import (
)
from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabled
from skyvern.forge.sdk.copilot.turn_context import TurnContextAssembler, TurnContextInputs, TurnContextPacket
+from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHalt, turn_halt_to_trace_data
from skyvern.forge.sdk.copilot.turn_intent import (
NO_MUTATION_TURN_INTENT_MODES,
RequiredContextKey,
@@ -132,49 +133,18 @@ WORKFLOW_KNOWLEDGE_BASE_PATH = (
_COPILOT_TURN_SPAN_NAME = "copilot.turn"
_USER_MESSAGE_PREVIEW_MAX_CHARS = 40
-_CODE_ONLY_BROWSER_AUTHORING_PROMPT = """
-ACTIVE BLOCK AUTHORING POLICY: CODE-ONLY BROWSER MODE
-
-Browser/page workflow block types are unavailable in this mode: `navigation`,
-`action`, `login`, `extraction`, `goto_url`, `validation`, `file_download`,
-`file_upload`, `print_page`, `task`, `task_v2`, and `browser_task`.
-
-If a durable step touches the browser page or browser session, author it as a
-focused `code` block. Allowed non-browser helper blocks remain available when
-they do not directly interact with the browser page, such as `conditional`,
-`for_loop`, `while_loop`, `send_email`, S3/Google Sheets helpers, and workflow
-triggers.
-
-Do not call `validate_block`, do not create dummy/probe code blocks, and do not
-call `get_run_results` before a real workflow run exists.
-
-Code block runtime facts:
-- `code` is async Python executed with a Playwright `page` object and workflow
- parameters available by key.
-- Workflow parameter keys that are valid Python identifiers are available as
- local variables. Normalize them with locals such as
- `name = str(person_name).strip()` before using them in page inputs.
-- Use deterministic, bounded Playwright calls such as `await page.goto(...)`,
- `await page.click(...)`, `await page.fill(...)`, `await page.press(...)`,
- `await page.wait_for_load_state(...)`, and `await page.evaluate(...)`.
-- A workflow parameter bound to a saved credential (`workflow_parameter_type:
- credential_id`) resolves at runtime to a credential object: read
- `<key>.username`, `<key>.password`, and `<key>.totp` (a fresh one-time code
- generated at block start). Scout saved-credential fields with the
- `fill_credential_field` tool; never type or embed literal secret values.
-- For extraction blocks, return precise JSON-safe structured data and the
- visible evidence text used to confirm it. Do not return only booleans for
- visible records, products, totals, confirmations, or identifiers.
-- For multi-line code in workflow YAML, always use a YAML block scalar
- (`code: |`). Never include placeholder text such as `[... truncated ...]`;
- pass complete workflow YAML to update tools.
-
-When a SYNTHESIZED CODE BLOCK is offered to you, it already encodes the page
-interactions you scouted as deterministic Playwright. Persist that block VERBATIM
-via update_workflow / update_and_run_blocks — do not rewrite, reorder, or
-re-derive its locators. Only hand-author the steps it does not cover, such as the
-extraction or report block that returns the structured result.
-"""
+
+def _render_code_only_browser_authoring_prompt() -> str:
+ from skyvern.forge.sdk.copilot.tools.banned_blocks import _code_only_browser_authoring_prompt
+
+ return (
+ _code_only_browser_authoring_prompt()
+ + "\n\nWhen a SYNTHESIZED CODE BLOCK is offered to you, it already encodes the page\n"
+ "interactions you scouted as deterministic Playwright. Persist that block VERBATIM\n"
+ "via update_workflow / update_and_run_blocks — do not rewrite, reorder, or\n"
+ "re-derive its locators. Only hand-author the steps it does not cover, such as the\n"
+ "extraction or report block that returns the structured result."
+ )
@runtime_checkable
@@ -469,7 +439,7 @@ def _build_system_prompt(
security_rules=copilot_config.security_rules,
)
if copilot_config.block_authoring_policy == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- prompt = f"{prompt}\n\n{_CODE_ONLY_BROWSER_AUTHORING_PROMPT.strip()}"
+ prompt = f"{prompt}\n\n{_render_code_only_browser_authoring_prompt()}"
return prompt
@@ -907,6 +877,8 @@ def _make_agent_result(
else global_llm_context
)
narrative_payload = kwargs.get("narrative_payload")
+ if ctx is not None and narrative_payload is None:
+ raise ValueError("_make_agent_result requires narrative_payload when ctx is provided")
response_type = kwargs.get("response_type", "REPLY")
proposal_disposition = kwargs.get("proposal_disposition")
if isinstance(narrative_payload, dict):
@@ -1165,11 +1137,36 @@ def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_context: s
turn_outcome=outcome,
turn_id=ctx.turn_id,
narrative_summary=ctx.narrative_summary,
+ narrative_payload=_build_narrative_payload(
+ ctx,
+ terminal="response",
+ terminal_message=final_text,
+ narrative_summary=ctx.narrative_summary,
+ ),
),
exit_site="verified_goal_satisfied",
)
+def _build_turn_halt_exit_result(
+ ctx: CopilotContext,
+ global_llm_context: str | None,
+ halt: TurnHalt,
+) -> AgentResult:
+ signal = halt.blocker_signal
+ user_response = (
+ signal.user_facing_reason
+ if isinstance(signal, CopilotToolBlockerSignal)
+ else "I could not continue this turn safely. Tell me what to change and I'll try again."
+ )
+ return _build_exit_result(
+ ctx,
+ user_response,
+ global_llm_context,
+ terminal_reason=f"turn_halt:{halt.kind.value}",
+ )
+
+
_TIMEOUT_REPLY_DEFAULT = "I ran out of time processing your request. Here's what I have so far."
_TIMEOUT_REPLY_UNVALIDATED = (
"I ran out of time before I could finish testing. I have a draft workflow you can keep — "
@@ -1955,16 +1952,21 @@ async def _translate_to_agent_result(
composition_page_evidence_error,
workflow_target_url,
)
+ from skyvern.forge.sdk.copilot.tools.banned_blocks import _copilot_banned_block_types
wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
if wait_block_error:
user_response = _with_inline_reject_note(user_response, wait_block_error)
ctx.last_test_ok = None
workflow_yaml = ""
- banned_items = _detect_new_banned_blocks(workflow_yaml, ctx.last_workflow_yaml)
+ banned_items = _detect_new_banned_blocks(
+ workflow_yaml,
+ ctx.last_workflow_yaml,
+ banned_types=_copilot_banned_block_types(ctx),
+ )
if banned_items:
_record_banned_block_reject_span("replace_workflow_inline", banned_items)
- user_response = _with_inline_reject_note(user_response, _banned_block_reject_message(banned_items))
+ user_response = _with_inline_reject_note(user_response, _banned_block_reject_message(banned_items, ctx))
workflow_yaml = ""
stale_metadata = _detect_stale_block_metadata(workflow_yaml, ctx.last_workflow_yaml or ctx.workflow_yaml)
if stale_metadata:
@@ -3248,6 +3250,13 @@ async def _run_copilot_turn_impl(
workflow_run_id=ctx.last_successful_run_blocks_workflow_run_id,
)
return _build_goal_satisfied_exit_result(ctx, global_llm_context)
+ except CopilotTurnHalt as exc:
+ LOG.info(
+ "Copilot run stopped after typed turn halt",
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ **turn_halt_to_trace_data(exc.halt),
+ )
+ return _build_turn_halt_exit_result(ctx, global_llm_context, exc.halt)
except MaxTurnsExceeded:
return _build_max_turns_exit_result(ctx, global_llm_context)
except CopilotTotalTimeoutError:
@@ -3315,6 +3324,15 @@ async def _run_copilot_turn_impl(
except Exception:
LOG.error("Copilot agent error", error=str(e), exc_info=True)
return _build_unexpected_error_exit_result(ctx, global_llm_context, error=e)
+ turn_halt = getattr(ctx, "turn_halt", None)
+ if isinstance(turn_halt, TurnHalt):
+ LOG.info(
+ "Copilot run stopped after typed turn halt from wrapped exception",
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ error_type=type(e).__name__,
+ **turn_halt_to_trace_data(turn_halt),
+ )
+ return _build_turn_halt_exit_result(ctx, global_llm_context, turn_halt)
if goal_satisfied:
# The Agents SDK can wrap exceptions raised from hooks; keep this
# fallback so a verified-goal stop is not rendered as a generic error.
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 12a50b2da..6f15d34fa 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -83,6 +83,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.narration import NarratorState
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.turn_context import TurnContextPacket
+ from skyvern.forge.sdk.copilot.turn_halt import TurnHalt
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent
from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
@@ -356,6 +357,7 @@ class CopilotContext(AgentContext):
tool_activity: list[dict[str, Any]] = field(default_factory=list)
latest_tool_blocker_signal: CopilotToolBlockerSignal | None = None
tool_blocker_signals: list[CopilotToolBlockerSignal] = field(default_factory=list)
+ turn_halt: TurnHalt | None = None
# ``None`` until usage is observed; ``0`` only when a provider explicitly
# reported zero. Distinct values let cost grading flag missing telemetry.
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 02693fad4..9dc4cdcc9 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -15,6 +15,7 @@ from agents.run import Runner
from skyvern.config import settings
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASES
from skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_block
from skyvern.forge.sdk.copilot.config import (
@@ -55,6 +56,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
)
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+from skyvern.forge.sdk.copilot.turn_halt import raise_if_turn_halt, stash_turn_halt_from_blocker_signal
from skyvern.utils.token_counter import count_tokens
if TYPE_CHECKING:
@@ -195,6 +197,13 @@ def _probable_site_block_stop_nudge(ctx: Any, config: CopilotConfig | None = Non
return _nudge(config, "post_probable_site_block_stop_prefix") + _probable_site_block_proxy_options(ctx)
+def _probable_site_block_stop_agent_text(ctx: Any, config: CopilotConfig | None = None) -> str:
+ return (
+ f"{_probable_site_block_stop_nudge(ctx, config)}\n"
+ f"Latest internal failure reason: {_single_line_failure_reason(ctx)}"
+ )
+
+
def _single_line_failure_reason(ctx: Any) -> str:
reason = getattr(ctx, "last_test_failure_reason", None)
if not isinstance(reason, str) or not reason.strip():
@@ -212,11 +221,31 @@ def build_probable_site_block_user_question(ctx: Any) -> str | None:
return (
"The site could not be loaded after repeated attempts. "
f'The latest failure_reason was: "{failure_reason}". '
- "Repeating the same IP/workflow shape is unlikely to help, so I should stop retrying that path.\n\n"
+ "Repeating the same IP/workflow shape is unlikely to help, so I should stop this path.\n\n"
f"Would you like me to {options}"
)
+def _probable_site_block_stop_signal(ctx: Any, config: CopilotConfig | None = None) -> CopilotToolBlockerSignal:
+ user_facing = build_probable_site_block_user_question(ctx)
+ if user_facing is None:
+ user_facing = (
+ "The site could not be loaded after repeated attempts. Tell me whether to try a different URL, "
+ "configure a proxy, or use an alternate entry point."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=_probable_site_block_stop_agent_text(ctx, config),
+ user_facing_reason=user_facing,
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=True,
+ renders_final_reply=True,
+ internal_reason_code="probable_site_block_stop",
+ blocked_tool="update_and_run_blocks",
+ )
+
+
class CopilotTotalTimeoutError(Exception):
"""Raised when the copilot agent exceeds the total allowed runtime."""
@@ -914,6 +943,10 @@ def _check_enforcement(
config: CopilotConfig | None = None,
) -> str | None:
# Terminal failure-mode signals must pre-empt tool-call hygiene nudges.
+ terminal_signal = getattr(ctx, "latest_tool_blocker_signal", None) or getattr(ctx, "blocker_signal", None)
+ if terminal_signal is not None:
+ stash_turn_halt_from_blocker_signal(ctx, terminal_signal, source="enforcement_backstop")
+ raise_if_turn_halt(ctx)
_raise_if_unrecoverable_contract_stop(ctx)
# A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be
@@ -988,7 +1021,10 @@ def _check_enforcement(
# failed_test_nudge_count slot.
if _needs_probable_site_block_stop_nudge(ctx):
ctx.probable_site_block_stop_nudge_count = getattr(ctx, "probable_site_block_stop_nudge_count", 0) + 1
- return _probable_site_block_stop_nudge(ctx, config)
+ signal = _probable_site_block_stop_signal(ctx, config)
+ stash_blocker_signal(ctx, signal)
+ stash_turn_halt_from_blocker_signal(ctx, signal, source="enforcement")
+ raise_if_turn_halt(ctx)
if _needs_failed_test_nudge(ctx):
ctx.failed_test_nudge_count += 1
diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.py
index fcc6d90f1..3637c5274 100644
--- a/skyvern/forge/sdk/copilot/hooks.py
+++ b/skyvern/forge/sdk/copilot/hooks.py
@@ -20,6 +20,10 @@ from skyvern.forge.sdk.copilot.enforcement import (
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_gate_decision
from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result
from skyvern.forge.sdk.copilot.streaming_adapter import parse_tool_output
+from skyvern.forge.sdk.copilot.turn_halt import (
+ raise_if_turn_halt,
+ stash_turn_halt_from_blocker_signal,
+)
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.context import CopilotContext
@@ -124,6 +128,13 @@ class CopilotRunHooks(RunHooksBase):
)
return
+ stash_turn_halt_from_blocker_signal(
+ self._ctx,
+ getattr(self._ctx, "latest_tool_blocker_signal", None) or getattr(self._ctx, "blocker_signal", None),
+ source="hook",
+ )
+ raise_if_turn_halt(self._ctx)
+
if _tool_completion_satisfies_turn(self._ctx, tool_name, parsed):
LOG.info(
"copilot tool satisfied goal; stopping agent loop",
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 7b1cc3373..145d396e4 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import json
from contextlib import AsyncExitStack
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Awaitable, Callable
+from typing import Any, Awaitable, Callable
import structlog
from agents.agent import AgentBase
@@ -27,7 +27,6 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
from skyvern.forge.sdk.copilot.loop_detection import (
detect_failed_tool_step_loop_for_ctx,
detect_tool_loop,
@@ -42,9 +41,7 @@ from skyvern.forge.sdk.copilot.runtime import (
)
from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure
-
-if TYPE_CHECKING:
- from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal
PreHook = Callable[[dict[str, Any], AgentContext], Awaitable[dict[str, Any] | None]]
PostHook = Callable[[dict[str, Any], dict[str, Any], AgentContext], Awaitable[dict[str, Any]]]
@@ -69,16 +66,11 @@ LOG = structlog.get_logger()
_INTERNAL_TOOL_ARG_KEYS = frozenset({"_summarized"})
-def _hide_code_only_tool(ctx: CopilotContext, tool_name: str) -> bool:
- return (
- normalize_block_authoring_policy(ctx.block_authoring_policy) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
- and tool_name == "validate_block"
- )
-
-
def _stash_and_emit_loop_blocker(ctx: Any, loop_message: str, tool_name: str) -> str:
signal = build_loop_blocker_signal(loop_message, tool_name=tool_name, evidence=loop_blocker_evidence_from_ctx(ctx))
- return stash_blocker_signal(ctx, signal)
+ payload = stash_blocker_signal(ctx, signal)
+ stash_turn_halt_from_blocker_signal(ctx, signal, source="mcp_loop_blocker")
+ return payload
def _apply_schema_overlay(
@@ -188,15 +180,12 @@ class SkyvernOverlayMCPServer(MCPServer):
self._cached_raw_tools = await self._client.list_tools()
raw_tools = self._cached_raw_tools
result: list[MCPTool] = []
- copilot_ctx = self._context_provider()
for tool in raw_tools:
if tool.name not in self._allowlist:
continue
copilot_name = self._reverse_alias.get(tool.name, tool.name)
- if _hide_code_only_tool(copilot_ctx, copilot_name):
- continue
overlay = self._overlays.get(copilot_name, SchemaOverlay())
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 424b8aa14..47ed80ced 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+ from skyvern.forge.sdk.copilot.turn_halt import TurnHalt
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
@@ -267,6 +268,7 @@ class AgentContext:
# turn end and overrides the AgentResult with a deterministic
# product-language reply. See blocker_signal.py for the contract.
blocker_signal: CopilotToolBlockerSignal | None = None
+ turn_halt: TurnHalt | None = None
# Most recently emitted blocker signal for the current tool output. Unlike
# blocker_signal, this is last-wins so the activity-log projection can
# render the current tool result from structured product text.
diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.py
index ffa10d974..dd0a3a4c5 100644
--- a/skyvern/forge/sdk/copilot/streaming_adapter.py
+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py
@@ -87,6 +87,11 @@ async def stream_to_sse(
CopilotUnrecoverableToolError,
_maybe_raise_unrecoverable_tool_error,
)
+ from skyvern.forge.sdk.copilot.turn_halt import (
+ CopilotTurnHalt,
+ raise_if_turn_halt,
+ stash_turn_halt_from_blocker_signal,
+ )
call_id_to_name: dict[str, str] = {}
# Counts completed tool round-trips (tool_called + tool_output pair), not
@@ -226,6 +231,16 @@ async def stream_to_sse(
except CopilotUnrecoverableToolError:
result.cancel()
raise
+ stash_turn_halt_from_blocker_signal(
+ ctx,
+ getattr(ctx, "latest_tool_blocker_signal", None) or getattr(ctx, "blocker_signal", None),
+ source="streaming_adapter",
+ )
+ try:
+ raise_if_turn_halt(ctx)
+ except CopilotTurnHalt:
+ result.cancel()
+ raise
# Keep latest_tool_blocker_signal scoped to the tool result
# that immediately follows the blocker-producing tool call.
ctx.latest_tool_blocker_signal = None
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index b03aa544c..75144e9a9 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -29,6 +29,7 @@ from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.schemas.workflows import BlockType
@@ -244,7 +245,9 @@ def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:
def _emit_tool_blocker_signal(ctx: AgentContext, signal: CopilotToolBlockerSignal) -> str:
- return stash_blocker_signal(ctx, signal)
+ payload = stash_blocker_signal(ctx, signal)
+ stash_turn_halt_from_blocker_signal(ctx, signal, source="tool_blocker_signal")
+ return payload
def _block_type_name(block: object) -> str:
diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
index 764db194f..467760a8f 100644
--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py
+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
@@ -1,6 +1,8 @@
from __future__ import annotations
import re
+from dataclasses import dataclass
+from enum import StrEnum
from typing import Any
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
@@ -11,27 +13,106 @@ from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from ._shared import _iter_yaml_blocks, _parse_workflow_blocks
-# Block types the copilot must never emit. They delegate the entire goal to
-# a separate agent, which bypasses copilot-level block decomposition and
-# obfuscates issues the copilot should surface/handle directly.
-_COPILOT_BANNED_BLOCK_TYPES: frozenset[str] = frozenset({"task", "task_v2"})
-# Code-only browser mode uses this temporary hard ban to force iteration on
-# code-first Copilot behavior. The GA policy is not settled: it may allow some
-# of these block types again, or make the allow/ban list configurable.
-_COPILOT_CODE_ONLY_BROWSER_BANNED_BLOCK_TYPES: frozenset[str] = _COPILOT_BANNED_BLOCK_TYPES | frozenset(
- {
- "action",
- "browser_task",
- "extraction",
- "file_download",
- "file_upload",
- "goto_url",
- "login",
- "navigation",
- "print_page",
- "validation",
- }
+
+class CopilotBlockPolicyStatus(StrEnum):
+ BANNED = "banned"
+ CODE_NATIVE_PENDING = "code_native_pending"
+ DECLARED_AI_LEAF = "declared_ai_leaf"
+
+
+class CopilotBlockPolicyScope(StrEnum):
+ ALL = "all"
+ CODE_ONLY_BROWSER = "code_only_browser"
+
+
+@dataclass(frozen=True)
+class CopilotBlockPolicy:
+ status: CopilotBlockPolicyStatus
+ scope: CopilotBlockPolicyScope
+ required_capability: str
+ guidance: str
+
+
+_P = CopilotBlockPolicy
+_ALL = CopilotBlockPolicyScope.ALL
+_CODE_ONLY = CopilotBlockPolicyScope.CODE_ONLY_BROWSER
+_BANNED = CopilotBlockPolicyStatus.BANNED
+_PENDING = CopilotBlockPolicyStatus.CODE_NATIVE_PENDING
+_AI_LEAF = CopilotBlockPolicyStatus.DECLARED_AI_LEAF
+
+_CODE_ONLY_FOCUSED_CODE_BLOCK_TYPES = (
+ "action",
+ "browser_task",
+ "extraction",
+ "goto_url",
+ "navigation",
+ "print_page",
+ "validation",
+)
+
+_COPILOT_BLOCK_TYPE_POLICIES: dict[str, CopilotBlockPolicy] = {
+ "task": _P(
+ _AI_LEAF,
+ _ALL,
+ "declared AI leaf support",
+ (
+ "The legacy task agent is not available in the workflow copilot; decompose the goal into explicit "
+ "workflow blocks or focused code blocks instead."
+ ),
+ ),
+ "task_v2": _P(
+ _AI_LEAF,
+ _ALL,
+ "declared AI leaf support",
+ (
+ "The legacy task_v2 agent is not available in the workflow copilot; decompose the goal into explicit "
+ "workflow blocks or focused code blocks instead."
+ ),
+ ),
+ **{
+ block_type: _P(
+ _BANNED,
+ _CODE_ONLY,
+ "focused `code` blocks for durable browser/page work",
+ "Use focused `code` blocks with concrete selectors, text anchors, outputs, and postconditions.",
+ )
+ for block_type in _CODE_ONLY_FOCUSED_CODE_BLOCK_TYPES
+ },
+ "login": _P(
+ _PENDING,
+ _CODE_ONLY,
+ "credential-typed code synthesis with runtime credential resolution",
+ (
+ "Use credential-typed code: scout saved-credential fields with fill_credential_field, bind the "
+ "credential as a credential_id workflow parameter, and read the resolved credential object in code."
+ ),
+ ),
+ "file_download": _P(
+ _PENDING,
+ _CODE_ONLY,
+ "code-block download registration and output chaining",
+ (
+ "Download chains require code-block download registration before downstream file_url_parser or "
+ "http_request file references can consume the output."
+ ),
+ ),
+ "file_upload": _P(
+ _PENDING,
+ _CODE_ONLY,
+ "same-run file path threading or workflow file materialization",
+ (
+ "Use code-native upload only when a local same-run path exists; workflow file parameters still need "
+ "file materialization before this rung is complete."
+ ),
+ ),
+}
+
+_COPILOT_BANNED_BLOCK_TYPES: frozenset[str] = frozenset(
+ block_type
+ for block_type, policy in _COPILOT_BLOCK_TYPE_POLICIES.items()
+ if policy.scope == CopilotBlockPolicyScope.ALL
)
+_COPILOT_CODE_ONLY_BROWSER_BANNED_BLOCK_TYPES: frozenset[str] = frozenset(_COPILOT_BLOCK_TYPE_POLICIES.keys())
# Shared suffix across every LLM-facing rejection message for banned
# block emission — the pre-hook (schema-lookup reject) and the post-
@@ -41,10 +122,6 @@ _COPILOT_BANNED_BLOCK_ALTERNATIVES = (
"`extraction` for data extraction, `validation` for completion checks, "
"`login` for authentication, or `goto_url` for pure URL navigation."
)
-_COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES = (
- "Browser/page workflow block types are unavailable in code-only browser mode. Use focused `code` "
- "blocks for durable page or browser-session work."
-)
_CODE_ONLY_TARGET_EVIDENCE_KEYS = frozenset(
{
"buttons",
@@ -77,21 +154,132 @@ def _copilot_banned_block_types(ctx: AgentContext | None) -> frozenset[str]:
return _COPILOT_BANNED_BLOCK_TYPES
+def _active_policy_scopes(ctx: AgentContext | None) -> frozenset[CopilotBlockPolicyScope]:
+ scopes = {CopilotBlockPolicyScope.ALL}
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ scopes.add(CopilotBlockPolicyScope.CODE_ONLY_BROWSER)
+ return frozenset(scopes)
+
+
+def _copilot_block_policy(
+ block_type: str,
+ ctx: AgentContext | None,
+) -> tuple[str, CopilotBlockPolicy] | None:
+ normalized = normalize_copilot_block_type_alias(block_type.strip().lower())
+ policy = _COPILOT_BLOCK_TYPE_POLICIES.get(normalized)
+ if policy is not None and policy.scope in _active_policy_scopes(ctx):
+ return normalized, policy
+ return None
+
+
+def _render_block_policy_detail(block_type: str, policy: CopilotBlockPolicy) -> str:
+ return f"`{block_type}` is {policy.status.value} and requires {policy.required_capability}. {policy.guidance}"
+
+
+def _code_only_browser_unavailable_types() -> list[str]:
+ return sorted(
+ block_type
+ for block_type, policy in _COPILOT_BLOCK_TYPE_POLICIES.items()
+ if policy.scope == CopilotBlockPolicyScope.CODE_ONLY_BROWSER
+ )
+
+
+def _code_only_browser_pending_details() -> list[str]:
+ return [
+ _render_block_policy_detail(block_type, policy)
+ for block_type, policy in sorted(_COPILOT_BLOCK_TYPE_POLICIES.items())
+ if policy.scope == CopilotBlockPolicyScope.CODE_ONLY_BROWSER
+ and policy.status == CopilotBlockPolicyStatus.CODE_NATIVE_PENDING
+ ]
+
+
+def _code_only_browser_unavailable_summary() -> str:
+ unavailable = ", ".join(f"`{block_type}`" for block_type in _code_only_browser_unavailable_types())
+ return (
+ f"Browser/page workflow block types are unavailable in code-only browser mode: {unavailable}. "
+ "Use focused `code` blocks for durable page or browser-session work."
+ )
+
+
+def _code_only_browser_validation_guidance() -> str:
+ return (
+ "Use validate_block only for allowed non-browser helper blocks. Do not use validate_block for `code` "
+ "blocks, dummy/probe code blocks, or browser/page native block types; validate real code blocks through "
+ "update_and_run_blocks."
+ )
+
+
+def _code_only_browser_schema_guidance() -> list[str]:
+ return [
+ "Use one focused code block per durable browser goal, such as open, search, submit, expand, or extract.",
+ _code_only_browser_unavailable_summary(),
+ "Use concrete selectors and text anchors found during exploration. If only intent targeting is available, inspect the page again before mutating.",
+ _code_only_browser_validation_guidance(),
+ "Keep block outputs JSON-safe and include visible evidence text when extracting records, products, totals, confirmations, or identifiers.",
+ "For saved credentials: bind the credential as a workflow parameter with workflow_parameter_type credential_id and the credential ID in default_value. At runtime the parameter key resolves to a credential object — read <key>.username, <key>.password, and <key>.totp (a fresh one-time code generated when the block starts). Never put literal secret values in code; scout credential fields with fill_credential_field.",
+ ]
+
+
+def _code_only_browser_authoring_prompt() -> str:
+ pending = "\n".join(f"- {detail}" for detail in _code_only_browser_pending_details())
+ return f"""
+ACTIVE BLOCK AUTHORING POLICY: CODE-ONLY BROWSER MODE
+
+{_code_only_browser_unavailable_summary()}
+
+Rules:
+- Browser/page/session durable steps must be focused `code` blocks.
+- Allowed non-browser helper blocks remain available: `conditional`, `for_loop`,
+ `while_loop`, `send_email`, S3/Google Sheets helpers, file parsers, and triggers.
+- {_code_only_browser_validation_guidance()}
+- Do not call `get_run_results` before a real workflow run exists.
+
+Code-native capabilities still pending plumbing:
+{pending}
+
+Runtime facts:
+- `code` is async Python with a Playwright `page` object and workflow parameters by key.
+- Valid Python identifier parameter keys are local variables; normalize before page inputs.
+- Use deterministic, bounded Playwright calls: `goto`, `click`, `fill`, `press`,
+ `wait_for_load_state`, and `evaluate`.
+- A `credential_id` workflow parameter resolves to a credential object with
+ `<key>.username`, `<key>.password`, and fresh `<key>.totp`; scout fields with
+ `fill_credential_field`, never embed literal secrets.
+- Return JSON-safe structured data plus visible evidence text for records, totals,
+ confirmations, and identifiers.
+- Use YAML block scalars (`code: |`) and pass complete workflow YAML to update tools.
+""".strip()
+
+
def _copilot_banned_block_alternatives(ctx: AgentContext | None) -> str:
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return _COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES
+ return _code_only_browser_unavailable_summary()
return _COPILOT_BANNED_BLOCK_ALTERNATIVES
def _banned_block_reject_message(items: list[tuple[str, str]], ctx: AgentContext | None = None) -> str:
"""Uniform error text for the post-emission reject, sharing the
alternatives suffix with the schema pre-hook."""
+ grouped: dict[str, list[str]] = {}
+ for label, block_type in items:
+ normalized = normalize_copilot_block_type_alias(block_type.strip().lower())
+ grouped.setdefault(normalized, []).append(label)
labels = ", ".join(sorted({label for label, _ in items}))
- types = sorted({block_type for _, block_type in items})
+ types = sorted(grouped)
types_part = " / ".join(repr(t) for t in types)
+ details = []
+ for block_type in types:
+ policy_entry = _copilot_block_policy(block_type, ctx)
+ if policy_entry is None:
+ continue
+ _normalized, policy = policy_entry
+ type_labels = ", ".join(sorted(grouped[block_type]))
+ details.append(f"{block_type} [{type_labels}]: {_render_block_policy_detail(block_type, policy)}")
+ details_part = " ".join(details)
return (
f"Block type {types_part} is not available in the workflow copilot. "
f"Offending labels: [{labels}]. "
+ f"{details_part} "
f"{_copilot_banned_block_alternatives(ctx)}"
)
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index c0a2e2050..82bc2f931 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import json
import re
from typing import Any
@@ -20,10 +21,13 @@ from ._shared import _DISCOVERY_PER_CALL_TIMEOUT_SECONDS
from .banned_blocks import (
_CODE_ONLY_SELECTOR_ACTION_TOOLS,
_CODE_ONLY_TARGET_EVIDENCE_KEYS,
- _COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES,
+ _code_only_browser_schema_guidance,
+ _code_only_browser_unavailable_summary,
_copilot_banned_block_alternatives,
_copilot_banned_block_types,
_copilot_block_authoring_policy,
+ _copilot_block_policy,
+ _render_block_policy_detail,
)
from .completion import _maybe_run_completion_verification_from_page_observation
from .page_observation import (
@@ -59,11 +63,16 @@ async def _get_block_schema_pre_hook(
normalized = normalize_copilot_block_type_alias(block_type)
if normalized != block_type.strip().lower():
params["block_type"] = normalized
- if normalized not in _copilot_banned_block_types(ctx):
+ policy_entry = _copilot_block_policy(normalized, ctx)
+ if policy_entry is None:
return None
+ normalized, policy = policy_entry
return {
"ok": False,
- "error": f"Block type {block_type!r} is not available in the workflow copilot. {_copilot_banned_block_alternatives(ctx)}",
+ "error": (
+ f"Block type {block_type!r} is not available in the workflow copilot. "
+ f"{_render_block_policy_detail(normalized, policy)} {_copilot_banned_block_alternatives(ctx)}"
+ ),
}
@@ -73,12 +82,37 @@ async def _validate_block_pre_hook(
) -> dict[str, Any] | None:
if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
return None
+ block_json = params.get("block_json")
+ if not isinstance(block_json, str):
+ return None
+ try:
+ raw = json.loads(block_json)
+ except (json.JSONDecodeError, TypeError):
+ return None
+ if not isinstance(raw, dict):
+ return None
+ block_type = raw.get("block_type")
+ if not isinstance(block_type, str):
+ return None
+ normalized = normalize_copilot_block_type_alias(block_type.strip().lower())
+ if normalized == "code":
+ return {
+ "ok": False,
+ "error": (
+ "CODE-ONLY CODE VALIDATION BLOCKED: do not use validate_block for `code` blocks, dummy code "
+ "blocks, or probe code blocks in code-only browser mode. validate real code blocks through "
+ "update_and_run_blocks."
+ ),
+ }
+ policy_entry = _copilot_block_policy(normalized, ctx)
+ if policy_entry is None:
+ return None
+ normalized, policy = policy_entry
return {
"ok": False,
"error": (
- "CODE-ONLY BLOCK VALIDATION DISABLED: do not use validate_block, dummy code blocks, or probe code "
- "blocks in code-only browser mode. Use MCP browser tools to explore the page, then call "
- "update_and_run_blocks with real focused code blocks that implement the workflow behavior."
+ f"Block type {block_type!r} is not available in the workflow copilot. "
+ f"{_render_block_policy_detail(normalized, policy)} {_copilot_banned_block_alternatives(ctx)}"
),
}
@@ -96,18 +130,12 @@ async def _get_block_schema_post_hook(
if isinstance(block_types, dict):
for banned in _copilot_banned_block_types(ctx):
block_types.pop(banned, None)
+ data["count"] = len(block_types)
block_type = data.get("block_type")
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and block_type == "code":
ctx.code_only_code_schema_seen = True
- data["code_only_note"] = _COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES
- data["code_only_guidance"] = [
- "Use one focused code block per durable browser goal, such as open, search, submit, expand, or extract.",
- "Do not persist navigation/action/login/extraction/validation blocks for browser page work.",
- "Use concrete selectors and text anchors found during exploration. If only intent targeting is available, inspect the page again before mutating.",
- "Call update_and_run_blocks with a connected runnable set of real code blocks instead of validating dummy or probe blocks.",
- "Keep block outputs JSON-safe and include visible evidence text when extracting records, products, totals, confirmations, or identifiers.",
- "For saved credentials: bind the credential as a workflow parameter with workflow_parameter_type credential_id and the credential ID in default_value. At runtime the parameter key resolves to a credential object — read <key>.username, <key>.password, and <key>.totp (a fresh one-time code generated when the block starts). Never put literal secret values in code; scout credential fields with fill_credential_field.",
- ]
+ data["code_only_note"] = _code_only_browser_unavailable_summary()
+ data["code_only_guidance"] = _code_only_browser_schema_guidance()
return result
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index d953a9144..753803e4f 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -54,6 +54,7 @@ from skyvern.forge.sdk.copilot.runtime import (
ensure_browser_session,
)
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotRunOutcomeUpdate, WorkflowCopilotStreamMessageType
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
@@ -1479,11 +1480,14 @@ def _record_run_blocks_result(
signal = _active_run_terminal_evidence_signal(copilot_ctx, "update_and_run_blocks")
if signal is not None:
stash_blocker_signal(copilot_ctx, signal)
+ stash_turn_halt_from_blocker_signal(copilot_ctx, signal, source="run_execution")
if run_ok:
_mark_page_inspected(copilot_ctx)
if structured_blocker:
failure_reason = f"Run completed, but extracted data reported a blocker: {structured_blocker}"
+ result["ok"] = False
+ result.setdefault("error", failure_reason)
copilot_ctx.last_test_ok = False
copilot_ctx.last_test_suspicious_success = True
copilot_ctx.last_test_failure_reason = failure_reason
@@ -1500,6 +1504,14 @@ def _record_run_blocks_result(
"reasoning": "Structured extracted data reported an anti-bot blocker.",
}
data["failure_categories"] = [anti_bot_category]
+ if _looks_like_anti_bot_blocker(structured_blocker):
+ signal = _run_output_terminal_blocker_signal(
+ structured_blocker=structured_blocker,
+ failure_reason=failure_reason,
+ tool_name="update_and_run_blocks",
+ )
+ stash_blocker_signal(copilot_ctx, signal)
+ stash_turn_halt_from_blocker_signal(copilot_ctx, signal, source="run_output_content_gate")
update_repeated_failure_state(copilot_ctx, result)
_update_verification_evidence_from_run_result(copilot_ctx, result)
return _stash_recorded_run_outcome(
@@ -1709,6 +1721,36 @@ def _record_diagnosis_repair_contract(
return contract
+def _run_output_terminal_blocker_signal(
+ *,
+ structured_blocker: str,
+ failure_reason: str,
+ tool_name: str,
+) -> CopilotToolBlockerSignal:
+ agent_steering = (
+ "The workflow run completed with structured anti-bot or challenge blocker evidence: "
+ f"{structured_blocker[:240]}. Do NOT call "
+ f"{tool_name} again in this turn. Reply from the recorded blocker and preserved draft; "
+ "do not claim the workflow is verified end-to-end."
+ )
+ user_facing = (
+ "The page is gated by a site verification challenge, so I stopped instead of retrying the same path. "
+ "The draft workflow is preserved, but it is not verified end-to-end."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=True,
+ renders_final_reply=True,
+ internal_reason_code="tool_error_run_output_terminal_blocker",
+ blocked_tool=tool_name,
+ extra={"failure_reason": failure_reason[:240]},
+ )
+
+
def _diagnosis_repair_tool_error(copilot_ctx: Any, source_tool: str, error: str) -> str:
result = {"ok": False, "error": error}
_record_diagnosis_repair_contract(copilot_ctx, source_tool=source_tool, result=result)
diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.py
new file mode 100644
index 000000000..c62f22900
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/turn_halt.py
@@ -0,0 +1,116 @@
+"""Typed terminal halt contract for Copilot turns."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+import structlog
+
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data
+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
+
+LOG = structlog.get_logger()
+
+
+class TurnHaltKind(StrEnum):
+ LOOP_DETECTED = "loop_detected"
+ ACTIVE_TERMINAL_CHALLENGE = "active_terminal_challenge"
+ PROBABLE_SITE_BLOCK = "probable_site_block"
+
+
+class TurnHaltVerdict(StrEnum):
+ BLOCKED = "blocked"
+
+
+_LOOP_TERMINAL_REASON_CODES = frozenset(
+ {
+ "loop_detected_credential_or_parameter_misconfig",
+ "loop_detected_repeated_failed_step",
+ "loop_detected_consecutive_same_tool",
+ "loop_detected_generic",
+ }
+)
+_ACTIVE_TERMINAL_CHALLENGE_REASON_CODES = frozenset(
+ {
+ ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
+ "tool_error_run_output_terminal_blocker",
+ "tool_error_post_budget_challenge_blocker",
+ "tool_error_challenge_gated_submit_disabled",
+ }
+)
+_PROBABLE_SITE_BLOCK_REASON_CODES = frozenset({"probable_site_block_stop"})
+
+
+@dataclass(frozen=True)
+class TurnHalt:
+ kind: TurnHaltKind
+ verdict: TurnHaltVerdict = TurnHaltVerdict.BLOCKED
+ blocker_signal: CopilotToolBlockerSignal | None = None
+ draft_state: dict[str, Any] = field(default_factory=dict)
+ run_refs: dict[str, Any] = field(default_factory=dict)
+ extra: dict[str, Any] = field(default_factory=dict)
+
+
+class CopilotTurnHalt(Exception):
+ def __init__(self, halt: TurnHalt) -> None:
+ self.halt = halt
+ super().__init__(f"Copilot turn halted: {halt.kind.value}")
+
+
+def _kind_for_blocker_signal(signal: CopilotToolBlockerSignal) -> TurnHaltKind | None:
+ reason = signal.internal_reason_code
+ if signal.blocker_kind == "loop_detected" and reason in _LOOP_TERMINAL_REASON_CODES:
+ return TurnHaltKind.LOOP_DETECTED
+ if reason in _ACTIVE_TERMINAL_CHALLENGE_REASON_CODES:
+ return TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
+ if reason in _PROBABLE_SITE_BLOCK_REASON_CODES:
+ return TurnHaltKind.PROBABLE_SITE_BLOCK
+ return None
+
+
+def turn_halt_from_blocker_signal(signal: object, *, source: str) -> TurnHalt | None:
+ if not isinstance(signal, CopilotToolBlockerSignal):
+ return None
+ if not signal.renders_final_reply:
+ return None
+ kind = _kind_for_blocker_signal(signal)
+ if kind is None:
+ return None
+ return TurnHalt(
+ kind=kind,
+ blocker_signal=signal,
+ draft_state={"preserves_workflow_draft": signal.preserves_workflow_draft},
+ extra={"source": source},
+ )
+
+
+def stash_turn_halt_from_blocker_signal(ctx: Any, signal: object, *, source: str) -> TurnHalt | None:
+ existing = getattr(ctx, "turn_halt", None)
+ if isinstance(existing, TurnHalt):
+ return existing
+ halt = turn_halt_from_blocker_signal(signal, source=source)
+ if halt is None:
+ return None
+ ctx.turn_halt = halt
+ LOG.info("copilot turn halt stashed", **turn_halt_to_trace_data(halt))
+ return halt
+
+
+def raise_if_turn_halt(ctx: Any) -> None:
+ halt = getattr(ctx, "turn_halt", None)
+ if isinstance(halt, TurnHalt):
+ raise CopilotTurnHalt(halt)
+
+
+def turn_halt_to_trace_data(halt: TurnHalt) -> dict[str, Any]:
+ data: dict[str, Any] = {
+ "turn_halt_kind": halt.kind.value,
+ "turn_halt_verdict": halt.verdict.value,
+ "turn_halt_extra": dict(halt.extra),
+ }
+ if halt.blocker_signal is not None:
+ data.update(blocker_signal_to_trace_data(halt.blocker_signal))
+ return data
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index b327c42f0..611867e4c 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -14,7 +14,7 @@ import yaml
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
-from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisInput,
DiagnosisRepairContract,
@@ -1189,6 +1189,38 @@ workflow_definition:
assert agent_result.updated_workflow is None
assert agent_result.workflow_yaml is None
+ def test_code_only_inline_replace_workflow_rejects_native_browser_block(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot.output_policy import OutputPolicyVerdict
+
+ process_mock = MagicMock(return_value=SimpleNamespace(name="new"))
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", process_mock)
+ monkeypatch.setattr(agent_module, "evaluate_output_policy", lambda **kwargs: OutputPolicyVerdict())
+
+ submitted_yaml = """
+title: Navigation example
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: open_step
+ navigation_goal: Open the example page.
+"""
+ ctx = _ctx(block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER)
+ result = _fake_run_result(
+ {"type": "REPLACE_WORKFLOW", "user_response": "Here you go.", "workflow_yaml": submitted_yaml}
+ )
+
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+ )
+
+ process_mock.assert_not_called()
+ assert "not available in the workflow copilot" in agent_result.user_response
+ assert "focused `code` blocks" in agent_result.user_response
+ assert agent_result.updated_workflow is None
+ assert agent_result.workflow_yaml is None
+
def test_inline_replace_with_invalid_yaml_keeps_prior_pass(self, monkeypatch) -> None:
# _process_workflow_yaml raising on a malformed REPLACE must leave
# ctx untouched — no spurious last_test_ok reset, no workflow swap —
@@ -1885,6 +1917,19 @@ class TestCredentialRefusalReachesAgent:
assert "MUST NOT build, update, or run a workflow" in prompt
assert "redacted from the outbound client stream" not in prompt
+ def test_code_only_prompt_renders_policy_table_and_helper_validation_guidance(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _build_system_prompt
+
+ config = CopilotConfig(block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER)
+ prompt = _build_system_prompt("", config=config)
+
+ assert "ACTIVE BLOCK AUTHORING POLICY: CODE-ONLY BROWSER MODE" in prompt
+ assert "credential-typed code" in prompt
+ assert "download registration" in prompt
+ assert "Use validate_block only for allowed non-browser helper blocks" in prompt
+ assert "Do not call `validate_block`" not in prompt
+ assert "native_allowed" not in prompt
+
def test_native_tools_carry_refusal_reference(self) -> None:
import re
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 04601f835..29cf7d88c 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -11,7 +11,9 @@ from unittest.mock import AsyncMock, MagicMock, call
import pytest
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
+from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, turn_halt_from_blocker_signal
@dataclass
@@ -36,6 +38,17 @@ def _mcp_text_output(payload: dict[str, Any]) -> list[dict[str, str]]:
return [{"type": "text", "text": json.dumps(payload)}]
+def _terminal_loop_signal() -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="loop_detected",
+ agent_steering_text="LOOP DETECTED: 'update_workflow' has already failed 3 times.",
+ user_facing_reason="I retried without making progress. Tell me what to change and I'll try again.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code="loop_detected_repeated_failed_step",
+ blocked_tool="update_workflow",
+ )
+
+
@pytest.mark.asyncio
async def test_on_tool_end_appends_generic_tool_entry() -> None:
ctx = _FakeContext()
@@ -51,6 +64,20 @@ async def test_on_tool_end_appends_generic_tool_entry() -> None:
assert "output_preview" not in entry # non-whitelisted tool
+async def test_on_tool_end_raises_turn_halt_after_activity_recording() -> None:
+ ctx = _FakeContext()
+ ctx.turn_halt = turn_halt_from_blocker_signal(_terminal_loop_signal(), source="test") # type: ignore[attr-defined]
+ hooks = CopilotRunHooks(ctx)
+
+ output = _mcp_text_output({"ok": False, "error": "terminal blocker"})
+ with pytest.raises(CopilotTurnHalt) as exc_info:
+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("update_workflow"), output)
+
+ assert exc_info.value.halt is ctx.turn_halt
+ assert ctx.tool_activity[0]["tool"] == "update_workflow"
+
+
@pytest.mark.asyncio
async def test_on_tool_end_whitelisted_tool_produces_output_preview() -> None:
ctx = _FakeContext()
diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.py
index 110c62314..40ce0a314 100644
--- a/tests/unit/test_copilot_loop_detection_blocker.py
+++ b/tests/unit/test_copilot_loop_detection_blocker.py
@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer, _stas
from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy
from skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_error
from skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_map
+from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
_LEAK_TOKENS = ("safe_reason_code", "LOOP DETECTED:")
@@ -102,6 +103,9 @@ def test_native_dispatch_failed_step_loop_sets_signal_and_returns_payload() -> N
# Renderer-side string is clean.
for token in _LEAK_TOKENS:
assert token not in ctx.blocker_signal.user_facing_reason
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind == TurnHaltKind.LOOP_DETECTED
+ assert ctx.turn_halt.blocker_signal is ctx.blocker_signal
def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:
@@ -111,6 +115,8 @@ def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:
assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"
assert ctx.blocker_signal.cleared_by_tools == frozenset()
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind == TurnHaltKind.LOOP_DETECTED
def test_native_and_mcp_paths_produce_equivalent_loop_signal() -> None:
diff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.py
index 7681ee269..42350e36b 100644
--- a/tests/unit/test_copilot_make_agent_result_backfill.py
+++ b/tests/unit/test_copilot_make_agent_result_backfill.py
@@ -6,6 +6,8 @@ from __future__ import annotations
from unittest.mock import MagicMock
+import pytest
+
from skyvern.forge.sdk.copilot.agent import _make_agent_result
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
@@ -117,5 +119,10 @@ def test_backfill_tolerates_ctx_none() -> None:
def test_backfill_tolerates_missing_payload() -> None:
- result = _result(_ctx(), turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=None)
+ with pytest.raises(ValueError, match="narrative_payload"):
+ _result(_ctx(), turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=None)
+
+
+def test_missing_payload_is_allowed_without_ctx() -> None:
+ result = _result(None, turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=None)
assert result.narrative_payload is None
diff --git a/tests/unit/test_copilot_probable_site_block.py b/tests/unit/test_copilot_probable_site_block.py
index 5ae497b00..a3e43e555 100644
--- a/tests/unit/test_copilot_probable_site_block.py
+++ b/tests/unit/test_copilot_probable_site_block.py
@@ -26,6 +26,7 @@ from skyvern.forge.sdk.copilot.tools import (
_timing_only_challenge_wait_reject_message,
_update_workflow,
)
+from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHaltKind
_SCRAPE_WALL_REASON = (
"Skyvern failed to load the website. The page may have navigated "
@@ -352,14 +353,18 @@ def test_stop_nudge_uses_different_proxy_advice_when_effective_proxy_is_active()
ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT
ctx.effective_workflow_proxy_location = "RESIDENTIAL"
- nudge = _check_enforcement(ctx)
+ with pytest.raises(CopilotTurnHalt) as exc_info:
+ _check_enforcement(ctx)
- assert nudge is not None
- assert "configure a proxy" not in nudge.lower()
- assert "different proxy location" in nudge.lower()
- assert "US-CA" in nudge
- assert "US-NY" in nudge
- assert "residential/ISP" in nudge
+ halt = exc_info.value.halt
+ assert halt.kind == TurnHaltKind.PROBABLE_SITE_BLOCK
+ assert halt.blocker_signal is ctx.blocker_signal
+ user_facing = halt.blocker_signal.user_facing_reason
+ assert "configure a proxy" not in user_facing.lower()
+ assert "different proxy location" in user_facing.lower()
+ assert "US-CA" in user_facing
+ assert "US-NY" in user_facing
+ assert "residential/ISP" in user_facing
def test_stop_nudge_keeps_configure_proxy_advice_when_proxy_is_none() -> None:
@@ -367,10 +372,11 @@ def test_stop_nudge_keeps_configure_proxy_advice_when_proxy_is_none() -> None:
ctx.probable_site_block_streak_count = PROBABLE_SITE_BLOCK_STREAK_STOP_AT
ctx.effective_workflow_proxy_location = "NONE"
- nudge = _check_enforcement(ctx)
+ with pytest.raises(CopilotTurnHalt) as exc_info:
+ _check_enforcement(ctx)
- assert nudge is not None
- assert "configure a proxy" in nudge.lower()
+ assert exc_info.value.halt.kind == TurnHaltKind.PROBABLE_SITE_BLOCK
+ assert "configure a proxy" in exc_info.value.halt.blocker_signal.user_facing_reason.lower()
def test_detects_challenge_named_wait_block() -> None:
diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.py
index 80be36fbf..58d37092c 100644
--- a/tests/unit/test_copilot_run_output_content_gate.py
+++ b/tests/unit/test_copilot_run_output_content_gate.py
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.copilot.tools import (
_record_run_blocks_result,
_run_blocks_structured_blocker_message,
)
+from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
def _code_block(label: str, extracted: Any, *, block_type: str = "CODE") -> dict[str, Any]:
@@ -149,6 +150,8 @@ def test_blocked_flag_run_records_suspicious_success() -> None:
_record_run_blocks_result(ctx, result, completion_verification=None)
+ assert result["ok"] is False
+ assert result["error"] == ctx.last_test_failure_reason
assert ctx.last_test_ok is False
assert ctx.last_test_suspicious_success is True
assert ctx.last_full_workflow_test_ok is False
@@ -156,6 +159,10 @@ def test_blocked_flag_run_records_suspicious_success() -> None:
assert ctx.last_failed_workflow_yaml == "blocks: []"
categories = result["data"]["failure_categories"]
assert any(category["category"] == "ANTI_BOT_DETECTION" for category in categories)
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == "tool_error_run_output_terminal_blocker"
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
assert verified_goal_satisfied_context(ctx) is False
assert _verified_workflow_or_none(ctx) == (None, None)
snapshot = getattr(ctx, "outcome_verification_trace_snapshot", {})
@@ -192,6 +199,11 @@ def test_blocked_status_value_rejected_deterministically(block_type: str) -> Non
assert _is_outcome_evidence_candidate(ctx, result) is False
_record_run_blocks_result(ctx, result, completion_verification=None)
+ assert result["ok"] is False
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == "tool_error_run_output_terminal_blocker"
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
assert ctx.last_test_ok is False
assert ctx.last_test_suspicious_success is True
assert ctx.last_full_workflow_test_ok is False
diff --git a/tests/unit/test_copilot_schema_overlay_ban.py b/tests/unit/test_copilot_schema_overlay_ban.py
index 1a1a64530..7385e2fba 100644
--- a/tests/unit/test_copilot_schema_overlay_ban.py
+++ b/tests/unit/test_copilot_schema_overlay_ban.py
@@ -7,16 +7,45 @@ from unittest.mock import MagicMock
import pytest
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.tools import (
_COPILOT_BANNED_BLOCK_TYPES,
_get_block_schema_post_hook,
_get_block_schema_pre_hook,
)
+from skyvern.forge.sdk.copilot.tools.banned_blocks import (
+ _COPILOT_CODE_ONLY_BROWSER_BANNED_BLOCK_TYPES,
+ CopilotBlockPolicyStatus,
+)
+from skyvern.forge.sdk.copilot.tools.mcp_hooks import _validate_block_pre_hook
+
+_CODE_ONLY_UNAVAILABLE = tuple(
+ "action browser_task extraction file_download file_upload goto_url login navigation print_page task task_v2 validation".split()
+)
+_CODE_ONLY_REQUIRED_TEXT = {
+ "file_download": "download registration",
+ "file_upload": "file materialization",
+ "login": "credential-typed code",
+ "task": "declared AI leaf",
+ "task_v2": "declared AI leaf",
+}
+_CODE_ONLY_HELPERS = tuple(
+ "conditional for_loop while_loop http_request send_email file_url_parser download_to_s3 upload_to_s3 google_sheets_read google_sheets_write".split()
+)
@pytest.fixture
def ctx() -> MagicMock:
- return MagicMock()
+ ctx = MagicMock()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
+ return ctx
+
+
+def code_only_ctx() -> MagicMock:
+ ctx = MagicMock()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ return ctx
@pytest.mark.parametrize("block_type", ["task", "task_v2", "TASK", "Task_V2", " task "])
@@ -104,6 +133,113 @@ def test_banned_types_set_contents() -> None:
assert _COPILOT_BANNED_BLOCK_TYPES == frozenset({"task", "task_v2"})
+def test_code_only_policy_table_derives_unavailable_types() -> None:
+ assert _COPILOT_CODE_ONLY_BROWSER_BANNED_BLOCK_TYPES == frozenset(_CODE_ONLY_UNAVAILABLE)
+ assert "native_allowed" not in {status.value for status in CopilotBlockPolicyStatus}
+
+
[email protected]("block_type", _CODE_ONLY_UNAVAILABLE)
+async def test_code_only_schema_pre_hook_rejects_table_entries(block_type: str, code_only_ctx: MagicMock) -> None:
+ result = await _get_block_schema_pre_hook({"block_type": block_type}, code_only_ctx)
+
+ assert result is not None
+ assert result["ok"] is False
+ assert "not available in the workflow copilot" in result["error"]
+ assert _CODE_ONLY_REQUIRED_TEXT.get(block_type, "focused `code` blocks") in result["error"]
+
+
+async def test_code_only_schema_pre_hook_normalizes_case_whitespace_and_alias(code_only_ctx: MagicMock) -> None:
+ params = {"block_type": " BROWSER_TASK "}
+
+ result = await _get_block_schema_pre_hook(params, code_only_ctx)
+
+ assert result is not None
+ assert result["ok"] is False
+ assert params["block_type"] == "navigation"
+ assert "focused `code` blocks" in result["error"]
+
+
+async def test_code_only_post_hook_scrubs_all_policy_table_entries(code_only_ctx: MagicMock) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "block_types": dict.fromkeys(
+ ("navigation", "code", "conditional", "task", "task_v2", "login", "file_download", "file_upload"),
+ "...",
+ ),
+ "count": 8,
+ },
+ }
+
+ out = await _get_block_schema_post_hook(result, raw={}, ctx=code_only_ctx)
+
+ assert set(out["data"]["block_types"]) == {"code", "conditional"}
+ assert out["data"]["count"] == 2
+
+
+async def test_code_schema_guidance_is_policy_rendered_and_allows_helper_validation(code_only_ctx: MagicMock) -> None:
+ result = {"ok": True, "data": {"block_type": "code", "summary": "..."}}
+
+ out = await _get_block_schema_post_hook(result, raw={}, ctx=code_only_ctx)
+
+ assert "Browser/page workflow block types are unavailable" in out["data"]["code_only_note"]
+ assert "validate_block only for allowed non-browser helper blocks" in " ".join(out["data"]["code_only_guidance"])
+ assert "Do not persist navigation/action/login" not in " ".join(out["data"]["code_only_guidance"])
+
+
[email protected]("block_type", ["task", "task_v2"])
+async def test_standard_validate_block_pre_hook_preserves_existing_behavior(block_type: str, ctx: MagicMock) -> None:
+ result = await _validate_block_pre_hook({"block_json": f'{{"block_type": "{block_type}", "label": "x"}}'}, ctx)
+
+ assert result is None
+
+
[email protected]("block_type", _CODE_ONLY_UNAVAILABLE + ("code", " LOGIN ", "BROWSER_TASK"))
+async def test_code_only_validate_block_pre_hook_rejects_unavailable_or_probe_types(
+ block_type: str, code_only_ctx: MagicMock
+) -> None:
+ result = await _validate_block_pre_hook(
+ {"block_json": f'{{"block_type": "{block_type}", "label": "candidate"}}'},
+ code_only_ctx,
+ )
+
+ assert result is not None
+ assert result["ok"] is False
+ if block_type.strip().lower() == "code":
+ assert "validate real code blocks through update_and_run_blocks" in result["error"]
+ else:
+ assert "not available in the workflow copilot" in result["error"]
+
+
[email protected]("block_type", _CODE_ONLY_HELPERS)
+async def test_code_only_validate_block_pre_hook_allows_non_browser_helpers(
+ block_type: str, code_only_ctx: MagicMock
+) -> None:
+ result = await _validate_block_pre_hook(
+ {"block_json": f'{{"block_type": "{block_type}", "label": "candidate"}}'},
+ code_only_ctx,
+ )
+
+ assert result is None
+
+
[email protected]("block_json", ["not json", "[]", '{"label": "missing_type"}'])
+async def test_code_only_validate_block_pre_hook_leaves_shape_errors_to_validator(
+ block_json: str, code_only_ctx: MagicMock
+) -> None:
+ result = await _validate_block_pre_hook({"block_json": block_json}, code_only_ctx)
+
+ assert result is None
+
+
def test_pre_hook_and_post_emission_reject_share_constant() -> None:
"""SKY-9174 Part F: the pre-emission SchemaOverlay hooks and the
post-emission YAML-level reject (in `_update_workflow` + REPLACE_WORKFLOW)
diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.py
index 68c6b117b..868068f4e 100644
--- a/tests/unit/test_copilot_streaming_adapter.py
+++ b/tests/unit/test_copilot_streaming_adapter.py
@@ -398,6 +398,60 @@ async def test_tool_result_sse_uses_latest_blocker_signal_for_activity_surface()
assert "Do NOT" not in activity[-1]["text"]
+async def test_stream_to_sse_cancels_and_stops_on_terminal_turn_halt() -> None:
+ from agents.items import RunItem
+ from agents.stream_events import RunItemStreamEvent
+
+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+ from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt
+
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="loop_detected",
+ agent_steering_text="LOOP DETECTED: 'update_workflow' has already failed 3 times.",
+ user_facing_reason="I retried without making progress. Tell me what to change and I'll try again.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code="loop_detected_repeated_failed_step",
+ blocked_tool="update_workflow",
+ )
+
+ call_item = MagicMock(spec=RunItem)
+ call_item.raw_item = {"call_id": "c1", "name": "update_workflow", "arguments": "{}"}
+ tool_call_event = RunItemStreamEvent(name="tool_called", item=call_item)
+
+ out_item = MagicMock(spec=RunItem)
+ out_item.raw_item = {"call_id": "c1", "name": "update_workflow"}
+ out_item.output = [{"type": "text", "text": json.dumps({"ok": False, "error": signal.agent_steering_text})}]
+ tool_output_event = RunItemStreamEvent(name="tool_output", item=out_item)
+
+ late_call_item = MagicMock(spec=RunItem)
+ late_call_item.raw_item = {"call_id": "c2", "name": "update_and_run_blocks", "arguments": "{}"}
+ late_tool_call_event = RunItemStreamEvent(name="tool_called", item=late_call_item)
+ consumed_late_event = False
+
+ async def _events() -> Any:
+ nonlocal consumed_late_event
+ yield tool_call_event
+ yield tool_output_event
+ consumed_late_event = True
+ yield late_tool_call_event
+
+ result = MagicMock()
+ result.stream_events = _events
+ result.cancel = MagicMock()
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = AsyncMock(return_value=True)
+ ctx = SimpleNamespace(latest_tool_blocker_signal=signal, tool_blocker_signals=[signal])
+
+ with pytest.raises(CopilotTurnHalt):
+ await stream_to_sse(result, stream, ctx)
+
+ result.cancel.assert_called_once()
+ assert consumed_late_event is False
+
+
@pytest.mark.asyncio
async def test_stream_to_sse_raises_and_cancels_on_repeated_unrecoverable_tool_error() -> None:
from agents.items import RunItem
diff --git a/tests/unit/test_copilot_task_block_rejection.py b/tests/unit/test_copilot_task_block_rejection.py
index 7fa4c7ade..c26fe4f4f 100644
--- a/tests/unit/test_copilot_task_block_rejection.py
+++ b/tests/unit/test_copilot_task_block_rejection.py
@@ -17,7 +17,9 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import yaml
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.tools import (
+ _banned_block_reject_message,
_detect_new_banned_blocks,
_proxy_location_trace_value,
_raw_yaml_proxy_location,
@@ -264,6 +266,7 @@ def test_block_without_label_is_skipped() -> None:
def _ctx(prior_yaml: str | None = None) -> MagicMock:
ctx = MagicMock()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
ctx.workflow_yaml = prior_yaml
ctx.workflow_id = "w_test"
ctx.workflow_permanent_id = "wpid_test"
@@ -271,6 +274,12 @@ def _ctx(prior_yaml: str | None = None) -> MagicMock:
return ctx
+def _code_only_ctx(prior_yaml: str | None = None) -> MagicMock:
+ ctx = _ctx(prior_yaml=prior_yaml)
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ return ctx
+
+
@pytest.mark.asyncio
async def test_update_workflow_rejects_new_task_block_and_emits_span() -> None:
submitted = _yaml({"block_type": "task", "label": "fill_contact_form", "navigation_goal": "do"})
@@ -372,3 +381,33 @@ async def test_update_workflow_allows_all_allowed_block_types() -> None:
result = await _update_workflow({"workflow_yaml": submitted}, ctx)
assert result["ok"] is True
+
+
+def test_code_only_reject_message_groups_per_type_capability_text() -> None:
+ ctx = _code_only_ctx()
+ message = _banned_block_reject_message(
+ [("login_step", "login"), ("download_step", "file_download"), ("open_step", "navigation")],
+ ctx,
+ )
+
+ assert "not available in the workflow copilot" in message
+ assert "login_step" in message
+ assert "download_step" in message
+ assert "open_step" in message
+ assert "credential-typed code" in message
+ assert "download registration" in message
+ assert "focused `code` blocks" in message
+
+
+async def test_code_only_update_workflow_rejects_new_browser_block_with_policy_text() -> None:
+ submitted = _yaml({"block_type": "login", "label": "login_step"})
+ ctx = _code_only_ctx(prior_yaml=None)
+
+ with patch("skyvern.forge.sdk.copilot.tools.workflow_update._record_banned_block_reject_span") as mock_span:
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "not available in the workflow copilot" in result["error"]
+ assert "credential-typed code" in result["error"]
+ mock_span.assert_called_once_with("_update_workflow", [("login_step", "login")])
diff --git a/tests/unit/test_copilot_turn_halt.py b/tests/unit/test_copilot_turn_halt.py
new file mode 100644
index 000000000..c8f4306a6
--- /dev/null
+++ b/tests/unit/test_copilot_turn_halt.py
@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
+from skyvern.forge.sdk.copilot.enforcement import _check_enforcement
+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
+from skyvern.forge.sdk.copilot.turn_halt import (
+ CopilotTurnHalt,
+ TurnHaltKind,
+ raise_if_turn_halt,
+ stash_turn_halt_from_blocker_signal,
+ turn_halt_from_blocker_signal,
+)
+
+
+def _signal(
+ *,
+ blocker_kind: str = "tool_error",
+ internal_reason_code: str = ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
+ renders_final_reply: bool = True,
+) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind=blocker_kind,
+ agent_steering_text="terminal blocker",
+ user_facing_reason="The page appears blocked by a site challenge.",
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=True,
+ renders_final_reply=renders_final_reply,
+ internal_reason_code=internal_reason_code,
+ blocked_tool="update_and_run_blocks",
+ )
+
+
+ ("signal", "expected_kind"),
+ [
+ (
+ _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step"),
+ TurnHaltKind.LOOP_DETECTED,
+ ),
+ (
+ _signal(internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE),
+ TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,
+ ),
+ (
+ _signal(internal_reason_code="tool_error_run_output_terminal_blocker"),
+ TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,
+ ),
+ (_signal(internal_reason_code="probable_site_block_stop"), TurnHaltKind.PROBABLE_SITE_BLOCK),
+ ],
+)
+def test_terminal_blockers_map_to_halts(signal: CopilotToolBlockerSignal, expected_kind: TurnHaltKind) -> None:
+ halt = turn_halt_from_blocker_signal(signal, source="hook")
+
+ assert halt is not None
+ assert halt.kind == expected_kind
+ assert halt.blocker_signal is signal
+
+
+def test_stash_and_raise_turn_halt_sets_context_once() -> None:
+ ctx = SimpleNamespace(turn_halt=None)
+ signal = _signal(blocker_kind="loop_detected", internal_reason_code="loop_detected_repeated_failed_step")
+
+ halt = stash_turn_halt_from_blocker_signal(ctx, signal, source="stream")
+
+ assert halt is ctx.turn_halt
+ assert halt is not None
+ with pytest.raises(CopilotTurnHalt) as exc_info:
+ raise_if_turn_halt(ctx)
+ assert exc_info.value.halt is ctx.turn_halt
+
+
+def test_enforcement_backstop_converts_existing_terminal_blocker_signal() -> None:
+ ctx = SimpleNamespace(turn_halt=None, latest_tool_blocker_signal=None, blocker_signal=None)
+ signal = _signal()
+ stash_blocker_signal(ctx, signal)
+
+ with pytest.raises(CopilotTurnHalt) as exc_info:
+ _check_enforcement(ctx)
+
+ assert ctx.turn_halt is exc_info.value.halt
+ assert exc_info.value.halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE