Skip to content

Automate browser based workflows with AI

fix(SKY-11971): reclassify precondition redirects out of the error stream (#7359)

Celal Zamanoğlu1507e2801e59
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index 207cf0f3f..51821740e 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.challenge_evidence import (
interactive_challenge_controls,
vision_challenge_carrier,
)
+from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIX
from skyvern.forge.sdk.copilot.reached_download_target import (
NAV_TARGET_DOWNLOAD_KIND_KEY,
classify_download_affordance,
@@ -595,7 +596,7 @@ def _post_run_observed_url_goto_error(
labels = ", ".join(block["label"] for block in offending[:5])
return (
- "Workflow validation failed: the draft is trying to persist a post-run browser URL as a new goto_url "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}the draft is trying to persist a post-run browser URL as a new goto_url "
"block after an incomplete or budgeted run. That URL may encode record-specific, result-page, or "
"session state. Keep the reusable entrypoint and verified upstream blocks, then either extract from "
"the observed current page, split or replace the budgeted frontier into smaller reusable UI actions, "
@@ -1071,14 +1072,14 @@ def composition_page_evidence_error(
)
if string_step is not None:
return (
- "Workflow validation failed: a block_observation_refs entry uses observation_step "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}a block_observation_refs entry uses observation_step "
f"{string_step!r} as a string. Pass the integer observation_step returned by "
"inspect_page_for_composition or evaluate for click-reached blocks. "
f"Offending blocks: {_format_page_block_findings([block])}"
)
if _required_observation_ref_missing(block, block_observation_refs):
return (
- "Workflow validation failed: a click-reached block requires a block_observation_refs entry. "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}a click-reached block requires a block_observation_refs entry. "
"Pass an interaction- or post_run-reached observation_step for click-reached blocks before "
"composing them. "
f"Offending blocks: {_format_page_block_findings([block])}"
@@ -1086,7 +1087,7 @@ def composition_page_evidence_error(
if wrong_reached_via is not None:
step, reached_via = wrong_reached_via
return (
- "Workflow validation failed: a block references observation_step "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}a block references observation_step "
f"{step}, but that observed page was reached via {reached_via!r}. "
"Pass an interaction- or post_run-reached observation_step for click-reached blocks. "
f"Offending blocks: {_format_page_block_findings([block])}"
@@ -1099,14 +1100,14 @@ def composition_page_evidence_error(
else "that observation step was not found in flow evidence"
)
return (
- "Workflow validation failed: a block references observation_step "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}a block references observation_step "
f"{missing_step}, but {missing_reason}. "
"Inspect or evaluate the reached page again and pass the new observation_step in "
"block_observation_refs before composing page-dependent blocks. "
f"Offending blocks: {_format_page_block_findings([block])}"
)
return (
- "Workflow validation failed: page-dependent build blocks need observed page evidence before they are "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}page-dependent build blocks need observed page evidence before they are "
f"authored. Call inspect_page_for_composition(target_url={target_url!r}) before composing page-dependent "
"blocks, or save only the initial goto_url block and inspect the reached page before the next mutation. "
f"Offending blocks: {_format_page_block_findings([block])}"
diff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.py
index 9dc60b530..f10b9ceb0 100644
--- a/skyvern/forge/sdk/copilot/narration.py
+++ b/skyvern/forge/sdk/copilot/narration.py
@@ -563,16 +563,21 @@ _TRANSITION_LABELS: dict[TransitionKind, str] = {
_MAX_DETAILS_CHARS = 240
-def extract_tool_details(tool_name: str, parsed: dict[str, Any]) -> str:
+def extract_tool_details(tool_name: str, parsed: dict[str, Any], *, success: bool | None = None) -> str:
"""Compact narrator-friendly excerpt from a tool's parsed payload.
Intentionally narrow: counts, domains, and high-level statuses only.
Raw labels (block names, field names, URL paths, page content) are excluded
so they can't reach the narrator prompt and be echoed at the user.
+
+ ``success`` lets a caller override the raw ``ok`` field (e.g. a precondition
+ redirect that streaming_adapter has already reclassified as non-failure) so this
+ detail line doesn't contradict the entry's own success status in the narrator prompt.
"""
if not isinstance(parsed, dict):
return ""
- if not parsed.get("ok", True):
+ ok = parsed.get("ok", True) if success is None else success
+ if not ok:
return "last action failed"
data = parsed.get("data")
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index caa8b04ea..74e821052 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -134,6 +134,31 @@ _INTERNAL_TOOL_PARAPHRASE_PHRASES = ("nudge turn", "nudge-turn")
_INTERNAL_COPILOT_SENTINEL_PREFIX = "[copilot:"
_LOOP_DETECTED_MARKER = "loop detected"
+# Copilot's own paraphrase for a formatting change it made — not standard YAML
+# vocabulary, so a legitimate clarification is not going to phrase itself this way.
+# Runs for every response type, including ASK_QUESTION.
+_YAML_AUTHORING_JARGON_PHRASES = (
+ "folded code formatting",
+ "literal code formatting",
+)
+# Standard YAML block-scalar terminology (folded `>` vs literal `|`). A legitimate
+# ASK_QUESTION can ask the user which style they want for a multi-line value, so these
+# only count as a leak once the reply is final and user-visible.
+_YAML_BLOCK_SCALAR_TERM_PHRASES = (
+ "folded block scalar",
+ "literal block scalar",
+)
+
+
+def _contains_yaml_authoring_vocab_leak(user_response: str, response_type: str) -> bool:
+ lower = user_response.lower()
+ if any(phrase in lower for phrase in _YAML_AUTHORING_JARGON_PHRASES):
+ return True
+ return response_type in _USER_VISIBLE_REPLY_TYPES and any(
+ phrase in lower for phrase in _YAML_BLOCK_SCALAR_TERM_PHRASES
+ )
+
+
_SELF_PRESCRIPTIVE_FIXED_PHRASE = "send me a normal instruction like"
_SELF_PRESCRIPTIVE_IMPERATIVE_VERBS = frozenset({"send", "reply", "type", "respond"})
_SELF_PRESCRIPTIVE_POLITE_PREFIXES = frozenset({"please", "kindly", "just", "now"})
@@ -669,6 +694,8 @@ def _contains_internal_block_taxonomy_leak(
return False
if _contains_deprecated_block_identifier(user_response):
return True
+ if _contains_yaml_authoring_vocab_leak(user_response, response_type):
+ return True
if response_type in _USER_VISIBLE_REPLY_TYPES and _contains_internal_tool_vocab_leak(user_response):
return True
if output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER:
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index fd52b53b7..37aa8eb97 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -425,6 +425,28 @@ def _structured_failure_summary_for_user(
return None
+# Blocker kinds where the tool was redirected before it ran (a precondition/authority
+# gate), not a real failure. `tool_error` and `loop_detected` keep failure affect —
+# something actually broke or the agent is stuck.
+_NEUTRAL_REDIRECT_BLOCKER_KINDS = frozenset({"phase_gated", "missing_required_context", "authority_denied"})
+
+
+def user_facing_success(
+ result: dict[str, Any],
+ *,
+ blocker_signal: CopilotToolBlockerSignal | Iterable[CopilotToolBlockerSignal] | None = None,
+) -> bool:
+ """Whether a tool result should render without failure affect in the user-facing
+ activity stream. A raw ``ok=False`` still counts as success here when it's explained
+ by a precondition/authority blocker signal — the agent was redirected, not broken."""
+ if result.get("ok", True):
+ return True
+ return any(
+ signal.blocker_kind in _NEUTRAL_REDIRECT_BLOCKER_KINDS and _blocker_signal_matches_result(signal, result)
+ for signal in _iter_blocker_signals(blocker_signal)
+ )
+
+
_HEADERS_BLOB_RE = re.compile(r"\s*headers:\s*\{[^{}]*\}\s*", re.IGNORECASE)
_LARGE_DICT_BLOB_RE = re.compile(r"\{[^{}]{40,}\}")
@@ -543,14 +565,26 @@ def summarize_tool_result_detail(
*,
tool_name: str | None = None,
blocker_signal: CopilotToolBlockerSignal | Iterable[CopilotToolBlockerSignal] | None = None,
+ success: bool | None = None,
) -> str | None:
- """Tooltip-grade failure detail (longer cap than ``summarize_tool_result``); None on success."""
- if result.get("ok", False):
+ """Tooltip-grade failure detail (longer cap than ``summarize_tool_result``); None on success.
+
+ ``success`` lets a caller pass the already-reclassified value (e.g. a phase/authority
+ redirect that ``user_facing_success`` upgraded from raw ``ok: false``) so this field
+ doesn't contradict the row's own success flag — same override shape as
+ ``narration.extract_tool_details``.
+ """
+ if result.get("ok", False) if success is None else success:
return None
structured = _structured_failure_summary_for_user(result, blocker_signal=blocker_signal, blocked_tool=tool_name)
if structured is not None:
return structured
- return _sanitize_failure_text(_extract_failure_message(result), max_chars=max_chars)
+ failure_message = _extract_failure_message(result)
+ # Same internal-validator convention _translate_failure_for_user maps to the generic
+ # summary — the tooltip-grade detail must not leak the raw text either.
+ if any(marker in failure_message.lower() for marker in _INTERNAL_VALIDATION_MARKERS):
+ return _USER_FACING_GENERIC_FAILURE
+ return _sanitize_failure_text(failure_message, max_chars=max_chars)
_JINJA_ERROR_MARKERS: tuple[str, ...] = ("Failed to format jinja", "Jinja style parameter")
@@ -565,6 +599,13 @@ _ENGINE_INSTRUCTION_MARKERS: tuple[str, ...] = (
"waiting for locator",
)
_USE_TOOL_NAME_RE = re.compile(r"use the ['\"]?[a-z_][a-z0-9_]*['\"]? tool", re.IGNORECASE)
+# Shared prefix for internal workflow-authoring validator rejects (stale block metadata,
+# banned block types, missing observation evidence, raw YAML/pydantic errors). Validators
+# import this constant rather than hand-typing the prefix, so a future validator can't
+# silently bypass the leak-suppression below by phrasing its reject text differently.
+# The full text is written for the agent to self-correct, never for the user.
+INTERNAL_VALIDATION_FAILURE_PREFIX = "Workflow validation failed: "
+_INTERNAL_VALIDATION_MARKERS: tuple[str, ...] = (INTERNAL_VALIDATION_FAILURE_PREFIX.strip(": ").lower(),)
_USER_FACING_LOOP_MESSAGE = "The agent got stuck retrying the same step — moving on."
_USER_FACING_JINJA_MESSAGE = "A workflow parameter could not be filled in."
@@ -579,6 +620,8 @@ def _translate_failure_for_user(error_text: str) -> str:
if any(marker in error_text for marker in _JINJA_ERROR_MARKERS):
return _USER_FACING_JINJA_MESSAGE
lowered = error_text.lower()
+ if any(marker in lowered for marker in _INTERNAL_VALIDATION_MARKERS):
+ return _USER_FACING_GENERIC_FAILURE
if any(marker in lowered for marker in _ENGINE_INSTRUCTION_MARKERS):
return _USER_FACING_GENERIC_FAILURE
if _USE_TOOL_NAME_RE.search(error_text):
diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.py
index e3cdda471..39538711c 100644
--- a/skyvern/forge/sdk/copilot/streaming_adapter.py
+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py
@@ -33,7 +33,11 @@ from skyvern.forge.sdk.copilot.narration import (
snapshot_ctx,
tool_activity_display_label,
)
-from skyvern.forge.sdk.copilot.output_utils import format_tool_result_for_user, summarize_tool_result_detail
+from skyvern.forge.sdk.copilot.output_utils import (
+ format_tool_result_for_user,
+ summarize_tool_result_detail,
+ user_facing_success,
+)
from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.turn_halt import (
CopilotTurnHalt,
@@ -376,8 +380,10 @@ async def stream_to_sse(
else:
blocker_signals = _tool_blocker_signal_candidates(ctx)
summary = format_tool_result_for_user(tool_name, parsed, blocker_signal=blocker_signals)
- success = parsed.get("ok", True)
- detail = summarize_tool_result_detail(parsed, tool_name=tool_name, blocker_signal=blocker_signals)
+ success = user_facing_success(parsed, blocker_signal=blocker_signals)
+ detail = summarize_tool_result_detail(
+ parsed, tool_name=tool_name, blocker_signal=blocker_signals, success=success
+ )
narrator_state.record_activity(
build_tool_result_activity(tool_name, summary, success, iteration, call_id)
)
@@ -409,7 +415,7 @@ async def stream_to_sse(
summary=summary,
success=success,
iteration=iteration,
- details=extract_tool_details(tool_name, parsed),
+ details=extract_tool_details(tool_name, parsed, success=success),
)
for transition in detect_transitions(ctx_before, ctx_after, tool_name, prior_tool_name):
narrator_state.record_transition(transition)
@@ -515,7 +521,7 @@ async def flush_goal_satisfied_tool_result(stream: EventSourceStream, ctx: Copil
return
blocker_signals = _tool_blocker_signal_candidates(ctx)
summary = format_tool_result_for_user(pending.tool_name, parsed, blocker_signal=blocker_signals)
- success = parsed.get("ok", True)
+ success = user_facing_success(parsed, blocker_signal=blocker_signals)
narrator_state = ctx.narrator_state
if narrator_state is not None:
narrator_state.record_activity(
@@ -531,7 +537,9 @@ async def flush_goal_satisfied_tool_result(stream: EventSourceStream, ctx: Copil
summary=summary,
iteration=pending.iteration,
tool_call_id=pending.call_id,
- detail=summarize_tool_result_detail(parsed, tool_name=pending.tool_name, blocker_signal=blocker_signals),
+ detail=summarize_tool_result_detail(
+ parsed, tool_name=pending.tool_name, blocker_signal=blocker_signals, success=success
+ ),
workflow_run_id=_tool_result_workflow_run_id(pending.tool_name, parsed),
)
)
diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
index 729ca6198..9ee722416 100644
--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py
+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
@@ -8,6 +8,7 @@ from typing import Any
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
from skyvern.forge.sdk.copilot.enforcement import PROBABLE_SITE_BLOCK_STREAK_STOP_AT
+from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIX
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -490,7 +491,7 @@ def _challenge_http_request_reject_message(
return None
labels_text = ", ".join(sorted(set(labels)))
return (
- "Workflow validation failed: raw http_request blocks are not allowed for a page with observed "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}raw http_request blocks are not allowed for a page with observed "
"anti-bot or human-verification challenge evidence. "
f"Offending labels: [{labels_text}]. "
"Use browser workflow blocks grounded in the observed page, include challenge handling only when visible, "
@@ -506,7 +507,7 @@ def _timing_only_challenge_wait_reject_message(ctx: Any, submitted_yaml: str | N
return None
labels_text = ", ".join(sorted(set(labels)))
return (
- "Workflow validation failed: timing-only challenge wait blocks are not allowed after confirmed "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}timing-only challenge wait blocks are not allowed after confirmed "
"anti-bot/WAF or repeated site-block evidence. "
f"Offending labels: [{labels_text}]. "
"Do not add wait/delay-only blocks for this blocker; use a conditional challenge check that takes a "
diff --git a/skyvern/forge/sdk/copilot/tools/frontier.py b/skyvern/forge/sdk/copilot/tools/frontier.py
index 4bcb604f0..7a7c33787 100644
--- a/skyvern/forge/sdk/copilot/tools/frontier.py
+++ b/skyvern/forge/sdk/copilot/tools/frontier.py
@@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.failure_tracking import (
_canonical_block_config,
)
+from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIX
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
)
@@ -264,7 +265,7 @@ def _stale_block_metadata_message(items: list[dict[str, Any]]) -> str:
details.append(f"(and {len(items) - _STALE_BLOCK_METADATA_MESSAGE_LIMIT} more)")
joined = "; ".join(details)
return (
- "Workflow validation failed: corrected block metadata still appears stale. "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}corrected block metadata still appears stale. "
"When changing a user's requested subject, URL, or action, rename affected block labels and titles "
"to match the revised goal, and update next_block_label, block_labels, and Jinja references accordingly. "
f"Stale metadata: {joined}"
@@ -961,7 +962,7 @@ def _frontier_run_size_error(
suggested = labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS]
remaining = labels_to_execute[_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:]
return (
- "Workflow validation failed: this browser test frontier is too long for a multi-stage "
+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}this browser test frontier is too long for a multi-stage "
"page-changing workflow. Keep the same complete workflow YAML, but shrink only the "
f"block_labels argument to the next 1-2 unverified labels: {suggested!r}. "
"If a prior run already advanced the browser, inspect that reached page "
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index b0b59ac61..6c7bb46a6 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -128,6 +128,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
output_policy_verdict_to_trace_data,
url_origin,
)
+from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIX
from skyvern.forge.sdk.copilot.reached_download_target import (
REGISTERED_DOWNLOAD_OUTPUT_KEYS,
ReachedDownloadTarget,
@@ -10215,7 +10216,7 @@ async def _update_workflow(
code_rejected=True,
)
return reject(
- error=f"Workflow validation failed: {e}",
+ error=f"{INTERNAL_VALIDATION_FAILURE_PREFIX}{e}",
user_facing_summary=user_facing_summary,
data=repair_data,
)
diff --git a/tests/unit/test_copilot_narration.py b/tests/unit/test_copilot_narration.py
index c4851d093..8c8bef1c6 100644
--- a/tests/unit/test_copilot_narration.py
+++ b/tests/unit/test_copilot_narration.py
@@ -352,6 +352,23 @@ def test_extract_tool_details_failure_is_generic() -> None:
assert "failed" in details.lower()
+def test_extract_tool_details_success_override_suppresses_failed_line() -> None:
+ """A caller-supplied success (e.g. a reclassified precondition redirect) must
+ win over the raw ok:False, so the detail line doesn't contradict the entry's
+ own success status in the narrator prompt."""
+ details = narration.extract_tool_details(
+ "evaluate",
+ {"ok": False, "error": "internal steering text"},
+ success=True,
+ )
+ assert details != "last action failed"
+
+
+def test_extract_tool_details_no_override_keeps_raw_ok() -> None:
+ details = narration.extract_tool_details("evaluate", {"ok": False, "error": "boom"})
+ assert details == "last action failed"
+
+
# ---------------------------------------------------------------------------
# _build_narrator_prompt — redacts raw tool identifiers
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 141cf3d25..58ff199ec 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
OutputPolicyReason,
OutputPolicyVerdict,
_contains_internal_tool_vocab_leak,
+ _contains_yaml_authoring_vocab_leak,
derive_output_kind,
evaluate_actuation_obligation,
evaluate_output_policy,
@@ -2755,6 +2756,46 @@ def test_contains_internal_tool_vocab_leak_helper(user_response: str, expected:
assert _contains_internal_tool_vocab_leak(user_response) is expected
+ "user_response,response_type,expected",
+ [
+ ("I changed the code block from folded code formatting to literal code formatting.", "REPLY", True),
+ ("Switched that step to use literal code formatting instead.", "REPLY", True),
+ ("Rewrote the block using a literal block scalar.", "REPLY", True),
+ ("Rewrote the block using a literal block scalar.", "REPLACE_WORKFLOW", True),
+ ("Switched that step to use literal code formatting instead.", "ASK_QUESTION", True),
+ ("Update the workflow to log in before extracting data.", "REPLY", False),
+ ("I formatted the code so it's easier to read.", "REPLY", False),
+ ("", "REPLY", False),
+ ],
+)
+def test_contains_yaml_authoring_vocab_leak_helper(user_response: str, response_type: str, expected: bool) -> None:
+ assert _contains_yaml_authoring_vocab_leak(user_response, response_type) is expected
+
+
+def test_yaml_authoring_vocab_leak_hard_blocks_ask_question() -> None:
+ """Regression guard: unlike the nudge/copilot-sentinel phrases, this leak class must
+ be caught in ASK_QUESTION turns too — a clarifying question is still user-visible."""
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="ASK_QUESTION",
+ user_response="Should I switch this block to literal code formatting?",
+ )
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in verdict.reason_codes
+
+
+def test_yaml_block_scalar_standard_term_does_not_hard_block_ask_question() -> None:
+ """Regression guard for PR #13274 review feedback (LawyZheng): 'literal/folded block
+ scalar' is standard YAML terminology a legitimate clarifying question can use — unlike
+ the code-formatting jargon phrases, it must not hard-replace an ASK_QUESTION turn."""
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="ASK_QUESTION",
+ user_response="Should this multi-line value use a literal block scalar or folded style?",
+ )
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK not in verdict.reason_codes
+
+
def test_translate_to_agent_result_prioritizes_unbacked_workflow_claim_over_taxonomy_rewrite() -> None:
result = _fake_run_result(
{
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index 19051076b..faeebaa1b 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
summarize_tool_result,
summarize_tool_result_detail,
truncate_output,
+ user_facing_success,
)
@@ -778,6 +779,59 @@ class TestFormatToolResultForUser:
assert agent_summary == "Clicked 'xpath=//button[2]'"
+class TestUserFacingSuccess:
+ @staticmethod
+ def _blocker(blocker_kind: str, *, steering: str = "internal steering text"):
+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+
+ return CopilotToolBlockerSignal(
+ blocker_kind=blocker_kind, # type: ignore[arg-type]
+ agent_steering_text=steering,
+ user_facing_reason="I need more information before I can continue.",
+ recovery_hint="ask_user_clarifying",
+ internal_reason_code="test_reason_code",
+ blocked_tool="evaluate",
+ )
+
+ def test_true_for_ok_result(self) -> None:
+ assert user_facing_success({"ok": True, "data": {}}) is True
+
+ def test_false_for_unclassified_failure(self) -> None:
+ assert user_facing_success({"ok": False, "error": "plain failure"}) is False
+
+ @pytest.mark.parametrize("blocker_kind", ["phase_gated", "missing_required_context", "authority_denied"])
+ def test_true_for_precondition_style_blockers(self, blocker_kind: str) -> None:
+ signal = self._blocker(blocker_kind)
+ result = {"ok": False, "error": signal.agent_steering_text}
+ assert user_facing_success(result, blocker_signal=signal) is True
+
+ @pytest.mark.parametrize("blocker_kind", ["tool_error", "loop_detected"])
+ def test_false_for_genuine_failure_blockers(self, blocker_kind: str) -> None:
+ """Regression guard: real tool errors and loop-detection halts keep failure affect."""
+ signal = self._blocker(blocker_kind)
+ result = {"ok": False, "error": signal.agent_steering_text}
+ assert user_facing_success(result, blocker_signal=signal) is False
+
+ def test_false_when_blocker_signal_does_not_match_result(self) -> None:
+ signal = self._blocker("phase_gated", steering="unrelated steering text")
+ result = {"ok": False, "error": "a totally different failure"}
+ assert user_facing_success(result, blocker_signal=signal) is False
+
+
+def test_format_tool_result_for_user_reframes_internal_validation_failure() -> None:
+ """Pins the SKY-11971 forensic leak: an unclassified internal validator reject must
+ never surface its raw agent-steering text (block labels, field names) to the user."""
+ raw_error = (
+ "Workflow validation failed: corrected block metadata still appears stale. "
+ "When changing a user's requested subject, URL, or action, rename affected block "
+ "labels and titles to match the revised goal. Stale metadata: extract_step: label mismatch"
+ )
+ summary = format_tool_result_for_user("update_workflow", {"ok": False, "error": raw_error})
+ assert summary == "Couldn't complete that step."
+ assert "stale" not in summary
+ assert "block" not in summary.lower()
+
+
class TestParseFinalResponse:
"""parse_final_response is the last mile between model output and the frontend.
@@ -1025,8 +1079,32 @@ def test_summarize_tool_result_detail_returns_none_on_success() -> None:
assert summarize_tool_result_detail({"ok": True, "data": {"block_count": 2}}) is None
+def test_summarize_tool_result_detail_omits_detail_for_reclassified_neutral_redirect() -> None:
+ """Regression guard (Codex, PR #13274): a phase/authority redirect reclassified to
+ success=True by user_facing_success must not still carry a non-None `detail` — the
+ schema documents `detail` as None on success, and this row renders without failure
+ affect. Without passing the reclassified `success` through, the raw `ok: false`
+ still drives a non-None structured detail here."""
+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text="internal steering text",
+ user_facing_reason="I need to know what page to inspect first.",
+ recovery_hint="ask_user_clarifying",
+ internal_reason_code="test_reason_code",
+ blocked_tool="evaluate",
+ )
+ result = {"ok": False, "error": signal.agent_steering_text}
+ reclassified_success = user_facing_success(result, blocker_signal=signal)
+ assert reclassified_success is True
+
+ assert summarize_tool_result_detail(result, blocker_signal=signal) is not None
+ assert summarize_tool_result_detail(result, blocker_signal=signal, success=reclassified_success) is None
+
+
def test_summarize_tool_result_detail_caps_at_max_chars() -> None:
- long_error = "Workflow validation failed: " + ("missing field 'foo'; " * 200)
+ long_error = "Element lookup failed: " + ("missing field 'foo'; " * 200)
detail = summarize_tool_result_detail({"ok": False, "error": long_error}, max_chars=400)
assert detail is not None
assert len(detail) <= 400
@@ -1034,10 +1112,18 @@ def test_summarize_tool_result_detail_caps_at_max_chars() -> None:
def test_summarize_tool_result_detail_preserves_short_full_message() -> None:
+ detail = summarize_tool_result_detail(
+ {"ok": False, "error": "Element lookup failed: title field required"},
+ )
+ assert detail == "Element lookup failed: title field required"
+
+
+def test_summarize_tool_result_detail_reframes_internal_validation_failure() -> None:
+ """Tooltip-grade detail must not leak raw internal validator text either."""
detail = summarize_tool_result_detail(
{"ok": False, "error": "Workflow validation failed: title field required"},
)
- assert detail == "Workflow validation failed: title field required"
+ assert detail == "Couldn't complete that step."
def test_summarize_tool_result_detail_strips_header_blobs() -> None:
diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.py
index b470bf4be..20b702070 100644
--- a/tests/unit/test_copilot_streaming_adapter.py
+++ b/tests/unit/test_copilot_streaming_adapter.py
@@ -523,12 +523,80 @@ async def test_tool_result_sse_uses_latest_blocker_signal_for_activity_surface()
assert tool_results[0].summary == signal.user_facing_reason
assert tool_results[0].detail == signal.user_facing_reason
assert "Do NOT" not in tool_results[0].summary
+ # tool_error blocker signals are genuine failures (something ran out of budget/broke,
+ # not a precondition redirect) and must keep their failure affect.
+ assert tool_results[0].success is False
activity = ctx.narrator_state.design_activity
assert len(activity) == 2
assert activity[-1]["kind"] == "tool_result"
assert activity[-1]["text"] == signal.user_facing_reason
assert "Do NOT" not in activity[-1]["text"]
+ assert activity[-1]["success"] is False
+
+
+async def test_tool_result_sse_success_true_for_phase_gated_precondition() -> None:
+ """A precondition redirect (agent tried a tool before its build phase allowed it)
+ was never a real failure — it must not render with error affect."""
+ from agents.items import RunItem
+ from agents.stream_events import RunItemStreamEvent
+
+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+ from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageType
+
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="phase_gated",
+ agent_steering_text=(
+ "Direct browser tools are not callable before composition. "
+ "Call discover_workflow_entrypoint to resolve the entrypoint URL."
+ ),
+ user_facing_reason="I need to know what site to work on before I can browse there. What URL should I use?",
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset({"discover_workflow_entrypoint"}),
+ internal_reason_code="build_phase_browser_blocked_pre_compose",
+ blocked_tool="evaluate",
+ )
+
+ call_item = MagicMock(spec=RunItem)
+ call_item.raw_item = {"call_id": "c1", "name": "evaluate", "arguments": "{}"}
+ tool_call_event = RunItemStreamEvent(name="tool_called", item=call_item)
+
+ out_item = MagicMock(spec=RunItem)
+ out_item.raw_item = {"call_id": "c1", "name": "evaluate"}
+ out_item.output = [{"type": "text", "text": json.dumps({"ok": False, "error": signal.agent_steering_text})}]
+ tool_output_event = RunItemStreamEvent(name="tool_output", item=out_item)
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(tool_call_event, tool_output_event)
+ result.cancel = MagicMock()
+
+ sent: list[Any] = []
+
+ async def _send(payload: Any) -> bool:
+ sent.append(payload)
+ return True
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = _send
+ ctx = SimpleNamespace(
+ latest_tool_blocker_signal=signal,
+ tool_blocker_signals=[signal],
+ last_artifact_health_blocker_reason=None,
+ completion_verification_result=None,
+ )
+
+ await stream_to_sse(result, stream, ctx)
+
+ tool_results = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.TOOL_RESULT]
+ assert len(tool_results) == 1
+ assert tool_results[0].summary == signal.user_facing_reason
+ assert tool_results[0].success is True
+
+ activity = ctx.narrator_state.design_activity
+ assert activity[-1]["kind"] == "tool_result"
+ assert activity[-1]["success"] is True
@pytest.mark.asyncio
@@ -1234,9 +1302,7 @@ async def _capture_tool_result(tool_name: str, parsed_output: dict[str, Any]) ->
@pytest.mark.asyncio
async def test_stream_emits_detail_for_failure() -> None:
- long_error = "Workflow validation failed: " + (
- "blocks.0.task expects 'navigation_goal' but the emitted YAML omitted it. " * 4
- )
+ long_error = "blocks.0.task expects 'navigation_goal' but the emitted YAML omitted it. " * 4
payload = await _capture_tool_result(
"update_workflow",
{"ok": False, "error": long_error},
@@ -1249,6 +1315,21 @@ async def test_stream_emits_detail_for_failure() -> None:
assert len(payload.detail) > len(payload.summary)
+async def test_stream_emits_generic_detail_for_internal_validation_failure() -> None:
+ """The tooltip-grade detail must not leak raw internal validator text either —
+ only the visible summary was sanitized before this regression test was added."""
+ long_error = "Workflow validation failed: " + (
+ "blocks.0.task expects 'navigation_goal' but the emitted YAML omitted it. " * 4
+ )
+ payload = await _capture_tool_result(
+ "update_workflow",
+ {"ok": False, "error": long_error},
+ )
+ assert payload.detail == "Couldn't complete that step."
+ assert "navigation_goal" not in payload.detail
+
+
@pytest.mark.asyncio
async def test_stream_emits_no_detail_for_success() -> None:
payload = await _capture_tool_result(