Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11971): reclassify precondition redirects out of the error stream (#7359)
Celal Zamanoğlu1507e2801e59
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.pyindex 207cf0f3f..51821740e 100644--- a/skyvern/forge/sdk/copilot/composition_evidence.py+++ b/skyvern/forge/sdk/copilot/composition_evidence.py@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.challenge_evidence import (interactive_challenge_controls,vision_challenge_carrier,)+from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIXfrom skyvern.forge.sdk.copilot.reached_download_target import (NAV_TARGET_DOWNLOAD_KIND_KEY,classify_download_affordance,@@ -595,7 +596,7 @@ def _post_run_observed_url_goto_error(labels = ", ".join(block["label"] for block in offending[:5])return (- "Workflow validation failed: the draft is trying to persist a post-run browser URL as a new goto_url "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}the draft is trying to persist a post-run browser URL as a new goto_url ""block after an incomplete or budgeted run. That URL may encode record-specific, result-page, or ""session state. Keep the reusable entrypoint and verified upstream blocks, then either extract from ""the observed current page, split or replace the budgeted frontier into smaller reusable UI actions, "@@ -1071,14 +1072,14 @@ def composition_page_evidence_error()if string_step is not None:return (- "Workflow validation failed: a block_observation_refs entry uses observation_step "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}a block_observation_refs entry uses observation_step "f"{string_step!r} as a string. Pass the integer observation_step returned by ""inspect_page_for_composition or evaluate for click-reached blocks. "f"Offending blocks: {_format_page_block_findings([block])}")if _required_observation_ref_missing(block, block_observation_refs):return (- "Workflow validation failed: a click-reached block requires a block_observation_refs entry. "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}a click-reached block requires a block_observation_refs entry. ""Pass an interaction- or post_run-reached observation_step for click-reached blocks before ""composing them. "f"Offending blocks: {_format_page_block_findings([block])}"@@ -1086,7 +1087,7 @@ def composition_page_evidence_error(if wrong_reached_via is not None:step, reached_via = wrong_reached_viareturn (- "Workflow validation failed: a block references observation_step "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}a block references observation_step "f"{step}, but that observed page was reached via {reached_via!r}. ""Pass an interaction- or post_run-reached observation_step for click-reached blocks. "f"Offending blocks: {_format_page_block_findings([block])}"@@ -1099,14 +1100,14 @@ def composition_page_evidence_error(else "that observation step was not found in flow evidence")return (- "Workflow validation failed: a block references observation_step "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}a block references observation_step "f"{missing_step}, but {missing_reason}. ""Inspect or evaluate the reached page again and pass the new observation_step in ""block_observation_refs before composing page-dependent blocks. "f"Offending blocks: {_format_page_block_findings([block])}")return (- "Workflow validation failed: page-dependent build blocks need observed page evidence before they are "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}page-dependent build blocks need observed page evidence before they are "f"authored. Call inspect_page_for_composition(target_url={target_url!r}) before composing page-dependent ""blocks, or save only the initial goto_url block and inspect the reached page before the next mutation. "f"Offending blocks: {_format_page_block_findings([block])}"diff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.pyindex 9dc60b530..f10b9ceb0 100644--- a/skyvern/forge/sdk/copilot/narration.py+++ b/skyvern/forge/sdk/copilot/narration.py@@ -563,16 +563,21 @@ _TRANSITION_LABELS: dict[TransitionKind, str] = {_MAX_DETAILS_CHARS = 240-def extract_tool_details(tool_name: str, parsed: dict[str, Any]) -> str:+def extract_tool_details(tool_name: str, parsed: dict[str, Any], *, success: bool | None = None) -> str:"""Compact narrator-friendly excerpt from a tool's parsed payload.Intentionally narrow: counts, domains, and high-level statuses only.Raw labels (block names, field names, URL paths, page content) are excludedso they can't reach the narrator prompt and be echoed at the user.++ ``success`` lets a caller override the raw ``ok`` field (e.g. a precondition+ redirect that streaming_adapter has already reclassified as non-failure) so this+ detail line doesn't contradict the entry's own success status in the narrator prompt."""if not isinstance(parsed, dict):return ""- if not parsed.get("ok", True):+ ok = parsed.get("ok", True) if success is None else success+ if not ok:return "last action failed"data = parsed.get("data")diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex caa8b04ea..74e821052 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -134,6 +134,31 @@ _INTERNAL_TOOL_PARAPHRASE_PHRASES = ("nudge turn", "nudge-turn")_INTERNAL_COPILOT_SENTINEL_PREFIX = "[copilot:"_LOOP_DETECTED_MARKER = "loop detected"+# Copilot's own paraphrase for a formatting change it made — not standard YAML+# vocabulary, so a legitimate clarification is not going to phrase itself this way.+# Runs for every response type, including ASK_QUESTION.+_YAML_AUTHORING_JARGON_PHRASES = (+ "folded code formatting",+ "literal code formatting",+)+# Standard YAML block-scalar terminology (folded `>` vs literal `|`). A legitimate+# ASK_QUESTION can ask the user which style they want for a multi-line value, so these+# only count as a leak once the reply is final and user-visible.+_YAML_BLOCK_SCALAR_TERM_PHRASES = (+ "folded block scalar",+ "literal block scalar",+)+++def _contains_yaml_authoring_vocab_leak(user_response: str, response_type: str) -> bool:+ lower = user_response.lower()+ if any(phrase in lower for phrase in _YAML_AUTHORING_JARGON_PHRASES):+ return True+ return response_type in _USER_VISIBLE_REPLY_TYPES and any(+ phrase in lower for phrase in _YAML_BLOCK_SCALAR_TERM_PHRASES+ )++_SELF_PRESCRIPTIVE_FIXED_PHRASE = "send me a normal instruction like"_SELF_PRESCRIPTIVE_IMPERATIVE_VERBS = frozenset({"send", "reply", "type", "respond"})_SELF_PRESCRIPTIVE_POLITE_PREFIXES = frozenset({"please", "kindly", "just", "now"})@@ -669,6 +694,8 @@ def _contains_internal_block_taxonomy_leak(return Falseif _contains_deprecated_block_identifier(user_response):return True+ if _contains_yaml_authoring_vocab_leak(user_response, response_type):+ return Trueif response_type in _USER_VISIBLE_REPLY_TYPES and _contains_internal_tool_vocab_leak(user_response):return Trueif output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER:diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex fd52b53b7..37aa8eb97 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -425,6 +425,28 @@ def _structured_failure_summary_for_user(return None+# Blocker kinds where the tool was redirected before it ran (a precondition/authority+# gate), not a real failure. `tool_error` and `loop_detected` keep failure affect —+# something actually broke or the agent is stuck.+_NEUTRAL_REDIRECT_BLOCKER_KINDS = frozenset({"phase_gated", "missing_required_context", "authority_denied"})+++def user_facing_success(+ result: dict[str, Any],+ *,+ blocker_signal: CopilotToolBlockerSignal | Iterable[CopilotToolBlockerSignal] | None = None,+) -> bool:+ """Whether a tool result should render without failure affect in the user-facing+ activity stream. A raw ``ok=False`` still counts as success here when it's explained+ by a precondition/authority blocker signal — the agent was redirected, not broken."""+ if result.get("ok", True):+ return True+ return any(+ signal.blocker_kind in _NEUTRAL_REDIRECT_BLOCKER_KINDS and _blocker_signal_matches_result(signal, result)+ for signal in _iter_blocker_signals(blocker_signal)+ )++_HEADERS_BLOB_RE = re.compile(r"\s*headers:\s*\{[^{}]*\}\s*", re.IGNORECASE)_LARGE_DICT_BLOB_RE = re.compile(r"\{[^{}]{40,}\}")@@ -543,14 +565,26 @@ def summarize_tool_result_detail(*,tool_name: str | None = None,blocker_signal: CopilotToolBlockerSignal | Iterable[CopilotToolBlockerSignal] | None = None,+ success: bool | None = None,) -> str | None:- """Tooltip-grade failure detail (longer cap than ``summarize_tool_result``); None on success."""- if result.get("ok", False):+ """Tooltip-grade failure detail (longer cap than ``summarize_tool_result``); None on success.++ ``success`` lets a caller pass the already-reclassified value (e.g. a phase/authority+ redirect that ``user_facing_success`` upgraded from raw ``ok: false``) so this field+ doesn't contradict the row's own success flag — same override shape as+ ``narration.extract_tool_details``.+ """+ if result.get("ok", False) if success is None else success:return Nonestructured = _structured_failure_summary_for_user(result, blocker_signal=blocker_signal, blocked_tool=tool_name)if structured is not None:return structured- return _sanitize_failure_text(_extract_failure_message(result), max_chars=max_chars)+ failure_message = _extract_failure_message(result)+ # Same internal-validator convention _translate_failure_for_user maps to the generic+ # summary — the tooltip-grade detail must not leak the raw text either.+ if any(marker in failure_message.lower() for marker in _INTERNAL_VALIDATION_MARKERS):+ return _USER_FACING_GENERIC_FAILURE+ return _sanitize_failure_text(failure_message, max_chars=max_chars)_JINJA_ERROR_MARKERS: tuple[str, ...] = ("Failed to format jinja", "Jinja style parameter")@@ -565,6 +599,13 @@ _ENGINE_INSTRUCTION_MARKERS: tuple[str, ...] = ("waiting for locator",)_USE_TOOL_NAME_RE = re.compile(r"use the ['\"]?[a-z_][a-z0-9_]*['\"]? tool", re.IGNORECASE)+# Shared prefix for internal workflow-authoring validator rejects (stale block metadata,+# banned block types, missing observation evidence, raw YAML/pydantic errors). Validators+# import this constant rather than hand-typing the prefix, so a future validator can't+# silently bypass the leak-suppression below by phrasing its reject text differently.+# The full text is written for the agent to self-correct, never for the user.+INTERNAL_VALIDATION_FAILURE_PREFIX = "Workflow validation failed: "+_INTERNAL_VALIDATION_MARKERS: tuple[str, ...] = (INTERNAL_VALIDATION_FAILURE_PREFIX.strip(": ").lower(),)_USER_FACING_LOOP_MESSAGE = "The agent got stuck retrying the same step — moving on."_USER_FACING_JINJA_MESSAGE = "A workflow parameter could not be filled in."@@ -579,6 +620,8 @@ def _translate_failure_for_user(error_text: str) -> str:if any(marker in error_text for marker in _JINJA_ERROR_MARKERS):return _USER_FACING_JINJA_MESSAGElowered = error_text.lower()+ if any(marker in lowered for marker in _INTERNAL_VALIDATION_MARKERS):+ return _USER_FACING_GENERIC_FAILUREif any(marker in lowered for marker in _ENGINE_INSTRUCTION_MARKERS):return _USER_FACING_GENERIC_FAILUREif _USE_TOOL_NAME_RE.search(error_text):diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.pyindex e3cdda471..39538711c 100644--- a/skyvern/forge/sdk/copilot/streaming_adapter.py+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py@@ -33,7 +33,11 @@ from skyvern.forge.sdk.copilot.narration import (snapshot_ctx,tool_activity_display_label,)-from skyvern.forge.sdk.copilot.output_utils import format_tool_result_for_user, summarize_tool_result_detail+from skyvern.forge.sdk.copilot.output_utils import (+ format_tool_result_for_user,+ summarize_tool_result_detail,+ user_facing_success,+)from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.turn_halt import (CopilotTurnHalt,@@ -376,8 +380,10 @@ async def stream_to_sse(else:blocker_signals = _tool_blocker_signal_candidates(ctx)summary = format_tool_result_for_user(tool_name, parsed, blocker_signal=blocker_signals)- success = parsed.get("ok", True)- detail = summarize_tool_result_detail(parsed, tool_name=tool_name, blocker_signal=blocker_signals)+ success = user_facing_success(parsed, blocker_signal=blocker_signals)+ detail = summarize_tool_result_detail(+ parsed, tool_name=tool_name, blocker_signal=blocker_signals, success=success+ )narrator_state.record_activity(build_tool_result_activity(tool_name, summary, success, iteration, call_id))@@ -409,7 +415,7 @@ async def stream_to_sse(summary=summary,success=success,iteration=iteration,- details=extract_tool_details(tool_name, parsed),+ details=extract_tool_details(tool_name, parsed, success=success),)for transition in detect_transitions(ctx_before, ctx_after, tool_name, prior_tool_name):narrator_state.record_transition(transition)@@ -515,7 +521,7 @@ async def flush_goal_satisfied_tool_result(stream: EventSourceStream, ctx: Copilreturnblocker_signals = _tool_blocker_signal_candidates(ctx)summary = format_tool_result_for_user(pending.tool_name, parsed, blocker_signal=blocker_signals)- success = parsed.get("ok", True)+ success = user_facing_success(parsed, blocker_signal=blocker_signals)narrator_state = ctx.narrator_stateif narrator_state is not None:narrator_state.record_activity(@@ -531,7 +537,9 @@ async def flush_goal_satisfied_tool_result(stream: EventSourceStream, ctx: Copilsummary=summary,iteration=pending.iteration,tool_call_id=pending.call_id,- detail=summarize_tool_result_detail(parsed, tool_name=pending.tool_name, blocker_signal=blocker_signals),+ detail=summarize_tool_result_detail(+ parsed, tool_name=pending.tool_name, blocker_signal=blocker_signals, success=success+ ),workflow_run_id=_tool_result_workflow_run_id(pending.tool_name, parsed),))diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.pyindex 729ca6198..9ee722416 100644--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py@@ -8,6 +8,7 @@ from typing import Anyfrom skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_aliasfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policyfrom skyvern.forge.sdk.copilot.enforcement import PROBABLE_SITE_BLOCK_STREAK_STOP_AT+from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIXfrom skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span@@ -490,7 +491,7 @@ def _challenge_http_request_reject_message(return Nonelabels_text = ", ".join(sorted(set(labels)))return (- "Workflow validation failed: raw http_request blocks are not allowed for a page with observed "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}raw http_request blocks are not allowed for a page with observed ""anti-bot or human-verification challenge evidence. "f"Offending labels: [{labels_text}]. ""Use browser workflow blocks grounded in the observed page, include challenge handling only when visible, "@@ -506,7 +507,7 @@ def _timing_only_challenge_wait_reject_message(ctx: Any, submitted_yaml: str | Nreturn Nonelabels_text = ", ".join(sorted(set(labels)))return (- "Workflow validation failed: timing-only challenge wait blocks are not allowed after confirmed "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}timing-only challenge wait blocks are not allowed after confirmed ""anti-bot/WAF or repeated site-block evidence. "f"Offending labels: [{labels_text}]. ""Do not add wait/delay-only blocks for this blocker; use a conditional challenge check that takes a "diff --git a/skyvern/forge/sdk/copilot/tools/frontier.py b/skyvern/forge/sdk/copilot/tools/frontier.pyindex 4bcb604f0..7a7c33787 100644--- a/skyvern/forge/sdk/copilot/tools/frontier.py+++ b/skyvern/forge/sdk/copilot/tools/frontier.py@@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.failure_tracking import (_canonical_block_config,)+from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIXfrom skyvern.forge.sdk.copilot.runtime import (AgentContext,)@@ -264,7 +265,7 @@ def _stale_block_metadata_message(items: list[dict[str, Any]]) -> str:details.append(f"(and {len(items) - _STALE_BLOCK_METADATA_MESSAGE_LIMIT} more)")joined = "; ".join(details)return (- "Workflow validation failed: corrected block metadata still appears stale. "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}corrected block metadata still appears stale. ""When changing a user's requested subject, URL, or action, rename affected block labels and titles ""to match the revised goal, and update next_block_label, block_labels, and Jinja references accordingly. "f"Stale metadata: {joined}"@@ -961,7 +962,7 @@ def _frontier_run_size_error(suggested = labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS]remaining = labels_to_execute[_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:]return (- "Workflow validation failed: this browser test frontier is too long for a multi-stage "+ f"{INTERNAL_VALIDATION_FAILURE_PREFIX}this browser test frontier is too long for a multi-stage ""page-changing workflow. Keep the same complete workflow YAML, but shrink only the "f"block_labels argument to the next 1-2 unverified labels: {suggested!r}. ""If a prior run already advanced the browser, inspect that reached page "diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex b0b59ac61..6c7bb46a6 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -128,6 +128,7 @@ from skyvern.forge.sdk.copilot.output_policy import (output_policy_verdict_to_trace_data,url_origin,)+from skyvern.forge.sdk.copilot.output_utils import INTERNAL_VALIDATION_FAILURE_PREFIXfrom skyvern.forge.sdk.copilot.reached_download_target import (REGISTERED_DOWNLOAD_OUTPUT_KEYS,ReachedDownloadTarget,@@ -10215,7 +10216,7 @@ async def _update_workflow(code_rejected=True,)return reject(- error=f"Workflow validation failed: {e}",+ error=f"{INTERNAL_VALIDATION_FAILURE_PREFIX}{e}",user_facing_summary=user_facing_summary,data=repair_data,)diff --git a/tests/unit/test_copilot_narration.py b/tests/unit/test_copilot_narration.pyindex c4851d093..8c8bef1c6 100644--- a/tests/unit/test_copilot_narration.py+++ b/tests/unit/test_copilot_narration.py@@ -352,6 +352,23 @@ def test_extract_tool_details_failure_is_generic() -> None:assert "failed" in details.lower()+def test_extract_tool_details_success_override_suppresses_failed_line() -> None:+ """A caller-supplied success (e.g. a reclassified precondition redirect) must+ win over the raw ok:False, so the detail line doesn't contradict the entry's+ own success status in the narrator prompt."""+ details = narration.extract_tool_details(+ "evaluate",+ {"ok": False, "error": "internal steering text"},+ success=True,+ )+ assert details != "last action failed"+++def test_extract_tool_details_no_override_keeps_raw_ok() -> None:+ details = narration.extract_tool_details("evaluate", {"ok": False, "error": "boom"})+ assert details == "last action failed"++# ---------------------------------------------------------------------------# _build_narrator_prompt — redacts raw tool identifiers# ---------------------------------------------------------------------------diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 141cf3d25..58ff199ec 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.output_policy import (OutputPolicyReason,OutputPolicyVerdict,_contains_internal_tool_vocab_leak,+ _contains_yaml_authoring_vocab_leak,derive_output_kind,evaluate_actuation_obligation,evaluate_output_policy,@@ -2755,6 +2756,46 @@ def test_contains_internal_tool_vocab_leak_helper(user_response: str, expected:assert _contains_internal_tool_vocab_leak(user_response) is expected+ "user_response,response_type,expected",+ [+ ("I changed the code block from folded code formatting to literal code formatting.", "REPLY", True),+ ("Switched that step to use literal code formatting instead.", "REPLY", True),+ ("Rewrote the block using a literal block scalar.", "REPLY", True),+ ("Rewrote the block using a literal block scalar.", "REPLACE_WORKFLOW", True),+ ("Switched that step to use literal code formatting instead.", "ASK_QUESTION", True),+ ("Update the workflow to log in before extracting data.", "REPLY", False),+ ("I formatted the code so it's easier to read.", "REPLY", False),+ ("", "REPLY", False),+ ],+)+def test_contains_yaml_authoring_vocab_leak_helper(user_response: str, response_type: str, expected: bool) -> None:+ assert _contains_yaml_authoring_vocab_leak(user_response, response_type) is expected+++def test_yaml_authoring_vocab_leak_hard_blocks_ask_question() -> None:+ """Regression guard: unlike the nudge/copilot-sentinel phrases, this leak class must+ be caught in ASK_QUESTION turns too — a clarifying question is still user-visible."""+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ response_type="ASK_QUESTION",+ user_response="Should I switch this block to literal code formatting?",+ )+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in verdict.reason_codes+++def test_yaml_block_scalar_standard_term_does_not_hard_block_ask_question() -> None:+ """Regression guard for PR #13274 review feedback (LawyZheng): 'literal/folded block+ scalar' is standard YAML terminology a legitimate clarifying question can use — unlike+ the code-formatting jargon phrases, it must not hard-replace an ASK_QUESTION turn."""+ verdict = evaluate_output_policy(+ request_policy=_policy(),+ response_type="ASK_QUESTION",+ user_response="Should this multi-line value use a literal block scalar or folded style?",+ )+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK not in verdict.reason_codes++def test_translate_to_agent_result_prioritizes_unbacked_workflow_claim_over_taxonomy_rewrite() -> None:result = _fake_run_result({diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex 19051076b..faeebaa1b 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.output_utils import (summarize_tool_result,summarize_tool_result_detail,truncate_output,+ user_facing_success,)@@ -778,6 +779,59 @@ class TestFormatToolResultForUser:assert agent_summary == "Clicked 'xpath=//button[2]'"+class TestUserFacingSuccess:+ @staticmethod+ def _blocker(blocker_kind: str, *, steering: str = "internal steering text"):+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal++ return CopilotToolBlockerSignal(+ blocker_kind=blocker_kind, # type: ignore[arg-type]+ agent_steering_text=steering,+ user_facing_reason="I need more information before I can continue.",+ recovery_hint="ask_user_clarifying",+ internal_reason_code="test_reason_code",+ blocked_tool="evaluate",+ )++ def test_true_for_ok_result(self) -> None:+ assert user_facing_success({"ok": True, "data": {}}) is True++ def test_false_for_unclassified_failure(self) -> None:+ assert user_facing_success({"ok": False, "error": "plain failure"}) is False++ @pytest.mark.parametrize("blocker_kind", ["phase_gated", "missing_required_context", "authority_denied"])+ def test_true_for_precondition_style_blockers(self, blocker_kind: str) -> None:+ signal = self._blocker(blocker_kind)+ result = {"ok": False, "error": signal.agent_steering_text}+ assert user_facing_success(result, blocker_signal=signal) is True++ @pytest.mark.parametrize("blocker_kind", ["tool_error", "loop_detected"])+ def test_false_for_genuine_failure_blockers(self, blocker_kind: str) -> None:+ """Regression guard: real tool errors and loop-detection halts keep failure affect."""+ signal = self._blocker(blocker_kind)+ result = {"ok": False, "error": signal.agent_steering_text}+ assert user_facing_success(result, blocker_signal=signal) is False++ def test_false_when_blocker_signal_does_not_match_result(self) -> None:+ signal = self._blocker("phase_gated", steering="unrelated steering text")+ result = {"ok": False, "error": "a totally different failure"}+ assert user_facing_success(result, blocker_signal=signal) is False+++def test_format_tool_result_for_user_reframes_internal_validation_failure() -> None:+ """Pins the SKY-11971 forensic leak: an unclassified internal validator reject must+ never surface its raw agent-steering text (block labels, field names) to the user."""+ raw_error = (+ "Workflow validation failed: corrected block metadata still appears stale. "+ "When changing a user's requested subject, URL, or action, rename affected block "+ "labels and titles to match the revised goal. Stale metadata: extract_step: label mismatch"+ )+ summary = format_tool_result_for_user("update_workflow", {"ok": False, "error": raw_error})+ assert summary == "Couldn't complete that step."+ assert "stale" not in summary+ assert "block" not in summary.lower()++class TestParseFinalResponse:"""parse_final_response is the last mile between model output and the frontend.@@ -1025,8 +1079,32 @@ def test_summarize_tool_result_detail_returns_none_on_success() -> None:assert summarize_tool_result_detail({"ok": True, "data": {"block_count": 2}}) is None+def test_summarize_tool_result_detail_omits_detail_for_reclassified_neutral_redirect() -> None:+ """Regression guard (Codex, PR #13274): a phase/authority redirect reclassified to+ success=True by user_facing_success must not still carry a non-None `detail` — the+ schema documents `detail` as None on success, and this row renders without failure+ affect. Without passing the reclassified `success` through, the raw `ok: false`+ still drives a non-None structured detail here."""+ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal++ signal = CopilotToolBlockerSignal(+ blocker_kind="phase_gated",+ agent_steering_text="internal steering text",+ user_facing_reason="I need to know what page to inspect first.",+ recovery_hint="ask_user_clarifying",+ internal_reason_code="test_reason_code",+ blocked_tool="evaluate",+ )+ result = {"ok": False, "error": signal.agent_steering_text}+ reclassified_success = user_facing_success(result, blocker_signal=signal)+ assert reclassified_success is True++ assert summarize_tool_result_detail(result, blocker_signal=signal) is not None+ assert summarize_tool_result_detail(result, blocker_signal=signal, success=reclassified_success) is None++def test_summarize_tool_result_detail_caps_at_max_chars() -> None:- long_error = "Workflow validation failed: " + ("missing field 'foo'; " * 200)+ long_error = "Element lookup failed: " + ("missing field 'foo'; " * 200)detail = summarize_tool_result_detail({"ok": False, "error": long_error}, max_chars=400)assert detail is not Noneassert len(detail) <= 400@@ -1034,10 +1112,18 @@ def test_summarize_tool_result_detail_caps_at_max_chars() -> None:def test_summarize_tool_result_detail_preserves_short_full_message() -> None:+ detail = summarize_tool_result_detail(+ {"ok": False, "error": "Element lookup failed: title field required"},+ )+ assert detail == "Element lookup failed: title field required"+++def test_summarize_tool_result_detail_reframes_internal_validation_failure() -> None:+ """Tooltip-grade detail must not leak raw internal validator text either."""detail = summarize_tool_result_detail({"ok": False, "error": "Workflow validation failed: title field required"},)- assert detail == "Workflow validation failed: title field required"+ assert detail == "Couldn't complete that step."def test_summarize_tool_result_detail_strips_header_blobs() -> None:diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.pyindex b470bf4be..20b702070 100644--- a/tests/unit/test_copilot_streaming_adapter.py+++ b/tests/unit/test_copilot_streaming_adapter.py@@ -523,12 +523,80 @@ async def test_tool_result_sse_uses_latest_blocker_signal_for_activity_surface()assert tool_results[0].summary == signal.user_facing_reasonassert tool_results[0].detail == signal.user_facing_reasonassert "Do NOT" not in tool_results[0].summary+ # tool_error blocker signals are genuine failures (something ran out of budget/broke,+ # not a precondition redirect) and must keep their failure affect.+ assert tool_results[0].success is Falseactivity = ctx.narrator_state.design_activityassert len(activity) == 2assert activity[-1]["kind"] == "tool_result"assert activity[-1]["text"] == signal.user_facing_reasonassert "Do NOT" not in activity[-1]["text"]+ assert activity[-1]["success"] is False+++async def test_tool_result_sse_success_true_for_phase_gated_precondition() -> None:+ """A precondition redirect (agent tried a tool before its build phase allowed it)+ was never a real failure — it must not render with error affect."""+ from agents.items import RunItem+ from agents.stream_events import RunItemStreamEvent++ from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal+ from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageType++ signal = CopilotToolBlockerSignal(+ blocker_kind="phase_gated",+ agent_steering_text=(+ "Direct browser tools are not callable before composition. "+ "Call discover_workflow_entrypoint to resolve the entrypoint URL."+ ),+ user_facing_reason="I need to know what site to work on before I can browse there. What URL should I use?",+ recovery_hint="ask_user_clarifying",+ cleared_by_tools=frozenset({"discover_workflow_entrypoint"}),+ internal_reason_code="build_phase_browser_blocked_pre_compose",+ blocked_tool="evaluate",+ )++ call_item = MagicMock(spec=RunItem)+ call_item.raw_item = {"call_id": "c1", "name": "evaluate", "arguments": "{}"}+ tool_call_event = RunItemStreamEvent(name="tool_called", item=call_item)++ out_item = MagicMock(spec=RunItem)+ out_item.raw_item = {"call_id": "c1", "name": "evaluate"}+ out_item.output = [{"type": "text", "text": json.dumps({"ok": False, "error": signal.agent_steering_text})}]+ tool_output_event = RunItemStreamEvent(name="tool_output", item=out_item)++ result = MagicMock()+ result.stream_events = lambda: _stream_events_from(tool_call_event, tool_output_event)+ result.cancel = MagicMock()++ sent: list[Any] = []++ async def _send(payload: Any) -> bool:+ sent.append(payload)+ return True++ stream = MagicMock()+ stream.is_disconnected = AsyncMock(return_value=False)+ stream.send = _send+ ctx = SimpleNamespace(+ latest_tool_blocker_signal=signal,+ tool_blocker_signals=[signal],+ last_artifact_health_blocker_reason=None,+ completion_verification_result=None,+ )++ await stream_to_sse(result, stream, ctx)++ tool_results = [p for p in sent if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.TOOL_RESULT]+ assert len(tool_results) == 1+ assert tool_results[0].summary == signal.user_facing_reason+ assert tool_results[0].success is True++ activity = ctx.narrator_state.design_activity+ assert activity[-1]["kind"] == "tool_result"+ assert activity[-1]["success"] is True@pytest.mark.asyncio@@ -1234,9 +1302,7 @@ async def _capture_tool_result(tool_name: str, parsed_output: dict[str, Any]) ->@pytest.mark.asyncioasync def test_stream_emits_detail_for_failure() -> None:- long_error = "Workflow validation failed: " + (- "blocks.0.task expects 'navigation_goal' but the emitted YAML omitted it. " * 4- )+ long_error = "blocks.0.task expects 'navigation_goal' but the emitted YAML omitted it. " * 4payload = await _capture_tool_result("update_workflow",{"ok": False, "error": long_error},@@ -1249,6 +1315,21 @@ async def test_stream_emits_detail_for_failure() -> None:assert len(payload.detail) > len(payload.summary)+async def test_stream_emits_generic_detail_for_internal_validation_failure() -> None:+ """The tooltip-grade detail must not leak raw internal validator text either —+ only the visible summary was sanitized before this regression test was added."""+ long_error = "Workflow validation failed: " + (+ "blocks.0.task expects 'navigation_goal' but the emitted YAML omitted it. " * 4+ )+ payload = await _capture_tool_result(+ "update_workflow",+ {"ok": False, "error": long_error},+ )+ assert payload.detail == "Couldn't complete that step."+ assert "navigation_goal" not in payload.detail++@pytest.mark.asyncioasync def test_stream_emits_no_detail_for_success() -> None:payload = await _capture_tool_result(