Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9136): fail fast on non-retriable navigation errors in workflow copilot (#5563)
Andrew Neilson834db2abe630
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex c1ccc8333..5bf271ee5 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -281,6 +281,7 @@ async def run_copilot_agent(from skyvern.cli.mcp_tools import mcp as skyvern_mcpfrom skyvern.forge.sdk.copilot.enforcement import (+ CopilotNonRetriableNavError,CopilotTotalTimeoutError,run_with_enforcement,)@@ -416,6 +417,23 @@ async def run_copilot_agent(workflow_yaml=ctx.last_workflow_yaml,workflow_was_persisted=ctx.workflow_persisted,)+ except CopilotNonRetriableNavError as exc:+ LOG.warning(+ "Copilot run halted on non-retriable navigation error",+ url=exc.url,+ error_message=exc.error_message,+ organization_id=organization_id,+ )+ return AgentResult(+ user_response=(+ f"The target URL could not be reached. Error: {exc.error_message}. "+ "Please verify the URL and try again."+ ),+ updated_workflow=ctx.last_workflow,+ global_llm_context=global_llm_context,+ workflow_yaml=ctx.last_workflow_yaml,+ workflow_was_persisted=ctx.workflow_persisted,+ )except Exception as e:LOG.error("Copilot agent error", error=str(e), exc_info=True)return AgentResult(diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex d6a7b730c..11bbc3606 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -181,6 +181,16 @@ class CopilotContext(AgentContext):# streak is still at 2, and guarantees the stop nudge fires exactly once# when the streak reaches 3.repeated_failure_nudge_emitted_at_streak: int = 0+ # Set by _record_run_blocks_result when the most recent failed run matches+ # SKIP_INNER_NAV_RETRY_ERRORS (DNS / cert / SSL / invalid URL). Drives the+ # one-shot non-retriable-nav stop nudge and the deterministic exit-path+ # exception in run_with_enforcement. Cleared at the top of every call to+ # _record_run_blocks_result so stale state can't leak across runs.+ last_test_non_retriable_nav_error: str | None = None+ # Normalized signature of the non-retriable nav error last nudged on.+ # Lets the stop nudge re-fire if the user retries with a different bad URL+ # (different signature) in the same session. Cleared on meaningful success.+ non_retriable_nav_error_last_emitted_signature: str | None = Nonelast_failure_category_top: str | None = None# Hash of the ordered (action_type, element_id) tuples from the last run's# action trace. When the same fingerprint repeats run-over-run with nodiff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 9b23e099f..fd69c76d6 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Anyimport structlogfrom agents.run import Runner+from skyvern.forge.sdk.copilot.failure_tracking import normalize_failure_reasonfrom skyvern.forge.sdk.copilot.output_utils import extract_final_text, parse_final_responsefrom skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntryfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span@@ -195,6 +196,14 @@ POST_PARAMETER_BINDING_WARN_NUDGE = ("`parameters` argument.")+POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE = (+ "STOP — the target URL is unreachable and further retries cannot succeed. "+ "The navigation failed with a permanent error (DNS resolution, SSL/cert, "+ "or invalid URL). Do NOT retry and do NOT edit the workflow. Reply to the "+ "user now: state that the URL could not be reached, quote the exact error "+ "message from the last failure_reason, and ask them to verify the URL."+)+POST_PARAMETER_BINDING_STOP_NUDGE = ("STOP — you have retried the same PARAMETER_BINDING_ERROR multiple times ""without reconciling the workflow configuration. Do NOT call "@@ -250,6 +259,41 @@ class CopilotTotalTimeoutError(Exception):"""Raised when the copilot agent exceeds the total allowed runtime."""+class CopilotNonRetriableNavError(Exception):+ """Raised from run_with_enforcement when the copilot's most recent run+ hit a permanent navigation error (DNS / cert / SSL / invalid URL) and+ the loop is about to exit without a successful test. Caught at the+ agent entrypoint and translated to a deterministic user-facing failure,+ mirroring the CopilotTotalTimeoutError handling pattern."""++ def __init__(self, url: str | None, error_message: str) -> None:+ self.url = url+ self.error_message = error_message+ super().__init__(f"Non-retriable navigation error: {error_message}")+++_FAILED_TO_NAVIGATE_URL_PATTERN = re.compile(r"Failed to navigate to url (\S+)\. Error message:")+++def _extract_url_from_nav_error(message: str) -> str | None:+ """Pull the URL out of a FailedToNavigateToUrl string. None on no match."""+ match = _FAILED_TO_NAVIGATE_URL_PATTERN.search(message)+ return match.group(1) if match else None+++def _maybe_raise_non_retriable_nav(ctx: Any) -> None:+ """Raise CopilotNonRetriableNavError if the most recent run was a+ permanent navigation failure and nothing else has succeeded. Called+ before both `return result` sites in run_with_enforcement so the loop+ cannot hand a failed run back to the caller as if it completed."""+ err = getattr(ctx, "last_test_non_retriable_nav_error", None)+ if not isinstance(err, str) or not err:+ return+ if getattr(ctx, "last_test_ok", None) is True:+ return+ raise CopilotNonRetriableNavError(url=_extract_url_from_nav_error(err), error_message=err)++_ACTION_CATEGORIES: list[list[str]] = [["navigate", "go to", "open", "visit"],["download", "save", "export"],@@ -362,6 +406,10 @@ def _needs_explore_without_workflow_nudge(ctx: Any) -> bool:def _needs_failed_test_nudge(ctx: Any) -> bool:"""Return True when the last test failed and the agent hasn't iterated yet."""+ # A permanent nav error cannot be 'fix the workflow and retry' material —+ # the dedicated non-retriable branch in _check_enforcement owns this case.+ if getattr(ctx, "last_test_non_retriable_nav_error", None):+ return Falseif getattr(ctx, "last_test_ok", None) is not False:return Falseif not getattr(ctx, "test_after_update_done", False):@@ -372,11 +420,18 @@ def _needs_failed_test_nudge(ctx: Any) -> bool:def _needs_suspicious_success_nudge(ctx: Any) -> bool:"""Return True when the last test 'completed' but data blocks had no output."""+ # A non-retriable nav failure cannot be "suspiciously successful" — defer+ # to the dedicated stop path rather than competing for the nudge slot.+ if getattr(ctx, "last_test_non_retriable_nav_error", None):+ return Falsereturn bool(getattr(ctx, "last_test_suspicious_success", False))def _needs_repeated_null_data_nudge(ctx: Any) -> bool:"""Return True when suspicious-success has happened enough times to escalate."""+ # Same as above: non-retriable nav state never belongs on this branch.+ if getattr(ctx, "last_test_non_retriable_nav_error", None):+ return Falseif not getattr(ctx, "last_test_suspicious_success", False):return Falsestreak = getattr(ctx, "null_data_streak_count", 0)@@ -393,6 +448,10 @@ def _repeated_frontier_failure_nudge(ctx: Any) -> str | None:keeps climbing on further identical failures (incremented elsewhere byupdate_repeated_failure_state), so the stop nudge fires naturally on thenext repeat after a warn."""+ # Non-retriable nav errors get their own dedicated stop path; don't let a+ # repeated-frontier nudge smuggle different retry advice past the gate.+ if getattr(ctx, "last_test_non_retriable_nav_error", None):+ return Nonestreak = _get_int(ctx, "repeated_failure_streak_count")emitted = _get_int(ctx, "repeated_failure_nudge_emitted_at_streak")top_category = getattr(ctx, "last_failure_category_top", None)@@ -408,7 +467,33 @@ def _repeated_frontier_failure_nudge(ctx: Any) -> str | None:_STOP_LEVEL_FRONTIER_NUDGES = frozenset({POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE, POST_PARAMETER_BINDING_STOP_NUDGE})+def _non_retriable_nav_error_nudge(ctx: Any) -> tuple[str, str] | None:+ """Emit POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE at most once per distinct+ non-retriable nav-error signature. Returns ``(nudge, signature)`` when it+ should fire, ``None`` otherwise. Signature normalization is shared with+ `failure_tracking.compute_failure_signature`, so a cert error after a DNS+ error (or vice versa) counts as a distinct signature and re-fires."""+ raw = getattr(ctx, "last_test_non_retriable_nav_error", None)+ if not isinstance(raw, str) or not raw:+ return None+ signature = normalize_failure_reason(raw)+ last_emitted = getattr(ctx, "non_retriable_nav_error_last_emitted_signature", None)+ if signature == last_emitted:+ return None+ return POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE, signature++def _check_enforcement(ctx: Any, result: RunResultStreaming | None = None) -> str | None:+ # Terminal failure-mode signals must pre-empt tool-call hygiene nudges.+ # A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be+ # resolved by observing a prior navigate or by testing an updated+ # workflow against the same bad URL, so let it speak first.+ non_retriable = _non_retriable_nav_error_nudge(ctx)+ if non_retriable is not None:+ nudge_msg, signature = non_retriable+ ctx.non_retriable_nav_error_last_emitted_signature = signature+ return nudge_msg+if ctx.navigate_called and not ctx.observation_after_navigate and not ctx.navigate_enforcement_done:ctx.navigate_enforcement_done = Truereturn POST_NAVIGATE_NUDGE@@ -734,6 +819,7 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {POST_REPEATED_NULL_DATA_NUDGE: "repeated_null_data",POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE: "repeated_frontier_failure_warn",POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE: "repeated_frontier_failure_stop",+ POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE: "non_retriable_nav_error_stop",POST_PARAMETER_BINDING_WARN_NUDGE: "parameter_binding_warn",POST_PARAMETER_BINDING_STOP_NUDGE: "parameter_binding_stop",POST_ANTI_BOT_FAILED_TEST_NUDGE: "anti_bot_block",@@ -940,6 +1026,7 @@ async def run_with_enforcement(else:nudge = _check_enforcement(ctx, result)if nudge is None:+ _maybe_raise_non_retriable_nav(ctx)return resultif nudge == POST_UPDATE_NUDGE:@@ -948,6 +1035,7 @@ async def run_with_enforcement("Enforcement exhausted post-update nudges, allowing response",nudge_count=ctx.post_update_nudge_count,)+ _maybe_raise_non_retriable_nav(ctx)return resultctx.post_update_nudge_count += 1diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.pyindex 8dec25667..4da5eef7e 100644--- a/skyvern/forge/sdk/copilot/failure_tracking.py+++ b/skyvern/forge/sdk/copilot/failure_tracking.py@@ -31,7 +31,7 @@ if TYPE_CHECKING:_FAILURE_REASON_MAX_CHARS = 200-def _normalize_failure_reason(raw: str | None) -> str:+def normalize_failure_reason(raw: str | None) -> str:if not raw:return ""collapsed = " ".join(raw.split())@@ -61,7 +61,7 @@ def compute_failure_signature(run (status=completed but data-producing blocks produced no output) stillgenerates a signature so repeated no-data runs can be counted as repeats."""- normalized = _normalize_failure_reason(failure_reason)+ normalized = normalize_failure_reason(failure_reason)has_signal = bool(normalized) or suspicious_successif not has_signal:return Nonediff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 333a64e55..150c7f7ba 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport base64import binasciiimport json+from collections.abc import Iteratorfrom typing import TYPE_CHECKING, Anyif TYPE_CHECKING:@@ -143,10 +144,40 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dictreturn sanitized+def iter_failure_reasons(result: dict[str, Any]) -> Iterator[str]:+ """Yield non-empty failure_reason strings from a copilot tool result:+ run-level ``data.failure_reason`` first, then each block's ``failure_reason``+ in order. Callers that only need the first match should wrap with ``next``."""+ data = result.get("data") if isinstance(result, dict) else None+ if not isinstance(data, dict):+ return+ run_level = data.get("failure_reason")+ if isinstance(run_level, str) and run_level:+ yield run_level+ blocks = data.get("blocks")+ if isinstance(blocks, list):+ for block in blocks:+ if not isinstance(block, dict):+ continue+ reason = block.get("failure_reason")+ if isinstance(reason, str) and reason:+ yield reason+++def _extract_failure_message(result: dict[str, Any]) -> str:+ """Prefer top-level ``error`` over nested failure_reason fields. Defense+ in depth: _run_blocks_and_collect_debug now populates ``error`` on+ failure, but other tool return shapes may still omit it."""+ top = result.get("error")+ if isinstance(top, str) and top:+ return top+ return next(iter_failure_reasons(result), "Unknown error")++def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:"""Create a brief human-readable summary of a tool result."""if not result.get("ok", False):- return f"Failed: {result.get('error', 'Unknown error')[:200]}"+ return f"Failed: {_extract_failure_message(result)[:200]}"data = result.get("data") or {}diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 5b5b5a09b..49bd2bbff 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -85,6 +85,8 @@ class AgentContext:last_failed_workflow_yaml: str | None = Nonerepeated_failure_streak_count: int = 0repeated_failure_nudge_emitted_at_streak: int = 0+ last_test_non_retriable_nav_error: str | None = None+ non_retriable_nav_error_last_emitted_signature: str | None = Noneworkflow_persisted: bool = Falselast_workflow: Any | None = Nonelast_workflow_yaml: str | None = Nonediff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex fafa04f19..aae464fb4 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import (from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loopfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlayfrom skyvern.forge.sdk.copilot.output_utils import (+ iter_failure_reasons,sanitize_tool_result_for_llm,truncate_output,)@@ -41,6 +42,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (WorkflowParameterType,)from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRunStatus+from skyvern.webeye.navigation import is_skip_inner_retry_errorLOG = structlog.get_logger()@@ -249,6 +251,23 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str) -> str | None:"detecting. Do NOT retry this tool — conclude the workflow is ""not automatable as-is and report back to the user.")++ # Within-turn fail-fast for permanent navigation errors (DNS / cert /+ # SSL / invalid URL). The enforcement-loop stop nudge only runs+ # BETWEEN agent turns, so without this check the LLM is free to make+ # speculative within-turn retries (e.g. drop the `www` subdomain and+ # try again) before the nudge fires. update_and_run_blocks internally+ # calls _update_workflow which clears the flag, so this check must+ # run before that — hence at the tool entrypoint, not inside the run+ # body.+ prior_nav_error = getattr(ctx, "last_test_non_retriable_nav_error", None)+ if isinstance(prior_nav_error, str) and prior_nav_error:+ return (+ f"Prior run in this turn hit a permanent navigation error "+ f"({prior_nav_error[:200]}). Do NOT retry — the URL is unreachable "+ "regardless of subdomain or path variations. Reply to the user "+ "explaining the failure and asking them to verify the URL."+ )return None@@ -978,10 +997,15 @@ async def _run_blocks_and_collect_debug(existing_set.add(label)ctx.verified_prefix_labels = existing_prefix- return {- "ok": run_ok,- "data": result_data,- }+ response: dict[str, Any] = {"ok": run_ok, "data": result_data}+ if not run_ok:+ # Promote the real failure message into a top-level ``error`` field so+ # every downstream consumer (summarize_tool_result, SSE frames, logs,+ # LLM context) sees it instead of defaulting to "Unknown error".+ # Wording matches ExtractionBlock's "Unknown error (no failure reason+ # provided)" fallback so error-message matching stays consistent.+ response["error"] = next(iter_failure_reasons(response), "Unknown error (no failure reason provided)")+ return responseasync def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:@@ -1383,6 +1407,12 @@ def _record_workflow_update_result(copilot_ctx: Any, result: dict[str, Any]) ->copilot_ctx.last_update_block_count = block_countcopilot_ctx.last_test_ok = Nonecopilot_ctx.last_test_failure_reason = None+ # A fresh workflow edit invalidates the prior test's failure state —+ # otherwise an exhausted POST_UPDATE_NUDGE on the new draft would raise+ # CopilotNonRetriableNavError with the old run's error, telling the user+ # to "verify the URL" for a URL they just corrected in the new draft.+ copilot_ctx.last_test_non_retriable_nav_error = None+ copilot_ctx.non_retriable_nav_error_last_emitted_signature = Nonecopilot_ctx.workflow_persisted = True@@ -1443,6 +1473,15 @@ def _analyze_run_blocks(result: dict[str, Any]) -> tuple[str | None, bool, list[return anti_bot_match, empty_data_blocks, categories+def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:+ """Return the first failure_reason that matches SKIP_INNER_NAV_RETRY_ERRORS+ (DNS / cert / SSL / invalid URL), preferring run-level over block-level.+ Same set is_skip_inner_retry_error uses at the browser layer, so the copilot+ classifies on exactly the patterns that already short-circuit retries in+ navigate_with_retry (skyvern/webeye/navigation.py)."""+ return next((reason for reason in iter_failure_reasons(result) if is_skip_inner_retry_error(reason)), None)++def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:run_ok = bool(result.get("ok", False))copilot_ctx.last_test_ok = run_ok@@ -1450,6 +1489,7 @@ def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:copilot_ctx.last_test_suspicious_success = Falsecopilot_ctx.last_test_anti_bot = Nonecopilot_ctx.last_failure_category_top = None+ copilot_ctx.last_test_non_retriable_nav_error = Noneanti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result)if anti_bot_match:@@ -1483,10 +1523,14 @@ def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:copilot_ctx.failed_test_nudge_count = 0copilot_ctx.null_data_streak_count = 0copilot_ctx.last_failed_workflow_yaml = None+ # Real success: clear the signature latch so a subsequent bad URL in+ # the same session can re-fire the stop nudge.+ copilot_ctx.non_retriable_nav_error_last_emitted_signature = Noneupdate_repeated_failure_state(copilot_ctx, result)returncopilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)+ copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)data = result.get("data")if isinstance(data, dict):diff --git a/tests/unit/test_copilot_non_retriable_nav.py b/tests/unit/test_copilot_non_retriable_nav.pynew file mode 100644index 000000000..d79b8e419--- /dev/null+++ b/tests/unit/test_copilot_non_retriable_nav.py@@ -0,0 +1,584 @@+"""Tests for non-retriable navigation error handling in the copilot layer.++Covers SKY-9136: when the browser layer raises ``FailedToNavigateToUrl`` with+a DNS / cert / SSL / invalid-URL pattern (``SKIP_INNER_NAV_RETRY_ERRORS``),+the copilot must surface the real error instead of "Unknown error", must not+keep retrying, and must fail deterministically even if the model tries to+narrate a completion.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest++from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.enforcement import (+ POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE,+ CopilotNonRetriableNavError,+ _check_enforcement,+ _extract_url_from_nav_error,+ _maybe_raise_non_retriable_nav,+ _needs_failed_test_nudge,+ _needs_repeated_null_data_nudge,+ _needs_suspicious_success_nudge,+ _non_retriable_nav_error_nudge,+ _repeated_frontier_failure_nudge,+)+from skyvern.forge.sdk.copilot.tools import (+ _detect_non_retriable_nav_error,+ _record_run_blocks_result,+ _record_workflow_update_result,+ _tool_loop_error,+)++_DNS_FAILURE_REASON = (+ "Failed to navigate to url https://www.example.invalid/path. Error message: net::ERR_NAME_NOT_RESOLVED"+)+_CERT_FAILURE_REASON = "Failed to navigate to url https://expired.example. Error message: net::ERR_CERT_DATE_INVALID"+_GENERIC_FAILURE_REASON = "Timeout waiting for element #submit"+++def _fresh_context() -> CopilotContext:+ return CopilotContext(+ organization_id="o",+ workflow_id="w",+ workflow_permanent_id="wp",+ workflow_yaml="",+ browser_session_id=None,+ stream=SimpleNamespace(), # type: ignore[arg-type]+ )+++# ---------------------------------------------------------------------------+# _detect_non_retriable_nav_error+# ---------------------------------------------------------------------------+++def test_detect_matches_dns_error_in_block_failure_reason() -> None:+ result = {"ok": False, "data": {"blocks": [{"failure_reason": _DNS_FAILURE_REASON}]}}+ assert _detect_non_retriable_nav_error(result) == _DNS_FAILURE_REASON+++def test_detect_matches_cert_error_in_run_level_failure_reason() -> None:+ result = {"ok": False, "data": {"failure_reason": _CERT_FAILURE_REASON, "blocks": []}}+ assert _detect_non_retriable_nav_error(result) == _CERT_FAILURE_REASON+++def test_detect_matches_invalid_url_error() -> None:+ invalid_url = "Failed to navigate to url not-a-url. Error message: net::ERR_INVALID_URL"+ result = {"ok": False, "data": {"blocks": [{"failure_reason": invalid_url}]}}+ assert _detect_non_retriable_nav_error(result) == invalid_url+++def test_detect_matches_name_resolution_failed() -> None:+ reason = "net::ERR_NAME_RESOLUTION_FAILED happened mid-flight"+ result = {"ok": False, "data": {"blocks": [{"failure_reason": reason}]}}+ assert _detect_non_retriable_nav_error(result) == reason+++def test_detect_matches_ssl_error() -> None:+ reason = "SSL error: net::ERR_SSL_PROTOCOL_ERROR"+ result = {"ok": False, "data": {"blocks": [{"failure_reason": reason}]}}+ assert _detect_non_retriable_nav_error(result) == reason+++def test_detect_returns_none_for_generic_failure() -> None:+ result = {"ok": False, "data": {"blocks": [{"failure_reason": _GENERIC_FAILURE_REASON}]}}+ assert _detect_non_retriable_nav_error(result) is None+++def test_detect_returns_none_for_missing_data() -> None:+ assert _detect_non_retriable_nav_error({"ok": False}) is None+++def test_detect_returns_none_for_empty_blocks() -> None:+ assert _detect_non_retriable_nav_error({"ok": False, "data": {"blocks": []}}) is None+++def test_detect_prefers_run_level_over_block_level() -> None:+ # When both match, the run-level reason wins (it is the authoritative+ # aggregate failure the workflow service recorded).+ result = {+ "ok": False,+ "data": {+ "failure_reason": _DNS_FAILURE_REASON,+ "blocks": [{"failure_reason": _CERT_FAILURE_REASON}],+ },+ }+ assert _detect_non_retriable_nav_error(result) == _DNS_FAILURE_REASON+++# ---------------------------------------------------------------------------+# _record_run_blocks_result — context flag plumbing+# ---------------------------------------------------------------------------+++def test_record_sets_flag_on_dns_failure() -> None:+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ _record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {"blocks": [{"failure_reason": _DNS_FAILURE_REASON}]},+ },+ )+ assert ctx.last_test_non_retriable_nav_error == _DNS_FAILURE_REASON+ assert ctx.last_test_ok is False+++def test_record_leaves_flag_none_for_generic_failure() -> None:+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ _record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {"blocks": [{"failure_reason": _GENERIC_FAILURE_REASON}]},+ },+ )+ assert ctx.last_test_non_retriable_nav_error is None+++def test_record_clears_flag_on_every_call() -> None:+ # Stale state from a prior run must not leak into the next run's context.+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.last_test_non_retriable_nav_error = "stale DNS error"+ _record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {"blocks": [{"failure_reason": _GENERIC_FAILURE_REASON}]},+ },+ )+ assert ctx.last_test_non_retriable_nav_error is None+++def test_workflow_update_clears_non_retriable_flag_and_signature_latch() -> None:+ # Codex review P2-2: after a DNS-failed run, if the agent edits the+ # workflow (e.g. fixing the URL), the stale flag must not survive the+ # edit — otherwise an exhausted POST_UPDATE_NUDGE on the new draft+ # would raise CopilotNonRetriableNavError with the OLD run's error+ # message, telling the user to verify a URL they just corrected.+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ ctx.non_retriable_nav_error_last_emitted_signature = "dns_signature_123"+ ctx.last_test_ok = False+ ctx.workflow_yaml = "updated yaml"++ _record_workflow_update_result(+ ctx,+ {+ "ok": True,+ "data": {"block_count": 2},+ "_workflow": SimpleNamespace(workflow_id="wf_new"),+ },+ )+ assert ctx.last_test_non_retriable_nav_error is None+ assert ctx.non_retriable_nav_error_last_emitted_signature is None+ # Consistency check: the other per-test fields are also reset (pre-existing behavior).+ assert ctx.last_test_ok is None+ assert ctx.last_test_failure_reason is None+++def test_workflow_update_does_not_clear_flag_on_failed_update() -> None:+ # Only a SUCCESSFUL update invalidates prior test state — a failed+ # validation attempt leaves the existing flags alone.+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ ctx.non_retriable_nav_error_last_emitted_signature = "dns_signature_123"+ ctx.last_test_ok = False++ _record_workflow_update_result(+ ctx,+ {"ok": False, "error": "validation failed"},+ )+ assert ctx.last_test_non_retriable_nav_error == _DNS_FAILURE_REASON+ assert ctx.non_retriable_nav_error_last_emitted_signature == "dns_signature_123"+++def test_record_clears_signature_latch_on_meaningful_success() -> None:+ # After a real success, the stop nudge must be able to re-fire if a new+ # bad URL fails later in the same session.+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.non_retriable_nav_error_last_emitted_signature = "some previous signature"+ _record_run_blocks_result(+ ctx,+ {+ "ok": True,+ "data": {+ "blocks": [+ {+ "label": "extract",+ "block_type": "extraction",+ "status": "completed",+ "extracted_data": [{"name": "widget", "price": 1.0}],+ }+ ],+ },+ },+ )+ assert ctx.non_retriable_nav_error_last_emitted_signature is None+++# ---------------------------------------------------------------------------+# _non_retriable_nav_error_nudge — signature latch+# ---------------------------------------------------------------------------+++def test_nudge_helper_fires_first_time() -> None:+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ result = _non_retriable_nav_error_nudge(ctx)+ assert result is not None+ nudge, signature = result+ assert nudge == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE+ assert signature # non-empty+++def test_nudge_helper_does_not_refire_same_signature() -> None:+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ first = _non_retriable_nav_error_nudge(ctx)+ assert first is not None+ ctx.non_retriable_nav_error_last_emitted_signature = first[1]+ assert _non_retriable_nav_error_nudge(ctx) is None+++def test_nudge_helper_refires_on_different_signature() -> None:+ # User corrects from one bad URL to another bad URL in the same session+ # without a successful run in between; the stop nudge must re-fire.+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ first = _non_retriable_nav_error_nudge(ctx)+ assert first is not None+ ctx.non_retriable_nav_error_last_emitted_signature = first[1]++ ctx.last_test_non_retriable_nav_error = _CERT_FAILURE_REASON+ second = _non_retriable_nav_error_nudge(ctx)+ assert second is not None+ assert second[1] != first[1]+++def test_nudge_helper_returns_none_when_flag_unset() -> None:+ ctx = _fresh_context()+ assert _non_retriable_nav_error_nudge(ctx) is None+++# ---------------------------------------------------------------------------+# _check_enforcement — branch ordering and latching+# ---------------------------------------------------------------------------+++def test_check_enforcement_returns_non_retriable_nudge_before_failed_test() -> None:+ # Conditions that would normally trigger POST_FAILED_TEST_NUDGE (last_test+ # failed, test_after_update_done=True) must yield the non-retriable stop+ # nudge instead when the flag is set.+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.last_test_ok = False+ ctx.last_test_failure_reason = _DNS_FAILURE_REASON+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ nudge = _check_enforcement(ctx)+ assert nudge == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE+ # Emission latched for future calls with same signature.+ assert ctx.non_retriable_nav_error_last_emitted_signature+++def test_check_enforcement_one_shot_per_signature() -> None:+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.last_test_ok = False+ ctx.last_test_failure_reason = _DNS_FAILURE_REASON+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ first = _check_enforcement(ctx)+ assert first == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE+ # Same signature on next iteration: nudge returns None (and does NOT+ # fall through to POST_FAILED_TEST_NUDGE thanks to competing-branch+ # suppression).+ assert _check_enforcement(ctx) is None+++def test_check_enforcement_pre_empts_post_navigate_nudge() -> None:+ # Codex review P2-1: a navigate-hygiene state (navigate_called=True,+ # observation_after_navigate=False) must not steal the nudge slot when+ # a non-retriable nav error is also present — POST_NAVIGATE_NUDGE tells+ # the agent to observe the page, which does not resolve a DNS failure+ # and merely delays the deterministic stop by at least one iteration.+ ctx = _fresh_context()+ ctx.navigate_called = True+ ctx.observation_after_navigate = False+ ctx.navigate_enforcement_done = False+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ assert _check_enforcement(ctx) == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE+++def test_check_enforcement_pre_empts_post_update_nudge() -> None:+ # Similar P2-1 case: an update-without-test state must not pre-empt the+ # terminal stop. In practice P2-2 clears the flag on update, but if a+ # race leaves the flag set, stop-nudge still wins.+ ctx = _fresh_context()+ ctx.update_workflow_called = True+ ctx.test_after_update_done = False+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ assert _check_enforcement(ctx) == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE+++# ---------------------------------------------------------------------------+# Competing-branch suppression — each failure helper+# ---------------------------------------------------------------------------+++def test_needs_failed_test_nudge_suppressed_when_flag_set() -> None:+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.last_test_ok = False+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ assert _needs_failed_test_nudge(ctx) is False+++def test_needs_failed_test_nudge_still_fires_without_flag() -> None:+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.last_test_ok = False+ assert _needs_failed_test_nudge(ctx) is True+++def test_needs_suspicious_success_nudge_suppressed_when_flag_set() -> None:+ ctx = _fresh_context()+ ctx.last_test_suspicious_success = True+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ assert _needs_suspicious_success_nudge(ctx) is False+++def test_needs_repeated_null_data_nudge_suppressed_when_flag_set() -> None:+ ctx = _fresh_context()+ ctx.last_test_suspicious_success = True+ ctx.null_data_streak_count = 5+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ assert _needs_repeated_null_data_nudge(ctx) is False+++def test_repeated_frontier_failure_nudge_suppressed_when_flag_set() -> None:+ ctx = _fresh_context()+ ctx.repeated_failure_streak_count = 5+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ assert _repeated_frontier_failure_nudge(ctx) is None+++# ---------------------------------------------------------------------------+# _extract_url_from_nav_error+# ---------------------------------------------------------------------------+++def test_extract_url_parses_standard_format() -> None:+ url = _extract_url_from_nav_error(_DNS_FAILURE_REASON)+ assert url == "https://www.example.invalid/path"+++def test_extract_url_returns_none_on_malformed_message() -> None:+ assert _extract_url_from_nav_error("some unrelated error text") is None+++# ---------------------------------------------------------------------------+# _maybe_raise_non_retriable_nav — deterministic exit-path+# ---------------------------------------------------------------------------+++def test_maybe_raise_noops_when_flag_unset() -> None:+ ctx = _fresh_context()+ _maybe_raise_non_retriable_nav(ctx) # must not raise+++def test_maybe_raise_noops_when_last_test_is_ok() -> None:+ # A prior successful run does NOT suppress the exception (per CORR-3),+ # but the MOST RECENT run being a real success does — because that+ # means this turn's test did not hit the non-retriable path.+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ ctx.last_test_ok = True+ _maybe_raise_non_retriable_nav(ctx) # must not raise+++def test_maybe_raise_raises_when_flag_and_last_test_failed() -> None:+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ ctx.last_test_ok = False+ with pytest.raises(CopilotNonRetriableNavError) as excinfo:+ _maybe_raise_non_retriable_nav(ctx)+ assert excinfo.value.error_message == _DNS_FAILURE_REASON+ assert excinfo.value.url == "https://www.example.invalid/path"+++def test_maybe_raise_raises_when_last_test_ok_is_none() -> None:+ # The guard is ``last_test_ok is not True`` (not ``is False``), so an+ # ambiguous None (e.g. a suspicious-success run) with the flag set still+ # surfaces the cached nav failure rather than letting the loop return+ # silently. Locks in the tri-state semantics.+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ ctx.last_test_ok = None+ with pytest.raises(CopilotNonRetriableNavError):+ _maybe_raise_non_retriable_nav(ctx)+++def test_maybe_raise_raises_when_prior_run_succeeded_but_current_failed() -> None:+ # Codex CORR-3: the guard must NOT be gated on session history. A fresh+ # non-retriable nav failure on the most recent run still raises, even if+ # an earlier run in the same session succeeded.+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ ctx.last_test_ok = False # most recent run+ # Simulate a prior successful run in the session — there is no+ # `any_test_ok_ever` flag; the helper only inspects current state.+ with pytest.raises(CopilotNonRetriableNavError):+ _maybe_raise_non_retriable_nav(ctx)+++# ---------------------------------------------------------------------------+# Sanity: exception carries the expected attributes for the agent handler+# ---------------------------------------------------------------------------+++def test_exception_carries_url_and_error_message() -> None:+ exc = CopilotNonRetriableNavError(url="https://x.test", error_message="some reason")+ assert exc.url == "https://x.test"+ assert exc.error_message == "some reason"+ assert "some reason" in str(exc)+++# ---------------------------------------------------------------------------+# Sanity: when flag is set, no failure-nudge branch competes+# ---------------------------------------------------------------------------+++def test_all_competing_branches_silent_after_latch() -> None:+ # Reproduce the full steady state after the one-shot stop nudge has+ # latched: ctx has a non-retriable error, last_test_ok=False,+ # test_after_update_done=True, and all counters are set such that the+ # other branches would normally fire. Assert all four helpers are silent.+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.last_test_ok = False+ ctx.last_test_suspicious_success = True+ ctx.null_data_streak_count = 5+ ctx.repeated_failure_streak_count = 5+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON++ assert _needs_failed_test_nudge(ctx) is False+ assert _needs_suspicious_success_nudge(ctx) is False+ assert _needs_repeated_null_data_nudge(ctx) is False+ assert _repeated_frontier_failure_nudge(ctx) is None+++def test_without_flag_competing_branches_still_active() -> None:+ # Inverse: same setup but without the flag — all relevant branches fire.+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ ctx.last_test_ok = False+ assert _needs_failed_test_nudge(ctx) is True+++# ---------------------------------------------------------------------------+# Integration-ish: record -> check -> exception flow+# ---------------------------------------------------------------------------+++def test_full_flow_record_then_check_then_raise() -> None:+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ # Simulate a failed run with a DNS error.+ _record_run_blocks_result(+ ctx,+ {+ "ok": False,+ "data": {"blocks": [{"failure_reason": _DNS_FAILURE_REASON}]},+ },+ )+ # Enforcement fires the stop nudge.+ nudge = _check_enforcement(ctx)+ assert nudge == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE+ # Same signature on next iteration: no nudge, and the exit-path guard+ # raises because last_test_ok is still False.+ assert _check_enforcement(ctx) is None+ with pytest.raises(CopilotNonRetriableNavError):+ _maybe_raise_non_retriable_nav(ctx)+++def test_full_flow_cleared_after_successful_run() -> None:+ ctx = _fresh_context()+ ctx.test_after_update_done = True+ _record_run_blocks_result(+ ctx,+ {"ok": False, "data": {"blocks": [{"failure_reason": _DNS_FAILURE_REASON}]}},+ )+ assert _check_enforcement(ctx) == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE+ # Then a real success happens.+ _record_run_blocks_result(+ ctx,+ {+ "ok": True,+ "data": {+ "blocks": [+ {+ "label": "extract",+ "block_type": "extraction",+ "status": "completed",+ "extracted_data": [{"x": 1}],+ }+ ],+ },+ },+ )+ # Signature latch cleared so a later bad URL can re-fire.+ assert ctx.non_retriable_nav_error_last_emitted_signature is None+ # Last-test fields now reflect success; the exit-path guard does nothing.+ _maybe_raise_non_retriable_nav(ctx) # must not raise+++# ---------------------------------------------------------------------------+# Within-turn fail-fast guard — _tool_loop_error+# ---------------------------------------------------------------------------+++def test_tool_loop_error_blocks_update_and_run_blocks_after_dns_failure() -> None:+ # Observed repro: agent called update_and_run_blocks on www.example.invalid,+ # got DNS failure, then the LLM tried again with the bare host on the next+ # tool call — still within the same agent turn. The enforcement-loop stop+ # nudge only fires between turns, so the guard must live at the tool+ # entrypoint to actually prevent the speculative retry.+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ err = _tool_loop_error(ctx, "update_and_run_blocks")+ assert err is not None+ assert "permanent navigation error" in err+ assert "net::ERR_NAME_NOT_RESOLVED" in err+ assert "Do NOT retry" in err+++def test_tool_loop_error_blocks_run_blocks_and_collect_debug_after_dns_failure() -> None:+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ err = _tool_loop_error(ctx, "run_blocks_and_collect_debug")+ assert err is not None+ assert "permanent navigation error" in err+++def test_tool_loop_error_does_not_block_planning_tools() -> None:+ # get_run_results / list_credentials / update_workflow are scoped out of+ # _BLOCK_RUNNING_TOOLS and should remain callable so the agent can inspect+ # the failure and decide how to respond to the user.+ ctx = _fresh_context()+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON+ for tool in ("get_run_results", "list_credentials", "update_workflow"):+ assert _tool_loop_error(ctx, tool) is None, f"{tool} should not be blocked"+++def test_tool_loop_error_does_not_block_when_flag_unset() -> None:+ ctx = _fresh_context()+ assert _tool_loop_error(ctx, "update_and_run_blocks") is None+ assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is Nonediff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex 1e42f96fd..52e1c6a8b 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -6,6 +6,7 @@ from unittest.mock import MagicMockfrom skyvern.forge.sdk.copilot.output_utils import (sanitize_tool_result_for_llm,+ summarize_tool_result,truncate_output,)@@ -216,8 +217,6 @@ class TestSanitization:class TestSummarizeToolResult:@staticmethoddef _summarize(tool_name: str, result: dict) -> str:- from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result-return summarize_tool_result(tool_name, result)def test_error_result(self) -> None:@@ -225,6 +224,62 @@ class TestSummarizeToolResult:assert "Failed" in summaryassert "oops" in summary+ def test_failed_run_surfaces_block_failure_reason_when_error_absent(self) -> None:+ summary = self._summarize(+ "run_blocks_and_collect_debug",+ {+ "ok": False,+ "data": {+ "overall_status": "failed",+ "blocks": [+ {+ "label": "navigate",+ "status": "failed",+ "failure_reason": (+ "Failed to navigate to url https://example.invalid. "+ "Error message: net::ERR_NAME_NOT_RESOLVED"+ ),+ }+ ],+ },+ },+ )+ assert "ERR_NAME_NOT_RESOLVED" in summary+ assert "Unknown error" not in summary++ def test_failed_run_prefers_top_level_error_over_nested(self) -> None:+ summary = self._summarize(+ "run_blocks_and_collect_debug",+ {+ "ok": False,+ "error": "top-level message",+ "data": {"blocks": [{"failure_reason": "nested message"}]},+ },+ )+ assert "top-level message" in summary+ assert "nested message" not in summary++ def test_failed_run_prefers_data_failure_reason_over_block_failure_reason(self) -> None:+ summary = self._summarize(+ "run_blocks_and_collect_debug",+ {+ "ok": False,+ "data": {+ "failure_reason": "run-level",+ "blocks": [{"failure_reason": "block-level"}],+ },+ },+ )+ assert "run-level" in summary+ assert "block-level" not in summary++ def test_failed_run_falls_back_to_unknown_error_when_nothing_present(self) -> None:+ summary = self._summarize(+ "run_blocks_and_collect_debug",+ {"ok": False, "data": {"blocks": []}},+ )+ assert "Unknown error" in summary+def test_update_workflow(self) -> None:summary = self._summarize("update_workflow",