Skip to content

Automate browser based workflows with AI

fix(SKY-9136): fail fast on non-retriable navigation errors in workflow copilot (#5563)

Andrew Neilson834db2abe630
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index c1ccc8333..5bf271ee5 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -281,6 +281,7 @@ async def run_copilot_agent(
from skyvern.cli.mcp_tools import mcp as skyvern_mcp
from skyvern.forge.sdk.copilot.enforcement import (
+ CopilotNonRetriableNavError,
CopilotTotalTimeoutError,
run_with_enforcement,
)
@@ -416,6 +417,23 @@ async def run_copilot_agent(
workflow_yaml=ctx.last_workflow_yaml,
workflow_was_persisted=ctx.workflow_persisted,
)
+ except CopilotNonRetriableNavError as exc:
+ LOG.warning(
+ "Copilot run halted on non-retriable navigation error",
+ url=exc.url,
+ error_message=exc.error_message,
+ organization_id=organization_id,
+ )
+ return AgentResult(
+ user_response=(
+ f"The target URL could not be reached. Error: {exc.error_message}. "
+ "Please verify the URL and try again."
+ ),
+ updated_workflow=ctx.last_workflow,
+ global_llm_context=global_llm_context,
+ workflow_yaml=ctx.last_workflow_yaml,
+ workflow_was_persisted=ctx.workflow_persisted,
+ )
except Exception as e:
LOG.error("Copilot agent error", error=str(e), exc_info=True)
return AgentResult(
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index d6a7b730c..11bbc3606 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -181,6 +181,16 @@ class CopilotContext(AgentContext):
# streak is still at 2, and guarantees the stop nudge fires exactly once
# when the streak reaches 3.
repeated_failure_nudge_emitted_at_streak: int = 0
+ # Set by _record_run_blocks_result when the most recent failed run matches
+ # SKIP_INNER_NAV_RETRY_ERRORS (DNS / cert / SSL / invalid URL). Drives the
+ # one-shot non-retriable-nav stop nudge and the deterministic exit-path
+ # exception in run_with_enforcement. Cleared at the top of every call to
+ # _record_run_blocks_result so stale state can't leak across runs.
+ last_test_non_retriable_nav_error: str | None = None
+ # Normalized signature of the non-retriable nav error last nudged on.
+ # Lets the stop nudge re-fire if the user retries with a different bad URL
+ # (different signature) in the same session. Cleared on meaningful success.
+ non_retriable_nav_error_last_emitted_signature: str | None = None
last_failure_category_top: str | None = None
# Hash of the ordered (action_type, element_id) tuples from the last run's
# action trace. When the same fingerprint repeats run-over-run with no
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 9b23e099f..fd69c76d6 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any
import structlog
from agents.run import Runner
+from skyvern.forge.sdk.copilot.failure_tracking import normalize_failure_reason
from skyvern.forge.sdk.copilot.output_utils import extract_final_text, parse_final_response
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -195,6 +196,14 @@ POST_PARAMETER_BINDING_WARN_NUDGE = (
"`parameters` argument."
)
+POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE = (
+ "STOP — the target URL is unreachable and further retries cannot succeed. "
+ "The navigation failed with a permanent error (DNS resolution, SSL/cert, "
+ "or invalid URL). Do NOT retry and do NOT edit the workflow. Reply to the "
+ "user now: state that the URL could not be reached, quote the exact error "
+ "message from the last failure_reason, and ask them to verify the URL."
+)
+
POST_PARAMETER_BINDING_STOP_NUDGE = (
"STOP — you have retried the same PARAMETER_BINDING_ERROR multiple times "
"without reconciling the workflow configuration. Do NOT call "
@@ -250,6 +259,41 @@ class CopilotTotalTimeoutError(Exception):
"""Raised when the copilot agent exceeds the total allowed runtime."""
+class CopilotNonRetriableNavError(Exception):
+ """Raised from run_with_enforcement when the copilot's most recent run
+ hit a permanent navigation error (DNS / cert / SSL / invalid URL) and
+ the loop is about to exit without a successful test. Caught at the
+ agent entrypoint and translated to a deterministic user-facing failure,
+ mirroring the CopilotTotalTimeoutError handling pattern."""
+
+ def __init__(self, url: str | None, error_message: str) -> None:
+ self.url = url
+ self.error_message = error_message
+ super().__init__(f"Non-retriable navigation error: {error_message}")
+
+
+_FAILED_TO_NAVIGATE_URL_PATTERN = re.compile(r"Failed to navigate to url (\S+)\. Error message:")
+
+
+def _extract_url_from_nav_error(message: str) -> str | None:
+ """Pull the URL out of a FailedToNavigateToUrl string. None on no match."""
+ match = _FAILED_TO_NAVIGATE_URL_PATTERN.search(message)
+ return match.group(1) if match else None
+
+
+def _maybe_raise_non_retriable_nav(ctx: Any) -> None:
+ """Raise CopilotNonRetriableNavError if the most recent run was a
+ permanent navigation failure and nothing else has succeeded. Called
+ before both `return result` sites in run_with_enforcement so the loop
+ cannot hand a failed run back to the caller as if it completed."""
+ err = getattr(ctx, "last_test_non_retriable_nav_error", None)
+ if not isinstance(err, str) or not err:
+ return
+ if getattr(ctx, "last_test_ok", None) is True:
+ return
+ raise CopilotNonRetriableNavError(url=_extract_url_from_nav_error(err), error_message=err)
+
+
_ACTION_CATEGORIES: list[list[str]] = [
["navigate", "go to", "open", "visit"],
["download", "save", "export"],
@@ -362,6 +406,10 @@ def _needs_explore_without_workflow_nudge(ctx: Any) -> bool:
def _needs_failed_test_nudge(ctx: Any) -> bool:
"""Return True when the last test failed and the agent hasn't iterated yet."""
+ # A permanent nav error cannot be 'fix the workflow and retry' material —
+ # the dedicated non-retriable branch in _check_enforcement owns this case.
+ if getattr(ctx, "last_test_non_retriable_nav_error", None):
+ return False
if getattr(ctx, "last_test_ok", None) is not False:
return False
if not getattr(ctx, "test_after_update_done", False):
@@ -372,11 +420,18 @@ def _needs_failed_test_nudge(ctx: Any) -> bool:
def _needs_suspicious_success_nudge(ctx: Any) -> bool:
"""Return True when the last test 'completed' but data blocks had no output."""
+ # A non-retriable nav failure cannot be "suspiciously successful" — defer
+ # to the dedicated stop path rather than competing for the nudge slot.
+ if getattr(ctx, "last_test_non_retriable_nav_error", None):
+ return False
return bool(getattr(ctx, "last_test_suspicious_success", False))
def _needs_repeated_null_data_nudge(ctx: Any) -> bool:
"""Return True when suspicious-success has happened enough times to escalate."""
+ # Same as above: non-retriable nav state never belongs on this branch.
+ if getattr(ctx, "last_test_non_retriable_nav_error", None):
+ return False
if not getattr(ctx, "last_test_suspicious_success", False):
return False
streak = getattr(ctx, "null_data_streak_count", 0)
@@ -393,6 +448,10 @@ def _repeated_frontier_failure_nudge(ctx: Any) -> str | None:
keeps climbing on further identical failures (incremented elsewhere by
update_repeated_failure_state), so the stop nudge fires naturally on the
next repeat after a warn."""
+ # Non-retriable nav errors get their own dedicated stop path; don't let a
+ # repeated-frontier nudge smuggle different retry advice past the gate.
+ if getattr(ctx, "last_test_non_retriable_nav_error", None):
+ return None
streak = _get_int(ctx, "repeated_failure_streak_count")
emitted = _get_int(ctx, "repeated_failure_nudge_emitted_at_streak")
top_category = getattr(ctx, "last_failure_category_top", None)
@@ -408,7 +467,33 @@ def _repeated_frontier_failure_nudge(ctx: Any) -> str | None:
_STOP_LEVEL_FRONTIER_NUDGES = frozenset({POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE, POST_PARAMETER_BINDING_STOP_NUDGE})
+def _non_retriable_nav_error_nudge(ctx: Any) -> tuple[str, str] | None:
+ """Emit POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE at most once per distinct
+ non-retriable nav-error signature. Returns ``(nudge, signature)`` when it
+ should fire, ``None`` otherwise. Signature normalization is shared with
+ `failure_tracking.compute_failure_signature`, so a cert error after a DNS
+ error (or vice versa) counts as a distinct signature and re-fires."""
+ raw = getattr(ctx, "last_test_non_retriable_nav_error", None)
+ if not isinstance(raw, str) or not raw:
+ return None
+ signature = normalize_failure_reason(raw)
+ last_emitted = getattr(ctx, "non_retriable_nav_error_last_emitted_signature", None)
+ if signature == last_emitted:
+ return None
+ return POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE, signature
+
+
def _check_enforcement(ctx: Any, result: RunResultStreaming | None = None) -> str | None:
+ # Terminal failure-mode signals must pre-empt tool-call hygiene nudges.
+ # A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be
+ # resolved by observing a prior navigate or by testing an updated
+ # workflow against the same bad URL, so let it speak first.
+ non_retriable = _non_retriable_nav_error_nudge(ctx)
+ if non_retriable is not None:
+ nudge_msg, signature = non_retriable
+ ctx.non_retriable_nav_error_last_emitted_signature = signature
+ return nudge_msg
+
if ctx.navigate_called and not ctx.observation_after_navigate and not ctx.navigate_enforcement_done:
ctx.navigate_enforcement_done = True
return POST_NAVIGATE_NUDGE
@@ -734,6 +819,7 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {
POST_REPEATED_NULL_DATA_NUDGE: "repeated_null_data",
POST_REPEATED_FRONTIER_FAILURE_WARN_NUDGE: "repeated_frontier_failure_warn",
POST_REPEATED_FRONTIER_FAILURE_STOP_NUDGE: "repeated_frontier_failure_stop",
+ POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE: "non_retriable_nav_error_stop",
POST_PARAMETER_BINDING_WARN_NUDGE: "parameter_binding_warn",
POST_PARAMETER_BINDING_STOP_NUDGE: "parameter_binding_stop",
POST_ANTI_BOT_FAILED_TEST_NUDGE: "anti_bot_block",
@@ -940,6 +1026,7 @@ async def run_with_enforcement(
else:
nudge = _check_enforcement(ctx, result)
if nudge is None:
+ _maybe_raise_non_retriable_nav(ctx)
return result
if nudge == POST_UPDATE_NUDGE:
@@ -948,6 +1035,7 @@ async def run_with_enforcement(
"Enforcement exhausted post-update nudges, allowing response",
nudge_count=ctx.post_update_nudge_count,
)
+ _maybe_raise_non_retriable_nav(ctx)
return result
ctx.post_update_nudge_count += 1
diff --git a/skyvern/forge/sdk/copilot/failure_tracking.py b/skyvern/forge/sdk/copilot/failure_tracking.py
index 8dec25667..4da5eef7e 100644
--- a/skyvern/forge/sdk/copilot/failure_tracking.py
+++ b/skyvern/forge/sdk/copilot/failure_tracking.py
@@ -31,7 +31,7 @@ if TYPE_CHECKING:
_FAILURE_REASON_MAX_CHARS = 200
-def _normalize_failure_reason(raw: str | None) -> str:
+def normalize_failure_reason(raw: str | None) -> str:
if not raw:
return ""
collapsed = " ".join(raw.split())
@@ -61,7 +61,7 @@ def compute_failure_signature(
run (status=completed but data-producing blocks produced no output) still
generates a signature so repeated no-data runs can be counted as repeats.
"""
- normalized = _normalize_failure_reason(failure_reason)
+ normalized = normalize_failure_reason(failure_reason)
has_signal = bool(normalized) or suspicious_success
if not has_signal:
return None
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 333a64e55..150c7f7ba 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import base64
import binascii
import json
+from collections.abc import Iterator
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
@@ -143,10 +144,40 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict
return sanitized
+def iter_failure_reasons(result: dict[str, Any]) -> Iterator[str]:
+ """Yield non-empty failure_reason strings from a copilot tool result:
+ run-level ``data.failure_reason`` first, then each block's ``failure_reason``
+ in order. Callers that only need the first match should wrap with ``next``."""
+ data = result.get("data") if isinstance(result, dict) else None
+ if not isinstance(data, dict):
+ return
+ run_level = data.get("failure_reason")
+ if isinstance(run_level, str) and run_level:
+ yield run_level
+ blocks = data.get("blocks")
+ if isinstance(blocks, list):
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ reason = block.get("failure_reason")
+ if isinstance(reason, str) and reason:
+ yield reason
+
+
+def _extract_failure_message(result: dict[str, Any]) -> str:
+ """Prefer top-level ``error`` over nested failure_reason fields. Defense
+ in depth: _run_blocks_and_collect_debug now populates ``error`` on
+ failure, but other tool return shapes may still omit it."""
+ top = result.get("error")
+ if isinstance(top, str) and top:
+ return top
+ return next(iter_failure_reasons(result), "Unknown error")
+
+
def summarize_tool_result(tool_name: str, result: dict[str, Any]) -> str:
"""Create a brief human-readable summary of a tool result."""
if not result.get("ok", False):
- return f"Failed: {result.get('error', 'Unknown error')[:200]}"
+ return f"Failed: {_extract_failure_message(result)[:200]}"
data = result.get("data") or {}
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 5b5b5a09b..49bd2bbff 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -85,6 +85,8 @@ class AgentContext:
last_failed_workflow_yaml: str | None = None
repeated_failure_streak_count: int = 0
repeated_failure_nudge_emitted_at_streak: int = 0
+ last_test_non_retriable_nav_error: str | None = None
+ non_retriable_nav_error_last_emitted_signature: str | None = None
workflow_persisted: bool = False
last_workflow: Any | None = None
last_workflow_yaml: str | None = None
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index fafa04f19..aae464fb4 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import (
from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loop
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay
from skyvern.forge.sdk.copilot.output_utils import (
+ iter_failure_reasons,
sanitize_tool_result_for_llm,
truncate_output,
)
@@ -41,6 +42,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
WorkflowParameterType,
)
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRunStatus
+from skyvern.webeye.navigation import is_skip_inner_retry_error
LOG = structlog.get_logger()
@@ -249,6 +251,23 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str) -> str | None:
"detecting. Do NOT retry this tool — conclude the workflow is "
"not automatable as-is and report back to the user."
)
+
+ # Within-turn fail-fast for permanent navigation errors (DNS / cert /
+ # SSL / invalid URL). The enforcement-loop stop nudge only runs
+ # BETWEEN agent turns, so without this check the LLM is free to make
+ # speculative within-turn retries (e.g. drop the `www` subdomain and
+ # try again) before the nudge fires. update_and_run_blocks internally
+ # calls _update_workflow which clears the flag, so this check must
+ # run before that — hence at the tool entrypoint, not inside the run
+ # body.
+ prior_nav_error = getattr(ctx, "last_test_non_retriable_nav_error", None)
+ if isinstance(prior_nav_error, str) and prior_nav_error:
+ return (
+ f"Prior run in this turn hit a permanent navigation error "
+ f"({prior_nav_error[:200]}). Do NOT retry — the URL is unreachable "
+ "regardless of subdomain or path variations. Reply to the user "
+ "explaining the failure and asking them to verify the URL."
+ )
return None
@@ -978,10 +997,15 @@ async def _run_blocks_and_collect_debug(
existing_set.add(label)
ctx.verified_prefix_labels = existing_prefix
- return {
- "ok": run_ok,
- "data": result_data,
- }
+ response: dict[str, Any] = {"ok": run_ok, "data": result_data}
+ if not run_ok:
+ # Promote the real failure message into a top-level ``error`` field so
+ # every downstream consumer (summarize_tool_result, SSE frames, logs,
+ # LLM context) sees it instead of defaulting to "Unknown error".
+ # Wording matches ExtractionBlock's "Unknown error (no failure reason
+ # provided)" fallback so error-message matching stays consistent.
+ response["error"] = next(iter_failure_reasons(response), "Unknown error (no failure reason provided)")
+ return response
async def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:
@@ -1383,6 +1407,12 @@ def _record_workflow_update_result(copilot_ctx: Any, result: dict[str, Any]) ->
copilot_ctx.last_update_block_count = block_count
copilot_ctx.last_test_ok = None
copilot_ctx.last_test_failure_reason = None
+ # A fresh workflow edit invalidates the prior test's failure state —
+ # otherwise an exhausted POST_UPDATE_NUDGE on the new draft would raise
+ # CopilotNonRetriableNavError with the old run's error, telling the user
+ # to "verify the URL" for a URL they just corrected in the new draft.
+ copilot_ctx.last_test_non_retriable_nav_error = None
+ copilot_ctx.non_retriable_nav_error_last_emitted_signature = None
copilot_ctx.workflow_persisted = True
@@ -1443,6 +1473,15 @@ def _analyze_run_blocks(result: dict[str, Any]) -> tuple[str | None, bool, list[
return anti_bot_match, empty_data_blocks, categories
+def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:
+ """Return the first failure_reason that matches SKIP_INNER_NAV_RETRY_ERRORS
+ (DNS / cert / SSL / invalid URL), preferring run-level over block-level.
+ Same set is_skip_inner_retry_error uses at the browser layer, so the copilot
+ classifies on exactly the patterns that already short-circuit retries in
+ navigate_with_retry (skyvern/webeye/navigation.py)."""
+ return next((reason for reason in iter_failure_reasons(result) if is_skip_inner_retry_error(reason)), None)
+
+
def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:
run_ok = bool(result.get("ok", False))
copilot_ctx.last_test_ok = run_ok
@@ -1450,6 +1489,7 @@ def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:
copilot_ctx.last_test_suspicious_success = False
copilot_ctx.last_test_anti_bot = None
copilot_ctx.last_failure_category_top = None
+ copilot_ctx.last_test_non_retriable_nav_error = None
anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result)
if anti_bot_match:
@@ -1483,10 +1523,14 @@ def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any]) -> None:
copilot_ctx.failed_test_nudge_count = 0
copilot_ctx.null_data_streak_count = 0
copilot_ctx.last_failed_workflow_yaml = None
+ # Real success: clear the signature latch so a subsequent bad URL in
+ # the same session can re-fire the stop nudge.
+ copilot_ctx.non_retriable_nav_error_last_emitted_signature = None
update_repeated_failure_state(copilot_ctx, result)
return
copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
+ copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)
data = result.get("data")
if isinstance(data, dict):
diff --git a/tests/unit/test_copilot_non_retriable_nav.py b/tests/unit/test_copilot_non_retriable_nav.py
new file mode 100644
index 000000000..d79b8e419
--- /dev/null
+++ b/tests/unit/test_copilot_non_retriable_nav.py
@@ -0,0 +1,584 @@
+"""Tests for non-retriable navigation error handling in the copilot layer.
+
+Covers SKY-9136: when the browser layer raises ``FailedToNavigateToUrl`` with
+a DNS / cert / SSL / invalid-URL pattern (``SKIP_INNER_NAV_RETRY_ERRORS``),
+the copilot must surface the real error instead of "Unknown error", must not
+keep retrying, and must fail deterministically even if the model tries to
+narrate a completion.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.enforcement import (
+ POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE,
+ CopilotNonRetriableNavError,
+ _check_enforcement,
+ _extract_url_from_nav_error,
+ _maybe_raise_non_retriable_nav,
+ _needs_failed_test_nudge,
+ _needs_repeated_null_data_nudge,
+ _needs_suspicious_success_nudge,
+ _non_retriable_nav_error_nudge,
+ _repeated_frontier_failure_nudge,
+)
+from skyvern.forge.sdk.copilot.tools import (
+ _detect_non_retriable_nav_error,
+ _record_run_blocks_result,
+ _record_workflow_update_result,
+ _tool_loop_error,
+)
+
+_DNS_FAILURE_REASON = (
+ "Failed to navigate to url https://www.example.invalid/path. Error message: net::ERR_NAME_NOT_RESOLVED"
+)
+_CERT_FAILURE_REASON = "Failed to navigate to url https://expired.example. Error message: net::ERR_CERT_DATE_INVALID"
+_GENERIC_FAILURE_REASON = "Timeout waiting for element #submit"
+
+
+def _fresh_context() -> CopilotContext:
+ return CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+
+
+# ---------------------------------------------------------------------------
+# _detect_non_retriable_nav_error
+# ---------------------------------------------------------------------------
+
+
+def test_detect_matches_dns_error_in_block_failure_reason() -> None:
+ result = {"ok": False, "data": {"blocks": [{"failure_reason": _DNS_FAILURE_REASON}]}}
+ assert _detect_non_retriable_nav_error(result) == _DNS_FAILURE_REASON
+
+
+def test_detect_matches_cert_error_in_run_level_failure_reason() -> None:
+ result = {"ok": False, "data": {"failure_reason": _CERT_FAILURE_REASON, "blocks": []}}
+ assert _detect_non_retriable_nav_error(result) == _CERT_FAILURE_REASON
+
+
+def test_detect_matches_invalid_url_error() -> None:
+ invalid_url = "Failed to navigate to url not-a-url. Error message: net::ERR_INVALID_URL"
+ result = {"ok": False, "data": {"blocks": [{"failure_reason": invalid_url}]}}
+ assert _detect_non_retriable_nav_error(result) == invalid_url
+
+
+def test_detect_matches_name_resolution_failed() -> None:
+ reason = "net::ERR_NAME_RESOLUTION_FAILED happened mid-flight"
+ result = {"ok": False, "data": {"blocks": [{"failure_reason": reason}]}}
+ assert _detect_non_retriable_nav_error(result) == reason
+
+
+def test_detect_matches_ssl_error() -> None:
+ reason = "SSL error: net::ERR_SSL_PROTOCOL_ERROR"
+ result = {"ok": False, "data": {"blocks": [{"failure_reason": reason}]}}
+ assert _detect_non_retriable_nav_error(result) == reason
+
+
+def test_detect_returns_none_for_generic_failure() -> None:
+ result = {"ok": False, "data": {"blocks": [{"failure_reason": _GENERIC_FAILURE_REASON}]}}
+ assert _detect_non_retriable_nav_error(result) is None
+
+
+def test_detect_returns_none_for_missing_data() -> None:
+ assert _detect_non_retriable_nav_error({"ok": False}) is None
+
+
+def test_detect_returns_none_for_empty_blocks() -> None:
+ assert _detect_non_retriable_nav_error({"ok": False, "data": {"blocks": []}}) is None
+
+
+def test_detect_prefers_run_level_over_block_level() -> None:
+ # When both match, the run-level reason wins (it is the authoritative
+ # aggregate failure the workflow service recorded).
+ result = {
+ "ok": False,
+ "data": {
+ "failure_reason": _DNS_FAILURE_REASON,
+ "blocks": [{"failure_reason": _CERT_FAILURE_REASON}],
+ },
+ }
+ assert _detect_non_retriable_nav_error(result) == _DNS_FAILURE_REASON
+
+
+# ---------------------------------------------------------------------------
+# _record_run_blocks_result — context flag plumbing
+# ---------------------------------------------------------------------------
+
+
+def test_record_sets_flag_on_dns_failure() -> None:
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ _record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {"blocks": [{"failure_reason": _DNS_FAILURE_REASON}]},
+ },
+ )
+ assert ctx.last_test_non_retriable_nav_error == _DNS_FAILURE_REASON
+ assert ctx.last_test_ok is False
+
+
+def test_record_leaves_flag_none_for_generic_failure() -> None:
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ _record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {"blocks": [{"failure_reason": _GENERIC_FAILURE_REASON}]},
+ },
+ )
+ assert ctx.last_test_non_retriable_nav_error is None
+
+
+def test_record_clears_flag_on_every_call() -> None:
+ # Stale state from a prior run must not leak into the next run's context.
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.last_test_non_retriable_nav_error = "stale DNS error"
+ _record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {"blocks": [{"failure_reason": _GENERIC_FAILURE_REASON}]},
+ },
+ )
+ assert ctx.last_test_non_retriable_nav_error is None
+
+
+def test_workflow_update_clears_non_retriable_flag_and_signature_latch() -> None:
+ # Codex review P2-2: after a DNS-failed run, if the agent edits the
+ # workflow (e.g. fixing the URL), the stale flag must not survive the
+ # edit — otherwise an exhausted POST_UPDATE_NUDGE on the new draft
+ # would raise CopilotNonRetriableNavError with the OLD run's error
+ # message, telling the user to verify a URL they just corrected.
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ ctx.non_retriable_nav_error_last_emitted_signature = "dns_signature_123"
+ ctx.last_test_ok = False
+ ctx.workflow_yaml = "updated yaml"
+
+ _record_workflow_update_result(
+ ctx,
+ {
+ "ok": True,
+ "data": {"block_count": 2},
+ "_workflow": SimpleNamespace(workflow_id="wf_new"),
+ },
+ )
+ assert ctx.last_test_non_retriable_nav_error is None
+ assert ctx.non_retriable_nav_error_last_emitted_signature is None
+ # Consistency check: the other per-test fields are also reset (pre-existing behavior).
+ assert ctx.last_test_ok is None
+ assert ctx.last_test_failure_reason is None
+
+
+def test_workflow_update_does_not_clear_flag_on_failed_update() -> None:
+ # Only a SUCCESSFUL update invalidates prior test state — a failed
+ # validation attempt leaves the existing flags alone.
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ ctx.non_retriable_nav_error_last_emitted_signature = "dns_signature_123"
+ ctx.last_test_ok = False
+
+ _record_workflow_update_result(
+ ctx,
+ {"ok": False, "error": "validation failed"},
+ )
+ assert ctx.last_test_non_retriable_nav_error == _DNS_FAILURE_REASON
+ assert ctx.non_retriable_nav_error_last_emitted_signature == "dns_signature_123"
+
+
+def test_record_clears_signature_latch_on_meaningful_success() -> None:
+ # After a real success, the stop nudge must be able to re-fire if a new
+ # bad URL fails later in the same session.
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.non_retriable_nav_error_last_emitted_signature = "some previous signature"
+ _record_run_blocks_result(
+ ctx,
+ {
+ "ok": True,
+ "data": {
+ "blocks": [
+ {
+ "label": "extract",
+ "block_type": "extraction",
+ "status": "completed",
+ "extracted_data": [{"name": "widget", "price": 1.0}],
+ }
+ ],
+ },
+ },
+ )
+ assert ctx.non_retriable_nav_error_last_emitted_signature is None
+
+
+# ---------------------------------------------------------------------------
+# _non_retriable_nav_error_nudge — signature latch
+# ---------------------------------------------------------------------------
+
+
+def test_nudge_helper_fires_first_time() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ result = _non_retriable_nav_error_nudge(ctx)
+ assert result is not None
+ nudge, signature = result
+ assert nudge == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE
+ assert signature # non-empty
+
+
+def test_nudge_helper_does_not_refire_same_signature() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ first = _non_retriable_nav_error_nudge(ctx)
+ assert first is not None
+ ctx.non_retriable_nav_error_last_emitted_signature = first[1]
+ assert _non_retriable_nav_error_nudge(ctx) is None
+
+
+def test_nudge_helper_refires_on_different_signature() -> None:
+ # User corrects from one bad URL to another bad URL in the same session
+ # without a successful run in between; the stop nudge must re-fire.
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ first = _non_retriable_nav_error_nudge(ctx)
+ assert first is not None
+ ctx.non_retriable_nav_error_last_emitted_signature = first[1]
+
+ ctx.last_test_non_retriable_nav_error = _CERT_FAILURE_REASON
+ second = _non_retriable_nav_error_nudge(ctx)
+ assert second is not None
+ assert second[1] != first[1]
+
+
+def test_nudge_helper_returns_none_when_flag_unset() -> None:
+ ctx = _fresh_context()
+ assert _non_retriable_nav_error_nudge(ctx) is None
+
+
+# ---------------------------------------------------------------------------
+# _check_enforcement — branch ordering and latching
+# ---------------------------------------------------------------------------
+
+
+def test_check_enforcement_returns_non_retriable_nudge_before_failed_test() -> None:
+ # Conditions that would normally trigger POST_FAILED_TEST_NUDGE (last_test
+ # failed, test_after_update_done=True) must yield the non-retriable stop
+ # nudge instead when the flag is set.
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.last_test_ok = False
+ ctx.last_test_failure_reason = _DNS_FAILURE_REASON
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ nudge = _check_enforcement(ctx)
+ assert nudge == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE
+ # Emission latched for future calls with same signature.
+ assert ctx.non_retriable_nav_error_last_emitted_signature
+
+
+def test_check_enforcement_one_shot_per_signature() -> None:
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.last_test_ok = False
+ ctx.last_test_failure_reason = _DNS_FAILURE_REASON
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ first = _check_enforcement(ctx)
+ assert first == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE
+ # Same signature on next iteration: nudge returns None (and does NOT
+ # fall through to POST_FAILED_TEST_NUDGE thanks to competing-branch
+ # suppression).
+ assert _check_enforcement(ctx) is None
+
+
+def test_check_enforcement_pre_empts_post_navigate_nudge() -> None:
+ # Codex review P2-1: a navigate-hygiene state (navigate_called=True,
+ # observation_after_navigate=False) must not steal the nudge slot when
+ # a non-retriable nav error is also present — POST_NAVIGATE_NUDGE tells
+ # the agent to observe the page, which does not resolve a DNS failure
+ # and merely delays the deterministic stop by at least one iteration.
+ ctx = _fresh_context()
+ ctx.navigate_called = True
+ ctx.observation_after_navigate = False
+ ctx.navigate_enforcement_done = False
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ assert _check_enforcement(ctx) == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE
+
+
+def test_check_enforcement_pre_empts_post_update_nudge() -> None:
+ # Similar P2-1 case: an update-without-test state must not pre-empt the
+ # terminal stop. In practice P2-2 clears the flag on update, but if a
+ # race leaves the flag set, stop-nudge still wins.
+ ctx = _fresh_context()
+ ctx.update_workflow_called = True
+ ctx.test_after_update_done = False
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ assert _check_enforcement(ctx) == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE
+
+
+# ---------------------------------------------------------------------------
+# Competing-branch suppression — each failure helper
+# ---------------------------------------------------------------------------
+
+
+def test_needs_failed_test_nudge_suppressed_when_flag_set() -> None:
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.last_test_ok = False
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ assert _needs_failed_test_nudge(ctx) is False
+
+
+def test_needs_failed_test_nudge_still_fires_without_flag() -> None:
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.last_test_ok = False
+ assert _needs_failed_test_nudge(ctx) is True
+
+
+def test_needs_suspicious_success_nudge_suppressed_when_flag_set() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_suspicious_success = True
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ assert _needs_suspicious_success_nudge(ctx) is False
+
+
+def test_needs_repeated_null_data_nudge_suppressed_when_flag_set() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_suspicious_success = True
+ ctx.null_data_streak_count = 5
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ assert _needs_repeated_null_data_nudge(ctx) is False
+
+
+def test_repeated_frontier_failure_nudge_suppressed_when_flag_set() -> None:
+ ctx = _fresh_context()
+ ctx.repeated_failure_streak_count = 5
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ assert _repeated_frontier_failure_nudge(ctx) is None
+
+
+# ---------------------------------------------------------------------------
+# _extract_url_from_nav_error
+# ---------------------------------------------------------------------------
+
+
+def test_extract_url_parses_standard_format() -> None:
+ url = _extract_url_from_nav_error(_DNS_FAILURE_REASON)
+ assert url == "https://www.example.invalid/path"
+
+
+def test_extract_url_returns_none_on_malformed_message() -> None:
+ assert _extract_url_from_nav_error("some unrelated error text") is None
+
+
+# ---------------------------------------------------------------------------
+# _maybe_raise_non_retriable_nav — deterministic exit-path
+# ---------------------------------------------------------------------------
+
+
+def test_maybe_raise_noops_when_flag_unset() -> None:
+ ctx = _fresh_context()
+ _maybe_raise_non_retriable_nav(ctx) # must not raise
+
+
+def test_maybe_raise_noops_when_last_test_is_ok() -> None:
+ # A prior successful run does NOT suppress the exception (per CORR-3),
+ # but the MOST RECENT run being a real success does — because that
+ # means this turn's test did not hit the non-retriable path.
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ ctx.last_test_ok = True
+ _maybe_raise_non_retriable_nav(ctx) # must not raise
+
+
+def test_maybe_raise_raises_when_flag_and_last_test_failed() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ ctx.last_test_ok = False
+ with pytest.raises(CopilotNonRetriableNavError) as excinfo:
+ _maybe_raise_non_retriable_nav(ctx)
+ assert excinfo.value.error_message == _DNS_FAILURE_REASON
+ assert excinfo.value.url == "https://www.example.invalid/path"
+
+
+def test_maybe_raise_raises_when_last_test_ok_is_none() -> None:
+ # The guard is ``last_test_ok is not True`` (not ``is False``), so an
+ # ambiguous None (e.g. a suspicious-success run) with the flag set still
+ # surfaces the cached nav failure rather than letting the loop return
+ # silently. Locks in the tri-state semantics.
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ ctx.last_test_ok = None
+ with pytest.raises(CopilotNonRetriableNavError):
+ _maybe_raise_non_retriable_nav(ctx)
+
+
+def test_maybe_raise_raises_when_prior_run_succeeded_but_current_failed() -> None:
+ # Codex CORR-3: the guard must NOT be gated on session history. A fresh
+ # non-retriable nav failure on the most recent run still raises, even if
+ # an earlier run in the same session succeeded.
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ ctx.last_test_ok = False # most recent run
+ # Simulate a prior successful run in the session — there is no
+ # `any_test_ok_ever` flag; the helper only inspects current state.
+ with pytest.raises(CopilotNonRetriableNavError):
+ _maybe_raise_non_retriable_nav(ctx)
+
+
+# ---------------------------------------------------------------------------
+# Sanity: exception carries the expected attributes for the agent handler
+# ---------------------------------------------------------------------------
+
+
+def test_exception_carries_url_and_error_message() -> None:
+ exc = CopilotNonRetriableNavError(url="https://x.test", error_message="some reason")
+ assert exc.url == "https://x.test"
+ assert exc.error_message == "some reason"
+ assert "some reason" in str(exc)
+
+
+# ---------------------------------------------------------------------------
+# Sanity: when flag is set, no failure-nudge branch competes
+# ---------------------------------------------------------------------------
+
+
+def test_all_competing_branches_silent_after_latch() -> None:
+ # Reproduce the full steady state after the one-shot stop nudge has
+ # latched: ctx has a non-retriable error, last_test_ok=False,
+ # test_after_update_done=True, and all counters are set such that the
+ # other branches would normally fire. Assert all four helpers are silent.
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.last_test_ok = False
+ ctx.last_test_suspicious_success = True
+ ctx.null_data_streak_count = 5
+ ctx.repeated_failure_streak_count = 5
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+
+ assert _needs_failed_test_nudge(ctx) is False
+ assert _needs_suspicious_success_nudge(ctx) is False
+ assert _needs_repeated_null_data_nudge(ctx) is False
+ assert _repeated_frontier_failure_nudge(ctx) is None
+
+
+def test_without_flag_competing_branches_still_active() -> None:
+ # Inverse: same setup but without the flag — all relevant branches fire.
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ ctx.last_test_ok = False
+ assert _needs_failed_test_nudge(ctx) is True
+
+
+# ---------------------------------------------------------------------------
+# Integration-ish: record -> check -> exception flow
+# ---------------------------------------------------------------------------
+
+
+def test_full_flow_record_then_check_then_raise() -> None:
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ # Simulate a failed run with a DNS error.
+ _record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {"blocks": [{"failure_reason": _DNS_FAILURE_REASON}]},
+ },
+ )
+ # Enforcement fires the stop nudge.
+ nudge = _check_enforcement(ctx)
+ assert nudge == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE
+ # Same signature on next iteration: no nudge, and the exit-path guard
+ # raises because last_test_ok is still False.
+ assert _check_enforcement(ctx) is None
+ with pytest.raises(CopilotNonRetriableNavError):
+ _maybe_raise_non_retriable_nav(ctx)
+
+
+def test_full_flow_cleared_after_successful_run() -> None:
+ ctx = _fresh_context()
+ ctx.test_after_update_done = True
+ _record_run_blocks_result(
+ ctx,
+ {"ok": False, "data": {"blocks": [{"failure_reason": _DNS_FAILURE_REASON}]}},
+ )
+ assert _check_enforcement(ctx) == POST_NON_RETRIABLE_NAV_ERROR_STOP_NUDGE
+ # Then a real success happens.
+ _record_run_blocks_result(
+ ctx,
+ {
+ "ok": True,
+ "data": {
+ "blocks": [
+ {
+ "label": "extract",
+ "block_type": "extraction",
+ "status": "completed",
+ "extracted_data": [{"x": 1}],
+ }
+ ],
+ },
+ },
+ )
+ # Signature latch cleared so a later bad URL can re-fire.
+ assert ctx.non_retriable_nav_error_last_emitted_signature is None
+ # Last-test fields now reflect success; the exit-path guard does nothing.
+ _maybe_raise_non_retriable_nav(ctx) # must not raise
+
+
+# ---------------------------------------------------------------------------
+# Within-turn fail-fast guard — _tool_loop_error
+# ---------------------------------------------------------------------------
+
+
+def test_tool_loop_error_blocks_update_and_run_blocks_after_dns_failure() -> None:
+ # Observed repro: agent called update_and_run_blocks on www.example.invalid,
+ # got DNS failure, then the LLM tried again with the bare host on the next
+ # tool call — still within the same agent turn. The enforcement-loop stop
+ # nudge only fires between turns, so the guard must live at the tool
+ # entrypoint to actually prevent the speculative retry.
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ err = _tool_loop_error(ctx, "update_and_run_blocks")
+ assert err is not None
+ assert "permanent navigation error" in err
+ assert "net::ERR_NAME_NOT_RESOLVED" in err
+ assert "Do NOT retry" in err
+
+
+def test_tool_loop_error_blocks_run_blocks_and_collect_debug_after_dns_failure() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ err = _tool_loop_error(ctx, "run_blocks_and_collect_debug")
+ assert err is not None
+ assert "permanent navigation error" in err
+
+
+def test_tool_loop_error_does_not_block_planning_tools() -> None:
+ # get_run_results / list_credentials / update_workflow are scoped out of
+ # _BLOCK_RUNNING_TOOLS and should remain callable so the agent can inspect
+ # the failure and decide how to respond to the user.
+ ctx = _fresh_context()
+ ctx.last_test_non_retriable_nav_error = _DNS_FAILURE_REASON
+ for tool in ("get_run_results", "list_credentials", "update_workflow"):
+ assert _tool_loop_error(ctx, tool) is None, f"{tool} should not be blocked"
+
+
+def test_tool_loop_error_does_not_block_when_flag_unset() -> None:
+ ctx = _fresh_context()
+ assert _tool_loop_error(ctx, "update_and_run_blocks") is None
+ assert _tool_loop_error(ctx, "run_blocks_and_collect_debug") is None
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index 1e42f96fd..52e1c6a8b 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock
from skyvern.forge.sdk.copilot.output_utils import (
sanitize_tool_result_for_llm,
+ summarize_tool_result,
truncate_output,
)
@@ -216,8 +217,6 @@ class TestSanitization:
class TestSummarizeToolResult:
@staticmethod
def _summarize(tool_name: str, result: dict) -> str:
- from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result
-
return summarize_tool_result(tool_name, result)
def test_error_result(self) -> None:
@@ -225,6 +224,62 @@ class TestSummarizeToolResult:
assert "Failed" in summary
assert "oops" in summary
+ def test_failed_run_surfaces_block_failure_reason_when_error_absent(self) -> None:
+ summary = self._summarize(
+ "run_blocks_and_collect_debug",
+ {
+ "ok": False,
+ "data": {
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "navigate",
+ "status": "failed",
+ "failure_reason": (
+ "Failed to navigate to url https://example.invalid. "
+ "Error message: net::ERR_NAME_NOT_RESOLVED"
+ ),
+ }
+ ],
+ },
+ },
+ )
+ assert "ERR_NAME_NOT_RESOLVED" in summary
+ assert "Unknown error" not in summary
+
+ def test_failed_run_prefers_top_level_error_over_nested(self) -> None:
+ summary = self._summarize(
+ "run_blocks_and_collect_debug",
+ {
+ "ok": False,
+ "error": "top-level message",
+ "data": {"blocks": [{"failure_reason": "nested message"}]},
+ },
+ )
+ assert "top-level message" in summary
+ assert "nested message" not in summary
+
+ def test_failed_run_prefers_data_failure_reason_over_block_failure_reason(self) -> None:
+ summary = self._summarize(
+ "run_blocks_and_collect_debug",
+ {
+ "ok": False,
+ "data": {
+ "failure_reason": "run-level",
+ "blocks": [{"failure_reason": "block-level"}],
+ },
+ },
+ )
+ assert "run-level" in summary
+ assert "block-level" not in summary
+
+ def test_failed_run_falls_back_to_unknown_error_when_nothing_present(self) -> None:
+ summary = self._summarize(
+ "run_blocks_and_collect_debug",
+ {"ok": False, "data": {"blocks": []}},
+ )
+ assert "Unknown error" in summary
+
def test_update_workflow(self) -> None:
summary = self._summarize(
"update_workflow",