Skip to content

Automate browser based workflows with AI

SKY-10138: stop Copilot on unrecoverable browser-session errors (#6106)

Andrew Neilson49c33573e616
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index a1a592ce7..1ef17f2f7 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -47,6 +47,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
normalize_response_scaffolding,
output_policy_verdict_from_trace_data,
output_policy_verdict_to_trace_data,
+ url_origin,
)
from skyvern.forge.sdk.copilot.output_utils import (
extract_final_text,
@@ -728,6 +729,93 @@ _UNVALIDATED_PROPOSAL_AFFORDANCE = (
"I have a draft workflow proposal. Use Review to inspect it, Accept to save it, or Reject to discard it. "
"It has not been tested or verified end-to-end."
)
+_RECORDED_FAILURE_URL_RE = re.compile(r"https?://[^\s)>,]+")
+
+
+def _workflow_block_count(ctx: CopilotContext) -> int | None:
+ count = getattr(ctx, "last_update_block_count", None)
+ if isinstance(count, int) and count > 0:
+ return count
+ workflow = getattr(ctx, "last_workflow", None)
+ definition = getattr(workflow, "workflow_definition", None)
+ blocks = getattr(definition, "blocks", None)
+ return len(blocks) if isinstance(blocks, list) and blocks else None
+
+
+def _clean_recorded_failure_text(value: Any, max_chars: int = 240) -> str:
+ from skyvern.forge.sdk.copilot.enforcement import redact_browser_session_references
+
+ if not isinstance(value, str):
+ return ""
+ text = redact_raw_secrets_for_prompt(" ".join(value.split()))
+ text = redact_browser_session_references(text)
+ text = _RECORDED_FAILURE_URL_RE.sub(lambda match: url_origin(match.group(0)) or "[URL]", text)
+ if len(text) > max_chars:
+ text = text[: max_chars - 3].rstrip() + "..."
+ return text.rstrip(".")
+
+
+def _recorded_failure_summary(ctx: CopilotContext) -> tuple[str, str]:
+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)
+ verification = getattr(contract, "verification_result", None)
+ diagnosis = getattr(contract, "diagnosis_result", None)
+ remaining_blocker = _clean_recorded_failure_text(getattr(verification, "remaining_blocker", None))
+ root_cause = _clean_recorded_failure_text(getattr(diagnosis, "root_cause_summary", None))
+ fallback_reason = _clean_recorded_failure_text(getattr(ctx, "last_test_failure_reason", None))
+ reason = remaining_blocker or root_cause or fallback_reason
+ run_status = _clean_recorded_failure_text(getattr(verification, "run_status", None), max_chars=80)
+ status_sentence = f" Last run status: {run_status}." if run_status else ""
+ return reason, status_sentence
+
+
+def _last_good_failure_reply(ctx: CopilotContext, tested_reply: str) -> str:
+ reason, status_sentence = _recorded_failure_summary(ctx)
+ if not reason:
+ return tested_reply
+ return f"{tested_reply} The latest attempted change did not verify: {reason}.{status_sentence}"
+
+
+def _recorded_failure_reply(ctx: CopilotContext, *, cancelled: bool = False) -> str | None:
+ if cancelled or ctx.last_test_ok is True:
+ return None
+
+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)
+ verification = getattr(contract, "verification_result", None)
+ diagnosis = getattr(contract, "diagnosis_result", None)
+ repair_decision = getattr(contract, "repair_decision", None)
+ diagnosis_input = getattr(contract, "diagnosis_input", None)
+ failure_type = getattr(getattr(diagnosis, "suspected_failure_type", None), "value", None) or getattr(
+ diagnosis,
+ "suspected_failure_type",
+ None,
+ )
+ next_action = getattr(getattr(repair_decision, "next_action", None), "value", None) or getattr(
+ repair_decision,
+ "next_action",
+ None,
+ )
+ reason, status_sentence = _recorded_failure_summary(ctx)
+ if not reason:
+ return None
+
+ run_status = _clean_recorded_failure_text(getattr(verification, "run_status", None), max_chars=80).lower()
+ block_count = _workflow_block_count(ctx)
+ block_phrase = f"a {block_count}-block draft" if block_count else "a draft"
+ test_attempted = bool(
+ getattr(ctx, "test_after_update_done", False)
+ or getattr(ctx, "last_test_ok", None) is not None
+ or getattr(diagnosis_input, "workflow_run_id", None)
+ )
+ test_failed = ctx.last_test_ok is False or run_status == "failed"
+ unrecoverable_stop = next_action == "stop" or failure_type == "unrecoverable_tool_error"
+
+ if getattr(ctx, "last_workflow", None) is not None:
+ if test_attempted and test_failed and not unrecoverable_stop:
+ return f"I built {block_phrase} and tested it, but the test failed: {reason}.{status_sentence}"
+ if test_attempted:
+ return f"I built {block_phrase} and tested it, but the test couldn't finish: {reason}.{status_sentence}"
+ return f"I built {block_phrase}, but I couldn't verify it: {reason}.{status_sentence}"
+ return f"I couldn't finish the Copilot turn: {reason}.{status_sentence}"
def _ensure_unvalidated_proposal_affordance(user_response: str) -> str:
@@ -753,6 +841,7 @@ def _build_wip_exit_result(
cancelled: bool = False,
) -> AgentResult:
"""Selected non-success exits surface the most recent successfully parsed workflow."""
+ recorded_failure_reply = _recorded_failure_reply(ctx, cancelled=cancelled)
# When an unverified edit/run has overwritten ``last_workflow``, prefer the
# verified shape while still forcing explicit review.
if (
@@ -762,7 +851,7 @@ def _build_wip_exit_result(
and not ctx.last_test_suspicious_success
):
return AgentResult(
- user_response=tested_reply,
+ user_response=_last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_reply,
updated_workflow=ctx.last_good_workflow,
global_llm_context=global_llm_context,
workflow_yaml=ctx.last_good_workflow_yaml,
@@ -778,7 +867,10 @@ def _build_wip_exit_result(
and not ctx.last_test_suspicious_success
):
unvalidated = ctx.last_test_ok is not True
- reply = unvalidated_reply if unvalidated else tested_reply
+ if unvalidated and recorded_failure_reply:
+ reply = recorded_failure_reply
+ else:
+ reply = unvalidated_reply if unvalidated else tested_reply
return AgentResult(
user_response=reply,
updated_workflow=ctx.last_workflow,
@@ -789,7 +881,7 @@ def _build_wip_exit_result(
proposal_disposition="review_untested" if unvalidated else "auto_applicable",
cancelled=cancelled,
)
- return _build_exit_result(ctx, default_reply, global_llm_context, cancelled=cancelled)
+ return _build_exit_result(ctx, recorded_failure_reply or default_reply, global_llm_context, cancelled=cancelled)
def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
@@ -1674,6 +1766,7 @@ async def _run_copilot_turn_impl(
from skyvern.forge.sdk.copilot.enforcement import (
CopilotNonRetriableNavError,
CopilotTotalTimeoutError,
+ CopilotUnrecoverableToolError,
run_with_enforcement,
)
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
@@ -1867,6 +1960,14 @@ async def _run_copilot_turn_impl(
return _build_max_turns_exit_result(ctx, global_llm_context)
except CopilotTotalTimeoutError:
return _build_timeout_exit_result(ctx, global_llm_context)
+ except CopilotUnrecoverableToolError as exc:
+ LOG.warning(
+ "Copilot run halted on unrecoverable tool error",
+ tool_name=exc.tool_name,
+ error_message=exc.error_message,
+ organization_id=organization_id,
+ )
+ return _build_unexpected_error_exit_result(ctx, global_llm_context)
except CopilotNonRetriableNavError as exc:
LOG.warning(
"Copilot run halted on non-retriable navigation error",
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 636b62d9b..dbb7586a4 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -26,6 +26,7 @@ class DiagnosisFailureType(StrEnum):
SUSPICIOUS_SUCCESS = "suspicious_success"
MISSING_CREDENTIAL_OR_INIT = "missing_credential_or_init"
REPAIRABLE_BLOCK_FAILURE = "repairable_block_failure"
+ UNRECOVERABLE_TOOL_ERROR = "unrecoverable_tool_error"
UNKNOWN = "unknown"
@@ -298,6 +299,14 @@ def _failure_type(
for value in (result.get("error"), data.get("failure_reason"), data.get("skip_reason"))
if value
)
+ if (
+ "UNRECOVERABLE_TOOL_ERROR" in categories
+ or "browser session not found" in error_text
+ or "no browser context" in error_text
+ or ("session not found" in error_text and "browser" in error_text)
+ or ("404" in error_text and "browser session" in error_text)
+ ):
+ return DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR
if (
data.get("skip_reason") == "workflow_credential_inputs_unbound"
or "credential" in error_text
@@ -320,6 +329,8 @@ def _next_action(failure_type: DiagnosisFailureType, ctx: Any, data: dict[str, A
or failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
):
return RepairNextAction.ASK
+ if failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR:
+ return RepairNextAction.STOP
if getattr(ctx, "last_test_non_retriable_nav_error", None):
return RepairNextAction.STOP
authority = getattr(getattr(ctx, "turn_intent", None), "authority", None)
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 39713fa83..0a109808a 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -77,6 +77,7 @@ REPEATED_FRONTIER_STREAK_STOP_AT = 3
# scraper could not read the page. Aligned with MAX_FAILED_TEST_NUDGES so the
# copilot gets one generic retry nudge, then stops on the second occurrence.
PROBABLE_SITE_BLOCK_STREAK_STOP_AT = 2
+UNRECOVERABLE_TOOL_ERROR_STOP_AT = 2
# Caps how many times the stop nudge can re-fire — without this, the streak
# stays latched while no new test runs reset it and every subsequent turn
# re-injects the same nudge until MAX_ITERATIONS. Independent of
@@ -249,6 +250,153 @@ def _maybe_raise_non_retriable_nav(ctx: Any) -> None:
raise CopilotNonRetriableNavError(url=_extract_url_from_nav_error(err), error_message=err)
+class CopilotUnrecoverableToolError(Exception):
+ """Raised when browser-session tool failures prove the current loop cannot recover."""
+
+ def __init__(self, tool_name: str, error_message: str) -> None:
+ self.tool_name = tool_name
+ self.error_message = error_message
+ super().__init__(f"Unrecoverable tool error in {tool_name}: {error_message}")
+
+
+_BROWSER_SESSION_TOOL_NAMES = frozenset(
+ {
+ "navigate_browser",
+ "get_browser_screenshot",
+ "evaluate",
+ "click",
+ "type_text",
+ "scroll",
+ "console_messages",
+ "select_option",
+ "press_key",
+ }
+)
+_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"
+_BROWSER_SESSION_ID_RE = re.compile(r"\bpbs_[A-Za-z0-9_-]+\b")
+_BROWSER_SESSION_WITH_ID_RE = re.compile(r"\bbrowser session\s+pbs_[A-Za-z0-9_-]+\b", re.IGNORECASE)
+
+
+def redact_browser_session_references(value: str) -> str:
+ value = _BROWSER_SESSION_WITH_ID_RE.sub("Browser session", value)
+ return _BROWSER_SESSION_ID_RE.sub("the browser session", value)
+
+
+def _result_text_values(value: Any) -> list[str]:
+ if isinstance(value, str):
+ return [value]
+ if isinstance(value, dict):
+ result: list[str] = []
+ for item in value.values():
+ result.extend(_result_text_values(item))
+ return result
+ if isinstance(value, list):
+ result = []
+ for item in value:
+ result.extend(_result_text_values(item))
+ return result
+ return []
+
+
+def _unrecoverable_tool_error_reason(output: dict[str, Any]) -> str:
+ raw_reason = output.get("error")
+ if not isinstance(raw_reason, str) or not raw_reason.strip():
+ data = output.get("data")
+ raw_reason = data.get("failure_reason") if isinstance(data, dict) else None
+ if not isinstance(raw_reason, str) or not raw_reason.strip():
+ raw_reason = " ".join(_result_text_values(output))
+ reason = " ".join(str(raw_reason or "Browser session was no longer reachable.").split())
+ reason = redact_browser_session_references(reason)
+ return reason[:240].rstrip()
+
+
+def _is_unrecoverable_browser_session_error(tool_name: str, output: dict[str, Any]) -> bool:
+ if tool_name not in _BROWSER_SESSION_TOOL_NAMES or output.get("ok", True):
+ return False
+ lowered = " ".join(_result_text_values(output)).lower()
+ if "no browser context" in lowered:
+ return True
+ has_session_signal = "browser session" in lowered or "browser context" in lowered
+ has_lost_signal = "not found" in lowered or "404" in lowered
+ return has_session_signal and has_lost_signal
+
+
+def _record_unrecoverable_tool_error_contract(ctx: Any, tool_name: str, reason: str) -> None:
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract
+
+ result = {
+ "ok": False,
+ "error": reason,
+ "data": {
+ "overall_status": "aborted",
+ "failure_reason": reason,
+ "failure_categories": [{"category": _UNRECOVERABLE_TOOL_ERROR_CATEGORY, "reasoning": reason}],
+ },
+ }
+ contract = build_diagnosis_repair_contract(source_tool=tool_name, result=result, ctx=ctx)
+ ctx.latest_diagnosis_repair_contract = contract
+ ctx.unrecoverable_tool_error_reason = reason
+ ctx.unrecoverable_tool_error_tool_name = tool_name
+ ctx.last_test_failure_reason = reason
+ trace_data = contract.to_trace_data()
+ LOG.warning(
+ "Copilot unrecoverable tool error stop",
+ tool_name=tool_name,
+ error_reason=reason,
+ **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},
+ )
+ with copilot_span("copilot_unrecoverable_tool_error", data={"tool_name": tool_name, **trace_data}):
+ pass
+
+
+def _maybe_raise_unrecoverable_tool_error(ctx: Any, tool_name: str, output: dict[str, Any]) -> None:
+ if not _is_unrecoverable_browser_session_error(tool_name, output):
+ if tool_name in _BROWSER_SESSION_TOOL_NAMES and output.get("ok", False):
+ ctx.unrecoverable_tool_error_streak_count = 0
+ ctx.unrecoverable_tool_error_signature = None
+ return
+
+ reason = _unrecoverable_tool_error_reason(output)
+ signature = "browser_session_unreachable"
+ prior_signature = getattr(ctx, "unrecoverable_tool_error_signature", None)
+ prior_count = getattr(ctx, "unrecoverable_tool_error_streak_count", 0)
+ prior_count = prior_count if isinstance(prior_count, int) else 0
+ count = prior_count + 1 if prior_signature == signature else 1
+ ctx.unrecoverable_tool_error_signature = signature
+ ctx.unrecoverable_tool_error_streak_count = count
+ ctx.unrecoverable_tool_error_reason = reason
+ ctx.unrecoverable_tool_error_tool_name = tool_name
+
+ if count >= UNRECOVERABLE_TOOL_ERROR_STOP_AT:
+ _record_unrecoverable_tool_error_contract(ctx, tool_name, reason)
+ raise CopilotUnrecoverableToolError(tool_name, reason)
+
+
+def _raise_if_unrecoverable_contract_stop(ctx: Any) -> None:
+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)
+ diagnosis = getattr(contract, "diagnosis_result", None)
+ repair_decision = getattr(contract, "repair_decision", None)
+ failure_type = getattr(getattr(diagnosis, "suspected_failure_type", None), "value", None) or getattr(
+ diagnosis,
+ "suspected_failure_type",
+ None,
+ )
+ next_action = getattr(getattr(repair_decision, "next_action", None), "value", None) or getattr(
+ repair_decision,
+ "next_action",
+ None,
+ )
+ if failure_type != "unrecoverable_tool_error" or next_action != "stop":
+ return
+ verification = getattr(contract, "verification_result", None)
+ reason = getattr(verification, "remaining_blocker", None) or getattr(diagnosis, "root_cause_summary", None)
+ if not isinstance(reason, str) or not reason.strip():
+ reason = "Browser session was no longer reachable."
+ source_tool = getattr(getattr(contract, "diagnosis_input", None), "source_tool", None)
+ tool_name = source_tool if isinstance(source_tool, str) and source_tool else "unknown"
+ raise CopilotUnrecoverableToolError(tool_name, reason)
+
+
_ACTION_CATEGORIES: list[list[str]] = [
["navigate", "go to", "open", "visit"],
["download", "save", "export"],
@@ -507,6 +655,8 @@ def _check_enforcement(
config: CopilotConfig | None = None,
) -> str | None:
# Terminal failure-mode signals must pre-empt tool-call hygiene nudges.
+ _raise_if_unrecoverable_contract_stop(ctx)
+
# A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be
# resolved by observing a prior navigate or by testing an updated
# workflow against the same bad URL, so let it speak first.
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 0f502f26f..6f5e62d33 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -53,6 +53,10 @@ class AgentContext:
pending_screenshots: list[ScreenshotEntry] = field(default_factory=list)
tool_activity: list[dict[str, Any]] = field(default_factory=list)
failed_tool_step_tracker: dict[str, int] = field(default_factory=dict)
+ unrecoverable_tool_error_streak_count: int = 0
+ unrecoverable_tool_error_signature: str | None = None
+ unrecoverable_tool_error_reason: str | None = None
+ unrecoverable_tool_error_tool_name: str | None = None
# Cross-turn agent state accumulated by tools.py as the agent runs.
# Read back by failure_tracking / loop_detection to detect stuck loops,
diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.py
index beb1bb846..1e52f48fb 100644
--- a/skyvern/forge/sdk/copilot/streaming_adapter.py
+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py
@@ -72,6 +72,11 @@ async def stream_to_sse(
"""
from agents.stream_events import RunItemStreamEvent
+ from skyvern.forge.sdk.copilot.enforcement import (
+ CopilotUnrecoverableToolError,
+ _maybe_raise_unrecoverable_tool_error,
+ )
+
call_id_to_name: dict[str, str] = {}
# Counts completed tool round-trips (tool_called + tool_output pair), not
# raw stream events. Both TOOL_CALL and TOOL_RESULT for the same round
@@ -188,6 +193,12 @@ async def stream_to_sse(
schedule_narration(narrator_state, stream, iteration)
else:
_update_enforcement_from_tool(ctx, tool_name, parsed)
+
+ try:
+ _maybe_raise_unrecoverable_tool_error(ctx, tool_name, parsed)
+ except CopilotUnrecoverableToolError:
+ result.cancel()
+ raise
iteration += 1
except asyncio.CancelledError:
# Real cancellation (server shutdown, upstream abort). Propagate so
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 2367e3b09..d7135236c 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -174,3 +174,24 @@ def test_stop_and_no_change_decisions_preserve_current_behavior_shadow_only() ->
assert no_change.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE
assert no_change.repair_decision.next_action == RepairNextAction.NO_CHANGE
assert no_change.verification_result.completion_contract_satisfied is True
+
+
+def test_unrecoverable_browser_session_contract_stops_with_blocker() -> None:
+ reason = "Browser session not found while taking screenshot (404)."
+ contract = build_diagnosis_repair_contract(
+ source_tool="get_browser_screenshot",
+ result={
+ "ok": False,
+ "error": reason,
+ "data": {
+ "overall_status": "aborted",
+ "failure_categories": [{"category": "UNRECOVERABLE_TOOL_ERROR"}],
+ },
+ },
+ ctx=_ctx(),
+ )
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR
+ assert contract.repair_decision.next_action == RepairNextAction.STOP
+ assert contract.verification_result.user_goal_satisfied is False
+ assert contract.verification_result.remaining_blocker == reason
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 2ec09e0f8..8e65d37eb 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -9,6 +9,7 @@ These cover three regressions observed in trace 019d7b5c884dff0ff648680b9f31f715
from __future__ import annotations
import json
+from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -112,6 +113,70 @@ def test_meaningful_data_string() -> None:
assert _is_meaningful_extracted_data("$260.48") is True
+def test_unrecoverable_browser_session_error_stops_after_second_failure() -> None:
+ from skyvern.forge.sdk.copilot.enforcement import (
+ CopilotUnrecoverableToolError,
+ _maybe_raise_unrecoverable_tool_error,
+ )
+
+ ctx = SimpleNamespace()
+ output = {"ok": False, "error": "Browser session not found while taking screenshot (404)."}
+
+ _maybe_raise_unrecoverable_tool_error(ctx, "get_browser_screenshot", output)
+ assert ctx.unrecoverable_tool_error_streak_count == 1
+
+ with pytest.raises(CopilotUnrecoverableToolError) as exc_info:
+ _maybe_raise_unrecoverable_tool_error(ctx, "get_browser_screenshot", output)
+
+ assert "Browser session not found" in str(exc_info.value)
+ assert ctx.unrecoverable_tool_error_streak_count == 2
+ contract = ctx.latest_diagnosis_repair_contract
+ assert contract.repair_decision.next_action == "stop"
+ assert contract.verification_result.remaining_blocker == "Browser session not found while taking screenshot (404)."
+
+
+def test_unrecoverable_tool_error_ignores_regular_website_404() -> None:
+ from skyvern.forge.sdk.copilot.enforcement import _maybe_raise_unrecoverable_tool_error
+
+ ctx = SimpleNamespace()
+
+ _maybe_raise_unrecoverable_tool_error(
+ ctx,
+ "navigate_browser",
+ {"ok": False, "error": "The page returned HTTP 404 page not found."},
+ )
+
+ assert getattr(ctx, "unrecoverable_tool_error_streak_count", 0) == 0
+ assert getattr(ctx, "latest_diagnosis_repair_contract", None) is None
+
+
+def test_unrecoverable_contract_stop_preempts_failed_test_nudge() -> None:
+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract
+ from skyvern.forge.sdk.copilot.enforcement import CopilotUnrecoverableToolError, _check_enforcement
+
+ ctx = _Ctx()
+ ctx.last_test_ok = False
+ reason = "Browser session not found while running blocks (404)."
+ ctx.latest_diagnosis_repair_contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result={
+ "ok": False,
+ "error": reason,
+ "data": {
+ "overall_status": "aborted",
+ "failure_reason": reason,
+ "failure_categories": [{"category": "UNRECOVERABLE_TOOL_ERROR"}],
+ },
+ },
+ ctx=ctx,
+ )
+
+ with pytest.raises(CopilotUnrecoverableToolError):
+ _check_enforcement(ctx)
+
+ assert ctx.failed_test_nudge_count == 0
+
+
# ---------------------------------------------------------------------------
# _analyze_run_blocks — envelope-unwrap for EXTRACTION blocks
#
diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.py
index c667f6444..d2d769ae4 100644
--- a/tests/unit/test_copilot_streaming_adapter.py
+++ b/tests/unit/test_copilot_streaming_adapter.py
@@ -293,6 +293,42 @@ async def test_tool_result_sse_summary_translates_loop_detected_failure() -> Non
assert "LOOP DETECTED" in agent_summary
+async def test_stream_to_sse_raises_and_cancels_on_repeated_unrecoverable_tool_error() -> None:
+ from agents.items import RunItem
+ from agents.stream_events import RunItemStreamEvent
+
+ from skyvern.forge.sdk.copilot.enforcement import CopilotUnrecoverableToolError
+
+ error_text = "Browser session pbs_123 not found while taking screenshot (404)."
+ events = []
+ for call_id in ("c1", "c2"):
+ call_item = MagicMock(spec=RunItem)
+ call_item.raw_item = {"call_id": call_id, "name": "get_browser_screenshot", "arguments": "{}"}
+ events.append(RunItemStreamEvent(name="tool_called", item=call_item))
+
+ output_item = MagicMock(spec=RunItem)
+ output_item.raw_item = {"call_id": call_id, "name": "get_browser_screenshot"}
+ output_item.output = [{"type": "text", "text": f'{{"ok": false, "error": "{error_text}"}}'}]
+ events.append(RunItemStreamEvent(name="tool_output", item=output_item))
+
+ result = MagicMock()
+ result.stream_events = lambda: _stream_events_from(*events)
+ result.cancel = MagicMock()
+
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ stream.send = AsyncMock(return_value=True)
+ ctx = SimpleNamespace()
+
+ with pytest.raises(CopilotUnrecoverableToolError):
+ await stream_to_sse(result, stream, ctx)
+
+ result.cancel.assert_called_once()
+ assert ctx.latest_diagnosis_repair_contract.repair_decision.next_action == "stop"
+ assert "pbs_" not in ctx.latest_diagnosis_repair_contract.verification_result.remaining_blocker
+
+
@pytest.mark.asyncio
async def test_tool_result_sse_summary_drops_click_selector_on_success() -> None:
"""A successful click emits an empty SSE summary; the call_id → tool_name
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
index ade6f32b2..27c9a59e1 100644
--- a/tests/unit/test_copilot_timeout_wip.py
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -23,6 +23,13 @@ from skyvern.forge.sdk.copilot.agent import (
_build_timeout_exit_result,
_build_unexpected_error_exit_result,
)
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisInput,
+ DiagnosisRepairContract,
+ DiagnosisResult,
+ RepairDecision,
+ VerificationResult,
+)
def _ctx(
@@ -44,9 +51,24 @@ def _ctx(
ctx.total_tokens_used = None
ctx.last_good_workflow = last_good_workflow
ctx.last_good_workflow_yaml = last_good_workflow_yaml
+ ctx.latest_diagnosis_repair_contract = None
+ ctx.test_after_update_done = last_test_ok is not None
+ ctx.last_update_block_count = None
return ctx
+def _blocker_contract(reason: str, *, run_status: str | None = "running") -> DiagnosisRepairContract:
+ return DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(source_tool="get_browser_screenshot", run_status=run_status),
+ diagnosis_result=DiagnosisResult(root_cause_summary=reason, confidence=0.9),
+ repair_decision=RepairDecision(next_action="stop"),
+ verification_result=VerificationResult(
+ run_status=run_status,
+ remaining_blocker=reason,
+ ),
+ )
+
+
class TestBuildTimeoutExitResult:
def test_no_workflow_falls_back_to_default_reply(self) -> None:
ctx = _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None)
@@ -257,6 +279,59 @@ class TestBuildUnexpectedErrorExitResult:
assert result.unvalidated is False
assert result.user_response == _UNEXPECTED_ERROR_REPLY_DEFAULT
+ def test_failed_test_uses_recorded_blocker_reply(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+ ctx.last_update_block_count = 3
+ ctx.latest_diagnosis_repair_contract = _blocker_contract("Browser session was no longer reachable.")
+
+ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
+
+ assert result.user_response == (
+ "I built a 3-block draft and tested it, but the test couldn't finish: "
+ "Browser session was no longer reachable. Last run status: running."
+ )
+
+ def test_aborted_test_surfaces_unvalidated_draft_with_recorded_blocker_reply(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx.test_after_update_done = True
+ ctx.last_update_block_count = 4
+ ctx.latest_diagnosis_repair_contract = _blocker_contract(
+ "The browser session disappeared before screenshot verification could complete.",
+ run_status="aborted",
+ )
+
+ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
+
+ assert result.unvalidated is True
+ assert result.user_response == (
+ "I built a 4-block draft and tested it, but the test couldn't finish: "
+ "The browser session disappeared before screenshot verification could complete. "
+ "Last run status: aborted."
+ )
+
+ def test_browser_only_blocker_does_not_claim_tested_and_redacts_internal_details(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+ ctx.test_after_update_done = False
+ ctx.last_update_block_count = 2
+ ctx.latest_diagnosis_repair_contract = _blocker_contract(
+ "Browser session pbs_123456 not found while reading https://example.test/path?token=secret.",
+ run_status="aborted",
+ )
+
+ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is wf
+ assert result.unvalidated is True
+ assert result.user_response == (
+ "I built a 2-block draft, but I couldn't verify it: "
+ "Browser session not found while reading https://example.test. Last run status: aborted."
+ )
+ assert "pbs_" not in result.user_response
+ assert "token=secret" not in result.user_response
+
def test_suspicious_success_drops_proposal(self) -> None:
wf = MagicMock(name="wf")
ctx = _ctx(
@@ -379,6 +454,26 @@ class TestWipExitSurfacesLastGoodWithForceReviewNotUnvalidated:
assert result.force_review is True
assert result.user_response == _UNEXPECTED_ERROR_REPLY_TESTED
+ def test_unexpected_error_with_overwrite_and_blocker_describes_latest_attempt_separately(self) -> None:
+ ctx = self._overwrite_ctx(last_test_ok=None)
+ ctx.last_update_block_count = 5
+ ctx.latest_diagnosis_repair_contract = _blocker_contract(
+ "Browser session pbs_789 not found during screenshot verification.",
+ run_status="aborted",
+ )
+
+ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is ctx.last_good_workflow
+ assert result.unvalidated is False
+ assert result.force_review is True
+ assert result.user_response == (
+ f"{_UNEXPECTED_ERROR_REPLY_TESTED} "
+ "The latest attempted change did not verify: "
+ "Browser session not found during screenshot verification. Last run status: aborted."
+ )
+ assert "pbs_" not in result.user_response
+
def test_cancelled_total_timeout_latch_uses_force_review_not_unvalidated(self) -> None:
ctx = self._overwrite_ctx(last_test_ok=None)
ctx.copilot_total_timeout_exceeded = True