Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10138: stop Copilot on unrecoverable browser-session errors (#6106)
Andrew Neilson49c33573e616
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex a1a592ce7..1ef17f2f7 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -47,6 +47,7 @@ from skyvern.forge.sdk.copilot.output_policy import (normalize_response_scaffolding,output_policy_verdict_from_trace_data,output_policy_verdict_to_trace_data,+ url_origin,)from skyvern.forge.sdk.copilot.output_utils import (extract_final_text,@@ -728,6 +729,93 @@ _UNVALIDATED_PROPOSAL_AFFORDANCE = ("I have a draft workflow proposal. Use Review to inspect it, Accept to save it, or Reject to discard it. ""It has not been tested or verified end-to-end.")+_RECORDED_FAILURE_URL_RE = re.compile(r"https?://[^\s)>,]+")+++def _workflow_block_count(ctx: CopilotContext) -> int | None:+ count = getattr(ctx, "last_update_block_count", None)+ if isinstance(count, int) and count > 0:+ return count+ workflow = getattr(ctx, "last_workflow", None)+ definition = getattr(workflow, "workflow_definition", None)+ blocks = getattr(definition, "blocks", None)+ return len(blocks) if isinstance(blocks, list) and blocks else None+++def _clean_recorded_failure_text(value: Any, max_chars: int = 240) -> str:+ from skyvern.forge.sdk.copilot.enforcement import redact_browser_session_references++ if not isinstance(value, str):+ return ""+ text = redact_raw_secrets_for_prompt(" ".join(value.split()))+ text = redact_browser_session_references(text)+ text = _RECORDED_FAILURE_URL_RE.sub(lambda match: url_origin(match.group(0)) or "[URL]", text)+ if len(text) > max_chars:+ text = text[: max_chars - 3].rstrip() + "..."+ return text.rstrip(".")+++def _recorded_failure_summary(ctx: CopilotContext) -> tuple[str, str]:+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)+ verification = getattr(contract, "verification_result", None)+ diagnosis = getattr(contract, "diagnosis_result", None)+ remaining_blocker = _clean_recorded_failure_text(getattr(verification, "remaining_blocker", None))+ root_cause = _clean_recorded_failure_text(getattr(diagnosis, "root_cause_summary", None))+ fallback_reason = _clean_recorded_failure_text(getattr(ctx, "last_test_failure_reason", None))+ reason = remaining_blocker or root_cause or fallback_reason+ run_status = _clean_recorded_failure_text(getattr(verification, "run_status", None), max_chars=80)+ status_sentence = f" Last run status: {run_status}." if run_status else ""+ return reason, status_sentence+++def _last_good_failure_reply(ctx: CopilotContext, tested_reply: str) -> str:+ reason, status_sentence = _recorded_failure_summary(ctx)+ if not reason:+ return tested_reply+ return f"{tested_reply} The latest attempted change did not verify: {reason}.{status_sentence}"+++def _recorded_failure_reply(ctx: CopilotContext, *, cancelled: bool = False) -> str | None:+ if cancelled or ctx.last_test_ok is True:+ return None++ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)+ verification = getattr(contract, "verification_result", None)+ diagnosis = getattr(contract, "diagnosis_result", None)+ repair_decision = getattr(contract, "repair_decision", None)+ diagnosis_input = getattr(contract, "diagnosis_input", None)+ failure_type = getattr(getattr(diagnosis, "suspected_failure_type", None), "value", None) or getattr(+ diagnosis,+ "suspected_failure_type",+ None,+ )+ next_action = getattr(getattr(repair_decision, "next_action", None), "value", None) or getattr(+ repair_decision,+ "next_action",+ None,+ )+ reason, status_sentence = _recorded_failure_summary(ctx)+ if not reason:+ return None++ run_status = _clean_recorded_failure_text(getattr(verification, "run_status", None), max_chars=80).lower()+ block_count = _workflow_block_count(ctx)+ block_phrase = f"a {block_count}-block draft" if block_count else "a draft"+ test_attempted = bool(+ getattr(ctx, "test_after_update_done", False)+ or getattr(ctx, "last_test_ok", None) is not None+ or getattr(diagnosis_input, "workflow_run_id", None)+ )+ test_failed = ctx.last_test_ok is False or run_status == "failed"+ unrecoverable_stop = next_action == "stop" or failure_type == "unrecoverable_tool_error"++ if getattr(ctx, "last_workflow", None) is not None:+ if test_attempted and test_failed and not unrecoverable_stop:+ return f"I built {block_phrase} and tested it, but the test failed: {reason}.{status_sentence}"+ if test_attempted:+ return f"I built {block_phrase} and tested it, but the test couldn't finish: {reason}.{status_sentence}"+ return f"I built {block_phrase}, but I couldn't verify it: {reason}.{status_sentence}"+ return f"I couldn't finish the Copilot turn: {reason}.{status_sentence}"def _ensure_unvalidated_proposal_affordance(user_response: str) -> str:@@ -753,6 +841,7 @@ def _build_wip_exit_result(cancelled: bool = False,) -> AgentResult:"""Selected non-success exits surface the most recent successfully parsed workflow."""+ recorded_failure_reply = _recorded_failure_reply(ctx, cancelled=cancelled)# When an unverified edit/run has overwritten ``last_workflow``, prefer the# verified shape while still forcing explicit review.if (@@ -762,7 +851,7 @@ def _build_wip_exit_result(and not ctx.last_test_suspicious_success):return AgentResult(- user_response=tested_reply,+ user_response=_last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_reply,updated_workflow=ctx.last_good_workflow,global_llm_context=global_llm_context,workflow_yaml=ctx.last_good_workflow_yaml,@@ -778,7 +867,10 @@ def _build_wip_exit_result(and not ctx.last_test_suspicious_success):unvalidated = ctx.last_test_ok is not True- reply = unvalidated_reply if unvalidated else tested_reply+ if unvalidated and recorded_failure_reply:+ reply = recorded_failure_reply+ else:+ reply = unvalidated_reply if unvalidated else tested_replyreturn AgentResult(user_response=reply,updated_workflow=ctx.last_workflow,@@ -789,7 +881,7 @@ def _build_wip_exit_result(proposal_disposition="review_untested" if unvalidated else "auto_applicable",cancelled=cancelled,)- return _build_exit_result(ctx, default_reply, global_llm_context, cancelled=cancelled)+ return _build_exit_result(ctx, recorded_failure_reply or default_reply, global_llm_context, cancelled=cancelled)def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:@@ -1674,6 +1766,7 @@ async def _run_copilot_turn_impl(from skyvern.forge.sdk.copilot.enforcement import (CopilotNonRetriableNavError,CopilotTotalTimeoutError,+ CopilotUnrecoverableToolError,run_with_enforcement,)from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks@@ -1867,6 +1960,14 @@ async def _run_copilot_turn_impl(return _build_max_turns_exit_result(ctx, global_llm_context)except CopilotTotalTimeoutError:return _build_timeout_exit_result(ctx, global_llm_context)+ except CopilotUnrecoverableToolError as exc:+ LOG.warning(+ "Copilot run halted on unrecoverable tool error",+ tool_name=exc.tool_name,+ error_message=exc.error_message,+ organization_id=organization_id,+ )+ return _build_unexpected_error_exit_result(ctx, global_llm_context)except CopilotNonRetriableNavError as exc:LOG.warning("Copilot run halted on non-retriable navigation error",diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex 636b62d9b..dbb7586a4 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -26,6 +26,7 @@ class DiagnosisFailureType(StrEnum):SUSPICIOUS_SUCCESS = "suspicious_success"MISSING_CREDENTIAL_OR_INIT = "missing_credential_or_init"REPAIRABLE_BLOCK_FAILURE = "repairable_block_failure"+ UNRECOVERABLE_TOOL_ERROR = "unrecoverable_tool_error"UNKNOWN = "unknown"@@ -298,6 +299,14 @@ def _failure_type(for value in (result.get("error"), data.get("failure_reason"), data.get("skip_reason"))if value)+ if (+ "UNRECOVERABLE_TOOL_ERROR" in categories+ or "browser session not found" in error_text+ or "no browser context" in error_text+ or ("session not found" in error_text and "browser" in error_text)+ or ("404" in error_text and "browser session" in error_text)+ ):+ return DiagnosisFailureType.UNRECOVERABLE_TOOL_ERRORif (data.get("skip_reason") == "workflow_credential_inputs_unbound"or "credential" in error_text@@ -320,6 +329,8 @@ def _next_action(failure_type: DiagnosisFailureType, ctx: Any, data: dict[str, Aor failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT):return RepairNextAction.ASK+ if failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR:+ return RepairNextAction.STOPif getattr(ctx, "last_test_non_retriable_nav_error", None):return RepairNextAction.STOPauthority = getattr(getattr(ctx, "turn_intent", None), "authority", None)diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 39713fa83..0a109808a 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -77,6 +77,7 @@ REPEATED_FRONTIER_STREAK_STOP_AT = 3# scraper could not read the page. Aligned with MAX_FAILED_TEST_NUDGES so the# copilot gets one generic retry nudge, then stops on the second occurrence.PROBABLE_SITE_BLOCK_STREAK_STOP_AT = 2+UNRECOVERABLE_TOOL_ERROR_STOP_AT = 2# Caps how many times the stop nudge can re-fire — without this, the streak# stays latched while no new test runs reset it and every subsequent turn# re-injects the same nudge until MAX_ITERATIONS. Independent of@@ -249,6 +250,153 @@ def _maybe_raise_non_retriable_nav(ctx: Any) -> None:raise CopilotNonRetriableNavError(url=_extract_url_from_nav_error(err), error_message=err)+class CopilotUnrecoverableToolError(Exception):+ """Raised when browser-session tool failures prove the current loop cannot recover."""++ def __init__(self, tool_name: str, error_message: str) -> None:+ self.tool_name = tool_name+ self.error_message = error_message+ super().__init__(f"Unrecoverable tool error in {tool_name}: {error_message}")+++_BROWSER_SESSION_TOOL_NAMES = frozenset(+ {+ "navigate_browser",+ "get_browser_screenshot",+ "evaluate",+ "click",+ "type_text",+ "scroll",+ "console_messages",+ "select_option",+ "press_key",+ }+)+_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"+_BROWSER_SESSION_ID_RE = re.compile(r"\bpbs_[A-Za-z0-9_-]+\b")+_BROWSER_SESSION_WITH_ID_RE = re.compile(r"\bbrowser session\s+pbs_[A-Za-z0-9_-]+\b", re.IGNORECASE)+++def redact_browser_session_references(value: str) -> str:+ value = _BROWSER_SESSION_WITH_ID_RE.sub("Browser session", value)+ return _BROWSER_SESSION_ID_RE.sub("the browser session", value)+++def _result_text_values(value: Any) -> list[str]:+ if isinstance(value, str):+ return [value]+ if isinstance(value, dict):+ result: list[str] = []+ for item in value.values():+ result.extend(_result_text_values(item))+ return result+ if isinstance(value, list):+ result = []+ for item in value:+ result.extend(_result_text_values(item))+ return result+ return []+++def _unrecoverable_tool_error_reason(output: dict[str, Any]) -> str:+ raw_reason = output.get("error")+ if not isinstance(raw_reason, str) or not raw_reason.strip():+ data = output.get("data")+ raw_reason = data.get("failure_reason") if isinstance(data, dict) else None+ if not isinstance(raw_reason, str) or not raw_reason.strip():+ raw_reason = " ".join(_result_text_values(output))+ reason = " ".join(str(raw_reason or "Browser session was no longer reachable.").split())+ reason = redact_browser_session_references(reason)+ return reason[:240].rstrip()+++def _is_unrecoverable_browser_session_error(tool_name: str, output: dict[str, Any]) -> bool:+ if tool_name not in _BROWSER_SESSION_TOOL_NAMES or output.get("ok", True):+ return False+ lowered = " ".join(_result_text_values(output)).lower()+ if "no browser context" in lowered:+ return True+ has_session_signal = "browser session" in lowered or "browser context" in lowered+ has_lost_signal = "not found" in lowered or "404" in lowered+ return has_session_signal and has_lost_signal+++def _record_unrecoverable_tool_error_contract(ctx: Any, tool_name: str, reason: str) -> None:+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract++ result = {+ "ok": False,+ "error": reason,+ "data": {+ "overall_status": "aborted",+ "failure_reason": reason,+ "failure_categories": [{"category": _UNRECOVERABLE_TOOL_ERROR_CATEGORY, "reasoning": reason}],+ },+ }+ contract = build_diagnosis_repair_contract(source_tool=tool_name, result=result, ctx=ctx)+ ctx.latest_diagnosis_repair_contract = contract+ ctx.unrecoverable_tool_error_reason = reason+ ctx.unrecoverable_tool_error_tool_name = tool_name+ ctx.last_test_failure_reason = reason+ trace_data = contract.to_trace_data()+ LOG.warning(+ "Copilot unrecoverable tool error stop",+ tool_name=tool_name,+ error_reason=reason,+ **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},+ )+ with copilot_span("copilot_unrecoverable_tool_error", data={"tool_name": tool_name, **trace_data}):+ pass+++def _maybe_raise_unrecoverable_tool_error(ctx: Any, tool_name: str, output: dict[str, Any]) -> None:+ if not _is_unrecoverable_browser_session_error(tool_name, output):+ if tool_name in _BROWSER_SESSION_TOOL_NAMES and output.get("ok", False):+ ctx.unrecoverable_tool_error_streak_count = 0+ ctx.unrecoverable_tool_error_signature = None+ return++ reason = _unrecoverable_tool_error_reason(output)+ signature = "browser_session_unreachable"+ prior_signature = getattr(ctx, "unrecoverable_tool_error_signature", None)+ prior_count = getattr(ctx, "unrecoverable_tool_error_streak_count", 0)+ prior_count = prior_count if isinstance(prior_count, int) else 0+ count = prior_count + 1 if prior_signature == signature else 1+ ctx.unrecoverable_tool_error_signature = signature+ ctx.unrecoverable_tool_error_streak_count = count+ ctx.unrecoverable_tool_error_reason = reason+ ctx.unrecoverable_tool_error_tool_name = tool_name++ if count >= UNRECOVERABLE_TOOL_ERROR_STOP_AT:+ _record_unrecoverable_tool_error_contract(ctx, tool_name, reason)+ raise CopilotUnrecoverableToolError(tool_name, reason)+++def _raise_if_unrecoverable_contract_stop(ctx: Any) -> None:+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)+ diagnosis = getattr(contract, "diagnosis_result", None)+ repair_decision = getattr(contract, "repair_decision", None)+ failure_type = getattr(getattr(diagnosis, "suspected_failure_type", None), "value", None) or getattr(+ diagnosis,+ "suspected_failure_type",+ None,+ )+ next_action = getattr(getattr(repair_decision, "next_action", None), "value", None) or getattr(+ repair_decision,+ "next_action",+ None,+ )+ if failure_type != "unrecoverable_tool_error" or next_action != "stop":+ return+ verification = getattr(contract, "verification_result", None)+ reason = getattr(verification, "remaining_blocker", None) or getattr(diagnosis, "root_cause_summary", None)+ if not isinstance(reason, str) or not reason.strip():+ reason = "Browser session was no longer reachable."+ source_tool = getattr(getattr(contract, "diagnosis_input", None), "source_tool", None)+ tool_name = source_tool if isinstance(source_tool, str) and source_tool else "unknown"+ raise CopilotUnrecoverableToolError(tool_name, reason)++_ACTION_CATEGORIES: list[list[str]] = [["navigate", "go to", "open", "visit"],["download", "save", "export"],@@ -507,6 +655,8 @@ def _check_enforcement(config: CopilotConfig | None = None,) -> str | None:# Terminal failure-mode signals must pre-empt tool-call hygiene nudges.+ _raise_if_unrecoverable_contract_stop(ctx)+# A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be# resolved by observing a prior navigate or by testing an updated# workflow against the same bad URL, so let it speak first.diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 0f502f26f..6f5e62d33 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -53,6 +53,10 @@ class AgentContext:pending_screenshots: list[ScreenshotEntry] = field(default_factory=list)tool_activity: list[dict[str, Any]] = field(default_factory=list)failed_tool_step_tracker: dict[str, int] = field(default_factory=dict)+ unrecoverable_tool_error_streak_count: int = 0+ unrecoverable_tool_error_signature: str | None = None+ unrecoverable_tool_error_reason: str | None = None+ unrecoverable_tool_error_tool_name: str | None = None# Cross-turn agent state accumulated by tools.py as the agent runs.# Read back by failure_tracking / loop_detection to detect stuck loops,diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.pyindex beb1bb846..1e52f48fb 100644--- a/skyvern/forge/sdk/copilot/streaming_adapter.py+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py@@ -72,6 +72,11 @@ async def stream_to_sse("""from agents.stream_events import RunItemStreamEvent+ from skyvern.forge.sdk.copilot.enforcement import (+ CopilotUnrecoverableToolError,+ _maybe_raise_unrecoverable_tool_error,+ )+call_id_to_name: dict[str, str] = {}# Counts completed tool round-trips (tool_called + tool_output pair), not# raw stream events. Both TOOL_CALL and TOOL_RESULT for the same round@@ -188,6 +193,12 @@ async def stream_to_sse(schedule_narration(narrator_state, stream, iteration)else:_update_enforcement_from_tool(ctx, tool_name, parsed)++ try:+ _maybe_raise_unrecoverable_tool_error(ctx, tool_name, parsed)+ except CopilotUnrecoverableToolError:+ result.cancel()+ raiseiteration += 1except asyncio.CancelledError:# Real cancellation (server shutdown, upstream abort). Propagate sodiff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 2367e3b09..d7135236c 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -174,3 +174,24 @@ def test_stop_and_no_change_decisions_preserve_current_behavior_shadow_only() ->assert no_change.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILUREassert no_change.repair_decision.next_action == RepairNextAction.NO_CHANGEassert no_change.verification_result.completion_contract_satisfied is True+++def test_unrecoverable_browser_session_contract_stops_with_blocker() -> None:+ reason = "Browser session not found while taking screenshot (404)."+ contract = build_diagnosis_repair_contract(+ source_tool="get_browser_screenshot",+ result={+ "ok": False,+ "error": reason,+ "data": {+ "overall_status": "aborted",+ "failure_categories": [{"category": "UNRECOVERABLE_TOOL_ERROR"}],+ },+ },+ ctx=_ctx(),+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR+ assert contract.repair_decision.next_action == RepairNextAction.STOP+ assert contract.verification_result.user_goal_satisfied is False+ assert contract.verification_result.remaining_blocker == reasondiff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex 2ec09e0f8..8e65d37eb 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -9,6 +9,7 @@ These cover three regressions observed in trace 019d7b5c884dff0ff648680b9f31f715from __future__ import annotationsimport json+from types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock@@ -112,6 +113,70 @@ def test_meaningful_data_string() -> None:assert _is_meaningful_extracted_data("$260.48") is True+def test_unrecoverable_browser_session_error_stops_after_second_failure() -> None:+ from skyvern.forge.sdk.copilot.enforcement import (+ CopilotUnrecoverableToolError,+ _maybe_raise_unrecoverable_tool_error,+ )++ ctx = SimpleNamespace()+ output = {"ok": False, "error": "Browser session not found while taking screenshot (404)."}++ _maybe_raise_unrecoverable_tool_error(ctx, "get_browser_screenshot", output)+ assert ctx.unrecoverable_tool_error_streak_count == 1++ with pytest.raises(CopilotUnrecoverableToolError) as exc_info:+ _maybe_raise_unrecoverable_tool_error(ctx, "get_browser_screenshot", output)++ assert "Browser session not found" in str(exc_info.value)+ assert ctx.unrecoverable_tool_error_streak_count == 2+ contract = ctx.latest_diagnosis_repair_contract+ assert contract.repair_decision.next_action == "stop"+ assert contract.verification_result.remaining_blocker == "Browser session not found while taking screenshot (404)."+++def test_unrecoverable_tool_error_ignores_regular_website_404() -> None:+ from skyvern.forge.sdk.copilot.enforcement import _maybe_raise_unrecoverable_tool_error++ ctx = SimpleNamespace()++ _maybe_raise_unrecoverable_tool_error(+ ctx,+ "navigate_browser",+ {"ok": False, "error": "The page returned HTTP 404 page not found."},+ )++ assert getattr(ctx, "unrecoverable_tool_error_streak_count", 0) == 0+ assert getattr(ctx, "latest_diagnosis_repair_contract", None) is None+++def test_unrecoverable_contract_stop_preempts_failed_test_nudge() -> None:+ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract+ from skyvern.forge.sdk.copilot.enforcement import CopilotUnrecoverableToolError, _check_enforcement++ ctx = _Ctx()+ ctx.last_test_ok = False+ reason = "Browser session not found while running blocks (404)."+ ctx.latest_diagnosis_repair_contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": False,+ "error": reason,+ "data": {+ "overall_status": "aborted",+ "failure_reason": reason,+ "failure_categories": [{"category": "UNRECOVERABLE_TOOL_ERROR"}],+ },+ },+ ctx=ctx,+ )++ with pytest.raises(CopilotUnrecoverableToolError):+ _check_enforcement(ctx)++ assert ctx.failed_test_nudge_count == 0++# ---------------------------------------------------------------------------# _analyze_run_blocks — envelope-unwrap for EXTRACTION blocks#diff --git a/tests/unit/test_copilot_streaming_adapter.py b/tests/unit/test_copilot_streaming_adapter.pyindex c667f6444..d2d769ae4 100644--- a/tests/unit/test_copilot_streaming_adapter.py+++ b/tests/unit/test_copilot_streaming_adapter.py@@ -293,6 +293,42 @@ async def test_tool_result_sse_summary_translates_loop_detected_failure() -> Nonassert "LOOP DETECTED" in agent_summary+async def test_stream_to_sse_raises_and_cancels_on_repeated_unrecoverable_tool_error() -> None:+ from agents.items import RunItem+ from agents.stream_events import RunItemStreamEvent++ from skyvern.forge.sdk.copilot.enforcement import CopilotUnrecoverableToolError++ error_text = "Browser session pbs_123 not found while taking screenshot (404)."+ events = []+ for call_id in ("c1", "c2"):+ call_item = MagicMock(spec=RunItem)+ call_item.raw_item = {"call_id": call_id, "name": "get_browser_screenshot", "arguments": "{}"}+ events.append(RunItemStreamEvent(name="tool_called", item=call_item))++ output_item = MagicMock(spec=RunItem)+ output_item.raw_item = {"call_id": call_id, "name": "get_browser_screenshot"}+ output_item.output = [{"type": "text", "text": f'{{"ok": false, "error": "{error_text}"}}'}]+ events.append(RunItemStreamEvent(name="tool_output", item=output_item))++ result = MagicMock()+ result.stream_events = lambda: _stream_events_from(*events)+ result.cancel = MagicMock()++ stream = MagicMock()+ stream.is_disconnected = AsyncMock(return_value=False)+ stream.send = AsyncMock(return_value=True)+ ctx = SimpleNamespace()++ with pytest.raises(CopilotUnrecoverableToolError):+ await stream_to_sse(result, stream, ctx)++ result.cancel.assert_called_once()+ assert ctx.latest_diagnosis_repair_contract.repair_decision.next_action == "stop"+ assert "pbs_" not in ctx.latest_diagnosis_repair_contract.verification_result.remaining_blocker++@pytest.mark.asyncioasync def test_tool_result_sse_summary_drops_click_selector_on_success() -> None:"""A successful click emits an empty SSE summary; the call_id → tool_namediff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex ade6f32b2..27c9a59e1 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -23,6 +23,13 @@ from skyvern.forge.sdk.copilot.agent import (_build_timeout_exit_result,_build_unexpected_error_exit_result,)+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (+ DiagnosisInput,+ DiagnosisRepairContract,+ DiagnosisResult,+ RepairDecision,+ VerificationResult,+)def _ctx(@@ -44,9 +51,24 @@ def _ctx(ctx.total_tokens_used = Nonectx.last_good_workflow = last_good_workflowctx.last_good_workflow_yaml = last_good_workflow_yaml+ ctx.latest_diagnosis_repair_contract = None+ ctx.test_after_update_done = last_test_ok is not None+ ctx.last_update_block_count = Nonereturn ctx+def _blocker_contract(reason: str, *, run_status: str | None = "running") -> DiagnosisRepairContract:+ return DiagnosisRepairContract(+ diagnosis_input=DiagnosisInput(source_tool="get_browser_screenshot", run_status=run_status),+ diagnosis_result=DiagnosisResult(root_cause_summary=reason, confidence=0.9),+ repair_decision=RepairDecision(next_action="stop"),+ verification_result=VerificationResult(+ run_status=run_status,+ remaining_blocker=reason,+ ),+ )++class TestBuildTimeoutExitResult:def test_no_workflow_falls_back_to_default_reply(self) -> None:ctx = _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None)@@ -257,6 +279,59 @@ class TestBuildUnexpectedErrorExitResult:assert result.unvalidated is Falseassert result.user_response == _UNEXPECTED_ERROR_REPLY_DEFAULT+ def test_failed_test_uses_recorded_blocker_reply(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)+ ctx.last_update_block_count = 3+ ctx.latest_diagnosis_repair_contract = _blocker_contract("Browser session was no longer reachable.")++ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)++ assert result.user_response == (+ "I built a 3-block draft and tested it, but the test couldn't finish: "+ "Browser session was no longer reachable. Last run status: running."+ )++ def test_aborted_test_surfaces_unvalidated_draft_with_recorded_blocker_reply(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx.test_after_update_done = True+ ctx.last_update_block_count = 4+ ctx.latest_diagnosis_repair_contract = _blocker_contract(+ "The browser session disappeared before screenshot verification could complete.",+ run_status="aborted",+ )++ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)++ assert result.unvalidated is True+ assert result.user_response == (+ "I built a 4-block draft and tested it, but the test couldn't finish: "+ "The browser session disappeared before screenshot verification could complete. "+ "Last run status: aborted."+ )++ def test_browser_only_blocker_does_not_claim_tested_and_redacts_internal_details(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)+ ctx.test_after_update_done = False+ ctx.last_update_block_count = 2+ ctx.latest_diagnosis_repair_contract = _blocker_contract(+ "Browser session pbs_123456 not found while reading https://example.test/path?token=secret.",+ run_status="aborted",+ )++ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is wf+ assert result.unvalidated is True+ assert result.user_response == (+ "I built a 2-block draft, but I couldn't verify it: "+ "Browser session not found while reading https://example.test. Last run status: aborted."+ )+ assert "pbs_" not in result.user_response+ assert "token=secret" not in result.user_response+def test_suspicious_success_drops_proposal(self) -> None:wf = MagicMock(name="wf")ctx = _ctx(@@ -379,6 +454,26 @@ class TestWipExitSurfacesLastGoodWithForceReviewNotUnvalidated:assert result.force_review is Trueassert result.user_response == _UNEXPECTED_ERROR_REPLY_TESTED+ def test_unexpected_error_with_overwrite_and_blocker_describes_latest_attempt_separately(self) -> None:+ ctx = self._overwrite_ctx(last_test_ok=None)+ ctx.last_update_block_count = 5+ ctx.latest_diagnosis_repair_contract = _blocker_contract(+ "Browser session pbs_789 not found during screenshot verification.",+ run_status="aborted",+ )++ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is ctx.last_good_workflow+ assert result.unvalidated is False+ assert result.force_review is True+ assert result.user_response == (+ f"{_UNEXPECTED_ERROR_REPLY_TESTED} "+ "The latest attempted change did not verify: "+ "Browser session not found during screenshot verification. Last run status: aborted."+ )+ assert "pbs_" not in result.user_response+def test_cancelled_total_timeout_latch_uses_force_review_not_unvalidated(self) -> None:ctx = self._overwrite_ctx(last_test_ok=None)ctx.copilot_total_timeout_exceeded = True