Skip to content

Automate browser based workflows with AI

fix(SKY-11618): drive native <select> via select_option under unrelated overlays (#7334)

Aaron Perezea8057d6ec71
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 6604096c9..6c9e8a5bf 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -17,6 +17,7 @@ from agents import ModelSettings, RunConfig
from agents.run import Runner
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
+from skyvern.forge.sdk.copilot import streaming_adapter
from skyvern.forge.sdk.copilot.blocker_signal import (
SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE,
@@ -151,6 +152,12 @@ from skyvern.forge.sdk.copilot.turn_ownership import (
claim_turn,
emit_blocker_signal_payload,
)
+from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (
+ CopilotUnrecoverableToolError as CopilotUnrecoverableToolError,
+)
+from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (
+ _maybe_raise_unrecoverable_tool_error as _maybe_raise_unrecoverable_tool_error,
+)
from skyvern.utils.token_counter import count_tokens
if TYPE_CHECKING:
@@ -185,7 +192,6 @@ REPEATED_FRONTIER_STREAK_STOP_AT = 3
# scraper could not read the page. Aligned with MAX_FAILED_TEST_NUDGES so the
# copilot gets one generic retry nudge, then stops on the second occurrence.
PROBABLE_SITE_BLOCK_STREAK_STOP_AT = 2
-UNRECOVERABLE_TOOL_ERROR_STOP_AT = 2
# Caps how many times the stop nudge can re-fire — without this, the streak
# stays latched while no new test runs reset it and every subsequent turn
# re-injects the same nudge until MAX_ITERATIONS. Independent of
@@ -980,127 +986,7 @@ def _maybe_raise_non_retriable_nav(ctx: Any) -> None:
raise CopilotNonRetriableNavError(url=_extract_url_from_nav_error(err), error_message=err)
-class CopilotUnrecoverableToolError(Exception):
- """Raised when browser-session tool failures prove the current loop cannot recover."""
-
- def __init__(self, tool_name: str, error_message: str) -> None:
- self.tool_name = tool_name
- self.error_message = error_message
- super().__init__(f"Unrecoverable tool error in {tool_name}: {error_message}")
-
-
-_BROWSER_SESSION_TOOL_NAMES = frozenset(
- {
- "navigate_browser",
- "get_browser_screenshot",
- "evaluate",
- "click",
- "type_text",
- "scroll",
- "console_messages",
- "select_option",
- "press_key",
- }
-)
_POST_RUN_PAGE_OBSERVATION_TOOLS = frozenset({"evaluate", "get_browser_screenshot", "inspect_page_for_composition"})
-_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"
-_BROWSER_SESSION_ID_RE = re.compile(r"\bpbs_[A-Za-z0-9_-]+\b")
-_BROWSER_SESSION_WITH_ID_RE = re.compile(r"\bbrowser session\s+pbs_[A-Za-z0-9_-]+\b", re.IGNORECASE)
-
-
-def redact_browser_session_references(value: str) -> str:
- value = _BROWSER_SESSION_WITH_ID_RE.sub("Browser session", value)
- return _BROWSER_SESSION_ID_RE.sub("the browser session", value)
-
-
-def _result_text_values(value: Any) -> list[str]:
- if isinstance(value, str):
- return [value]
- if isinstance(value, dict):
- result: list[str] = []
- for item in value.values():
- result.extend(_result_text_values(item))
- return result
- if isinstance(value, list):
- result = []
- for item in value:
- result.extend(_result_text_values(item))
- return result
- return []
-
-
-def _unrecoverable_tool_error_reason(output: dict[str, Any]) -> str:
- raw_reason = output.get("error")
- if not isinstance(raw_reason, str) or not raw_reason.strip():
- data = output.get("data")
- raw_reason = data.get("failure_reason") if isinstance(data, dict) else None
- if not isinstance(raw_reason, str) or not raw_reason.strip():
- raw_reason = " ".join(_result_text_values(output))
- reason = " ".join(str(raw_reason or "Browser session was no longer reachable.").split())
- reason = redact_browser_session_references(reason)
- return reason[:240].rstrip()
-
-
-def _is_unrecoverable_browser_session_error(tool_name: str, output: dict[str, Any]) -> bool:
- if tool_name not in _BROWSER_SESSION_TOOL_NAMES or output.get("ok", True):
- return False
- lowered = " ".join(_result_text_values(output)).lower()
- if "no browser context" in lowered:
- return True
- has_session_signal = "browser session" in lowered or "browser context" in lowered
- has_lost_signal = "not found" in lowered or "404" in lowered
- return has_session_signal and has_lost_signal
-
-
-def _record_unrecoverable_tool_error_contract(ctx: Any, tool_name: str, reason: str) -> None:
- from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract
-
- result = {
- "ok": False,
- "error": reason,
- "data": {
- "overall_status": "aborted",
- "failure_reason": reason,
- "failure_categories": [{"category": _UNRECOVERABLE_TOOL_ERROR_CATEGORY, "reasoning": reason}],
- },
- }
- contract = build_diagnosis_repair_contract(source_tool=tool_name, result=result, ctx=ctx)
- ctx.latest_diagnosis_repair_contract = contract
- ctx.unrecoverable_tool_error_reason = reason
- ctx.unrecoverable_tool_error_tool_name = tool_name
- ctx.last_test_failure_reason = reason
- trace_data = contract.to_trace_data()
- LOG.warning(
- "Copilot unrecoverable tool error stop",
- tool_name=tool_name,
- error_reason=reason,
- **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},
- )
- with copilot_span("copilot_unrecoverable_tool_error", data={"tool_name": tool_name, **trace_data}):
- pass
-
-
-def _maybe_raise_unrecoverable_tool_error(ctx: Any, tool_name: str, output: dict[str, Any]) -> None:
- if not _is_unrecoverable_browser_session_error(tool_name, output):
- if tool_name in _BROWSER_SESSION_TOOL_NAMES and output.get("ok", False):
- ctx.unrecoverable_tool_error_streak_count = 0
- ctx.unrecoverable_tool_error_signature = None
- return
-
- reason = _unrecoverable_tool_error_reason(output)
- signature = "browser_session_unreachable"
- prior_signature = getattr(ctx, "unrecoverable_tool_error_signature", None)
- prior_count = getattr(ctx, "unrecoverable_tool_error_streak_count", 0)
- prior_count = prior_count if isinstance(prior_count, int) else 0
- count = prior_count + 1 if prior_signature == signature else 1
- ctx.unrecoverable_tool_error_signature = signature
- ctx.unrecoverable_tool_error_streak_count = count
- ctx.unrecoverable_tool_error_reason = reason
- ctx.unrecoverable_tool_error_tool_name = tool_name
-
- if count >= UNRECOVERABLE_TOOL_ERROR_STOP_AT:
- _record_unrecoverable_tool_error_contract(ctx, tool_name, reason)
- raise CopilotUnrecoverableToolError(tool_name, reason)
def _raise_if_unrecoverable_contract_stop(ctx: Any) -> None:
@@ -2209,14 +2095,12 @@ async def _run_streamed_with_deadline(
``MIN_DEADLINE_REMAINING_SECONDS`` floors ``remaining`` so
``wait_for(timeout=0)`` never panics on an already-spent budget.
"""
- from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse
-
elapsed = time.monotonic() - start_time
remaining = max(MIN_DEADLINE_REMAINING_SECONDS, TOTAL_TIMEOUT_SECONDS - elapsed)
result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)
try:
try:
- await asyncio.wait_for(stream_to_sse(result, tracked_stream, ctx), timeout=remaining)
+ await asyncio.wait_for(streaming_adapter.stream_to_sse(result, tracked_stream, ctx), timeout=remaining)
finally:
_accumulate_usage(result, ctx)
except asyncio.TimeoutError:
diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.py
index bf4b62602..3ea7b1ca2 100644
--- a/skyvern/forge/sdk/copilot/streaming_adapter.py
+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py
@@ -10,6 +10,7 @@ from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any
import structlog
+from agents.stream_events import RawResponsesStreamEvent, RunItemStreamEvent
from skyvern.config import settings
@@ -33,6 +34,16 @@ from skyvern.forge.sdk.copilot.narration import (
tool_activity_display_label,
)
from skyvern.forge.sdk.copilot.output_utils import format_tool_result_for_user, summarize_tool_result_detail
+from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
+from skyvern.forge.sdk.copilot.turn_halt import (
+ CopilotTurnHalt,
+ raise_if_turn_halt,
+ stash_turn_halt_from_blocker_signal,
+)
+from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (
+ CopilotUnrecoverableToolError,
+ _maybe_raise_unrecoverable_tool_error,
+)
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotCodegenProgressUpdate,
WorkflowCopilotDesignEndUpdate,
@@ -251,19 +262,6 @@ async def stream_to_sse(
reasons unrelated to a dropped client) is re-raised unchanged so
asyncio's cancellation machinery still runs normally.
"""
- from agents.stream_events import RawResponsesStreamEvent, RunItemStreamEvent
-
- from skyvern.forge.sdk.copilot.enforcement import (
- CopilotUnrecoverableToolError,
- _maybe_raise_unrecoverable_tool_error,
- outcome_fully_verified,
- )
- from skyvern.forge.sdk.copilot.turn_halt import (
- CopilotTurnHalt,
- raise_if_turn_halt,
- stash_turn_halt_from_blocker_signal,
- )
-
call_id_to_name: dict[str, str] = {}
# Counts completed tool round-trips (tool_called + tool_output pair), not
# raw stream events. Both TOOL_CALL and TOOL_RESULT for the same round
diff --git a/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py b/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py
new file mode 100644
index 000000000..9bfed44dd
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py
@@ -0,0 +1,133 @@
+"""Unrecoverable browser-session tool-error detection, shared by enforcement and the stream adapter."""
+
+import re
+from typing import Any
+
+import structlog
+
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+
+LOG = structlog.get_logger()
+
+UNRECOVERABLE_TOOL_ERROR_STOP_AT = 2
+
+
+class CopilotUnrecoverableToolError(Exception):
+ """Raised when browser-session tool failures prove the current loop cannot recover."""
+
+ def __init__(self, tool_name: str, error_message: str) -> None:
+ self.tool_name = tool_name
+ self.error_message = error_message
+ super().__init__(f"Unrecoverable tool error in {tool_name}: {error_message}")
+
+
+_BROWSER_SESSION_TOOL_NAMES = frozenset(
+ {
+ "navigate_browser",
+ "get_browser_screenshot",
+ "evaluate",
+ "click",
+ "type_text",
+ "scroll",
+ "console_messages",
+ "select_option",
+ "press_key",
+ }
+)
+_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"
+_BROWSER_SESSION_ID_RE = re.compile(r"\bpbs_[A-Za-z0-9_-]+\b")
+_BROWSER_SESSION_WITH_ID_RE = re.compile(r"\bbrowser session\s+pbs_[A-Za-z0-9_-]+\b", re.IGNORECASE)
+
+
+def redact_browser_session_references(value: str) -> str:
+ value = _BROWSER_SESSION_WITH_ID_RE.sub("Browser session", value)
+ return _BROWSER_SESSION_ID_RE.sub("the browser session", value)
+
+
+def _result_text_values(value: Any) -> list[str]:
+ if isinstance(value, str):
+ return [value]
+ if isinstance(value, dict):
+ result: list[str] = []
+ for item in value.values():
+ result.extend(_result_text_values(item))
+ return result
+ if isinstance(value, list):
+ result = []
+ for item in value:
+ result.extend(_result_text_values(item))
+ return result
+ return []
+
+
+def _unrecoverable_tool_error_reason(output: dict[str, Any]) -> str:
+ raw_reason = output.get("error")
+ if not isinstance(raw_reason, str) or not raw_reason.strip():
+ data = output.get("data")
+ raw_reason = data.get("failure_reason") if isinstance(data, dict) else None
+ if not isinstance(raw_reason, str) or not raw_reason.strip():
+ raw_reason = " ".join(_result_text_values(output))
+ reason = " ".join(str(raw_reason or "Browser session was no longer reachable.").split())
+ reason = redact_browser_session_references(reason)
+ return reason[:240].rstrip()
+
+
+def _is_unrecoverable_browser_session_error(tool_name: str, output: dict[str, Any]) -> bool:
+ if tool_name not in _BROWSER_SESSION_TOOL_NAMES or output.get("ok", True):
+ return False
+ lowered = " ".join(_result_text_values(output)).lower()
+ if "no browser context" in lowered:
+ return True
+ has_session_signal = "browser session" in lowered or "browser context" in lowered
+ has_lost_signal = "not found" in lowered or "404" in lowered
+ return has_session_signal and has_lost_signal
+
+
+def _record_unrecoverable_tool_error_contract(ctx: Any, tool_name: str, reason: str) -> None:
+ result = {
+ "ok": False,
+ "error": reason,
+ "data": {
+ "overall_status": "aborted",
+ "failure_reason": reason,
+ "failure_categories": [{"category": _UNRECOVERABLE_TOOL_ERROR_CATEGORY, "reasoning": reason}],
+ },
+ }
+ contract = build_diagnosis_repair_contract(source_tool=tool_name, result=result, ctx=ctx)
+ ctx.latest_diagnosis_repair_contract = contract
+ ctx.unrecoverable_tool_error_reason = reason
+ ctx.unrecoverable_tool_error_tool_name = tool_name
+ ctx.last_test_failure_reason = reason
+ trace_data = contract.to_trace_data()
+ LOG.warning(
+ "Copilot unrecoverable tool error stop",
+ tool_name=tool_name,
+ error_reason=reason,
+ **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},
+ )
+ with copilot_span("copilot_unrecoverable_tool_error", data={"tool_name": tool_name, **trace_data}):
+ pass
+
+
+def _maybe_raise_unrecoverable_tool_error(ctx: Any, tool_name: str, output: dict[str, Any]) -> None:
+ if not _is_unrecoverable_browser_session_error(tool_name, output):
+ if tool_name in _BROWSER_SESSION_TOOL_NAMES and output.get("ok", False):
+ ctx.unrecoverable_tool_error_streak_count = 0
+ ctx.unrecoverable_tool_error_signature = None
+ return
+
+ reason = _unrecoverable_tool_error_reason(output)
+ signature = "browser_session_unreachable"
+ prior_signature = getattr(ctx, "unrecoverable_tool_error_signature", None)
+ prior_count = getattr(ctx, "unrecoverable_tool_error_streak_count", 0)
+ prior_count = prior_count if isinstance(prior_count, int) else 0
+ count = prior_count + 1 if prior_signature == signature else 1
+ ctx.unrecoverable_tool_error_signature = signature
+ ctx.unrecoverable_tool_error_streak_count = count
+ ctx.unrecoverable_tool_error_reason = reason
+ ctx.unrecoverable_tool_error_tool_name = tool_name
+
+ if count >= UNRECOVERABLE_TOOL_ERROR_STOP_AT:
+ _record_unrecoverable_tool_error_contract(ctx, tool_name, reason)
+ raise CopilotUnrecoverableToolError(tool_name, reason)
diff --git a/tests/unit/test_enforcement_streaming_adapter_layering.py b/tests/unit/test_enforcement_streaming_adapter_layering.py
new file mode 100644
index 000000000..30c5f5f8c
--- /dev/null
+++ b/tests/unit/test_enforcement_streaming_adapter_layering.py
@@ -0,0 +1,39 @@
+"""Guard: enforcement → streaming_adapter is a one-way, module-level dependency.
+
+The two modules used to import each other lazily (a hidden cycle):
+`streaming_adapter` reached into `enforcement` for the unrecoverable-tool-error
+helpers, while `enforcement` lazily imported `stream_to_sse`. SKY-11695 extracts
+that cluster into the `unrecoverable_tool_error` leaf module, so
+`streaming_adapter` needs nothing from `enforcement`, and `enforcement` imports
+`streaming_adapter` once at module level (calling it attribute-style so the
+existing monkeypatch sites keep working). This guard fails if either direction
+regresses.
+"""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+COPILOT_DIR = Path(__file__).resolve().parents[2] / "skyvern" / "forge" / "sdk" / "copilot"
+
+
+def _imports_matching(tree: ast.Module, suffix: str) -> list[ast.stmt]:
+ matches: list[ast.stmt] = []
+ for node in ast.walk(tree):
+ if isinstance(node, ast.ImportFrom) and (node.module or "").endswith(suffix):
+ matches.append(node)
+ elif isinstance(node, ast.Import) and any(alias.name.endswith(suffix) for alias in node.names):
+ matches.append(node)
+ return matches
+
+
+def test_enforcement_streaming_adapter_dependency_is_one_way() -> None:
+ adapter_tree = ast.parse((COPILOT_DIR / "streaming_adapter.py").read_text(encoding="utf-8"))
+ adapter_offenders = [node.lineno for node in _imports_matching(adapter_tree, "copilot.enforcement")]
+ assert adapter_offenders == [], f"streaming_adapter imports enforcement at lines {adapter_offenders}"
+
+ enforcement_tree = ast.parse((COPILOT_DIR / "enforcement.py").read_text(encoding="utf-8"))
+ top_level = set(enforcement_tree.body)
+ nested = [node.lineno for node in _imports_matching(enforcement_tree, "streaming_adapter") if node not in top_level]
+ assert nested == [], f"enforcement imports streaming_adapter below module level at lines {nested}"