Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11618): drive native <select> via select_option under unrelated overlays (#7334)
Aaron Perezea8057d6ec71
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 6604096c9..6c9e8a5bf 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -17,6 +17,7 @@ from agents import ModelSettings, RunConfigfrom agents.run import Runnerfrom skyvern.forge.sdk.copilot import config as copilot_config_defaults+from skyvern.forge.sdk.copilot import streaming_adapterfrom skyvern.forge.sdk.copilot.blocker_signal import (SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE,@@ -151,6 +152,12 @@ from skyvern.forge.sdk.copilot.turn_ownership import (claim_turn,emit_blocker_signal_payload,)+from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (+ CopilotUnrecoverableToolError as CopilotUnrecoverableToolError,+)+from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (+ _maybe_raise_unrecoverable_tool_error as _maybe_raise_unrecoverable_tool_error,+)from skyvern.utils.token_counter import count_tokensif TYPE_CHECKING:@@ -185,7 +192,6 @@ REPEATED_FRONTIER_STREAK_STOP_AT = 3# scraper could not read the page. Aligned with MAX_FAILED_TEST_NUDGES so the# copilot gets one generic retry nudge, then stops on the second occurrence.PROBABLE_SITE_BLOCK_STREAK_STOP_AT = 2-UNRECOVERABLE_TOOL_ERROR_STOP_AT = 2# Caps how many times the stop nudge can re-fire — without this, the streak# stays latched while no new test runs reset it and every subsequent turn# re-injects the same nudge until MAX_ITERATIONS. Independent of@@ -980,127 +986,7 @@ def _maybe_raise_non_retriable_nav(ctx: Any) -> None:raise CopilotNonRetriableNavError(url=_extract_url_from_nav_error(err), error_message=err)-class CopilotUnrecoverableToolError(Exception):- """Raised when browser-session tool failures prove the current loop cannot recover."""-- def __init__(self, tool_name: str, error_message: str) -> None:- self.tool_name = tool_name- self.error_message = error_message- super().__init__(f"Unrecoverable tool error in {tool_name}: {error_message}")---_BROWSER_SESSION_TOOL_NAMES = frozenset(- {- "navigate_browser",- "get_browser_screenshot",- "evaluate",- "click",- "type_text",- "scroll",- "console_messages",- "select_option",- "press_key",- }-)_POST_RUN_PAGE_OBSERVATION_TOOLS = frozenset({"evaluate", "get_browser_screenshot", "inspect_page_for_composition"})-_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"-_BROWSER_SESSION_ID_RE = re.compile(r"\bpbs_[A-Za-z0-9_-]+\b")-_BROWSER_SESSION_WITH_ID_RE = re.compile(r"\bbrowser session\s+pbs_[A-Za-z0-9_-]+\b", re.IGNORECASE)---def redact_browser_session_references(value: str) -> str:- value = _BROWSER_SESSION_WITH_ID_RE.sub("Browser session", value)- return _BROWSER_SESSION_ID_RE.sub("the browser session", value)---def _result_text_values(value: Any) -> list[str]:- if isinstance(value, str):- return [value]- if isinstance(value, dict):- result: list[str] = []- for item in value.values():- result.extend(_result_text_values(item))- return result- if isinstance(value, list):- result = []- for item in value:- result.extend(_result_text_values(item))- return result- return []---def _unrecoverable_tool_error_reason(output: dict[str, Any]) -> str:- raw_reason = output.get("error")- if not isinstance(raw_reason, str) or not raw_reason.strip():- data = output.get("data")- raw_reason = data.get("failure_reason") if isinstance(data, dict) else None- if not isinstance(raw_reason, str) or not raw_reason.strip():- raw_reason = " ".join(_result_text_values(output))- reason = " ".join(str(raw_reason or "Browser session was no longer reachable.").split())- reason = redact_browser_session_references(reason)- return reason[:240].rstrip()---def _is_unrecoverable_browser_session_error(tool_name: str, output: dict[str, Any]) -> bool:- if tool_name not in _BROWSER_SESSION_TOOL_NAMES or output.get("ok", True):- return False- lowered = " ".join(_result_text_values(output)).lower()- if "no browser context" in lowered:- return True- has_session_signal = "browser session" in lowered or "browser context" in lowered- has_lost_signal = "not found" in lowered or "404" in lowered- return has_session_signal and has_lost_signal---def _record_unrecoverable_tool_error_contract(ctx: Any, tool_name: str, reason: str) -> None:- from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract-- result = {- "ok": False,- "error": reason,- "data": {- "overall_status": "aborted",- "failure_reason": reason,- "failure_categories": [{"category": _UNRECOVERABLE_TOOL_ERROR_CATEGORY, "reasoning": reason}],- },- }- contract = build_diagnosis_repair_contract(source_tool=tool_name, result=result, ctx=ctx)- ctx.latest_diagnosis_repair_contract = contract- ctx.unrecoverable_tool_error_reason = reason- ctx.unrecoverable_tool_error_tool_name = tool_name- ctx.last_test_failure_reason = reason- trace_data = contract.to_trace_data()- LOG.warning(- "Copilot unrecoverable tool error stop",- tool_name=tool_name,- error_reason=reason,- **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},- )- with copilot_span("copilot_unrecoverable_tool_error", data={"tool_name": tool_name, **trace_data}):- pass---def _maybe_raise_unrecoverable_tool_error(ctx: Any, tool_name: str, output: dict[str, Any]) -> None:- if not _is_unrecoverable_browser_session_error(tool_name, output):- if tool_name in _BROWSER_SESSION_TOOL_NAMES and output.get("ok", False):- ctx.unrecoverable_tool_error_streak_count = 0- ctx.unrecoverable_tool_error_signature = None- return-- reason = _unrecoverable_tool_error_reason(output)- signature = "browser_session_unreachable"- prior_signature = getattr(ctx, "unrecoverable_tool_error_signature", None)- prior_count = getattr(ctx, "unrecoverable_tool_error_streak_count", 0)- prior_count = prior_count if isinstance(prior_count, int) else 0- count = prior_count + 1 if prior_signature == signature else 1- ctx.unrecoverable_tool_error_signature = signature- ctx.unrecoverable_tool_error_streak_count = count- ctx.unrecoverable_tool_error_reason = reason- ctx.unrecoverable_tool_error_tool_name = tool_name-- if count >= UNRECOVERABLE_TOOL_ERROR_STOP_AT:- _record_unrecoverable_tool_error_contract(ctx, tool_name, reason)- raise CopilotUnrecoverableToolError(tool_name, reason)def _raise_if_unrecoverable_contract_stop(ctx: Any) -> None:@@ -2209,14 +2095,12 @@ async def _run_streamed_with_deadline(``MIN_DEADLINE_REMAINING_SECONDS`` floors ``remaining`` so``wait_for(timeout=0)`` never panics on an already-spent budget."""- from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse-elapsed = time.monotonic() - start_timeremaining = max(MIN_DEADLINE_REMAINING_SECONDS, TOTAL_TIMEOUT_SECONDS - elapsed)result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)try:try:- await asyncio.wait_for(stream_to_sse(result, tracked_stream, ctx), timeout=remaining)+ await asyncio.wait_for(streaming_adapter.stream_to_sse(result, tracked_stream, ctx), timeout=remaining)finally:_accumulate_usage(result, ctx)except asyncio.TimeoutError:diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.pyindex bf4b62602..3ea7b1ca2 100644--- a/skyvern/forge/sdk/copilot/streaming_adapter.py+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py@@ -10,6 +10,7 @@ from datetime import datetime, timezonefrom typing import TYPE_CHECKING, Anyimport structlog+from agents.stream_events import RawResponsesStreamEvent, RunItemStreamEventfrom skyvern.config import settings@@ -33,6 +34,16 @@ from skyvern.forge.sdk.copilot.narration import (tool_activity_display_label,)from skyvern.forge.sdk.copilot.output_utils import format_tool_result_for_user, summarize_tool_result_detail+from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified+from skyvern.forge.sdk.copilot.turn_halt import (+ CopilotTurnHalt,+ raise_if_turn_halt,+ stash_turn_halt_from_blocker_signal,+)+from skyvern.forge.sdk.copilot.unrecoverable_tool_error import (+ CopilotUnrecoverableToolError,+ _maybe_raise_unrecoverable_tool_error,+)from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotCodegenProgressUpdate,WorkflowCopilotDesignEndUpdate,@@ -251,19 +262,6 @@ async def stream_to_sse(reasons unrelated to a dropped client) is re-raised unchanged soasyncio's cancellation machinery still runs normally."""- from agents.stream_events import RawResponsesStreamEvent, RunItemStreamEvent-- from skyvern.forge.sdk.copilot.enforcement import (- CopilotUnrecoverableToolError,- _maybe_raise_unrecoverable_tool_error,- outcome_fully_verified,- )- from skyvern.forge.sdk.copilot.turn_halt import (- CopilotTurnHalt,- raise_if_turn_halt,- stash_turn_halt_from_blocker_signal,- )-call_id_to_name: dict[str, str] = {}# Counts completed tool round-trips (tool_called + tool_output pair), not# raw stream events. Both TOOL_CALL and TOOL_RESULT for the same rounddiff --git a/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py b/skyvern/forge/sdk/copilot/unrecoverable_tool_error.pynew file mode 100644index 000000000..9bfed44dd--- /dev/null+++ b/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py@@ -0,0 +1,133 @@+"""Unrecoverable browser-session tool-error detection, shared by enforcement and the stream adapter."""++import re+from typing import Any++import structlog++from skyvern.forge.sdk.copilot.diagnosis_repair_contract import build_diagnosis_repair_contract+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span++LOG = structlog.get_logger()++UNRECOVERABLE_TOOL_ERROR_STOP_AT = 2+++class CopilotUnrecoverableToolError(Exception):+ """Raised when browser-session tool failures prove the current loop cannot recover."""++ def __init__(self, tool_name: str, error_message: str) -> None:+ self.tool_name = tool_name+ self.error_message = error_message+ super().__init__(f"Unrecoverable tool error in {tool_name}: {error_message}")+++_BROWSER_SESSION_TOOL_NAMES = frozenset(+ {+ "navigate_browser",+ "get_browser_screenshot",+ "evaluate",+ "click",+ "type_text",+ "scroll",+ "console_messages",+ "select_option",+ "press_key",+ }+)+_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"+_BROWSER_SESSION_ID_RE = re.compile(r"\bpbs_[A-Za-z0-9_-]+\b")+_BROWSER_SESSION_WITH_ID_RE = re.compile(r"\bbrowser session\s+pbs_[A-Za-z0-9_-]+\b", re.IGNORECASE)+++def redact_browser_session_references(value: str) -> str:+ value = _BROWSER_SESSION_WITH_ID_RE.sub("Browser session", value)+ return _BROWSER_SESSION_ID_RE.sub("the browser session", value)+++def _result_text_values(value: Any) -> list[str]:+ if isinstance(value, str):+ return [value]+ if isinstance(value, dict):+ result: list[str] = []+ for item in value.values():+ result.extend(_result_text_values(item))+ return result+ if isinstance(value, list):+ result = []+ for item in value:+ result.extend(_result_text_values(item))+ return result+ return []+++def _unrecoverable_tool_error_reason(output: dict[str, Any]) -> str:+ raw_reason = output.get("error")+ if not isinstance(raw_reason, str) or not raw_reason.strip():+ data = output.get("data")+ raw_reason = data.get("failure_reason") if isinstance(data, dict) else None+ if not isinstance(raw_reason, str) or not raw_reason.strip():+ raw_reason = " ".join(_result_text_values(output))+ reason = " ".join(str(raw_reason or "Browser session was no longer reachable.").split())+ reason = redact_browser_session_references(reason)+ return reason[:240].rstrip()+++def _is_unrecoverable_browser_session_error(tool_name: str, output: dict[str, Any]) -> bool:+ if tool_name not in _BROWSER_SESSION_TOOL_NAMES or output.get("ok", True):+ return False+ lowered = " ".join(_result_text_values(output)).lower()+ if "no browser context" in lowered:+ return True+ has_session_signal = "browser session" in lowered or "browser context" in lowered+ has_lost_signal = "not found" in lowered or "404" in lowered+ return has_session_signal and has_lost_signal+++def _record_unrecoverable_tool_error_contract(ctx: Any, tool_name: str, reason: str) -> None:+ result = {+ "ok": False,+ "error": reason,+ "data": {+ "overall_status": "aborted",+ "failure_reason": reason,+ "failure_categories": [{"category": _UNRECOVERABLE_TOOL_ERROR_CATEGORY, "reasoning": reason}],+ },+ }+ contract = build_diagnosis_repair_contract(source_tool=tool_name, result=result, ctx=ctx)+ ctx.latest_diagnosis_repair_contract = contract+ ctx.unrecoverable_tool_error_reason = reason+ ctx.unrecoverable_tool_error_tool_name = tool_name+ ctx.last_test_failure_reason = reason+ trace_data = contract.to_trace_data()+ LOG.warning(+ "Copilot unrecoverable tool error stop",+ tool_name=tool_name,+ error_reason=reason,+ **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},+ )+ with copilot_span("copilot_unrecoverable_tool_error", data={"tool_name": tool_name, **trace_data}):+ pass+++def _maybe_raise_unrecoverable_tool_error(ctx: Any, tool_name: str, output: dict[str, Any]) -> None:+ if not _is_unrecoverable_browser_session_error(tool_name, output):+ if tool_name in _BROWSER_SESSION_TOOL_NAMES and output.get("ok", False):+ ctx.unrecoverable_tool_error_streak_count = 0+ ctx.unrecoverable_tool_error_signature = None+ return++ reason = _unrecoverable_tool_error_reason(output)+ signature = "browser_session_unreachable"+ prior_signature = getattr(ctx, "unrecoverable_tool_error_signature", None)+ prior_count = getattr(ctx, "unrecoverable_tool_error_streak_count", 0)+ prior_count = prior_count if isinstance(prior_count, int) else 0+ count = prior_count + 1 if prior_signature == signature else 1+ ctx.unrecoverable_tool_error_signature = signature+ ctx.unrecoverable_tool_error_streak_count = count+ ctx.unrecoverable_tool_error_reason = reason+ ctx.unrecoverable_tool_error_tool_name = tool_name++ if count >= UNRECOVERABLE_TOOL_ERROR_STOP_AT:+ _record_unrecoverable_tool_error_contract(ctx, tool_name, reason)+ raise CopilotUnrecoverableToolError(tool_name, reason)diff --git a/tests/unit/test_enforcement_streaming_adapter_layering.py b/tests/unit/test_enforcement_streaming_adapter_layering.pynew file mode 100644index 000000000..30c5f5f8c--- /dev/null+++ b/tests/unit/test_enforcement_streaming_adapter_layering.py@@ -0,0 +1,39 @@+"""Guard: enforcement → streaming_adapter is a one-way, module-level dependency.++The two modules used to import each other lazily (a hidden cycle):+`streaming_adapter` reached into `enforcement` for the unrecoverable-tool-error+helpers, while `enforcement` lazily imported `stream_to_sse`. SKY-11695 extracts+that cluster into the `unrecoverable_tool_error` leaf module, so+`streaming_adapter` needs nothing from `enforcement`, and `enforcement` imports+`streaming_adapter` once at module level (calling it attribute-style so the+existing monkeypatch sites keep working). This guard fails if either direction+regresses.+"""++from __future__ import annotations++import ast+from pathlib import Path++COPILOT_DIR = Path(__file__).resolve().parents[2] / "skyvern" / "forge" / "sdk" / "copilot"+++def _imports_matching(tree: ast.Module, suffix: str) -> list[ast.stmt]:+ matches: list[ast.stmt] = []+ for node in ast.walk(tree):+ if isinstance(node, ast.ImportFrom) and (node.module or "").endswith(suffix):+ matches.append(node)+ elif isinstance(node, ast.Import) and any(alias.name.endswith(suffix) for alias in node.names):+ matches.append(node)+ return matches+++def test_enforcement_streaming_adapter_dependency_is_one_way() -> None:+ adapter_tree = ast.parse((COPILOT_DIR / "streaming_adapter.py").read_text(encoding="utf-8"))+ adapter_offenders = [node.lineno for node in _imports_matching(adapter_tree, "copilot.enforcement")]+ assert adapter_offenders == [], f"streaming_adapter imports enforcement at lines {adapter_offenders}"++ enforcement_tree = ast.parse((COPILOT_DIR / "enforcement.py").read_text(encoding="utf-8"))+ top_level = set(enforcement_tree.body)+ nested = [node.lineno for node in _imports_matching(enforcement_tree, "streaming_adapter") if node not in top_level]+ assert nested == [], f"enforcement imports streaming_adapter below module level at lines {nested}"