Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[oncall] Capture transient download error popups (#6080)
Marc Kelechavac03df7ebd80d
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 1fdd34426..831f366eb 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -4683,6 +4683,7 @@ class ForgeAgent:step.output.actions_and_results.append((persisted_action, action_results))if isinstance(persisted_action, DecisiveAction) and persisted_action.errors:step.output.errors.extend(persisted_action.errors)+ step.output.terminal_user_errors = Trueif isinstance(persisted_action, TerminateAction):LOG.warning(@@ -4871,6 +4872,39 @@ class ForgeAgent:if organization.max_retries_per_step is not Noneelse settings.MAX_RETRIES_PER_STEP)+ step_errors = (+ step.output.errors if step.output and step.output.errors and step.output.terminal_user_errors else []+ )+ if step_errors:+ failure_reason = "; ".join(error.reasoning for error in step_errors)+ failure_category = classify_from_failure_reason(failure_reason, fallback_to_unknown=True)+ existing_errors = {+ (error.get("error_code"), error.get("reasoning"))+ for error in (task.errors or [])+ if isinstance(error, dict)+ }+ new_step_errors = [+ error.model_dump()+ for error in step_errors+ if (error.error_code, error.reasoning) not in existing_errors+ ]+ LOG.warning(+ "Step failed with user-defined errors, marking task as failed without retry",+ task_id=task.task_id,+ step_id=step.step_id,+ step_order=step.order,+ step_retry=step.retry_index,+ error_codes=[error.error_code for error in step_errors],+ )+ await self.update_task(+ task,+ TaskStatus.failed,+ failure_reason=failure_reason,+ errors=new_step_errors,+ failure_category=failure_category,+ )+ return None+if step.retry_index >= max_retries_per_step:LOG.warning("Step failed after max retries, marking task as failed",diff --git a/skyvern/schemas/steps.py b/skyvern/schemas/steps.pyindex c9f8cd97b..9b77dc7a1 100644--- a/skyvern/schemas/steps.py+++ b/skyvern/schemas/steps.py@@ -21,6 +21,8 @@ class AgentStepOutput(BaseModel):# Nullable for backwards compatibility, once backfill is done, this won't be nullable anymoreactions_and_results: list[tuple[Action, list[ActionResult]]] | None = Noneerrors: list[UserDefinedError] = []+ # Explicit no-retry signal; historical/plain errors still use the normal retry budget.+ terminal_user_errors: bool = Falsebrowser_metadata: BrowserMetadata | None = Nonestep_exception: str | None = Nonediff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.pyindex 3ea88725b..98b6ebfc3 100644--- a/skyvern/webeye/actions/actions.py+++ b/skyvern/webeye/actions/actions.py@@ -131,6 +131,8 @@ class Action(BaseModel):# DecisiveAction (CompleteAction, TerminateAction) fieldserrors: list[UserDefinedError] | None = None+ # Explicit signal that this action's user-defined errors should bypass step retry.+ terminal_user_errors: bool = Falsedata_extraction_goal: str | None = None# WebAction fieldsdiff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 9e3629022..f056a0142 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -126,6 +126,10 @@ from skyvern.webeye.scraper.scraped_page import (json_to_html,)from skyvern.webeye.scraper.scraper import IncrementalScrapePage, hash_element, trim_element_tree+from skyvern.webeye.transient_page_observer import (+ TransientPageTextObserver,+ match_user_defined_errors_from_transient_text,+)from skyvern.webeye.utils.dom import COMMON_INPUT_TAGS, DomUtil, InteractiveElement, SkyvernElementfrom skyvern.webeye.utils.page import SkyvernFrame@@ -722,8 +726,15 @@ class ActionHandler:xhr_capture = ScopedXhrDownloadCapture(page, download_dir)download_triggered = False+ transient_text_observer = TransientPageTextObserver(+ page,+ task_id=task.task_id,+ step_id=step.step_id,+ workflow_run_id=task.workflow_run_id,+ )page.on("download", _capture_download_event)try:+ await transient_text_observer.start()xhr_capture.enable()with _tracer.start_as_current_span("skyvern.agent.action.handle_inner") as _hi_span:apply_context_attrs(_hi_span)@@ -736,6 +747,7 @@ class ActionHandler:)if not results:return results+ await transient_text_observer.start()_download_timeout = task.download_timeout or BROWSER_DOWNLOAD_MAX_WAIT_TIME_download_event_grace_seconds = min(DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS, _download_timeout)with _tracer.start_as_current_span("skyvern.agent.action.download_wait") as _dl_wait_span:@@ -755,6 +767,7 @@ class ActionHandler:download_signal_source: str | None = Nonedownload_signal_elapsed_seconds: float | None = Nonedownload_signal_poll_iterations: int | None = None+ download_wait_matched_errors: list[UserDefinedError] = []download_wait_started_at = time.monotonic()def _record_download_signal(source: str) -> None:@@ -850,6 +863,24 @@ class ActionHandler:download_event_fallback_failed = Truebreakelapsed_since_action = time.monotonic() - download_wait_started_at+ if not download_signal_observed:+ download_wait_matched_errors = match_user_defined_errors_from_transient_text(+ task,+ step,+ transient_text_observer.events,+ )+ if download_wait_matched_errors:+ action.errors = (action.errors or []) + download_wait_matched_errors+ action.terminal_user_errors = True+ LOG.warning(+ "Stopping download wait after transient user-defined error text",+ task_id=task.task_id,+ step_id=step.step_id,+ workflow_run_id=task.workflow_run_id,+ error_codes=[error.error_code for error in download_wait_matched_errors],+ )+ break+if not download_signal_observed and elapsed_since_action >= no_signal_grace_seconds:LOG.warning("No download signal observed after action",@@ -881,9 +912,28 @@ class ActionHandler:_dl_wait_span.set_attribute("download_event_fallback_attempted", download_event_fallback_attempted)_dl_wait_span.set_attribute("download_event_fallback_used", download_event_fallback_used)_dl_wait_span.set_attribute("download_event_fallback_failed", download_event_fallback_failed)+ _dl_wait_span.set_attribute(+ "download_wait_observed_text_count",+ len(transient_text_observer.events),+ )+ _dl_wait_span.set_attribute(+ "download_wait_user_error_detected",+ bool(download_wait_matched_errors),+ )+ if download_wait_matched_errors:+ _dl_wait_span.set_attribute(+ "download_wait_user_error_codes",+ ",".join(error.error_code for error in download_wait_matched_errors),+ )if not download_triggered:- results[-1].download_triggered = False+ if action.errors:+ results[-1] = ActionFailure(+ Exception("; ".join(error.reasoning for error in action.errors)),+ download_triggered=False,+ )+ else:+ results[-1].download_triggered = Falseaction.download_triggered = Falsereturn resultsresults[-1].download_triggered = True@@ -918,6 +968,7 @@ class ActionHandler:return resultsfinally:+ await transient_text_observer.stop()xhr_capture.disable()if browser_state is not None and download_triggered:# get the page count after downloaddiff --git a/skyvern/webeye/actions/models.py b/skyvern/webeye/actions/models.pyindex 4e1281189..924f5d20b 100644--- a/skyvern/webeye/actions/models.py+++ b/skyvern/webeye/actions/models.py@@ -41,8 +41,8 @@ class DetailedAgentStepOutput(BaseModel):def extract_errors(self) -> list[UserDefinedError]:errors = []if self.actions_and_results:- for action, action_results in self.actions_and_results:- if isinstance(action, DecisiveAction):+ for action, _action_results in self.actions_and_results:+ if action.errors and (isinstance(action, DecisiveAction) or action.terminal_user_errors):errors.extend(action.errors)return errors@@ -62,6 +62,7 @@ class DetailedAgentStepOutput(BaseModel):def to_agent_step_output(self) -> AgentStepOutput:clean_output = self.get_clean_detailed_output()+ errors = clean_output.extract_errors()browser_metadata: BrowserMetadata | None = Noneif clean_output.scraped_page and clean_output.scraped_page.url:@@ -70,7 +71,8 @@ class DetailedAgentStepOutput(BaseModel):return AgentStepOutput(action_results=clean_output.action_results if clean_output.action_results else [],actions_and_results=(clean_output.actions_and_results if clean_output.actions_and_results else []),- errors=clean_output.extract_errors(),+ errors=errors,+ terminal_user_errors=bool(errors),browser_metadata=browser_metadata,step_exception=clean_output.step_exception,)diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.pyindex 7443c0f1a..c83f41955 100644--- a/skyvern/webeye/scraper/scraped_page.py+++ b/skyvern/webeye/scraper/scraped_page.py@@ -308,9 +308,24 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):return element_str[: int(len(element_str) * percent_to_keep)]if fmt == ElementTreeFormat.HTML:+ elements_to_render = self.economy_element_tree+ if percent_to_keep < 1:+ # Portals (popper menus, dialogs) are appended near the end of+ # <body> and appear last in the root-level list. A naïve front+ # slice drops them disproportionately. Move overlay root elements+ # to the front so they survive character-level truncation.+ overlay_roots = [e for e in self.economy_element_tree if self._element_subtree_has_overlay(e)]+ if overlay_roots:+ regular_roots = [e for e in self.economy_element_tree if not self._element_subtree_has_overlay(e)]+ elements_to_render = overlay_roots + regular_roots+ LOG.info(+ "economy_tree_portal_priority: moved overlay roots to front before truncation",+ overlay_count=len(overlay_roots),+ regular_count=len(regular_roots),+ percent_to_keep=percent_to_keep,+ )element_str = "".join(- json_to_html(element, need_skyvern_attrs=html_need_skyvern_attrs)- for element in self.economy_element_tree+ json_to_html(element, need_skyvern_attrs=html_need_skyvern_attrs) for element in elements_to_render)result = element_str[: int(len(element_str) * percent_to_keep)]self.last_used_element_tree_html = result@@ -337,6 +352,22 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):element["children"] = new_childrenreturn element+ @staticmethod+ def _element_subtree_has_overlay(element: dict, _depth: int = 0) -> bool:+ """Return True if this element or any descendant carries role=listbox/option.++ These roles are preserved by `_trimmed_attributes` and are reliable+ indicators of popper/dialog portal content. Recursion is limited to+ avoid spending time walking deep table trees — portals are typically+ shallow (1-3 levels).+ """+ if _depth > 4:+ return False+ role = (element.get("attributes") or {}).get("role", "")+ if role in ("listbox", "option"):+ return True+ return any(ScrapedPage._element_subtree_has_overlay(child, _depth + 1) for child in element.get("children", []))+def build_lean_elements_tree(self,fmt: ElementTreeFormat = ElementTreeFormat.HTML,diff --git a/skyvern/webeye/transient_page_observer.py b/skyvern/webeye/transient_page_observer.pynew file mode 100644index 000000000..0f147d2a1--- /dev/null+++ b/skyvern/webeye/transient_page_observer.py@@ -0,0 +1,286 @@+"""Capture short-lived visible page text during explicitly scoped browser waits."""++import re+import weakref+from dataclasses import dataclass+from typing import Any++import structlog+from playwright.async_api import Page++from skyvern.errors.errors import UserDefinedError+from skyvern.forge.sdk.models import Step+from skyvern.forge.sdk.schemas.tasks import Task++LOG = structlog.get_logger()++TRANSIENT_TEXT_EVENT_LIMIT = 100+TRANSIENT_TEXT_MIN_LENGTH = 8+TRANSIENT_TEXT_MATCH_MIN_LENGTH = 12+TRANSIENT_TEXT_MATCH_WORD_WINDOW = 6+TRANSIENT_TEXT_MAX_LENGTH = 500+TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT = 160+# Heuristic DOM text matches are strong signals, but not equivalent to explicit LLM classification.+TRANSIENT_TEXT_MATCH_CONFIDENCE = 0.9+TRANSIENT_TEXT_BINDING_NAME = "__skyvernRecordTransientText"+TRANSIENT_TEXT_OBSERVER_STATE_KEY = "__skyvernTransientTextObserver"+++@dataclass+class _TransientPageTextBinding:+ active_observer: "TransientPageTextObserver | None" = None+ registered: bool = False+++_PAGE_TRANSIENT_TEXT_BINDINGS: weakref.WeakKeyDictionary[Any, _TransientPageTextBinding] = weakref.WeakKeyDictionary()+++def _get_transient_text_binding(page: Page) -> _TransientPageTextBinding:+ binding = _PAGE_TRANSIENT_TEXT_BINDINGS.get(page)+ if binding is None:+ binding = _TransientPageTextBinding()+ _PAGE_TRANSIENT_TEXT_BINDINGS[page] = binding+ return binding+++def _normalize_text(text: str) -> str:+ return re.sub(r"\s+", " ", text).strip()+++def _normalize_for_match(text: str) -> str:+ return _normalize_text(text).casefold()+++def _has_meaningful_text_overlap(observed_text: str, mapped_description: str) -> bool:+ observed_text = _normalize_for_match(observed_text)+ mapped_description = _normalize_for_match(mapped_description)+ if observed_text in mapped_description or mapped_description in observed_text:+ return True++ observed_words = re.findall(r"\w+", observed_text)+ mapped_words = re.findall(r"\w+", mapped_description)+ mapped_word_text = f" {' '.join(mapped_words)} "+ return len(observed_words) >= TRANSIENT_TEXT_MATCH_WORD_WINDOW and any(+ f" {' '.join(observed_words[idx : idx + TRANSIENT_TEXT_MATCH_WORD_WINDOW])} " in mapped_word_text+ for idx in range(len(observed_words) - TRANSIENT_TEXT_MATCH_WORD_WINDOW + 1)+ )+++def _append_text_event(events: list[dict[str, Any]], payload: Any) -> None:+ if not isinstance(payload, dict) or not isinstance(raw_text := payload.get("text"), str):+ return++ text = _normalize_text(raw_text)+ if len(text) < TRANSIENT_TEXT_MIN_LENGTH:+ return++ text = text[:TRANSIENT_TEXT_MAX_LENGTH]+ if any(event.get("text") == text for event in events):+ return++ event: dict[str, Any] = {}+ for key in ("timestamp_ms", "tag", "role", "aria_live"):+ if key not in payload:+ continue+ value = payload[key]+ if isinstance(value, str | int | float) or value is None:+ event[key] = value+ event["text"] = text+ events.append(event)+ if len(events) > TRANSIENT_TEXT_EVENT_LIMIT:+ del events[: len(events) - TRANSIENT_TEXT_EVENT_LIMIT]+++def _format_observed_text_reasoning(observed_texts: list[str]) -> str:+ snippets = []+ for text in observed_texts[:3]:+ if len(text) > TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT:+ text = f"{text[:TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT]}..."+ snippets.append(text)+ return " | ".join(snippets)+++class TransientPageTextObserver:+ def __init__(+ self,+ page: Page,+ *,+ task_id: str | None = None,+ step_id: str | None = None,+ workflow_run_id: str | None = None,+ ) -> None:+ self.page = page+ self.events: list[dict[str, Any]] = []+ self._binding_state: _TransientPageTextBinding | None = None+ self._log_context = {+ "task_id": task_id,+ "step_id": step_id,+ "workflow_run_id": workflow_run_id,+ }++ async def start(self) -> None:+ binding_state = _get_transient_text_binding(self.page)++ def record_text_event(_source: dict[str, Any], payload: Any) -> None:+ if binding_state.active_observer is not None:+ _append_text_event(binding_state.active_observer.events, payload)++ try:+ if not binding_state.registered:+ await self.page.expose_binding(TRANSIENT_TEXT_BINDING_NAME, record_text_event)+ binding_state.registered = True+ binding_state.active_observer = self+ await self.page.evaluate(+ """+ ({ bindingName, stateKey, minLength, maxLength }) => {+ const key = stateKey;+ const normalize = (value) => String(value || "").replace(/\\s+/g, " ").trim();+ try { window[key]?.observer?.disconnect?.(); } catch (e) {}++ const isVisible = (element) => {+ if (!element || !(element instanceof Element)) return false;+ const style = window.getComputedStyle(element);+ if (+ !style ||+ style.display === "none" ||+ style.visibility === "hidden" ||+ style.opacity === "0"+ ) {+ return false;+ }+ const rect = element.getBoundingClientRect();+ return rect.width > 0 && rect.height > 0;+ };++ const emit = (node) => {+ const element = node?.nodeType === Node.ELEMENT_NODE ? node : node?.parentElement;+ if (!element || !isVisible(element)) return;+ let text = element instanceof HTMLElement && typeof element.innerText === "string"+ ? element.innerText+ : element.textContent || "";+ text = normalize(text);+ if (text.length < minLength) return;+ if (text.length > maxLength) text = text.slice(0, maxLength);+ Promise.resolve(window[bindingName]({+ text,+ timestamp_ms: Date.now(),+ tag: element.tagName || null,+ role: element.getAttribute("role"),+ aria_live: element.getAttribute("aria-live"),+ })).catch(() => {});+ };++ const observer = new MutationObserver((mutations) => {+ for (const mutation of mutations) {+ for (const node of mutation.addedNodes) emit(node);+ if (mutation.type === "characterData" || mutation.type === "attributes") {+ emit(mutation.target);+ }+ }+ });+ observer.observe(document.documentElement || document.body, {+ subtree: true,+ childList: true,+ characterData: true,+ attributes: true,+ attributeFilter: ["class", "style", "hidden", "aria-hidden", "aria-live"],+ });+ window[key] = { observer, bindingName };+ }+ """,+ {+ "bindingName": TRANSIENT_TEXT_BINDING_NAME,+ "stateKey": TRANSIENT_TEXT_OBSERVER_STATE_KEY,+ "minLength": TRANSIENT_TEXT_MIN_LENGTH,+ "maxLength": TRANSIENT_TEXT_MAX_LENGTH,+ },+ )+ self._binding_state = binding_state+ except Exception:+ if binding_state.active_observer is self:+ binding_state.active_observer = None+ LOG.warning("Failed to start transient page text observer", **self._log_context, exc_info=True)++ async def stop(self) -> None:+ binding_state = self._binding_state+ if binding_state is None:+ return+ if binding_state.active_observer is self:+ try:+ await self.page.evaluate(+ """+ ({ bindingName, stateKey }) => {+ const state = window[stateKey];+ if (state?.bindingName === bindingName) {+ state.observer?.disconnect?.();+ delete window[stateKey];+ }+ }+ """,+ {"bindingName": TRANSIENT_TEXT_BINDING_NAME, "stateKey": TRANSIENT_TEXT_OBSERVER_STATE_KEY},+ )+ except Exception:+ LOG.warning("Failed to stop transient page text observer", **self._log_context, exc_info=True)+ finally:+ binding_state.active_observer = None+ self._binding_state = None+++def match_user_defined_errors_from_transient_text(+ task: Task,+ step: Step,+ observed_text_events: list[dict[str, Any]],+) -> list[UserDefinedError]:+ if not task.error_code_mapping or not observed_text_events:+ return []++ observed_texts = [+ text+ for event in observed_text_events+ if isinstance(text := event.get("text"), str) and len(text) >= TRANSIENT_TEXT_MATCH_MIN_LENGTH+ ]+ if not observed_texts:+ return []++ normalized_observed_texts = [_normalize_for_match(text) for text in observed_texts]+ combined_observed_text = "\n".join(normalized_observed_texts)+ matched_errors: list[UserDefinedError] = []+ for error_code, error_description in task.error_code_mapping.items():+ normalized_code = error_code.casefold()+ # Restrict code matching to machine-style codes; natural words are too collision-prone in page text.+ code_matches = (+ "_" in error_code and re.search(rf"\b{re.escape(normalized_code)}\b", combined_observed_text) is not None+ )+ normalized_description = _normalize_for_match(error_description) if isinstance(error_description, str) else ""+ description_matches = bool(+ normalized_description+ and (+ normalized_description in combined_observed_text+ or any(+ _has_meaningful_text_overlap(observed_text, normalized_description)+ for observed_text in normalized_observed_texts+ )+ )+ )+ if code_matches or description_matches:+ matched_errors.append(+ UserDefinedError(+ error_code=error_code,+ reasoning=f"Observed transient text during browser wait: {_format_observed_text_reasoning(observed_texts)}",+ confidence_float=TRANSIENT_TEXT_MATCH_CONFIDENCE,+ )+ )++ if matched_errors:+ if len(matched_errors) > 1:+ LOG.warning(+ "Multiple user-defined error mappings matched transient browser text; using first match",+ task_id=task.task_id,+ step_id=step.step_id,+ matched_error_codes=[error.error_code for error in matched_errors],+ selected_error_code=matched_errors[0].error_code,+ tie_breaker="task.error_code_mapping_order",+ )+ # If multiple mappings match, preserve task.error_code_mapping order so user-authored priority is deterministic.+ return [matched_errors[0]]+ return []diff --git a/tests/unit/test_action_error_extraction.py b/tests/unit/test_action_error_extraction.pynew file mode 100644index 000000000..64f28ef90--- /dev/null+++ b/tests/unit/test_action_error_extraction.py@@ -0,0 +1,63 @@+from skyvern.errors.errors import UserDefinedError+from skyvern.webeye.actions.actions import Action, ClickAction, CompleteAction+from skyvern.webeye.actions.models import DetailedAgentStepOutput+from skyvern.webeye.actions.responses import ActionSuccess+++def _error(code: str) -> UserDefinedError:+ return UserDefinedError(error_code=code, reasoning="mapped error", confidence_float=0.9)+++def _output_for_action(action: Action) -> DetailedAgentStepOutput:+ return DetailedAgentStepOutput(+ scraped_page=None,+ extract_action_prompt=None,+ llm_response=None,+ actions=None,+ action_results=None,+ actions_and_results=[(action, [ActionSuccess()])],+ )+++def test_extract_errors_preserves_decisive_action_errors() -> None:+ error = _error("task_failed")+ action = CompleteAction(errors=[error])++ assert _output_for_action(action).extract_errors() == [error]+++def test_extract_errors_includes_download_action_errors() -> None:+ error = _error("data_not_downloadable")+ action = ClickAction(element_id="download-link", download=True, errors=[error], terminal_user_errors=True)++ assert _output_for_action(action).extract_errors() == [error]+++def test_agent_step_output_marks_extracted_errors_terminal() -> None:+ error = _error("data_not_downloadable")+ action = ClickAction(element_id="download-link", download=True, errors=[error], terminal_user_errors=True)++ output = _output_for_action(action).to_agent_step_output()++ assert output.errors == [error]+ assert output.terminal_user_errors is True+++def test_extract_errors_ignores_download_action_errors_without_terminal_flag() -> None:+ error = _error("inline_download_error")+ action = ClickAction(element_id="download-link", download=True, errors=[error])++ output = _output_for_action(action).to_agent_step_output()++ assert output.errors == []+ assert output.terminal_user_errors is False+++def test_extract_errors_ignores_non_download_web_action_errors() -> None:+ error = _error("inline_form_error")+ action = ClickAction(element_id="submit", download=False, errors=[error])++ output = _output_for_action(action).to_agent_step_output()++ assert output.errors == []+ assert output.terminal_user_errors is Falsediff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.pyindex 2afae2e6d..c7c7bccf9 100644--- a/tests/unit/test_download_file_action_handler.py+++ b/tests/unit/test_download_file_action_handler.py@@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from skyvern.errors.errors import UserDefinedErrorfrom skyvern.forge.sdk.models import StepStatusfrom skyvern.webeye.actions.actions import ClickAction, DownloadFileActionfrom skyvern.webeye.actions.handler import ActionHandler, _remove_download_listener, handle_download_file_action@@ -25,6 +26,45 @@ def _download_wait_span_attrs(span_exporter: InMemorySpanExporter) -> dict:return dict(span.attributes or {})+def _make_download_click_context(+ *,+ now: datetime,+ organization,+ page_url: str,+ task_overrides: dict | None = None,+) -> tuple:+ task = make_task(+ now,+ organization,+ workflow_run_id="wr-1",+ browser_session_id=None,+ download_timeout=30.0,+ **(task_overrides or {}),+ )+ step = make_step(now, task, step_id="step-1", status=StepStatus.created, order=0, output=None)+ page = MagicMock()+ page.url = page_url+ page.context.browser = None+ browser_state = MagicMock()+ browser_state.list_valid_pages = AsyncMock(return_value=[page])+ scraped_page = ScrapedPage(+ elements=[],+ element_tree=[],+ element_tree_trimmed=[],+ _browser_state=browser_state,+ _clean_up_func=AsyncMock(return_value=[]),+ _scrape_exclude=None,+ )+ action = ClickAction(+ element_id="download-link",+ download=True,+ organization_id=task.organization_id,+ task_id=task.task_id,+ step_id=step.step_id,+ )+ return task, step, page, browser_state, scraped_page, action++def test_remove_download_listener_uses_playwright_remove_listener_when_off_unavailable() -> None:page = MagicMock(spec=["remove_listener"])callback = MagicMock()@@ -662,6 +702,100 @@ async def test_handle_action_download_no_signal_fails_fast(span_exporter: InMemoassert "download_signal_poll_iterations" not in span_attrs+async def test_handle_action_download_fails_on_transient_user_defined_error_text(+ span_exporter: InMemorySpanExporter,+) -> None:+ now = datetime.now(UTC)+ organization = make_organization(now)+ task, step, page, browser_state, scraped_page, action = _make_download_click_context(+ now=now,+ organization=organization,+ page_url="https://example.com/portal/invoices",+ task_overrides={+ "error_code_mapping": {+ "data_not_downloadable": (+ "Return this error if the page displays "+ "download failure says the generated archive could not be saved"+ ),+ },+ },+ )+ existing_error = UserDefinedError(+ error_code="previous_error",+ reasoning="Earlier action error",+ confidence_float=0.8,+ )+ action.errors = [existing_error]+ page.evaluate = AsyncMock()++ async def expose_binding(_name: str, callback: Callable[[dict, dict], None]) -> None:+ page._transient_text_callback = callback++ page.expose_binding = AsyncMock(side_effect=expose_binding)++ async def mock_inner_handle_action(*args: object, **kwargs: object) -> list[ActionSuccess]:+ page._transient_text_callback(+ {},+ {+ "text": "Example download failure says the generated archive could not be saved",+ "timestamp_ms": 1,+ "tag": "DIV",+ "role": "alert",+ },+ )+ return [ActionSuccess()]++ with tempfile.TemporaryDirectory() as temp_dir:+ mock_app = MagicMock()+ mock_app.BROWSER_MANAGER.get_for_task.return_value = browser_state+ mock_app.DATABASE.workflow_params.create_action = AsyncMock(return_value=action)+ mock_app.STORAGE = MagicMock()+ wait_for_downloads = AsyncMock()++ started_at = time.monotonic()+ with (+ patch.object(ActionHandler, "_handle_action", side_effect=mock_inner_handle_action),+ patch("skyvern.webeye.actions.handler.get_download_dir", return_value=temp_dir),+ patch("skyvern.webeye.actions.handler.list_files_in_directory", return_value=[]),+ patch("skyvern.webeye.actions.handler.skyvern_context.current", return_value=None),+ patch(+ "skyvern.webeye.actions.handler.check_downloading_files_and_wait_for_download_to_complete",+ new=wait_for_downloads,+ ),+ patch("skyvern.webeye.actions.handler.app", mock_app),+ ):+ results = await ActionHandler.handle_action(+ scraped_page=scraped_page,+ task=task,+ step=step,+ page=page,+ action=action,+ )+ elapsed = time.monotonic() - started_at++ assert elapsed < 1.0+ assert isinstance(results[-1], ActionFailure)+ assert results[-1].download_triggered is False+ assert "download failure says the generated archive could not be saved" in (results[-1].exception_message or "")+ assert action.download_triggered is False+ assert action.errors is not None+ assert [error.error_code for error in action.errors] == ["previous_error", "data_not_downloadable"]+ assert action.terminal_user_errors is True+ assert wait_for_downloads.await_count == 0+ page.off.assert_called_once()+ assert page.expose_binding.await_count == 1+ observer_install_count = sum(+ "new MutationObserver" in call.args[0] for call in page.evaluate.await_args_list if call.args+ )+ assert observer_install_count == 2+ span_attrs = _download_wait_span_attrs(span_exporter)+ assert span_attrs["download_signal_observed"] is False+ assert span_attrs["download_wait_observed_text_count"] == 1+ assert span_attrs["download_wait_user_error_detected"] is True+ assert span_attrs["download_wait_user_error_codes"] == "data_not_downloadable"++@pytest.mark.asyncioasync def test_handle_action_prefers_observed_file_over_download_event_copy(span_exporter: InMemorySpanExporter,diff --git a/tests/unit/test_economy_tree_portal_priority.py b/tests/unit/test_economy_tree_portal_priority.pynew file mode 100644index 000000000..2f1d61573--- /dev/null+++ b/tests/unit/test_economy_tree_portal_priority.py@@ -0,0 +1,187 @@+"""SKY-10011 — economy tree portal-priority truncation.++When the economy tree still exceeds the token budget and is sliced at+percent_to_keep=2/3, portal/overlay elements (those containing+role="listbox" or role="option" descendants) must survive truncation+even if they are positioned at the end of <body> (i.e. last root elements).+"""++from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.webeye.scraper.scraped_page import ElementTreeFormat, ScrapedPage+++def _make_scraped_page(element_tree_trimmed: list[dict]) -> ScrapedPage:+ return ScrapedPage(+ elements=[],+ element_tree=[],+ element_tree_trimmed=element_tree_trimmed,+ _browser_state=MagicMock(),+ _clean_up_func=AsyncMock(return_value=[]),+ _scrape_exclude=None,+ )++[email protected](autouse=True)+def _scoped_context():+ with skyvern_context.scoped(SkyvernContext(organization_id="o_test", workflow_run_id="wr_test")):+ yield+++def _table_row_element(row_index: int) -> dict:+ """A large, non-overlay table row element."""+ return {+ "tagName": "tr",+ "attributes": {},+ "children": [+ {+ "tagName": "td",+ "attributes": {},+ "text": f"Row {row_index} " + ("cell data " * 30),+ "children": [],+ }+ ],+ }+++def _listbox_portal_element() -> dict:+ """A portal element with a listbox child, mimicking a popper dropdown."""+ return {+ "tagName": "div",+ "attributes": {},+ "children": [+ {+ "tagName": "ul",+ "attributes": {"role": "listbox"},+ "children": [+ {+ "tagName": "li",+ "attributes": {"role": "option"},+ "text": "Option A",+ "children": [],+ "interactable": True,+ "id": "opt_a",+ },+ {+ "tagName": "li",+ "attributes": {"role": "option"},+ "text": "Option B",+ "children": [],+ "interactable": True,+ "id": "opt_b",+ },+ ],+ }+ ],+ }+++def test_portal_element_survives_economy_truncation() -> None:+ """Portal at end of tree must survive when first 2/3 of chars are kept."""+ # Build a tree: many large table rows first, portal last (as portals+ # are appended to <body> by React portals).+ rows = [_table_row_element(i) for i in range(100)]+ portal = _listbox_portal_element()+ page = _make_scraped_page(rows + [portal])++ # 2/3 truncation at character level would drop the portal (it's last).+ result = page.build_economy_elements_tree(+ fmt=ElementTreeFormat.HTML,+ percent_to_keep=2 / 3,+ )++ assert "Option A" in result, "Portal listbox option must be retained after truncation"+ assert "Option B" in result, "Portal listbox option must be retained after truncation"+++def test_portal_element_survives_when_nested_one_level_deep() -> None:+ """Portal heuristic works even when listbox is nested one level inside the root div."""+ rows = [_table_row_element(i) for i in range(100)]+ portal = {+ "tagName": "div",+ "attributes": {},+ "children": [+ {+ "tagName": "div",+ "attributes": {},+ "children": [+ {+ "tagName": "ul",+ "attributes": {"role": "listbox"},+ "children": [+ {+ "tagName": "li",+ "attributes": {"role": "option"},+ "text": "Nested Option",+ "children": [],+ }+ ],+ }+ ],+ }+ ],+ }+ page = _make_scraped_page(rows + [portal])++ result = page.build_economy_elements_tree(+ fmt=ElementTreeFormat.HTML,+ percent_to_keep=2 / 3,+ )++ assert "Nested Option" in result, "Nested portal must survive truncation"+++def test_no_overlay_elements_behavior_unchanged() -> None:+ """Without any overlay elements, truncation keeps the front (unchanged behavior)."""+ # Make elements with distinctive content that can be checked+ first_elem = {+ "tagName": "div",+ "attributes": {},+ "text": "FIRST_ELEMENT_MARKER",+ "children": [],+ }+ rows = [_table_row_element(i) for i in range(100)]+ last_elem = {+ "tagName": "div",+ "attributes": {},+ "text": "LAST_ELEMENT_MARKER",+ "children": [],+ }+ page = _make_scraped_page([first_elem] + rows + [last_elem])++ result = page.build_economy_elements_tree(+ fmt=ElementTreeFormat.HTML,+ percent_to_keep=2 / 3,+ )++ assert "FIRST_ELEMENT_MARKER" in result, "Front elements must be retained"+ # Last element may or may not be dropped — just verify no crash+++def test_portal_retained_and_cache_cleared_after_refetch() -> None:+ """economy_element_tree cache is reset between calls with different trees."""+ rows = [_table_row_element(i) for i in range(50)]+ portal = _listbox_portal_element()+ page = _make_scraped_page(rows + [portal])++ # First call with no truncation — must also work+ result_full = page.build_economy_elements_tree(+ fmt=ElementTreeFormat.HTML,+ percent_to_keep=1,+ )+ assert "Option A" in result_full++ # Reset cache (simulates refresh)+ page.economy_element_tree = None++ result_truncated = page.build_economy_elements_tree(+ fmt=ElementTreeFormat.HTML,+ percent_to_keep=2 / 3,+ )+ assert "Option A" in result_truncateddiff --git a/tests/unit/test_error_detection_integration.py b/tests/unit/test_error_detection_integration.pyindex 7922780a9..46d5ebdfb 100644--- a/tests/unit/test_error_detection_integration.py+++ b/tests/unit/test_error_detection_integration.py@@ -12,6 +12,8 @@ import pytestfrom skyvern.errors.errors import ReachMaxRetriesError, UserDefinedErrorfrom skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.models import StepStatus+from skyvern.forge.sdk.schemas.tasks import TaskStatus+from skyvern.schemas.steps import AgentStepOutputfrom skyvern.utils.prompt_engine import MaxStepsReasonResponsefrom tests.unit.helpers import make_organization, make_step, make_task@@ -109,6 +111,137 @@ async def test_navigate_failure_with_error_detection(agent, mock_browser_state):assert call_kwargs["errors"][0]["error_code"] == "page_not_found"+async def test_failed_step_with_user_defined_error_does_not_retry(agent):+ now = datetime.now()+ organization = make_organization(now).model_copy(update={"max_retries_per_step": 3})+ task = make_task(+ now,+ organization,+ error_code_mapping={"data_not_downloadable": "Download error displayed"},+ )+ detected_error = UserDefinedError(+ error_code="data_not_downloadable",+ reasoning="Observed transient text during download wait: Download error displayed",+ confidence_float=1.0,+ )+ step = make_step(+ now,+ task,+ step_id="step-user-error",+ status=StepStatus.failed,+ order=1,+ retry_index=0,+ output=AgentStepOutput(+ action_results=[],+ actions_and_results=[],+ errors=[detected_error],+ terminal_user_errors=True,+ ),+ )++ with patch.object(agent, "update_task", new_callable=AsyncMock) as mock_update_task:+ mock_update_task.return_value = task++ result = await agent.handle_failed_step(organization, task, step)++ assert result is None+ mock_update_task.assert_awaited_once()+ call_task, call_status = mock_update_task.call_args.args+ assert call_task == task+ assert call_status == TaskStatus.failed+ call_kwargs = mock_update_task.call_args.kwargs+ assert call_kwargs["failure_reason"] == detected_error.reasoning+ assert call_kwargs["errors"][0]["error_code"] == "data_not_downloadable"+++async def test_failed_step_with_duplicate_user_defined_error_preserves_empty_update(agent):+ now = datetime.now()+ organization = make_organization(now).model_copy(update={"max_retries_per_step": 3})+ detected_error = UserDefinedError(+ error_code="data_not_downloadable",+ reasoning="Observed transient text during download wait: Download error displayed",+ confidence_float=0.9,+ )+ task = make_task(+ now,+ organization,+ error_code_mapping={"data_not_downloadable": "Download error displayed"},+ errors=[detected_error.model_dump()],+ )+ step = make_step(+ now,+ task,+ step_id="step-user-error",+ status=StepStatus.failed,+ order=1,+ retry_index=0,+ output=AgentStepOutput(+ action_results=[],+ actions_and_results=[],+ errors=[detected_error],+ terminal_user_errors=True,+ ),+ )++ with patch.object(agent, "update_task", new_callable=AsyncMock) as mock_update_task:+ mock_update_task.return_value = task++ result = await agent.handle_failed_step(organization, task, step)++ assert result is None+ mock_update_task.assert_awaited_once()+ assert mock_update_task.call_args.kwargs["errors"] == []+++async def test_failed_step_with_non_terminal_output_errors_uses_retry_budget(agent):+ now = datetime.now()+ organization = make_organization(now).model_copy(update={"max_retries_per_step": 3})+ task = make_task(+ now,+ organization,+ error_code_mapping={"inline_form_error": "Inline form error"},+ )+ detected_error = UserDefinedError(+ error_code="inline_form_error",+ reasoning="Inline validation text was visible",+ confidence_float=0.9,+ )+ step = make_step(+ now,+ task,+ step_id="step-non-terminal-error",+ status=StepStatus.failed,+ order=1,+ retry_index=0,+ output=AgentStepOutput(action_results=[], actions_and_results=[], errors=[detected_error]),+ )+ next_step = make_step(+ now,+ task,+ step_id="step-retry",+ status=StepStatus.created,+ order=step.order,+ retry_index=step.retry_index + 1,+ output=None,+ )++ with patch("skyvern.forge.agent.app") as mock_app:+ mock_app.DATABASE.tasks.create_step = AsyncMock(return_value=next_step)++ result = await agent.handle_failed_step(organization, task, step)++ assert result == next_step+ mock_app.DATABASE.tasks.create_step.assert_awaited_once_with(+ task_id=task.task_id,+ organization_id=task.organization_id,+ order=step.order,+ retry_index=step.retry_index + 1,+ )++@pytest.mark.asyncioasync def test_max_retries_with_error_detection(agent, mock_browser_state):"""Test error detection when max retries are exceeded."""diff --git a/tests/unit/test_transient_page_observer.py b/tests/unit/test_transient_page_observer.pynew file mode 100644index 000000000..acfe8010a--- /dev/null+++ b/tests/unit/test_transient_page_observer.py@@ -0,0 +1,124 @@+from collections.abc import Callable+from datetime import UTC, datetime+from typing import Any+from unittest.mock import AsyncMock++import pytest++from skyvern.forge.sdk.models import StepStatus+from skyvern.webeye.transient_page_observer import (+ TRANSIENT_TEXT_BINDING_NAME,+ TRANSIENT_TEXT_MATCH_CONFIDENCE,+ TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT,+ TransientPageTextObserver,+ _append_text_event,+ _format_observed_text_reasoning,+ _has_meaningful_text_overlap,+ match_user_defined_errors_from_transient_text,+)+from tests.unit.helpers import make_organization, make_step, make_task+++class _FakePage:+ def __init__(self) -> None:+ self.expose_binding = AsyncMock()+ self.evaluate = AsyncMock()+++async def test_transient_text_observer_reuses_binding_and_hands_off_active_observer() -> None:+ page = _FakePage()+ captured: dict[str, Any] = {}++ async def expose_binding(name: str, callback: Callable[[dict[str, Any], Any], None]) -> None:+ captured["name"] = name+ captured["callback"] = callback++ page.expose_binding.side_effect = expose_binding++ first_observer = TransientPageTextObserver(page) # type: ignore[arg-type]+ await first_observer.start()+ captured["callback"]({}, {"text": "First transient error", "timestamp_ms": 1})+ await first_observer.stop()++ second_observer = TransientPageTextObserver(page) # type: ignore[arg-type]+ await second_observer.start()+ captured["callback"]({}, {"text": "Second transient error", "timestamp_ms": 2})+ await second_observer.stop()++ assert captured["name"] == TRANSIENT_TEXT_BINDING_NAME+ page.expose_binding.assert_awaited_once()+ assert [event["text"] for event in first_observer.events] == ["First transient error"]+ assert [event["text"] for event in second_observer.events] == ["Second transient error"]+++def test_transient_text_overlap_requires_longer_word_window() -> None:+ assert not _has_meaningful_text_overlap(+ "the file is ready to download now",+ "show an error when the file is ready for review",+ )+ assert _has_meaningful_text_overlap(+ "Example download failure says the generated archive could not be saved",+ "Return this error if the page displays download failure says the generated archive could not be saved",+ )+++def test_transient_text_overlap_normalizes_inputs() -> None:+ assert _has_meaningful_text_overlap(+ "DOWNLOAD FAILURE SAYS THE GENERATED ARCHIVE COULD NOT BE SAVED",+ "Return this error if the page displays download failure says the generated archive could not be saved",+ )+++def test_append_text_event_omits_absent_metadata() -> None:+ events: list[dict[str, Any]] = []++ _append_text_event(+ events,+ {+ "text": "Download failure says the generated archive could not be saved",+ "tag": None,+ "role": "alert",+ },+ )++ assert events == [+ {+ "tag": None,+ "role": "alert",+ "text": "Download failure says the generated archive could not be saved",+ }+ ]+++def test_format_observed_text_reasoning_truncates_snippets() -> None:+ text = "x" * (TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT + 20)++ reasoning = _format_observed_text_reasoning([text])++ assert reasoning == f"{'x' * TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT}..."+++def test_match_user_defined_error_from_transient_text_uses_heuristic_confidence() -> None:+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(+ now,+ organization,+ error_code_mapping={+ "data_not_downloadable": (+ "Return this error if the page displays download failure says the generated archive could not be saved"+ ),+ },+ )+ step = make_step(now, task, step_id="step-1", status=StepStatus.running, order=1, output=None)++ errors = match_user_defined_errors_from_transient_text(+ task,+ step,+ [{"text": "Example download failure says the generated archive could not be saved"}],+ )++ assert len(errors) == 1+ assert errors[0].error_code == "data_not_downloadable"+ assert errors[0].confidence_float == TRANSIENT_TEXT_MATCH_CONFIDENCE