Skip to content

Automate browser based workflows with AI

[oncall] Capture transient download error popups (#6080)

Marc Kelechavac03df7ebd80d
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 1fdd34426..831f366eb 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -4683,6 +4683,7 @@ class ForgeAgent:
step.output.actions_and_results.append((persisted_action, action_results))
if isinstance(persisted_action, DecisiveAction) and persisted_action.errors:
step.output.errors.extend(persisted_action.errors)
+ step.output.terminal_user_errors = True
if isinstance(persisted_action, TerminateAction):
LOG.warning(
@@ -4871,6 +4872,39 @@ class ForgeAgent:
if organization.max_retries_per_step is not None
else settings.MAX_RETRIES_PER_STEP
)
+ step_errors = (
+ step.output.errors if step.output and step.output.errors and step.output.terminal_user_errors else []
+ )
+ if step_errors:
+ failure_reason = "; ".join(error.reasoning for error in step_errors)
+ failure_category = classify_from_failure_reason(failure_reason, fallback_to_unknown=True)
+ existing_errors = {
+ (error.get("error_code"), error.get("reasoning"))
+ for error in (task.errors or [])
+ if isinstance(error, dict)
+ }
+ new_step_errors = [
+ error.model_dump()
+ for error in step_errors
+ if (error.error_code, error.reasoning) not in existing_errors
+ ]
+ LOG.warning(
+ "Step failed with user-defined errors, marking task as failed without retry",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ step_order=step.order,
+ step_retry=step.retry_index,
+ error_codes=[error.error_code for error in step_errors],
+ )
+ await self.update_task(
+ task,
+ TaskStatus.failed,
+ failure_reason=failure_reason,
+ errors=new_step_errors,
+ failure_category=failure_category,
+ )
+ return None
+
if step.retry_index >= max_retries_per_step:
LOG.warning(
"Step failed after max retries, marking task as failed",
diff --git a/skyvern/schemas/steps.py b/skyvern/schemas/steps.py
index c9f8cd97b..9b77dc7a1 100644
--- a/skyvern/schemas/steps.py
+++ b/skyvern/schemas/steps.py
@@ -21,6 +21,8 @@ class AgentStepOutput(BaseModel):
# Nullable for backwards compatibility, once backfill is done, this won't be nullable anymore
actions_and_results: list[tuple[Action, list[ActionResult]]] | None = None
errors: list[UserDefinedError] = []
+ # Explicit no-retry signal; historical/plain errors still use the normal retry budget.
+ terminal_user_errors: bool = False
browser_metadata: BrowserMetadata | None = None
step_exception: str | None = None
diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py
index 3ea88725b..98b6ebfc3 100644
--- a/skyvern/webeye/actions/actions.py
+++ b/skyvern/webeye/actions/actions.py
@@ -131,6 +131,8 @@ class Action(BaseModel):
# DecisiveAction (CompleteAction, TerminateAction) fields
errors: list[UserDefinedError] | None = None
+ # Explicit signal that this action's user-defined errors should bypass step retry.
+ terminal_user_errors: bool = False
data_extraction_goal: str | None = None
# WebAction fields
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 9e3629022..f056a0142 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -126,6 +126,10 @@ from skyvern.webeye.scraper.scraped_page import (
json_to_html,
)
from skyvern.webeye.scraper.scraper import IncrementalScrapePage, hash_element, trim_element_tree
+from skyvern.webeye.transient_page_observer import (
+ TransientPageTextObserver,
+ match_user_defined_errors_from_transient_text,
+)
from skyvern.webeye.utils.dom import COMMON_INPUT_TAGS, DomUtil, InteractiveElement, SkyvernElement
from skyvern.webeye.utils.page import SkyvernFrame
@@ -722,8 +726,15 @@ class ActionHandler:
xhr_capture = ScopedXhrDownloadCapture(page, download_dir)
download_triggered = False
+ transient_text_observer = TransientPageTextObserver(
+ page,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ workflow_run_id=task.workflow_run_id,
+ )
page.on("download", _capture_download_event)
try:
+ await transient_text_observer.start()
xhr_capture.enable()
with _tracer.start_as_current_span("skyvern.agent.action.handle_inner") as _hi_span:
apply_context_attrs(_hi_span)
@@ -736,6 +747,7 @@ class ActionHandler:
)
if not results:
return results
+ await transient_text_observer.start()
_download_timeout = task.download_timeout or BROWSER_DOWNLOAD_MAX_WAIT_TIME
_download_event_grace_seconds = min(DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS, _download_timeout)
with _tracer.start_as_current_span("skyvern.agent.action.download_wait") as _dl_wait_span:
@@ -755,6 +767,7 @@ class ActionHandler:
download_signal_source: str | None = None
download_signal_elapsed_seconds: float | None = None
download_signal_poll_iterations: int | None = None
+ download_wait_matched_errors: list[UserDefinedError] = []
download_wait_started_at = time.monotonic()
def _record_download_signal(source: str) -> None:
@@ -850,6 +863,24 @@ class ActionHandler:
download_event_fallback_failed = True
break
elapsed_since_action = time.monotonic() - download_wait_started_at
+ if not download_signal_observed:
+ download_wait_matched_errors = match_user_defined_errors_from_transient_text(
+ task,
+ step,
+ transient_text_observer.events,
+ )
+ if download_wait_matched_errors:
+ action.errors = (action.errors or []) + download_wait_matched_errors
+ action.terminal_user_errors = True
+ LOG.warning(
+ "Stopping download wait after transient user-defined error text",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ workflow_run_id=task.workflow_run_id,
+ error_codes=[error.error_code for error in download_wait_matched_errors],
+ )
+ break
+
if not download_signal_observed and elapsed_since_action >= no_signal_grace_seconds:
LOG.warning(
"No download signal observed after action",
@@ -881,9 +912,28 @@ class ActionHandler:
_dl_wait_span.set_attribute("download_event_fallback_attempted", download_event_fallback_attempted)
_dl_wait_span.set_attribute("download_event_fallback_used", download_event_fallback_used)
_dl_wait_span.set_attribute("download_event_fallback_failed", download_event_fallback_failed)
+ _dl_wait_span.set_attribute(
+ "download_wait_observed_text_count",
+ len(transient_text_observer.events),
+ )
+ _dl_wait_span.set_attribute(
+ "download_wait_user_error_detected",
+ bool(download_wait_matched_errors),
+ )
+ if download_wait_matched_errors:
+ _dl_wait_span.set_attribute(
+ "download_wait_user_error_codes",
+ ",".join(error.error_code for error in download_wait_matched_errors),
+ )
if not download_triggered:
- results[-1].download_triggered = False
+ if action.errors:
+ results[-1] = ActionFailure(
+ Exception("; ".join(error.reasoning for error in action.errors)),
+ download_triggered=False,
+ )
+ else:
+ results[-1].download_triggered = False
action.download_triggered = False
return results
results[-1].download_triggered = True
@@ -918,6 +968,7 @@ class ActionHandler:
return results
finally:
+ await transient_text_observer.stop()
xhr_capture.disable()
if browser_state is not None and download_triggered:
# get the page count after download
diff --git a/skyvern/webeye/actions/models.py b/skyvern/webeye/actions/models.py
index 4e1281189..924f5d20b 100644
--- a/skyvern/webeye/actions/models.py
+++ b/skyvern/webeye/actions/models.py
@@ -41,8 +41,8 @@ class DetailedAgentStepOutput(BaseModel):
def extract_errors(self) -> list[UserDefinedError]:
errors = []
if self.actions_and_results:
- for action, action_results in self.actions_and_results:
- if isinstance(action, DecisiveAction):
+ for action, _action_results in self.actions_and_results:
+ if action.errors and (isinstance(action, DecisiveAction) or action.terminal_user_errors):
errors.extend(action.errors)
return errors
@@ -62,6 +62,7 @@ class DetailedAgentStepOutput(BaseModel):
def to_agent_step_output(self) -> AgentStepOutput:
clean_output = self.get_clean_detailed_output()
+ errors = clean_output.extract_errors()
browser_metadata: BrowserMetadata | None = None
if clean_output.scraped_page and clean_output.scraped_page.url:
@@ -70,7 +71,8 @@ class DetailedAgentStepOutput(BaseModel):
return AgentStepOutput(
action_results=clean_output.action_results if clean_output.action_results else [],
actions_and_results=(clean_output.actions_and_results if clean_output.actions_and_results else []),
- errors=clean_output.extract_errors(),
+ errors=errors,
+ terminal_user_errors=bool(errors),
browser_metadata=browser_metadata,
step_exception=clean_output.step_exception,
)
diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.py
index 7443c0f1a..c83f41955 100644
--- a/skyvern/webeye/scraper/scraped_page.py
+++ b/skyvern/webeye/scraper/scraped_page.py
@@ -308,9 +308,24 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):
return element_str[: int(len(element_str) * percent_to_keep)]
if fmt == ElementTreeFormat.HTML:
+ elements_to_render = self.economy_element_tree
+ if percent_to_keep < 1:
+ # Portals (popper menus, dialogs) are appended near the end of
+ # <body> and appear last in the root-level list. A naïve front
+ # slice drops them disproportionately. Move overlay root elements
+ # to the front so they survive character-level truncation.
+ overlay_roots = [e for e in self.economy_element_tree if self._element_subtree_has_overlay(e)]
+ if overlay_roots:
+ regular_roots = [e for e in self.economy_element_tree if not self._element_subtree_has_overlay(e)]
+ elements_to_render = overlay_roots + regular_roots
+ LOG.info(
+ "economy_tree_portal_priority: moved overlay roots to front before truncation",
+ overlay_count=len(overlay_roots),
+ regular_count=len(regular_roots),
+ percent_to_keep=percent_to_keep,
+ )
element_str = "".join(
- json_to_html(element, need_skyvern_attrs=html_need_skyvern_attrs)
- for element in self.economy_element_tree
+ json_to_html(element, need_skyvern_attrs=html_need_skyvern_attrs) for element in elements_to_render
)
result = element_str[: int(len(element_str) * percent_to_keep)]
self.last_used_element_tree_html = result
@@ -337,6 +352,22 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):
element["children"] = new_children
return element
+ @staticmethod
+ def _element_subtree_has_overlay(element: dict, _depth: int = 0) -> bool:
+ """Return True if this element or any descendant carries role=listbox/option.
+
+ These roles are preserved by `_trimmed_attributes` and are reliable
+ indicators of popper/dialog portal content. Recursion is limited to
+ avoid spending time walking deep table trees — portals are typically
+ shallow (1-3 levels).
+ """
+ if _depth > 4:
+ return False
+ role = (element.get("attributes") or {}).get("role", "")
+ if role in ("listbox", "option"):
+ return True
+ return any(ScrapedPage._element_subtree_has_overlay(child, _depth + 1) for child in element.get("children", []))
+
def build_lean_elements_tree(
self,
fmt: ElementTreeFormat = ElementTreeFormat.HTML,
diff --git a/skyvern/webeye/transient_page_observer.py b/skyvern/webeye/transient_page_observer.py
new file mode 100644
index 000000000..0f147d2a1
--- /dev/null
+++ b/skyvern/webeye/transient_page_observer.py
@@ -0,0 +1,286 @@
+"""Capture short-lived visible page text during explicitly scoped browser waits."""
+
+import re
+import weakref
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+from playwright.async_api import Page
+
+from skyvern.errors.errors import UserDefinedError
+from skyvern.forge.sdk.models import Step
+from skyvern.forge.sdk.schemas.tasks import Task
+
+LOG = structlog.get_logger()
+
+TRANSIENT_TEXT_EVENT_LIMIT = 100
+TRANSIENT_TEXT_MIN_LENGTH = 8
+TRANSIENT_TEXT_MATCH_MIN_LENGTH = 12
+TRANSIENT_TEXT_MATCH_WORD_WINDOW = 6
+TRANSIENT_TEXT_MAX_LENGTH = 500
+TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT = 160
+# Heuristic DOM text matches are strong signals, but not equivalent to explicit LLM classification.
+TRANSIENT_TEXT_MATCH_CONFIDENCE = 0.9
+TRANSIENT_TEXT_BINDING_NAME = "__skyvernRecordTransientText"
+TRANSIENT_TEXT_OBSERVER_STATE_KEY = "__skyvernTransientTextObserver"
+
+
+@dataclass
+class _TransientPageTextBinding:
+ active_observer: "TransientPageTextObserver | None" = None
+ registered: bool = False
+
+
+_PAGE_TRANSIENT_TEXT_BINDINGS: weakref.WeakKeyDictionary[Any, _TransientPageTextBinding] = weakref.WeakKeyDictionary()
+
+
+def _get_transient_text_binding(page: Page) -> _TransientPageTextBinding:
+ binding = _PAGE_TRANSIENT_TEXT_BINDINGS.get(page)
+ if binding is None:
+ binding = _TransientPageTextBinding()
+ _PAGE_TRANSIENT_TEXT_BINDINGS[page] = binding
+ return binding
+
+
+def _normalize_text(text: str) -> str:
+ return re.sub(r"\s+", " ", text).strip()
+
+
+def _normalize_for_match(text: str) -> str:
+ return _normalize_text(text).casefold()
+
+
+def _has_meaningful_text_overlap(observed_text: str, mapped_description: str) -> bool:
+ observed_text = _normalize_for_match(observed_text)
+ mapped_description = _normalize_for_match(mapped_description)
+ if observed_text in mapped_description or mapped_description in observed_text:
+ return True
+
+ observed_words = re.findall(r"\w+", observed_text)
+ mapped_words = re.findall(r"\w+", mapped_description)
+ mapped_word_text = f" {' '.join(mapped_words)} "
+ return len(observed_words) >= TRANSIENT_TEXT_MATCH_WORD_WINDOW and any(
+ f" {' '.join(observed_words[idx : idx + TRANSIENT_TEXT_MATCH_WORD_WINDOW])} " in mapped_word_text
+ for idx in range(len(observed_words) - TRANSIENT_TEXT_MATCH_WORD_WINDOW + 1)
+ )
+
+
+def _append_text_event(events: list[dict[str, Any]], payload: Any) -> None:
+ if not isinstance(payload, dict) or not isinstance(raw_text := payload.get("text"), str):
+ return
+
+ text = _normalize_text(raw_text)
+ if len(text) < TRANSIENT_TEXT_MIN_LENGTH:
+ return
+
+ text = text[:TRANSIENT_TEXT_MAX_LENGTH]
+ if any(event.get("text") == text for event in events):
+ return
+
+ event: dict[str, Any] = {}
+ for key in ("timestamp_ms", "tag", "role", "aria_live"):
+ if key not in payload:
+ continue
+ value = payload[key]
+ if isinstance(value, str | int | float) or value is None:
+ event[key] = value
+ event["text"] = text
+ events.append(event)
+ if len(events) > TRANSIENT_TEXT_EVENT_LIMIT:
+ del events[: len(events) - TRANSIENT_TEXT_EVENT_LIMIT]
+
+
+def _format_observed_text_reasoning(observed_texts: list[str]) -> str:
+ snippets = []
+ for text in observed_texts[:3]:
+ if len(text) > TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT:
+ text = f"{text[:TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT]}..."
+ snippets.append(text)
+ return " | ".join(snippets)
+
+
+class TransientPageTextObserver:
+ def __init__(
+ self,
+ page: Page,
+ *,
+ task_id: str | None = None,
+ step_id: str | None = None,
+ workflow_run_id: str | None = None,
+ ) -> None:
+ self.page = page
+ self.events: list[dict[str, Any]] = []
+ self._binding_state: _TransientPageTextBinding | None = None
+ self._log_context = {
+ "task_id": task_id,
+ "step_id": step_id,
+ "workflow_run_id": workflow_run_id,
+ }
+
+ async def start(self) -> None:
+ binding_state = _get_transient_text_binding(self.page)
+
+ def record_text_event(_source: dict[str, Any], payload: Any) -> None:
+ if binding_state.active_observer is not None:
+ _append_text_event(binding_state.active_observer.events, payload)
+
+ try:
+ if not binding_state.registered:
+ await self.page.expose_binding(TRANSIENT_TEXT_BINDING_NAME, record_text_event)
+ binding_state.registered = True
+ binding_state.active_observer = self
+ await self.page.evaluate(
+ """
+ ({ bindingName, stateKey, minLength, maxLength }) => {
+ const key = stateKey;
+ const normalize = (value) => String(value || "").replace(/\\s+/g, " ").trim();
+ try { window[key]?.observer?.disconnect?.(); } catch (e) {}
+
+ const isVisible = (element) => {
+ if (!element || !(element instanceof Element)) return false;
+ const style = window.getComputedStyle(element);
+ if (
+ !style ||
+ style.display === "none" ||
+ style.visibility === "hidden" ||
+ style.opacity === "0"
+ ) {
+ return false;
+ }
+ const rect = element.getBoundingClientRect();
+ return rect.width > 0 && rect.height > 0;
+ };
+
+ const emit = (node) => {
+ const element = node?.nodeType === Node.ELEMENT_NODE ? node : node?.parentElement;
+ if (!element || !isVisible(element)) return;
+ let text = element instanceof HTMLElement && typeof element.innerText === "string"
+ ? element.innerText
+ : element.textContent || "";
+ text = normalize(text);
+ if (text.length < minLength) return;
+ if (text.length > maxLength) text = text.slice(0, maxLength);
+ Promise.resolve(window[bindingName]({
+ text,
+ timestamp_ms: Date.now(),
+ tag: element.tagName || null,
+ role: element.getAttribute("role"),
+ aria_live: element.getAttribute("aria-live"),
+ })).catch(() => {});
+ };
+
+ const observer = new MutationObserver((mutations) => {
+ for (const mutation of mutations) {
+ for (const node of mutation.addedNodes) emit(node);
+ if (mutation.type === "characterData" || mutation.type === "attributes") {
+ emit(mutation.target);
+ }
+ }
+ });
+ observer.observe(document.documentElement || document.body, {
+ subtree: true,
+ childList: true,
+ characterData: true,
+ attributes: true,
+ attributeFilter: ["class", "style", "hidden", "aria-hidden", "aria-live"],
+ });
+ window[key] = { observer, bindingName };
+ }
+ """,
+ {
+ "bindingName": TRANSIENT_TEXT_BINDING_NAME,
+ "stateKey": TRANSIENT_TEXT_OBSERVER_STATE_KEY,
+ "minLength": TRANSIENT_TEXT_MIN_LENGTH,
+ "maxLength": TRANSIENT_TEXT_MAX_LENGTH,
+ },
+ )
+ self._binding_state = binding_state
+ except Exception:
+ if binding_state.active_observer is self:
+ binding_state.active_observer = None
+ LOG.warning("Failed to start transient page text observer", **self._log_context, exc_info=True)
+
+ async def stop(self) -> None:
+ binding_state = self._binding_state
+ if binding_state is None:
+ return
+ if binding_state.active_observer is self:
+ try:
+ await self.page.evaluate(
+ """
+ ({ bindingName, stateKey }) => {
+ const state = window[stateKey];
+ if (state?.bindingName === bindingName) {
+ state.observer?.disconnect?.();
+ delete window[stateKey];
+ }
+ }
+ """,
+ {"bindingName": TRANSIENT_TEXT_BINDING_NAME, "stateKey": TRANSIENT_TEXT_OBSERVER_STATE_KEY},
+ )
+ except Exception:
+ LOG.warning("Failed to stop transient page text observer", **self._log_context, exc_info=True)
+ finally:
+ binding_state.active_observer = None
+ self._binding_state = None
+
+
+def match_user_defined_errors_from_transient_text(
+ task: Task,
+ step: Step,
+ observed_text_events: list[dict[str, Any]],
+) -> list[UserDefinedError]:
+ if not task.error_code_mapping or not observed_text_events:
+ return []
+
+ observed_texts = [
+ text
+ for event in observed_text_events
+ if isinstance(text := event.get("text"), str) and len(text) >= TRANSIENT_TEXT_MATCH_MIN_LENGTH
+ ]
+ if not observed_texts:
+ return []
+
+ normalized_observed_texts = [_normalize_for_match(text) for text in observed_texts]
+ combined_observed_text = "\n".join(normalized_observed_texts)
+ matched_errors: list[UserDefinedError] = []
+ for error_code, error_description in task.error_code_mapping.items():
+ normalized_code = error_code.casefold()
+ # Restrict code matching to machine-style codes; natural words are too collision-prone in page text.
+ code_matches = (
+ "_" in error_code and re.search(rf"\b{re.escape(normalized_code)}\b", combined_observed_text) is not None
+ )
+ normalized_description = _normalize_for_match(error_description) if isinstance(error_description, str) else ""
+ description_matches = bool(
+ normalized_description
+ and (
+ normalized_description in combined_observed_text
+ or any(
+ _has_meaningful_text_overlap(observed_text, normalized_description)
+ for observed_text in normalized_observed_texts
+ )
+ )
+ )
+ if code_matches or description_matches:
+ matched_errors.append(
+ UserDefinedError(
+ error_code=error_code,
+ reasoning=f"Observed transient text during browser wait: {_format_observed_text_reasoning(observed_texts)}",
+ confidence_float=TRANSIENT_TEXT_MATCH_CONFIDENCE,
+ )
+ )
+
+ if matched_errors:
+ if len(matched_errors) > 1:
+ LOG.warning(
+ "Multiple user-defined error mappings matched transient browser text; using first match",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ matched_error_codes=[error.error_code for error in matched_errors],
+ selected_error_code=matched_errors[0].error_code,
+ tie_breaker="task.error_code_mapping_order",
+ )
+ # If multiple mappings match, preserve task.error_code_mapping order so user-authored priority is deterministic.
+ return [matched_errors[0]]
+ return []
diff --git a/tests/unit/test_action_error_extraction.py b/tests/unit/test_action_error_extraction.py
new file mode 100644
index 000000000..64f28ef90
--- /dev/null
+++ b/tests/unit/test_action_error_extraction.py
@@ -0,0 +1,63 @@
+from skyvern.errors.errors import UserDefinedError
+from skyvern.webeye.actions.actions import Action, ClickAction, CompleteAction
+from skyvern.webeye.actions.models import DetailedAgentStepOutput
+from skyvern.webeye.actions.responses import ActionSuccess
+
+
+def _error(code: str) -> UserDefinedError:
+ return UserDefinedError(error_code=code, reasoning="mapped error", confidence_float=0.9)
+
+
+def _output_for_action(action: Action) -> DetailedAgentStepOutput:
+ return DetailedAgentStepOutput(
+ scraped_page=None,
+ extract_action_prompt=None,
+ llm_response=None,
+ actions=None,
+ action_results=None,
+ actions_and_results=[(action, [ActionSuccess()])],
+ )
+
+
+def test_extract_errors_preserves_decisive_action_errors() -> None:
+ error = _error("task_failed")
+ action = CompleteAction(errors=[error])
+
+ assert _output_for_action(action).extract_errors() == [error]
+
+
+def test_extract_errors_includes_download_action_errors() -> None:
+ error = _error("data_not_downloadable")
+ action = ClickAction(element_id="download-link", download=True, errors=[error], terminal_user_errors=True)
+
+ assert _output_for_action(action).extract_errors() == [error]
+
+
+def test_agent_step_output_marks_extracted_errors_terminal() -> None:
+ error = _error("data_not_downloadable")
+ action = ClickAction(element_id="download-link", download=True, errors=[error], terminal_user_errors=True)
+
+ output = _output_for_action(action).to_agent_step_output()
+
+ assert output.errors == [error]
+ assert output.terminal_user_errors is True
+
+
+def test_extract_errors_ignores_download_action_errors_without_terminal_flag() -> None:
+ error = _error("inline_download_error")
+ action = ClickAction(element_id="download-link", download=True, errors=[error])
+
+ output = _output_for_action(action).to_agent_step_output()
+
+ assert output.errors == []
+ assert output.terminal_user_errors is False
+
+
+def test_extract_errors_ignores_non_download_web_action_errors() -> None:
+ error = _error("inline_form_error")
+ action = ClickAction(element_id="submit", download=False, errors=[error])
+
+ output = _output_for_action(action).to_agent_step_output()
+
+ assert output.errors == []
+ assert output.terminal_user_errors is False
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index 2afae2e6d..c7c7bccf9 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from skyvern.errors.errors import UserDefinedError
from skyvern.forge.sdk.models import StepStatus
from skyvern.webeye.actions.actions import ClickAction, DownloadFileAction
from skyvern.webeye.actions.handler import ActionHandler, _remove_download_listener, handle_download_file_action
@@ -25,6 +26,45 @@ def _download_wait_span_attrs(span_exporter: InMemorySpanExporter) -> dict:
return dict(span.attributes or {})
+def _make_download_click_context(
+ *,
+ now: datetime,
+ organization,
+ page_url: str,
+ task_overrides: dict | None = None,
+) -> tuple:
+ task = make_task(
+ now,
+ organization,
+ workflow_run_id="wr-1",
+ browser_session_id=None,
+ download_timeout=30.0,
+ **(task_overrides or {}),
+ )
+ step = make_step(now, task, step_id="step-1", status=StepStatus.created, order=0, output=None)
+ page = MagicMock()
+ page.url = page_url
+ page.context.browser = None
+ browser_state = MagicMock()
+ browser_state.list_valid_pages = AsyncMock(return_value=[page])
+ scraped_page = ScrapedPage(
+ elements=[],
+ element_tree=[],
+ element_tree_trimmed=[],
+ _browser_state=browser_state,
+ _clean_up_func=AsyncMock(return_value=[]),
+ _scrape_exclude=None,
+ )
+ action = ClickAction(
+ element_id="download-link",
+ download=True,
+ organization_id=task.organization_id,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ )
+ return task, step, page, browser_state, scraped_page, action
+
+
def test_remove_download_listener_uses_playwright_remove_listener_when_off_unavailable() -> None:
page = MagicMock(spec=["remove_listener"])
callback = MagicMock()
@@ -662,6 +702,100 @@ async def test_handle_action_download_no_signal_fails_fast(span_exporter: InMemo
assert "download_signal_poll_iterations" not in span_attrs
+async def test_handle_action_download_fails_on_transient_user_defined_error_text(
+ span_exporter: InMemorySpanExporter,
+) -> None:
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task, step, page, browser_state, scraped_page, action = _make_download_click_context(
+ now=now,
+ organization=organization,
+ page_url="https://example.com/portal/invoices",
+ task_overrides={
+ "error_code_mapping": {
+ "data_not_downloadable": (
+ "Return this error if the page displays "
+ "download failure says the generated archive could not be saved"
+ ),
+ },
+ },
+ )
+ existing_error = UserDefinedError(
+ error_code="previous_error",
+ reasoning="Earlier action error",
+ confidence_float=0.8,
+ )
+ action.errors = [existing_error]
+ page.evaluate = AsyncMock()
+
+ async def expose_binding(_name: str, callback: Callable[[dict, dict], None]) -> None:
+ page._transient_text_callback = callback
+
+ page.expose_binding = AsyncMock(side_effect=expose_binding)
+
+ async def mock_inner_handle_action(*args: object, **kwargs: object) -> list[ActionSuccess]:
+ page._transient_text_callback(
+ {},
+ {
+ "text": "Example download failure says the generated archive could not be saved",
+ "timestamp_ms": 1,
+ "tag": "DIV",
+ "role": "alert",
+ },
+ )
+ return [ActionSuccess()]
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ mock_app = MagicMock()
+ mock_app.BROWSER_MANAGER.get_for_task.return_value = browser_state
+ mock_app.DATABASE.workflow_params.create_action = AsyncMock(return_value=action)
+ mock_app.STORAGE = MagicMock()
+ wait_for_downloads = AsyncMock()
+
+ started_at = time.monotonic()
+ with (
+ patch.object(ActionHandler, "_handle_action", side_effect=mock_inner_handle_action),
+ patch("skyvern.webeye.actions.handler.get_download_dir", return_value=temp_dir),
+ patch("skyvern.webeye.actions.handler.list_files_in_directory", return_value=[]),
+ patch("skyvern.webeye.actions.handler.skyvern_context.current", return_value=None),
+ patch(
+ "skyvern.webeye.actions.handler.check_downloading_files_and_wait_for_download_to_complete",
+ new=wait_for_downloads,
+ ),
+ patch("skyvern.webeye.actions.handler.app", mock_app),
+ ):
+ results = await ActionHandler.handle_action(
+ scraped_page=scraped_page,
+ task=task,
+ step=step,
+ page=page,
+ action=action,
+ )
+ elapsed = time.monotonic() - started_at
+
+ assert elapsed < 1.0
+ assert isinstance(results[-1], ActionFailure)
+ assert results[-1].download_triggered is False
+ assert "download failure says the generated archive could not be saved" in (results[-1].exception_message or "")
+ assert action.download_triggered is False
+ assert action.errors is not None
+ assert [error.error_code for error in action.errors] == ["previous_error", "data_not_downloadable"]
+ assert action.terminal_user_errors is True
+ assert wait_for_downloads.await_count == 0
+ page.off.assert_called_once()
+ assert page.expose_binding.await_count == 1
+ observer_install_count = sum(
+ "new MutationObserver" in call.args[0] for call in page.evaluate.await_args_list if call.args
+ )
+ assert observer_install_count == 2
+ span_attrs = _download_wait_span_attrs(span_exporter)
+ assert span_attrs["download_signal_observed"] is False
+ assert span_attrs["download_wait_observed_text_count"] == 1
+ assert span_attrs["download_wait_user_error_detected"] is True
+ assert span_attrs["download_wait_user_error_codes"] == "data_not_downloadable"
+
+
@pytest.mark.asyncio
async def test_handle_action_prefers_observed_file_over_download_event_copy(
span_exporter: InMemorySpanExporter,
diff --git a/tests/unit/test_economy_tree_portal_priority.py b/tests/unit/test_economy_tree_portal_priority.py
new file mode 100644
index 000000000..2f1d61573
--- /dev/null
+++ b/tests/unit/test_economy_tree_portal_priority.py
@@ -0,0 +1,187 @@
+"""SKY-10011 — economy tree portal-priority truncation.
+
+When the economy tree still exceeds the token budget and is sliced at
+percent_to_keep=2/3, portal/overlay elements (those containing
+role="listbox" or role="option" descendants) must survive truncation
+even if they are positioned at the end of <body> (i.e. last root elements).
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.webeye.scraper.scraped_page import ElementTreeFormat, ScrapedPage
+
+
+def _make_scraped_page(element_tree_trimmed: list[dict]) -> ScrapedPage:
+ return ScrapedPage(
+ elements=[],
+ element_tree=[],
+ element_tree_trimmed=element_tree_trimmed,
+ _browser_state=MagicMock(),
+ _clean_up_func=AsyncMock(return_value=[]),
+ _scrape_exclude=None,
+ )
+
+
[email protected](autouse=True)
+def _scoped_context():
+ with skyvern_context.scoped(SkyvernContext(organization_id="o_test", workflow_run_id="wr_test")):
+ yield
+
+
+def _table_row_element(row_index: int) -> dict:
+ """A large, non-overlay table row element."""
+ return {
+ "tagName": "tr",
+ "attributes": {},
+ "children": [
+ {
+ "tagName": "td",
+ "attributes": {},
+ "text": f"Row {row_index} " + ("cell data " * 30),
+ "children": [],
+ }
+ ],
+ }
+
+
+def _listbox_portal_element() -> dict:
+ """A portal element with a listbox child, mimicking a popper dropdown."""
+ return {
+ "tagName": "div",
+ "attributes": {},
+ "children": [
+ {
+ "tagName": "ul",
+ "attributes": {"role": "listbox"},
+ "children": [
+ {
+ "tagName": "li",
+ "attributes": {"role": "option"},
+ "text": "Option A",
+ "children": [],
+ "interactable": True,
+ "id": "opt_a",
+ },
+ {
+ "tagName": "li",
+ "attributes": {"role": "option"},
+ "text": "Option B",
+ "children": [],
+ "interactable": True,
+ "id": "opt_b",
+ },
+ ],
+ }
+ ],
+ }
+
+
+def test_portal_element_survives_economy_truncation() -> None:
+ """Portal at end of tree must survive when first 2/3 of chars are kept."""
+ # Build a tree: many large table rows first, portal last (as portals
+ # are appended to <body> by React portals).
+ rows = [_table_row_element(i) for i in range(100)]
+ portal = _listbox_portal_element()
+ page = _make_scraped_page(rows + [portal])
+
+ # 2/3 truncation at character level would drop the portal (it's last).
+ result = page.build_economy_elements_tree(
+ fmt=ElementTreeFormat.HTML,
+ percent_to_keep=2 / 3,
+ )
+
+ assert "Option A" in result, "Portal listbox option must be retained after truncation"
+ assert "Option B" in result, "Portal listbox option must be retained after truncation"
+
+
+def test_portal_element_survives_when_nested_one_level_deep() -> None:
+ """Portal heuristic works even when listbox is nested one level inside the root div."""
+ rows = [_table_row_element(i) for i in range(100)]
+ portal = {
+ "tagName": "div",
+ "attributes": {},
+ "children": [
+ {
+ "tagName": "div",
+ "attributes": {},
+ "children": [
+ {
+ "tagName": "ul",
+ "attributes": {"role": "listbox"},
+ "children": [
+ {
+ "tagName": "li",
+ "attributes": {"role": "option"},
+ "text": "Nested Option",
+ "children": [],
+ }
+ ],
+ }
+ ],
+ }
+ ],
+ }
+ page = _make_scraped_page(rows + [portal])
+
+ result = page.build_economy_elements_tree(
+ fmt=ElementTreeFormat.HTML,
+ percent_to_keep=2 / 3,
+ )
+
+ assert "Nested Option" in result, "Nested portal must survive truncation"
+
+
+def test_no_overlay_elements_behavior_unchanged() -> None:
+ """Without any overlay elements, truncation keeps the front (unchanged behavior)."""
+ # Make elements with distinctive content that can be checked
+ first_elem = {
+ "tagName": "div",
+ "attributes": {},
+ "text": "FIRST_ELEMENT_MARKER",
+ "children": [],
+ }
+ rows = [_table_row_element(i) for i in range(100)]
+ last_elem = {
+ "tagName": "div",
+ "attributes": {},
+ "text": "LAST_ELEMENT_MARKER",
+ "children": [],
+ }
+ page = _make_scraped_page([first_elem] + rows + [last_elem])
+
+ result = page.build_economy_elements_tree(
+ fmt=ElementTreeFormat.HTML,
+ percent_to_keep=2 / 3,
+ )
+
+ assert "FIRST_ELEMENT_MARKER" in result, "Front elements must be retained"
+ # Last element may or may not be dropped — just verify no crash
+
+
+def test_portal_retained_and_cache_cleared_after_refetch() -> None:
+ """economy_element_tree cache is reset between calls with different trees."""
+ rows = [_table_row_element(i) for i in range(50)]
+ portal = _listbox_portal_element()
+ page = _make_scraped_page(rows + [portal])
+
+ # First call with no truncation — must also work
+ result_full = page.build_economy_elements_tree(
+ fmt=ElementTreeFormat.HTML,
+ percent_to_keep=1,
+ )
+ assert "Option A" in result_full
+
+ # Reset cache (simulates refresh)
+ page.economy_element_tree = None
+
+ result_truncated = page.build_economy_elements_tree(
+ fmt=ElementTreeFormat.HTML,
+ percent_to_keep=2 / 3,
+ )
+ assert "Option A" in result_truncated
diff --git a/tests/unit/test_error_detection_integration.py b/tests/unit/test_error_detection_integration.py
index 7922780a9..46d5ebdfb 100644
--- a/tests/unit/test_error_detection_integration.py
+++ b/tests/unit/test_error_detection_integration.py
@@ -12,6 +12,8 @@ import pytest
from skyvern.errors.errors import ReachMaxRetriesError, UserDefinedError
from skyvern.forge.agent import ForgeAgent
from skyvern.forge.sdk.models import StepStatus
+from skyvern.forge.sdk.schemas.tasks import TaskStatus
+from skyvern.schemas.steps import AgentStepOutput
from skyvern.utils.prompt_engine import MaxStepsReasonResponse
from tests.unit.helpers import make_organization, make_step, make_task
@@ -109,6 +111,137 @@ async def test_navigate_failure_with_error_detection(agent, mock_browser_state):
assert call_kwargs["errors"][0]["error_code"] == "page_not_found"
+async def test_failed_step_with_user_defined_error_does_not_retry(agent):
+ now = datetime.now()
+ organization = make_organization(now).model_copy(update={"max_retries_per_step": 3})
+ task = make_task(
+ now,
+ organization,
+ error_code_mapping={"data_not_downloadable": "Download error displayed"},
+ )
+ detected_error = UserDefinedError(
+ error_code="data_not_downloadable",
+ reasoning="Observed transient text during download wait: Download error displayed",
+ confidence_float=1.0,
+ )
+ step = make_step(
+ now,
+ task,
+ step_id="step-user-error",
+ status=StepStatus.failed,
+ order=1,
+ retry_index=0,
+ output=AgentStepOutput(
+ action_results=[],
+ actions_and_results=[],
+ errors=[detected_error],
+ terminal_user_errors=True,
+ ),
+ )
+
+ with patch.object(agent, "update_task", new_callable=AsyncMock) as mock_update_task:
+ mock_update_task.return_value = task
+
+ result = await agent.handle_failed_step(organization, task, step)
+
+ assert result is None
+ mock_update_task.assert_awaited_once()
+ call_task, call_status = mock_update_task.call_args.args
+ assert call_task == task
+ assert call_status == TaskStatus.failed
+ call_kwargs = mock_update_task.call_args.kwargs
+ assert call_kwargs["failure_reason"] == detected_error.reasoning
+ assert call_kwargs["errors"][0]["error_code"] == "data_not_downloadable"
+
+
+async def test_failed_step_with_duplicate_user_defined_error_preserves_empty_update(agent):
+ now = datetime.now()
+ organization = make_organization(now).model_copy(update={"max_retries_per_step": 3})
+ detected_error = UserDefinedError(
+ error_code="data_not_downloadable",
+ reasoning="Observed transient text during download wait: Download error displayed",
+ confidence_float=0.9,
+ )
+ task = make_task(
+ now,
+ organization,
+ error_code_mapping={"data_not_downloadable": "Download error displayed"},
+ errors=[detected_error.model_dump()],
+ )
+ step = make_step(
+ now,
+ task,
+ step_id="step-user-error",
+ status=StepStatus.failed,
+ order=1,
+ retry_index=0,
+ output=AgentStepOutput(
+ action_results=[],
+ actions_and_results=[],
+ errors=[detected_error],
+ terminal_user_errors=True,
+ ),
+ )
+
+ with patch.object(agent, "update_task", new_callable=AsyncMock) as mock_update_task:
+ mock_update_task.return_value = task
+
+ result = await agent.handle_failed_step(organization, task, step)
+
+ assert result is None
+ mock_update_task.assert_awaited_once()
+ assert mock_update_task.call_args.kwargs["errors"] == []
+
+
+async def test_failed_step_with_non_terminal_output_errors_uses_retry_budget(agent):
+ now = datetime.now()
+ organization = make_organization(now).model_copy(update={"max_retries_per_step": 3})
+ task = make_task(
+ now,
+ organization,
+ error_code_mapping={"inline_form_error": "Inline form error"},
+ )
+ detected_error = UserDefinedError(
+ error_code="inline_form_error",
+ reasoning="Inline validation text was visible",
+ confidence_float=0.9,
+ )
+ step = make_step(
+ now,
+ task,
+ step_id="step-non-terminal-error",
+ status=StepStatus.failed,
+ order=1,
+ retry_index=0,
+ output=AgentStepOutput(action_results=[], actions_and_results=[], errors=[detected_error]),
+ )
+ next_step = make_step(
+ now,
+ task,
+ step_id="step-retry",
+ status=StepStatus.created,
+ order=step.order,
+ retry_index=step.retry_index + 1,
+ output=None,
+ )
+
+ with patch("skyvern.forge.agent.app") as mock_app:
+ mock_app.DATABASE.tasks.create_step = AsyncMock(return_value=next_step)
+
+ result = await agent.handle_failed_step(organization, task, step)
+
+ assert result == next_step
+ mock_app.DATABASE.tasks.create_step.assert_awaited_once_with(
+ task_id=task.task_id,
+ organization_id=task.organization_id,
+ order=step.order,
+ retry_index=step.retry_index + 1,
+ )
+
+
@pytest.mark.asyncio
async def test_max_retries_with_error_detection(agent, mock_browser_state):
"""Test error detection when max retries are exceeded."""
diff --git a/tests/unit/test_transient_page_observer.py b/tests/unit/test_transient_page_observer.py
new file mode 100644
index 000000000..acfe8010a
--- /dev/null
+++ b/tests/unit/test_transient_page_observer.py
@@ -0,0 +1,124 @@
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Any
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.webeye.transient_page_observer import (
+ TRANSIENT_TEXT_BINDING_NAME,
+ TRANSIENT_TEXT_MATCH_CONFIDENCE,
+ TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT,
+ TransientPageTextObserver,
+ _append_text_event,
+ _format_observed_text_reasoning,
+ _has_meaningful_text_overlap,
+ match_user_defined_errors_from_transient_text,
+)
+from tests.unit.helpers import make_organization, make_step, make_task
+
+
+class _FakePage:
+ def __init__(self) -> None:
+ self.expose_binding = AsyncMock()
+ self.evaluate = AsyncMock()
+
+
+async def test_transient_text_observer_reuses_binding_and_hands_off_active_observer() -> None:
+ page = _FakePage()
+ captured: dict[str, Any] = {}
+
+ async def expose_binding(name: str, callback: Callable[[dict[str, Any], Any], None]) -> None:
+ captured["name"] = name
+ captured["callback"] = callback
+
+ page.expose_binding.side_effect = expose_binding
+
+ first_observer = TransientPageTextObserver(page) # type: ignore[arg-type]
+ await first_observer.start()
+ captured["callback"]({}, {"text": "First transient error", "timestamp_ms": 1})
+ await first_observer.stop()
+
+ second_observer = TransientPageTextObserver(page) # type: ignore[arg-type]
+ await second_observer.start()
+ captured["callback"]({}, {"text": "Second transient error", "timestamp_ms": 2})
+ await second_observer.stop()
+
+ assert captured["name"] == TRANSIENT_TEXT_BINDING_NAME
+ page.expose_binding.assert_awaited_once()
+ assert [event["text"] for event in first_observer.events] == ["First transient error"]
+ assert [event["text"] for event in second_observer.events] == ["Second transient error"]
+
+
+def test_transient_text_overlap_requires_longer_word_window() -> None:
+ assert not _has_meaningful_text_overlap(
+ "the file is ready to download now",
+ "show an error when the file is ready for review",
+ )
+ assert _has_meaningful_text_overlap(
+ "Example download failure says the generated archive could not be saved",
+ "Return this error if the page displays download failure says the generated archive could not be saved",
+ )
+
+
+def test_transient_text_overlap_normalizes_inputs() -> None:
+ assert _has_meaningful_text_overlap(
+ "DOWNLOAD FAILURE SAYS THE GENERATED ARCHIVE COULD NOT BE SAVED",
+ "Return this error if the page displays download failure says the generated archive could not be saved",
+ )
+
+
+def test_append_text_event_omits_absent_metadata() -> None:
+ events: list[dict[str, Any]] = []
+
+ _append_text_event(
+ events,
+ {
+ "text": "Download failure says the generated archive could not be saved",
+ "tag": None,
+ "role": "alert",
+ },
+ )
+
+ assert events == [
+ {
+ "tag": None,
+ "role": "alert",
+ "text": "Download failure says the generated archive could not be saved",
+ }
+ ]
+
+
+def test_format_observed_text_reasoning_truncates_snippets() -> None:
+ text = "x" * (TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT + 20)
+
+ reasoning = _format_observed_text_reasoning([text])
+
+ assert reasoning == f"{'x' * TRANSIENT_TEXT_REASONING_SNIPPET_LIMIT}..."
+
+
+def test_match_user_defined_error_from_transient_text_uses_heuristic_confidence() -> None:
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task = make_task(
+ now,
+ organization,
+ error_code_mapping={
+ "data_not_downloadable": (
+ "Return this error if the page displays download failure says the generated archive could not be saved"
+ ),
+ },
+ )
+ step = make_step(now, task, step_id="step-1", status=StepStatus.running, order=1, output=None)
+
+ errors = match_user_defined_errors_from_transient_text(
+ task,
+ step,
+ [{"text": "Example download failure says the generated archive could not be saved"}],
+ )
+
+ assert len(errors) == 1
+ assert errors[0].error_code == "data_not_downloadable"
+ assert errors[0].confidence_float == TRANSIENT_TEXT_MATCH_CONFIDENCE