Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop re-clicking a failed download in the same document (#8863)
LawyZheng4536feaf1cde
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 82c9aa3ad..d748fae33 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Mappingfrom contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontextfrom dataclasses import dataclass, fieldfrom datetime import datetime, timedelta, timezone-from typing import TYPE_CHECKING, Any, Callable, Literal, Protocol, TypedDict+from typing import TYPE_CHECKING, Any, Callable, Literal, TypedDictimport httpximport structlog@@ -97,7 +97,7 @@ from skyvern.webeye.utils.dom import SkyvernElementfrom skyvern.webeye.utils.page import SkyvernFrame, take_element_screenshotif TYPE_CHECKING:- from playwright.async_api import BrowserContext, Locator, Request, Response+ from playwright.async_api import BrowserContextfrom skyvern.forge.sdk.db.enums import WorkflowRunTriggerTypefrom skyvern.forge.sdk.schemas.totp_codes import OTPType@@ -116,7 +116,6 @@ if TYPE_CHECKING:from skyvern.schemas.workflows import WorkflowStatusfrom skyvern.services.otp_service import OTPValuefrom skyvern.webeye.browser_artifacts import DownloadBinding- from skyvern.webeye.scraper.scraped_page import ScrapedPageLOG = structlog.get_logger()@@ -974,32 +973,6 @@ class RecordingVideoSizeResolution:raw_output_bound: dict[str, int] | None-@dataclass(frozen=True)-class DownloadRecoveryRemap:- """Bounded outcome of a download-recovery ``remap``.-- ``locator`` is the retry target on the pinned page, or ``None`` when recovery must not click.- ``resolution`` names how the locator was found (``cached_css`` or ``fresh_scrape``) and is set only- when ``locator`` is present. ``reason`` is a bounded, non-sensitive rejection code when ``locator``- is ``None`` so the single structured log line can distinguish sub-outcomes without leaking URLs,- query parameters, or customer data.- """-- locator: Locator | None- resolution: str | None = None- reason: str | None = None---class DownloadRecoveryHook(Protocol):- def matches_target(self, request: Request) -> bool: ...-- def matches_failure(self, response: Response) -> bool: ...-- async def remap(self, page: Page) -> DownloadRecoveryRemap: ...-- async def reverify(self, page: Page) -> str | None: ...--class AgentFunction:async def record_audit_event(self, principal: RequestPrincipal | None, event: AuditEvent) -> None:"""OSS has no customer audit store; cloud overrides this best-effort hook."""@@ -1013,11 +986,6 @@ class AgentFunction:) -> str | None:return None- def build_download_recovery(- self, *, action: Action, scraped_page: ScrapedPage, page: Page- ) -> DownloadRecoveryHook | None:- return None-# OSS default honors the requested engine; cloud overrides to A/B-route eligible# traffic onto the native task_v3 engine.async def resolve_run_engine(diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex cedbac4bc..b01d918a0 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -17,7 +17,7 @@ from contextvars import ContextVarfrom datetime import UTC, datetime, timedelta, timezonefrom enum import StrEnumfrom pathlib import Path-from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, Callable, List, NamedTuple, TypedDict, TypeGuard, cast+from typing import Any, AsyncIterator, Awaitable, Callable, List, NamedTuple, TypedDict, TypeGuard, castimport structlogfrom cachetools import TTLCache@@ -274,9 +274,6 @@ from skyvern.webeye.utils.page import (teardown_blob_url_retention,)-if TYPE_CHECKING:- from skyvern.forge.agent_functions import DownloadRecoveryHook-LOG = structlog.get_logger()_DISPATCHER_OWNED_INPUT_EXCEPTIONS = (MissingElement,@@ -403,28 +400,6 @@ class CustomSelectFamilyOutcome(StrEnum):DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS = 60DOWNLOAD_IN_FLIGHT_EXTENSION_MAX_SECONDS = 120DOWNLOAD_IN_FLIGHT_POLL_INTERVAL_SECONDS = 1.0-# Fire the one-shot download-recovery retry this many seconds before the no-signal grace expires, so a-# slow-but-real download has the whole grace to arrive on its own before we spend the single retry.-# Clamped to the grace, so a short grace still fires the retry inside the existing wait.-DOWNLOAD_RECOVERY_LATE_FIRE_LEAD_SECONDS = 15.0-# Playwright call-log markers for the actionability check that blocked a click. The message itself is never-# logged: it can embed the intercepting element's HTML.-_CLICK_BLOCK_MARKERS = (- ("element is not visible", "not_visible"),- ("element is not enabled", "not_enabled"),- ("element is not stable", "not_stable"),- ("intercepts pointer events", "intercepted"),- ("outside of the viewport", "not_in_viewport"),-)---def _click_block_reason(exc: BaseException) -> str:- message = str(exc)- positions = [(message.rfind(marker), reason) for marker, reason in _CLICK_BLOCK_MARKERS]- position, reason = max(positions)- return reason if position >= 0 else "timeout_other"--LARGE_DOWNLOAD_LOG_THRESHOLD_BYTES = 100 * 1024 * 1024# Synchronous FileDownloadBlock false-click start-signal detection window: how long to wait for a first local# download signal (a new .crdownload/final file) before giving up, so a legitimate non-download popup is not@@ -4347,12 +4322,6 @@ class ScopedXhrDownloadCapture:monotonic: Callable[[], float] = time.monotonic,) -> None:self._page = page- self.recovery_hook: "DownloadRecoveryHook | None" = None- self.recovery_requested = False- # Retry-phase outcome of the hook's target request, kept apart from the original failure status.- self.recovery_retry_started = False- self._recovery_retry_requests: set[Request] = set()- self.recovery_retry_status: int | None = Noneself._download_dir = download_dirself._timeout_seconds = timeout_secondsself._monotonic = monotonic@@ -4448,9 +4417,6 @@ class ScopedXhrDownloadCapture:if request_page is not None:self._status_observation_child_pages.discard(request_page)- if self._is_retry_target(request):- self._recovery_retry_requests.add(request)-if self._accept_new_requests or redirected_from_admitted_request or child_page_has_bootstrap_allowance:self._in_flight_requests.add(request)self._admitted_requests.add(request)@@ -4502,35 +4468,6 @@ class ScopedXhrDownloadCapture:self._accept_new_requests = Falseself._status_observation_deadline = self._monotonic() + _STATUS_OBSERVATION_POST_SEAL_GRACE_SECONDS- def resume_in_flight_requests(self) -> None:- self._accept_new_requests = True- # Reopen the status-observation window for the one resumed retry so its own request (and the- # retry's exact 500/200-class response) is observable again. The window was sealed after the- # original action; without this the late retry's requests would start past the deadline and be- # dropped. The window is re-sealed by the caller's seal_in_flight_requests() after the retry- # click, still bounded by the original download-wait hard deadline.- self._status_observation_deadline = None- self.recovery_retry_started = True-- def recovery_retry_receipt(self) -> dict[str, bool | int | None]:- return {- "retry_request_seen": bool(self._recovery_retry_requests),- "retry_request_admitted": any(r in self._admitted_requests for r in self._recovery_retry_requests),- "retry_status": self.recovery_retry_status,- "observation_capped": self._status_observation_capped,- }-- def _is_retry_target(self, request: Request) -> bool:- try:- return (- self.recovery_retry_started- and self.recovery_hook is not None- and request in self._status_observation_requests- and self.recovery_hook.matches_target(request)- )- except Exception:- return False-def _is_xhr_download(self, headers: dict[str, str], status: int) -> bool:"""Check if an XHR response carries a downloadable file body.@@ -4573,13 +4510,9 @@ class ScopedXhrDownloadCapture:if response.request not in self._status_observation_requests:returnstatus = response.status- if self.recovery_retry_status is None and response.request in self._recovery_retry_requests:- self.recovery_retry_status = statusif not isinstance(status, int) or status not in _OBSERVED_DOWNLOAD_FAILURE_STATUSES:returnself._observed_download_failure_status = status- if self.recovery_hook is not None and self.recovery_hook.matches_failure(response):- self.recovery_requested = Trueexcept Exception:return@@ -5398,12 +5331,6 @@ class ActionHandler:staging_dir = Path(tempfile.mkdtemp(prefix="xhr_staging_", dir=get_run_temp_dir(task.organization_id, run_id or task.task_id)))- try:- recovery_hook = app.AGENT_FUNCTION.build_download_recovery(- action=action, scraped_page=scraped_page, page=page- )- except Exception:- recovery_hook = Nonexhr_capture = ScopedXhrDownloadCapture(page,staging_dir,@@ -5411,7 +5338,6 @@ class ActionHandler:if task.download_timeout is not Noneelse BROWSER_DOWNLOAD_TIMEOUT,)- xhr_capture.recovery_hook = recovery_hookdownload_triggered = Falseworking_page_recovery_attempted = Falseworking_page_replaced_after_close = False@@ -5493,20 +5419,10 @@ class ActionHandler:download_wait_hard_timeout_seconds = no_signal_grace_seconds + DOWNLOAD_IN_FLIGHT_EXTENSION_MAX_SECONDSdownload_wait_started_at = time.monotonic()download_wait_deadline = download_wait_started_at + download_wait_hard_timeout_seconds- # Late-fire the one-shot recovery retry near the end of the no-signal grace (clamped to the- # grace so a short timeout still fires inside the existing wait), giving a slow real download- # the whole grace to land on its own before the single retry is spent.- recovery_fire_after_seconds = max(0.0, no_signal_grace_seconds - DOWNLOAD_RECOVERY_LATE_FIRE_LEAD_SECONDS)def _remaining_download_wait_seconds() -> float:return max(0.0, download_wait_deadline - time.monotonic())- async def _recovery_signal_arrived() -> bool:- if download_event.done() or any(staging_dir.iterdir()):- return True- files = await _list_download_signal_files()- return bool({_download_signal_identity(file) for file in files} - signal_file_identities_before)-_download_completion_timeout = task.download_timeout or BROWSER_DOWNLOAD_TIMEOUT_download_event_grace_seconds = min(DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS, download_wait_hard_timeout_seconds@@ -5718,170 +5634,6 @@ class ActionHandler:)download_event_fallback_failed = Truebreak- recovery_elapsed_seconds = time.monotonic() - download_wait_started_at- if (- recovery_hook is not None- and xhr_capture.recovery_requested- and recovery_elapsed_seconds >= recovery_fire_after_seconds- ):- # Consume the one-shot only now that a real attempt is about to begin: an- # earlier poll where the fire time had not arrived leaves the still-eligible- # hook untouched. Every branch below emits exactly one structured- # "Download recovery click" receipt with a bounded sub-outcome.- hook, recovery_hook = recovery_hook, None- if download_event.done():- LOG.info(- "Download recovery click",- attempt=1,- result="not_attempted",- reason="event_done",- elapsed_seconds=recovery_elapsed_seconds,- )- elif any(staging_dir.iterdir()):- LOG.info(- "Download recovery click",- attempt=1,- result="not_attempted",- reason="staging_nonempty",- elapsed_seconds=recovery_elapsed_seconds,- )- elif _remaining_download_wait_seconds() <= 0:- LOG.info(- "Download recovery click",- attempt=1,- result="not_attempted",- reason="budget_exhausted",- elapsed_seconds=recovery_elapsed_seconds,- )- else:- phase = "remap"- phase_started_at = time.monotonic()- wait_ms: int | None = None- click_budget_ms: int | None = None- try:- remap = await hook.remap(page)- if remap.locator is None:- LOG.info(- "Download recovery click",- attempt=1,- result="remap_none",- reason=remap.reason,- elapsed_seconds=recovery_elapsed_seconds,- )- else:- phase = "recheck"- phase_started_at = time.monotonic()- signal_arrived = await _recovery_signal_arrived()- # Sampled once: Playwright treats timeout=0 as "no timeout".- budget_ms = _remaining_download_wait_seconds() * 1000- if signal_arrived:- LOG.info(- "Download recovery click",- attempt=1,- result="stale_signal",- resolution=remap.resolution,- elapsed_seconds=recovery_elapsed_seconds,- )- elif budget_ms < 1:- LOG.info(- "Download recovery click",- attempt=1,- result="no_budget_at_click",- resolution=remap.resolution,- elapsed_seconds=recovery_elapsed_seconds,- )- else:- # The actionability wait and the real click share one bound. The- # wait never skips the click by itself; every guard is re-proved- # after it so a signal or row re-render during the wait cannot be- # followed by a duplicate or wrong-target click.- bound_ms = min(settings.BROWSER_ACTION_TIMEOUT_MS, budget_ms)- phase = "wait"- wait_started_at = phase_started_at = time.monotonic()- wait_exc: Exception | None = None- try:- await remap.locator.click(trial=True, timeout=bound_ms)- except Exception as exc:- wait_exc = exc- waited_ms = (time.monotonic() - wait_started_at) * 1000- wait_ms = int(waited_ms)- phase = "reverify"- phase_started_at = time.monotonic()- reverify_reason = await hook.reverify(page)- signal_arrived = await _recovery_signal_arrived()- click_ms = min(- bound_ms - waited_ms, _remaining_download_wait_seconds() * 1000- )- if reverify_reason is not None:- LOG.info(- "Download recovery click",- attempt=1,- result="reverify_none",- reason=reverify_reason,- elapsed_seconds=recovery_elapsed_seconds,- )- elif signal_arrived:- LOG.info(- "Download recovery click",- attempt=1,- result="stale_signal",- phase="post_wait",- resolution=remap.resolution,- elapsed_seconds=recovery_elapsed_seconds,- )- elif click_ms < 1 and wait_exc is not None:- phase = "wait"- raise wait_exc- elif click_ms < 1:- LOG.info(- "Download recovery click",- attempt=1,- result="no_budget_at_click",- phase="post_wait",- resolution=remap.resolution,- elapsed_seconds=recovery_elapsed_seconds,- )- else:- phase = "click"- phase_started_at = time.monotonic()- click_budget_ms = int(click_ms)- xhr_capture.resume_in_flight_requests()- try:- await remap.locator.click(timeout=click_ms)- LOG.info(- "Download recovery click",- attempt=1,- result="clicked",- resolution=remap.resolution,- wait_ms=wait_ms,- click_budget_ms=click_budget_ms,- elapsed_seconds=recovery_elapsed_seconds,- )- await asyncio.sleep(0)- finally:- xhr_capture.seal_in_flight_requests()- continue- except Exception as recovery_exc:- LOG.info(- "Download recovery click",- attempt=1,- result="failed",- phase=phase,- error_type=type(recovery_exc).__name__,- reason="timeout" if "Timeout" in type(recovery_exc).__name__ else "error",- click_block_reason=_click_block_reason(recovery_exc)- if phase in ("wait", "click") and "Timeout" in type(recovery_exc).__name__- else None,- # The wait's own duration: a re-raised wait failure is logged only after- # the re-verification, which must not be counted against the bound.- phase_ms=wait_ms- if phase == "wait" and wait_ms is not None- else int((time.monotonic() - phase_started_at) * 1000),- wait_ms=wait_ms,- click_budget_ms=click_budget_ms,- elapsed_seconds=recovery_elapsed_seconds,- )-elapsed_since_action = time.monotonic() - download_wait_started_atif elapsed_since_action >= download_wait_hard_timeout_seconds:raise asyncio.TimeoutError@@ -5907,9 +5659,6 @@ class ActionHandler:workflow_run_id=task.workflow_run_id,)finally:- if xhr_capture.recovery_retry_started:- with contained_effect("record download recovery retry outcome"):- LOG.info("Download recovery retry", **xhr_capture.recovery_retry_receipt())_dl_wait_span.set_attribute("download_signal_observed", download_signal_observed)if download_signal_source:_dl_wait_span.set_attribute("download_signal_source", download_signal_source)diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 70c452d53..8d5ca5134 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -552,7 +552,6 @@ async def scrape_web_unsafe(wait_seconds: float = 0,must_included_tags: list[str] | None = None,allow_transient_ui_suppression: bool = False,- page: Page | None = None,) -> ScrapedPage:"""Asynchronous function that performs web scraping without any built-in error handling. This function is intended@@ -561,15 +560,12 @@ async def scrape_web_unsafe(:param browser_context: BrowserContext instance used for scraping.:param url: URL of the web page to be scraped. Used only when creating a new page.- :param page: Optional Page instance to scrape directly. When provided, the global working page is NOT- reacquired, so a page opened concurrently (e.g. a popup) cannot become the scrape target.:return: Tuple containing Page instance, base64 encoded screenshot, and page elements.:note: This function does not handle exceptions. Ensure proper error handling in the calling context."""# browser state must have the page instance, otherwise we should not do scraping- if page is None:- page = await browser_state.must_get_working_page()+ page = await browser_state.must_get_working_page()# Take screenshots of the page with the bounding boxes. We will remove the bounding boxes later.# Scroll to the top of the page and take a screenshot.# Scroll to the next page and take a screenshot until we reach the end of the page.diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.pyindex ff363e858..fe8d050e1 100644--- a/tests/unit/test_browser_effect_sink_enumeration.py+++ b/tests/unit/test_browser_effect_sink_enumeration.py@@ -22,8 +22,7 @@ _DISCOVERED_BROWSER_API_CALLS = {{"bring_to_front": 2,"check": 2,- # 26 = 25 prior + the download-recovery retry's bounded actionability wait before its bounded real click.- "click": 26,+ "click": 24,"clear": 6,# 6 = 5 prior + the bounded v4 FileDownloadBlock synchronous popup close (SKY-15371)."close": 6,@@ -265,14 +264,14 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:}assert observed == _DISCOVERED_BROWSER_API_CALLS- assert sum(sum(methods.values()) for methods in observed.values()) == 210+ assert sum(sum(methods.values()) for methods in observed.values()) == 208handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)classified_non_browser = Counter({signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES})assert classified_non_browser == _NON_BROWSER_CANDIDATESassert sum(_NON_BROWSER_CANDIDATES.values()) == 7- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 203+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 201def test_every_raw_evaluate_call_is_classified() -> None: