Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Switch on Flutter web accessibility semantics so both readers can see canvas-drawn apps (SKY-18263) (#8928)
LawyZhengd5e79ebc985e
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex a372dd827..20e373744 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -1984,7 +1984,9 @@ class NoTOTPSecretFound(SkyvernException):class NoElementFound(SkyvernException):- def __init__(self) -> None:+ def __init__(self, *, canvas_app: bool = False) -> None:+ # Set when the page is an app that draws its interface on a canvas: it loaded, it just exposed nothing to read.+ self.canvas_app = canvas_appsuper().__init__("No element found.")diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex a0eb20231..c72c14049 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -123,7 +123,18 @@ from skyvern.webeye.browser_driver_errors import is_driver_error, is_driver_timefrom skyvern.webeye.browser_state import BLANK_PAGE_URLS, is_selectable_page_urlfrom skyvern.webeye.navigation import driver_nav_error_code, record_task_nav_error_code, redact_url_secretsfrom skyvern.webeye.utils.challenge_signature import CHALLENGE_VENDOR_FRAME_URL, CHALLENGE_VENDOR_SIGNATURE-from skyvern.webeye.utils.page import OTP_INPUT_PRIVACY_JS, OTP_SAFE_FRAGMENT_HTML_JS, mask_otp_values_in_html+from skyvern.webeye.utils.flutter_semantics import (+ FLUTTER_STATE_JS,+ FlutterSemanticsState,+ SemanticsActivation,+ activate_flutter_semantics,+)+from skyvern.webeye.utils.page import (+ OTP_INPUT_PRIVACY_JS,+ OTP_SAFE_FRAGMENT_HTML_JS,+ SkyvernFrame,+ mask_otp_values_in_html,+)if TYPE_CHECKING:# opaque_refs imports auth_tools which imports this module, so it can only be referenced for@@ -10937,7 +10948,7 @@ async (__tv3TextArgs) => {} catch (e) { /* a count the page broke is not worth the reading */ }let readyState = '';try { readyState = String(_getter(Document.prototype, 'readyState').call(document)); } catch (e) { readyState = ''; }- const payload = _ownStringify({ textDelta: textDelta, refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, iconsTruncated: iconsTruncated, truncatedInComponents: truncatedInComponents, pointerCapped: pointerCapped, pointerTruncated: pointerTruncated, pointerDropped: pointerDropped, pointerListed: pointerListed, pointerScanStopped: _pointerState.scanStopped ? 1 : 0, pointerScanFailed: _pointerState.scanFailed ? 1 : 0, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, modalOpen: modalOpen, modalListed: modalListed, modalTruncated: modalTruncated, readyState: readyState, elements: out });+ const payload = _ownStringify({ textDelta: textDelta, refsFresh: refsFresh, flutter: __OBSERVE_FLUTTER__, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, iconsTruncated: iconsTruncated, truncatedInComponents: truncatedInComponents, pointerCapped: pointerCapped, pointerTruncated: pointerTruncated, pointerDropped: pointerDropped, pointerListed: pointerListed, pointerScanStopped: _pointerState.scanStopped ? 1 : 0, pointerScanFailed: _pointerState.scanFailed ? 1 : 0, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, modalOpen: modalOpen, modalListed: modalListed, modalTruncated: modalTruncated, readyState: readyState, elements: out });return __OBSERVE_RETURN__;}"""@@ -10954,6 +10965,7 @@ def _observe_js_returning(expression: str, retain_width: int) -> str:.replace("__OBSERVE_RETURN__", expression, 1).replace("__OBSERVE_EL_KEY__", key).replace("__OBSERVE_TEXT_DELTA__", _OBSERVE_TEXT_DELTA_JS, 1)+ .replace("__OBSERVE_FLUTTER__", FLUTTER_STATE_JS, 1))@@ -12269,6 +12281,8 @@ class _Observation(NamedTuple):# Frames left out by the walk's own caps (count or depth), which is a different fact from a frame# that refused to be read and is disclosed separately.capped_frames: int = 0+ # Whether this reading switched a canvas-drawn app's accessibility tree on, or found it still off after trying.+ semantics: SemanticsActivation | None = None# Summed across realms: each is a count of something the reading gave up, and the model needs the@@ -12850,6 +12864,46 @@ def build_browser_tools(return data, handles, afterasync def _read_observation(page: Any) -> _Observation:+ """The page's reading, taken again whenever switching on a canvas-drawn app's accessibility tree may have+ changed the page under the first one."""+ observation = await _read_settled_observation(page)+ first = FlutterSemanticsState.from_raw(observation.data.get("flutter"))+ if not first.needs_activation:+ return observation+ keep_first = False+ try:+ activation = await activate_flutter_semantics(SkyvernFrame(frame=page))+ if not activation.reading_stale:+ keep_first = True+ # Clicked earlier in this document, so the note says only what this reading shows.+ return observation if first.tree else observation._replace(semantics=SemanticsActivation.NOT_ACTIVATED)+ reading = await _read_settled_observation(page)+ finally:+ # Released here, even when the second read raises or is cancelled: the pairing step only ever+ # sees the reading that is returned.+ if not keep_first:+ for discarded in observation.handles:+ if discarded is None:+ continue+ try:+ await discarded.dispose()+ except Exception:+ pass+ # A realm whose numbering restarted on the first reading looks unchanged on the second, which would keep refs a+ # reload made stale. Carried whatever document the second reading found: a restart reported that did not happen+ # only renumbers that realm's refs.+ refs_fresh = {+ realm: fresh or observation.refs_fresh.get(realm, False) for realm, fresh in reading.refs_fresh.items()+ }+ if activation is SemanticsActivation.PLACEHOLDER_GONE:+ return reading._replace(refs_fresh=refs_fresh)+ state = FlutterSemanticsState.from_raw(reading.data.get("flutter"))+ semantics = (+ SemanticsActivation.ACTIVATED if state.tree and not state.placeholder else SemanticsActivation.NOT_ACTIVATED+ )+ return reading._replace(refs_fresh=refs_fresh, semantics=semantics)++ async def _read_settled_observation(page: Any) -> _Observation:"""One reading, retried once if a document was replaced while it was being assembled.A page or frame navigating across a read is ordinary; navigating across two consecutive reads is@@ -13086,6 +13140,18 @@ def build_browser_tools(f" (url truncated from {len(sanitized_url)} chars)" if len(sanitized_url) > OBSERVE_URL_MAX_CHARS else "")lines = [f"url={shown_url}{url_note} title={data.get('title')!r} ({len(elements)} interactive elements)"]+ # Said either way: observe clicked the page to get this reading, and when that did not work the model would+ # otherwise read the framework's placeholder as the page.+ if observation.semantics is SemanticsActivation.ACTIVATED:+ lines.append(+ "note: this page draws its interface on a canvas; observe switched on its accessibility tree so its "+ "controls could be listed"+ )+ elif observation.semantics is SemanticsActivation.NOT_ACTIVATED:+ lines.append(+ "note: this page draws its interface on a canvas and its accessibility tree did not come up, so its "+ "controls are not listed; use look to see the page"+ )hidden_kept = data.get("hiddenListed") or 0if hidden_kept:lines.append(diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 8d5ca5134..6d57cb4ed 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -46,6 +46,11 @@ from skyvern.webeye.scraper.scraped_page import (json_to_html,)from skyvern.webeye.utils.document import get_main_document_loader_id+from skyvern.webeye.utils.flutter_semantics import (+ FlutterSemanticsState,+ SemanticsActivation,+ activate_flutter_semantics,+)from skyvern.webeye.utils.page import SkyvernFrame, load_js_script, with_dom_utilsif TYPE_CHECKING:@@ -55,7 +60,7 @@ LOG = structlog.get_logger()async def build_scraping_failed_reason(- browser_state: BrowserState, requested_url: str, *, timed_out: bool = False+ browser_state: BrowserState, requested_url: str, *, timed_out: bool = False, canvas_app: bool = False) -> str:"""Build the user-facing ScrapingFailed reason with the requested URL plus the landed URL when they differ.@@ -79,6 +84,12 @@ async def build_scraping_failed_reason("Skyvern hit a page-analysis timeout while loading the website. ""The page took too long to load or become responsive during analysis.")+ elif canvas_app:+ # Keeps "failed to load the website" as a substring: the page-load log filter and reports still match on it.+ base = (+ "Skyvern failed to load the website's controls. The page loaded, but it draws its interface on a "+ "canvas and its accessibility tree did not come up, so there was nothing on it Skyvern could read."+ )else:base = ("Skyvern failed to load the website. "@@ -389,7 +400,10 @@ async def scrape_website(else:raise ScrapingFailed(reason=await build_scraping_failed_reason(- browser_state, url, timed_out=_scrape_timed_out(browser_state, e)+ browser_state,+ url,+ timed_out=_scrape_timed_out(browser_state, e),+ canvas_app=isinstance(e, NoElementFound) and e.canvas_app,)) from eLOG.info("Scraping failed, will retry", max_retries=max_retries, num_retry=num_retry, url=url, wait_seconds=0.5)@@ -590,23 +604,43 @@ async def scrape_web_unsafe(document_loader_id_before = await get_main_document_loader_id(page)- elements, element_tree, destinations = await get_interactable_element_tree(- page,- scrape_exclude,- must_included_tags,- engine_selection=browser_state.engine_selection,- )- empty_page_retry = False- if not elements and not support_empty_page:- LOG.warning("No elements found on the page, wait and retry")- await empty_page_retry_wait()- empty_page_retry = True+ async def read_tree() -> tuple[list[dict], list[dict], dict[str, dict], FlutterSemanticsState]:+ flutter_state = FlutterSemanticsState()elements, element_tree, destinations = await get_interactable_element_tree(page,scrape_exclude,must_included_tags,engine_selection=browser_state.engine_selection,+ flutter_state=flutter_state,)+ return elements, element_tree, destinations, flutter_state++ async def enable_semantics(flutter_state: FlutterSemanticsState) -> SemanticsActivation | None:+ # Keyed on the placeholder rather than on what was listed: banners, host controls and child frames can all be+ # readable while the app itself is not.+ if not flutter_state.needs_activation:+ return None+ return await activate_flutter_semantics(skyvern_frame)++ elements, element_tree, destinations, flutter_state = await read_tree()+ activation = await enable_semantics(flutter_state)+ if activation is not None and activation.reading_stale:+ # Switching on can rewrite the URL (an app's hash routing), and the reading must carry the URL it was taken at.+ url = page.url+ elements, element_tree, destinations, flutter_state = await read_tree()+ empty_page_retry = False+ # An activation never stands in for this wait: an app that draws its tree after the activation's budget is read+ # here, where skipping ahead would reach the reload that switches its semantics off again.+ if not elements and not support_empty_page:+ LOG.warning("No elements found on the page, wait and retry")+ await empty_page_retry_wait()+ empty_page_retry = True+ elements, element_tree, destinations, flutter_state = await read_tree()+ # An app that mounted during the wait shows its placeholder only now.+ activation = await enable_semantics(flutter_state)+ if activation is not None and activation.reading_stale:+ url = page.url+ elements, element_tree, destinations, flutter_state = await read_tree()element_tree = await cleanup_element_tree(page, url, _deepcopy_element_tree(element_tree))element_tree_trimmed = trim_element_tree(_deepcopy_element_tree(element_tree))@@ -694,7 +728,7 @@ async def scrape_web_unsafe(# if there are no elements, fail the scraping unless support_empty_page is Trueif not elements and not support_empty_page:- raise NoElementFound()+ raise NoElementFound(canvas_app=flutter_state.tree_missing)text_content = await get_frame_text(page.main_frame, scrape_exclude)@@ -906,17 +940,20 @@ async def get_interactable_element_tree(scrape_exclude: ScrapeExcludeFunc | None = None,must_included_tags: list[str] | None = None,engine_selection: "BrowserEngineSelection | None" = None,+ *,+ flutter_state: FlutterSemanticsState | None = None,) -> tuple[list[dict], list[dict], dict[str, dict]]:"""Get the element tree of the page, including all the elements that are interactable.:param page: Page instance to get the element tree from.+ :param flutter_state: filled in with what the main frame showed of a Flutter app, when given.:return: Tuple of the interactable elements, the element tree, and the per-elementdestination-facts sidecar stripped out of them at the SkyvernFrame boundary."""# main page index is 0skyvern_page = await SkyvernFrame.create_instance(page, engine_selection=engine_selection)elements, element_tree, destinations = await skyvern_page.build_tree_from_body(- frame_name="main.frame", frame_index=0, must_included_tags=must_included_tags+ frame_name="main.frame", frame_index=0, must_included_tags=must_included_tags, flutter_state=flutter_state)context = skyvern_context.ensure_context()diff --git a/skyvern/webeye/utils/flutter_semantics.py b/skyvern/webeye/utils/flutter_semantics.pynew file mode 100644index 000000000..ceca94dd5--- /dev/null+++ b/skyvern/webeye/utils/flutter_semantics.py@@ -0,0 +1,184 @@+"""Switch on a Flutter web app's accessibility semantics so its canvas-drawn interface gets a DOM to read."""++from __future__ import annotations++import asyncio+from dataclasses import dataclass+from enum import StrEnum+from typing import TYPE_CHECKING, Any++import structlog++from skyvern.webeye.browser_driver_errors import is_driver_error+from skyvern.webeye.browser_errors import BrowserTargetClosedError, is_target_closed_message++if TYPE_CHECKING:+ from skyvern.webeye.utils.page import SkyvernFrame++LOG = structlog.get_logger()++# One deadline spans every evaluate and sleep of an activation, so a page that never answers costs this and no more.+ACTIVATION_BUDGET_SECONDS = 3.0+_POLL_SECONDS = 0.25+# Room for one round trip to a remote browser: a poll started with less can only end in a page-analysis timeout, which+# counts against browser health.+_MIN_EVALUATE_SECONDS = 0.5+_ATTEMPT_MARK = "__semanticsActivationAttempted"++# The placeholder exists only while semantics are off, so finding it does not depend on any reader listing it. Older+# engines host the view in flt-glass-pane's open shadow root, so the known hosts' roots are searched as well.+_FIND_JS = """const roots = [document];+ const hosts = document.querySelectorAll('flutter-view, flt-glass-pane');+ for (let i = 0; i < hosts.length && i < 8; i++) {+ if (hosts[i].shadowRoot) roots.push(hosts[i].shadowRoot);+ }+ const find = (selector) => {+ for (const root of roots) {+ const match = root.querySelector(selector);+ if (match) return match;+ }+ return null;+ };"""++# An expression rather than a function, so a reader can embed it in the script it already runs.+FLUTTER_STATE_JS = f"""(() => {{+ try {{+ {_FIND_JS}+ return {{+ host: hosts.length > 0,+ placeholder: find('flt-semantics-placeholder') !== null,+ tree: find('flt-semantics') !== null,+ }};+ }} catch (e) {{+ return null;+ }}+}})()"""++_STATE_FN_JS = f"() => {FLUTTER_STATE_JS}"++# One click per placeholder element: the engine listens for it from the moment it creates the placeholder, and a+# framework that starts later still picks up the enabled state, so an ignored click is not one that came too early. A+# rebuilt placeholder or a reload brings a new element to try, and the mobile engine deletes its placeholder after+# repeated misses.+_ACTIVATE_JS = f"""() => {{+ {_FIND_JS}+ const placeholder = hosts.length > 0 ? find('flt-semantics-placeholder') : null;+ // Read through the prototype, so a page that answers the query with some other element never gets clicked.+ const localName = Object.getOwnPropertyDescriptor(Element.prototype, 'localName').get;+ if (placeholder === null || localName.call(placeholder) !== 'flt-semantics-placeholder') return 'absent';+ if (placeholder['{_ATTEMPT_MARK}'] === true) return 'attempted';+ Object.defineProperty(placeholder, '{_ATTEMPT_MARK}', {{ value: true }});+ // The mobile engine only accepts a click within a pixel of the placeholder's centre; the desktop one takes any.+ const rect = placeholder.getBoundingClientRect();+ // The engine listens on the placeholder itself, so the click neither bubbles nor leaves a shadow root, and the+ // page's bubbling handlers and enclosing links never see it.+ placeholder.dispatchEvent(new MouseEvent('click', {{+ bubbles: false,+ cancelable: true,+ composed: false,+ view: window,+ clientX: rect.left + rect.width / 2,+ clientY: rect.top + rect.height / 2,+ }}));+ return 'dispatched';+}}"""+++@dataclass+class FlutterSemanticsState:+ host: bool = False+ placeholder: bool = False+ tree: bool = False++ @classmethod+ def from_raw(cls, raw: Any) -> FlutterSemanticsState:+ state = cls()+ state.record(raw)+ return state++ def record(self, raw: Any) -> None:+ if isinstance(raw, dict):+ self.host = raw.get("host") is True+ self.placeholder = raw.get("placeholder") is True+ self.tree = raw.get("tree") is True++ @property+ def needs_activation(self) -> bool:+ # The current engine puts its placeholder on the body, outside its hosts, so the element's name alone does not+ # make a page an app.+ return self.host and self.placeholder++ @property+ def tree_missing(self) -> bool:+ return self.host and not self.tree+++class SemanticsActivation(StrEnum):+ ACTIVATED = "activated"+ NOT_ACTIVATED = "not_activated"+ ALREADY_ATTEMPTED = "already_attempted"+ PLACEHOLDER_GONE = "placeholder_gone"+ FAILED = "failed"++ @property+ def reading_stale(self) -> bool:+ # A click, a failure partway through one, or a placeholder that vanished since the reading all mean the page+ # moved under it; only a placeholder already clicked in this document leaves it as it was read.+ return self is not SemanticsActivation.ALREADY_ATTEMPTED+++async def activate_flutter_semantics(skyvern_frame: SkyvernFrame) -> SemanticsActivation:+ """Click the app's 'Enable accessibility' placeholder, then wait for its semantics tree, within one deadline."""+ loop = asyncio.get_running_loop()+ started = loop.time()+ deadline = started + ACTIVATION_BUDGET_SECONDS+ try:+ answer = await _evaluate(skyvern_frame, _ACTIVATE_JS, deadline)+ if answer == "dispatched":+ activated = await _wait_for_tree(skyvern_frame, deadline)+ outcome = SemanticsActivation.ACTIVATED if activated else SemanticsActivation.NOT_ACTIVATED+ elif answer == "attempted":+ outcome = SemanticsActivation.ALREADY_ATTEMPTED+ else:+ outcome = SemanticsActivation.PLACEHOLDER_GONE+ except Exception as error:+ # A page, context or browser that went away is the reader's failure to report, not a failed activation.+ if _target_closed(skyvern_frame, error):+ raise+ LOG.warning("Could not switch on a Flutter app's accessibility semantics", exc_info=True)+ outcome = SemanticsActivation.FAILED+ LOG.info(+ "Flutter accessibility semantics activation",+ outcome=outcome.value,+ seconds=round(loop.time() - started, 2),+ )+ return outcome+++async def _wait_for_tree(skyvern_frame: SkyvernFrame, deadline: float) -> bool:+ loop = asyncio.get_running_loop()+ while True:+ remaining = deadline - loop.time()+ if remaining <= _MIN_EVALUATE_SECONDS:+ return False+ await asyncio.sleep(min(_POLL_SECONDS, remaining - _MIN_EVALUATE_SECONDS))+ if FlutterSemanticsState.from_raw(await _evaluate(skyvern_frame, _STATE_FN_JS, deadline)).tree:+ return True+++async def _evaluate(skyvern_frame: SkyvernFrame, expression: str, deadline: float) -> Any:+ return await skyvern_frame.evaluate(+ frame=skyvern_frame.frame,+ expression=expression,+ engine_selection=skyvern_frame.engine_selection,+ deadline=deadline,+ )+++def _target_closed(skyvern_frame: SkyvernFrame, error: Exception) -> bool:+ engine_selection = skyvern_frame.engine_selection+ if engine_selection is None:+ return is_driver_error(error) and is_target_closed_message(str(error))+ if isinstance(engine_selection.classify_error(error), BrowserTargetClosedError):+ return True+ return engine_selection.is_engine_error(error) and is_target_closed_message(str(error))diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 7a5f07012..14d470a20 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -49,6 +49,7 @@ from skyvern.webeye.browser_runtime_events import ()from skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefixfrom skyvern.webeye.navigation import redact_url_secrets+from skyvern.webeye.utils.flutter_semantics import FLUTTER_STATE_JS, FlutterSemanticsStateif TYPE_CHECKING:from skyvern.webeye.browser_state import BrowserState@@ -2939,6 +2940,8 @@ class SkyvernFrame:frame_index: int,must_included_tags: list[str] | None = None,timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,+ *,+ flutter_state: FlutterSemanticsState | None = None,) -> tuple[list[dict], list[dict], dict[str, dict]]:must_included_tags = must_included_tags or []# Capture is flag-gated so a disabled-mode build does no destination-fact work on any page@@ -2947,11 +2950,23 @@ class SkyvernFrame:# unconditional: it is protection against a hostile wrapper injecting the key, not capture# cost.capture_destination_facts = policy_observation_enabled()+ build_call = "await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)"+ if flutter_state is not None:+ # Read in the same evaluate as the tree, so knowing whether the page is a Flutter app costs no round trip.+ build_call = (+ f"((tree) => Array.isArray(tree) ? [tree[0], tree[1], {FLUTTER_STATE_JS}] : tree)({build_call})"+ )js_script = with_dom_utils(- "async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)",+ f"async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => {build_call}",("buildTreeFromBody",),)+ def split_flutter_state(result: Any) -> Any:+ if flutter_state is not None and isinstance(result, (list, tuple)) and len(result) == 3:+ flutter_state.record(result[2])+ return result[:2]+ return result+# One monotonic budget across the flag write, both build attempts and the re-injection# between them -- as _evaluate_with_navigation_recovery does -- so the retry cannot double# what a stuck frame costs. A step that starts with nothing left times out immediately,@@ -2973,7 +2988,7 @@ class SkyvernFrame:deadline=deadline,)- tree = _as_element_tree_pair(await build())+ tree = _as_element_tree_pair(split_flutter_state(await build()))if tree is None:# Re-bootstrap once for a malformed tree result; missing helpers are prepared before build().LOG.warning(@@ -2987,7 +3002,7 @@ class SkyvernFrame:timeout_ms=remaining_ms(),deadline=deadline,)- retried = await build()+ retried = split_flutter_state(await build())tree = _as_element_tree_pair(retried)if tree is None:raise ElementTreeBuildFailed(returned=_describe_non_pair(retried))diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.pyindex 0bbabebb5..a610cec9a 100644--- a/tests/unit/test_failure_classifier.py+++ b/tests/unit/test_failure_classifier.py@@ -764,12 +764,17 @@ async def test_page_analysis_timeout_reason_ranks_page_load_timeout() -> None:@pytest.mark.asyncio-async def test_non_timeout_scraping_reason_stays_data_extraction() -> None:[email protected]("canvas_app", [False, True])+async def test_non_timeout_scraping_reason_stays_data_extraction(canvas_app: bool) -> None:browser_state = MagicMock()browser_state.get_working_page = AsyncMock(return_value=None)- generic_reason = await build_scraping_failed_reason(browser_state, "https://example.com/path", timed_out=False)+ generic_reason = await build_scraping_failed_reason(+ browser_state, "https://example.com/path", timed_out=False, canvas_app=canvas_app+ )assert "timeout" not in generic_reason.lower()+ # The page-load log filter and reporting queries match this phrase, so the canvas wording must keep it.+ assert "failed to load the website" in generic_reasonassert _categories_for(generic_reason, ScrapingFailed(reason=generic_reason)) == ["DATA_EXTRACTION_FAILURE"]