Skip to content

Automate browser based workflows with AI

Switch on Flutter web accessibility semantics so both readers can see canvas-drawn apps (SKY-18263) (#8928)

LawyZhengd5e79ebc985e
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index a372dd827..20e373744 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -1984,7 +1984,9 @@ class NoTOTPSecretFound(SkyvernException):
class NoElementFound(SkyvernException):
- def __init__(self) -> None:
+ def __init__(self, *, canvas_app: bool = False) -> None:
+ # Set when the page is an app that draws its interface on a canvas: it loaded, it just exposed nothing to read.
+ self.canvas_app = canvas_app
super().__init__("No element found.")
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index a0eb20231..c72c14049 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -123,7 +123,18 @@ from skyvern.webeye.browser_driver_errors import is_driver_error, is_driver_time
from skyvern.webeye.browser_state import BLANK_PAGE_URLS, is_selectable_page_url
from skyvern.webeye.navigation import driver_nav_error_code, record_task_nav_error_code, redact_url_secrets
from skyvern.webeye.utils.challenge_signature import CHALLENGE_VENDOR_FRAME_URL, CHALLENGE_VENDOR_SIGNATURE
-from skyvern.webeye.utils.page import OTP_INPUT_PRIVACY_JS, OTP_SAFE_FRAGMENT_HTML_JS, mask_otp_values_in_html
+from skyvern.webeye.utils.flutter_semantics import (
+ FLUTTER_STATE_JS,
+ FlutterSemanticsState,
+ SemanticsActivation,
+ activate_flutter_semantics,
+)
+from skyvern.webeye.utils.page import (
+ OTP_INPUT_PRIVACY_JS,
+ OTP_SAFE_FRAGMENT_HTML_JS,
+ SkyvernFrame,
+ mask_otp_values_in_html,
+)
if TYPE_CHECKING:
# opaque_refs imports auth_tools which imports this module, so it can only be referenced for
@@ -10937,7 +10948,7 @@ async (__tv3TextArgs) => {
} catch (e) { /* a count the page broke is not worth the reading */ }
let readyState = '';
try { readyState = String(_getter(Document.prototype, 'readyState').call(document)); } catch (e) { readyState = ''; }
- const payload = _ownStringify({ textDelta: textDelta, refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, iconsTruncated: iconsTruncated, truncatedInComponents: truncatedInComponents, pointerCapped: pointerCapped, pointerTruncated: pointerTruncated, pointerDropped: pointerDropped, pointerListed: pointerListed, pointerScanStopped: _pointerState.scanStopped ? 1 : 0, pointerScanFailed: _pointerState.scanFailed ? 1 : 0, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, modalOpen: modalOpen, modalListed: modalListed, modalTruncated: modalTruncated, readyState: readyState, elements: out });
+ const payload = _ownStringify({ textDelta: textDelta, refsFresh: refsFresh, flutter: __OBSERVE_FLUTTER__, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, iconsTruncated: iconsTruncated, truncatedInComponents: truncatedInComponents, pointerCapped: pointerCapped, pointerTruncated: pointerTruncated, pointerDropped: pointerDropped, pointerListed: pointerListed, pointerScanStopped: _pointerState.scanStopped ? 1 : 0, pointerScanFailed: _pointerState.scanFailed ? 1 : 0, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, modalOpen: modalOpen, modalListed: modalListed, modalTruncated: modalTruncated, readyState: readyState, elements: out });
return __OBSERVE_RETURN__;
}
"""
@@ -10954,6 +10965,7 @@ def _observe_js_returning(expression: str, retain_width: int) -> str:
.replace("__OBSERVE_RETURN__", expression, 1)
.replace("__OBSERVE_EL_KEY__", key)
.replace("__OBSERVE_TEXT_DELTA__", _OBSERVE_TEXT_DELTA_JS, 1)
+ .replace("__OBSERVE_FLUTTER__", FLUTTER_STATE_JS, 1)
)
@@ -12269,6 +12281,8 @@ class _Observation(NamedTuple):
# Frames left out by the walk's own caps (count or depth), which is a different fact from a frame
# that refused to be read and is disclosed separately.
capped_frames: int = 0
+ # Whether this reading switched a canvas-drawn app's accessibility tree on, or found it still off after trying.
+ semantics: SemanticsActivation | None = None
# Summed across realms: each is a count of something the reading gave up, and the model needs the
@@ -12850,6 +12864,46 @@ def build_browser_tools(
return data, handles, after
async def _read_observation(page: Any) -> _Observation:
+ """The page's reading, taken again whenever switching on a canvas-drawn app's accessibility tree may have
+ changed the page under the first one."""
+ observation = await _read_settled_observation(page)
+ first = FlutterSemanticsState.from_raw(observation.data.get("flutter"))
+ if not first.needs_activation:
+ return observation
+ keep_first = False
+ try:
+ activation = await activate_flutter_semantics(SkyvernFrame(frame=page))
+ if not activation.reading_stale:
+ keep_first = True
+ # Clicked earlier in this document, so the note says only what this reading shows.
+ return observation if first.tree else observation._replace(semantics=SemanticsActivation.NOT_ACTIVATED)
+ reading = await _read_settled_observation(page)
+ finally:
+ # Released here, even when the second read raises or is cancelled: the pairing step only ever
+ # sees the reading that is returned.
+ if not keep_first:
+ for discarded in observation.handles:
+ if discarded is None:
+ continue
+ try:
+ await discarded.dispose()
+ except Exception:
+ pass
+ # A realm whose numbering restarted on the first reading looks unchanged on the second, which would keep refs a
+ # reload made stale. Carried whatever document the second reading found: a restart reported that did not happen
+ # only renumbers that realm's refs.
+ refs_fresh = {
+ realm: fresh or observation.refs_fresh.get(realm, False) for realm, fresh in reading.refs_fresh.items()
+ }
+ if activation is SemanticsActivation.PLACEHOLDER_GONE:
+ return reading._replace(refs_fresh=refs_fresh)
+ state = FlutterSemanticsState.from_raw(reading.data.get("flutter"))
+ semantics = (
+ SemanticsActivation.ACTIVATED if state.tree and not state.placeholder else SemanticsActivation.NOT_ACTIVATED
+ )
+ return reading._replace(refs_fresh=refs_fresh, semantics=semantics)
+
+ async def _read_settled_observation(page: Any) -> _Observation:
"""One reading, retried once if a document was replaced while it was being assembled.
A page or frame navigating across a read is ordinary; navigating across two consecutive reads is
@@ -13086,6 +13140,18 @@ def build_browser_tools(
f" (url truncated from {len(sanitized_url)} chars)" if len(sanitized_url) > OBSERVE_URL_MAX_CHARS else ""
)
lines = [f"url={shown_url}{url_note} title={data.get('title')!r} ({len(elements)} interactive elements)"]
+ # Said either way: observe clicked the page to get this reading, and when that did not work the model would
+ # otherwise read the framework's placeholder as the page.
+ if observation.semantics is SemanticsActivation.ACTIVATED:
+ lines.append(
+ "note: this page draws its interface on a canvas; observe switched on its accessibility tree so its "
+ "controls could be listed"
+ )
+ elif observation.semantics is SemanticsActivation.NOT_ACTIVATED:
+ lines.append(
+ "note: this page draws its interface on a canvas and its accessibility tree did not come up, so its "
+ "controls are not listed; use look to see the page"
+ )
hidden_kept = data.get("hiddenListed") or 0
if hidden_kept:
lines.append(
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 8d5ca5134..6d57cb4ed 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -46,6 +46,11 @@ from skyvern.webeye.scraper.scraped_page import (
json_to_html,
)
from skyvern.webeye.utils.document import get_main_document_loader_id
+from skyvern.webeye.utils.flutter_semantics import (
+ FlutterSemanticsState,
+ SemanticsActivation,
+ activate_flutter_semantics,
+)
from skyvern.webeye.utils.page import SkyvernFrame, load_js_script, with_dom_utils
if TYPE_CHECKING:
@@ -55,7 +60,7 @@ LOG = structlog.get_logger()
async def build_scraping_failed_reason(
- browser_state: BrowserState, requested_url: str, *, timed_out: bool = False
+ browser_state: BrowserState, requested_url: str, *, timed_out: bool = False, canvas_app: bool = False
) -> str:
"""Build the user-facing ScrapingFailed reason with the requested URL plus the landed URL when they differ.
@@ -79,6 +84,12 @@ async def build_scraping_failed_reason(
"Skyvern hit a page-analysis timeout while loading the website. "
"The page took too long to load or become responsive during analysis."
)
+ elif canvas_app:
+ # Keeps "failed to load the website" as a substring: the page-load log filter and reports still match on it.
+ base = (
+ "Skyvern failed to load the website's controls. The page loaded, but it draws its interface on a "
+ "canvas and its accessibility tree did not come up, so there was nothing on it Skyvern could read."
+ )
else:
base = (
"Skyvern failed to load the website. "
@@ -389,7 +400,10 @@ async def scrape_website(
else:
raise ScrapingFailed(
reason=await build_scraping_failed_reason(
- browser_state, url, timed_out=_scrape_timed_out(browser_state, e)
+ browser_state,
+ url,
+ timed_out=_scrape_timed_out(browser_state, e),
+ canvas_app=isinstance(e, NoElementFound) and e.canvas_app,
)
) from e
LOG.info("Scraping failed, will retry", max_retries=max_retries, num_retry=num_retry, url=url, wait_seconds=0.5)
@@ -590,23 +604,43 @@ async def scrape_web_unsafe(
document_loader_id_before = await get_main_document_loader_id(page)
- elements, element_tree, destinations = await get_interactable_element_tree(
- page,
- scrape_exclude,
- must_included_tags,
- engine_selection=browser_state.engine_selection,
- )
- empty_page_retry = False
- if not elements and not support_empty_page:
- LOG.warning("No elements found on the page, wait and retry")
- await empty_page_retry_wait()
- empty_page_retry = True
+ async def read_tree() -> tuple[list[dict], list[dict], dict[str, dict], FlutterSemanticsState]:
+ flutter_state = FlutterSemanticsState()
elements, element_tree, destinations = await get_interactable_element_tree(
page,
scrape_exclude,
must_included_tags,
engine_selection=browser_state.engine_selection,
+ flutter_state=flutter_state,
)
+ return elements, element_tree, destinations, flutter_state
+
+ async def enable_semantics(flutter_state: FlutterSemanticsState) -> SemanticsActivation | None:
+ # Keyed on the placeholder rather than on what was listed: banners, host controls and child frames can all be
+ # readable while the app itself is not.
+ if not flutter_state.needs_activation:
+ return None
+ return await activate_flutter_semantics(skyvern_frame)
+
+ elements, element_tree, destinations, flutter_state = await read_tree()
+ activation = await enable_semantics(flutter_state)
+ if activation is not None and activation.reading_stale:
+ # Switching on can rewrite the URL (an app's hash routing), and the reading must carry the URL it was taken at.
+ url = page.url
+ elements, element_tree, destinations, flutter_state = await read_tree()
+ empty_page_retry = False
+ # An activation never stands in for this wait: an app that draws its tree after the activation's budget is read
+ # here, where skipping ahead would reach the reload that switches its semantics off again.
+ if not elements and not support_empty_page:
+ LOG.warning("No elements found on the page, wait and retry")
+ await empty_page_retry_wait()
+ empty_page_retry = True
+ elements, element_tree, destinations, flutter_state = await read_tree()
+ # An app that mounted during the wait shows its placeholder only now.
+ activation = await enable_semantics(flutter_state)
+ if activation is not None and activation.reading_stale:
+ url = page.url
+ elements, element_tree, destinations, flutter_state = await read_tree()
element_tree = await cleanup_element_tree(page, url, _deepcopy_element_tree(element_tree))
element_tree_trimmed = trim_element_tree(_deepcopy_element_tree(element_tree))
@@ -694,7 +728,7 @@ async def scrape_web_unsafe(
# if there are no elements, fail the scraping unless support_empty_page is True
if not elements and not support_empty_page:
- raise NoElementFound()
+ raise NoElementFound(canvas_app=flutter_state.tree_missing)
text_content = await get_frame_text(page.main_frame, scrape_exclude)
@@ -906,17 +940,20 @@ async def get_interactable_element_tree(
scrape_exclude: ScrapeExcludeFunc | None = None,
must_included_tags: list[str] | None = None,
engine_selection: "BrowserEngineSelection | None" = None,
+ *,
+ flutter_state: FlutterSemanticsState | None = None,
) -> tuple[list[dict], list[dict], dict[str, dict]]:
"""
Get the element tree of the page, including all the elements that are interactable.
:param page: Page instance to get the element tree from.
+ :param flutter_state: filled in with what the main frame showed of a Flutter app, when given.
:return: Tuple of the interactable elements, the element tree, and the per-element
destination-facts sidecar stripped out of them at the SkyvernFrame boundary.
"""
# main page index is 0
skyvern_page = await SkyvernFrame.create_instance(page, engine_selection=engine_selection)
elements, element_tree, destinations = await skyvern_page.build_tree_from_body(
- frame_name="main.frame", frame_index=0, must_included_tags=must_included_tags
+ frame_name="main.frame", frame_index=0, must_included_tags=must_included_tags, flutter_state=flutter_state
)
context = skyvern_context.ensure_context()
diff --git a/skyvern/webeye/utils/flutter_semantics.py b/skyvern/webeye/utils/flutter_semantics.py
new file mode 100644
index 000000000..ceca94dd5
--- /dev/null
+++ b/skyvern/webeye/utils/flutter_semantics.py
@@ -0,0 +1,184 @@
+"""Switch on a Flutter web app's accessibility semantics so its canvas-drawn interface gets a DOM to read."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any
+
+import structlog
+
+from skyvern.webeye.browser_driver_errors import is_driver_error
+from skyvern.webeye.browser_errors import BrowserTargetClosedError, is_target_closed_message
+
+if TYPE_CHECKING:
+ from skyvern.webeye.utils.page import SkyvernFrame
+
+LOG = structlog.get_logger()
+
+# One deadline spans every evaluate and sleep of an activation, so a page that never answers costs this and no more.
+ACTIVATION_BUDGET_SECONDS = 3.0
+_POLL_SECONDS = 0.25
+# Room for one round trip to a remote browser: a poll started with less can only end in a page-analysis timeout, which
+# counts against browser health.
+_MIN_EVALUATE_SECONDS = 0.5
+_ATTEMPT_MARK = "__semanticsActivationAttempted"
+
+# The placeholder exists only while semantics are off, so finding it does not depend on any reader listing it. Older
+# engines host the view in flt-glass-pane's open shadow root, so the known hosts' roots are searched as well.
+_FIND_JS = """const roots = [document];
+ const hosts = document.querySelectorAll('flutter-view, flt-glass-pane');
+ for (let i = 0; i < hosts.length && i < 8; i++) {
+ if (hosts[i].shadowRoot) roots.push(hosts[i].shadowRoot);
+ }
+ const find = (selector) => {
+ for (const root of roots) {
+ const match = root.querySelector(selector);
+ if (match) return match;
+ }
+ return null;
+ };"""
+
+# An expression rather than a function, so a reader can embed it in the script it already runs.
+FLUTTER_STATE_JS = f"""(() => {{
+ try {{
+ {_FIND_JS}
+ return {{
+ host: hosts.length > 0,
+ placeholder: find('flt-semantics-placeholder') !== null,
+ tree: find('flt-semantics') !== null,
+ }};
+ }} catch (e) {{
+ return null;
+ }}
+}})()"""
+
+_STATE_FN_JS = f"() => {FLUTTER_STATE_JS}"
+
+# One click per placeholder element: the engine listens for it from the moment it creates the placeholder, and a
+# framework that starts later still picks up the enabled state, so an ignored click is not one that came too early. A
+# rebuilt placeholder or a reload brings a new element to try, and the mobile engine deletes its placeholder after
+# repeated misses.
+_ACTIVATE_JS = f"""() => {{
+ {_FIND_JS}
+ const placeholder = hosts.length > 0 ? find('flt-semantics-placeholder') : null;
+ // Read through the prototype, so a page that answers the query with some other element never gets clicked.
+ const localName = Object.getOwnPropertyDescriptor(Element.prototype, 'localName').get;
+ if (placeholder === null || localName.call(placeholder) !== 'flt-semantics-placeholder') return 'absent';
+ if (placeholder['{_ATTEMPT_MARK}'] === true) return 'attempted';
+ Object.defineProperty(placeholder, '{_ATTEMPT_MARK}', {{ value: true }});
+ // The mobile engine only accepts a click within a pixel of the placeholder's centre; the desktop one takes any.
+ const rect = placeholder.getBoundingClientRect();
+ // The engine listens on the placeholder itself, so the click neither bubbles nor leaves a shadow root, and the
+ // page's bubbling handlers and enclosing links never see it.
+ placeholder.dispatchEvent(new MouseEvent('click', {{
+ bubbles: false,
+ cancelable: true,
+ composed: false,
+ view: window,
+ clientX: rect.left + rect.width / 2,
+ clientY: rect.top + rect.height / 2,
+ }}));
+ return 'dispatched';
+}}"""
+
+
+@dataclass
+class FlutterSemanticsState:
+ host: bool = False
+ placeholder: bool = False
+ tree: bool = False
+
+ @classmethod
+ def from_raw(cls, raw: Any) -> FlutterSemanticsState:
+ state = cls()
+ state.record(raw)
+ return state
+
+ def record(self, raw: Any) -> None:
+ if isinstance(raw, dict):
+ self.host = raw.get("host") is True
+ self.placeholder = raw.get("placeholder") is True
+ self.tree = raw.get("tree") is True
+
+ @property
+ def needs_activation(self) -> bool:
+ # The current engine puts its placeholder on the body, outside its hosts, so the element's name alone does not
+ # make a page an app.
+ return self.host and self.placeholder
+
+ @property
+ def tree_missing(self) -> bool:
+ return self.host and not self.tree
+
+
+class SemanticsActivation(StrEnum):
+ ACTIVATED = "activated"
+ NOT_ACTIVATED = "not_activated"
+ ALREADY_ATTEMPTED = "already_attempted"
+ PLACEHOLDER_GONE = "placeholder_gone"
+ FAILED = "failed"
+
+ @property
+ def reading_stale(self) -> bool:
+ # A click, a failure partway through one, or a placeholder that vanished since the reading all mean the page
+ # moved under it; only a placeholder already clicked in this document leaves it as it was read.
+ return self is not SemanticsActivation.ALREADY_ATTEMPTED
+
+
+async def activate_flutter_semantics(skyvern_frame: SkyvernFrame) -> SemanticsActivation:
+ """Click the app's 'Enable accessibility' placeholder, then wait for its semantics tree, within one deadline."""
+ loop = asyncio.get_running_loop()
+ started = loop.time()
+ deadline = started + ACTIVATION_BUDGET_SECONDS
+ try:
+ answer = await _evaluate(skyvern_frame, _ACTIVATE_JS, deadline)
+ if answer == "dispatched":
+ activated = await _wait_for_tree(skyvern_frame, deadline)
+ outcome = SemanticsActivation.ACTIVATED if activated else SemanticsActivation.NOT_ACTIVATED
+ elif answer == "attempted":
+ outcome = SemanticsActivation.ALREADY_ATTEMPTED
+ else:
+ outcome = SemanticsActivation.PLACEHOLDER_GONE
+ except Exception as error:
+ # A page, context or browser that went away is the reader's failure to report, not a failed activation.
+ if _target_closed(skyvern_frame, error):
+ raise
+ LOG.warning("Could not switch on a Flutter app's accessibility semantics", exc_info=True)
+ outcome = SemanticsActivation.FAILED
+ LOG.info(
+ "Flutter accessibility semantics activation",
+ outcome=outcome.value,
+ seconds=round(loop.time() - started, 2),
+ )
+ return outcome
+
+
+async def _wait_for_tree(skyvern_frame: SkyvernFrame, deadline: float) -> bool:
+ loop = asyncio.get_running_loop()
+ while True:
+ remaining = deadline - loop.time()
+ if remaining <= _MIN_EVALUATE_SECONDS:
+ return False
+ await asyncio.sleep(min(_POLL_SECONDS, remaining - _MIN_EVALUATE_SECONDS))
+ if FlutterSemanticsState.from_raw(await _evaluate(skyvern_frame, _STATE_FN_JS, deadline)).tree:
+ return True
+
+
+async def _evaluate(skyvern_frame: SkyvernFrame, expression: str, deadline: float) -> Any:
+ return await skyvern_frame.evaluate(
+ frame=skyvern_frame.frame,
+ expression=expression,
+ engine_selection=skyvern_frame.engine_selection,
+ deadline=deadline,
+ )
+
+
+def _target_closed(skyvern_frame: SkyvernFrame, error: Exception) -> bool:
+ engine_selection = skyvern_frame.engine_selection
+ if engine_selection is None:
+ return is_driver_error(error) and is_target_closed_message(str(error))
+ if isinstance(engine_selection.classify_error(error), BrowserTargetClosedError):
+ return True
+ return engine_selection.is_engine_error(error) and is_target_closed_message(str(error))
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 7a5f07012..14d470a20 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -49,6 +49,7 @@ from skyvern.webeye.browser_runtime_events import (
)
from skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefix
from skyvern.webeye.navigation import redact_url_secrets
+from skyvern.webeye.utils.flutter_semantics import FLUTTER_STATE_JS, FlutterSemanticsState
if TYPE_CHECKING:
from skyvern.webeye.browser_state import BrowserState
@@ -2939,6 +2940,8 @@ class SkyvernFrame:
frame_index: int,
must_included_tags: list[str] | None = None,
timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
+ *,
+ flutter_state: FlutterSemanticsState | None = None,
) -> tuple[list[dict], list[dict], dict[str, dict]]:
must_included_tags = must_included_tags or []
# Capture is flag-gated so a disabled-mode build does no destination-fact work on any page
@@ -2947,11 +2950,23 @@ class SkyvernFrame:
# unconditional: it is protection against a hostile wrapper injecting the key, not capture
# cost.
capture_destination_facts = policy_observation_enabled()
+ build_call = "await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)"
+ if flutter_state is not None:
+ # Read in the same evaluate as the tree, so knowing whether the page is a Flutter app costs no round trip.
+ build_call = (
+ f"((tree) => Array.isArray(tree) ? [tree[0], tree[1], {FLUTTER_STATE_JS}] : tree)({build_call})"
+ )
js_script = with_dom_utils(
- "async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)",
+ f"async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => {build_call}",
("buildTreeFromBody",),
)
+ def split_flutter_state(result: Any) -> Any:
+ if flutter_state is not None and isinstance(result, (list, tuple)) and len(result) == 3:
+ flutter_state.record(result[2])
+ return result[:2]
+ return result
+
# One monotonic budget across the flag write, both build attempts and the re-injection
# between them -- as _evaluate_with_navigation_recovery does -- so the retry cannot double
# what a stuck frame costs. A step that starts with nothing left times out immediately,
@@ -2973,7 +2988,7 @@ class SkyvernFrame:
deadline=deadline,
)
- tree = _as_element_tree_pair(await build())
+ tree = _as_element_tree_pair(split_flutter_state(await build()))
if tree is None:
# Re-bootstrap once for a malformed tree result; missing helpers are prepared before build().
LOG.warning(
@@ -2987,7 +3002,7 @@ class SkyvernFrame:
timeout_ms=remaining_ms(),
deadline=deadline,
)
- retried = await build()
+ retried = split_flutter_state(await build())
tree = _as_element_tree_pair(retried)
if tree is None:
raise ElementTreeBuildFailed(returned=_describe_non_pair(retried))
diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.py
index 0bbabebb5..a610cec9a 100644
--- a/tests/unit/test_failure_classifier.py
+++ b/tests/unit/test_failure_classifier.py
@@ -764,12 +764,17 @@ async def test_page_analysis_timeout_reason_ranks_page_load_timeout() -> None:
@pytest.mark.asyncio
-async def test_non_timeout_scraping_reason_stays_data_extraction() -> None:
[email protected]("canvas_app", [False, True])
+async def test_non_timeout_scraping_reason_stays_data_extraction(canvas_app: bool) -> None:
browser_state = MagicMock()
browser_state.get_working_page = AsyncMock(return_value=None)
- generic_reason = await build_scraping_failed_reason(browser_state, "https://example.com/path", timed_out=False)
+ generic_reason = await build_scraping_failed_reason(
+ browser_state, "https://example.com/path", timed_out=False, canvas_app=canvas_app
+ )
assert "timeout" not in generic_reason.lower()
+ # The page-load log filter and reporting queries match this phrase, so the canvas wording must keep it.
+ assert "failed to load the website" in generic_reason
assert _categories_for(generic_reason, ScrapingFailed(reason=generic_reason)) == ["DATA_EXTRACTION_FAILURE"]