Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Capture download-wait error overlays that animate in after insertion (#8072)
LawyZheng3140edda23cc
diff --git a/skyvern/webeye/transient_page_observer.py b/skyvern/webeye/transient_page_observer.pyindex a1c0964db..eaf809809 100644--- a/skyvern/webeye/transient_page_observer.py+++ b/skyvern/webeye/transient_page_observer.py@@ -160,13 +160,13 @@ class TransientPageTextObserver:return rect.width > 0 && rect.height > 0;};- const emit = (node) => {- const element = node?.nodeType === Node.ELEMENT_NODE ? node : node?.parentElement;- if (!element || !isVisible(element)) return;- let text = element instanceof HTMLElement && typeof element.innerText === "string"+ const innerTextOf = (element) =>+ element instanceof HTMLElement && typeof element.innerText === "string"? element.innerText: element.textContent || "";- text = normalize(text);++ const capture = (element) => {+ let text = normalize(innerTextOf(element));if (text.length < minLength) return;if (text.length > maxLength) text = text.slice(0, maxLength);Promise.resolve(window[bindingName]({@@ -178,6 +178,73 @@ class TransientPageTextObserver:})).catch(() => {});};+ let state;+ // element -> deadline (ms epoch). Each deferred node carries its own recheck budget so a+ // node admitted late still gets the full window, independent of when other nodes were added.+ const pendingVisible = new Map();+ // The download handler reinstalls this observer after the action and before the wait; carry+ // not-yet-visible overlays across that reinstall so one inserted invisible under the previous+ // generation is not dropped before it animates into visibility.+ if (previousState && previousState.pendingVisible instanceof Map) {+ const carryNow = Date.now();+ for (const [carriedElement, carriedDeadline] of previousState.pendingVisible) {+ if (carriedElement && carriedElement.isConnected && carriedDeadline > carryNow) {+ pendingVisible.set(carriedElement, carriedDeadline);+ }+ }+ }+ const RECHECK_BUDGET_MS = 900;+ const SWEEP_INTERVAL_MS = 100;+ let sweepTimer = 0;+ const sweepPending = () => {+ sweepTimer = 0;+ if (window[key] !== state) {+ pendingVisible.clear();+ return;+ }+ const now = Date.now();+ for (const [element, deadline] of Array.from(pendingVisible)) {+ if (!element.isConnected) {+ pendingVisible.delete(element);+ } else if (isVisible(element)) {+ pendingVisible.delete(element);+ capture(element);+ } else if (now >= deadline) {+ pendingVisible.delete(element);+ }+ }+ if (pendingVisible.size > 0) scheduleSweep();+ };+ const scheduleSweep = () => {+ if (sweepTimer || pendingVisible.size === 0) return;+ sweepTimer = setTimeout(sweepPending, SWEEP_INTERVAL_MS);+ };+ // Overlays (toasts/dialogs) are frequently inserted invisible (opacity:0 / off-screen) and+ // animate in without another observed mutation, so a point-in-time visibility check at+ // insertion misses them. Re-check each admitted node until its own deadline (one shared+ // timer, no per-node timers) so it still becomes evidence once it turns visible.+ const deferVisibility = (element) => {+ if (pendingVisible.has(element)) {+ scheduleSweep();+ return;+ }+ if (pendingVisible.size >= 50) return;+ if (normalize(innerTextOf(element)).length < minLength) return;+ pendingVisible.set(element, Date.now() + RECHECK_BUDGET_MS);+ scheduleSweep();+ };++ const emit = (node) => {+ const element = node?.nodeType === Node.ELEMENT_NODE ? node : node?.parentElement;+ if (!element) return;+ if (isVisible(element)) {+ pendingVisible.delete(element);+ capture(element);+ } else if (element.isConnected) {+ deferVisibility(element);+ }+ };+let observer;try {observer = new MutationObserver((mutations) => {@@ -197,20 +264,17 @@ class TransientPageTextObserver:});const selectors = "[role='alert'],[role='status'],[aria-live]:not([aria-live='off'])";const visibleSemanticElements = Array.from(document.querySelectorAll(selectors)).filter(isVisible);- const textForElement = (element) => normalize(- element instanceof HTMLElement && typeof element.innerText === "string"- ? element.innerText- : element.textContent || ""- );- const visibleSemanticTexts = visibleSemanticElements.map(textForElement);+ const visibleSemanticTexts = visibleSemanticElements.map((element) => normalize(innerTextOf(element)));if (scanInitialVisibleState) {const staleVisibleTexts = new Set(previousState?.visibleSemanticTexts || []);for (const element of visibleSemanticElements) {- if (!staleVisibleTexts.has(textForElement(element))) emit(element);+ if (!staleVisibleTexts.has(normalize(innerTextOf(element)))) capture(element);}}- window[key] = { observer, bindingName, visibleSemanticTexts };+ state = { observer, bindingName, visibleSemanticTexts, pendingVisible };+ window[key] = state;try { previousState?.observer?.disconnect?.(); } catch (e) {}+ scheduleSweep();} catch (error) {try { observer?.disconnect?.(); } catch (e) {}throw error;diff --git a/tests/unit/test_transient_page_observer.py b/tests/unit/test_transient_page_observer.pyindex e9f6205a5..57f0a4016 100644--- a/tests/unit/test_transient_page_observer.py+++ b/tests/unit/test_transient_page_observer.py@@ -158,6 +158,159 @@ async def test_transient_text_observer_can_observe_mutations_without_scanning_viassert [event["text"] for event in observer.events] == ["A new generated document error"]+async def test_transient_text_observer_captures_overlay_that_animates_in_after_insertion(+ chromium_browser: Browser,+) -> None:+ # Production-shaped miss: an error toast is inserted invisible and animates in (opacity 0 -> 1) with+ # no further observed mutation, so a point-in-time visibility check at the insertion mutation drops+ # it and it never becomes step evidence.+ page = await chromium_browser.new_page()+ await page.set_content(+ """+ <style>+ @keyframes skyReveal { from { opacity: 0; } to { opacity: 1; } }+ .sky-toast {+ position: fixed; top: 10px; right: 10px; width: 240px; height: 64px;+ animation: skyReveal 200ms linear 80ms both;+ }+ </style>+ <div id="host"></div>+ """+ )+ observer = TransientPageTextObserver(page)++ await observer.start(scan_initial_visible_state=True)+ await page.evaluate(+ """+ () => document.getElementById("host").insertAdjacentHTML(+ "beforeend",+ '<div class="sky-toast" role="alert">Fehlermeldung Beim Herunterladen der Datei ist ein Fehler aufgetreten. Bitte versuchen Sie es spater noch einmal.</div>'+ )+ """+ )+ await page.wait_for_timeout(700)+ await observer.stop()++ assert any(+ "Beim Herunterladen der Datei ist ein Fehler aufgetreten" in event["text"] for event in observer.events+ ), observer.events+++async def test_transient_text_observer_ignores_element_that_never_becomes_visible(+ chromium_browser: Browser,+) -> None:+ # Negative coverage: a stale/hidden overlay that never becomes visible must not be captured, so a+ # successful download (no error surfaced) or an unrelated invisible node cannot cause a false match.+ page = await chromium_browser.new_page()+ await page.set_content('<div id="host"></div>')+ observer = TransientPageTextObserver(page)++ await observer.start(scan_initial_visible_state=True)+ await page.evaluate(+ """+ () => document.getElementById("host").insertAdjacentHTML(+ "beforeend",+ '<div style="position:fixed;top:10px;right:10px;width:240px;height:64px;opacity:0" role="alert">Hidden overlay text that never becomes visible</div>'+ )+ """+ )+ await page.wait_for_timeout(700)+ await observer.stop()++ assert observer.events == []+++async def test_transient_text_observer_captures_alert_inserted_near_an_earlier_node_deadline(+ chromium_browser: Browser,+) -> None:+ # A first invisible node is admitted early; a real alert then appears ~850ms later (near the first+ # node's recheck budget) and animates in. Each admitted node must carry its own recheck deadline, so+ # the late alert still gets its full window instead of being dropped by a schedule shared with the+ # earlier node.+ page = await chromium_browser.new_page()+ await page.set_content(+ """+ <style>+ @keyframes skyLateReveal { from { opacity: 0; } to { opacity: 1; } }+ .sky-late-toast {+ position: fixed; top: 90px; right: 10px; width: 240px; height: 64px;+ animation: skyLateReveal 200ms linear 250ms both;+ }+ </style>+ <div id="host"></div>+ """+ )+ observer = TransientPageTextObserver(page)++ await observer.start(scan_initial_visible_state=True)+ await page.evaluate(+ """+ () => document.getElementById("host").insertAdjacentHTML(+ "beforeend",+ '<div style="position:fixed;top:10px;right:10px;width:240px;height:64px;opacity:0" role="status">Earlier node that stays invisible the whole time</div>'+ )+ """+ )+ await page.wait_for_timeout(850)+ await page.evaluate(+ """+ () => document.getElementById("host").insertAdjacentHTML(+ "beforeend",+ '<div class="sky-late-toast" role="alert">Late arriving download error alert text</div>'+ )+ """+ )+ await page.wait_for_timeout(800)+ await observer.stop()++ texts = [event["text"] for event in observer.events]+ assert any("Late arriving download error alert text" in text for text in texts), texts+ assert all("Earlier node that stays invisible" not in text for text in texts), texts+++async def test_transient_text_observer_captures_overlay_straddling_post_action_reinstall(+ chromium_browser: Browser,+) -> None:+ # Mirrors the download handler lifecycle: the observer is installed before the action+ # (scan_initial_visible_state=False), then reinstalled after the action+ # (scan_initial_visible_state=True) before the download wait. An error overlay inserted+ # invisible during the action, still mid animation-delay when the reinstall lands, must+ # survive the reinstall and be captured once it animates into visibility.+ page = await chromium_browser.new_page()+ await page.set_content(+ """+ <style>+ @keyframes skyStraddleReveal { from { opacity: 0; } to { opacity: 1; } }+ .sky-straddle-toast {+ position: fixed; top: 10px; right: 10px; width: 240px; height: 64px;+ animation: skyStraddleReveal 200ms linear 250ms both;+ }+ </style>+ <div id="host"></div>+ """+ )+ observer = TransientPageTextObserver(page)++ await observer.start(scan_initial_visible_state=False)+ await page.evaluate(+ """+ () => document.getElementById("host").insertAdjacentHTML(+ "beforeend",+ '<div class="sky-straddle-toast" role="alert">Straddle download error alert text</div>'+ )+ """+ )+ await observer.start(scan_initial_visible_state=True)+ await page.wait_for_timeout(700)+ await observer.stop()++ assert any("Straddle download error alert text" in event["text"] for event in observer.events), observer.events++@pytest.mark.asyncioasync def test_transient_text_observer_failed_reinstall_preserves_routing_and_cleanup_ownership() -> None:page = _FakePage()