Skip to content

Automate browser based workflows with AI

Capture download-wait error overlays that animate in after insertion (#8072)

LawyZheng3140edda23cc
diff --git a/skyvern/webeye/transient_page_observer.py b/skyvern/webeye/transient_page_observer.py
index a1c0964db..eaf809809 100644
--- a/skyvern/webeye/transient_page_observer.py
+++ b/skyvern/webeye/transient_page_observer.py
@@ -160,13 +160,13 @@ class TransientPageTextObserver:
return rect.width > 0 && rect.height > 0;
};
- const emit = (node) => {
- const element = node?.nodeType === Node.ELEMENT_NODE ? node : node?.parentElement;
- if (!element || !isVisible(element)) return;
- let text = element instanceof HTMLElement && typeof element.innerText === "string"
+ const innerTextOf = (element) =>
+ element instanceof HTMLElement && typeof element.innerText === "string"
? element.innerText
: element.textContent || "";
- text = normalize(text);
+
+ const capture = (element) => {
+ let text = normalize(innerTextOf(element));
if (text.length < minLength) return;
if (text.length > maxLength) text = text.slice(0, maxLength);
Promise.resolve(window[bindingName]({
@@ -178,6 +178,73 @@ class TransientPageTextObserver:
})).catch(() => {});
};
+ let state;
+ // element -> deadline (ms epoch). Each deferred node carries its own recheck budget so a
+ // node admitted late still gets the full window, independent of when other nodes were added.
+ const pendingVisible = new Map();
+ // The download handler reinstalls this observer after the action and before the wait; carry
+ // not-yet-visible overlays across that reinstall so one inserted invisible under the previous
+ // generation is not dropped before it animates into visibility.
+ if (previousState && previousState.pendingVisible instanceof Map) {
+ const carryNow = Date.now();
+ for (const [carriedElement, carriedDeadline] of previousState.pendingVisible) {
+ if (carriedElement && carriedElement.isConnected && carriedDeadline > carryNow) {
+ pendingVisible.set(carriedElement, carriedDeadline);
+ }
+ }
+ }
+ const RECHECK_BUDGET_MS = 900;
+ const SWEEP_INTERVAL_MS = 100;
+ let sweepTimer = 0;
+ const sweepPending = () => {
+ sweepTimer = 0;
+ if (window[key] !== state) {
+ pendingVisible.clear();
+ return;
+ }
+ const now = Date.now();
+ for (const [element, deadline] of Array.from(pendingVisible)) {
+ if (!element.isConnected) {
+ pendingVisible.delete(element);
+ } else if (isVisible(element)) {
+ pendingVisible.delete(element);
+ capture(element);
+ } else if (now >= deadline) {
+ pendingVisible.delete(element);
+ }
+ }
+ if (pendingVisible.size > 0) scheduleSweep();
+ };
+ const scheduleSweep = () => {
+ if (sweepTimer || pendingVisible.size === 0) return;
+ sweepTimer = setTimeout(sweepPending, SWEEP_INTERVAL_MS);
+ };
+ // Overlays (toasts/dialogs) are frequently inserted invisible (opacity:0 / off-screen) and
+ // animate in without another observed mutation, so a point-in-time visibility check at
+ // insertion misses them. Re-check each admitted node until its own deadline (one shared
+ // timer, no per-node timers) so it still becomes evidence once it turns visible.
+ const deferVisibility = (element) => {
+ if (pendingVisible.has(element)) {
+ scheduleSweep();
+ return;
+ }
+ if (pendingVisible.size >= 50) return;
+ if (normalize(innerTextOf(element)).length < minLength) return;
+ pendingVisible.set(element, Date.now() + RECHECK_BUDGET_MS);
+ scheduleSweep();
+ };
+
+ const emit = (node) => {
+ const element = node?.nodeType === Node.ELEMENT_NODE ? node : node?.parentElement;
+ if (!element) return;
+ if (isVisible(element)) {
+ pendingVisible.delete(element);
+ capture(element);
+ } else if (element.isConnected) {
+ deferVisibility(element);
+ }
+ };
+
let observer;
try {
observer = new MutationObserver((mutations) => {
@@ -197,20 +264,17 @@ class TransientPageTextObserver:
});
const selectors = "[role='alert'],[role='status'],[aria-live]:not([aria-live='off'])";
const visibleSemanticElements = Array.from(document.querySelectorAll(selectors)).filter(isVisible);
- const textForElement = (element) => normalize(
- element instanceof HTMLElement && typeof element.innerText === "string"
- ? element.innerText
- : element.textContent || ""
- );
- const visibleSemanticTexts = visibleSemanticElements.map(textForElement);
+ const visibleSemanticTexts = visibleSemanticElements.map((element) => normalize(innerTextOf(element)));
if (scanInitialVisibleState) {
const staleVisibleTexts = new Set(previousState?.visibleSemanticTexts || []);
for (const element of visibleSemanticElements) {
- if (!staleVisibleTexts.has(textForElement(element))) emit(element);
+ if (!staleVisibleTexts.has(normalize(innerTextOf(element)))) capture(element);
}
}
- window[key] = { observer, bindingName, visibleSemanticTexts };
+ state = { observer, bindingName, visibleSemanticTexts, pendingVisible };
+ window[key] = state;
try { previousState?.observer?.disconnect?.(); } catch (e) {}
+ scheduleSweep();
} catch (error) {
try { observer?.disconnect?.(); } catch (e) {}
throw error;
diff --git a/tests/unit/test_transient_page_observer.py b/tests/unit/test_transient_page_observer.py
index e9f6205a5..57f0a4016 100644
--- a/tests/unit/test_transient_page_observer.py
+++ b/tests/unit/test_transient_page_observer.py
@@ -158,6 +158,159 @@ async def test_transient_text_observer_can_observe_mutations_without_scanning_vi
assert [event["text"] for event in observer.events] == ["A new generated document error"]
+async def test_transient_text_observer_captures_overlay_that_animates_in_after_insertion(
+ chromium_browser: Browser,
+) -> None:
+ # Production-shaped miss: an error toast is inserted invisible and animates in (opacity 0 -> 1) with
+ # no further observed mutation, so a point-in-time visibility check at the insertion mutation drops
+ # it and it never becomes step evidence.
+ page = await chromium_browser.new_page()
+ await page.set_content(
+ """
+ <style>
+ @keyframes skyReveal { from { opacity: 0; } to { opacity: 1; } }
+ .sky-toast {
+ position: fixed; top: 10px; right: 10px; width: 240px; height: 64px;
+ animation: skyReveal 200ms linear 80ms both;
+ }
+ </style>
+ <div id="host"></div>
+ """
+ )
+ observer = TransientPageTextObserver(page)
+
+ await observer.start(scan_initial_visible_state=True)
+ await page.evaluate(
+ """
+ () => document.getElementById("host").insertAdjacentHTML(
+ "beforeend",
+ '<div class="sky-toast" role="alert">Fehlermeldung Beim Herunterladen der Datei ist ein Fehler aufgetreten. Bitte versuchen Sie es spater noch einmal.</div>'
+ )
+ """
+ )
+ await page.wait_for_timeout(700)
+ await observer.stop()
+
+ assert any(
+ "Beim Herunterladen der Datei ist ein Fehler aufgetreten" in event["text"] for event in observer.events
+ ), observer.events
+
+
+async def test_transient_text_observer_ignores_element_that_never_becomes_visible(
+ chromium_browser: Browser,
+) -> None:
+ # Negative coverage: a stale/hidden overlay that never becomes visible must not be captured, so a
+ # successful download (no error surfaced) or an unrelated invisible node cannot cause a false match.
+ page = await chromium_browser.new_page()
+ await page.set_content('<div id="host"></div>')
+ observer = TransientPageTextObserver(page)
+
+ await observer.start(scan_initial_visible_state=True)
+ await page.evaluate(
+ """
+ () => document.getElementById("host").insertAdjacentHTML(
+ "beforeend",
+ '<div style="position:fixed;top:10px;right:10px;width:240px;height:64px;opacity:0" role="alert">Hidden overlay text that never becomes visible</div>'
+ )
+ """
+ )
+ await page.wait_for_timeout(700)
+ await observer.stop()
+
+ assert observer.events == []
+
+
+async def test_transient_text_observer_captures_alert_inserted_near_an_earlier_node_deadline(
+ chromium_browser: Browser,
+) -> None:
+ # A first invisible node is admitted early; a real alert then appears ~850ms later (near the first
+ # node's recheck budget) and animates in. Each admitted node must carry its own recheck deadline, so
+ # the late alert still gets its full window instead of being dropped by a schedule shared with the
+ # earlier node.
+ page = await chromium_browser.new_page()
+ await page.set_content(
+ """
+ <style>
+ @keyframes skyLateReveal { from { opacity: 0; } to { opacity: 1; } }
+ .sky-late-toast {
+ position: fixed; top: 90px; right: 10px; width: 240px; height: 64px;
+ animation: skyLateReveal 200ms linear 250ms both;
+ }
+ </style>
+ <div id="host"></div>
+ """
+ )
+ observer = TransientPageTextObserver(page)
+
+ await observer.start(scan_initial_visible_state=True)
+ await page.evaluate(
+ """
+ () => document.getElementById("host").insertAdjacentHTML(
+ "beforeend",
+ '<div style="position:fixed;top:10px;right:10px;width:240px;height:64px;opacity:0" role="status">Earlier node that stays invisible the whole time</div>'
+ )
+ """
+ )
+ await page.wait_for_timeout(850)
+ await page.evaluate(
+ """
+ () => document.getElementById("host").insertAdjacentHTML(
+ "beforeend",
+ '<div class="sky-late-toast" role="alert">Late arriving download error alert text</div>'
+ )
+ """
+ )
+ await page.wait_for_timeout(800)
+ await observer.stop()
+
+ texts = [event["text"] for event in observer.events]
+ assert any("Late arriving download error alert text" in text for text in texts), texts
+ assert all("Earlier node that stays invisible" not in text for text in texts), texts
+
+
+async def test_transient_text_observer_captures_overlay_straddling_post_action_reinstall(
+ chromium_browser: Browser,
+) -> None:
+ # Mirrors the download handler lifecycle: the observer is installed before the action
+ # (scan_initial_visible_state=False), then reinstalled after the action
+ # (scan_initial_visible_state=True) before the download wait. An error overlay inserted
+ # invisible during the action, still mid animation-delay when the reinstall lands, must
+ # survive the reinstall and be captured once it animates into visibility.
+ page = await chromium_browser.new_page()
+ await page.set_content(
+ """
+ <style>
+ @keyframes skyStraddleReveal { from { opacity: 0; } to { opacity: 1; } }
+ .sky-straddle-toast {
+ position: fixed; top: 10px; right: 10px; width: 240px; height: 64px;
+ animation: skyStraddleReveal 200ms linear 250ms both;
+ }
+ </style>
+ <div id="host"></div>
+ """
+ )
+ observer = TransientPageTextObserver(page)
+
+ await observer.start(scan_initial_visible_state=False)
+ await page.evaluate(
+ """
+ () => document.getElementById("host").insertAdjacentHTML(
+ "beforeend",
+ '<div class="sky-straddle-toast" role="alert">Straddle download error alert text</div>'
+ )
+ """
+ )
+ await observer.start(scan_initial_visible_state=True)
+ await page.wait_for_timeout(700)
+ await observer.stop()
+
+ assert any("Straddle download error alert text" in event["text"] for event in observer.events), observer.events
+
+
@pytest.mark.asyncio
async def test_transient_text_observer_failed_reinstall_preserves_routing_and_cleanup_ownership() -> None:
page = _FakePage()