Skip to content

Automate browser based workflows with AI

Prevent incremental DOM observer memory growth (#8594)

LawyZhengad220bca6239
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index cf3e6e0be..1cfc5bd05 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -3498,6 +3498,10 @@ if (window.globalOneTimeIncrementElements === undefined) {
window.globalOneTimeIncrementElements = [];
}
+if (window.globalIncrementalJobCount === undefined) {
+ window.globalIncrementalJobCount = 0;
+}
+
if (window.globalDomDepthMap === undefined) {
window.globalDomDepthMap = new Map();
}
@@ -3506,6 +3510,12 @@ if (window.globalParsedElementCounter === undefined) {
window.globalParsedElementCounter = new SafeCounter();
}
+// Bumped whenever the incremental observer's callback/drain/count contract changes. A persistent
+// page can outlive a rolling deploy, so a running observer created by another bundle carries no
+// stamp or a different one; the current bundle compares this value exactly and stays compatible
+// with the older contract instead of retiring the observer mid-callback.
+const INCREMENTAL_OBSERVER_VERSION = 1;
+
function isClassNameIncludesHidden(className) {
// some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`
return (
@@ -3581,8 +3591,24 @@ async function addIncrementalNodeToMap(parentNode, childrenNode) {
await window.globalParsedElementCounter.add();
}
-if (window.globalObserverForDOMIncrement === undefined) {
- window.globalObserverForDOMIncrement = new MutationObserver(async function (
+async function processIncrementalNode(parentNode, childrenNode) {
+ // A listener restart can replace the global array while this job is parsing.
+ const pendingEntries = window.globalOneTimeIncrementElements;
+ const entry = { targetNode: parentNode, newNodes: childrenNode };
+ pendingEntries.push(entry);
+ window.globalIncrementalJobCount += 1;
+ try {
+ await addIncrementalNodeToMap(parentNode, childrenNode);
+ } finally {
+ const index = pendingEntries.indexOf(entry);
+ if (index !== -1) {
+ pendingEntries.splice(index, 1);
+ }
+ }
+}
+
+function createIncrementalObserver() {
+ const observer = new MutationObserver(async function (
mutationsList,
observer,
) {
@@ -3601,11 +3627,7 @@ if (window.globalObserverForDOMIncrement === undefined) {
isDropdownRelatedElement(node) &&
getElementComputedStyle(node)?.display !== "none"
) {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
+ await processIncrementalNode(node, [node]);
continue;
}
@@ -3616,11 +3638,7 @@ if (window.globalObserverForDOMIncrement === undefined) {
switch (mutation.attributeName) {
case "hidden": {
if (!node.hidden) {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
+ await processIncrementalNode(node, [node]);
}
break;
}
@@ -3628,11 +3646,7 @@ if (window.globalObserverForDOMIncrement === undefined) {
// TODO: need to confirm that elemnent is hidden previously
if (tagName === "body") continue;
if (getElementComputedStyle(node)?.display !== "none") {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
+ await processIncrementalNode(node, [node]);
}
break;
}
@@ -3654,11 +3668,7 @@ if (window.globalObserverForDOMIncrement === undefined) {
)
continue;
if (getElementComputedStyle(node)?.display !== "none") {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
+ await processIncrementalNode(node, [node]);
}
break;
}
@@ -3666,9 +3676,6 @@ if (window.globalObserverForDOMIncrement === undefined) {
break;
}
case "childList": {
- let changedNode = {
- targetNode: node, // TODO: for future usage, when we want to parse new elements into a tree
- };
let newNodes = [];
if (mutation.addedNodes && mutation.addedNodes.length > 0) {
for (const node of mutation.addedNodes) {
@@ -3688,24 +3695,63 @@ if (window.globalObserverForDOMIncrement === undefined) {
}
if (newNodes.length > 0) {
- changedNode.newNodes = newNodes;
- window.globalOneTimeIncrementElements.push(changedNode);
- await addIncrementalNodeToMap(
- changedNode.targetNode,
- changedNode.newNodes,
- );
+ await processIncrementalNode(node, newNodes);
}
break;
}
}
}
});
+ observer.skyvernObserverVersion = INCREMENTAL_OBSERVER_VERSION;
+ return observer;
+}
+
+function isCurrentIncrementalObserver(observer) {
+ return Boolean(
+ observer &&
+ observer.skyvernObserverVersion === INCREMENTAL_OBSERVER_VERSION,
+ );
+}
+
+function hasSplicingObserverContract(observer) {
+ // The current observer splices each finished job out of the pending array and bumps the scalar
+ // count. A transitional prior build that is unstamped (or carries a mismatched stamp) but already
+ // bumped the scalar proves it splices the same way, so both drain once the array empties. An
+ // observer whose scalar is still 0 is indistinguishable from a pre-splice build that only ever
+ // pushed a monotonic history, so it is treated as pre-splice until a job bumps the scalar.
+ return (
+ isCurrentIncrementalObserver(observer) ||
+ window.globalIncrementalJobCount > 0
+ );
+}
+
+if (window.globalObserverForDOMIncrement === undefined) {
+ window.globalObserverForDOMIncrement = createIncrementalObserver();
+}
+
+async function waitForIncrementalDrain() {
+ // Reevaluate the contract each poll: a transitional splicing observer that starts with a 0 scalar
+ // is drained by the pre-splice predicate until its first completed job bumps the scalar, then
+ // switches to pending-length draining so an in-flight parse is never abandoned. A pre-splice build
+ // only ever pushed to a monotonic history, so it drains when the parsed counter catches its length.
+ while (true) {
+ const elements = window.globalOneTimeIncrementElements;
+ if (!elements) return;
+ if (hasSplicingObserverContract(window.globalObserverForDOMIncrement)) {
+ if (elements.length === 0) return;
+ } else {
+ const counter = window.globalParsedElementCounter;
+ if (!counter || (await counter.get()) >= elements.length) return;
+ }
+ await asyncSleepFor(100);
+ }
}
async function startGlobalIncrementalObserver(element = null) {
window.globalListnerFlag = true;
window.globalDomDepthMap = new Map();
window.globalOneTimeIncrementElements = [];
+ window.globalIncrementalJobCount = 0;
await getHoverStylesMap();
window.globalParsedElementCounter = new SafeCounter();
window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data
@@ -3731,30 +3777,27 @@ async function startGlobalIncrementalObserver(element = null) {
async function stopGlobalIncrementalObserver() {
window.globalListnerFlag = false;
+ // Any observer that is not the exact current version (unstamped pre-splice, unstamped splicing, or
+ // a mismatched stamp) is drained under its own contract and then replaced below.
+ const needsReplacement = !isCurrentIncrementalObserver(
+ window.globalObserverForDOMIncrement,
+ );
window.globalObserverForDOMIncrement.disconnect();
window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data
- while (
- window.globalParsedElementCounter &&
- window.globalOneTimeIncrementElements &&
- (await window.globalParsedElementCounter.get()) <
- window.globalOneTimeIncrementElements.length
- ) {
- await asyncSleepFor(100);
- }
+ await waitForIncrementalDrain();
window.globalOneTimeIncrementElements = [];
+ window.globalIncrementalJobCount = 0;
window.globalDomDepthMap = new Map();
+ // A never-navigating page adopts the memory-safe callback here: replace a drained older observer
+ // with a current one so the next action's parsing splices and counts under the current contract.
+ if (needsReplacement) {
+ window.globalObserverForDOMIncrement = createIncrementalObserver();
+ }
}
async function getIncrementElements(wait_until_finished = true) {
if (wait_until_finished) {
- while (
- window.globalParsedElementCounter &&
- window.globalOneTimeIncrementElements &&
- (await window.globalParsedElementCounter.get()) <
- window.globalOneTimeIncrementElements.length
- ) {
- await asyncSleepFor(100);
- }
+ await waitForIncrementalDrain();
}
otpContainerCounts = new WeakMap();
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 616b79e51..0eedd1005 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -1023,12 +1023,28 @@ class IncrementalScrapePage(ElementTreeBuilder):
)
async def get_incremental_elements_num(self) -> int:
- # check if the DOM has navigated away or refreshed
- js_script = "() => window.globalOneTimeIncrementElements === undefined"
- if await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script):
- return 0
-
- js_script = "() => window.globalOneTimeIncrementElements.length"
+ # A persistent browser page may still run an observer injected by another rolling-deploy build.
+ # The current observer (exact version match) and a transitional unstamped/mismatched build that
+ # already bumped the scalar both keep a correct cumulative count in globalIncrementalJobCount.
+ # A pre-splice build never bumped it, leaving a reinjection-zeroed scalar, so for that case fall
+ # back to the length of the monotonic history array the pre-splice callback only ever pushed to.
+ js_script = """() => {
+ const observer = window.globalObserverForDOMIncrement;
+ const currentVersion = window.INCREMENTAL_OBSERVER_VERSION;
+ const isCurrent = Boolean(
+ observer &&
+ currentVersion !== undefined &&
+ observer.skyvernObserverVersion === currentVersion
+ );
+ const jobCount = window.globalIncrementalJobCount;
+ if ((isCurrent || jobCount > 0) && jobCount !== undefined) {
+ return jobCount;
+ }
+ if (window.globalOneTimeIncrementElements !== undefined) {
+ return window.globalOneTimeIncrementElements.length;
+ }
+ return 0;
+ }"""
return await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)
async def __validate_element_by_value(self, value: str, element: dict) -> tuple[Locator | None, bool]:
diff --git a/tests/unit/test_increment_elements_global_guard.js b/tests/unit/test_increment_elements_global_guard.js
index 7c53357c7..745294ca4 100644
--- a/tests/unit/test_increment_elements_global_guard.js
+++ b/tests/unit/test_increment_elements_global_guard.js
@@ -1,8 +1,8 @@
/**
* Regression test for the getIncrementElements crash path in domUtils.js: a
* mid-click context reset can wipe window.globalParsedElementCounter, and the
- * wait loop dereferenced `.get()` on undefined and threw a TypeError. The guard
- * must skip the wait when the counter is gone and return an empty result.
+ * wait loop dereferenced `.get()` on undefined and threw a TypeError. Waiting
+ * now tracks pending entries independently of the parsed counter.
* Exit 0 = pass, exit 1 = failures on stderr.
*/
@@ -34,6 +34,13 @@ function extractFn(name) {
return src.substring(fnStart, fnEnd);
}
+const OBSERVER_VERSION_MATCH = src.match(
+ /const INCREMENTAL_OBSERVER_VERSION = (\d+);/,
+);
+if (!OBSERVER_VERSION_MATCH)
+ throw new Error("INCREMENTAL_OBSERVER_VERSION not found");
+const OBSERVER_VERSION = Number(OBSERVER_VERSION_MATCH[1]);
+
const makeGetIncrementElements = new Function(
"window",
"asyncSleepFor",
@@ -42,9 +49,16 @@ const makeGetIncrementElements = new Function(
"buildElementObject",
"stampOtpInputBoxes",
"otpContainerCounts",
- `${extractFn("getIncrementElements")}\nreturn getIncrementElements;`,
+ // getIncrementElements now drains through the observer-aware helpers, so bind them (and the version
+ // const the exact-match check closes over) too.
+ `const INCREMENTAL_OBSERVER_VERSION = ${OBSERVER_VERSION};\n` +
+ `${extractFn("isCurrentIncrementalObserver")}\n${extractFn("hasSplicingObserverContract")}\n` +
+ `${extractFn("waitForIncrementalDrain")}\n` +
+ `${extractFn("getIncrementElements")}\nreturn getIncrementElements;`,
);
+const CURRENT_OBSERVER = { skyvernObserverVersion: OBSERVER_VERSION };
+
const immediateSleep = () => Promise.resolve();
const throwingDocument = {
querySelector() {
@@ -120,20 +134,97 @@ function assertEmptyResult(result) {
assertEmptyResult(result);
});
- // --- guard must not break the normal wait path when the counter exists ---
- await test("getIncrementElements: defined counter still waits until parsed catches up", async () => {
- let getCalls = 0;
+ await test("getIncrementElements: undefined pending entries with waiting enabled is safe", async () => {
+ const win = {
+ globalOneTimeIncrementElements: undefined,
+ globalDomDepthMap: new Map(),
+ };
+ assertEmptyResult(await bind(win)(true));
+ });
+
+ await test("getIncrementElements: current observer waits for pending work without a parsed counter", async () => {
+ let waited = false;
const win = {
- globalParsedElementCounter: {
- get: async () => (getCalls++ === 0 ? 0 : 1),
- },
+ globalObserverForDOMIncrement: CURRENT_OBSERVER,
+ globalParsedElementCounter: undefined,
globalOneTimeIncrementElements: [{}],
globalDomDepthMap: new Map(),
};
- const result = await bind(win)(true);
+ const result = await bind(win, async () => {
+ waited = true;
+ win.globalOneTimeIncrementElements.pop();
+ })(true);
+ assert(waited, "read must wait for pending work to finish");
+ assertEmptyResult(result);
+ });
+
+ await test("getIncrementElements: legacy observer drains by parsed counter, not array length", async () => {
+ let sleeps = 0;
+ const win = {
+ globalObserverForDOMIncrement: {}, // unmarked -> legacy contract
+ globalParsedElementCounter: { get: async () => win._parsed },
+ _parsed: 0,
+ globalOneTimeIncrementElements: [{}, {}], // monotonic history the legacy callback never splices
+ globalDomDepthMap: new Map(),
+ };
+ const result = await bind(win, async () => {
+ sleeps += 1;
+ win._parsed = win.globalOneTimeIncrementElements.length;
+ })(true);
+ assert(
+ sleeps === 1,
+ "legacy drain must wait until the parsed counter catches the history length",
+ );
+ assertEmptyResult(result);
+ });
+
+ await test("getIncrementElements: unstamped splicing observer drains by pending length, not the parsed counter", async () => {
+ let sleeps = 0;
+ const win = {
+ globalObserverForDOMIncrement: {}, // unstamped, but a transitional splicing build
+ globalIncrementalJobCount: 3, // scalar already bumped -> splicing contract proven
+ globalParsedElementCounter: { get: async () => 9 }, // counter raced past the pending length
+ globalOneTimeIncrementElements: [{}], // one job still in flight, not yet spliced
+ globalDomDepthMap: new Map(),
+ };
+ const result = await bind(win, async () => {
+ sleeps += 1;
+ win.globalOneTimeIncrementElements.pop();
+ })(true);
+ // The pre-splice predicate (parsed 9 >= length 1) would exit immediately and abandon the pending
+ // entry; the splicing contract must wait for the array to drain instead.
+ assert(
+ sleeps === 1,
+ "splicing observer must wait for the pending entry to splice out, not exit on the parsed counter",
+ );
+ assertEmptyResult(result);
+ });
+
+ await test("getIncrementElements: drain reevaluates the contract once the scalar proves splicing", async () => {
+ let sleeps = 0;
+ const win = {
+ globalObserverForDOMIncrement: {}, // unstamped
+ globalIncrementalJobCount: 0, // idle at entry -> pre-splice contract
+ globalParsedElementCounter: { get: async () => win._parsed },
+ _parsed: 0,
+ globalOneTimeIncrementElements: [{}], // a pending entry from a job about to prove itself
+ globalDomDepthMap: new Map(),
+ };
+ const result = await bind(win, async () => {
+ sleeps += 1;
+ if (sleeps === 1) {
+ // first job completes: bumps the scalar and races the parsed counter past the pending length
+ win.globalIncrementalJobCount = 1;
+ win._parsed = 5;
+ } else {
+ win.globalOneTimeIncrementElements.pop();
+ }
+ })(true);
+ // A contract captured once at entry would stay pre-splice and exit on the raced counter, leaking
+ // the pending entry; reevaluating each poll switches to pending-length draining after the bump.
assert(
- getCalls >= 2,
- "wait loop ran at least one iteration before exiting",
+ sleeps === 2,
+ "drain must switch to pending-length semantics after the scalar proves splicing",
);
assertEmptyResult(result);
});
diff --git a/tests/unit/test_increment_elements_global_guard.py b/tests/unit/test_increment_elements_global_guard.py
index 6d3b4d721..88482fdb6 100644
--- a/tests/unit/test_increment_elements_global_guard.py
+++ b/tests/unit/test_increment_elements_global_guard.py
@@ -1,19 +1,21 @@
-"""
-Regression tests for the getIncrementElements crash guard in domUtils.js.
+"""Incremental DOM observer source-guard and rolling-deploy getter compatibility tests.
-A mid-click context reset (SPA re-render, iframe document swap, soft nav) can
-wipe window.globalParsedElementCounter while the top-frame URL guard still
-passes, so getIncrementElements used to dereference `.get()` on undefined and
-throw a TypeError. The behavioral Node test proves the wait loop now skips when
-the counter is gone, and the source check pins the top-level guard initializer.
+Node/source/getter checks that run without a browser. The real-Chromium observer lifecycle tests
+live in tests/browser_e2e/test_taskv3_incremental_observer_memory_e2e.py.
"""
+import json
import re
import shutil
import subprocess
from pathlib import Path
+from unittest.mock import MagicMock
import pytest
+from playwright.async_api import Page
+
+from skyvern.webeye.scraper.scraper import IncrementalScrapePage
+from skyvern.webeye.utils.page import SkyvernFrame
_REPO_ROOT = Path(__file__).parent.parent.parent
_DOMUTILS = _REPO_ROOT / "skyvern" / "webeye" / "scraper" / "domUtils.js"
@@ -47,3 +49,115 @@ class TestIncrementElementsGlobalGuard:
r"if\s*\(\s*window\.globalParsedElementCounter\s*===\s*undefined\s*\)",
source,
), "globalParsedElementCounter must be guard-initialized at top level like its sibling globals"
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ ("window_state", "expected_count"),
+ [
+ (
+ {
+ "globalObserverForDOMIncrement": {"skyvernObserverVersion": 1},
+ "INCREMENTAL_OBSERVER_VERSION": 1,
+ "globalIncrementalJobCount": 7,
+ },
+ 7,
+ ),
+ (
+ {
+ "globalObserverForDOMIncrement": {"skyvernObserverVersion": 1},
+ "INCREMENTAL_OBSERVER_VERSION": 1,
+ "globalIncrementalJobCount": 7,
+ "globalOneTimeIncrementElements": [{}, {}],
+ },
+ 7,
+ ),
+ (
+ {
+ "globalObserverForDOMIncrement": {"skyvernObserverVersion": 1},
+ "INCREMENTAL_OBSERVER_VERSION": 1,
+ "globalIncrementalJobCount": 0,
+ "globalOneTimeIncrementElements": [{}, {}],
+ },
+ 0,
+ ),
+ (
+ {
+ "globalObserverForDOMIncrement": {},
+ "INCREMENTAL_OBSERVER_VERSION": 1,
+ "globalIncrementalJobCount": 5,
+ "globalOneTimeIncrementElements": [{}],
+ },
+ 5,
+ ),
+ (
+ {
+ "globalObserverForDOMIncrement": {"skyvernObserverVersion": 999},
+ "INCREMENTAL_OBSERVER_VERSION": 1,
+ "globalIncrementalJobCount": 5,
+ "globalOneTimeIncrementElements": [{}],
+ },
+ 5,
+ ),
+ (
+ {
+ "globalObserverForDOMIncrement": {"skyvernObserverVersion": 999},
+ "INCREMENTAL_OBSERVER_VERSION": 1,
+ "globalIncrementalJobCount": 0,
+ "globalOneTimeIncrementElements": [{}, {}],
+ },
+ 2,
+ ),
+ (
+ {
+ "globalObserverForDOMIncrement": {},
+ "globalIncrementalJobCount": 0,
+ "globalOneTimeIncrementElements": [{}, {}],
+ },
+ 2,
+ ),
+ ({"globalObserverForDOMIncrement": {}, "globalOneTimeIncrementElements": [{}, {}]}, 2),
+ ({"globalObserverForDOMIncrement": {}, "globalOneTimeIncrementElements": []}, 0),
+ ({}, 0),
+ ],
+ ids=[
+ "current-scalar",
+ "current-scalar-wins",
+ "current-scalar-zero",
+ "unstamped-splicing-scalar-wins",
+ "version-mismatch-splicing-scalar-wins",
+ "version-mismatch-scalar-zero-array-wins",
+ "legacy-scalar-zero-array-wins",
+ "legacy-array",
+ "legacy-empty",
+ "navigation",
+ ],
+ )
+ async def test_incremental_elements_num_rolling_deploy_compatibility(
+ self, window_state: dict[str, object], expected_count: int
+ ) -> None:
+ def evaluate(expression: str, arg: object = None) -> int | bool:
+ # Execute the getter's JavaScript; a canned evaluate result would miss compatibility regressions.
+ result = subprocess.run(
+ [
+ _NODE,
+ "-e",
+ (
+ "const vm = require('node:vm');"
+ "const window = JSON.parse(process.argv[1]);"
+ "const read = vm.runInNewContext(process.argv[2], {window});"
+ "process.stdout.write(JSON.stringify(read()));"
+ ),
+ json.dumps(window_state),
+ expression,
+ ],
+ capture_output=True,
+ text=True,
+ timeout=10,
+ check=True,
+ )
+ return json.loads(result.stdout)
+
+ page = MagicMock(spec=Page, context=None)
+ page.evaluate.side_effect = evaluate
+ scraped = IncrementalScrapePage(SkyvernFrame(page))
+ assert await scraped.get_incremental_elements_num() == expected_count