Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

perf: fix O(n²) in build_snapshot_lookup — 39x speedup at 100k elements

MagMueller6a927805868d
diff --git a/browser_use/dom/enhanced_snapshot.py b/browser_use/dom/enhanced_snapshot.py
index f9433145c..4d9c3b438 100644
--- a/browser_use/dom/enhanced_snapshot.py
+++ b/browser_use/dom/enhanced_snapshot.py
@@ -30,9 +30,9 @@ REQUIRED_COMPUTED_STYLES = [
]
-def _parse_rare_boolean_data(rare_data: RareBooleanData, index: int) -> bool | None:
- """Parse rare boolean data from snapshot - returns True if index is in the rare data."""
- return index in rare_data['index']
+def _parse_rare_boolean_data(rare_data_set: set[int], index: int) -> bool | None:
+ """Parse rare boolean data from snapshot - returns True if index is in the rare data set."""
+ return index in rare_data_set
def _parse_computed_styles(strings: list[str], style_indices: list[int]) -> dict[str, str]:
@@ -85,11 +85,17 @@ def build_snapshot_lookup(
if node_index not in layout_index_map: # Only store first occurrence
layout_index_map[node_index] = layout_idx
+ # Pre-convert rare boolean data from list to set for O(1) lookups.
+ # The raw CDP data uses List[int] which makes `index in list` O(n).
+ # Called once per node, this was O(n²) total — the #1 bottleneck.
+ # At 20k elements: 5,925ms (list) → 2ms (set) = 3,000x speedup.
+ is_clickable_set: set[int] = set(nodes['isClickable']['index']) if 'isClickable' in nodes else set()
+
# Build snapshot lookup for each backend node id
for backend_node_id, snapshot_index in backend_node_to_snapshot_index.items():
is_clickable = None
- if 'isClickable' in nodes:
- is_clickable = _parse_rare_boolean_data(nodes['isClickable'], snapshot_index)
+ if is_clickable_set:
+ is_clickable = _parse_rare_boolean_data(is_clickable_set, snapshot_index)
# Find corresponding layout node
cursor_style = None
diff --git a/tests/heavy_pages/bench_pipeline_breakdown.py b/tests/heavy_pages/bench_pipeline_breakdown.py
new file mode 100644
index 000000000..aa0e96241
--- /dev/null
+++ b/tests/heavy_pages/bench_pipeline_breakdown.py
@@ -0,0 +1,248 @@
+"""
+Pipeline breakdown: time EVERY stage separately.
+
+Q1: Where do the 8 seconds go at 5k elements?
+Q2: Would AX tree alone be enough (and faster)?
+Q3: Can we use partial AX tree (viewport-scoped)?
+Q4: What if we skip paint order? Skip AX tree? Skip snapshot?
+Q5: Can we get interactive elements via a fast JS query instead?
+Q6: How fast is each raw CDP call vs the Python processing on top?
+"""
+
+import asyncio, json, logging, os, sys, time
+from pathlib import Path
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+from threading import Thread
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+from dotenv import load_dotenv
+load_dotenv(Path('/Users/magnus/Developer/cloud/backend/.env'))
+os.environ['TIMEOUT_BrowserStateRequestEvent'] = '120'
+
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+from browser_use.dom.service import DomService
+from browser_use.dom.enhanced_snapshot import build_snapshot_lookup, REQUIRED_COMPUTED_STYLES
+from browser_use.dom.serializer.serializer import DOMTreeSerializer
+
+logging.basicConfig(level=logging.WARNING)
+
+def gen(n):
+ # Mix of interactive elements: buttons, inputs, links, selects, divs with handlers
+ return f'''<html><head><title>Pipeline Bench {n}</title></head><body>
+<h1>Pipeline Bench</h1>
+<nav>{"".join(f'<a href="#">Link {i}</a> ' for i in range(min(n//10, 50)))}</nav>
+<form>{"".join(f'<div><label>F{i}</label><input type="text" value="v{i}" /><select><option>A</option><option>B</option></select></div>' for i in range(min(n//5, 200)))}</form>
+<div id="bulk"></div>
+<script>
+const f=document.createDocumentFragment();
+for(let i=0;i<{n};i++){{
+ const d=document.createElement('div');
+ d.innerHTML='<span>'+i+'</span><input value=v'+i+' /><button onclick="this.textContent=\\'ok\\'">btn'+i+'</button>';
+ if(i%3===0) d.addEventListener('click',()=>{{}});
+ f.appendChild(d);
+}}
+document.getElementById('bulk').appendChild(f);
+</script></body></html>'''
+
+class Q(SimpleHTTPRequestHandler):
+ def log_message(self,*a):pass
+
+async def t(label, coro, timeout=60):
+ t0=time.time()
+ try:
+ r = await asyncio.wait_for(coro, timeout=timeout)
+ ms=(time.time()-t0)*1000
+ return r, ms, None
+ except Exception as e:
+ return None, (time.time()-t0)*1000, type(e).__name__
+
+async def bench(n, base):
+ s = BrowserSession(browser_profile=BrowserProfile(headless=True, cross_origin_iframes=False))
+ await s.start()
+ cdp = await s.get_or_create_cdp_session(focus=True)
+ sid = cdp.session_id
+ send = cdp.cdp_client.send
+
+ await send.Page.navigate(params={'url':f'{base}/pipe_{n}.html'}, session_id=sid)
+ await asyncio.sleep(2)
+
+ r = await send.Runtime.evaluate(params={'expression':'document.querySelectorAll("*").length','returnByValue':True}, session_id=sid)
+ elems = r.get('result',{}).get('value',0)
+
+ target_id = s.agent_focus_target_id
+
+ print(f'\n{"="*90}')
+ print(f' {n:,} target elements ({elems:,} DOM nodes)')
+ print(f'{"="*90}')
+
+ # ── RAW CDP CALLS (what Chrome does) ──────────────────────────────
+ print(f'\n {"RAW CDP CALLS":-<70}')
+
+ # 1. DOMSnapshot.captureSnapshot
+ async def do_snap():
+ return await send.DOMSnapshot.captureSnapshot(params={
+ 'computedStyles': REQUIRED_COMPUTED_STYLES,
+ 'includePaintOrder':True,'includeDOMRects':True,
+ 'includeBlendedBackgroundColors':False,'includeTextColorOpacities':False
+ }, session_id=sid)
+ snapshot, ms, err = await t('DOMSnapshot.captureSnapshot', do_snap())
+ snap_nodes = sum(len(d.get('nodes',{}).get('nodeName',[])) for d in (snapshot or {}).get('documents',[])) if snapshot else 0
+ print(f' DOMSnapshot.captureSnapshot {ms:>8.0f}ms {snap_nodes:,} nodes {err or ""}')
+ snap_ms = ms
+
+ # 2. DOM.getDocument
+ async def do_dom():
+ return await send.DOM.getDocument(params={'depth':-1,'pierce':True}, session_id=sid)
+ dom_tree, ms, err = await t('DOM.getDocument', do_dom())
+ print(f' DOM.getDocument(depth=-1) {ms:>8.0f}ms {err or ""}')
+ dom_ms = ms
+
+ # 3. Full AX tree
+ async def do_ax_full():
+ return await send.Accessibility.getFullAXTree(params={}, session_id=sid)
+ ax_full, ms, err = await t('Accessibility.getFullAXTree', do_ax_full())
+ ax_full_count = len(ax_full.get('nodes',[])) if ax_full else 0
+ print(f' Accessibility.getFullAXTree {ms:>8.0f}ms {ax_full_count:,} nodes {err or ""}')
+ ax_full_ms = ms
+
+ # 4. Partial AX tree (single node, to see if the API exists)
+ async def do_ax_partial():
+ # Get the root node's backendNodeId
+ root_id = dom_tree['root']['backendNodeId'] if dom_tree else 1
+ return await send.Accessibility.getPartialAXTree(params={
+ 'backendNodeId': root_id, 'fetchRelatives': False
+ }, session_id=sid)
+ ax_partial, ms, err = await t('Accessibility.getPartialAXTree', do_ax_partial())
+ ax_partial_count = len(ax_partial.get('nodes',[])) if ax_partial else 0
+ print(f' Accessibility.getPartialAXTree {ms:>8.0f}ms {ax_partial_count:,} nodes {err or ""}')
+
+ # 5. Screenshot
+ async def do_ss():
+ return await send.Page.captureScreenshot(params={'format':'png','quality':80}, session_id=sid)
+ _, ms, err = await t('Screenshot', do_ss())
+ print(f' Page.captureScreenshot {ms:>8.0f}ms {err or ""}')
+
+ # 6. JS query for interactive elements (viewport-scoped)
+ async def do_js_interactive():
+ return await send.Runtime.evaluate(params={
+ 'expression': '''
+ (() => {
+ const sel = 'a, button, input, select, textarea, [onclick], [role="button"], [role="link"], [role="tab"], [tabindex]';
+ const all = document.querySelectorAll(sel);
+ const vh = window.innerHeight;
+ const visible = [];
+ const offscreen = [];
+ for (const el of all) {
+ const r = el.getBoundingClientRect();
+ const entry = {tag: el.tagName, id: el.id || undefined, name: el.name || undefined,
+ type: el.type || undefined, value: (el.value || '').slice(0,30),
+ text: (el.textContent || '').slice(0,30).trim(),
+ x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height)};
+ if (r.bottom > 0 && r.top < vh) visible.push(entry);
+ else offscreen.push(entry);
+ }
+ return {visible: visible.length, offscreen: offscreen.length, total: all.length,
+ sample_visible: visible.slice(0, 5)};
+ })()
+ ''', 'returnByValue': True
+ }, session_id=sid)
+ js_r, ms, err = await t('JS interactive query', do_js_interactive())
+ js_data = js_r.get('result',{}).get('value',{}) if js_r else {}
+ print(f' JS interactive query (viewport) {ms:>8.0f}ms {js_data.get("visible",0)} visible, {js_data.get("total",0)} total {err or ""}')
+ js_ms = ms
+
+ # 7. JS query ALL elements (to compare with querySelectorAll('*'))
+ async def do_js_all():
+ return await send.Runtime.evaluate(params={
+ 'expression': 'document.querySelectorAll("*").length', 'returnByValue': True
+ }, session_id=sid)
+ _, ms, err = await t('JS querySelectorAll(*)', do_js_all())
+ print(f' JS querySelectorAll("*").length {ms:>8.0f}ms {err or ""}')
+
+ # ── PYTHON PROCESSING (what browser-use adds) ─────────────────────
+ if snapshot and dom_tree:
+ print(f'\n {"PYTHON PROCESSING":-<70}')
+
+ # 8. build_snapshot_lookup
+ t0=time.time()
+ device_pixel_ratio = 1.0
+ snapshot_lookup = build_snapshot_lookup(snapshot, device_pixel_ratio)
+ ms = (time.time()-t0)*1000
+ print(f' build_snapshot_lookup {ms:>8.0f}ms {len(snapshot_lookup):,} entries')
+
+ # 9. Build AX tree lookup
+ t0=time.time()
+ ax_tree_data = ax_full if ax_full else {'nodes': []}
+ ax_tree_lookup = {n['backendDOMNodeId']: n for n in ax_tree_data['nodes'] if 'backendDOMNodeId' in n}
+ ms = (time.time()-t0)*1000
+ print(f' build AX tree lookup {ms:>8.0f}ms {len(ax_tree_lookup):,} entries')
+
+ # 10. Full _construct_enhanced_node + get_dom_tree
+ dom_service = DomService(
+ browser_session=s, cross_origin_iframes=False,
+ paint_order_filtering=True, max_iframes=100, max_iframe_depth=5
+ )
+ t0=time.time()
+ try:
+ enhanced_tree, timing = await asyncio.wait_for(
+ dom_service.get_dom_tree(target_id=target_id), timeout=60
+ )
+ ms = (time.time()-t0)*1000
+ tree_ok = True
+ except Exception as e:
+ ms = (time.time()-t0)*1000
+ tree_ok = False
+ enhanced_tree = None
+ timing = {}
+ print(f' get_dom_tree (full) {ms:>8.0f}ms')
+ # Print sub-timings from the timing dict
+ for k, v in sorted(timing.items()):
+ print(f' {k:<40} {v:>8.1f}ms')
+
+ # 11. Serialization
+ if tree_ok and enhanced_tree:
+ t0=time.time()
+ serialized, ser_timing = DOMTreeSerializer(
+ enhanced_tree, None, paint_order_filtering=True, session_id=s.id
+ ).serialize_accessible_elements()
+ ms = (time.time()-t0)*1000
+ print(f' serialize_accessible_elements {ms:>8.0f}ms {len(serialized.selector_map):,} selectors')
+ for k, v in sorted(ser_timing.items()):
+ print(f' {k:<40} {v*1000:>8.1f}ms')
+
+ # 12. Serialization WITHOUT paint order
+ t0=time.time()
+ serialized2, ser_timing2 = DOMTreeSerializer(
+ enhanced_tree, None, paint_order_filtering=False, session_id=s.id
+ ).serialize_accessible_elements()
+ ms = (time.time()-t0)*1000
+ print(f' serialize (NO paint order) {ms:>8.0f}ms {len(serialized2.selector_map):,} selectors')
+ for k, v in sorted(ser_timing2.items()):
+ print(f' {k:<40} {v*1000:>8.1f}ms')
+
+ # ── COMPARISON SUMMARY ────────────────────────────────────────────
+ print(f'\n {"COMPARISON":-<70}')
+ print(f' Raw CDP snapshot+DOM+AX: {snap_ms+dom_ms+ax_full_ms:>8.0f}ms (Chrome work)')
+ print(f' JS interactive query: {js_ms:>8.0f}ms (alternative)')
+ print(f' Full pipeline overhead: Python processing on top of CDP')
+
+ await s.kill()
+
+
+async def main():
+ d = Path(__file__).parent / 'generated'; d.mkdir(exist_ok=True)
+ scales = [5000, 20000, 100000]
+ for n in scales:
+ (d/f'pipe_{n}.html').write_text(gen(n))
+ os.chdir(str(d))
+ srv = HTTPServer(('127.0.0.1', 8775), Q)
+ Thread(target=srv.serve_forever, daemon=True).start()
+
+ for n in scales:
+ await bench(n, 'http://127.0.0.1:8775')
+
+ srv.shutdown()
+
+if __name__ == '__main__':
+ asyncio.run(main())
diff --git a/tests/heavy_pages/bench_snapshot_profile.py b/tests/heavy_pages/bench_snapshot_profile.py
new file mode 100644
index 000000000..19324d45c
--- /dev/null
+++ b/tests/heavy_pages/bench_snapshot_profile.py
@@ -0,0 +1,157 @@
+"""Profile build_snapshot_lookup to find the exact O(n²) bottleneck."""
+import asyncio, os, sys, time, cProfile, pstats, io
+from pathlib import Path
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+from threading import Thread
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+from dotenv import load_dotenv
+load_dotenv(Path('/Users/magnus/Developer/cloud/backend/.env'))
+
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+from browser_use.dom.enhanced_snapshot import build_snapshot_lookup, REQUIRED_COMPUTED_STYLES, _parse_rare_boolean_data
+
+import logging
+logging.basicConfig(level=logging.WARNING)
+
+def gen(n):
+ return f'<html><body><div id=b></div><script>const f=document.createDocumentFragment();for(let i=0;i<{n};i++){{const d=document.createElement("div");d.innerHTML="<span>"+i+"</span><input value=v"+i+" /><button>X</button>";f.appendChild(d)}}document.getElementById("b").appendChild(f)</script></body></html>'
+
+class Q(SimpleHTTPRequestHandler):
+ def log_message(self,*a):pass
+
+async def main():
+ d = Path(__file__).parent / 'generated'; d.mkdir(exist_ok=True)
+ for n in [5000, 20000]:
+ (d/f'prof_{n}.html').write_text(gen(n))
+ os.chdir(str(d))
+ srv = HTTPServer(('127.0.0.1', 8774), Q)
+ Thread(target=srv.serve_forever, daemon=True).start()
+
+ for n in [5000, 20000]:
+ s = BrowserSession(browser_profile=BrowserProfile(headless=True))
+ await s.start()
+ cdp = await s.get_or_create_cdp_session(focus=True)
+ sid = cdp.session_id
+ send = cdp.cdp_client.send
+
+ await send.Page.navigate(params={'url':f'http://127.0.0.1:8774/prof_{n}.html'}, session_id=sid)
+ await asyncio.sleep(2)
+
+ # Get snapshot
+ snapshot = await send.DOMSnapshot.captureSnapshot(params={
+ 'computedStyles': REQUIRED_COMPUTED_STYLES,
+ 'includePaintOrder':True,'includeDOMRects':True,
+ 'includeBlendedBackgroundColors':False,'includeTextColorOpacities':False
+ }, session_id=sid)
+
+ total_nodes = sum(len(d.get('nodes',{}).get('nodeName',[])) for d in snapshot.get('documents',[]))
+ print(f'\n{"="*80}')
+ print(f' {n:,} elements, {total_nodes:,} snapshot nodes')
+ print(f'{"="*80}')
+
+ # Check isClickable data size
+ for doc_idx, doc in enumerate(snapshot['documents']):
+ nodes_data = doc['nodes']
+ if 'isClickable' in nodes_data:
+ clickable_list = nodes_data['isClickable']['index']
+ print(f' doc[{doc_idx}] isClickable index list length: {len(clickable_list)}')
+
+ # Manual breakdown of build_snapshot_lookup
+ strings = snapshot['strings']
+ documents = snapshot['documents']
+
+ for doc_idx, document in enumerate(documents):
+ nodes_data = document['nodes']
+ layout = document['layout']
+
+ # Time: build backend_node_to_snapshot_index
+ t0 = time.time()
+ backend_node_to_snapshot_index = {}
+ if 'backendNodeId' in nodes_data:
+ for i, bid in enumerate(nodes_data['backendNodeId']):
+ backend_node_to_snapshot_index[bid] = i
+ ms = (time.time()-t0)*1000
+ print(f' doc[{doc_idx}] build backend_node_to_snapshot_index: {ms:.1f}ms ({len(backend_node_to_snapshot_index)} entries)')
+
+ # Time: build layout_index_map
+ t0 = time.time()
+ layout_index_map = {}
+ if layout and 'nodeIndex' in layout:
+ for layout_idx, node_index in enumerate(layout['nodeIndex']):
+ if node_index not in layout_index_map:
+ layout_index_map[node_index] = layout_idx
+ ms = (time.time()-t0)*1000
+ print(f' doc[{doc_idx}] build layout_index_map: {ms:.1f}ms ({len(layout_index_map)} entries)')
+
+ # Time: isClickable parsing (the suspected O(n²))
+ if 'isClickable' in nodes_data:
+ clickable_index_list = nodes_data['isClickable']['index']
+
+ # Method 1: current (list scan per node)
+ t0 = time.time()
+ count = 0
+ for snapshot_index in range(len(nodes_data.get('backendNodeId', []))):
+ if snapshot_index in clickable_index_list: # O(len(clickable_index_list)) per call!
+ count += 1
+ ms = (time.time()-t0)*1000
+ print(f' doc[{doc_idx}] isClickable via LIST scan: {ms:.1f}ms (found {count} clickable)')
+
+ # Method 2: convert to set first
+ t0 = time.time()
+ clickable_set = set(clickable_index_list)
+ count2 = 0
+ for snapshot_index in range(len(nodes_data.get('backendNodeId', []))):
+ if snapshot_index in clickable_set: # O(1) per call
+ count2 += 1
+ ms = (time.time()-t0)*1000
+ print(f' doc[{doc_idx}] isClickable via SET scan: {ms:.1f}ms (found {count2} clickable)')
+ assert count == count2
+
+ # Time: the main loop (creating EnhancedSnapshotNode objects)
+ t0 = time.time()
+ dummy_count = 0
+ for backend_node_id, snapshot_index in backend_node_to_snapshot_index.items():
+ # Simulate the work without isClickable
+ if snapshot_index in layout_index_map:
+ layout_idx = layout_index_map[snapshot_index]
+ if layout_idx < len(layout.get('bounds', [])):
+ bounds = layout['bounds'][layout_idx]
+ _ = bounds[0] if len(bounds) >= 4 else None
+ if layout_idx < len(layout.get('styles', [])):
+ style_indices = layout['styles'][layout_idx]
+ # Parse styles
+ styles = {}
+ for i, si in enumerate(style_indices):
+ if i < len(REQUIRED_COMPUTED_STYLES) and 0 <= si < len(strings):
+ styles[REQUIRED_COMPUTED_STYLES[i]] = strings[si]
+ dummy_count += 1
+ ms = (time.time()-t0)*1000
+ print(f' doc[{doc_idx}] main loop (no isClickable): {ms:.1f}ms ({dummy_count} iterations)')
+
+ # Time: full build_snapshot_lookup
+ t0 = time.time()
+ result = build_snapshot_lookup(snapshot, 1.0)
+ ms = (time.time()-t0)*1000
+ print(f'\n FULL build_snapshot_lookup: {ms:.0f}ms ({len(result)} entries)')
+
+ # Profile it
+ pr = cProfile.Profile()
+ pr.enable()
+ result2 = build_snapshot_lookup(snapshot, 1.0)
+ pr.disable()
+
+ stream = io.StringIO()
+ ps = pstats.Stats(pr, stream=stream).sort_stats('cumulative')
+ ps.print_stats(15)
+ print(f'\n cProfile top 15:')
+ for line in stream.getvalue().split('\n')[:20]:
+ print(f' {line}')
+
+ await s.kill()
+
+ srv.shutdown()
+
+if __name__ == '__main__':
+ asyncio.run(main())