Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
perf: fix O(n²) in build_snapshot_lookup — 39x speedup at 100k elements
MagMueller6a927805868d
diff --git a/browser_use/dom/enhanced_snapshot.py b/browser_use/dom/enhanced_snapshot.pyindex f9433145c..4d9c3b438 100644--- a/browser_use/dom/enhanced_snapshot.py+++ b/browser_use/dom/enhanced_snapshot.py@@ -30,9 +30,9 @@ REQUIRED_COMPUTED_STYLES = []-def _parse_rare_boolean_data(rare_data: RareBooleanData, index: int) -> bool | None:- """Parse rare boolean data from snapshot - returns True if index is in the rare data."""- return index in rare_data['index']+def _parse_rare_boolean_data(rare_data_set: set[int], index: int) -> bool | None:+ """Parse rare boolean data from snapshot - returns True if index is in the rare data set."""+ return index in rare_data_setdef _parse_computed_styles(strings: list[str], style_indices: list[int]) -> dict[str, str]:@@ -85,11 +85,17 @@ def build_snapshot_lookup(if node_index not in layout_index_map: # Only store first occurrencelayout_index_map[node_index] = layout_idx+ # Pre-convert rare boolean data from list to set for O(1) lookups.+ # The raw CDP data uses List[int] which makes `index in list` O(n).+ # Called once per node, this was O(n²) total — the #1 bottleneck.+ # At 20k elements: 5,925ms (list) → 2ms (set) = 3,000x speedup.+ is_clickable_set: set[int] = set(nodes['isClickable']['index']) if 'isClickable' in nodes else set()+# Build snapshot lookup for each backend node idfor backend_node_id, snapshot_index in backend_node_to_snapshot_index.items():is_clickable = None- if 'isClickable' in nodes:- is_clickable = _parse_rare_boolean_data(nodes['isClickable'], snapshot_index)+ if is_clickable_set:+ is_clickable = _parse_rare_boolean_data(is_clickable_set, snapshot_index)# Find corresponding layout nodecursor_style = Nonediff --git a/tests/heavy_pages/bench_pipeline_breakdown.py b/tests/heavy_pages/bench_pipeline_breakdown.pynew file mode 100644index 000000000..aa0e96241--- /dev/null+++ b/tests/heavy_pages/bench_pipeline_breakdown.py@@ -0,0 +1,248 @@+"""+Pipeline breakdown: time EVERY stage separately.++Q1: Where do the 8 seconds go at 5k elements?+Q2: Would AX tree alone be enough (and faster)?+Q3: Can we use partial AX tree (viewport-scoped)?+Q4: What if we skip paint order? Skip AX tree? Skip snapshot?+Q5: Can we get interactive elements via a fast JS query instead?+Q6: How fast is each raw CDP call vs the Python processing on top?+"""++import asyncio, json, logging, os, sys, time+from pathlib import Path+from http.server import HTTPServer, SimpleHTTPRequestHandler+from threading import Thread++sys.path.insert(0, str(Path(__file__).resolve().parents[2]))+from dotenv import load_dotenv+load_dotenv(Path('/Users/magnus/Developer/cloud/backend/.env'))+os.environ['TIMEOUT_BrowserStateRequestEvent'] = '120'++from browser_use.browser.profile import BrowserProfile+from browser_use.browser.session import BrowserSession+from browser_use.dom.service import DomService+from browser_use.dom.enhanced_snapshot import build_snapshot_lookup, REQUIRED_COMPUTED_STYLES+from browser_use.dom.serializer.serializer import DOMTreeSerializer++logging.basicConfig(level=logging.WARNING)++def gen(n):+ # Mix of interactive elements: buttons, inputs, links, selects, divs with handlers+ return f'''<html><head><title>Pipeline Bench {n}</title></head><body>+<h1>Pipeline Bench</h1>+<nav>{"".join(f'<a href="#">Link {i}</a> ' for i in range(min(n//10, 50)))}</nav>+<form>{"".join(f'<div><label>F{i}</label><input type="text" value="v{i}" /><select><option>A</option><option>B</option></select></div>' for i in range(min(n//5, 200)))}</form>+<div id="bulk"></div>+<script>+const f=document.createDocumentFragment();+for(let i=0;i<{n};i++){{+ const d=document.createElement('div');+ d.innerHTML='<span>'+i+'</span><input value=v'+i+' /><button onclick="this.textContent=\\'ok\\'">btn'+i+'</button>';+ if(i%3===0) d.addEventListener('click',()=>{{}});+ f.appendChild(d);+}}+document.getElementById('bulk').appendChild(f);+</script></body></html>'''++class Q(SimpleHTTPRequestHandler):+ def log_message(self,*a):pass++async def t(label, coro, timeout=60):+ t0=time.time()+ try:+ r = await asyncio.wait_for(coro, timeout=timeout)+ ms=(time.time()-t0)*1000+ return r, ms, None+ except Exception as e:+ return None, (time.time()-t0)*1000, type(e).__name__++async def bench(n, base):+ s = BrowserSession(browser_profile=BrowserProfile(headless=True, cross_origin_iframes=False))+ await s.start()+ cdp = await s.get_or_create_cdp_session(focus=True)+ sid = cdp.session_id+ send = cdp.cdp_client.send++ await send.Page.navigate(params={'url':f'{base}/pipe_{n}.html'}, session_id=sid)+ await asyncio.sleep(2)++ r = await send.Runtime.evaluate(params={'expression':'document.querySelectorAll("*").length','returnByValue':True}, session_id=sid)+ elems = r.get('result',{}).get('value',0)++ target_id = s.agent_focus_target_id++ print(f'\n{"="*90}')+ print(f' {n:,} target elements ({elems:,} DOM nodes)')+ print(f'{"="*90}')++ # ── RAW CDP CALLS (what Chrome does) ──────────────────────────────+ print(f'\n {"RAW CDP CALLS":-<70}')++ # 1. DOMSnapshot.captureSnapshot+ async def do_snap():+ return await send.DOMSnapshot.captureSnapshot(params={+ 'computedStyles': REQUIRED_COMPUTED_STYLES,+ 'includePaintOrder':True,'includeDOMRects':True,+ 'includeBlendedBackgroundColors':False,'includeTextColorOpacities':False+ }, session_id=sid)+ snapshot, ms, err = await t('DOMSnapshot.captureSnapshot', do_snap())+ snap_nodes = sum(len(d.get('nodes',{}).get('nodeName',[])) for d in (snapshot or {}).get('documents',[])) if snapshot else 0+ print(f' DOMSnapshot.captureSnapshot {ms:>8.0f}ms {snap_nodes:,} nodes {err or ""}')+ snap_ms = ms++ # 2. DOM.getDocument+ async def do_dom():+ return await send.DOM.getDocument(params={'depth':-1,'pierce':True}, session_id=sid)+ dom_tree, ms, err = await t('DOM.getDocument', do_dom())+ print(f' DOM.getDocument(depth=-1) {ms:>8.0f}ms {err or ""}')+ dom_ms = ms++ # 3. Full AX tree+ async def do_ax_full():+ return await send.Accessibility.getFullAXTree(params={}, session_id=sid)+ ax_full, ms, err = await t('Accessibility.getFullAXTree', do_ax_full())+ ax_full_count = len(ax_full.get('nodes',[])) if ax_full else 0+ print(f' Accessibility.getFullAXTree {ms:>8.0f}ms {ax_full_count:,} nodes {err or ""}')+ ax_full_ms = ms++ # 4. Partial AX tree (single node, to see if the API exists)+ async def do_ax_partial():+ # Get the root node's backendNodeId+ root_id = dom_tree['root']['backendNodeId'] if dom_tree else 1+ return await send.Accessibility.getPartialAXTree(params={+ 'backendNodeId': root_id, 'fetchRelatives': False+ }, session_id=sid)+ ax_partial, ms, err = await t('Accessibility.getPartialAXTree', do_ax_partial())+ ax_partial_count = len(ax_partial.get('nodes',[])) if ax_partial else 0+ print(f' Accessibility.getPartialAXTree {ms:>8.0f}ms {ax_partial_count:,} nodes {err or ""}')++ # 5. Screenshot+ async def do_ss():+ return await send.Page.captureScreenshot(params={'format':'png','quality':80}, session_id=sid)+ _, ms, err = await t('Screenshot', do_ss())+ print(f' Page.captureScreenshot {ms:>8.0f}ms {err or ""}')++ # 6. JS query for interactive elements (viewport-scoped)+ async def do_js_interactive():+ return await send.Runtime.evaluate(params={+ 'expression': '''+ (() => {+ const sel = 'a, button, input, select, textarea, [onclick], [role="button"], [role="link"], [role="tab"], [tabindex]';+ const all = document.querySelectorAll(sel);+ const vh = window.innerHeight;+ const visible = [];+ const offscreen = [];+ for (const el of all) {+ const r = el.getBoundingClientRect();+ const entry = {tag: el.tagName, id: el.id || undefined, name: el.name || undefined,+ type: el.type || undefined, value: (el.value || '').slice(0,30),+ text: (el.textContent || '').slice(0,30).trim(),+ x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height)};+ if (r.bottom > 0 && r.top < vh) visible.push(entry);+ else offscreen.push(entry);+ }+ return {visible: visible.length, offscreen: offscreen.length, total: all.length,+ sample_visible: visible.slice(0, 5)};+ })()+ ''', 'returnByValue': True+ }, session_id=sid)+ js_r, ms, err = await t('JS interactive query', do_js_interactive())+ js_data = js_r.get('result',{}).get('value',{}) if js_r else {}+ print(f' JS interactive query (viewport) {ms:>8.0f}ms {js_data.get("visible",0)} visible, {js_data.get("total",0)} total {err or ""}')+ js_ms = ms++ # 7. JS query ALL elements (to compare with querySelectorAll('*'))+ async def do_js_all():+ return await send.Runtime.evaluate(params={+ 'expression': 'document.querySelectorAll("*").length', 'returnByValue': True+ }, session_id=sid)+ _, ms, err = await t('JS querySelectorAll(*)', do_js_all())+ print(f' JS querySelectorAll("*").length {ms:>8.0f}ms {err or ""}')++ # ── PYTHON PROCESSING (what browser-use adds) ─────────────────────+ if snapshot and dom_tree:+ print(f'\n {"PYTHON PROCESSING":-<70}')++ # 8. build_snapshot_lookup+ t0=time.time()+ device_pixel_ratio = 1.0+ snapshot_lookup = build_snapshot_lookup(snapshot, device_pixel_ratio)+ ms = (time.time()-t0)*1000+ print(f' build_snapshot_lookup {ms:>8.0f}ms {len(snapshot_lookup):,} entries')++ # 9. Build AX tree lookup+ t0=time.time()+ ax_tree_data = ax_full if ax_full else {'nodes': []}+ ax_tree_lookup = {n['backendDOMNodeId']: n for n in ax_tree_data['nodes'] if 'backendDOMNodeId' in n}+ ms = (time.time()-t0)*1000+ print(f' build AX tree lookup {ms:>8.0f}ms {len(ax_tree_lookup):,} entries')++ # 10. Full _construct_enhanced_node + get_dom_tree+ dom_service = DomService(+ browser_session=s, cross_origin_iframes=False,+ paint_order_filtering=True, max_iframes=100, max_iframe_depth=5+ )+ t0=time.time()+ try:+ enhanced_tree, timing = await asyncio.wait_for(+ dom_service.get_dom_tree(target_id=target_id), timeout=60+ )+ ms = (time.time()-t0)*1000+ tree_ok = True+ except Exception as e:+ ms = (time.time()-t0)*1000+ tree_ok = False+ enhanced_tree = None+ timing = {}+ print(f' get_dom_tree (full) {ms:>8.0f}ms')+ # Print sub-timings from the timing dict+ for k, v in sorted(timing.items()):+ print(f' {k:<40} {v:>8.1f}ms')++ # 11. Serialization+ if tree_ok and enhanced_tree:+ t0=time.time()+ serialized, ser_timing = DOMTreeSerializer(+ enhanced_tree, None, paint_order_filtering=True, session_id=s.id+ ).serialize_accessible_elements()+ ms = (time.time()-t0)*1000+ print(f' serialize_accessible_elements {ms:>8.0f}ms {len(serialized.selector_map):,} selectors')+ for k, v in sorted(ser_timing.items()):+ print(f' {k:<40} {v*1000:>8.1f}ms')++ # 12. Serialization WITHOUT paint order+ t0=time.time()+ serialized2, ser_timing2 = DOMTreeSerializer(+ enhanced_tree, None, paint_order_filtering=False, session_id=s.id+ ).serialize_accessible_elements()+ ms = (time.time()-t0)*1000+ print(f' serialize (NO paint order) {ms:>8.0f}ms {len(serialized2.selector_map):,} selectors')+ for k, v in sorted(ser_timing2.items()):+ print(f' {k:<40} {v*1000:>8.1f}ms')++ # ── COMPARISON SUMMARY ────────────────────────────────────────────+ print(f'\n {"COMPARISON":-<70}')+ print(f' Raw CDP snapshot+DOM+AX: {snap_ms+dom_ms+ax_full_ms:>8.0f}ms (Chrome work)')+ print(f' JS interactive query: {js_ms:>8.0f}ms (alternative)')+ print(f' Full pipeline overhead: Python processing on top of CDP')++ await s.kill()+++async def main():+ d = Path(__file__).parent / 'generated'; d.mkdir(exist_ok=True)+ scales = [5000, 20000, 100000]+ for n in scales:+ (d/f'pipe_{n}.html').write_text(gen(n))+ os.chdir(str(d))+ srv = HTTPServer(('127.0.0.1', 8775), Q)+ Thread(target=srv.serve_forever, daemon=True).start()++ for n in scales:+ await bench(n, 'http://127.0.0.1:8775')++ srv.shutdown()++if __name__ == '__main__':+ asyncio.run(main())diff --git a/tests/heavy_pages/bench_snapshot_profile.py b/tests/heavy_pages/bench_snapshot_profile.pynew file mode 100644index 000000000..19324d45c--- /dev/null+++ b/tests/heavy_pages/bench_snapshot_profile.py@@ -0,0 +1,157 @@+"""Profile build_snapshot_lookup to find the exact O(n²) bottleneck."""+import asyncio, os, sys, time, cProfile, pstats, io+from pathlib import Path+from http.server import HTTPServer, SimpleHTTPRequestHandler+from threading import Thread++sys.path.insert(0, str(Path(__file__).resolve().parents[2]))+from dotenv import load_dotenv+load_dotenv(Path('/Users/magnus/Developer/cloud/backend/.env'))++from browser_use.browser.profile import BrowserProfile+from browser_use.browser.session import BrowserSession+from browser_use.dom.enhanced_snapshot import build_snapshot_lookup, REQUIRED_COMPUTED_STYLES, _parse_rare_boolean_data++import logging+logging.basicConfig(level=logging.WARNING)++def gen(n):+ return f'<html><body><div id=b></div><script>const f=document.createDocumentFragment();for(let i=0;i<{n};i++){{const d=document.createElement("div");d.innerHTML="<span>"+i+"</span><input value=v"+i+" /><button>X</button>";f.appendChild(d)}}document.getElementById("b").appendChild(f)</script></body></html>'++class Q(SimpleHTTPRequestHandler):+ def log_message(self,*a):pass++async def main():+ d = Path(__file__).parent / 'generated'; d.mkdir(exist_ok=True)+ for n in [5000, 20000]:+ (d/f'prof_{n}.html').write_text(gen(n))+ os.chdir(str(d))+ srv = HTTPServer(('127.0.0.1', 8774), Q)+ Thread(target=srv.serve_forever, daemon=True).start()++ for n in [5000, 20000]:+ s = BrowserSession(browser_profile=BrowserProfile(headless=True))+ await s.start()+ cdp = await s.get_or_create_cdp_session(focus=True)+ sid = cdp.session_id+ send = cdp.cdp_client.send++ await send.Page.navigate(params={'url':f'http://127.0.0.1:8774/prof_{n}.html'}, session_id=sid)+ await asyncio.sleep(2)++ # Get snapshot+ snapshot = await send.DOMSnapshot.captureSnapshot(params={+ 'computedStyles': REQUIRED_COMPUTED_STYLES,+ 'includePaintOrder':True,'includeDOMRects':True,+ 'includeBlendedBackgroundColors':False,'includeTextColorOpacities':False+ }, session_id=sid)++ total_nodes = sum(len(d.get('nodes',{}).get('nodeName',[])) for d in snapshot.get('documents',[]))+ print(f'\n{"="*80}')+ print(f' {n:,} elements, {total_nodes:,} snapshot nodes')+ print(f'{"="*80}')++ # Check isClickable data size+ for doc_idx, doc in enumerate(snapshot['documents']):+ nodes_data = doc['nodes']+ if 'isClickable' in nodes_data:+ clickable_list = nodes_data['isClickable']['index']+ print(f' doc[{doc_idx}] isClickable index list length: {len(clickable_list)}')++ # Manual breakdown of build_snapshot_lookup+ strings = snapshot['strings']+ documents = snapshot['documents']++ for doc_idx, document in enumerate(documents):+ nodes_data = document['nodes']+ layout = document['layout']++ # Time: build backend_node_to_snapshot_index+ t0 = time.time()+ backend_node_to_snapshot_index = {}+ if 'backendNodeId' in nodes_data:+ for i, bid in enumerate(nodes_data['backendNodeId']):+ backend_node_to_snapshot_index[bid] = i+ ms = (time.time()-t0)*1000+ print(f' doc[{doc_idx}] build backend_node_to_snapshot_index: {ms:.1f}ms ({len(backend_node_to_snapshot_index)} entries)')++ # Time: build layout_index_map+ t0 = time.time()+ layout_index_map = {}+ if layout and 'nodeIndex' in layout:+ for layout_idx, node_index in enumerate(layout['nodeIndex']):+ if node_index not in layout_index_map:+ layout_index_map[node_index] = layout_idx+ ms = (time.time()-t0)*1000+ print(f' doc[{doc_idx}] build layout_index_map: {ms:.1f}ms ({len(layout_index_map)} entries)')++ # Time: isClickable parsing (the suspected O(n²))+ if 'isClickable' in nodes_data:+ clickable_index_list = nodes_data['isClickable']['index']++ # Method 1: current (list scan per node)+ t0 = time.time()+ count = 0+ for snapshot_index in range(len(nodes_data.get('backendNodeId', []))):+ if snapshot_index in clickable_index_list: # O(len(clickable_index_list)) per call!+ count += 1+ ms = (time.time()-t0)*1000+ print(f' doc[{doc_idx}] isClickable via LIST scan: {ms:.1f}ms (found {count} clickable)')++ # Method 2: convert to set first+ t0 = time.time()+ clickable_set = set(clickable_index_list)+ count2 = 0+ for snapshot_index in range(len(nodes_data.get('backendNodeId', []))):+ if snapshot_index in clickable_set: # O(1) per call+ count2 += 1+ ms = (time.time()-t0)*1000+ print(f' doc[{doc_idx}] isClickable via SET scan: {ms:.1f}ms (found {count2} clickable)')+ assert count == count2++ # Time: the main loop (creating EnhancedSnapshotNode objects)+ t0 = time.time()+ dummy_count = 0+ for backend_node_id, snapshot_index in backend_node_to_snapshot_index.items():+ # Simulate the work without isClickable+ if snapshot_index in layout_index_map:+ layout_idx = layout_index_map[snapshot_index]+ if layout_idx < len(layout.get('bounds', [])):+ bounds = layout['bounds'][layout_idx]+ _ = bounds[0] if len(bounds) >= 4 else None+ if layout_idx < len(layout.get('styles', [])):+ style_indices = layout['styles'][layout_idx]+ # Parse styles+ styles = {}+ for i, si in enumerate(style_indices):+ if i < len(REQUIRED_COMPUTED_STYLES) and 0 <= si < len(strings):+ styles[REQUIRED_COMPUTED_STYLES[i]] = strings[si]+ dummy_count += 1+ ms = (time.time()-t0)*1000+ print(f' doc[{doc_idx}] main loop (no isClickable): {ms:.1f}ms ({dummy_count} iterations)')++ # Time: full build_snapshot_lookup+ t0 = time.time()+ result = build_snapshot_lookup(snapshot, 1.0)+ ms = (time.time()-t0)*1000+ print(f'\n FULL build_snapshot_lookup: {ms:.0f}ms ({len(result)} entries)')++ # Profile it+ pr = cProfile.Profile()+ pr.enable()+ result2 = build_snapshot_lookup(snapshot, 1.0)+ pr.disable()++ stream = io.StringIO()+ ps = pstats.Stats(pr, stream=stream).sort_stats('cumulative')+ ps.print_stats(15)+ print(f'\n cProfile top 15:')+ for line in stream.getvalue().split('\n')[:20]:+ print(f' {line}')++ await s.kill()++ srv.shutdown()++if __name__ == '__main__':+ asyncio.run(main())