Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(accessibility): resolve AX object IDs in their owning session
ldm0d63da2d76c2d
diff --git a/moli-cdp-smoke/moli_cdp_smoke/puppeteer_aria_queries.mjs b/moli-cdp-smoke/moli_cdp_smoke/puppeteer_aria_queries.mjsnew file mode 100644index 000000000..45bc6cc4b--- /dev/null+++ b/moli-cdp-smoke/moli_cdp_smoke/puppeteer_aria_queries.mjs@@ -0,0 +1,97 @@+export async function runPuppeteerAriaQuerySmoke(page) {+ await page.evaluate(async () => {+ document.body.innerHTML = `+ <main id="aria-root"><button id="aria-go">Go</button></main>+ <button id="aria-stay">Stay</button>+ `;+ globalThis.__puppeteerAriaClicks = 0;+ document.querySelector('#aria-go').addEventListener('click', () => {+ globalThis.__puppeteerAriaClicks += 1;+ });+ const iframe = document.createElement('iframe');+ iframe.id = 'aria-frame';+ iframe.srcdoc = '<button id="aria-child">Child Go</button>';+ await new Promise(resolve => {+ iframe.addEventListener('load', resolve, { once: true });+ document.body.append(iframe);+ });+ });++ for (const selector of ['aria/Go', 'aria/Go[role="button"]']) {+ const handle = await page.$(selector);+ if (!handle) {+ throw new Error(`Puppeteer ARIA selector did not match: ${selector}`);+ }+ try {+ const id = await handle.evaluate(element => element.id);+ if (id !== 'aria-go') {+ throw new Error(`Puppeteer ARIA selector returned ${id}: ${selector}`);+ }+ } finally {+ await handle.dispose();+ }+ }++ const buttons = await page.$$('aria/[role="button"]');+ try {+ const ids = (await Promise.all(buttons.map(handle => handle.evaluate(element => element.id)))).sort();+ if (JSON.stringify(ids) !== JSON.stringify(['aria-go', 'aria-stay'])) {+ throw new Error(`Puppeteer ARIA role query returned unexpected elements: ${JSON.stringify(ids)}`);+ }+ } finally {+ await Promise.all(buttons.map(handle => handle.dispose()));+ }++ if (await page.$('aria/Missing')) {+ throw new Error('Puppeteer ARIA query must return null for a missing name');+ }++ const root = await page.$('#aria-root');+ if (!root) {+ throw new Error('Puppeteer ARIA query fixture root is missing');+ }+ try {+ if (await root.$('aria/Stay')) {+ throw new Error('Puppeteer ARIA query escaped its element subtree');+ }+ const go = await root.waitForSelector('aria/Go[role="button"]', { timeout: 3000 });+ if (!go) {+ throw new Error('Puppeteer scoped ARIA wait returned no element');+ }+ try {+ await go.evaluate(element => element.click());+ } finally {+ await go.dispose();+ }+ } finally {+ await root.dispose();+ }+ const clicks = await page.evaluate(() => globalThis.__puppeteerAriaClicks);+ if (clicks !== 1) {+ throw new Error(`Puppeteer ARIA handle must activate the matched element once: ${clicks}`);+ }++ const iframe = await page.$('#aria-frame');+ if (!iframe) {+ throw new Error('Puppeteer ARIA query fixture iframe is missing');+ }+ try {+ const frame = await iframe.contentFrame();+ const child = await frame?.$('aria/Child Go[role="button"]');+ if (!child) {+ throw new Error('Puppeteer ARIA selector did not match in the child frame');+ }+ try {+ const id = await child.evaluate(element => element.id);+ if (id !== 'aria-child') {+ throw new Error(`Puppeteer child-frame ARIA selector returned ${id}`);+ }+ } finally {+ await child.dispose();+ }+ } finally {+ await iframe.dispose();+ }++ return { selectors: ['name', 'name-and-role', 'role', 'missing', 'scoped-wait', 'iframe'] };+}diff --git a/moli-cdp-smoke/moli_cdp_smoke/puppeteer_smoke.mjs b/moli-cdp-smoke/moli_cdp_smoke/puppeteer_smoke.mjsindex 7f11b2499..bfcbea330 100644--- a/moli-cdp-smoke/moli_cdp_smoke/puppeteer_smoke.mjs+++ b/moli-cdp-smoke/moli_cdp_smoke/puppeteer_smoke.mjs@@ -4,6 +4,7 @@ import { createRequire } from 'node:module';import os from 'node:os';import path from 'node:path';+import { runPuppeteerAriaQuerySmoke } from './puppeteer_aria_queries.mjs';import {activateXPathElement,runPuppeteerDomInteractionSmoke,@@ -437,6 +438,12 @@ async function main() {}record('puppeteer_evaluate_fetch');+ const ariaQueryResult = await withTimeout(+ 'Puppeteer ARIA selectors',+ runPuppeteerAriaQuerySmoke(page),+ );+ record('puppeteer_aria_selector_workflow', ariaQueryResult);+const domInteractionResult = await withTimeout('Puppeteer DOM interaction matrix',runPuppeteerDomInteractionSmoke(page),diff --git a/moli-protocol/src/domains/accessibility.rs b/moli-protocol/src/domains/accessibility.rsindex e9cae6c68..2876dbf0b 100644--- a/moli-protocol/src/domains/accessibility.rs+++ b/moli-protocol/src/domains/accessibility.rs@@ -40,7 +40,7 @@ enum PendingAccessibilityCommandKind {},ChildFrameRoot,ObjectAccessibilityPayloads {- frame_id: String,+ frame_id: Option<String>,top_frame_id: String,operation: AccessibilityNodeOperation,},@@ -412,28 +412,25 @@ fn start_pending_object_reference_command(let Some(top_frame_id) = helpers::top_frame_id_for_session(conn, cmd.session_id) else {return Err(PendingAccessibilityCommandStartError::no_document_loaded());};- let resolved_frame_id = frame_id.unwrap_or(top_frame_id.as_str()).to_owned();let Some(object_id) = reference.object_id.as_deref() else {return Err(PendingAccessibilityCommandStartError {code: -32000,message: "Could not find node with given id".to_owned(),});};+ let inspector_session_id =+ conn.target_renderer_runtime_inspector_session_id_for_session(cmd.session_id);let Some(page) = helpers::loaded_page_mut_for_session(conn, cmd.session_id) else {return Err(PendingAccessibilityCommandStartError::no_document_loaded());};- let pending = start_accessibility_object_page_command(- page,- cmd.session_id.map(str::to_owned),- object_id,- &operation,- )- .map_err(PendingAccessibilityCommandStartError::renderer_error)?;+ let pending =+ start_accessibility_object_page_command(page, inspector_session_id, object_id, &operation)+ .map_err(PendingAccessibilityCommandStartError::renderer_error)?;Ok(Some(PendingAccessibilityCommandDispatch::from_command(conn,cmd,PendingAccessibilityCommandKind::ObjectAccessibilityPayloads {- frame_id: resolved_frame_id,+ frame_id: frame_id.map(str::to_owned),top_frame_id,operation,},@@ -720,7 +717,7 @@ pub(crate) async fn complete_pending_accessibility_command(fn complete_object_accessibility_payloads_command(page: &mut moli_core::page::Page,completion: CompletedPageCommand,- frame_id: String,+ frame_id: Option<String>,top_frame_id: String,operation: AccessibilityNodeOperation,) -> CommandOutputPlan {@@ -740,7 +737,11 @@ fn complete_object_accessibility_payloads_command(.frame_id.as_deref().unwrap_or(top_frame_id.as_str());- if object_frame_id != frame_id {+ // Object IDs identify their own frame unless the caller explicitly constrains it.+ if frame_id+ .as_deref()+ .is_some_and(|frame_id| object_frame_id != frame_id)+ {return CommandOutputPlan::error(-32000, "Could not find node with given id");}let Some(mut nodes) = payloads.payloads else {diff --git a/moli-protocol/src/domains/accessibility/native.rs b/moli-protocol/src/domains/accessibility/native.rsindex 0bf45c8c2..7c0d2b2e9 100644--- a/moli-protocol/src/domains/accessibility/native.rs+++ b/moli-protocol/src/domains/accessibility/native.rs@@ -197,9 +197,8 @@ fn reference_operation(top_frame_id: String,operation: AccessibilityNodeOperation,) -> Result<Operation, StartError> {- let frame_id = frame_id.unwrap_or_else(|| top_frame_id.clone());if let Some(object_id) = reference.object_id {- let session = cmd.session_id.map(str::to_owned);+ let session = conn.target_renderer_runtime_inspector_session_id_for_session(cmd.session_id);let command = match &operation {AccessibilityNodeOperation::Ancestors => {Command::accessibility_node_and_ancestor_payloads_for_object_id(session, object_id)@@ -220,6 +219,7 @@ fn reference_operation(project_nodes(reply, frame_id, top_frame_id, operation)}));}+ let frame_id = frame_id.unwrap_or_else(|| top_frame_id.clone());let reference = if let Some(id) = renderer_backend_node_id_for_reference(&reference) {DevToolsDomNodeReference::BackendNodeId(id)} else {@@ -261,13 +261,13 @@ fn node_operation(}};Operation::new(command, move |reply| {- project_nodes(reply, frame_id, top_frame_id, operation)+ project_nodes(reply, Some(frame_id), top_frame_id, operation)})}fn project_nodes(reply: anyhow::Result<Reply>,- frame_id: String,+ frame_id: Option<String>,top_frame_id: String,operation: AccessibilityNodeOperation,) -> Response {@@ -282,7 +282,11 @@ fn project_nodes(}_ => unreachable!("accessibility node reply"),};- if payload.frame_id.as_deref().unwrap_or(&top_frame_id) != frame_id {+ // Object IDs identify their own frame unless the caller explicitly constrains it.+ if frame_id+ .as_deref()+ .is_some_and(|frame_id| payload.frame_id.as_deref().unwrap_or(&top_frame_id) != frame_id)+ {return node_not_found();}let Some(mut nodes) = payload.payloads else {diff --git a/moli-protocol/src/domains/accessibility/tests.rs b/moli-protocol/src/domains/accessibility/tests.rsindex 647f20c25..a3b9af97b 100644--- a/moli-protocol/src/domains/accessibility/tests.rs+++ b/moli-protocol/src/domains/accessibility/tests.rs@@ -2517,6 +2517,131 @@ async fn query_ax_tree_supports_object_id_and_empty_match() {assert_eq!(empty["result"]["nodes"], json!([]));}+#[tokio::test(flavor = "multi_thread")]+async fn query_ax_tree_object_id_uses_owning_inspector_session() {+ let mut ctx = TestContext::new();+ load_page_async(+ &mut ctx,+ "<!doctype html><html><body><button id='go'>Go</button></body></html>",+ )+ .await;+ let browser_context = ctx.conn.browser_context.as_mut().expect("browser context");+ browser_context.attach_active_session("SID-primary");+ assert!(browser_context.assign_attached_session_to_target("TID-1", "SID-attached".to_owned()));++ for (session_id, other_session_id) in [+ ("SID-primary", "SID-attached"),+ ("SID-attached", "SID-primary"),+ ] {+ ctx.process_async(json!({+ "id": 560,+ "method": "Runtime.enable",+ "sessionId": session_id+ }))+ .await;+ ctx.take_all();+ ctx.process_async(json!({+ "id": 561,+ "method": "Page.createIsolatedWorld",+ "sessionId": session_id,+ "params": { "frameId": "TID-1", "worldName": format!("ax-query-{session_id}") }+ }))+ .await;+ let context_id = ctx.take_response_by_id(561)["result"]["executionContextId"]+ .as_i64()+ .expect("isolated world id");+ ctx.take_all();+ ctx.process_async(json!({+ "id": 562,+ "method": "Runtime.evaluate",+ "sessionId": session_id,+ "params": { "expression": "document", "contextId": context_id }+ }))+ .await;+ let object_id = ctx.take_response_by_id(562)["result"]["result"]["objectId"]+ .as_str()+ .expect("document object id")+ .to_owned();++ ctx.process_async(json!({+ "id": 563,+ "method": "Accessibility.queryAXTree",+ "sessionId": session_id,+ "params": { "objectId": object_id, "role": "button", "accessibleName": "Go" }+ }))+ .await;+ let message = ctx.take_response_by_id(563);+ let nodes = message["result"]["nodes"]+ .as_array()+ .unwrap_or_else(|| panic!("{session_id} AX query failed: {message}"));+ assert_eq!(nodes.len(), 1);+ assert_eq!(nodes[0]["role"]["value"], "button");+ assert_eq!(nodes[0]["name"]["value"], "Go");++ ctx.process_async(json!({+ "id": 564,+ "method": "Accessibility.queryAXTree",+ "sessionId": other_session_id,+ "params": { "objectId": object_id, "role": "button" }+ }))+ .await;+ ctx.expect_error(564, -32000, "Could not find node with given id");+ }+}++#[tokio::test(flavor = "multi_thread")]+async fn query_ax_tree_object_id_infers_child_frame_and_checks_explicit_frame() {+ let mut ctx = TestContext::new();+ load_page_async(+ &mut ctx,+ "<!doctype html><html><body><button>Parent</button><iframe srcdoc=\"<button>Go</button>\"></iframe></body></html>",+ )+ .await;+ enable_runtime_async(&mut ctx).await;+ let child_frame_id = child_frame_id_for_single_iframe_async(&mut ctx).await;+ ctx.process_async(json!({+ "id": 565,+ "method": "Page.createIsolatedWorld",+ "params": { "frameId": child_frame_id, "worldName": "ax-query-child-document" }+ }))+ .await;+ let context_id = ctx.take_response_by_id(565)["result"]["executionContextId"]+ .as_i64()+ .expect("child isolated world id");+ ctx.take_all();+ ctx.process_async(json!({+ "id": 566,+ "method": "Runtime.evaluate",+ "params": { "expression": "document", "contextId": context_id }+ }))+ .await;+ let object_id = ctx.take_response_by_id(566)["result"]["result"]["objectId"]+ .as_str()+ .expect("child document object id")+ .to_owned();++ ctx.process_async(json!({+ "id": 567,+ "method": "Accessibility.queryAXTree",+ "params": { "objectId": object_id, "role": "button" }+ }))+ .await;+ let message = ctx.take_response_by_id(567);+ let nodes = message["result"]["nodes"]+ .as_array()+ .unwrap_or_else(|| panic!("child AX query failed: {message}"));+ assert_eq!(nodes.len(), 1);+ assert_eq!(nodes[0]["name"]["value"], "Go");++ ctx.process_async(json!({+ "id": 568,+ "method": "Accessibility.queryAXTree",+ "params": { "objectId": object_id, "frameId": "TID-1", "role": "button" }+ }))+ .await;+ ctx.expect_error(568, -32000, "Could not find node with given id");+}+#[tokio::test(flavor = "multi_thread")]async fn query_ax_tree_requires_context_loaded_page_and_bound_node() {let mut ctx = TestContext::new();