Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
perf(accessibility): avoid JSON intermediates in semantic dumps
ldm01404fa344be4
diff --git a/Cargo.lock b/Cargo.lockindex 438ac0bc2..c57bb3b37 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2618,6 +2618,7 @@ dependencies = ["moli-html2md","percent-encoding","selectors",+ "serde","serde_json","servo_arc","thin-vec",diff --git a/moli-benchmark/scripts/semantic-dump-memory.py b/moli-benchmark/scripts/semantic-dump-memory.pynew file mode 100644index 000000000..c3de662b8--- /dev/null+++ b/moli-benchmark/scripts/semantic-dump-memory.py@@ -0,0 +1,141 @@+#!/usr/bin/env python3+"""Compare semantic dump time/RSS and verify output bytes on a generated list.++Build both revisions with `cargo build --release -p moli`, then run:+ python3 moli-benchmark/scripts/semantic-dump-memory.py \+ --before /tmp/moli-before --after target/release/moli \+ --output-dir /tmp/semantic-dump-results++Requires Unix wait4; RSS is reported in bytes on both Linux and macOS.+Both executables are copied into the output directory so executable mappings+have the same filesystem backing when their RSS is compared.+"""++import argparse+import functools+import hashlib+import http.server+import json+import os+from pathlib import Path+import shutil+import subprocess+import sys+import threading+import time+++class QuietHandler(http.server.SimpleHTTPRequestHandler):+ def log_message(self, *_args):+ pass+++def positive_int(value):+ result = int(value)+ if result <= 0:+ raise argparse.ArgumentTypeError("must be positive")+ return result+++def measure(binary, label, fmt, repeat, rows, url, output_dir, env):+ output = output_dir / f"{label}-{fmt}.out"+ errors = output_dir / f"{label}-{fmt}.stderr"+ command = [+ str(binary), "fetch", "--timeout", "120000", "--dump", fmt, url+ ]+ started = time.monotonic()+ with output.open("wb") as out, errors.open("wb") as err:+ with subprocess.Popen(command, stdout=out, stderr=err, env=env) as process:+ _, status, usage = os.wait4(process.pid, 0)+ process.returncode = os.waitstatus_to_exitcode(status)+ wall_seconds = time.monotonic() - started+ with output.open("rb") as stream:+ digest = hashlib.file_digest(stream, "sha256").hexdigest()+ record = {+ "binary": label,+ "executable": str(binary),+ "rows": rows,+ "format": fmt,+ "repeat": repeat,+ "returncode": process.returncode,+ "maxrss_bytes": usage.ru_maxrss * (1 if sys.platform == "darwin" else 1024),+ "wall_seconds": wall_seconds,+ "output_bytes": output.stat().st_size,+ "sha256": digest,+ }+ with (output_dir / "results.jsonl").open("a") as results:+ results.write(json.dumps(record) + "\n")+ print(json.dumps(record), flush=True)+ if process.returncode:+ raise RuntimeError(f"{label} {fmt} failed; see {errors}")+ return record["output_bytes"], digest+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--before", required=True, type=Path)+ parser.add_argument("--after", required=True, type=Path)+ parser.add_argument("--output-dir", required=True, type=Path)+ parser.add_argument("--rows", type=positive_int, default=40000)+ parser.add_argument("--runs", type=positive_int, default=3)+ args = parser.parse_args()+ binaries = [("before", args.before.resolve()), ("after", args.after.resolve())]+ for label, binary in binaries:+ if not os.access(binary, os.X_OK):+ parser.error(f"{label} binary is not executable: {binary}")+ args.output_dir.mkdir(parents=True, exist_ok=True)+ staged_binaries = []+ manifest = {"rows": args.rows, "runs": args.runs, "binaries": {}}+ for label, binary in binaries:+ staged = (args.output_dir / f"moli-{label}").resolve()+ if binary != staged:+ shutil.copy2(binary, staged)+ with staged.open("rb") as stream:+ digest = hashlib.file_digest(stream, "sha256").hexdigest()+ manifest["binaries"][label] = {+ "source": str(binary), "staged": str(staged), "sha256": digest+ }+ staged_binaries.append((label, staged))+ binaries = staged_binaries+ (args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")+ fixture = args.output_dir / "large-list.html"+ with fixture.open("w") as page:+ page.write("<!doctype html><html><head><title>Large list</title></head><body><main>\n")+ for index in range(args.rows):+ page.write(+ f'<div class="row"><a href="/item/{index}">Item {index}</a>'+ f'<span> description {index}</span></div>\n'+ )+ page.write("</main></body></html>\n")+ handler = functools.partial(QuietHandler, directory=str(args.output_dir.resolve()))+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ url = f"http://127.0.0.1:{server.server_port}/{fixture.name}"+ env = {+ key: value for key, value in os.environ.items()+ if key.lower() not in ("http_proxy", "https_proxy", "all_proxy")+ }+ try:+ for fmt in ("semantic_tree_text", "semantic_tree", "markdown"):+ expected = None+ for repeat in range(args.runs):+ # Alternate the order to reduce warm-cache/order bias.+ order = binaries if repeat % 2 == 0 else list(reversed(binaries))+ for label, binary in order:+ actual = measure(+ binary, label, fmt, repeat, args.rows, url, args.output_dir, env+ )+ if expected is None:+ expected = actual+ elif actual != expected:+ raise RuntimeError(f"{fmt} output differs: {label}, run {repeat}")+ print("All dump outputs are byte-for-byte identical.", flush=True)+ finally:+ server.shutdown()+ thread.join()+ server.server_close()+++if __name__ == "__main__":+ main()diff --git a/moli-core/src/page/accessibility_support/mod.rs b/moli-core/src/page/accessibility_support/mod.rsindex 1e043a28f..a44068f20 100644--- a/moli-core/src/page/accessibility_support/mod.rs+++ b/moli-core/src/page/accessibility_support/mod.rs@@ -6,6 +6,24 @@ use crate::renderer::{use serde_json::Value;impl Page {+ /// Observe the same live AX tree as the protocol without materializing a+ /// JSON object for every node. `frame_id` selects a child Document.+ pub async fn accessibility_tree_nodes_async(+ &mut self,+ frame_id: Option<String>,+ ) -> anyhow::Result<Option<Vec<moli_dom::accessibility::AccessibilityNode>>> {+ let pending =+ self.start_page_command(RendererPageCommand::AccessibilityTreeNodes { frame_id })?;+ let completion = pending.wait().await?;+ let reply = self.finish_page_command(completion);+ expect_page_reply!(+ reply,+ "accessibility tree nodes page command",+ "optional accessibility nodes",+ RendererPageReply::OptionalAccessibilityNodes(nodes) => Ok(nodes),+ )+ }+pub fn start_accessibility_tree_payloads_for_document(&self,max_depth: Option<i32>,diff --git a/moli-core/src/page/command_dispatch.rs b/moli-core/src/page/command_dispatch.rsindex a48c341f1..65efcc600 100644--- a/moli-core/src/page/command_dispatch.rs+++ b/moli-core/src/page/command_dispatch.rs@@ -316,6 +316,7 @@ impl Page {RendererPageReply::DocumentNodePropertyResolution(_) => {"a document node property resolution"}+ RendererPageReply::OptionalAccessibilityNodes(_) => "optional accessibility nodes",RendererPageReply::OptionalAccessibilityPayloads(_) => {"optional accessibility payloads"}diff --git a/moli-core/src/page/mod.rs b/moli-core/src/page/mod.rsindex e691bf082..2c1bcb093 100644--- a/moli-core/src/page/mod.rs+++ b/moli-core/src/page/mod.rs@@ -68,6 +68,7 @@ pub use input_support::{decode_element_click_preparation_completion, decode_input_dispatch_outcome_completion,decode_insert_text_completion,};+pub use moli_dom::accessibility::{AccessibilityNode, AccessibilityNodeId, AccessibilityValue};pub use moli_page_types::DomScrollIntoViewRect;pub use moli_page_types::{RendererDomDebuggerDomBreakpointType, RendererDomDebuggerEventListenerBreakpoint,diff --git a/moli-dom/Cargo.toml b/moli-dom/Cargo.tomlindex c87fe4257..cbc046214 100644--- a/moli-dom/Cargo.toml+++ b/moli-dom/Cargo.toml@@ -13,6 +13,7 @@ moli-html2md = { path = "../moli-html2md" }moli-html-input-temporal = { path = "../moli-html-input-temporal" }percent-encoding = "2.3"selectors = "0.40"+serde = { version = "1.0.228", features = ["derive"] }serde_json = "1.0.145"servo_arc = "0.4.3"thin-vec = "0.2.14"diff --git a/moli-dom/src/accessibility/ax_node.rs b/moli-dom/src/accessibility/ax_node.rsnew file mode 100644index 000000000..28490b603--- /dev/null+++ b/moli-dom/src/accessibility/ax_node.rs@@ -0,0 +1,151 @@+use std::fmt;++use serde::{Serialize, Serializer, ser::SerializeMap};+use serde_json::Value;++/// Keep references numeric until a protocol or dump actually serializes them.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+pub enum AccessibilityNodeId {+ Dom(u32),+ ListMarker(u32),+}++impl fmt::Display for AccessibilityNodeId {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match self {+ Self::Dom(id) => write!(f, "AX-{id}"),+ Self::ListMarker(id) => write!(f, "AX-LM-{id}"),+ }+ }+}++impl Serialize for AccessibilityNodeId {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ serializer.collect_str(self)+ }+}++#[derive(Debug, Serialize)]+pub struct AccessibilityValue<T> {+ #[serde(rename = "type")]+ pub kind: &'static str,+ pub value: T,+}++/// The canonical AX payload, without a JSON object and allocated field names+/// for every node. Serialization retains the protocol's field insertion order.+#[derive(Debug)]+pub struct AccessibilityNode {+ pub node_id: AccessibilityNodeId,+ pub backend_node_id: Option<u32>,+ pub ignored: bool,+ pub ignored_reasons: Option<Value>,+ pub role: AccessibilityValue<&'static str>,+ pub name: Option<AccessibilityValue<String>>,+ pub value: Option<Value>,+ pub properties: Option<Vec<Value>>,+ pub parent_id: Option<AccessibilityNodeId>,+ pub child_ids: Option<Vec<AccessibilityNodeId>>,+}++impl Serialize for AccessibilityNode {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let mut payload = serializer.serialize_map(None)?;+ payload.serialize_entry("nodeId", &self.node_id)?;+ if let Some(id) = self.backend_node_id {+ payload.serialize_entry("backendDOMNodeId", &id)?;+ }+ payload.serialize_entry("ignored", &self.ignored)?;+ // DOM nodes without an AX object have a smaller synthetic payload,+ // whose role historically precedes its ignored reason.+ if self.properties.is_none() {+ payload.serialize_entry("role", &self.role)?;+ }+ if let Some(reasons) = &self.ignored_reasons {+ payload.serialize_entry("ignoredReasons", reasons)?;+ }+ if self.properties.is_some() {+ payload.serialize_entry("role", &self.role)?;+ }+ if let Some(name) = &self.name {+ payload.serialize_entry("name", name)?;+ }+ if let Some(value) = &self.value {+ payload.serialize_entry("value", value)?;+ }+ if let Some(properties) = &self.properties {+ payload.serialize_entry("properties", properties)?;+ }+ // Joining a child frame appends parentId to its existing root payload.+ let mut parent_id = self.parent_id;+ if self.role.value != "RootWebArea"+ && let Some(parent) = parent_id.take()+ {+ payload.serialize_entry("parentId", &parent)?;+ }+ if let Some(children) = &self.child_ids {+ payload.serialize_entry("childIds", children)?;+ }+ if let Some(parent) = parent_id {+ payload.serialize_entry("parentId", &parent)?;+ }+ payload.end()+ }+}++impl From<AccessibilityNode> for Value {+ fn from(node: AccessibilityNode) -> Self {+ let mut payload = serde_json::Map::new();+ payload.insert("nodeId".to_owned(), Value::String(node.node_id.to_string()));+ if let Some(id) = node.backend_node_id {+ payload.insert("backendDOMNodeId".to_owned(), id.into());+ }+ payload.insert("ignored".to_owned(), node.ignored.into());+ let mut ignored_reasons = node.ignored_reasons;+ if node.properties.is_some()+ && let Some(reasons) = ignored_reasons.take()+ {+ payload.insert("ignoredReasons".to_owned(), reasons);+ }+ payload.insert(+ "role".to_owned(),+ serde_json::json!({"type":node.role.kind, "value":node.role.value}),+ );+ if let Some(reasons) = ignored_reasons {+ payload.insert("ignoredReasons".to_owned(), reasons);+ }+ if let Some(name) = node.name {+ payload.insert(+ "name".to_owned(),+ serde_json::json!({"type":name.kind, "value":name.value}),+ );+ }+ if let Some(value) = node.value {+ payload.insert("value".to_owned(), value);+ }+ if let Some(properties) = node.properties {+ payload.insert("properties".to_owned(), Value::Array(properties));+ }+ let mut parent_id = node.parent_id;+ if node.role.value != "RootWebArea"+ && let Some(parent) = parent_id.take()+ {+ payload.insert("parentId".to_owned(), Value::String(parent.to_string()));+ }+ if let Some(children) = node.child_ids {+ payload.insert(+ "childIds".to_owned(),+ Value::Array(+ children+ .into_iter()+ .map(|id| Value::String(id.to_string()))+ .collect(),+ ),+ );+ }+ if let Some(parent) = parent_id {+ payload.insert("parentId".to_owned(), Value::String(parent.to_string()));+ }+ Self::Object(payload)+ }+}diff --git a/moli-dom/src/accessibility/ax_tree.rs b/moli-dom/src/accessibility/ax_tree.rsindex 4dcf75635..5b573f43a 100644--- a/moli-dom/src/accessibility/ax_tree.rs+++ b/moli-dom/src/accessibility/ax_tree.rs@@ -5,10 +5,10 @@ use crate::{use serde_json::{Value, json};use std::collections::VecDeque;-use super::AccessibilityInput;use super::ax_projection::{AxIgnoredReason, AxTreeProjection};use super::ax_properties::{ax_name, ax_properties, ax_value};use super::ax_roles::{ax_role, ordered_list_item_index};+use super::{AccessibilityInput, AccessibilityNode, AccessibilityNodeId, AccessibilityValue};/// The semantic traversal requested by one synchronous AX command.#[derive(Clone, Copy, Debug)]@@ -27,6 +27,26 @@ pub fn accessibility_payloads_for_document(request: AccessibilityRequest,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,) -> Option<Vec<Value>> {+ build_nodes(document, input, node_id, request, backend_node_id_for_node)+}++pub fn accessibility_nodes_for_document(+ document: &DomHost,+ input: &mut AccessibilityInput<'_>,+ node_id: NodeId,+ request: AccessibilityRequest,+ backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,+) -> Option<Vec<AccessibilityNode>> {+ build_nodes(document, input, node_id, request, backend_node_id_for_node)+}++fn build_nodes<T: From<AccessibilityNode>>(+ document: &DomHost,+ input: &mut AccessibilityInput<'_>,+ node_id: NodeId,+ request: AccessibilityRequest,+ backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,+) -> Option<Vec<T>> {// A ShadowRoot contributes children but has no AX object of its own.let node_id = if matches!(request, AccessibilityRequest::Tree { .. }) {document.shadow_root_host(node_id).unwrap_or(node_id)@@ -41,11 +61,9 @@ pub fn accessibility_payloads_for_document(max_depth,backend_node_id_for_node,),- AccessibilityRequest::Node => Some(vec![node_payload(- &mut projection,- node_id,- backend_node_id_for_node,- )?]),+ AccessibilityRequest::Node => Some(vec![+ node_payload(&mut projection, node_id, backend_node_id_for_node)?.into(),+ ]),AccessibilityRequest::Children => {let mut nodes = Vec::new();if projection.is_rendered(node_id).ok()? {@@ -60,10 +78,9 @@ pub fn accessibility_payloads_for_document(}AccessibilityRequest::Ancestors => {if projection.node(node_id).ok()?.is_none() {- return Some(vec![ax_dom_node_without_object_payload(- node_id,- backend_node_id_for_node,- )?]);+ return Some(vec![+ ax_dom_node_without_object_payload(node_id, backend_node_id_for_node)?.into(),+ ]);}let mut nodes = Vec::new();add_ax_ancestors(@@ -76,11 +93,8 @@ pub fn accessibility_payloads_for_document(}AccessibilityRequest::Partial { fetch_relatives } => {let projected = projection.node(node_id).ok()?;- let mut nodes = vec![node_payload(- &mut projection,- node_id,- backend_node_id_for_node,- )?];+ let mut nodes =+ vec![node_payload(&mut projection, node_id, backend_node_id_for_node)?.into()];if fetch_relatives {if projected.is_some_and(|node| node.ignored_reason.is_none()) {add_ax_children(@@ -114,12 +128,12 @@ pub fn accessibility_payloads_for_document(}}-fn tree_payloads(+fn tree_payloads<T: From<AccessibilityNode>>(projection: &mut AxTreeProjection<'_, '_>,root: NodeId,max_depth: Option<i32>,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,-) -> Option<Vec<Value>> {+) -> Option<Vec<T>> {if !projection.is_rendered(root).ok()? {return Some(Vec::new());}@@ -149,7 +163,7 @@ fn node_payload(projection: &mut AxTreeProjection<'_, '_>,node_id: NodeId,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,-) -> Option<Value> {+) -> Option<AccessibilityNode> {if projection.node(node_id).ok()?.is_none() {ax_dom_node_without_object_payload(node_id, backend_node_id_for_node)} else {@@ -157,19 +171,15 @@ fn node_payload(}}-fn add_ax_ancestors(+fn add_ax_ancestors<T: From<AccessibilityNode>>(projection: &mut AxTreeProjection<'_, '_>,mut current: Option<NodeId>,- out: &mut Vec<Value>,+ out: &mut Vec<T>,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,) -> Option<()> {while let Some(node_id) = current {let projected = projection.node(node_id).ok()??;- out.push(ax_node_payload(- projection,- node_id,- backend_node_id_for_node,- )?);+ out.push(ax_node_payload(projection, node_id, backend_node_id_for_node)?.into());current = projected.parent;}Some(())@@ -178,21 +188,31 @@ fn add_ax_ancestors(fn ax_dom_node_without_object_payload(node_id: NodeId,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,-) -> Option<Value> {+) -> Option<AccessibilityNode> {let backend_node_id = backend_node_id_for_node(node_id)?;- Some(json!({- "nodeId": format!("AX-{backend_node_id}"),- "backendDOMNodeId": backend_node_id,- "ignored": true,- "role": {"type": "role", "value": "none"},- "ignoredReasons": [{"name": "notRendered", "value": {"type": "boolean", "value": true}}],- }))+ Some(AccessibilityNode {+ node_id: AccessibilityNodeId::Dom(backend_node_id),+ backend_node_id: Some(backend_node_id),+ ignored: true,+ role: AccessibilityValue {+ kind: "role",+ value: "none",+ },+ ignored_reasons: Some(+ json!([{"name": "notRendered", "value": {"type": "boolean", "value": true}}]),+ ),+ name: None,+ value: None,+ properties: None,+ parent_id: None,+ child_ids: None,+ })}-fn add_ax_children(+fn add_ax_children<T: From<AccessibilityNode>>(projection: &mut AxTreeProjection<'_, '_>,node_id: NodeId,- out: &mut Vec<Value>,+ out: &mut Vec<T>,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,) -> Option<()> {let mut reachable = projection.children(node_id).ok()?;@@ -210,18 +230,14 @@ fn add_ax_children(Some(())}-fn push_ax_node_payload(+fn push_ax_node_payload<T: From<AccessibilityNode>>(projection: &mut AxTreeProjection<'_, '_>,node_id: NodeId,- out: &mut Vec<Value>,+ out: &mut Vec<T>,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,) -> Option<()> {let projected = projection.node(node_id).ok()??;- out.push(ax_node_payload(- projection,- node_id,- backend_node_id_for_node,- )?);+ out.push(ax_node_payload(projection, node_id, backend_node_id_for_node)?.into());if projected.ignored_reason.is_none()&& let Some(marker) = ax_list_marker_payload(projection.document,@@ -230,7 +246,7 @@ fn push_ax_node_payload(backend_node_id_for_node,){- out.push(marker);+ out.push(marker.into());}Some(())}@@ -239,70 +255,66 @@ fn ax_node_payload(projection: &mut AxTreeProjection<'_, '_>,node_id: NodeId,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,-) -> Option<Value> {+) -> Option<AccessibilityNode> {let document = projection.document;let node = document.node(node_id)?;let projected = projection.node(node_id).ok()??;let backend_node_id = backend_node_id_for_node(node_id)?;- let mut payload = serde_json::Map::new();- payload.insert("nodeId".to_owned(), json!(format!("AX-{backend_node_id}")));- payload.insert("backendDOMNodeId".to_owned(), json!(backend_node_id));let ignored = projected.ignored_reason.is_some();- payload.insert("ignored".to_owned(), json!(ignored));- if let Some(reason) = projected.ignored_reason {- payload.insert(- "ignoredReasons".to_owned(),- ax_ignored_reasons_payload(document, node_id, reason, backend_node_id_for_node),- );- }- payload.insert(- "role".to_owned(),- json!({- "type": "role",- "value": if ignored { "none" } else { ax_role(node) },- }),- );-- if !ignored {- let name = ax_name(document, projection, node).ok()?;- if !name.is_empty() {- payload.insert(- "name".to_owned(),- json!({"type": "computedString", "value": name}),- );- }- if let Some(value) = ax_value(document, node_id, node) {- payload.insert("value".to_owned(), value);- }- }- payload.insert(- "properties".to_owned(),- Value::Array(if ignored {- Vec::new()- } else {- ax_properties(document, node_id, node)- }),- );-- if let Some(parent_id) = projected.parent {- let parent_backend_node_id = backend_node_id_for_node(parent_id)?;- payload.insert(- "parentId".to_owned(),- json!(format!("AX-{parent_backend_node_id}")),- );- }+ let ignored_reasons = projected.ignored_reason.map(|reason| {+ ax_ignored_reasons_payload(document, node_id, reason, backend_node_id_for_node)+ });+ let name = if ignored {+ String::new()+ } else {+ ax_name(document, projection, node).ok()?+ };+ let value = if ignored {+ None+ } else {+ ax_value(document, node_id, node)+ };+ let properties = if ignored {+ Vec::new()+ } else {+ ax_properties(document, node_id, node)+ };+ let parent_id = match projected.parent {+ Some(parent) => Some(AccessibilityNodeId::Dom(backend_node_id_for_node(parent)?)),+ None => None,+ };let children = projection.children(node_id).ok()?;- if !children.is_empty() {- let mut child_ids = Vec::new();+ let child_ids = if children.is_empty() {+ None+ } else {+ let mut child_ids = Vec::with_capacity(children.len());for child_id in children {if projection.is_rendered(child_id).ok()? {- let backend_node_id = backend_node_id_for_node(child_id)?;- child_ids.push(format!("AX-{backend_node_id}"));+ child_ids.push(AccessibilityNodeId::Dom(backend_node_id_for_node(+ child_id,+ )?));}}- payload.insert("childIds".to_owned(), json!(child_ids));- }- Some(Value::Object(payload))+ Some(child_ids)+ };+ Some(AccessibilityNode {+ node_id: AccessibilityNodeId::Dom(backend_node_id),+ backend_node_id: Some(backend_node_id),+ ignored,+ ignored_reasons,+ role: AccessibilityValue {+ kind: "role",+ value: if ignored { "none" } else { ax_role(node) },+ },+ name: (!name.is_empty()).then_some(AccessibilityValue {+ kind: "computedString",+ value: name,+ }),+ value,+ properties: Some(properties),+ parent_id,+ child_ids,+ })}fn ax_ignored_reasons_payload(@@ -362,7 +374,7 @@ fn ax_list_marker_payload(node_id: NodeId,node: &Node,backend_node_id_for_node: &mut dyn FnMut(NodeId) -> Option<u32>,-) -> Option<Value> {+) -> Option<AccessibilityNode> {let element = node.as_element()?;if !element.is_html_element("li") {return None;@@ -381,19 +393,22 @@ fn ax_list_marker_payload(};let parent_backend_node_id = backend_node_id_for_node(node_id)?;- Some(json!({- "nodeId": format!("AX-LM-{parent_backend_node_id}"),- "ignored": false,- "role": {- "type": "role",- "value": "ListMarker",- },- "name": {- "type": "computedString",- "value": marker_text,+ Some(AccessibilityNode {+ node_id: AccessibilityNodeId::ListMarker(parent_backend_node_id),+ backend_node_id: None,+ ignored: false,+ ignored_reasons: None,+ role: AccessibilityValue {+ kind: "role",+ value: "ListMarker",},- "properties": [],- "parentId": format!("AX-{parent_backend_node_id}"),- "childIds": [],- }))+ name: Some(AccessibilityValue {+ kind: "computedString",+ value: marker_text,+ }),+ value: None,+ properties: Some(Vec::new()),+ parent_id: Some(AccessibilityNodeId::Dom(parent_backend_node_id)),+ child_ids: Some(Vec::new()),+ })}diff --git a/moli-dom/src/accessibility/mod.rs b/moli-dom/src/accessibility/mod.rsindex fb8c7fb71..62718a012 100644--- a/moli-dom/src/accessibility/mod.rs+++ b/moli-dom/src/accessibility/mod.rs@@ -1,15 +1,19 @@mod ax_dom;+mod ax_node;mod ax_projection;mod ax_properties;mod ax_roles;mod ax_styles;mod ax_tree;+pub use ax_node::{AccessibilityNode, AccessibilityNodeId, AccessibilityValue};pub use ax_styles::{AccessibilityFrameState, AccessibilityInput, AccessibilityStyle, AccessibilityStyleSource,};-pub use ax_tree::{AccessibilityRequest, accessibility_payloads_for_document};+pub use ax_tree::{+ AccessibilityRequest, accessibility_nodes_for_document, accessibility_payloads_for_document,+};#[cfg(test)]mod tests;diff --git a/moli-renderer-v8/src/devtools/command/nested.rs b/moli-renderer-v8/src/devtools/command/nested.rsindex 54fedc25a..302776468 100644--- a/moli-renderer-v8/src/devtools/command/nested.rs+++ b/moli-renderer-v8/src/devtools/command/nested.rs@@ -58,6 +58,7 @@ impl RendererPageCommand {| Self::DocumentNodeAttributes { .. }| Self::DocumentNodeText { .. }| Self::DocumentNodeProperty { .. }+ | Self::AccessibilityTreeNodes { .. }| Self::AccessibilityTreePayloadsForDocument { .. }| Self::AccessibilityNodePayloadForDocument| Self::AccessibilityTreePayloadsForNode { .. }diff --git a/moli-renderer-v8/src/runtime/page_commands.rs b/moli-renderer-v8/src/runtime/page_commands.rsindex 6d6ee8968..956207395 100644--- a/moli-renderer-v8/src/runtime/page_commands.rs+++ b/moli-renderer-v8/src/runtime/page_commands.rs@@ -733,6 +733,11 @@ impl PageVm {} => Ok(RendererPageReply::DocumentNodePropertyResolution(self.document_node_property(reference, &name),)),+ RendererPageCommand::AccessibilityTreeNodes { frame_id } => Ok(+ RendererPageReply::OptionalAccessibilityNodes(+ self.accessibility_tree_nodes(frame_id.as_deref()),+ ),+ ),RendererPageCommand::AccessibilityTreePayloadsForDocument { max_depth } => Ok(RendererPageReply::OptionalAccessibilityPayloads(self.accessibility_tree_payloads_for_document(max_depth),diff --git a/moli-renderer-v8/src/runtime/page_dom.rs b/moli-renderer-v8/src/runtime/page_dom.rsindex 7441b734a..feef92393 100644--- a/moli-renderer-v8/src/runtime/page_dom.rs+++ b/moli-renderer-v8/src/runtime/page_dom.rs@@ -3109,6 +3109,23 @@ impl PageVm {self.document_node_property_for_live_handle(handle, name)}+ pub(crate) fn accessibility_tree_nodes(+ &self,+ frame_id: Option<&str>,+ ) -> Option<Vec<moli_dom::accessibility::AccessibilityNode>> {+ let document = match frame_id {+ Some(frame_id) => self+ .vm()+ .child_browsing_context_document_handle_by_frame_id(frame_id)?,+ None => self.vm().document_runtime.dom_host().document_node_id(),+ };+ self.vm().accessibility_nodes(+ document,+ AccessibilityRequest::Tree { max_depth: None },+ &mut |node_id| self.renderer_backend_node_id_for_live_handle(node_id),+ )+ }+fn build_accessibility_payloads(&self,handle: DomHandle,diff --git a/moli-renderer-v8/src/runtime/page_surface.rs b/moli-renderer-v8/src/runtime/page_surface.rsindex c0e760b03..2e3924a0d 100644--- a/moli-renderer-v8/src/runtime/page_surface.rs+++ b/moli-renderer-v8/src/runtime/page_surface.rs@@ -5066,6 +5066,9 @@ pub enum RendererPageCommand {reference: RendererDomNodeReference,name: String,},+ AccessibilityTreeNodes {+ frame_id: Option<String>,+ },AccessibilityTreePayloadsForDocument {max_depth: Option<i32>,},@@ -5705,6 +5708,7 @@ impl RendererPageCommand {Self::DocumentNodeAttributes { .. } => Some("DocumentNodeAttributes"),Self::DocumentNodeText { .. } => Some("DocumentNodeText"),Self::DocumentNodeProperty { .. } => Some("DocumentNodeProperty"),+ Self::AccessibilityTreeNodes { .. } => Some("AccessibilityTreeNodes"),Self::AccessibilityTreePayloadsForDocument { .. } => {Some("AccessibilityTreePayloadsForDocument")}@@ -6089,6 +6093,7 @@ pub enum RendererPageReply {DocumentNodeAttributesResolution(RendererDocumentNodeAttributesResolution),DocumentNodeTextResolution(RendererDocumentNodeTextResolution),DocumentNodePropertyResolution(RendererDocumentNodePropertyResolution),+ OptionalAccessibilityNodes(Option<Vec<moli_dom::accessibility::AccessibilityNode>>),OptionalAccessibilityPayloads(Option<Vec<Value>>),OptionalAccessibilityPayload(Option<Value>),OptionalAccessibilityPayloadsForObjectId(Option<RendererAccessibilityPayloadsForObjectId>),diff --git a/moli-renderer-v8/src/script_vm/layout.rs b/moli-renderer-v8/src/script_vm/layout.rsindex 62094cf42..8e584f553 100644--- a/moli-renderer-v8/src/script_vm/layout.rs+++ b/moli-renderer-v8/src/script_vm/layout.rs@@ -18,6 +18,42 @@ impl ScriptVm {request: moli_dom::accessibility::AccessibilityRequest,backend_node_id_for_node: &mut dyn FnMut(DomHandle) -> Option<u32>,) -> Option<Vec<serde_json::Value>> {+ self.with_accessibility_input(handle, |document, input| {+ moli_dom::accessibility::accessibility_payloads_for_document(+ document,+ input,+ handle,+ request,+ backend_node_id_for_node,+ )+ })+ }++ pub(crate) fn accessibility_nodes(+ &self,+ handle: DomHandle,+ request: moli_dom::accessibility::AccessibilityRequest,+ backend_node_id_for_node: &mut dyn FnMut(DomHandle) -> Option<u32>,+ ) -> Option<Vec<moli_dom::accessibility::AccessibilityNode>> {+ self.with_accessibility_input(handle, |document, input| {+ moli_dom::accessibility::accessibility_nodes_for_document(+ document,+ input,+ handle,+ request,+ backend_node_id_for_node,+ )+ })+ }++ fn with_accessibility_input<T>(+ &self,+ handle: DomHandle,+ read: impl FnOnce(+ &crate::dom::native::DomHost,+ &mut moli_dom::accessibility::AccessibilityInput<'_>,+ ) -> Option<T>,+ ) -> Option<T> {let host = self._context_host.borrow();let mut document = host.dom_host().owner_document_handle(handle)?;let mut frame_state = moli_dom::accessibility::AccessibilityFrameState::Active;@@ -31,13 +67,7 @@ impl ScriptVm {let mut observation = crate::native_bridge::element::StyleObservation::new(&host);let mut input =moli_dom::accessibility::AccessibilityInput::new(&mut observation, frame_state);- moli_dom::accessibility::accessibility_payloads_for_document(- host.dom_host(),- &mut input,- handle,- request,- backend_node_id_for_node,- )+ read(host.dom_host(), &mut input)}pub(crate) fn sync_live_document_style_sources(&mut self) {diff --git a/moli/src/fetch_dump/semantic.rs b/moli/src/fetch_dump/semantic.rsindex 623364013..d9eab948a 100644--- a/moli/src/fetch_dump/semantic.rs+++ b/moli/src/fetch_dump/semantic.rs@@ -1,8 +1,7 @@-use std::collections::HashMap;+use std::{collections::HashMap, fmt::Write};use anyhow::Result;-use moli_core::page::{ChildFrameTreeSnapshot, Page};-use serde_json::{Value, json};+use moli_core::page::{AccessibilityNode, AccessibilityNodeId, ChildFrameTreeSnapshot, Page};pub(super) async fn render_json(page: &mut Page, with_frames: bool) -> Result<String> {let payloads = collect_payloads(page, with_frames).await?;@@ -14,10 +13,11 @@ pub(super) async fn render_text(page: &mut Page, with_frames: bool) -> Result<StOk(render_payloads_text(&payloads))}-async fn collect_payloads(page: &mut Page, with_frames: bool) -> Result<Vec<Value>> {+async fn collect_payloads(page: &mut Page, with_frames: bool) -> Result<Vec<AccessibilityNode>> {let mut payloads = page- .accessibility_tree_payloads_for_document_async(None)- .await?;+ .accessibility_tree_nodes_async(None)+ .await?+ .unwrap_or_default();if !with_frames {return Ok(payloads);}@@ -33,9 +33,7 @@ async fn collect_payloads(page: &mut Page, with_frames: bool) -> Result<Vec<Valuelse {continue;};- let Some(child_payloads) = page- .child_frame_accessibility_tree_payloads_async(&frame_id, None)- .await?+ let Some(child_payloads) = page.accessibility_tree_nodes_async(Some(frame_id)).await?else {continue;};@@ -53,109 +51,73 @@ fn collect_child_frame_ids(frames: &[ChildFrameTreeSnapshot], frame_ids: &mut Ve}fn attach_child_frame_accessibility_tree(- payloads: &mut Vec<Value>,+ payloads: &mut Vec<AccessibilityNode>,owner_backend_node_id: u32,- mut child_payloads: Vec<Value>,+ mut child_payloads: Vec<AccessibilityNode>,) {- let Some(child_root_id) = child_payloads- .first()- .and_then(|payload| payload.get("nodeId"))- .and_then(Value::as_str)- .map(str::to_owned)- else {- return;- };- let Some(owner_index) = payloads.iter().position(|payload| {- payload.get("backendDOMNodeId").and_then(Value::as_u64)- == Some(u64::from(owner_backend_node_id))- }) else {+ let Some(child_root) = child_payloads.first_mut() else {return;};- let Some(owner_node_id) = payloads[owner_index]- .get("nodeId")- .and_then(Value::as_str)- .map(str::to_owned)+ let Some(owner) = payloads+ .iter_mut()+ .find(|node| node.backend_node_id == Some(owner_backend_node_id))else {return;};- let Some(child_root) = child_payloads.first_mut().and_then(Value::as_object_mut) else {- return;- };- child_root.insert("parentId".to_owned(), json!(owner_node_id));-- let Some(owner) = payloads[owner_index].as_object_mut() else {- return;- };- let child_ids = owner- .entry("childIds".to_owned())- .or_insert_with(|| Value::Array(Vec::new()));- let Some(child_ids) = child_ids.as_array_mut() else {- return;- };- if !child_ids.iter().any(|child_id| child_id == &child_root_id) {- child_ids.push(json!(child_root_id));+ child_root.parent_id = Some(owner.node_id);+ let child_ids = owner.child_ids.get_or_insert_with(Vec::new);+ if !child_ids.contains(&child_root.node_id) {+ child_ids.push(child_root.node_id);}payloads.append(&mut child_payloads);}-fn render_payloads_text(payloads: &[Value]) -> String {- if payloads.is_empty() {+fn render_payloads_text(payloads: &[AccessibilityNode]) -> String {+ let Some(root) = payloads.first() else {return String::new();- }-- let mut by_id = HashMap::new();- for payload in payloads {- if let Some(id) = payload.get("nodeId").and_then(Value::as_str) {- by_id.insert(id.to_owned(), payload);- }- }-- let mut out = String::new();- if let Some(root_id) = payloads[0].get("nodeId").and_then(Value::as_str) {- render_node_text(root_id, &by_id, 0, &mut out);- }- out.trim_end().to_owned()-}--fn render_node_text(- node_id: &str,- by_id: &HashMap<String, &Value>,- depth: usize,- out: &mut String,-) {- let Some(payload) = by_id.get(node_id) else {- return;};-- let role = payload["role"]["value"].as_str().unwrap_or("Unknown");- let name = payload["name"]["value"].as_str().unwrap_or_default();- let value = payload["value"]["value"].as_str().unwrap_or_default();- let backend = payload["backendDOMNodeId"].as_u64().unwrap_or(0);-- out.push_str(&" ".repeat(depth));- out.push_str("- ");- out.push_str(role);- if !name.is_empty() {- out.push_str(": ");- out.push_str(name);- }- if !value.is_empty() {- out.push_str(" = ");- out.push_str(value);- }- if backend != 0 {- out.push_str(&format!(" [backendNodeId={backend}]"));- }- out.push('\n');-- for child_id in payload["childIds"]- .as_array()- .into_iter()- .flatten()- .filter_map(Value::as_str)- {- render_node_text(child_id, by_id, depth + 1, out);+ let by_id: HashMap<AccessibilityNodeId, &AccessibilityNode> =+ payloads.iter().map(|node| (node.node_id, node)).collect();+ let mut out = String::new();+ let mut pending = vec![(root.node_id, 0)];+ while let Some((node_id, depth)) = pending.pop() {+ let Some(node) = by_id.get(&node_id) else {+ continue;+ };+ for _ in 0..depth {+ out.push_str(" ");+ }+ out.push_str("- ");+ out.push_str(node.role.value);+ if let Some(name) = &node.name+ && !name.value.is_empty()+ {+ out.push_str(": ");+ out.push_str(&name.value);+ }+ if let Some(value) = node+ .value+ .as_ref()+ .and_then(|value| value["value"].as_str())+ && !value.is_empty()+ {+ out.push_str(" = ");+ out.push_str(value);+ }+ if let Some(backend) = node.backend_node_id.filter(|id| *id != 0) {+ write!(&mut out, " [backendNodeId={backend}]").expect("writing to a String");+ }+ out.push('\n');+ pending.extend(+ node.child_ids+ .iter()+ .flatten()+ .rev()+ .map(|child| (*child, depth + 1)),+ );}+ out.truncate(out.trim_end().len());+ out}#[cfg(test)]@@ -189,64 +151,95 @@ mod tests {assert_eq!(frame_ids, ["first", "nested", "second"]);}+ fn node(+ id: u32,+ role: &'static str,+ parent: Option<u32>,+ children: &[u32],+ ) -> AccessibilityNode {+ AccessibilityNode {+ node_id: AccessibilityNodeId::Dom(id),+ backend_node_id: Some(id),+ ignored: false,+ ignored_reasons: None,+ role: moli_core::page::AccessibilityValue {+ kind: "role",+ value: role,+ },+ name: None,+ value: None,+ properties: Some(Vec::new()),+ parent_id: parent.map(AccessibilityNodeId::Dom),+ child_ids: Some(+ children+ .iter()+ .copied()+ .map(AccessibilityNodeId::Dom)+ .collect(),+ ),+ }+ }+#[test]fn child_tree_is_attached_to_its_iframe_owner() {let mut payloads = vec![- json!({- "nodeId": "AX-1",- "backendDOMNodeId": 1,- "role": { "value": "RootWebArea" },- "childIds": ["AX-2"]- }),- json!({- "nodeId": "AX-2",- "parentId": "AX-1",- "backendDOMNodeId": 2,- "role": { "value": "Iframe" }- }),- ];- let child_payloads = vec![- json!({- "nodeId": "AX-3",- "backendDOMNodeId": 3,- "role": { "value": "RootWebArea" },- "childIds": ["AX-4"]- }),- json!({- "nodeId": "AX-4",- "parentId": "AX-3",- "backendDOMNodeId": 4,- "role": { "value": "button" },- "name": { "value": "Child action" }- }),+ node(1, "RootWebArea", None, &[2]),+ node(2, "Iframe", Some(1), &[]),];+ let mut child_button = node(4, "button", Some(3), &[]);+ child_button.name = Some(moli_core::page::AccessibilityValue {+ kind: "computedString",+ value: "Child action".to_owned(),+ });+ let child_payloads = vec![node(3, "RootWebArea", None, &[4]), child_button];attach_child_frame_accessibility_tree(&mut payloads, 2, child_payloads);- assert_eq!(payloads[1]["childIds"], json!(["AX-3"]));- assert_eq!(payloads[2]["parentId"], "AX-2");+ assert_eq!(+ payloads[1].child_ids,+ Some(vec![AccessibilityNodeId::Dom(3)])+ );+ assert_eq!(payloads[2].parent_id, Some(AccessibilityNodeId::Dom(2)));assert!(render_payloads_text(&payloads).contains("button: Child action"));}#[test]fn child_tree_without_a_matching_owner_is_not_appended() {- let mut payloads = vec![json!({- "nodeId": "AX-1",- "backendDOMNodeId": 1,- "role": { "value": "RootWebArea" }- })];- let original = payloads.clone();-+ let mut payloads = vec![node(1, "RootWebArea", None, &[])];+ let original = serde_json::to_value(&payloads).expect("fixture JSON");attach_child_frame_accessibility_tree(&mut payloads,99,- vec![json!({- "nodeId": "AX-2",- "backendDOMNodeId": 2,- "role": { "value": "RootWebArea" }- })],+ vec![node(2, "RootWebArea", None, &[])],+ );+ assert_eq!(+ serde_json::to_value(payloads).expect("fixture JSON"),+ original);+ }- assert_eq!(payloads, original);+ #[test]+ fn text_preserves_child_order_and_value_rules_without_recursive_traversal() {+ const DEPTH: u32 = 2048;+ let mut nodes = vec![node(1, "RootWebArea", None, &[2, DEPTH + 1])];+ for id in 2..=DEPTH {+ let next = [id + 1];+ nodes.push(node(+ id,+ "generic",+ Some(id - 1),+ if id < DEPTH { &next } else { &[] },+ ));+ }+ let mut last = node(DEPTH + 1, "textbox", Some(1), &[]);+ last.name = Some(moli_core::page::AccessibilityValue {+ kind: "computedString",+ value: "Name".to_owned(),+ });+ last.value = Some(serde_json::json!({"type":"string", "value":"value"}));+ nodes.push(last);+ let text = render_payloads_text(&nodes);+ assert_eq!(text.lines().count(), (DEPTH + 1) as usize);+ assert!(text.ends_with(" - textbox: Name = value [backendNodeId=2049]"));}}diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rsindex b2f93f81c..0d9e2a6a6 100644--- a/moli/src/fetch_dump/tests.rs+++ b/moli/src/fetch_dump/tests.rs@@ -281,6 +281,72 @@ async fn render_semantic_tree_dump_uses_renderer_live_accessibility_tree() -> ReOk(())}+#[tokio::test]+async fn semantic_dump_preserves_protocol_json_bytes_and_text_fields() -> Result<()> {+ let (_browser, mut page, http_server) = load_page(+ r#"<!doctype html><html><head><title>AX serialization</title></head><body>+ <ul><li>List item</li></ul>+ <input aria-label="Text field" value="field value">+ <input type="range" aria-label="Numeric field" value="42">+ <button disabled>Disabled action</button>+ <div aria-hidden="true"><span>Ignored text</span></div>+ <div hidden>Not rendered</div>+ </body></html>"#,+ )+ .await?;+ let protocol = page+ .accessibility_tree_payloads_for_document_async(None)+ .await?;+ let json = super::semantic::render_json(&mut page, false).await?;+ assert_eq!(json, serde_json::to_string_pretty(&protocol)?);+ let nodes: Vec<serde_json::Value> = serde_json::from_str(&json)?;+ let field_names = |node: &serde_json::Value| {+ node.as_object()+ .expect("AX object")+ .keys()+ .cloned()+ .collect::<Vec<_>>()+ };+ assert_eq!(+ field_names(&nodes[0]),+ [+ "nodeId",+ "backendDOMNodeId",+ "ignored",+ "role",+ "name",+ "properties",+ "childIds",+ ]+ );+ let marker = nodes+ .iter()+ .find(|node| node["role"]["value"] == "ListMarker")+ .expect("list marker");+ assert_eq!(+ field_names(marker),+ [+ "nodeId",+ "ignored",+ "role",+ "name",+ "properties",+ "parentId",+ "childIds",+ ]+ );+ let text = super::semantic::render_text(&mut page, false).await?;+ assert!(text.contains("textbox: Text field = field value [backendNodeId="));+ assert!(text.contains("textbox: Numeric field = 42 [backendNodeId="));+ assert!(+ !text.contains("ListMarker"),+ "markers have no child reference in the protocol tree"+ );+ assert!(!text.contains("Not rendered"));+ http_server.abort();+ Ok(())+}+#[tokio::test]async fn render_semantic_tree_dumps_include_child_frames_only_when_requested() -> Result<()> {let (_browser, mut page, http_server) = load_page(@@ -318,6 +384,16 @@ async fn render_semantic_tree_dumps_include_child_frames_only_when_requested() -.filter(|payload| payload["role"]["value"] == "RootWebArea").nth(1).expect("child frame RootWebArea");+ assert_eq!(+ child_root+ .as_object()+ .expect("child root object")+ .keys()+ .next_back()+ .map(String::as_str),+ Some("parentId"),+ "frame attachment must preserve the existing JSON field order"+ );let child_root_id = child_root["nodeId"].as_str().expect("child root nodeId");let owner_id = child_root["parentId"].as_str()