Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
Expose image coordinates in untransformed root space
ldm0d0b3d4ef0a1c
diff --git a/moli-layout/src/layout_tree/model.rs b/moli-layout/src/layout_tree/model.rsindex 6689ae31f..6d520fb93 100644--- a/moli-layout/src/layout_tree/model.rs+++ b/moli-layout/src/layout_tree/model.rs@@ -255,6 +255,12 @@ pub(crate) struct LayoutCoordinateSpace {pub(crate) local_to_document: LayoutTransform2D,/// Maps local coordinates directly to viewport CSS pixels.pub(crate) local_to_viewport: LayoutTransform2D,+ /// Maps local coordinates to the visual document coordinate system while+ /// omitting authored CSS transforms. Layout placement, scrolling, CSS+ /// zoom, and fixed-position anchoring are retained.+ pub(crate) local_to_document_ignoring_css_transforms: LayoutTransform2D,+ /// The corresponding transform-free mapping to viewport CSS pixels.+ pub(crate) local_to_viewport_ignoring_css_transforms: LayoutTransform2D,}/// Query-facing coordinate data retained for one frozen box-tree node.@@ -262,6 +268,7 @@ pub(crate) struct LayoutCoordinateSpace {pub struct FrozenCoordinateSpace {pub owner: Option<LayoutOutputBoxId>,pub local_to_viewport: LayoutTransform2D,+ pub(crate) local_to_viewport_ignoring_css_transforms: LayoutTransform2D,}impl From<LayoutCoordinateSpace> for FrozenCoordinateSpace {@@ -269,6 +276,8 @@ impl From<LayoutCoordinateSpace> for FrozenCoordinateSpace {Self {owner: space.owner,local_to_viewport: space.local_to_viewport,+ local_to_viewport_ignoring_css_transforms: space+ .local_to_viewport_ignoring_css_transforms,}}}diff --git a/moli-layout/src/layout_tree/query.rs b/moli-layout/src/layout_tree/query.rsindex 1bb30ccdc..328db9da3 100644--- a/moli-layout/src/layout_tree/query.rs+++ b/moli-layout/src/layout_tree/query.rs@@ -41,6 +41,9 @@ pub struct LayoutDocumentMetrics {pub struct LayoutElementMetrics<N> {pub offset_parent: Option<N>,pub offset_position: LayoutPoint,+ /// Border-box origin in viewport CSS pixels after layout placement and+ /// scrolling, but before authored CSS transforms.+ pub border_origin_in_viewport_ignoring_css_transforms: LayoutPoint,pub offset_size: LayoutSize,pub content_size: LayoutSize,pub client_size: LayoutSize,diff --git a/moli-layout/src/layout_tree/source_query.rs b/moli-layout/src/layout_tree/source_query.rsindex 371fd36b2..0c34bad41 100644--- a/moli-layout/src/layout_tree/source_query.rs+++ b/moli-layout/src/layout_tree/source_query.rs@@ -109,30 +109,45 @@ where}}).unwrap_or(LayoutPoint::ZERO);- let layout_origin = output- .fragments- .iter()- .find_map(|id| {- let fragment = self.fragment(*id)?;- let LayoutFragmentKind::InlineBox {- box_id: fragment_box,- ..- } = fragment.kind- else {- return None;- };- if fragment_box != box_id {- return None;- }- let border = fragment.box_model?.border;- let owner = self.coordinate_space(fragment.coordinate_space)?.owner?;- let owner_origin = self.box_geometry(owner)?.layout_origin_in_document;- Some(LayoutPoint::new(- owner_origin.x + border.x,- owner_origin.y + border.y,- ))- })- .unwrap_or(geometry.layout_origin_in_document);+ let fragment_origins = output.fragments.iter().find_map(|id| {+ let fragment = self.fragment(*id)?;+ let LayoutFragmentKind::InlineBox {+ box_id: fragment_box,+ ..+ } = fragment.kind+ else {+ return None;+ };+ if fragment_box != box_id {+ return None;+ }+ let border = fragment.box_model?.border;+ let fragment_space = self.coordinate_space(fragment.coordinate_space)?;+ let owner = fragment_space.owner?;+ let owner_origin = self.box_geometry(owner)?.layout_origin_in_document;+ let local_border_origin = LayoutPoint::new(border.x, border.y);+ Some((+ LayoutPoint::new(+ owner_origin.x + local_border_origin.x,+ owner_origin.y + local_border_origin.y,+ ),+ fragment_space+ .local_to_viewport_ignoring_css_transforms+ .map_point(local_border_origin),+ ))+ });+ let (layout_origin, border_origin_in_viewport_ignoring_css_transforms) = fragment_origins+ .unwrap_or_else(|| {+ (+ geometry.layout_origin_in_document,+ coordinate_space+ .local_to_viewport_ignoring_css_transforms+ .map_point(LayoutPoint::new(+ geometry.border_box.x,+ geometry.border_box.y,+ )),+ )+ });let client_size = if is_root {// CSSOM defines root client dimensions from the viewport and only// subtracts an actually-present viewport scrollbar. Stable empty@@ -160,6 +175,7 @@ wherelayout_origin.x - offset_parent_origin.x,layout_origin.y - offset_parent_origin.y,),+ border_origin_in_viewport_ignoring_css_transforms,offset_size: LayoutSize::new(geometry.border_box.width, geometry.border_box.height),content_size: LayoutSize::new(geometry.content_box.width, geometry.content_box.height),client_size,diff --git a/moli-layout/src/projection.rs b/moli-layout/src/projection.rsindex 7c80e6c9d..cddddcd6c 100644--- a/moli-layout/src/projection.rs+++ b/moli-layout/src/projection.rs@@ -776,6 +776,8 @@ wherelocal_to_parent: LayoutTransform2D::IDENTITY,local_to_document: LayoutTransform2D::IDENTITY,local_to_viewport: LayoutTransform2D::IDENTITY,+ local_to_document_ignoring_css_transforms: LayoutTransform2D::IDENTITY,+ local_to_viewport_ignoring_css_transforms: LayoutTransform2D::IDENTITY,},paint: PaintSpace::ROOT,});@@ -1277,15 +1279,20 @@ wherelet parent_space = parent_box.map_or(LayoutCoordinateSpaceId::from_index(0), |parent| {LayoutCoordinateSpaceId::from_index(parent.index() + 1)});- let parent_transform = match parent_box {+ let (parent_transform, parent_transform_ignoring_css_transforms) = match parent_box {Some(parent) => spaces.get(parent.index() + 1).and_then(Option::as_ref)- .map(|space| space.layout.local_to_document)+ .map(|space| {+ (+ space.layout.local_to_document,+ space.layout.local_to_document_ignoring_css_transforms,+ )+ }).ok_or(LayoutError::InvalidBoxReference {index: parent.index(),})?,- None => LayoutTransform2D::IDENTITY,+ None => (LayoutTransform2D::IDENTITY, LayoutTransform2D::IDENTITY),};let is_viewport_anchored = parent_box.map_or_else(|| layout_box.style.is_fixed_positioned(),@@ -1303,12 +1310,24 @@ wherelocation.y - applied_parent_scroll.y,).concatenate(resolved_transforms[index].transform);+ let local_to_parent_ignoring_css_transforms = LayoutTransform2D::translation(+ location.x - applied_parent_scroll.x,+ location.y - applied_parent_scroll.y,+ );let local_to_document = if parent_box.is_none() && is_viewport_anchored {LayoutTransform2D::translation(viewport_scroll.x, viewport_scroll.y).concatenate(local_to_parent)} else {parent_transform.concatenate(local_to_parent)};+ let local_to_document_ignoring_css_transforms = if parent_box.is_none() && is_viewport_anchored+ {+ LayoutTransform2D::translation(viewport_scroll.x, viewport_scroll.y)+ .concatenate(local_to_parent_ignoring_css_transforms)+ } else {+ parent_transform_ignoring_css_transforms+ .concatenate(local_to_parent_ignoring_css_transforms)+ };let parent_paint_state = match parent_box {Some(parent) => Some(spaces@@ -1332,6 +1351,9 @@ wherelet exact_local_to_viewport =LayoutTransform2D::translation(-viewport_scroll.x, -viewport_scroll.y).concatenate(local_to_document);+ let local_to_viewport_ignoring_css_transforms =+ LayoutTransform2D::translation(-viewport_scroll.x, -viewport_scroll.y)+ .concatenate(local_to_document_ignoring_css_transforms);spaces[index + 1] = Some(ProjectedCoordinateSpace {layout: LayoutCoordinateSpace {id: LayoutCoordinateSpaceId::from_index(index + 1),@@ -1340,6 +1362,8 @@ wherelocal_to_parent,local_to_document,local_to_viewport: exact_local_to_viewport,+ local_to_document_ignoring_css_transforms,+ local_to_viewport_ignoring_css_transforms,},paint,});diff --git a/moli-layout/tests/phase5_output_contract.rs b/moli-layout/tests/phase5_output_contract.rsindex 82ee75ba3..8c71751c6 100644--- a/moli-layout/tests/phase5_output_contract.rs+++ b/moli-layout/tests/phase5_output_contract.rs@@ -1587,6 +1587,13 @@ fn scroll_is_sampled_per_pass_and_updates_geometry_clip_and_hit_testing() {.x,-40.0,);+ let first_child_metrics = first+ .element_metrics_for_source(2)+ .expect("the scrolled child has element metrics");+ assert_eq!(+ first_child_metrics.border_origin_in_viewport_ignoring_css_transforms,+ LayoutPoint::new(-40.0, -30.0)+ );assert_eq!(first.hit_test(LayoutPoint::new(10.0, 10.0), false)@@ -1669,6 +1676,14 @@ fn transforms_and_semantic_paint_order_share_the_hit_test_projection() {.bounding_rect(),LayoutRect::new(30.0, 25.0, 80.0, 80.0),);+ assert_eq!(+ output+ .element_metrics_for_source(2)+ .expect("the transformed box has element metrics")+ .border_origin_in_viewport_ignoring_css_transforms,+ LayoutPoint::new(20.0, 20.0),+ "the untransformed mapping must retain layout placement while skipping CSS transforms"+ );assert_eq!(output.hit_test(LayoutPoint::new(40.0, 40.0), false)@@ -1816,6 +1831,14 @@ fn viewport_fixed_geometry_does_not_move_with_root_scroll() {.bounding_rect();assert_close(fixed.x, 10.0);assert_close(fixed.y, 15.0);+ assert_eq!(+ output+ .element_metrics_for_source(2)+ .expect("the fixed box has element metrics")+ .border_origin_in_viewport_ignoring_css_transforms,+ LayoutPoint::new(10.0, 15.0),+ "the transform-free viewport mapping must preserve fixed positioning"+ );assert_close(output.box_model_for_source(1)diff --git a/moli-renderer-v8/src/native_bridge/element.rs b/moli-renderer-v8/src/native_bridge/element.rsindex 33d6f128a..c15381178 100644--- a/moli-renderer-v8/src/native_bridge/element.rs+++ b/moli-renderer-v8/src/native_bridge/element.rs@@ -512,6 +512,7 @@ pub(in crate::native_bridge) use images::{image_height_getter_function, image_height_setter_function, image_is_map_getter_function,image_is_map_setter_function, image_natural_height_getter_function,image_natural_width_getter_function, image_width_getter_function, image_width_setter_function,+ image_x_getter_function, image_y_getter_function,};pub(in crate::native_bridge) use media::queue_text_track_load_if_source;pub(crate) use media::{@@ -3605,6 +3606,10 @@ struct HtmlImageElementUrlPrototypeDeclaration {natural_width: (),#[webapi(accessor_property = "naturalHeight", getter = image_natural_height_getter_function)]natural_height: (),+ #[webapi(accessor_property, getter = image_x_getter_function)]+ x: (),+ #[webapi(accessor_property, getter = image_y_getter_function)]+ y: (),#[webapi(accessor_property = "isMap",getter = image_is_map_getter_function,diff --git a/moli-renderer-v8/src/native_bridge/element/geometry/mock/queries.rs b/moli-renderer-v8/src/native_bridge/element/geometry/mock/queries.rsindex 1c2b49d8f..578a6bc5a 100644--- a/moli-renderer-v8/src/native_bridge/element/geometry/mock/queries.rs+++ b/moli-renderer-v8/src/native_bridge/element/geometry/mock/queries.rs@@ -179,6 +179,10 @@ fn mock_element_metrics(Some(LayoutElementMetrics {offset_parent: compute_mock_offset_parent(runtime, source),offset_position: LayoutPoint::new(rect.left as f32, rect.top as f32),+ border_origin_in_viewport_ignoring_css_transforms: LayoutPoint::new(+ rect.left as f32,+ rect.top as f32,+ ),offset_size: size,content_size: size,client_size: size,diff --git a/moli-renderer-v8/src/native_bridge/element/images.rs b/moli-renderer-v8/src/native_bridge/element/images.rsindex 7087744b2..b78ed7b6f 100644--- a/moli-renderer-v8/src/native_bridge/element/images.rs+++ b/moli-renderer-v8/src/native_bridge/element/images.rs@@ -1,6 +1,7 @@mod attributes;mod decode;mod dimensions;+mod geometry;mod lazy;mod src;mod state;@@ -15,6 +16,7 @@ pub(in crate::native_bridge) use dimensions::{image_natural_height_getter_function, image_natural_width_getter_function,image_width_getter_function, image_width_setter_function,};+pub(in crate::native_bridge) use geometry::{image_x_getter_function, image_y_getter_function};pub(in crate::native_bridge) use src::image_current_src_getter_function;pub(crate) use src::{apply_authorized_image_load_event_in_context, apply_image_attribute_mutation_plan,diff --git a/moli-renderer-v8/src/native_bridge/element/images/geometry.rs b/moli-renderer-v8/src/native_bridge/element/images/geometry.rsnew file mode 100644index 000000000..0e54afbbc--- /dev/null+++ b/moli-renderer-v8/src/native_bridge/element/images/geometry.rs@@ -0,0 +1,84 @@+use super::super::{geometry::observable_element_metrics, html_element_getter_receiver};++#[derive(Clone, Copy)]+enum ImageCoordinateAxis {+ X,+ Y,+}++impl ImageCoordinateAxis {+ const fn member(self) -> &'static str {+ match self {+ Self::X => "x",+ Self::Y => "y",+ }+ }++ const fn select(self, point: moli_layout::LayoutPoint) -> f32 {+ match self {+ Self::X => point.x,+ Self::Y => point.y,+ }+ }+}++fn image_root_coordinate<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ object: v8::Local<'s, v8::Object>,+ axis: ImageCoordinateAxis,+) -> Result<i32, moli_layout::LayoutError> {+ let Some((runtime_ptr, handle)) =+ html_element_getter_receiver(scope, object, "HTMLImageElement", axis.member(), "img")+ else {+ return Ok(0);+ };+ let runtime = unsafe { &*runtime_ptr };++ let origin = observable_element_metrics(+ runtime,+ handle,+ moli_layout::LayoutFlushReason::SynchronousGeometry,+ )?+ .map(|metrics| metrics.border_origin_in_viewport_ignoring_css_transforms)+ .unwrap_or(moli_layout::LayoutPoint::ZERO);+ // Blink returns LayoutUnit::ToInt(), whose integer division truncates+ // toward zero. Rust's float-to-integer cast has the same truncation and+ // saturates at the WebIDL `long` bounds.+ Ok(axis.select(origin) as i32)+}++fn return_image_root_coordinate<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ object: v8::Local<'s, v8::Object>,+ axis: ImageCoordinateAxis,+ mut rv: v8::ReturnValue<'_, v8::Value>,+) {+ match image_root_coordinate(scope, object, axis) {+ Ok(value) => rv.set_int32(value),+ Err(error) => {+ let member = axis.member();+ let message = format!("Layout failed while reading HTMLImageElement.{member}: {error}");+ if let Some(message) = crate::util::v8_string(scope, &message) {+ let exception = v8::Exception::error(scope, message);+ scope.throw_exception(exception);+ }+ rv.set_int32(0);+ }+ }+}++pub(in crate::native_bridge) fn image_x_getter_function<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ args: v8::FunctionCallbackArguments<'s>,+ rv: v8::ReturnValue<'_, v8::Value>,+) {+ return_image_root_coordinate(scope, args.this(), ImageCoordinateAxis::X, rv);+}++pub(in crate::native_bridge) fn image_y_getter_function<'s>(+ scope: &mut v8::PinScope<'s, '_>,+ args: v8::FunctionCallbackArguments<'s>,+ rv: v8::ReturnValue<'_, v8::Value>,+) {+ return_image_root_coordinate(scope, args.this(), ImageCoordinateAxis::Y, rv);+}diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/image_root_coordinates.rs b/moli-renderer-v8/src/runtime/page_vm/tests/image_root_coordinates.rsnew file mode 100644index 000000000..c3e20647c--- /dev/null+++ b/moli-renderer-v8/src/runtime/page_vm/tests/image_root_coordinates.rs@@ -0,0 +1,125 @@+use super::*;++#[tokio::test(flavor = "current_thread")]+async fn screenshot_exposes_image_coordinates_in_untransformed_root_space() {+ run_page_vm_async_test(async move {+ let loader =+ crate::network::ResourceRequestClient::new(&FetchConfig::default()).expect("loader");+ let mut page_vm = test_page_vm_with_loader_and_document_url(+ &loader,+ Vec::new(),+ Url::parse("https://example.com/image-root-coordinates.html")?,+ );+ page_vm+ .vm_mut()+ .set_layout_policy(crate::real_layout_test_policy());+ page_vm.vm_mut().eval(+ r#"+document.head.innerHTML = `<style>+img{display:block;width:1px;height:1px}+.container{height:100px;width:100px}+#x4-zoom-container{zoom:4}+#ancestor-transform{position:absolute;left:220px;top:20px;transform:translate(40px,30px)}+#ancestor-transform img{width:10px;height:10px}+</style>`;+document.body.innerHTML = `+<div class=container>+ <img id=no-zoom>+ <img id=x2-zoom style="zoom:2">+</div>+<div class=container id=x4-zoom-container>+ <img id=x4-relative style="position:relative;top:10px">+ <img id=x8-effective style="zoom:2">+ <img id=transformed style="transform:scale(5)">+</div>+<div id=ancestor-transform><img id=ancestor-transformed></div>+<img id=negative style="position:absolute;left:-10.75px;top:-20.75px">+<img id=hidden style="display:none">+`;+'installed'+"#,+ )?;+ page_vm.vm_mut().sync_live_document_style_sources();+ page_vm+ .vm_mut()+ .screenshot_layout_snapshot(moli_layout::PaintViewport::new(480, 320, 1.0))?+ .expect("image-coordinate fixture must retain a layout root");++ let result = page_vm.vm_mut().eval(+ r#"JSON.stringify((()=>{+const ids=['no-zoom','x2-zoom','x4-relative','x8-effective','transformed','ancestor-transformed','negative'];+const geometry=Object.fromEntries(ids.map(id=>{+ const image=document.getElementById(id);+ const rect=image.getBoundingClientRect();+ return [id,[image.x,image.y,rect.left,rect.top]];+}));+const xDescriptor=Object.getOwnPropertyDescriptor(HTMLImageElement.prototype,'x');+const yDescriptor=Object.getOwnPropertyDescriptor(HTMLImageElement.prototype,'y');+const noZoom=document.getElementById('no-zoom');+let wrongReceiver;+try{xDescriptor.get.call(document.createElement('div'))}catch(error){wrongReceiver=error.name}+const detached=document.createElement('img');+return {+ geometry,+ hidden:[hidden.x,hidden.y],+ detached:[detached.x,detached.y],+ descriptors:[+ typeof xDescriptor.get,xDescriptor.get.name,xDescriptor.get.length,xDescriptor.set,+ xDescriptor.enumerable,xDescriptor.configurable,+ typeof yDescriptor.get,yDescriptor.get.name,yDescriptor.get.length,yDescriptor.set,+ yDescriptor.enumerable,yDescriptor.configurable,+ Object.hasOwn(noZoom,'x'),Object.hasOwn(noZoom,'y')+ ],+ wrongReceiver+};+})())"#,+ )?;+ let result: serde_json::Value = serde_json::from_str(&result)?;++ for (id, expected) in [+ ("no-zoom", [8, 8, 8, 8]),+ ("x2-zoom", [8, 9, 8, 9]),+ ("x4-relative", [8, 148, 8, 148]),+ ("x8-effective", [8, 112, 8, 112]),+ ] {+ assert_eq!(result["geometry"][id], serde_json::json!(expected), "{id}");+ }+ let transformed = result["geometry"]["transformed"]+ .as_array()+ .expect("transformed image geometry");+ assert_eq!(transformed[0], serde_json::json!(8));+ assert_eq!(transformed[1], serde_json::json!(120));+ assert_ne!(+ transformed[0], transformed[2],+ "image x ignores the image's own CSS transform"+ );+ assert_ne!(+ transformed[1], transformed[3],+ "image y ignores the image's own CSS transform"+ );+ assert_eq!(+ result["geometry"]["ancestor-transformed"],+ serde_json::json!([220, 20, 260, 50]),+ "only getBoundingClientRect includes the ancestor transform"+ );+ let negative = result["geometry"]["negative"]+ .as_array()+ .expect("negative image geometry");+ assert_eq!(negative[0], serde_json::json!(-10));+ assert_eq!(negative[1], serde_json::json!(-20));+ assert_eq!(result["hidden"], serde_json::json!([0, 0]));+ assert_eq!(result["detached"], serde_json::json!([0, 0]));+ assert_eq!(+ result["descriptors"],+ serde_json::json!([+ "function", "get x", 0, null, true, true,+ "function", "get y", 0, null, true, true,+ false, false+ ])+ );+ assert_eq!(result["wrongReceiver"], "TypeError");+ Ok::<_, anyhow::Error>(())+ })+ .await+ .expect("image root-coordinate fixture should run");+}diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/mod.rs b/moli-renderer-v8/src/runtime/page_vm/tests/mod.rsindex f40d7cb51..ad12b9ada 100644--- a/moli-renderer-v8/src/runtime/page_vm/tests/mod.rs+++ b/moli-renderer-v8/src/runtime/page_vm/tests/mod.rs@@ -125,6 +125,7 @@ mod file_system_directory_reader;mod hash_change_delivery;mod history_traversal;mod image_load_event;+mod image_root_coordinates;mod indexed_db;mod inline_svg_paint;mod internal_loading_completion;