Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

Expose image coordinates in untransformed root space

ldm0d0b3d4ef0a1c
diff --git a/moli-layout/src/layout_tree/model.rs b/moli-layout/src/layout_tree/model.rs
index 6689ae31f..6d520fb93 100644
--- a/moli-layout/src/layout_tree/model.rs
+++ b/moli-layout/src/layout_tree/model.rs
@@ -255,6 +255,12 @@ pub(crate) struct LayoutCoordinateSpace {
pub(crate) local_to_document: LayoutTransform2D,
/// Maps local coordinates directly to viewport CSS pixels.
pub(crate) local_to_viewport: LayoutTransform2D,
+ /// Maps local coordinates to the visual document coordinate system while
+ /// omitting authored CSS transforms. Layout placement, scrolling, CSS
+ /// zoom, and fixed-position anchoring are retained.
+ pub(crate) local_to_document_ignoring_css_transforms: LayoutTransform2D,
+ /// The corresponding transform-free mapping to viewport CSS pixels.
+ pub(crate) local_to_viewport_ignoring_css_transforms: LayoutTransform2D,
}
/// Query-facing coordinate data retained for one frozen box-tree node.
@@ -262,6 +268,7 @@ pub(crate) struct LayoutCoordinateSpace {
pub struct FrozenCoordinateSpace {
pub owner: Option<LayoutOutputBoxId>,
pub local_to_viewport: LayoutTransform2D,
+ pub(crate) local_to_viewport_ignoring_css_transforms: LayoutTransform2D,
}
impl From<LayoutCoordinateSpace> for FrozenCoordinateSpace {
@@ -269,6 +276,8 @@ impl From<LayoutCoordinateSpace> for FrozenCoordinateSpace {
Self {
owner: space.owner,
local_to_viewport: space.local_to_viewport,
+ local_to_viewport_ignoring_css_transforms: space
+ .local_to_viewport_ignoring_css_transforms,
}
}
}
diff --git a/moli-layout/src/layout_tree/query.rs b/moli-layout/src/layout_tree/query.rs
index 1bb30ccdc..328db9da3 100644
--- a/moli-layout/src/layout_tree/query.rs
+++ b/moli-layout/src/layout_tree/query.rs
@@ -41,6 +41,9 @@ pub struct LayoutDocumentMetrics {
pub struct LayoutElementMetrics<N> {
pub offset_parent: Option<N>,
pub offset_position: LayoutPoint,
+ /// Border-box origin in viewport CSS pixels after layout placement and
+ /// scrolling, but before authored CSS transforms.
+ pub border_origin_in_viewport_ignoring_css_transforms: LayoutPoint,
pub offset_size: LayoutSize,
pub content_size: LayoutSize,
pub client_size: LayoutSize,
diff --git a/moli-layout/src/layout_tree/source_query.rs b/moli-layout/src/layout_tree/source_query.rs
index 371fd36b2..0c34bad41 100644
--- a/moli-layout/src/layout_tree/source_query.rs
+++ b/moli-layout/src/layout_tree/source_query.rs
@@ -109,30 +109,45 @@ where
}
})
.unwrap_or(LayoutPoint::ZERO);
- let layout_origin = output
- .fragments
- .iter()
- .find_map(|id| {
- let fragment = self.fragment(*id)?;
- let LayoutFragmentKind::InlineBox {
- box_id: fragment_box,
- ..
- } = fragment.kind
- else {
- return None;
- };
- if fragment_box != box_id {
- return None;
- }
- let border = fragment.box_model?.border;
- let owner = self.coordinate_space(fragment.coordinate_space)?.owner?;
- let owner_origin = self.box_geometry(owner)?.layout_origin_in_document;
- Some(LayoutPoint::new(
- owner_origin.x + border.x,
- owner_origin.y + border.y,
- ))
- })
- .unwrap_or(geometry.layout_origin_in_document);
+ let fragment_origins = output.fragments.iter().find_map(|id| {
+ let fragment = self.fragment(*id)?;
+ let LayoutFragmentKind::InlineBox {
+ box_id: fragment_box,
+ ..
+ } = fragment.kind
+ else {
+ return None;
+ };
+ if fragment_box != box_id {
+ return None;
+ }
+ let border = fragment.box_model?.border;
+ let fragment_space = self.coordinate_space(fragment.coordinate_space)?;
+ let owner = fragment_space.owner?;
+ let owner_origin = self.box_geometry(owner)?.layout_origin_in_document;
+ let local_border_origin = LayoutPoint::new(border.x, border.y);
+ Some((
+ LayoutPoint::new(
+ owner_origin.x + local_border_origin.x,
+ owner_origin.y + local_border_origin.y,
+ ),
+ fragment_space
+ .local_to_viewport_ignoring_css_transforms
+ .map_point(local_border_origin),
+ ))
+ });
+ let (layout_origin, border_origin_in_viewport_ignoring_css_transforms) = fragment_origins
+ .unwrap_or_else(|| {
+ (
+ geometry.layout_origin_in_document,
+ coordinate_space
+ .local_to_viewport_ignoring_css_transforms
+ .map_point(LayoutPoint::new(
+ geometry.border_box.x,
+ geometry.border_box.y,
+ )),
+ )
+ });
let client_size = if is_root {
// CSSOM defines root client dimensions from the viewport and only
// subtracts an actually-present viewport scrollbar. Stable empty
@@ -160,6 +175,7 @@ where
layout_origin.x - offset_parent_origin.x,
layout_origin.y - offset_parent_origin.y,
),
+ border_origin_in_viewport_ignoring_css_transforms,
offset_size: LayoutSize::new(geometry.border_box.width, geometry.border_box.height),
content_size: LayoutSize::new(geometry.content_box.width, geometry.content_box.height),
client_size,
diff --git a/moli-layout/src/projection.rs b/moli-layout/src/projection.rs
index 7c80e6c9d..cddddcd6c 100644
--- a/moli-layout/src/projection.rs
+++ b/moli-layout/src/projection.rs
@@ -776,6 +776,8 @@ where
local_to_parent: LayoutTransform2D::IDENTITY,
local_to_document: LayoutTransform2D::IDENTITY,
local_to_viewport: LayoutTransform2D::IDENTITY,
+ local_to_document_ignoring_css_transforms: LayoutTransform2D::IDENTITY,
+ local_to_viewport_ignoring_css_transforms: LayoutTransform2D::IDENTITY,
},
paint: PaintSpace::ROOT,
});
@@ -1277,15 +1279,20 @@ where
let parent_space = parent_box.map_or(LayoutCoordinateSpaceId::from_index(0), |parent| {
LayoutCoordinateSpaceId::from_index(parent.index() + 1)
});
- let parent_transform = match parent_box {
+ let (parent_transform, parent_transform_ignoring_css_transforms) = match parent_box {
Some(parent) => spaces
.get(parent.index() + 1)
.and_then(Option::as_ref)
- .map(|space| space.layout.local_to_document)
+ .map(|space| {
+ (
+ space.layout.local_to_document,
+ space.layout.local_to_document_ignoring_css_transforms,
+ )
+ })
.ok_or(LayoutError::InvalidBoxReference {
index: parent.index(),
})?,
- None => LayoutTransform2D::IDENTITY,
+ None => (LayoutTransform2D::IDENTITY, LayoutTransform2D::IDENTITY),
};
let is_viewport_anchored = parent_box.map_or_else(
|| layout_box.style.is_fixed_positioned(),
@@ -1303,12 +1310,24 @@ where
location.y - applied_parent_scroll.y,
)
.concatenate(resolved_transforms[index].transform);
+ let local_to_parent_ignoring_css_transforms = LayoutTransform2D::translation(
+ location.x - applied_parent_scroll.x,
+ location.y - applied_parent_scroll.y,
+ );
let local_to_document = if parent_box.is_none() && is_viewport_anchored {
LayoutTransform2D::translation(viewport_scroll.x, viewport_scroll.y)
.concatenate(local_to_parent)
} else {
parent_transform.concatenate(local_to_parent)
};
+ let local_to_document_ignoring_css_transforms = if parent_box.is_none() && is_viewport_anchored
+ {
+ LayoutTransform2D::translation(viewport_scroll.x, viewport_scroll.y)
+ .concatenate(local_to_parent_ignoring_css_transforms)
+ } else {
+ parent_transform_ignoring_css_transforms
+ .concatenate(local_to_parent_ignoring_css_transforms)
+ };
let parent_paint_state = match parent_box {
Some(parent) => Some(
spaces
@@ -1332,6 +1351,9 @@ where
let exact_local_to_viewport =
LayoutTransform2D::translation(-viewport_scroll.x, -viewport_scroll.y)
.concatenate(local_to_document);
+ let local_to_viewport_ignoring_css_transforms =
+ LayoutTransform2D::translation(-viewport_scroll.x, -viewport_scroll.y)
+ .concatenate(local_to_document_ignoring_css_transforms);
spaces[index + 1] = Some(ProjectedCoordinateSpace {
layout: LayoutCoordinateSpace {
id: LayoutCoordinateSpaceId::from_index(index + 1),
@@ -1340,6 +1362,8 @@ where
local_to_parent,
local_to_document,
local_to_viewport: exact_local_to_viewport,
+ local_to_document_ignoring_css_transforms,
+ local_to_viewport_ignoring_css_transforms,
},
paint,
});
diff --git a/moli-layout/tests/phase5_output_contract.rs b/moli-layout/tests/phase5_output_contract.rs
index 82ee75ba3..8c71751c6 100644
--- a/moli-layout/tests/phase5_output_contract.rs
+++ b/moli-layout/tests/phase5_output_contract.rs
@@ -1587,6 +1587,13 @@ fn scroll_is_sampled_per_pass_and_updates_geometry_clip_and_hit_testing() {
.x,
-40.0,
);
+ let first_child_metrics = first
+ .element_metrics_for_source(2)
+ .expect("the scrolled child has element metrics");
+ assert_eq!(
+ first_child_metrics.border_origin_in_viewport_ignoring_css_transforms,
+ LayoutPoint::new(-40.0, -30.0)
+ );
assert_eq!(
first
.hit_test(LayoutPoint::new(10.0, 10.0), false)
@@ -1669,6 +1676,14 @@ fn transforms_and_semantic_paint_order_share_the_hit_test_projection() {
.bounding_rect(),
LayoutRect::new(30.0, 25.0, 80.0, 80.0),
);
+ assert_eq!(
+ output
+ .element_metrics_for_source(2)
+ .expect("the transformed box has element metrics")
+ .border_origin_in_viewport_ignoring_css_transforms,
+ LayoutPoint::new(20.0, 20.0),
+ "the untransformed mapping must retain layout placement while skipping CSS transforms"
+ );
assert_eq!(
output
.hit_test(LayoutPoint::new(40.0, 40.0), false)
@@ -1816,6 +1831,14 @@ fn viewport_fixed_geometry_does_not_move_with_root_scroll() {
.bounding_rect();
assert_close(fixed.x, 10.0);
assert_close(fixed.y, 15.0);
+ assert_eq!(
+ output
+ .element_metrics_for_source(2)
+ .expect("the fixed box has element metrics")
+ .border_origin_in_viewport_ignoring_css_transforms,
+ LayoutPoint::new(10.0, 15.0),
+ "the transform-free viewport mapping must preserve fixed positioning"
+ );
assert_close(
output
.box_model_for_source(1)
diff --git a/moli-renderer-v8/src/native_bridge/element.rs b/moli-renderer-v8/src/native_bridge/element.rs
index 33d6f128a..c15381178 100644
--- a/moli-renderer-v8/src/native_bridge/element.rs
+++ b/moli-renderer-v8/src/native_bridge/element.rs
@@ -512,6 +512,7 @@ pub(in crate::native_bridge) use images::{
image_height_getter_function, image_height_setter_function, image_is_map_getter_function,
image_is_map_setter_function, image_natural_height_getter_function,
image_natural_width_getter_function, image_width_getter_function, image_width_setter_function,
+ image_x_getter_function, image_y_getter_function,
};
pub(in crate::native_bridge) use media::queue_text_track_load_if_source;
pub(crate) use media::{
@@ -3605,6 +3606,10 @@ struct HtmlImageElementUrlPrototypeDeclaration {
natural_width: (),
#[webapi(accessor_property = "naturalHeight", getter = image_natural_height_getter_function)]
natural_height: (),
+ #[webapi(accessor_property, getter = image_x_getter_function)]
+ x: (),
+ #[webapi(accessor_property, getter = image_y_getter_function)]
+ y: (),
#[webapi(
accessor_property = "isMap",
getter = image_is_map_getter_function,
diff --git a/moli-renderer-v8/src/native_bridge/element/geometry/mock/queries.rs b/moli-renderer-v8/src/native_bridge/element/geometry/mock/queries.rs
index 1c2b49d8f..578a6bc5a 100644
--- a/moli-renderer-v8/src/native_bridge/element/geometry/mock/queries.rs
+++ b/moli-renderer-v8/src/native_bridge/element/geometry/mock/queries.rs
@@ -179,6 +179,10 @@ fn mock_element_metrics(
Some(LayoutElementMetrics {
offset_parent: compute_mock_offset_parent(runtime, source),
offset_position: LayoutPoint::new(rect.left as f32, rect.top as f32),
+ border_origin_in_viewport_ignoring_css_transforms: LayoutPoint::new(
+ rect.left as f32,
+ rect.top as f32,
+ ),
offset_size: size,
content_size: size,
client_size: size,
diff --git a/moli-renderer-v8/src/native_bridge/element/images.rs b/moli-renderer-v8/src/native_bridge/element/images.rs
index 7087744b2..b78ed7b6f 100644
--- a/moli-renderer-v8/src/native_bridge/element/images.rs
+++ b/moli-renderer-v8/src/native_bridge/element/images.rs
@@ -1,6 +1,7 @@
mod attributes;
mod decode;
mod dimensions;
+mod geometry;
mod lazy;
mod src;
mod state;
@@ -15,6 +16,7 @@ pub(in crate::native_bridge) use dimensions::{
image_natural_height_getter_function, image_natural_width_getter_function,
image_width_getter_function, image_width_setter_function,
};
+pub(in crate::native_bridge) use geometry::{image_x_getter_function, image_y_getter_function};
pub(in crate::native_bridge) use src::image_current_src_getter_function;
pub(crate) use src::{
apply_authorized_image_load_event_in_context, apply_image_attribute_mutation_plan,
diff --git a/moli-renderer-v8/src/native_bridge/element/images/geometry.rs b/moli-renderer-v8/src/native_bridge/element/images/geometry.rs
new file mode 100644
index 000000000..0e54afbbc
--- /dev/null
+++ b/moli-renderer-v8/src/native_bridge/element/images/geometry.rs
@@ -0,0 +1,84 @@
+use super::super::{geometry::observable_element_metrics, html_element_getter_receiver};
+
+#[derive(Clone, Copy)]
+enum ImageCoordinateAxis {
+ X,
+ Y,
+}
+
+impl ImageCoordinateAxis {
+ const fn member(self) -> &'static str {
+ match self {
+ Self::X => "x",
+ Self::Y => "y",
+ }
+ }
+
+ const fn select(self, point: moli_layout::LayoutPoint) -> f32 {
+ match self {
+ Self::X => point.x,
+ Self::Y => point.y,
+ }
+ }
+}
+
+fn image_root_coordinate<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ object: v8::Local<'s, v8::Object>,
+ axis: ImageCoordinateAxis,
+) -> Result<i32, moli_layout::LayoutError> {
+ let Some((runtime_ptr, handle)) =
+ html_element_getter_receiver(scope, object, "HTMLImageElement", axis.member(), "img")
+ else {
+ return Ok(0);
+ };
+ let runtime = unsafe { &*runtime_ptr };
+
+ let origin = observable_element_metrics(
+ runtime,
+ handle,
+ moli_layout::LayoutFlushReason::SynchronousGeometry,
+ )?
+ .map(|metrics| metrics.border_origin_in_viewport_ignoring_css_transforms)
+ .unwrap_or(moli_layout::LayoutPoint::ZERO);
+ // Blink returns LayoutUnit::ToInt(), whose integer division truncates
+ // toward zero. Rust's float-to-integer cast has the same truncation and
+ // saturates at the WebIDL `long` bounds.
+ Ok(axis.select(origin) as i32)
+}
+
+fn return_image_root_coordinate<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ object: v8::Local<'s, v8::Object>,
+ axis: ImageCoordinateAxis,
+ mut rv: v8::ReturnValue<'_, v8::Value>,
+) {
+ match image_root_coordinate(scope, object, axis) {
+ Ok(value) => rv.set_int32(value),
+ Err(error) => {
+ let member = axis.member();
+ let message = format!("Layout failed while reading HTMLImageElement.{member}: {error}");
+ if let Some(message) = crate::util::v8_string(scope, &message) {
+ let exception = v8::Exception::error(scope, message);
+ scope.throw_exception(exception);
+ }
+ rv.set_int32(0);
+ }
+ }
+}
+
+pub(in crate::native_bridge) fn image_x_getter_function<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ args: v8::FunctionCallbackArguments<'s>,
+ rv: v8::ReturnValue<'_, v8::Value>,
+) {
+ return_image_root_coordinate(scope, args.this(), ImageCoordinateAxis::X, rv);
+}
+
+pub(in crate::native_bridge) fn image_y_getter_function<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ args: v8::FunctionCallbackArguments<'s>,
+ rv: v8::ReturnValue<'_, v8::Value>,
+) {
+ return_image_root_coordinate(scope, args.this(), ImageCoordinateAxis::Y, rv);
+}
diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/image_root_coordinates.rs b/moli-renderer-v8/src/runtime/page_vm/tests/image_root_coordinates.rs
new file mode 100644
index 000000000..c3e20647c
--- /dev/null
+++ b/moli-renderer-v8/src/runtime/page_vm/tests/image_root_coordinates.rs
@@ -0,0 +1,125 @@
+use super::*;
+
+#[tokio::test(flavor = "current_thread")]
+async fn screenshot_exposes_image_coordinates_in_untransformed_root_space() {
+ run_page_vm_async_test(async move {
+ let loader =
+ crate::network::ResourceRequestClient::new(&FetchConfig::default()).expect("loader");
+ let mut page_vm = test_page_vm_with_loader_and_document_url(
+ &loader,
+ Vec::new(),
+ Url::parse("https://example.com/image-root-coordinates.html")?,
+ );
+ page_vm
+ .vm_mut()
+ .set_layout_policy(crate::real_layout_test_policy());
+ page_vm.vm_mut().eval(
+ r#"
+document.head.innerHTML = `<style>
+img{display:block;width:1px;height:1px}
+.container{height:100px;width:100px}
+#x4-zoom-container{zoom:4}
+#ancestor-transform{position:absolute;left:220px;top:20px;transform:translate(40px,30px)}
+#ancestor-transform img{width:10px;height:10px}
+</style>`;
+document.body.innerHTML = `
+<div class=container>
+ <img id=no-zoom>
+ <img id=x2-zoom style="zoom:2">
+</div>
+<div class=container id=x4-zoom-container>
+ <img id=x4-relative style="position:relative;top:10px">
+ <img id=x8-effective style="zoom:2">
+ <img id=transformed style="transform:scale(5)">
+</div>
+<div id=ancestor-transform><img id=ancestor-transformed></div>
+<img id=negative style="position:absolute;left:-10.75px;top:-20.75px">
+<img id=hidden style="display:none">
+`;
+'installed'
+"#,
+ )?;
+ page_vm.vm_mut().sync_live_document_style_sources();
+ page_vm
+ .vm_mut()
+ .screenshot_layout_snapshot(moli_layout::PaintViewport::new(480, 320, 1.0))?
+ .expect("image-coordinate fixture must retain a layout root");
+
+ let result = page_vm.vm_mut().eval(
+ r#"JSON.stringify((()=>{
+const ids=['no-zoom','x2-zoom','x4-relative','x8-effective','transformed','ancestor-transformed','negative'];
+const geometry=Object.fromEntries(ids.map(id=>{
+ const image=document.getElementById(id);
+ const rect=image.getBoundingClientRect();
+ return [id,[image.x,image.y,rect.left,rect.top]];
+}));
+const xDescriptor=Object.getOwnPropertyDescriptor(HTMLImageElement.prototype,'x');
+const yDescriptor=Object.getOwnPropertyDescriptor(HTMLImageElement.prototype,'y');
+const noZoom=document.getElementById('no-zoom');
+let wrongReceiver;
+try{xDescriptor.get.call(document.createElement('div'))}catch(error){wrongReceiver=error.name}
+const detached=document.createElement('img');
+return {
+ geometry,
+ hidden:[hidden.x,hidden.y],
+ detached:[detached.x,detached.y],
+ descriptors:[
+ typeof xDescriptor.get,xDescriptor.get.name,xDescriptor.get.length,xDescriptor.set,
+ xDescriptor.enumerable,xDescriptor.configurable,
+ typeof yDescriptor.get,yDescriptor.get.name,yDescriptor.get.length,yDescriptor.set,
+ yDescriptor.enumerable,yDescriptor.configurable,
+ Object.hasOwn(noZoom,'x'),Object.hasOwn(noZoom,'y')
+ ],
+ wrongReceiver
+};
+})())"#,
+ )?;
+ let result: serde_json::Value = serde_json::from_str(&result)?;
+
+ for (id, expected) in [
+ ("no-zoom", [8, 8, 8, 8]),
+ ("x2-zoom", [8, 9, 8, 9]),
+ ("x4-relative", [8, 148, 8, 148]),
+ ("x8-effective", [8, 112, 8, 112]),
+ ] {
+ assert_eq!(result["geometry"][id], serde_json::json!(expected), "{id}");
+ }
+ let transformed = result["geometry"]["transformed"]
+ .as_array()
+ .expect("transformed image geometry");
+ assert_eq!(transformed[0], serde_json::json!(8));
+ assert_eq!(transformed[1], serde_json::json!(120));
+ assert_ne!(
+ transformed[0], transformed[2],
+ "image x ignores the image's own CSS transform"
+ );
+ assert_ne!(
+ transformed[1], transformed[3],
+ "image y ignores the image's own CSS transform"
+ );
+ assert_eq!(
+ result["geometry"]["ancestor-transformed"],
+ serde_json::json!([220, 20, 260, 50]),
+ "only getBoundingClientRect includes the ancestor transform"
+ );
+ let negative = result["geometry"]["negative"]
+ .as_array()
+ .expect("negative image geometry");
+ assert_eq!(negative[0], serde_json::json!(-10));
+ assert_eq!(negative[1], serde_json::json!(-20));
+ assert_eq!(result["hidden"], serde_json::json!([0, 0]));
+ assert_eq!(result["detached"], serde_json::json!([0, 0]));
+ assert_eq!(
+ result["descriptors"],
+ serde_json::json!([
+ "function", "get x", 0, null, true, true,
+ "function", "get y", 0, null, true, true,
+ false, false
+ ])
+ );
+ assert_eq!(result["wrongReceiver"], "TypeError");
+ Ok::<_, anyhow::Error>(())
+ })
+ .await
+ .expect("image root-coordinate fixture should run");
+}
diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/mod.rs b/moli-renderer-v8/src/runtime/page_vm/tests/mod.rs
index f40d7cb51..ad12b9ada 100644
--- a/moli-renderer-v8/src/runtime/page_vm/tests/mod.rs
+++ b/moli-renderer-v8/src/runtime/page_vm/tests/mod.rs
@@ -125,6 +125,7 @@ mod file_system_directory_reader;
mod hash_change_delivery;
mod history_traversal;
mod image_load_event;
+mod image_root_coordinates;
mod indexed_db;
mod inline_svg_paint;
mod internal_loading_completion;