Skip to content

Lightpanda: the headless browser designed for AI and automation

fix: route native mouse input into hit iframe documents [skip ci]

zzwong8ef4541c6388
diff --git a/src/browser/frame/user_input.zig b/src/browser/frame/user_input.zig
index 0a0c23a19..6202e3fcf 100644
--- a/src/browser/frame/user_input.zig
+++ b/src/browser/frame/user_input.zig
@@ -59,6 +59,20 @@ const HoverContext = struct {
y: f64 = 0,
buttons: u16 = 0,
modifiers: Modifiers = .{},
+ root_point: ?Point = null,
+
+ fn freeze(self: HoverContext, frame: *Frame) HoverContext {
+ var ctx = self;
+ const origin = viewportOrigin(frame);
+ ctx.root_point = .{ .x = self.x + origin.x, .y = self.y + origin.y };
+ return ctx;
+ }
+
+ fn pointInOwner(self: HoverContext, frame: *Frame, owner: *Frame) Point {
+ const point = self.root_point orelse return pointInFrame(frame, owner, self.x, self.y);
+ const origin = viewportOrigin(owner);
+ return .{ .x = point.x - origin.x, .y = point.y - origin.y };
+ }
fn emit(ctx: HoverContext, frame: *Frame, target: *Element, comptime suffix: []const u8, related: ?*Element, bubbling: bool) void {
dispatchPointerBoundary(frame, target, "pointer" ++ suffix, related, bubbling, ctx);
@@ -71,6 +85,10 @@ const HoverContext = struct {
const CompatHover = struct {
ctx: HoverContext,
+ fn freeze(self: CompatHover, frame: *Frame) CompatHover {
+ return .{ .ctx = self.ctx.freeze(frame) };
+ }
+
fn emit(self: CompatHover, frame: *Frame, target: *Element, comptime suffix: []const u8, related: ?*Element, bubbling: bool) void {
dispatchMouseBoundary(frame, target, "mouse" ++ suffix, related, bubbling, self.ctx);
}
@@ -80,19 +98,72 @@ const CompatHover = struct {
// hovered item so that trasitions can fire the correct events. e.g. mouseout
// bubbles up normally, but mouseleave will only fire on parents where the new
// target isn't part of.
-pub fn updateHoverTarget(frame: *Frame, to: ?*Element, emitter: anytype) void {
+pub fn updateHoverTarget(frame: *Frame, to: ?*Element, input_emitter: anytype) void {
+ // Keep the physical point stable if a boundary listener moves an iframe.
+ const emitter = input_emitter.freeze(frame);
const page = frame.page;
const from = page.input_hover_target;
if (from == to) {
return;
}
page.input_hover_target = to;
- dispatchBoundaryEvents(frame, from, to, emitter);
+ const from_owner = if (from) |el| el.ownerFrame(frame) else null;
+ const to_owner = if (to) |el| el.ownerFrame(frame) else null;
+ if (from_owner == to_owner) {
+ dispatchBoundaryEvents(frame, from, to, emitter);
+ return;
+ }
+
+ // Each document has its own hover path. In an ancestor document, a
+ // descendant's target is represented by the iframe containing it.
+ var common = from_owner;
+ while (common) |candidate| : (common = candidate.parent) {
+ var current = to_owner;
+ while (current) |owner| : (current = owner.parent) {
+ if (owner == candidate) break;
+ }
+ if (current != null) break;
+ }
+
+ var old = from;
+ var current = from_owner;
+ while (current != common) {
+ const owner = current orelse break;
+ dispatchBoundaryEvents(frame, old, null, emitter);
+ old = if (owner.iframe) |iframe| iframe.asElement() else null;
+ current = owner.parent;
+ }
+
+ var new = to;
+ current = to_owner;
+ var depth: usize = 0;
+ while (current != common) {
+ const owner = current orelse break;
+ new = if (owner.iframe) |iframe| iframe.asElement() else null;
+ current = owner.parent;
+ depth += 1;
+ }
+ if (common != null) dispatchBoundaryEvents(frame, old, new, emitter);
+
+ // Enter the new browsing contexts outermost first. A context crossed in
+ // either direction has no relatedTarget from the other document.
+ while (depth > 0) : (depth -= 1) {
+ new = to;
+ current = to_owner;
+ var remaining = depth - 1;
+ while (remaining > 0) : (remaining -= 1) {
+ const owner = current orelse break;
+ new = if (owner.iframe) |iframe| iframe.asElement() else null;
+ current = owner.parent;
+ }
+ dispatchBoundaryEvents(frame, null, new, emitter);
+ }
}
/// `emitter.emit` fires one boundary event, named by its suffix ("out",
/// "leave", "over", "enter").
fn dispatchBoundaryEvents(frame: *Frame, from: ?*Element, to: ?*Element, emitter: anytype) void {
+ if (from == to) return;
const pivot: ?*Node = blk: {
const a = from orelse break :blk null;
const b = to orelse break :blk null;
@@ -144,14 +215,20 @@ fn dispatchBoundaryEvents(frame: *Frame, from: ?*Element, to: ?*Element, emitter
}
fn dispatchPointerBoundary(frame: *Frame, target: *Element, comptime pointer_typ: []const u8, related: ?*Element, bubbling: bool, ctx: HoverContext) void {
+ const owner = target.ownerFrame(frame) orelse return;
+ if (owner.window._closed) return;
+ const point = ctx.pointInOwner(frame, owner);
const modifiers = ctx.modifiers;
- const related_target = if (related) |r| r.asEventTarget() else null;
+ const related_target = if (related) |r|
+ if (r.ownerFrame(frame) == owner) r.asEventTarget() else null
+ else
+ null;
const pointer_event = PointerEvent.initTrusted(pointer_typ, .{
.bubbles = bubbling,
.cancelable = bubbling,
.composed = bubbling,
- .clientX = ctx.x,
- .clientY = ctx.y,
+ .clientX = point.x,
+ .clientY = point.y,
// No button changed state: https://www.w3.org/TR/pointerevents3/#the-button-property
.button = -1,
.buttons = ctx.buttons,
@@ -163,35 +240,41 @@ fn dispatchPointerBoundary(frame: *Frame, target: *Element, comptime pointer_typ
.shiftKey = modifiers.shift,
.altKey = modifiers.alt,
.metaKey = modifiers.meta,
- }, frame) catch |err| {
+ }, owner) catch |err| {
log.debug(.frame, "boundary pointer event", .{ .err = err, .type = pointer_typ });
return;
};
- frame._event_manager.dispatch(target.asEventTarget(), pointer_event.asEvent()) catch |err| {
+ owner._event_manager.dispatch(target.asEventTarget(), pointer_event.asEvent()) catch |err| {
log.debug(.frame, "boundary pointer dispatch", .{ .err = err, .type = pointer_typ });
};
}
fn dispatchMouseBoundary(frame: *Frame, target: *Element, comptime mouse_typ: []const u8, related: ?*Element, bubbling: bool, ctx: HoverContext) void {
+ const owner = target.ownerFrame(frame) orelse return;
+ if (owner.window._closed) return;
+ const point = ctx.pointInOwner(frame, owner);
const modifiers = ctx.modifiers;
- const related_target = if (related) |r| r.asEventTarget() else null;
+ const related_target = if (related) |r|
+ if (r.ownerFrame(frame) == owner) r.asEventTarget() else null
+ else
+ null;
const mouse_event = MouseEvent.initTrusted(comptime .wrap(mouse_typ), .{
.bubbles = bubbling,
.cancelable = bubbling,
.composed = bubbling,
- .clientX = ctx.x,
- .clientY = ctx.y,
+ .clientX = point.x,
+ .clientY = point.y,
.buttons = ctx.buttons,
.relatedTarget = related_target,
.ctrlKey = modifiers.ctrl,
.shiftKey = modifiers.shift,
.altKey = modifiers.alt,
.metaKey = modifiers.meta,
- }, frame) catch |err| {
+ }, owner) catch |err| {
log.debug(.frame, "boundary mouse event", .{ .err = err, .type = mouse_typ });
return;
};
- frame._event_manager.dispatch(target.asEventTarget(), mouse_event.asEvent()) catch |err| {
+ owner._event_manager.dispatch(target.asEventTarget(), mouse_event.asEvent()) catch |err| {
log.debug(.frame, "boundary mouse dispatch", .{ .err = err, .type = mouse_typ });
};
}
@@ -219,6 +302,7 @@ fn emitPointer(
detail: u32,
) !bool {
const owner = target.ownerFrame(frame) orelse frame;
+ if (owner.window._closed) return false;
const boundary = comptime std.mem.eql(u8, typ, "pointerenter") or std.mem.eql(u8, typ, "pointerleave");
const event: *PointerEvent = try .initTrusted(typ, .{
.bubbles = !boundary,
@@ -252,6 +336,7 @@ fn emitMouse(
detail: u32,
) !bool {
const owner = target.ownerFrame(frame) orelse frame;
+ if (owner.window._closed) return false;
const event: *MouseEvent = try .initTrusted(comptime .wrap(typ), .{
.bubbles = true,
.cancelable = true,
@@ -306,7 +391,8 @@ pub const PointerButtons = struct {
var pg = g;
pg.buttons_down = self.held;
const suppressed = try pressSequence(frame, target, pg, self.mousedown_suppressed);
- if (starts_gesture) {
+ // A listener may have discarded the target's frame and reset us.
+ if (starts_gesture and self.down_target == target) {
self.mousedown_suppressed = suppressed;
}
}
@@ -357,12 +443,12 @@ fn commonAncestor(a: *Node, b: *Node) ?*Node {
/// A click whose mousedown and mouseup landed on different elements fires at
/// their nearest common inclusive ancestor element.
-fn commonClickTarget(down: *Element, up: *Element) *Element {
+fn commonClickTarget(down: *Element, up: *Element) ?*Element {
if (down == up) {
return up;
}
- const ancestor = commonAncestor(down.asNode(), up.asNode()) orelse return up;
- return ancestor.is(Element) orelse up;
+ const ancestor = commonAncestor(down.asNode(), up.asNode()) orelse return null;
+ return ancestor.is(Element);
}
pub fn moveSequence(frame: *Frame, target: *Element, g: Gesture) !void {
@@ -386,6 +472,8 @@ fn pressSequence(frame: *Frame, target: *Element, g: Gesture, chord_suppressed:
_ = try emitPointer(frame, target, "pointermove", g, 0);
break :blk chord_suppressed;
};
+ // A pointerdown listener may remove the iframe containing the target.
+ if ((target.ownerFrame(frame) orelse frame).window._closed) return suppressed;
if (!suppressed) {
try mouseDown(frame, target, g);
}
@@ -401,6 +489,7 @@ fn pressSequence(frame: *Frame, target: *Element, g: Gesture, chord_suppressed:
/// no mouse events, click or focus, as in Chrome.
fn mouseDown(frame: *Frame, target: *Element, g: Gesture) !void {
if (!target.isDisabled() and !try emitMouse(frame, target, "mousedown", g, g.click_count)) {
+ if ((target.ownerFrame(frame) orelse frame).window._closed) return;
focusForMouseDown(frame, target) catch |err| log.debug(.app, "mousedown focus", .{ .err = err });
}
}
@@ -454,35 +543,92 @@ pub fn triggerClick(frame: *Frame, target: *Element, modifiers: Modifiers) !void
_ = try pointer.release(frame, target, g);
}
+const MouseHit = struct {
+ frame: *Frame,
+ target: *Element,
+ x: f64,
+ y: f64,
+};
+
+/// Input coordinates start in the caller's viewport. Unlike the DOM API,
+/// native hit testing descends into the hit iframe's document.
+fn mouseHitTest(frame: *Frame, x: f64, y: f64) !?MouseHit {
+ if (frame.window._closed) return null;
+ var hit: MouseHit = .{
+ .frame = frame,
+ .target = (try frame.document.elementFromPoint(x, y, frame)) orelse return null,
+ .x = x,
+ .y = y,
+ };
+ while (hit.target.is(Element.Html.IFrame)) |iframe| {
+ const window = iframe._window orelse break;
+ if (window._closed) break;
+ const child = window._frame;
+ const rect = hit.target.boundingClientRectValues(hit.frame);
+ const child_x = hit.x - rect.x - hit.target.getClientLeft();
+ const child_y = hit.y - rect.y - hit.target.getClientTop();
+ const target = (try child.document.elementFromPoint(child_x, child_y, child)) orelse
+ child.document.getDocumentElement() orelse break;
+ hit = .{ .frame = child, .target = target, .x = child_x, .y = child_y };
+ }
+ return hit;
+}
+
+const Point = struct { x: f64, y: f64 };
+
+// The existing faux layout has no transform scaling and client borders are
+// zero. Use the same rects as hit testing, including each parent's scroll.
+fn viewportOrigin(frame: *Frame) Point {
+ var point: Point = .{ .x = 0, .y = 0 };
+ var current = frame;
+ while (current.parent) |parent| {
+ const element = (current.iframe orelse break).asElement();
+ const rect = element.boundingClientRectValues(parent);
+ point.x += rect.x + element.getClientLeft();
+ point.y += rect.y + element.getClientTop();
+ current = parent;
+ }
+ return point;
+}
+
+fn pointInFrame(from: *Frame, to: *Frame, x: f64, y: f64) Point {
+ if (from == to) return .{ .x = x, .y = y };
+ const from_origin = viewportOrigin(from);
+ const to_origin = viewportOrigin(to);
+ return .{ .x = x + from_origin.x - to_origin.x, .y = y + from_origin.y - to_origin.y };
+}
+
pub fn triggerMousePress(frame: *Frame, x: f64, y: f64, button: i32, click_count: i32) !void {
- const target = (try frame.window._document.elementFromPoint(x, y, frame)) orelse {
+ const hit = (try mouseHitTest(frame, x, y)) orelse {
// Don't leave a prior gesture's state for the next message to misread.
frame.page.input_pointer.reset();
return;
};
+ const target = hit.target;
if (comptime lp.IS_DEBUG) {
log.debug(.frame, "frame mouse press", .{
.url = frame.url,
.node = target,
- .x = x,
- .y = y,
+ .x = hit.x,
+ .y = hit.y,
.button = button,
.type = frame._type,
});
}
- try frame.page.input_pointer.press(frame, target, .{
+ try frame.page.input_pointer.press(hit.frame, target, .{
.button = button,
// clickCount 0 (omitted) stays 0, not forced to 1: Chrome and Firefox
// both fire mousedown with detail 0 in that case.
.click_count = if (click_count > 0) @intCast(click_count) else 0,
- .x = x,
- .y = y,
+ .x = hit.x,
+ .y = hit.y,
});
}
pub fn triggerMouseMove(frame: *Frame, x: f64, y: f64) !void {
- const target = (try frame.window._document.elementFromPoint(x, y, frame)) orelse return;
+ const hit = (try mouseHitTest(frame, x, y)) orelse return;
+ const target = hit.target;
if (comptime lp.IS_DEBUG) {
log.debug(.frame, "frame mouse move", .{
.url = frame.url,
@@ -493,44 +639,50 @@ pub fn triggerMouseMove(frame: *Frame, x: f64, y: f64) !void {
});
}
- try moveSequence(frame, target, .{ .buttons_down = frame.page.input_pointer.held, .x = x, .y = y });
+ try moveSequence(hit.frame, target, .{ .buttons_down = frame.page.input_pointer.held, .x = hit.x, .y = hit.y });
}
pub fn triggerMouseRelease(frame: *Frame, x: f64, y: f64, button: i32, click_count: i32) !void {
const pointer = &frame.page.input_pointer;
// A release that misses every element still lets go of the button, so
// the next message doesn't read it as held.
- const hit = frame.window._document.elementFromPoint(x, y, frame) catch |err| {
+ const result = mouseHitTest(frame, x, y) catch |err| {
pointer.releaseButton(button);
return err;
};
- const target = hit orelse {
+ const hit = result orelse {
pointer.releaseButton(button);
return;
};
+ const target = hit.target;
if (comptime lp.IS_DEBUG) {
log.debug(.frame, "frame mouse release", .{
.url = frame.url,
.node = target,
- .x = x,
- .y = y,
+ .x = hit.x,
+ .y = hit.y,
.button = button,
.type = frame._type,
});
}
- _ = try pointer.release(frame, target, .{
+ _ = try pointer.release(hit.frame, target, .{
.button = button,
.click_count = if (click_count > 0) @intCast(click_count) else 0,
- .x = x,
- .y = y,
+ .x = hit.x,
+ .y = hit.y,
});
}
pub fn triggerMouseWheel(frame: *Frame, x: f64, y: f64, delta_x: f64, delta_y: f64) !void {
const document = frame.window._document;
- const target = (try document.elementFromPoint(x, y, frame)) orelse
- document.getDocumentElement() orelse return;
+ const hit = (try mouseHitTest(frame, x, y)) orelse MouseHit{
+ .frame = frame,
+ .target = document.getDocumentElement() orelse return,
+ .x = x,
+ .y = y,
+ };
+ const target = hit.target;
if (comptime lp.IS_DEBUG) {
log.debug(.frame, "frame mouse wheel", .{
.url = frame.url,
@@ -543,7 +695,7 @@ pub fn triggerMouseWheel(frame: *Frame, x: f64, y: f64, delta_x: f64, delta_y: f
});
}
- try wheel(frame, target, x, y, delta_x, delta_y);
+ try wheel(hit.frame, target, hit.x, hit.y, delta_x, delta_y);
}
/// A wheel over `target`: a trusted `wheel`, then the scroll unless it was
@@ -554,6 +706,7 @@ pub fn wheel(frame: *Frame, target: *Element, x: f64, y: f64, delta_x: f64, delt
// Listeners live in the event manager of the element's own frame, which
// is not the caller's when the element belongs to an iframe's document.
const owner = target.ownerFrame(frame) orelse return;
+ if (owner.window._closed) return;
const event: *WheelEvent = try .initTrusted("wheel", .{
.bubbles = true,
@@ -568,6 +721,9 @@ pub fn wheel(frame: *Frame, target: *Element, x: f64, y: f64, delta_x: f64, delt
return;
}
+ // A wheel listener may remove the iframe before the default action.
+ if (owner.window._closed) return;
+
// Deltas come from the wire, so guard NaN and saturate the addition.
try scrollAxis(target, .width, deltaToScroll(delta_x), owner);
try scrollAxis(target, .height, deltaToScroll(delta_y), owner);
@@ -1761,3 +1917,90 @@ test "user_input: dispatchTouchEvent a closed-shadow touch target retargets to t
const result = try ls.local.compileAndRun("window.eventTargetOK === true && window.touchTargetOK === true", null);
try testing.expect(result.isTrue());
}
+
+test "user_input: mouse hit testing descends nested iframe viewports" {
+ const page = try testing.pageTest("input_mouse_frames.html", .{});
+ defer page.close();
+ const main = page.frame().?;
+ const child = main.child_frames.items[0];
+ const nested = child.child_frames.items[0];
+ const target = nested.document.getElementById("inner", nested).?;
+ const rect = target.boundingClientRectValues(nested);
+ const child_rect = child.iframe.?.asElement().boundingClientRectValues(main);
+ const nested_rect = nested.iframe.?.asElement().boundingClientRectValues(child);
+ const x = rect.x + nested_rect.x + child_rect.x + 1;
+ const y = rect.y + nested_rect.y + child_rect.y + 1;
+
+ // The DOM API intentionally returns the iframe in the calling document.
+ try testing.expect((try main.document.elementFromPoint(x, y, main)) == child.iframe.?.asElement());
+ var ls: js.Local.Scope = undefined;
+ nested.js.localScope(&ls);
+ defer ls.deinit();
+ _ = try ls.local.compileAndRun(
+ \\window.events = [];
+ \\const target = document.getElementById('inner');
+ \\for (const type of ['pointerover', 'mouseover', 'pointermove', 'mousemove', 'pointerdown', 'mousedown', 'pointerup', 'mouseup', 'click']) {
+ \\ target.addEventListener(type, e => events.push({type, x:e.clientX, y:e.clientY, trusted:e.isTrusted}));
+ \\}
+ , null);
+ try triggerMouseMove(main, x, y);
+ try triggerMousePress(main, x, y, mouse_button.main, 1);
+ try triggerMouseRelease(main, x, y, mouse_button.main, 1);
+ try testing.expect(focusedFrame(main) == nested);
+ try insertText(focusedFrame(main), "typed");
+ try testing.expect((try ls.local.compileAndRun(
+ \\target.value === 'typed' &&
+ \\events.map(e => e.type).join(',') === 'pointerover,mouseover,pointermove,mousemove,pointerdown,mousedown,pointerup,mouseup,click' &&
+ \\events.every(e => e.trusted && e.x === target.getBoundingClientRect().x + 1 && e.y === target.getBoundingClientRect().y + 1)
+ , null)).isTrue());
+}
+
+test "user_input: mouse release in another document does not click" {
+ const page = try testing.pageTest("input_mouse_frames.html", .{});
+ defer page.close();
+ const main = page.frame().?;
+ const child = main.child_frames.items[0];
+ const nested = child.child_frames.items[0];
+ const inner = nested.document.getElementById("inner", nested).?;
+ const outer = main.document.getElementById("outer", main).?;
+ const inner_rect = inner.boundingClientRectValues(nested);
+ const child_rect = child.iframe.?.asElement().boundingClientRectValues(main);
+ const nested_rect = nested.iframe.?.asElement().boundingClientRectValues(child);
+ const outer_rect = outer.boundingClientRectValues(main);
+ var ls: js.Local.Scope = undefined;
+ main.js.localScope(&ls);
+ defer ls.deinit();
+ _ = try ls.local.compileAndRun("window.clicks = 0; document.addEventListener('click', () => clicks++);", null);
+ try triggerMousePress(main, inner_rect.x + nested_rect.x + child_rect.x + 1, inner_rect.y + nested_rect.y + child_rect.y + 1, mouse_button.main, 1);
+ try testing.expect(main.page.input_pointer.down_target == inner);
+ try triggerMouseRelease(main, outer_rect.x + 1, outer_rect.y + 1, mouse_button.main, 1);
+ try testing.expect((try ls.local.compileAndRun("clicks === 0", null)).isTrue());
+ try testing.expectEqual(0, main.page.input_pointer.held);
+ try testing.expect(main.page.input_pointer.down_target == null);
+}
+
+test "user_input: removing hit iframe in pointerdown stops its mouse sequence" {
+ const page = try testing.pageTest("input_mouse_frames.html", .{});
+ defer page.close();
+ const main = page.frame().?;
+ const child = main.child_frames.items[0];
+ const target = child.document.getElementById("middle", child).?;
+ const rect = target.boundingClientRectValues(child);
+ const iframe_rect = child.iframe.?.asElement().boundingClientRectValues(main);
+ var ls: js.Local.Scope = undefined;
+ main.js.localScope(&ls);
+ defer ls.deinit();
+ _ = try ls.local.compileAndRun(
+ \\window.events = [];
+ \\const childInput = document.getElementById('child').contentDocument.getElementById('middle');
+ \\childInput.addEventListener('pointerdown', e => { e.preventDefault(); events.push('pointerdown'); document.getElementById('child').remove(); });
+ \\for (const type of ['mousedown', 'focus', 'mouseup', 'click']) childInput.addEventListener(type, () => events.push(type));
+ , null);
+ try testing.expect((try main.document.elementFromPoint(rect.x + iframe_rect.x + 1, rect.y + iframe_rect.y + 1, main)) == child.iframe.?.asElement());
+ try triggerMousePress(main, rect.x + iframe_rect.x + 1, rect.y + iframe_rect.y + 1, mouse_button.main, 1);
+ try testing.expect((try ls.local.compileAndRun("events.join(',') === 'pointerdown'", null)).isTrue());
+ try testing.expect(child.window._closed);
+ try testing.expect(main.page.input_pointer.down_target == null);
+ try testing.expectEqual(0, main.page.input_pointer.held);
+ try testing.expect(!main.page.input_pointer.mousedown_suppressed);
+}
diff --git a/src/browser/tests/input_mouse_frames.html b/src/browser/tests/input_mouse_frames.html
new file mode 100644
index 000000000..d823c100f
--- /dev/null
+++ b/src/browser/tests/input_mouse_frames.html
@@ -0,0 +1,3 @@
+<!doctype html>
+<input id="outer" style="width: 80px; height: 10px">
+<iframe id="child" style="width: 300px; height: 300px" srcdoc="<!doctype html><input id=middle style='width:80px;height:10px'><iframe id=nested style='width:200px;height:200px' srcdoc='<!doctype html><input id=inner style=width:80px;height:10px>'></iframe>"></iframe>