Lightpanda: the headless browser designed for AI and automation
fix: route native mouse input into hit iframe documents [skip ci]
zzwong8ef4541c6388
diff --git a/src/browser/frame/user_input.zig b/src/browser/frame/user_input.zigindex 0a0c23a19..6202e3fcf 100644--- a/src/browser/frame/user_input.zig+++ b/src/browser/frame/user_input.zig@@ -59,6 +59,20 @@ const HoverContext = struct {y: f64 = 0,buttons: u16 = 0,modifiers: Modifiers = .{},+ root_point: ?Point = null,++ fn freeze(self: HoverContext, frame: *Frame) HoverContext {+ var ctx = self;+ const origin = viewportOrigin(frame);+ ctx.root_point = .{ .x = self.x + origin.x, .y = self.y + origin.y };+ return ctx;+ }++ fn pointInOwner(self: HoverContext, frame: *Frame, owner: *Frame) Point {+ const point = self.root_point orelse return pointInFrame(frame, owner, self.x, self.y);+ const origin = viewportOrigin(owner);+ return .{ .x = point.x - origin.x, .y = point.y - origin.y };+ }fn emit(ctx: HoverContext, frame: *Frame, target: *Element, comptime suffix: []const u8, related: ?*Element, bubbling: bool) void {dispatchPointerBoundary(frame, target, "pointer" ++ suffix, related, bubbling, ctx);@@ -71,6 +85,10 @@ const HoverContext = struct {const CompatHover = struct {ctx: HoverContext,+ fn freeze(self: CompatHover, frame: *Frame) CompatHover {+ return .{ .ctx = self.ctx.freeze(frame) };+ }+fn emit(self: CompatHover, frame: *Frame, target: *Element, comptime suffix: []const u8, related: ?*Element, bubbling: bool) void {dispatchMouseBoundary(frame, target, "mouse" ++ suffix, related, bubbling, self.ctx);}@@ -80,19 +98,72 @@ const CompatHover = struct {// hovered item so that trasitions can fire the correct events. e.g. mouseout// bubbles up normally, but mouseleave will only fire on parents where the new// target isn't part of.-pub fn updateHoverTarget(frame: *Frame, to: ?*Element, emitter: anytype) void {+pub fn updateHoverTarget(frame: *Frame, to: ?*Element, input_emitter: anytype) void {+ // Keep the physical point stable if a boundary listener moves an iframe.+ const emitter = input_emitter.freeze(frame);const page = frame.page;const from = page.input_hover_target;if (from == to) {return;}page.input_hover_target = to;- dispatchBoundaryEvents(frame, from, to, emitter);+ const from_owner = if (from) |el| el.ownerFrame(frame) else null;+ const to_owner = if (to) |el| el.ownerFrame(frame) else null;+ if (from_owner == to_owner) {+ dispatchBoundaryEvents(frame, from, to, emitter);+ return;+ }++ // Each document has its own hover path. In an ancestor document, a+ // descendant's target is represented by the iframe containing it.+ var common = from_owner;+ while (common) |candidate| : (common = candidate.parent) {+ var current = to_owner;+ while (current) |owner| : (current = owner.parent) {+ if (owner == candidate) break;+ }+ if (current != null) break;+ }++ var old = from;+ var current = from_owner;+ while (current != common) {+ const owner = current orelse break;+ dispatchBoundaryEvents(frame, old, null, emitter);+ old = if (owner.iframe) |iframe| iframe.asElement() else null;+ current = owner.parent;+ }++ var new = to;+ current = to_owner;+ var depth: usize = 0;+ while (current != common) {+ const owner = current orelse break;+ new = if (owner.iframe) |iframe| iframe.asElement() else null;+ current = owner.parent;+ depth += 1;+ }+ if (common != null) dispatchBoundaryEvents(frame, old, new, emitter);++ // Enter the new browsing contexts outermost first. A context crossed in+ // either direction has no relatedTarget from the other document.+ while (depth > 0) : (depth -= 1) {+ new = to;+ current = to_owner;+ var remaining = depth - 1;+ while (remaining > 0) : (remaining -= 1) {+ const owner = current orelse break;+ new = if (owner.iframe) |iframe| iframe.asElement() else null;+ current = owner.parent;+ }+ dispatchBoundaryEvents(frame, null, new, emitter);+ }}/// `emitter.emit` fires one boundary event, named by its suffix ("out",/// "leave", "over", "enter").fn dispatchBoundaryEvents(frame: *Frame, from: ?*Element, to: ?*Element, emitter: anytype) void {+ if (from == to) return;const pivot: ?*Node = blk: {const a = from orelse break :blk null;const b = to orelse break :blk null;@@ -144,14 +215,20 @@ fn dispatchBoundaryEvents(frame: *Frame, from: ?*Element, to: ?*Element, emitter}fn dispatchPointerBoundary(frame: *Frame, target: *Element, comptime pointer_typ: []const u8, related: ?*Element, bubbling: bool, ctx: HoverContext) void {+ const owner = target.ownerFrame(frame) orelse return;+ if (owner.window._closed) return;+ const point = ctx.pointInOwner(frame, owner);const modifiers = ctx.modifiers;- const related_target = if (related) |r| r.asEventTarget() else null;+ const related_target = if (related) |r|+ if (r.ownerFrame(frame) == owner) r.asEventTarget() else null+ else+ null;const pointer_event = PointerEvent.initTrusted(pointer_typ, .{.bubbles = bubbling,.cancelable = bubbling,.composed = bubbling,- .clientX = ctx.x,- .clientY = ctx.y,+ .clientX = point.x,+ .clientY = point.y,// No button changed state: https://www.w3.org/TR/pointerevents3/#the-button-property.button = -1,.buttons = ctx.buttons,@@ -163,35 +240,41 @@ fn dispatchPointerBoundary(frame: *Frame, target: *Element, comptime pointer_typ.shiftKey = modifiers.shift,.altKey = modifiers.alt,.metaKey = modifiers.meta,- }, frame) catch |err| {+ }, owner) catch |err| {log.debug(.frame, "boundary pointer event", .{ .err = err, .type = pointer_typ });return;};- frame._event_manager.dispatch(target.asEventTarget(), pointer_event.asEvent()) catch |err| {+ owner._event_manager.dispatch(target.asEventTarget(), pointer_event.asEvent()) catch |err| {log.debug(.frame, "boundary pointer dispatch", .{ .err = err, .type = pointer_typ });};}fn dispatchMouseBoundary(frame: *Frame, target: *Element, comptime mouse_typ: []const u8, related: ?*Element, bubbling: bool, ctx: HoverContext) void {+ const owner = target.ownerFrame(frame) orelse return;+ if (owner.window._closed) return;+ const point = ctx.pointInOwner(frame, owner);const modifiers = ctx.modifiers;- const related_target = if (related) |r| r.asEventTarget() else null;+ const related_target = if (related) |r|+ if (r.ownerFrame(frame) == owner) r.asEventTarget() else null+ else+ null;const mouse_event = MouseEvent.initTrusted(comptime .wrap(mouse_typ), .{.bubbles = bubbling,.cancelable = bubbling,.composed = bubbling,- .clientX = ctx.x,- .clientY = ctx.y,+ .clientX = point.x,+ .clientY = point.y,.buttons = ctx.buttons,.relatedTarget = related_target,.ctrlKey = modifiers.ctrl,.shiftKey = modifiers.shift,.altKey = modifiers.alt,.metaKey = modifiers.meta,- }, frame) catch |err| {+ }, owner) catch |err| {log.debug(.frame, "boundary mouse event", .{ .err = err, .type = mouse_typ });return;};- frame._event_manager.dispatch(target.asEventTarget(), mouse_event.asEvent()) catch |err| {+ owner._event_manager.dispatch(target.asEventTarget(), mouse_event.asEvent()) catch |err| {log.debug(.frame, "boundary mouse dispatch", .{ .err = err, .type = mouse_typ });};}@@ -219,6 +302,7 @@ fn emitPointer(detail: u32,) !bool {const owner = target.ownerFrame(frame) orelse frame;+ if (owner.window._closed) return false;const boundary = comptime std.mem.eql(u8, typ, "pointerenter") or std.mem.eql(u8, typ, "pointerleave");const event: *PointerEvent = try .initTrusted(typ, .{.bubbles = !boundary,@@ -252,6 +336,7 @@ fn emitMouse(detail: u32,) !bool {const owner = target.ownerFrame(frame) orelse frame;+ if (owner.window._closed) return false;const event: *MouseEvent = try .initTrusted(comptime .wrap(typ), .{.bubbles = true,.cancelable = true,@@ -306,7 +391,8 @@ pub const PointerButtons = struct {var pg = g;pg.buttons_down = self.held;const suppressed = try pressSequence(frame, target, pg, self.mousedown_suppressed);- if (starts_gesture) {+ // A listener may have discarded the target's frame and reset us.+ if (starts_gesture and self.down_target == target) {self.mousedown_suppressed = suppressed;}}@@ -357,12 +443,12 @@ fn commonAncestor(a: *Node, b: *Node) ?*Node {/// A click whose mousedown and mouseup landed on different elements fires at/// their nearest common inclusive ancestor element.-fn commonClickTarget(down: *Element, up: *Element) *Element {+fn commonClickTarget(down: *Element, up: *Element) ?*Element {if (down == up) {return up;}- const ancestor = commonAncestor(down.asNode(), up.asNode()) orelse return up;- return ancestor.is(Element) orelse up;+ const ancestor = commonAncestor(down.asNode(), up.asNode()) orelse return null;+ return ancestor.is(Element);}pub fn moveSequence(frame: *Frame, target: *Element, g: Gesture) !void {@@ -386,6 +472,8 @@ fn pressSequence(frame: *Frame, target: *Element, g: Gesture, chord_suppressed:_ = try emitPointer(frame, target, "pointermove", g, 0);break :blk chord_suppressed;};+ // A pointerdown listener may remove the iframe containing the target.+ if ((target.ownerFrame(frame) orelse frame).window._closed) return suppressed;if (!suppressed) {try mouseDown(frame, target, g);}@@ -401,6 +489,7 @@ fn pressSequence(frame: *Frame, target: *Element, g: Gesture, chord_suppressed:/// no mouse events, click or focus, as in Chrome.fn mouseDown(frame: *Frame, target: *Element, g: Gesture) !void {if (!target.isDisabled() and !try emitMouse(frame, target, "mousedown", g, g.click_count)) {+ if ((target.ownerFrame(frame) orelse frame).window._closed) return;focusForMouseDown(frame, target) catch |err| log.debug(.app, "mousedown focus", .{ .err = err });}}@@ -454,35 +543,92 @@ pub fn triggerClick(frame: *Frame, target: *Element, modifiers: Modifiers) !void_ = try pointer.release(frame, target, g);}+const MouseHit = struct {+ frame: *Frame,+ target: *Element,+ x: f64,+ y: f64,+};++/// Input coordinates start in the caller's viewport. Unlike the DOM API,+/// native hit testing descends into the hit iframe's document.+fn mouseHitTest(frame: *Frame, x: f64, y: f64) !?MouseHit {+ if (frame.window._closed) return null;+ var hit: MouseHit = .{+ .frame = frame,+ .target = (try frame.document.elementFromPoint(x, y, frame)) orelse return null,+ .x = x,+ .y = y,+ };+ while (hit.target.is(Element.Html.IFrame)) |iframe| {+ const window = iframe._window orelse break;+ if (window._closed) break;+ const child = window._frame;+ const rect = hit.target.boundingClientRectValues(hit.frame);+ const child_x = hit.x - rect.x - hit.target.getClientLeft();+ const child_y = hit.y - rect.y - hit.target.getClientTop();+ const target = (try child.document.elementFromPoint(child_x, child_y, child)) orelse+ child.document.getDocumentElement() orelse break;+ hit = .{ .frame = child, .target = target, .x = child_x, .y = child_y };+ }+ return hit;+}++const Point = struct { x: f64, y: f64 };++// The existing faux layout has no transform scaling and client borders are+// zero. Use the same rects as hit testing, including each parent's scroll.+fn viewportOrigin(frame: *Frame) Point {+ var point: Point = .{ .x = 0, .y = 0 };+ var current = frame;+ while (current.parent) |parent| {+ const element = (current.iframe orelse break).asElement();+ const rect = element.boundingClientRectValues(parent);+ point.x += rect.x + element.getClientLeft();+ point.y += rect.y + element.getClientTop();+ current = parent;+ }+ return point;+}++fn pointInFrame(from: *Frame, to: *Frame, x: f64, y: f64) Point {+ if (from == to) return .{ .x = x, .y = y };+ const from_origin = viewportOrigin(from);+ const to_origin = viewportOrigin(to);+ return .{ .x = x + from_origin.x - to_origin.x, .y = y + from_origin.y - to_origin.y };+}+pub fn triggerMousePress(frame: *Frame, x: f64, y: f64, button: i32, click_count: i32) !void {- const target = (try frame.window._document.elementFromPoint(x, y, frame)) orelse {+ const hit = (try mouseHitTest(frame, x, y)) orelse {// Don't leave a prior gesture's state for the next message to misread.frame.page.input_pointer.reset();return;};+ const target = hit.target;if (comptime lp.IS_DEBUG) {log.debug(.frame, "frame mouse press", .{.url = frame.url,.node = target,- .x = x,- .y = y,+ .x = hit.x,+ .y = hit.y,.button = button,.type = frame._type,});}- try frame.page.input_pointer.press(frame, target, .{+ try frame.page.input_pointer.press(hit.frame, target, .{.button = button,// clickCount 0 (omitted) stays 0, not forced to 1: Chrome and Firefox// both fire mousedown with detail 0 in that case..click_count = if (click_count > 0) @intCast(click_count) else 0,- .x = x,- .y = y,+ .x = hit.x,+ .y = hit.y,});}pub fn triggerMouseMove(frame: *Frame, x: f64, y: f64) !void {- const target = (try frame.window._document.elementFromPoint(x, y, frame)) orelse return;+ const hit = (try mouseHitTest(frame, x, y)) orelse return;+ const target = hit.target;if (comptime lp.IS_DEBUG) {log.debug(.frame, "frame mouse move", .{.url = frame.url,@@ -493,44 +639,50 @@ pub fn triggerMouseMove(frame: *Frame, x: f64, y: f64) !void {});}- try moveSequence(frame, target, .{ .buttons_down = frame.page.input_pointer.held, .x = x, .y = y });+ try moveSequence(hit.frame, target, .{ .buttons_down = frame.page.input_pointer.held, .x = hit.x, .y = hit.y });}pub fn triggerMouseRelease(frame: *Frame, x: f64, y: f64, button: i32, click_count: i32) !void {const pointer = &frame.page.input_pointer;// A release that misses every element still lets go of the button, so// the next message doesn't read it as held.- const hit = frame.window._document.elementFromPoint(x, y, frame) catch |err| {+ const result = mouseHitTest(frame, x, y) catch |err| {pointer.releaseButton(button);return err;};- const target = hit orelse {+ const hit = result orelse {pointer.releaseButton(button);return;};+ const target = hit.target;if (comptime lp.IS_DEBUG) {log.debug(.frame, "frame mouse release", .{.url = frame.url,.node = target,- .x = x,- .y = y,+ .x = hit.x,+ .y = hit.y,.button = button,.type = frame._type,});}- _ = try pointer.release(frame, target, .{+ _ = try pointer.release(hit.frame, target, .{.button = button,.click_count = if (click_count > 0) @intCast(click_count) else 0,- .x = x,- .y = y,+ .x = hit.x,+ .y = hit.y,});}pub fn triggerMouseWheel(frame: *Frame, x: f64, y: f64, delta_x: f64, delta_y: f64) !void {const document = frame.window._document;- const target = (try document.elementFromPoint(x, y, frame)) orelse- document.getDocumentElement() orelse return;+ const hit = (try mouseHitTest(frame, x, y)) orelse MouseHit{+ .frame = frame,+ .target = document.getDocumentElement() orelse return,+ .x = x,+ .y = y,+ };+ const target = hit.target;if (comptime lp.IS_DEBUG) {log.debug(.frame, "frame mouse wheel", .{.url = frame.url,@@ -543,7 +695,7 @@ pub fn triggerMouseWheel(frame: *Frame, x: f64, y: f64, delta_x: f64, delta_y: f});}- try wheel(frame, target, x, y, delta_x, delta_y);+ try wheel(hit.frame, target, hit.x, hit.y, delta_x, delta_y);}/// A wheel over `target`: a trusted `wheel`, then the scroll unless it was@@ -554,6 +706,7 @@ pub fn wheel(frame: *Frame, target: *Element, x: f64, y: f64, delta_x: f64, delt// Listeners live in the event manager of the element's own frame, which// is not the caller's when the element belongs to an iframe's document.const owner = target.ownerFrame(frame) orelse return;+ if (owner.window._closed) return;const event: *WheelEvent = try .initTrusted("wheel", .{.bubbles = true,@@ -568,6 +721,9 @@ pub fn wheel(frame: *Frame, target: *Element, x: f64, y: f64, delta_x: f64, deltreturn;}+ // A wheel listener may remove the iframe before the default action.+ if (owner.window._closed) return;+// Deltas come from the wire, so guard NaN and saturate the addition.try scrollAxis(target, .width, deltaToScroll(delta_x), owner);try scrollAxis(target, .height, deltaToScroll(delta_y), owner);@@ -1761,3 +1917,90 @@ test "user_input: dispatchTouchEvent a closed-shadow touch target retargets to tconst result = try ls.local.compileAndRun("window.eventTargetOK === true && window.touchTargetOK === true", null);try testing.expect(result.isTrue());}++test "user_input: mouse hit testing descends nested iframe viewports" {+ const page = try testing.pageTest("input_mouse_frames.html", .{});+ defer page.close();+ const main = page.frame().?;+ const child = main.child_frames.items[0];+ const nested = child.child_frames.items[0];+ const target = nested.document.getElementById("inner", nested).?;+ const rect = target.boundingClientRectValues(nested);+ const child_rect = child.iframe.?.asElement().boundingClientRectValues(main);+ const nested_rect = nested.iframe.?.asElement().boundingClientRectValues(child);+ const x = rect.x + nested_rect.x + child_rect.x + 1;+ const y = rect.y + nested_rect.y + child_rect.y + 1;++ // The DOM API intentionally returns the iframe in the calling document.+ try testing.expect((try main.document.elementFromPoint(x, y, main)) == child.iframe.?.asElement());+ var ls: js.Local.Scope = undefined;+ nested.js.localScope(&ls);+ defer ls.deinit();+ _ = try ls.local.compileAndRun(+ \\window.events = [];+ \\const target = document.getElementById('inner');+ \\for (const type of ['pointerover', 'mouseover', 'pointermove', 'mousemove', 'pointerdown', 'mousedown', 'pointerup', 'mouseup', 'click']) {+ \\ target.addEventListener(type, e => events.push({type, x:e.clientX, y:e.clientY, trusted:e.isTrusted}));+ \\}+ , null);+ try triggerMouseMove(main, x, y);+ try triggerMousePress(main, x, y, mouse_button.main, 1);+ try triggerMouseRelease(main, x, y, mouse_button.main, 1);+ try testing.expect(focusedFrame(main) == nested);+ try insertText(focusedFrame(main), "typed");+ try testing.expect((try ls.local.compileAndRun(+ \\target.value === 'typed' &&+ \\events.map(e => e.type).join(',') === 'pointerover,mouseover,pointermove,mousemove,pointerdown,mousedown,pointerup,mouseup,click' &&+ \\events.every(e => e.trusted && e.x === target.getBoundingClientRect().x + 1 && e.y === target.getBoundingClientRect().y + 1)+ , null)).isTrue());+}++test "user_input: mouse release in another document does not click" {+ const page = try testing.pageTest("input_mouse_frames.html", .{});+ defer page.close();+ const main = page.frame().?;+ const child = main.child_frames.items[0];+ const nested = child.child_frames.items[0];+ const inner = nested.document.getElementById("inner", nested).?;+ const outer = main.document.getElementById("outer", main).?;+ const inner_rect = inner.boundingClientRectValues(nested);+ const child_rect = child.iframe.?.asElement().boundingClientRectValues(main);+ const nested_rect = nested.iframe.?.asElement().boundingClientRectValues(child);+ const outer_rect = outer.boundingClientRectValues(main);+ var ls: js.Local.Scope = undefined;+ main.js.localScope(&ls);+ defer ls.deinit();+ _ = try ls.local.compileAndRun("window.clicks = 0; document.addEventListener('click', () => clicks++);", null);+ try triggerMousePress(main, inner_rect.x + nested_rect.x + child_rect.x + 1, inner_rect.y + nested_rect.y + child_rect.y + 1, mouse_button.main, 1);+ try testing.expect(main.page.input_pointer.down_target == inner);+ try triggerMouseRelease(main, outer_rect.x + 1, outer_rect.y + 1, mouse_button.main, 1);+ try testing.expect((try ls.local.compileAndRun("clicks === 0", null)).isTrue());+ try testing.expectEqual(0, main.page.input_pointer.held);+ try testing.expect(main.page.input_pointer.down_target == null);+}++test "user_input: removing hit iframe in pointerdown stops its mouse sequence" {+ const page = try testing.pageTest("input_mouse_frames.html", .{});+ defer page.close();+ const main = page.frame().?;+ const child = main.child_frames.items[0];+ const target = child.document.getElementById("middle", child).?;+ const rect = target.boundingClientRectValues(child);+ const iframe_rect = child.iframe.?.asElement().boundingClientRectValues(main);+ var ls: js.Local.Scope = undefined;+ main.js.localScope(&ls);+ defer ls.deinit();+ _ = try ls.local.compileAndRun(+ \\window.events = [];+ \\const childInput = document.getElementById('child').contentDocument.getElementById('middle');+ \\childInput.addEventListener('pointerdown', e => { e.preventDefault(); events.push('pointerdown'); document.getElementById('child').remove(); });+ \\for (const type of ['mousedown', 'focus', 'mouseup', 'click']) childInput.addEventListener(type, () => events.push(type));+ , null);+ try testing.expect((try main.document.elementFromPoint(rect.x + iframe_rect.x + 1, rect.y + iframe_rect.y + 1, main)) == child.iframe.?.asElement());+ try triggerMousePress(main, rect.x + iframe_rect.x + 1, rect.y + iframe_rect.y + 1, mouse_button.main, 1);+ try testing.expect((try ls.local.compileAndRun("events.join(',') === 'pointerdown'", null)).isTrue());+ try testing.expect(child.window._closed);+ try testing.expect(main.page.input_pointer.down_target == null);+ try testing.expectEqual(0, main.page.input_pointer.held);+ try testing.expect(!main.page.input_pointer.mousedown_suppressed);+}diff --git a/src/browser/tests/input_mouse_frames.html b/src/browser/tests/input_mouse_frames.htmlnew file mode 100644index 000000000..d823c100f--- /dev/null+++ b/src/browser/tests/input_mouse_frames.html@@ -0,0 +1,3 @@+<!doctype html>+<input id="outer" style="width: 80px; height: 10px">+<iframe id="child" style="width: 300px; height: 300px" srcdoc="<!doctype html><input id=middle style='width:80px;height:10px'><iframe id=nested style='width:200px;height:200px' srcdoc='<!doctype html><input id=inner style=width:80px;height:10px>'></iframe>"></iframe>