Lightpanda: the headless browser designed for AI and automation
bidi: user input
Karl Seguin877316a3e81a
diff --git a/src/browser/tests/bidi/input.html b/src/browser/tests/bidi/input.htmlnew file mode 100644index 000000000..6cbf255c9--- /dev/null+++ b/src/browser/tests/bidi/input.html@@ -0,0 +1,16 @@+<!DOCTYPE html>+<title>bidi input</title>+<button id=btn>go</button>+<input id=inp>+<script>+window.events = [];+for (const t of ['mousemove', 'mousedown', 'mouseup', 'click', 'dblclick', 'keydown', 'keyup', 'wheel']) {+ document.addEventListener(t, (e) => {+ let s = t;+ if (e.key !== undefined) s += ':' + e.key + (e.shiftKey ? '+shift' : '') + (e.ctrlKey ? '+ctrl' : '');+ if (e.button) s += ':b' + e.button;+ if (e.target && e.target.id) s += '@' + e.target.id;+ window.events.push(s);+ });+}+</script>diff --git a/src/server/bidi/BiDi.zig b/src/server/bidi/BiDi.zigindex 0f19c0fe0..1caaa0e29 100644--- a/src/server/bidi/BiDi.zig+++ b/src/server/bidi/BiDi.zig@@ -75,6 +75,8 @@ handles: remote_value.Handles,// Commands awaiting promise resolutionpending: std.ArrayList(*script.Pending) = .empty,+input_state: @import("input.zig").State = .{},+subscriptions: std.ArrayList(Subscription) = .empty,const Subscription = struct {@@ -136,6 +138,7 @@ pub fn deinit(self: *BiDi) void {// Now we can destroyscript.Pending.destroyAll(self);self.pending.deinit(allocator);+ self.input_state.deinit(allocator);self.node_registry.deinit();self.notification.deinit();@@ -175,6 +178,10 @@ pub fn resetRealm(self: *BiDi) void {self.handles.releaseAll();self.node_registry.reset();if (self.browsing_context) |*ctx| {+ if (ctx.realm_announced) {+ ctx.realm_announced = false;+ self.sendEvent("script.realmDestroyed", .{ .realm = &ctx.realm_id }) catch {};+ }uuidv4(&ctx.realm_id);}}@@ -224,6 +231,7 @@ fn dispatch(self: *BiDi, arena: Allocator, id: u64, method: []const u8, data: []script,browser,browsingContext,+ input,}, method[0..i]) orelse return error.UnknownCommand;// Only the session module is reachable without a session (it's what@@ -245,6 +253,7 @@ fn dispatch(self: *BiDi, arena: Allocator, id: u64, method: []const u8, data: [].script => return @import("script.zig").processMessage(&cmd),.browser => return @import("browser.zig").processMessage(&cmd),.browsingContext => return @import("browsing_context.zig").processMessage(&cmd),+ .input => return @import("input.zig").processMessage(&cmd),}}diff --git a/src/server/bidi/browsing_context.zig b/src/server/bidi/browsing_context.zigindex a554dfe7e..9902e3531 100644--- a/src/server/bidi/browsing_context.zig+++ b/src/server/bidi/browsing_context.zig@@ -53,6 +53,10 @@ pub const Context = struct {// A navigate command whose `wait` hasn't been satisfied yet.pending_navigate: ?PendingNavigate = null,++ // Whether script.realmCreated was sent or not. If sent, we'll send+ // script.realmDestroyed on teardown.+ realm_announced: bool = false,};const PendingNavigate = struct {@@ -146,11 +150,28 @@ fn create(cmd: *const BiDi.Command) !void {}) catch |err| {log.warn(.bidi, "blank navigate", .{ .err = err });};+ try announceRealm(bidi, frame);}return cmd.sendResult(.{ .context = &ctx.id });}+fn announceRealm(bidi: *BiDi, frame: *const Frame) !void {+ const ctx = &(bidi.browsing_context orelse return);+ if (ctx.realm_announced) {+ return;+ }+ ctx.realm_announced = true;++ const origin = (URL.getOrigin(bidi.message_arena.allocator(), frame.url) catch null) orelse "null";+ return bidi.sendEvent("script.realmCreated", .{+ .realm = &ctx.realm_id,+ .origin = origin,+ .type = "window",+ .context = &ctx.id,+ });+}+fn navigate(cmd: *const BiDi.Command) !void {const p = try cmd.params(struct {url: [:0]const u8,@@ -218,6 +239,9 @@ pub fn destroy(cmd: *const BiDi.Command, ctx: *Context) !void {// ctx points into bidi.browsing_context, which is about to be clearedconst gone = ctx.*;+ // the input state map is keyed by top-level browsing context+ bidi.input_state.reset(bidi.app.allocator);+// the frame's url doesn't survive closePageconst url: []const u8 = if (bidi.user_context.session.currentFrame()) |frame|try cmd.arena.dupe(u8, frame.url)@@ -390,6 +414,7 @@ pub fn registerNotifications(bidi: *BiDi) !void {const notification = bidi.notification;try notification.register(.frame_loaded, bidi, onFrameLoaded);try notification.register(.frame_remove, bidi, onFrameRemove);+ try notification.register(.frame_created, bidi, onFrameCreated);try notification.register(.frame_destroyed, bidi, onFrameDestroyed);try notification.register(.frame_navigate, bidi, onFrameNavigate);try notification.register(.frame_navigate_failed, bidi, onFrameNavigateFailed);@@ -402,6 +427,15 @@ fn onFrameRemove(ptr: *anyopaque, _: Notification.FrameRemove) !void {bidi.resetRealm();}+// A new root document, so a new window realm.+fn onFrameCreated(ptr: *anyopaque, frame: *Frame) !void {+ const bidi: *BiDi = @ptrCast(@alignCast(ptr));+ if (frame.parent != null or rootContext(bidi, frame._frame_id) == null) {+ return;+ }+ return announceRealm(bidi, frame);+}+fn onFrameDestroyed(ptr: *anyopaque, frame: *const Frame) !void {const bidi: *BiDi = @ptrCast(@alignCast(ptr));script.Pending.contextDestroyed(bidi, frame.js.id);@@ -661,3 +695,30 @@ test "bidi.browsing_context: locateNodes" {});try ctx.expectSentError("no such frame", null, .{ .id = 12 });}++test "bidi.browsing_context: window realm lifecycle" {+ var ctx = try testing.context();+ defer ctx.deinit();++ try ctx.startSession();+ try ctx.processMessage(.{ .id = 1, .method = "session.subscribe", .params = .{ .events = .{"script"} } });+ try ctx.processMessage(.{ .id = 2, .method = "browsingContext.create", .params = .{ .type = "tab" } });++ const bc = &(ctx.bidi().browsing_context orelse return error.NoBrowsingContext);+ const context_id = try testing.arena.dupe(u8, &bc.id);+ const realm1 = try testing.arena.dupe(u8, &bc.realm_id);+ try ctx.expectSentEvent("script.realmCreated", .{ .realm = realm1, .type = "window", .context = context_id, .origin = "null" });++ // a navigation is a new document and so a new realm+ try ctx.processMessage(.{+ .id = 3,+ .method = "browsingContext.navigate",+ .params = .{ .context = context_id, .url = testing.test_server ++ "bidi/values.html", .wait = "complete" },+ });+ try ctx.wait();+ const realm2 = try testing.arena.dupe(u8, &bc.realm_id);+ try testing.expect(std.mem.eql(u8, realm1, realm2) == false);+ try ctx.expectSentEvent("script.realmDestroyed", .{ .realm = realm1 });+ try ctx.expectSentEvent("script.realmCreated", .{ .realm = realm2, .type = "window", .context = context_id, .origin = "http://127.0.0.1:9582" });+ try ctx.expectSentResult(.{ .navigation = &bc.navigation_id }, .{ .id = 3 });+}diff --git a/src/server/bidi/input.zig b/src/server/bidi/input.zignew file mode 100644index 000000000..0eadcb1ec--- /dev/null+++ b/src/server/bidi/input.zig@@ -0,0 +1,1090 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const std = @import("std");+const lp = @import("lightpanda");++const Frame = @import("../../browser/Frame.zig");+const Node = @import("../../browser/webapi/Node.zig");+const KeyboardEvent = @import("../../browser/webapi/event/KeyboardEvent.zig");++const BiDi = @import("BiDi.zig");+const remote_value = @import("remote_value.zig");+const browsing_context = @import("browsing_context.zig");++const log = lp.log;+const user_input = Frame.user_input;+const Allocator = std.mem.Allocator;++pub fn processMessage(cmd: *const BiDi.Command) !void {+ const command = std.meta.stringToEnum(enum {+ performActions,+ releaseActions,+ }, cmd.action) orelse return error.UnknownCommand;++ switch (command) {+ .performActions => return performActions(cmd),+ .releaseActions => return releaseActions(cmd),+ }+}++fn performActions(cmd: *const BiDi.Command) !void {+ const p = try cmd.params(struct {+ context: []const u8,+ actions: []const std.json.Value,+ });++ if ((try browsing_context.requireContext(cmd, p.context)) == null) {+ return;+ }++ const bidi = cmd.bidi;+ const arena = try bidi.app.arena_pool.acquire(.small, "bidi input.Pending");+ errdefer arena.release();++ const ticks = parseTicks(bidi, arena.allocator(), p.actions) catch |err| {+ if (err == error.OutOfMemory) {+ return err;+ }+ arena.release();+ return cmd.sendError("invalid argument", errorMessage(err));+ };++ const pending = try arena.allocator().create(Pending);+ pending.* = .{+ .bidi = bidi,+ .arena = arena,+ .ticks = ticks,+ .command_id = cmd.id,+ .generation = bidi.input_state.generation,+ };+ return pending.run();+}++// Undo everything still held: keys and buttons in reverse press order.+fn releaseActions(cmd: *const BiDi.Command) !void {+ const p = try cmd.params(struct {+ context: []const u8,+ });++ if ((try browsing_context.requireContext(cmd, p.context)) == null) {+ return;+ }++ const bidi = cmd.bidi;+ const frame = bidi.user_context.session.currentFrame() orelse {+ return cmd.sendError("no such frame", "no frame");+ };++ // dispatch pops the released key/button off `pressed` itself+ const state = &bidi.input_state;+ for (state.sources.items) |*source| {+ switch (source.kind) {+ .key => while (source.key.pressed.getLastOrNull()) |cp| {+ try dispatch(bidi, frame, source, &.{ .key_up = cp });+ },+ .pointer => while (source.pointer.pressed.getLastOrNull()) |button| {+ try dispatch(bidi, frame, source, &.{ .pointer_up = button });+ },+ .none, .wheel => {},+ }+ }+ state.reset(bidi.app.allocator);++ return cmd.sendResult(struct {}{});+}++// Spec: the input state map, keyed by top-level browsing context. We have one+// browsing context, so this lives on the BiDi and survives navigations until+// input.releaseActions or the browsing context closes.+pub const State = struct {+ // incremented on reset so that performActions+ // Bumped by reset so a parked performActions can tell its sources are gone.+ generation: u32 = 0,+ sources: std.ArrayList(Source) = .empty,++ pub fn deinit(self: *State, allocator: Allocator) void {+ self.reset(allocator);+ self.sources.deinit(allocator);+ }++ pub fn reset(self: *State, allocator: Allocator) void {+ for (self.sources.items) |*source| {+ source.deinit(allocator);+ }+ self.sources.clearRetainingCapacity();+ self.generation +%= 1;+ }++ fn find(self: *State, id: []const u8) ?usize {+ for (self.sources.items, 0..) |*source, i| {+ if (std.mem.eql(u8, source.id, id)) {+ return i;+ }+ }+ return null;+ }++ fn findOrCreate(self: *State, allocator: Allocator, id: []const u8, kind: Source.Kind) !usize {+ if (self.find(id)) |i| {+ if (self.sources.items[i].kind != kind) {+ return error.SourceKindMismatch;+ }+ return i;+ }+ const owned = try allocator.dupe(u8, id);+ errdefer allocator.free(owned);+ try self.sources.append(allocator, .{ .id = owned, .kind = kind });+ return self.sources.items.len - 1;+ }+};++const Modifiers = struct {+ alt: bool = false,+ ctrl: bool = false,+ meta: bool = false,+ shift: bool = false,+};++const Source = struct {+ id: []const u8,+ kind: Kind,+ key: KeyState = .{},+ pointer: PointerState = .{},++ const Kind = enum { none, key, pointer, wheel };++ const KeyState = struct {+ // in press order, so releaseActions can undo them in reverse+ pressed: std.ArrayList(u21) = .empty,+ modifiers: Modifiers = .{},+ };++ const PointerState = struct {+ x: f64 = 0,+ y: f64 = 0,+ // DOM button numbers, in press order+ pressed: std.ArrayList(u8) = .empty,+ // for the click count+ last_click_ms: u64 = 0,+ last_click_x: f64 = 0,+ last_click_y: f64 = 0,+ click_count: i32 = 0,+ };++ fn deinit(self: *Source, allocator: Allocator) void {+ allocator.free(self.id);+ self.key.pressed.deinit(allocator);+ self.pointer.pressed.deinit(allocator);+ }+};++const Origin = union(enum) {+ viewport,+ pointer,+ // Resolved when the action runs, not when it's parsed: an earlier tick can+ // pause long enough for the page to change under us.+ element: []const u8,+};++const Action = union(enum) {+ pause: u32,+ key_down: u21,+ key_up: u21,+ pointer_down: u8,+ pointer_up: u8,+ pointer_move: struct { x: f64, y: f64, duration: u32, origin: Origin },+ scroll: struct { x: f64, y: f64, delta_x: f64, delta_y: f64, duration: u32, origin: Origin },++ fn duration(self: *const Action) u32 {+ return switch (self.*) {+ .pause => |d| d,+ .pointer_move => |m| m.duration,+ .scroll => |s| s.duration,+ else => 0,+ };+ }+};++const TickAction = struct {+ source: usize,+ action: Action,+};++const Pending = struct {+ bidi: *BiDi,+ arena: *lp.Arena,+ command_id: u64,+ generation: u32,+ ticks: []const []const TickAction,+ next: usize = 0,++ fn deinit(self: *Pending) void {+ self.arena.release();+ }++ // Runs ticks until one has a duration+ fn run(self: *Pending) !void {+ const bidi = self.bidi;+ while (self.next < self.ticks.len) {+ if (bidi.input_state.generation != self.generation) {+ defer self.deinit();+ return bidi.sendError(self.command_id, "unknown error", "input state was released while actions were pending");+ }+ const frame = bidi.user_context.session.currentFrame() orelse {+ defer self.deinit();+ return bidi.sendError(self.command_id, "no such frame", "no frame");+ };++ const next = self.next;+ const tick = self.ticks[next];+ self.next = next + 1;++ var wait: u32 = 0;+ for (tick) |ta| {+ wait = @max(wait, ta.action.duration());+ dispatch(bidi, frame, &bidi.input_state.sources.items[ta.source], &ta.action) catch |err| {+ defer self.deinit();+ if (err == error.OutOfMemory) {+ return err;+ }+ return bidi.sendError(self.command_id, errorCode(err), errorMessage(err));+ };+ }++ if (wait > 0) {+ return frame.js.scheduler.add(self, resumeFromScheduler, wait, .{ .name = "bidi.performActions", .finalizer = cancelled });+ }+ }++ defer self.deinit();+ try bidi.sendResult(self.command_id, struct {}{});+ }++ fn resumeFromScheduler(ctx: *anyopaque) !?u32 {+ const self: *Pending = @ptrCast(@alignCast(ctx));+ self.run() catch |err| {+ log.err(.bidi, "performActions", .{ .err = err, .id = self.command_id });+ self.deinit();+ };+ return null;+ }++ fn cancelled(ctx: *anyopaque) void {+ const self: *Pending = @ptrCast(@alignCast(ctx));+ self.bidi.sendError(self.command_id, "no such frame", "frame destroyed while actions were pending") catch {};+ self.deinit();+ }+};++const ParseError = error{+ OutOfMemory,+ InvalidActions,+ SourceKindMismatch,+ UnknownActionType,+ ActionNotAllowedForSource,+ MissingField,+ InvalidField,+ InvalidKeyValue,+ InvalidOrigin,+};++const SourceActions = struct {+ id: []const u8,+ type: Source.Kind,+ parameters: ?std.json.Value = null,+ actions: []const std.json.Value,+};++// Given 2 actions:+// {id: "keyboard", actions: [+// {type: "keyDown", value: "a"},+// {type: "keyUp", value: "a"}+// ]},+// {id: "mouse", actions: [+// { type: "pointerDown", button: 0 },+// { type: "pointerUp", button: 0 },+// ]}+//+// We'll flatten it rotate the actions into:+// [+// {tick: 1, actions: {+// keyboard: { type: "keyDown", value: "a" },+// mouse: { type: "pointerDown", button: 0 },+// }},+// {tick: 2, actions: {+// keyboard: { type: "keyUp", value: "a" },+// mouse: { type: "pointerUp", button: 0 },+// }}+// ]+//+// If the action arrays aren' the same length, a "pause" is inserteds+fn parseTicks(bidi: *BiDi, arena: Allocator, actions: []const std.json.Value) ParseError![]const []const TickAction {+ const state = &bidi.input_state;+ const allocator = bidi.app.allocator;++ var columns: std.ArrayList(struct { source: usize, actions: []const Action }) = try .initCapacity(arena, actions.len);++ var tick_count: usize = 0;+ for (actions) |value| {+ const sa = std.json.parseFromValueLeaky(SourceActions, arena, value, .{ .ignore_unknown_fields = true }) catch {+ return error.InvalidActions;+ };+ const source = try state.findOrCreate(allocator, sa.id, sa.type);++ const parsed = try arena.alloc(Action, sa.actions.len);+ for (sa.actions, parsed) |raw, *action| {+ action.* = try parseAction(sa.type, raw);+ }+ columns.appendAssumeCapacity(.{ .source = source, .actions = parsed });++ // the number of ticks we'll have is the max # of actions the actions have+ tick_count = @max(tick_count, parsed.len);+ }++ const ticks = try arena.alloc([]const TickAction, tick_count);+ for (ticks, 0..) |*tick, i| {+ var row: std.ArrayList(TickAction) = try .initCapacity(arena, columns.items.len);+ for (columns.items) |column| {+ if (i < column.actions.len) {+ row.appendAssumeCapacity(.{ .source = column.source, .action = column.actions[i] });+ }+ }+ tick.* = row.items;+ }+ return ticks;+}++fn parseAction(kind: Source.Kind, raw: std.json.Value) ParseError!Action {+ const obj = switch (raw) {+ .object => |o| o,+ else => return error.InvalidActions,+ };+ const type_name = switch (obj.get("type") orelse return error.MissingField) {+ .string => |s| s,+ else => return error.InvalidField,+ };+ const typ = std.meta.stringToEnum(enum {+ pause,+ keyDown,+ keyUp,+ pointerDown,+ pointerUp,+ pointerMove,+ scroll,+ }, type_name) orelse return error.UnknownActionType;++ const allowed = switch (typ) {+ .pause => true,+ .keyDown, .keyUp => kind == .key,+ .pointerDown, .pointerUp, .pointerMove => kind == .pointer,+ .scroll => kind == .wheel,+ };+ if (!allowed) {+ return error.ActionNotAllowedForSource;+ }++ return switch (typ) {+ .pause => .{ .pause = try uintField(obj, "duration", 0) },+ .keyDown => .{ .key_down = try keyValue(obj) },+ .keyUp => .{ .key_up = try keyValue(obj) },+ .pointerDown => .{ .pointer_down = try buttonField(obj) },+ .pointerUp => .{ .pointer_up = try buttonField(obj) },+ .pointerMove => .{ .pointer_move = .{+ .x = try numberField(obj, "x"),+ .y = try numberField(obj, "y"),+ .duration = try uintField(obj, "duration", 0),+ .origin = try originField(obj, .pointer),+ } },+ .scroll => .{ .scroll = .{+ .x = try numberField(obj, "x"),+ .y = try numberField(obj, "y"),+ .delta_x = try numberField(obj, "deltaX"),+ .delta_y = try numberField(obj, "deltaY"),+ .duration = try uintField(obj, "duration", 0),+ .origin = try originField(obj, .wheel),+ } },+ };+}++fn keyValue(obj: std.json.ObjectMap) ParseError!u21 {+ const str = switch (obj.get("value") orelse return error.MissingField) {+ .string => |s| s,+ else => return error.InvalidField,+ };+ // one code point, no more, no less+ if (str.len == 0) {+ return error.InvalidKeyValue;+ }+ const len = std.unicode.utf8ByteSequenceLength(str[0]) catch return error.InvalidKeyValue;+ if (str.len != len) {+ return error.InvalidKeyValue;+ }+ return std.unicode.utf8Decode(str) catch return error.InvalidKeyValue;+}++fn buttonField(obj: std.json.ObjectMap) ParseError!u8 {+ const button = try uintField(obj, "button", 0);+ if (button > std.math.maxInt(u8)) {+ return error.InvalidField;+ }+ return @intCast(button);+}++fn uintField(obj: std.json.ObjectMap, name: []const u8, default: u32) ParseError!u32 {+ const value = obj.get(name) orelse return default;+ return switch (value) {+ .integer => |i| if (i >= 0 and i <= std.math.maxInt(u32)) @intCast(i) else error.InvalidField,+ .null => default,+ else => error.InvalidField,+ };+}++fn numberField(obj: std.json.ObjectMap, name: []const u8) ParseError!f64 {+ const value = obj.get(name) orelse return error.MissingField;+ return switch (value) {+ .integer => |i| @floatFromInt(i),+ .float => |f| f,+ else => error.InvalidField,+ };+}++fn originField(obj: std.json.ObjectMap, source: Source.Kind) ParseError!Origin {+ const value = obj.get("origin") orelse return .viewport;+ switch (value) {+ .null => return .viewport,+ .string => |s| {+ if (std.mem.eql(u8, s, "viewport")) {+ return .viewport;+ }+ if (std.mem.eql(u8, s, "pointer") and source == .pointer) {+ return .pointer;+ }+ return error.InvalidOrigin;+ },+ .object => |o| {+ const typ = o.get("type") orelse return error.InvalidOrigin;+ if (typ != .string or std.mem.eql(u8, typ.string, "element") == false) {+ return error.InvalidOrigin;+ }+ const element = switch (o.get("element") orelse return error.InvalidOrigin) {+ .object => |e| e,+ else => return error.InvalidOrigin,+ };+ return switch (element.get("sharedId") orelse return error.InvalidOrigin) {+ .string => |s| .{ .element = s },+ else => error.InvalidOrigin,+ };+ },+ else => return error.InvalidOrigin,+ }+}++const DispatchError = error{+ OutOfMemory,+ NoSuchElement,+ NotAnElement,+ StringTooLarge,+ CompilationError,+ JsException,+ ExecutionTerminated,+};++fn dispatch(bidi: *BiDi, frame: *Frame, source: *Source, action: *const Action) DispatchError!void {+ const allocator = bidi.app.allocator;+ switch (action.*) {+ .pause => {},+ .key_down => |cp| {+ const key = &source.key;+ const info = keyInfo(cp, key.modifiers.shift);+ setModifier(&key.modifiers, info.modifier, true);+ if (std.mem.indexOfScalar(u21, key.pressed.items, cp) == null) {+ try key.pressed.append(allocator, cp);+ }+ try dispatchKey(frame, "keydown", &info, &key.modifiers);+ },+ .key_up => |cp| {+ const key = &source.key;+ const info = keyInfo(cp, key.modifiers.shift);+ setModifier(&key.modifiers, info.modifier, false);+ if (std.mem.indexOfScalar(u21, key.pressed.items, cp)) |i| {+ _ = key.pressed.orderedRemove(i);+ }+ try dispatchKey(frame, "keyup", &info, &key.modifiers);+ },+ .pointer_down => |button| {+ const pointer = &source.pointer;+ if (std.mem.indexOfScalar(u8, pointer.pressed.items, button) != null) {+ return; // already down; the spec makes this a no-op+ }+ try pointer.pressed.append(allocator, button);++ // A press near the last click, soon enough after it, counts up+ const now = lp.datetime.milliTimestamp(.boot);+ const near = @abs(pointer.x - pointer.last_click_x) <= 2 and @abs(pointer.y - pointer.last_click_y) <= 2;+ pointer.click_count = if (near and now - pointer.last_click_ms <= 500) pointer.click_count + 1 else 1;+ pointer.last_click_ms = now;+ pointer.last_click_x = pointer.x;+ pointer.last_click_y = pointer.y;++ try user_input.triggerMousePress(frame, pointer.x, pointer.y, button);+ },+ .pointer_up => |button| {+ const pointer = &source.pointer;+ const i = std.mem.indexOfScalar(u8, pointer.pressed.items, button) orelse return;+ _ = pointer.pressed.orderedRemove(i);+ try user_input.triggerMouseRelease(frame, pointer.x, pointer.y, button, pointer.click_count);+ },+ .pointer_move => |move| {+ const pointer = &source.pointer;+ const target = try resolveOrigin(bidi, frame, source, move.origin, move.x, move.y);+ pointer.x = target.x;+ pointer.y = target.y;+ try user_input.triggerMouseMove(frame, target.x, target.y);+ },+ .scroll => |scroll| {+ const target = try resolveOrigin(bidi, frame, source, scroll.origin, scroll.x, scroll.y);+ try user_input.triggerMouseWheel(frame, target.x, target.y, scroll.delta_x, scroll.delta_y);+ },+ }+}++const Point = struct { x: f64, y: f64 };++fn resolveOrigin(bidi: *BiDi, frame: *Frame, source: *const Source, origin: Origin, x: f64, y: f64) !Point {+ switch (origin) {+ .viewport => return .{ .x = x, .y = y },+ .pointer => return .{ .x = source.pointer.x + x, .y = source.pointer.y + y },+ .element => |shared_id| {+ const node = remote_value.nodeFromSharedId(&bidi.node_registry, .{ .string = shared_id }) catch {+ return error.NoSuchElement;+ };+ const element = node.is(Node.Element) orelse return error.NotAnElement;+ // relative to the element's center+ const rect = element.boundingClientRectValues(frame);+ return .{ .x = rect.x + rect.width / 2 + x, .y = rect.y + rect.height / 2 + y };+ },+ }+}++fn dispatchKey(frame: *Frame, comptime typ: []const u8, info: *const KeyInfo, modifiers: *const Modifiers) !void {+ var buf: [4]u8 = undefined;+ const key: []const u8 = switch (info.key) {+ .name => |name| name,+ .char => |cp| buf[0 .. std.unicode.utf8Encode(cp, &buf) catch unreachable],+ };+ const event = try KeyboardEvent.initTrusted(comptime .wrap(typ), .{+ .key = key,+ .code = info.code,+ .location = info.location,+ .altKey = modifiers.alt,+ .ctrlKey = modifiers.ctrl,+ .metaKey = modifiers.meta,+ .shiftKey = modifiers.shift,+ }, frame);+ try user_input.triggerKeyboard(frame, event);+}++fn setModifier(modifiers: *Modifiers, which: ?Modifier, down: bool) void {+ switch (which orelse return) {+ .alt => modifiers.alt = down,+ .ctrl => modifiers.ctrl = down,+ .meta => modifiers.meta = down,+ .shift => modifiers.shift = down,+ }+}++const Modifier = enum { alt, ctrl, meta, shift };++const KeyInfo = struct {+ // a named key, or the character itself+ key: union(enum) { name: []const u8, char: u21 },+ code: []const u8,+ location: u32 = 0,+ modifier: ?Modifier = null,+};++// WebDriver's key table: the Private Use Area - names the+// non-printing keys, anything else is the character itself.+// https://w3c.github.io/webdriver/#keyboard-actions+fn keyInfo(cp: u21, shift: bool) KeyInfo {+ if (cp >= 0xE000 and cp <= 0xE05D) {+ return specialKey(cp);+ }++ const char: u21 = if (shift and cp < 128) shiftedAscii(@intCast(cp)) else cp;+ return .{ .key = .{ .char = char }, .code = asciiCode(cp) };+}++fn specialKey(cp: u21) KeyInfo {+ const k = struct {+ fn k(key: []const u8, code: []const u8) KeyInfo {+ return .{ .key = .{ .name = key }, .code = code };+ }+ fn m(key: []const u8, code: []const u8, location: u32, modifier: Modifier) KeyInfo {+ return .{ .key = .{ .name = key }, .code = code, .location = location, .modifier = modifier };+ }+ fn n(key: []const u8, code: []const u8) KeyInfo {+ return .{ .key = .{ .name = key }, .code = code, .location = 3 };+ }+ };+ return switch (cp) {+ 0xE000 => k.k("Unidentified", ""),+ 0xE001 => k.k("Cancel", "Abort"),+ 0xE002 => k.k("Help", "Help"),+ 0xE003 => k.k("Backspace", "Backspace"),+ 0xE004 => k.k("Tab", "Tab"),+ 0xE005 => k.k("Clear", "NumLock"),+ 0xE006 => k.k("Enter", "Enter"),+ 0xE007 => k.n("Enter", "NumpadEnter"),+ 0xE008 => k.m("Shift", "ShiftLeft", 1, .shift),+ 0xE009 => k.m("Control", "ControlLeft", 1, .ctrl),+ 0xE00A => k.m("Alt", "AltLeft", 1, .alt),+ 0xE00B => k.k("Pause", "Pause"),+ 0xE00C => k.k("Escape", "Escape"),+ 0xE00D => k.k(" ", "Space"),+ 0xE00E => k.k("PageUp", "PageUp"),+ 0xE00F => k.k("PageDown", "PageDown"),+ 0xE010 => k.k("End", "End"),+ 0xE011 => k.k("Home", "Home"),+ 0xE012 => k.k("ArrowLeft", "ArrowLeft"),+ 0xE013 => k.k("ArrowUp", "ArrowUp"),+ 0xE014 => k.k("ArrowRight", "ArrowRight"),+ 0xE015 => k.k("ArrowDown", "ArrowDown"),+ 0xE016 => k.k("Insert", "Insert"),+ 0xE017 => k.k("Delete", "Delete"),+ 0xE018 => k.k(";", "Semicolon"),+ 0xE019 => k.k("=", "Equal"),+ 0xE01A => k.n("0", "Numpad0"),+ 0xE01B => k.n("1", "Numpad1"),+ 0xE01C => k.n("2", "Numpad2"),+ 0xE01D => k.n("3", "Numpad3"),+ 0xE01E => k.n("4", "Numpad4"),+ 0xE01F => k.n("5", "Numpad5"),+ 0xE020 => k.n("6", "Numpad6"),+ 0xE021 => k.n("7", "Numpad7"),+ 0xE022 => k.n("8", "Numpad8"),+ 0xE023 => k.n("9", "Numpad9"),+ 0xE024 => k.n("*", "NumpadMultiply"),+ 0xE025 => k.n("+", "NumpadAdd"),+ 0xE026 => k.n(",", "NumpadComma"),+ 0xE027 => k.n("-", "NumpadSubtract"),+ 0xE028 => k.n(".", "NumpadDecimal"),+ 0xE029 => k.n("/", "NumpadDivide"),+ 0xE031 => k.k("F1", "F1"),+ 0xE032 => k.k("F2", "F2"),+ 0xE033 => k.k("F3", "F3"),+ 0xE034 => k.k("F4", "F4"),+ 0xE035 => k.k("F5", "F5"),+ 0xE036 => k.k("F6", "F6"),+ 0xE037 => k.k("F7", "F7"),+ 0xE038 => k.k("F8", "F8"),+ 0xE039 => k.k("F9", "F9"),+ 0xE03A => k.k("F10", "F10"),+ 0xE03B => k.k("F11", "F11"),+ 0xE03C => k.k("F12", "F12"),+ 0xE03D => k.m("Meta", "MetaLeft", 1, .meta),+ 0xE040 => k.k("ZenkakuHankaku", ""),+ 0xE050 => k.m("Shift", "ShiftRight", 2, .shift),+ 0xE051 => k.m("Control", "ControlRight", 2, .ctrl),+ 0xE052 => k.m("Alt", "AltRight", 2, .alt),+ 0xE053 => k.m("Meta", "MetaRight", 2, .meta),+ 0xE054 => k.n("PageUp", "Numpad9"),+ 0xE055 => k.n("PageDown", "Numpad3"),+ 0xE056 => k.n("End", "Numpad1"),+ 0xE057 => k.n("Home", "Numpad7"),+ 0xE058 => k.n("ArrowLeft", "Numpad4"),+ 0xE059 => k.n("ArrowUp", "Numpad8"),+ 0xE05A => k.n("ArrowRight", "Numpad6"),+ 0xE05B => k.n("ArrowDown", "Numpad2"),+ 0xE05C => k.n("Insert", "Numpad0"),+ 0xE05D => k.n("Delete", "NumpadDecimal"),+ else => k.k("Unidentified", ""),+ };+}++// The `code` of a printable character on a US layout.+fn asciiCode(cp: u21) []const u8 {+ if (cp >= 128) {+ return "";+ }+ const c: u8 = @intCast(cp);+ return switch (c) {+ 'a'...'z' => &[_]u8{ 'K', 'e', 'y', std.ascii.toUpper(c) },+ 'A'...'Z' => &[_]u8{ 'K', 'e', 'y', c },+ '0'...'9' => &[_]u8{ 'D', 'i', 'g', 'i', 't', c },+ ' ' => "Space",+ '\n', '\r' => "Enter",+ '\t' => "Tab",+ '`', '~' => "Backquote",+ '-', '_' => "Minus",+ '=', '+' => "Equal",+ '[', '{' => "BracketLeft",+ ']', '}' => "BracketRight",+ '\\', '|' => "Backslash",+ ';', ':' => "Semicolon",+ '\'', '"' => "Quote",+ ',', '<' => "Comma",+ '.', '>' => "Period",+ '/', '?' => "Slash",+ '!' => "Digit1",+ '@' => "Digit2",+ '#' => "Digit3",+ '$' => "Digit4",+ '%' => "Digit5",+ '^' => "Digit6",+ '&' => "Digit7",+ '*' => "Digit8",+ '(' => "Digit9",+ ')' => "Digit0",+ else => "",+ };+}++// What a held Shift turns a US-layout key into.+fn shiftedAscii(c: u8) u8 {+ return switch (c) {+ 'a'...'z' => std.ascii.toUpper(c),+ '`' => '~',+ '1' => '!',+ '2' => '@',+ '3' => '#',+ '4' => '$',+ '5' => '%',+ '6' => '^',+ '7' => '&',+ '8' => '*',+ '9' => '(',+ '0' => ')',+ '-' => '_',+ '=' => '+',+ '[' => '{',+ ']' => '}',+ '\\' => '|',+ ';' => ':',+ '\'' => '"',+ ',' => '<',+ '.' => '>',+ '/' => '?',+ else => c,+ };+}++fn errorCode(err: anyerror) []const u8 {+ return switch (err) {+ error.NoSuchElement => "no such element",+ error.NotAnElement => "no such element",+ else => "unknown error",+ };+}++fn errorMessage(err: anyerror) []const u8 {+ return switch (err) {+ error.InvalidActions => "actions must be a list of source actions",+ error.SourceKindMismatch => "input source id already used with a different type",+ error.UnknownActionType => "unknown action type",+ error.ActionNotAllowedForSource => "action type not allowed for this input source",+ error.MissingField => "missing required action field",+ error.InvalidField => "invalid action field",+ error.InvalidKeyValue => "key value must be a single code point",+ error.InvalidOrigin => "invalid origin",+ error.NoSuchElement => "origin element not found",+ error.NotAnElement => "origin must be an element",+ else => "action failed",+ };+}++const testing = @import("testing.zig");+test "bidi.input: click via element origin" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const context_id = try ctx.createContext(.{ .url = "bidi/input.html" });++ try ctx.processMessage(.{+ .id = 1,+ .method = "browsingContext.locateNodes",+ .params = .{ .context = context_id, .locator = .{ .type = "css", .value = "#btn" } },+ });+ try ctx.expectSentResult(.{ .nodes = .{.{ .sharedId = "1" }} }, .{ .id = 1 });++ // puppeteer's page.click shape: move, down, up+ try ctx.processMessage(.{+ .id = 2,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "pointer",+ .id = "mouse",+ .actions = .{+ .{ .type = "pointerMove", .x = 0, .y = 0, .origin = .{ .type = "element", .element = .{ .sharedId = "1" } } },+ .{ .type = "pointerDown", .button = 0 },+ .{ .type = "pointerUp", .button = 0 },+ },+ }} },+ });+ try ctx.expectSentResult(null, .{ .id = 2 });++ try evaluate(&ctx, 3, context_id, "window.events.join(' ')");+ try ctx.expectSentResult(.{ .type = "success", .result = .{+ .type = "string",+ .value = "mousemove@btn mousedown@btn mouseup@btn click@btn",+ } }, .{ .id = 3 });++ // The pointer keeps its position across commands; a second click there+ // is a double click.+ try ctx.processMessage(.{+ .id = 4,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "pointer",+ .id = "mouse",+ .actions = .{+ .{ .type = "pointerDown", .button = 0 },+ .{ .type = "pointerUp", .button = 0 },+ },+ }} },+ });+ try ctx.expectSentResult(null, .{ .id = 4 });+ try evaluate(&ctx, 5, context_id, "window.events.slice(4).join(' ')");+ try ctx.expectSentResult(.{ .type = "success", .result = .{+ .type = "string",+ .value = "mousedown@btn mouseup@btn click@btn dblclick@btn",+ } }, .{ .id = 5 });++ // right button+ try ctx.processMessage(.{+ .id = 6,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "pointer",+ .id = "mouse",+ .actions = .{+ .{ .type = "pointerMove", .x = 0, .y = 0, .origin = "pointer" },+ .{ .type = "pointerDown", .button = 2 },+ .{ .type = "pointerUp", .button = 2 },+ },+ }} },+ });+ try ctx.expectSentResult(null, .{ .id = 6 });+ try evaluate(&ctx, 7, context_id, "window.events.slice(8).join(' ')");+ try ctx.expectSentResult(.{ .type = "success", .result = .{+ .type = "string",+ .value = "mousemove@btn mousedown:b2@btn mouseup:b2@btn",+ } }, .{ .id = 7 });+}++test "bidi.input: keys and modifiers" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const context_id = try ctx.createContext(.{ .url = "bidi/input.html" });++ try evaluate(&ctx, 1, context_id, "document.getElementById('inp').focus()");+ try ctx.expectSentResult(.{ .type = "success" }, .{ .id = 1 });++ // a, Shift+b (Shift = ), Enter ()+ try ctx.processMessage(.{+ .id = 2,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "key",+ .id = "kb",+ .actions = .{+ .{ .type = "keyDown", .value = "a" },+ .{ .type = "keyUp", .value = "a" },+ .{ .type = "keyDown", .value = "\u{E008}" },+ .{ .type = "keyDown", .value = "b" },+ .{ .type = "keyUp", .value = "b" },+ .{ .type = "keyUp", .value = "\u{E008}" },+ .{ .type = "keyDown", .value = "\u{E007}" },+ .{ .type = "keyUp", .value = "\u{E007}" },+ },+ }} },+ });+ try ctx.expectSentResult(null, .{ .id = 2 });++ try evaluate(&ctx, 3, context_id, "window.events.join(' ')");+ try ctx.expectSentResult(.{ .type = "success", .result = .{+ .type = "string",+ .value = "keydown:a@inp keyup:a@inp keydown:Shift+shift@inp keydown:B+shift@inp keyup:B+shift@inp keyup:Shift@inp keydown:Enter@inp keyup:Enter@inp",+ } }, .{ .id = 3 });++ try evaluate(&ctx, 4, context_id, "document.getElementById('inp').value");+ try ctx.expectSentResult(.{ .type = "success", .result = .{ .type = "string", .value = "aB" } }, .{ .id = 4 });+}++test "bidi.input: pause parks the command, releaseActions undoes held input" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const context_id = try ctx.createContext(.{ .url = "bidi/input.html" });++ try evaluate(&ctx, 1, context_id, "document.getElementById('inp').focus()");+ try ctx.expectSentResult(.{ .type = "success" }, .{ .id = 1 });++ // Two sources in lockstep: the key source holds Control while the "none"+ // source pauses. Only the ticks before the pause run synchronously.+ try ctx.processMessage(.{+ .id = 2,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{+ .{+ .type = "key",+ .id = "kb",+ .actions = .{+ .{ .type = "keyDown", .value = "\u{E009}" },+ .{ .type = "pause" },+ .{ .type = "keyDown", .value = "x" },+ },+ },+ .{+ .type = "none",+ .id = "clock",+ .actions = .{+ .{ .type = "pause" },+ .{ .type = "pause", .duration = 20 },+ },+ },+ } },+ });+ try ctx.expectNotAnswered(2);+ try evaluate(&ctx, 3, context_id, "window.events.join(' ')");+ try ctx.expectSentResult(.{ .type = "success", .result = .{ .type = "string", .value = "keydown:Control+ctrl@inp" } }, .{ .id = 3 });++ try ctx.wait();+ try ctx.expectSentResult(null, .{ .id = 2 });+ try evaluate(&ctx, 4, context_id, "window.events.join(' ')");+ try ctx.expectSentResult(.{ .type = "success", .result = .{+ .type = "string",+ .value = "keydown:Control+ctrl@inp keydown:x+ctrl@inp",+ } }, .{ .id = 4 });++ // Control and x are still held; release lets go in reverse order.+ try ctx.processMessage(.{ .id = 5, .method = "input.releaseActions", .params = .{ .context = context_id } });+ try ctx.expectSentResult(null, .{ .id = 5 });+ try evaluate(&ctx, 6, context_id, "window.events.slice(2).join(' ')");+ try ctx.expectSentResult(.{ .type = "success", .result = .{+ .type = "string",+ .value = "keyup:x+ctrl@inp keyup:Control@inp",+ } }, .{ .id = 6 });+}++test "bidi.input: errors" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const context_id = try ctx.createContext(.{ .url = "bidi/input.html" });++ try ctx.processMessage(.{+ .id = 1,+ .method = "input.performActions",+ .params = .{ .context = "nope", .actions = .{} },+ });+ try ctx.expectSentError("no such frame", null, .{ .id = 1 });++ // key action on a pointer source+ try ctx.processMessage(.{+ .id = 2,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "pointer",+ .id = "mouse",+ .actions = .{.{ .type = "keyDown", .value = "a" }},+ }} },+ });+ try ctx.expectSentError("invalid argument", null, .{ .id = 2 });++ // an id can't change type+ try ctx.processMessage(.{+ .id = 3,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{ .type = "key", .id = "mouse", .actions = .{} }} },+ });+ try ctx.expectSentError("invalid argument", null, .{ .id = 3 });++ try ctx.processMessage(.{+ .id = 4,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "key",+ .id = "kb",+ .actions = .{.{ .type = "keyDown", .value = "ab" }},+ }} },+ });+ try ctx.expectSentError("invalid argument", null, .{ .id = 4 });++ try ctx.processMessage(.{+ .id = 5,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "pointer",+ .id = "mouse",+ .actions = .{.{ .type = "pointerMove", .x = 0, .y = 0, .origin = .{ .type = "element", .element = .{ .sharedId = "99" } } }},+ }} },+ });+ try ctx.expectSentError("no such element", null, .{ .id = 5 });++ try ctx.processMessage(.{+ .id = 7,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{ .type = "key", .id = "kb", .actions = .{.{ .type = "keyDown", .value = "" }} }} },+ });+ try ctx.expectSentError("invalid argument", null, .{ .id = 7 });++ try ctx.processMessage(.{+ .id = 8,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{ .type = "pointer", .id = "mouse", .actions = .{.{ .type = "pointerDown", .button = 300 }} }} },+ });+ try ctx.expectSentError("invalid argument", null, .{ .id = 8 });++ try ctx.processMessage(.{+ .id = 6,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "wheel",+ .id = "w",+ .actions = .{.{ .type = "scroll", .x = 0, .y = 0, .deltaX = 0, .deltaY = 10, .origin = "pointer" }},+ }} },+ });+ try ctx.expectSentError("invalid argument", null, .{ .id = 6 });+}++test "bidi.input: teardown while actions are parked" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const context_id = try ctx.createContext(.{ .url = "bidi/input.html" });++ // A long pause parks the command on the frame's scheduler; tearing the+ // context down must finalize it (answer + free), which the leak checker+ // verifies.+ try ctx.processMessage(.{+ .id = 1,+ .method = "input.performActions",+ .params = .{ .context = context_id, .actions = .{.{+ .type = "none",+ .id = "clock",+ .actions = .{.{ .type = "pause", .duration = 60_000 }},+ }} },+ });+ try ctx.expectNotAnswered(1);+}++fn evaluate(ctx: *testing.TestContext, id: u64, context_id: []const u8, expression: []const u8) !void {+ return ctx.processMessage(.{+ .id = id,+ .method = "script.evaluate",+ .params = .{ .expression = expression, .target = .{ .context = context_id } },+ });+}diff --git a/src/server/bidi/remote_value.zig b/src/server/bidi/remote_value.zigindex 2de002514..1d30de3b7 100644--- a/src/server/bidi/remote_value.zig+++ b/src/server/bidi/remote_value.zig@@ -329,6 +329,7 @@ pub const Serializer = struct {JsException,ExecutionTerminated,MethodNotFound,+ DeadFunctionHandle,};fn remote(self: *Serializer, value: js.Value, depth: u32, own: bool) Error!Remote {diff --git a/src/server/bidi/testing.zig b/src/server/bidi/testing.zigindex 7e1c4f8d0..2db75fa84 100644--- a/src/server/bidi/testing.zig+++ b/src/server/bidi/testing.zig@@ -39,7 +39,7 @@ pub const silenceLog = base.silenceLog;// Where the test http server serves src/browser/tests from.pub const test_server = "http://127.0.0.1:9582/src/browser/tests/";-const TestContext = struct {+pub const TestContext = struct {read_at: usize = 0,read_buf: [1024 * 32]u8 = undefined,bidi_: BiDi = undefined,