Lightpanda: the headless browser designed for AI and automation
// Copyright (C) 2023-2025 Lightpanda (Selecy SAS)
// Francis Bouvier <[email protected]>
// Pierre Tachoire <[email protected]>
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
// The CDP-specific views over the NodeRegistry: DOM.performSearch state and
// the DOM.Node wire-format serializer.
const std = @import("std");
const lp = @import("lightpanda");
const id = @import("id.zig");
const DOMNode = @import("../../browser/webapi/Node.zig");
const Frame = @import("../../browser/Frame.zig");
const IFrame = @import("../../browser/webapi/element/html/IFrame.zig");
const NodeRegistry = @import("../../NodeRegistry.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
// Searches are a 3 step process:
// 1 - Dom.performSearch
// 2 - Dom.getSearchResults
// 3 - Dom.discardSearchResults
//
// For a given browser context, we can have multiple active searches. I.e.
// performSearch could be called multiple times without getSearchResults or
// discardSearchResults being called. We keep these active searches in the
// browser context's node_search_list, which is a SearchList. Since we don't
// expect many active searches (mostly just 1), a list is fine to scan through.
pub const Search = struct {
name: []const u8,
node_ids: []const NodeRegistry.Id,
pub const List = struct {
search_id: u16 = 0,
registry: *NodeRegistry,
arena: std.heap.ArenaAllocator,
searches: std.ArrayList(Search) = .empty,
pub fn init(allocator: Allocator, registry: *NodeRegistry) List {
return .{
.registry = registry,
.arena = std.heap.ArenaAllocator.init(allocator),
};
}
pub fn deinit(self: *List) void {
self.arena.deinit();
}
pub fn reset(self: *List) void {
self.search_id = 0;
self.searches = .empty;
_ = self.arena.reset(.{ .retain_with_limit = 4096 });
}
pub fn create(self: *List, nodes: []const *DOMNode) !Search {
const search_id = self.search_id;
defer self.search_id = search_id +% 1;
const arena = self.arena.allocator();
const name = switch (search_id) {
0 => "0",
1 => "1",
2 => "2",
3 => "3",
4 => "4",
5 => "5",
6 => "6",
7 => "7",
8 => "8",
9 => "9",
else => try arena.print("{d}", .{search_id}),
};
var registry = self.registry;
const node_ids = try arena.alloc(NodeRegistry.Id, nodes.len);
for (nodes, node_ids) |node, *node_id| {
node_id.* = (try registry.register(node)).id;
}
const search = Search{
.name = name,
.node_ids = node_ids,
};
try self.searches.append(arena, search);
return search;
}
pub fn remove(self: *List, name: []const u8) void {
for (self.searches.items, 0..) |search, i| {
if (std.mem.eql(u8, name, search.name)) {
_ = self.searches.swapRemove(i);
return;
}
}
}
pub fn get(self: *const List, name: []const u8) ?Search {
for (self.searches.items) |search| {
if (std.mem.eql(u8, name, search.name)) {
return search;
}
}
return null;
}
};
};
// Need a custom writer, because we can't just serialize the node as-is.
// Sometimes we want to serialize the node without children, sometimes with just
// its direct children, and sometimes the entire tree.
// (For now, we only support direct children)
pub const Writer = struct {
depth: i32,
exclude_root: bool,
pierce: bool = false,
root: *const NodeRegistry.Node,
registry: *NodeRegistry,
frame: *const Frame,
pub const Opts = struct {
depth: i32 = 0,
exclude_root: bool = false,
pierce: bool = false,
};
pub fn jsonStringify(self: *const Writer, w: anytype) error{WriteFailed}!void {
if (self.exclude_root) {
_ = self.writeChildren(self.root, 1, w) catch |err| {
log.err(.cdp, "node writeChildren", .{ .err = err });
return error.WriteFailed;
};
} else {
self.toJSON(self.root, 0, w) catch |err| {
// The only error our jsonStringify method can return is
// @TypeOf(w).Error. In other words, our code can't return its own
// error, we can only return a writer error. Kinda sucks.
log.err(.cdp, "node toJSON stringify", .{ .err = err });
return error.WriteFailed;
};
}
}
fn toJSON(self: *const Writer, node: *const NodeRegistry.Node, depth: usize, w: anytype) !void {
try w.beginObject();
try self.writeCommon(node, false, w);
try self.writeShadowRoots(node, self.pierce, depth, w);
try w.objectField("children");
const child_count = try self.writeChildren(node, depth, w);
try w.objectField("childNodeCount");
try w.write(child_count);
try w.endObject();
}
fn writeShadowRoots(self: *const Writer, node: *const NodeRegistry.Node, pierce: bool, depth: usize, w: anytype) anyerror!void {
const el = node.dom.is(DOMNode.Element) orelse return;
const shadow = el.hostedShadowRoot(self.frame) orelse return;
const shadow_node = try self.registry.register(shadow.asNode());
try w.objectField("shadowRoots");
try w.beginArray();
if (pierce and self.isFull(depth)) {
try self.toJSON(shadow_node, depth + 1, w);
} else {
try w.beginObject();
try self.writeCommon(shadow_node, true, w);
try w.endObject();
}
try w.endArray();
}
fn isFull(self: *const Writer, depth: usize) bool {
return self.depth < 0 or self.depth < depth;
}
fn writeChildren(self: *const Writer, node: *const NodeRegistry.Node, depth: usize, w: anytype) anyerror!usize {
var count: usize = 0;
var it = node.dom.childrenIterator();
var registry = self.registry;
const full_child = self.isFull(depth);
try w.beginArray();
while (it.next()) |dom_child| {
const child_node = try registry.register(dom_child);
if (full_child) {
try self.toJSON(child_node, depth + 1, w);
} else {
try w.beginObject();
try self.writeCommon(child_node, true, w);
try self.writeShadowRoots(child_node, false, depth + 1, w);
try w.endObject();
}
count += 1;
}
try w.endArray();
return count;
}
fn writeCommon(self: *const Writer, node: *const NodeRegistry.Node, include_child_count: bool, w: anytype) !void {
try w.objectField("nodeId");
try w.write(node.id);
try w.objectField("backendNodeId");
try w.write(node.id);
const dom_node = node.dom;
if (dom_node._parent) |dom_parent| {
const parent_node = try self.registry.register(dom_parent);
try w.objectField("parentId");
try w.write(parent_node.id);
}
if (dom_node.is(DOMNode.Element)) |element| {
if (element.hasAttributes()) {
try w.objectField("attributes");
try w.beginArray();
for (element.attributeEntries()) |*attr| {
try w.write(attr.name());
try w.write(attr.value());
}
try w.endArray();
}
try w.objectField("localName");
try w.write(element.getLocalName());
// Chrome names the hosted frame on frame-owner elements and the own
// frame on the document element; that's how clients pair an <iframe>
// with its Page.frameAttached id.
if (element.is(IFrame)) |iframe| {
if (iframe.getContentDocument()) |document| {
if (document._frame) |child| {
try w.objectField("frameId");
try w.write(&id.toFrameId(child._frame_id));
}
}
} else if (dom_node._parent) |dom_parent| {
if (dom_parent._type == .document) {
if (dom_parent.subtype(DOMNode.Document)._frame) |frame| {
try w.objectField("frameId");
try w.write(&id.toFrameId(frame._frame_id));
}
}
}
} else {
try w.objectField("localName");
try w.write("");
}
try w.objectField("nodeType");
try w.write(dom_node.getNodeType());
try w.objectField("nodeName");
var name_buf: [1024]u8 = undefined;
var name_fba = std.heap.FixedBufferAllocator.init(&name_buf);
try w.write(dom_node.getNodeName(name_fba.allocator()));
try w.objectField("nodeValue");
if (dom_node.getNodeValue()) |nv| {
try w.write(nv.str());
} else {
try w.write("");
}
if (dom_node.is(DOMNode.ShadowRoot)) |shadow| {
try w.objectField("shadowRootType");
try w.write(@tagName(shadow._mode));
}
if (include_child_count) {
try w.objectField("childNodeCount");
try w.write(dom_node.getChildrenCount());
}
try w.objectField("documentURL");
try w.write(null);
try w.objectField("baseURL");
try w.write(null);
try w.objectField("xmlVersion");
try w.write("");
try w.objectField("compatibilityMode");
try w.write("NoQuirksMode");
try w.objectField("isScrollable");
try w.write(false);
}
};
const testing = @import("testing.zig");
test "cdp Node: search list" {
var registry = NodeRegistry.init(testing.allocator);
defer registry.deinit();
var search_list = Search.List.init(testing.allocator, ®istry);
defer search_list.deinit();
{
// empty search list, noops
search_list.remove("0");
try testing.expectEqual(null, search_list.get("0"));
}
{
// empty nodes
const s1 = try search_list.create(&.{});
try testing.expectEqual("0", s1.name);
try testing.expectEqual(0, s1.node_ids.len);
const s2 = search_list.get("0").?;
try testing.expectEqual("0", s2.name);
try testing.expectEqual(0, s2.node_ids.len);
search_list.remove("0");
try testing.expectEqual(null, search_list.get("0"));
}
{
var page = try testing.pageTest("cdp/registry2.html", .{});
defer page.close();
const frame = page.frame().?;
var doc = frame.window._document;
{
const l1 = try doc.querySelectorAll(.wrap("a"), frame);
defer l1.deinit(frame.page);
const s1 = try search_list.create(l1._nodes);
try testing.expectEqual("1", s1.name);
try testing.expectEqualSlices(u32, &.{ 1, 2 }, s1.node_ids);
}
try testing.expectEqual(2, registry.lookup_by_id.count());
try testing.expectEqual(2, registry.lookup_by_node.count());
{
const l2 = try doc.querySelectorAll(.wrap("#a1"), frame);
defer l2.deinit(frame.page);
const s2 = try search_list.create(l2._nodes);
try testing.expectEqual("2", s2.name);
try testing.expectEqualSlices(u32, &.{1}, s2.node_ids);
}
{
const l3 = try doc.querySelectorAll(.wrap("#a2"), frame);
defer l3.deinit(frame.page);
const s3 = try search_list.create(l3._nodes);
try testing.expectEqual("3", s3.name);
try testing.expectEqualSlices(u32, &.{2}, s3.node_ids);
}
try testing.expectEqual(2, registry.lookup_by_id.count());
try testing.expectEqual(2, registry.lookup_by_node.count());
}
}
test "cdp Node: Writer" {
var registry = NodeRegistry.init(testing.allocator);
defer registry.deinit();
var page = try testing.pageTest("cdp/registry3.html", .{});
defer page.close();
const frame = page.frame().?;
var doc = frame.window._document;
{
const node = try registry.register(doc.asNode());
const json = try std.json.Stringify.valueAlloc(testing.allocator, Writer{
.root = node,
.depth = 0,
.exclude_root = false,
.registry = ®istry,
.frame = frame,
}, .{});
defer testing.allocator.free(json);
try testing.expectJson(.{
.nodeId = 1,
.backendNodeId = 1,
.nodeType = 9,
.nodeName = "#document",
.localName = "",
.nodeValue = "",
.documentURL = null,
.baseURL = null,
.xmlVersion = "",
.isScrollable = false,
.compatibilityMode = "NoQuirksMode",
.childNodeCount = 1,
.children = &.{.{
.nodeId = 2,
.backendNodeId = 2,
.nodeType = 1,
.nodeName = "HTML",
.localName = "html",
.nodeValue = "",
.childNodeCount = 2,
.documentURL = null,
.baseURL = null,
.xmlVersion = "",
.compatibilityMode = "NoQuirksMode",
.isScrollable = false,
}},
}, json);
}
{
const node = registry.lookup_by_id.get(2).?;
const json = try std.json.Stringify.valueAlloc(testing.allocator, Writer{
.root = node,
.depth = 1,
.exclude_root = false,
.registry = ®istry,
.frame = frame,
}, .{});
defer testing.allocator.free(json);
try testing.expectJson(.{
.nodeId = 2,
.backendNodeId = 2,
.nodeType = 1,
.nodeName = "HTML",
.localName = "html",
.nodeValue = "",
.childNodeCount = 2,
.documentURL = null,
.baseURL = null,
.xmlVersion = "",
.compatibilityMode = "NoQuirksMode",
.isScrollable = false,
.children = &.{ .{
.nodeId = 3,
.backendNodeId = 3,
.nodeType = 1,
.nodeName = "HEAD",
.localName = "head",
.nodeValue = "",
.childNodeCount = 0,
.documentURL = null,
.baseURL = null,
.xmlVersion = "",
.compatibilityMode = "NoQuirksMode",
.isScrollable = false,
.parentId = 2,
}, .{
.nodeId = 4,
.backendNodeId = 4,
.nodeType = 1,
.nodeName = "BODY",
.localName = "body",
.nodeValue = "",
.childNodeCount = 3,
.documentURL = null,
.baseURL = null,
.xmlVersion = "",
.compatibilityMode = "NoQuirksMode",
.isScrollable = false,
.parentId = 2,
} },
}, json);
}
{
const node = registry.lookup_by_id.get(2).?;
const json = try std.json.Stringify.valueAlloc(testing.allocator, Writer{
.root = node,
.depth = -1,
.exclude_root = true,
.registry = ®istry,
.frame = frame,
}, .{});
defer testing.allocator.free(json);
try testing.expectJson(&.{ .{
.nodeId = 3,
.backendNodeId = 3,
.nodeType = 1,
.nodeName = "HEAD",
.localName = "head",
.nodeValue = "",
.childNodeCount = 0,
.documentURL = null,
.baseURL = null,
.xmlVersion = "",
.compatibilityMode = "NoQuirksMode",
.isScrollable = false,
.parentId = 2,
}, .{
.nodeId = 4,
.backendNodeId = 4,
.nodeType = 1,
.nodeName = "BODY",
.localName = "body",
.nodeValue = "",
.childNodeCount = 3,
.documentURL = null,
.baseURL = null,
.xmlVersion = "",
.compatibilityMode = "NoQuirksMode",
.isScrollable = false,
.children = &.{ .{
.nodeId = 5,
.localName = "a",
.childNodeCount = 0,
.attributes = &.{ "id", "a1" },
.parentId = 4,
}, .{
.nodeId = 6,
.localName = "div",
.childNodeCount = 1,
.parentId = 4,
.children = &.{.{
.nodeId = 7,
.localName = "a",
.childNodeCount = 0,
.parentId = 6,
.attributes = &.{ "id", "a2" },
}},
}, .{
.nodeId = 8,
.backendNodeId = 8,
.nodeName = "#text",
.localName = "",
.childNodeCount = 0,
.parentId = 4,
.nodeValue = "\n",
} },
} }, json);
}
}
test "cdp Node: Writer shadowRoots" {
var registry = NodeRegistry.init(testing.allocator);
defer registry.deinit();
var page = try testing.pageTest("cdp/shadow_dom.html", .{});
defer page.close();
const frame = page.frame().?;
const host = try registry.register(frame.window._document.getElementById("coupon").?.asNode());
{
const json = try std.json.Stringify.valueAlloc(testing.allocator, Writer{
.root = host,
.depth = -1,
.exclude_root = false,
.pierce = true,
.registry = ®istry,
.frame = frame,
}, .{});
defer testing.allocator.free(json);
try testing.expectJson(.{
.localName = "x-coupon",
.childNodeCount = 2,
.shadowRoots = &.{.{
.nodeType = 11,
.nodeName = "#document-fragment",
.shadowRootType = "open",
.childNodeCount = 3,
.children = &.{
.{ .localName = "label", .childNodeCount = 2 },
.{ .localName = "button", .attributes = &.{ "id", "apply" } },
.{ .localName = "slot" },
},
}},
}, json);
}
{
// without pierce, the root is listed but not walked
const json = try std.json.Stringify.valueAlloc(testing.allocator, Writer{
.root = host,
.depth = -1,
.exclude_root = false,
.registry = ®istry,
.frame = frame,
}, .{});
defer testing.allocator.free(json);
try testing.expectJson(.{ .shadowRoots = &.{.{ .shadowRootType = "open", .childNodeCount = 3 }} }, json);
try testing.expect(std.mem.find(u8, json, "\"apply\"") == null);
}
{
// closed roots too
const closed = try registry.register(frame.window._document.getElementById("closed-host").?.asNode());
const json = try std.json.Stringify.valueAlloc(testing.allocator, Writer{
.root = closed,
.depth = -1,
.exclude_root = false,
.pierce = true,
.registry = ®istry,
.frame = frame,
}, .{});
defer testing.allocator.free(json);
try testing.expectJson(.{ .shadowRoots = &.{.{
.shadowRootType = "closed",
.children = &.{.{ .localName = "button", .attributes = &.{ "id", "in-closed" } }},
}} }, json);
}
}