Skip to content

Lightpanda: the headless browser designed for AI and automation

18.5 KB
Raw Blame
//
// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
// Francis Bouvier <[email protected]>
// Pierre Tachoire <[email protected]>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

const std = @import("std");
const lp = @import("lightpanda");

const App = @import("../../App.zig");
const Inbox = @import("../../Inbox.zig");

const sys_net = @import("../../sys/net.zig");
const uuidv4 = @import("../../id.zig").uuidv4;
const Notification = @import("../../Notification.zig");
const NodeRegistry = @import("../../NodeRegistry.zig");

const Browser = @import("../../browser/Browser.zig");
const Session = @import("../../browser/Session.zig");

const http = @import("../http.zig");
const Link = @import("../Link.zig");
const Server = @import("../Server.zig");

const script = @import("script.zig");
const http_command = @import("http_command.zig");
const execute = @import("execute.zig");
const remote_value = @import("remote_value.zig");

const posix = std.posix;
const Allocator = std.mem.Allocator;

const BiDi = @This();

app: *App,

// The websocket, when a client is connected. Null for an HTTP WebDriver
// session (until it optionally connects via WebSocket)
link: ?*Link,

// The worker's mailbox, owned by the Worker and thus outliving the link.
inbox: *Inbox,

// WebDriver can be BiDi only, or HTTP WebDriver + BiDi or HTTP WebDriver only.
mode: Mode,

// Re-used arena for processing a message. Works because we strictly process
// one message at a time.
message_arena: std.heap.ArenaAllocator,

// Allocations that live for the whole session (subscriptions).
session_arena: std.heap.ArenaAllocator,

// The active session
session_id: ?[36]u8,

browser: Browser,
notification: *Notification,

// browsing context is the page
browsing_context: ?@import("browsing_context.zig").Context = null,

// A user context is a Session. There's a "default" and we allow 1 more to
// be created (limited because a Browser can only have 1 Session).
user_context: UserContext,

// Issues the script module's sharedIds
node_registry: NodeRegistry,

// Values the client owns via `resultOwnership: "root"`
handles: remote_value.Handles,

// Commands awaiting promise resolution
pending: std.ArrayList(*script.Pending) = .empty,

// The HTTP session's execute/sync and execute/async, awaiting a promise or
// the callback an async script was handed.
execute_pending: std.ArrayList(*execute.Pending) = .empty,

// The HTTP session's timeouts. BiDi has no equivalent.
timeouts: Timeouts = .{},

input_state: @import("input.zig").State = .{},

subscriptions: std.ArrayList(Subscription) = .empty,

const Subscription = struct {
    id: [36]u8,
    event: []const u8,
};

pub const Mode = union(enum) {
    // Directly created via websocket upgrade, tied to the websocket's lifetime
    bidi_only: void,

    // created via HTTP, a websocket may or may not web associated with it (it
    // can come and go), but the lifetime is explicit: either removed via HTTP
    // (DELETE /session/:id) or by the HTTP reaper
    http: *Server.Worker,
};

pub const Reply = union(enum) {
    bidi: u64, // reply goes to websocket with this id
    http: void, // reply is sent as an http response
};

// What a worker is born from: a websocket upgrade (the session comes later
// via session.new) or an HTTP session (a websocket may come later via
// GET /session/{id}); never both.
pub const Origin = union(enum) {
    socket: posix.socket_t,
    session: struct { id: [36]u8, worker: *Server.Worker },
};

const InputMessage = struct {
    id: ?u64 = null,
    method: ?[]const u8 = null,
};

pub const Timeouts = struct {
    script: ?u32 = 30_000,
    pageLoad: u32 = 300_000,
    implicit: u32 = 0,
};

pub fn init(self: *BiDi, app: *App, inbox: *Inbox, origin: Origin) !void {
    const allocator = app.allocator;
    {
        // this is documentation, and future-proofing, to show exactly where
        // the socket's ownership is
        errdefer if (origin == .socket) {
            sys_net.close(origin.socket);
        };

        self.* = .{
            .app = app,
            .link = null,
            .inbox = inbox,
            .mode = switch (origin) {
                .socket => .bidi_only,
                .session => |session| .{ .http = session.worker },
            },
            .browser = undefined,
            .user_context = undefined,
            .notification = undefined,
            .session_id = switch (origin) {
                .socket => null,
                .session => |session| session.id,
            },
            .node_registry = .init(allocator),
            .handles = .{ .allocator = allocator },
            .message_arena = std.heap.ArenaAllocator.init(allocator),
            .session_arena = std.heap.ArenaAllocator.init(allocator),
        };
    }

    // Link.create takes ownership of the socket
    switch (origin) {
        .socket => |socket| self.link = try Link.create(app, socket, .bidi, inbox),
        .session => {},
    }
    errdefer if (self.link) |l| l.destroy();

    try self.browser.init(app, .{});
    errdefer self.browser.deinit();

    self.notification = try Notification.init(allocator);
    errdefer self.notification.deinit();

    try self.newUserContext("default");
    try @import("browsing_context.zig").registerNotifications(self);
}

pub fn deinit(self: *BiDi) void {
    const allocator = self.app.allocator;

    self.notification.unregisterAll(self);

    // Cancel first, so that any completions during session teardown are still valid
    script.Pending.cancelAll(self);
    execute.Pending.cancelAll(self);
    self.handles.deinit();
    self.browser.closeSession();
    // Now we can destroy
    script.Pending.destroyAll(self);
    execute.Pending.destroyAll(self);
    self.pending.deinit(allocator);
    self.execute_pending.deinit(allocator);
    self.input_state.deinit(allocator);

    self.node_registry.deinit();
    self.notification.deinit();
    self.browser.deinit();
    // The loop let go of the link before we got here (Server.Worker.run)
    if (self.link) |l| {
        l.destroy();
    }
    self.message_arena.deinit();
    self.session_arena.deinit();
}

// Worker thread, from the inbox: the loop is already reading from it.
pub fn adoptLink(self: *BiDi, l: *Link) void {
    if (self.link != null) {
        // the loop only hands one over once it has seen the previous one
        // released (Server.Worker.link is null)
        if (comptime lp.IS_DEBUG) {
            lp.assert(false, "BiDi.adoptLink held", .{});
        }
        l.destroy();
        return;
    }
    self.link = l;
}

// Worker thread. The link is gone (peer closed, or the loop dropped it).
// Returns true when the worker is done with it: a bidi-only session dies
// with its connection, an HTTP session just drops the link and waits
// for the next one, or for DELETE / the idle reaper.
pub fn onLinkGone(self: *BiDi) bool {
    const worker = switch (self.mode) {
        .bidi_only => return true,
        .http => |worker| worker,
    };
    self.releaseLink(worker);
    return false;
}

fn releaseLink(self: *BiDi, worker: *Server.Worker) void {
    const l = self.link orelse {
        // the loop only tells us the link is gone while we hold it
        if (comptime lp.IS_DEBUG) {
            lp.assert(false, "BiDi.releaseLink empty", .{});
        }
        return;
    };
    self.link = null;
    // blocks until the loop has stopped reading from it
    worker.releaseLink();
    l.destroy();
}

pub fn replaceSession(self: *BiDi, id: []const u8) !void {
    self.resetRealm();
    try self.newUserContext(id);
}

fn newUserContext(self: *BiDi, id: []const u8) !void {
    const session = try self.browser.newSession(self.notification);
    self.user_context = .{ .session = session, .id_len = @intCast(id.len), .id_buf = undefined };
    @memcpy(self.user_context.id_buf[0..id.len], id);
}

const UserContext = struct {
    id_len: u8,
    id_buf: [36]u8, // "default" or uuid
    session: *Session,

    pub fn id(self: *const UserContext) []const u8 {
        return self.id_buf[0..self.id_len];
    }

    pub fn isDefault(self: *const UserContext) bool {
        return std.mem.eql(u8, self.id(), "default");
    }
};

pub fn resetRealm(self: *BiDi) void {
    script.Pending.realmReset(self);
    execute.Pending.realmReset(self);
    self.handles.releaseAll();
    self.node_registry.reset();
    if (self.browsing_context) |*ctx| {
        if (ctx.realm_announced) {
            ctx.realm_announced = false;
            self.sendEvent("script.realmDestroyed", .{ .realm = &ctx.realm_id }) catch {};
        }
        uuidv4(&ctx.realm_id);
    }
}

// Dispatch a single BiDi frame from the inbox. Unlike CDP, frames aren't
// parsed on the Network thread, so `data` is the raw JSON.
pub fn onMessage(self: *BiDi, data: []const u8) anyerror!void {
    if (self.browser.env.terminatePending()) {
        return;
    }

    defer _ = self.message_arena.reset(.{ .retain_with_limit = 4096 });
    const arena = self.message_arena.allocator();

    const input = std.json.parseFromSliceLeaky(InputMessage, arena, data, .{
        .ignore_unknown_fields = true,
    }) catch {
        return self.sendError(null, "invalid argument", "invalid JSON message");
    };

    const id = input.id orelse {
        return self.sendError(null, "invalid argument", "missing command id");
    };
    var cmd: Command = .{
        .bidi = self,
        .arena = arena,
        .input = .{ .bidi = .{ .id = id, .json = data } },
    };

    const method = input.method orelse {
        return cmd.sendError("invalid argument", "missing command method");
    };
    lp.metrics.serve_commands.incr(.bidi);

    // A BiDi method is always "<module>.<command>".
    const i = std.mem.findScalar(u8, method, '.') orelse {
        return unknownCommand(&cmd, method);
    };
    const module = std.meta.stringToEnum(enum {
        session,
        script,
        browser,
        browsingContext,
        input,
        storage,
        emulation,
    }, method[0..i]) orelse return unknownCommand(&cmd, method);

    // Only the session module is reachable without a session (it's what
    // creates one); it gates its own commands.
    if (self.session_id == null and module != .session) {
        return cmd.sendError("invalid session id", "no active session");
    }

    const action = method[i + 1 ..];
    const result = switch (module) {
        .session => @import("session.zig").processMessage(&cmd, action),
        .script => @import("script.zig").processMessage(&cmd, action),
        .browser => @import("browser.zig").processMessage(&cmd, action),
        .browsingContext => @import("browsing_context.zig").processMessage(&cmd, action),
        .input => @import("input.zig").processMessage(&cmd, action),
        .storage => @import("storage.zig").processMessage(&cmd, action),
        .emulation => @import("emulation.zig").processMessage(&cmd, action),
    };
    result catch |err| {
        if (err == error.UnknownCommand and cmd.answered == false) {
            return unknownCommand(&cmd, method);
        }
        cmd.failed(err);
    };
}

fn unknownCommand(cmd: *Command, method: []const u8) !void {
    lp.metrics.serve_unknown_commands.incr(.bidi);
    return cmd.sendError("unknown command", method);
}

// Dispatch an HTTP WebDriver command from the inbox. Its connection is parked
// until we respond.
pub fn onHttpCommand(self: *BiDi, command: http_command.Command) void {
    if (self.mode != .http) {
        // the loop only parks commands on an HTTP session's worker
        if (comptime lp.IS_DEBUG) {
            lp.assert(false, "BiDi.onHttpCommand mode", .{});
        }
        return;
    }
    if (self.browser.env.terminatePending()) {
        // the loop answers it once the worker releases
        return;
    }

    defer _ = self.message_arena.reset(.{ .retain_with_limit = 4096 });
    lp.metrics.serve_commands.incr(.bidi);

    var cmd: Command = .{
        .bidi = self,
        .arena = self.message_arena.allocator(),
        .input = .{ .http = command },
    };
    http_command.process(&cmd) catch |err| {
        cmd.failed(err);
    };
}

// One command being processed, from either transport. Handlers answer
// through it so they don't have to thread where the answer goes.
pub const Command = struct {
    bidi: *BiDi,
    arena: Allocator, // The message_arena; valid for the lifetime of the command.
    input: Input,
    answered: bool = false,

    pub const Input = union(enum) {
        // A websocket frame: the id is echoed back, and `params` is parsed
        // out of the raw json on demand.
        bidi: struct {
            id: u64,
            json: []const u8,
        },

        // Parsed on the loop, params included.
        http: http_command.Command,
    };

    pub fn reply(self: *const Command) Reply {
        return switch (self.input) {
            .bidi => |b| .{ .bidi = b.id },
            .http => .http,
        };
    }

    // Parses the websocket command's params object. Answers the client and
    // returns error.InvalidParams when the message has no params or they
    // don't match T, so callers can just `try`. `failed` ignores that error.
    pub fn params(self: *Command, comptime T: type) !T {
        const json = switch (self.input) {
            .bidi => |b| b.json,
            .http => {
                // HTTP handlers get their params from input.http
                if (comptime lp.IS_DEBUG) {
                    lp.assert(false, "Command.params http", .{});
                }
                return error.InvalidParams;
            },
        };
        const wrapper = std.json.parseFromSliceLeaky(struct { params: T }, self.arena, json, .{
            .ignore_unknown_fields = true,
        }) catch {
            try self.sendError("invalid argument", "invalid params");
            return error.InvalidParams;
        };
        return wrapper.params;
    }

    // `result` is in the reply's shape: a BiDi result, or HTTP's value.
    pub fn sendResult(self: *Command, result: anytype) !void {
        self.answered = true;
        return self.bidi.replyResult(self.reply(), result);
    }

    pub fn sendDone(self: *Command) !void {
        self.answered = true;
        return self.bidi.replyDone(self.reply());
    }

    pub fn sendError(self: *Command, code: []const u8, message: []const u8) !void {
        self.answered = true;
        return self.bidi.replyError(self.reply(), code, message);
    }

    // For an answer sent after the command is gone: whoever holds the reply
    // answers it.
    pub fn takeReply(self: *Command) Reply {
        self.answered = true;
        return self.reply();
    }

    // Events aren't tied to the command, but handlers that emit one always
    // have a cmd on hand.
    pub fn sendEvent(self: *const Command, method: []const u8, p: anytype) !void {
        return self.bidi.sendEvent(method, p);
    }

    fn failed(self: *Command, err: anyerror) void {
        if (self.answered) {
            if (err != error.InvalidParams) {
                // params answers its own error.InvalidParams
                lp.log.warn(.bidi, "command failed after answering", .{ .err = err, .reply = self.reply() });
            }
            return;
        }
        lp.log.err(.bidi, "command failed", .{ .err = err, .reply = self.reply() });
        self.sendError("unknown error", @errorName(err)) catch {};
    }
};

// An event is delivered when its name is subscribed exactly, or its module
// is ("browsingContext" covers "browsingContext.load").
fn subscribed(self: *const BiDi, name: []const u8) bool {
    for (self.subscriptions.items) |sub| {
        if (std.mem.eql(u8, sub.event, name)) {
            return true;
        }
        if (sub.event.len < name.len and name[sub.event.len] == '.' and std.mem.startsWith(u8, name, sub.event)) {
            return true;
        }
    }
    return false;
}

pub fn sendEvent(self: *BiDi, method: []const u8, p: anytype) !void {
    if (!self.subscribed(method)) {
        return;
    }
    return self.sendJSON(.{ .type = "event", .method = method, .params = p });
}

pub fn sendResult(self: *BiDi, id: u64, result: anytype) !void {
    return self.sendJSON(.{ .type = "success", .id = id, .result = result });
}

pub fn sendError(self: *BiDi, id: ?u64, code: []const u8, message: []const u8) !void {
    return self.sendJSON(.{ .type = "error", .id = id, .@"error" = code, .message = message });
}

pub fn replyResult(self: *BiDi, reply: Reply, result: anytype) !void {
    switch (reply) {
        .bidi => |id| return self.sendResult(id, result),
        .http => return self.respondHTTP(result),
    }
}

// A command's empty success: BiDi's {}, HTTP's null.
pub fn replyDone(self: *BiDi, reply: Reply) !void {
    switch (reply) {
        .bidi => |id| return self.sendResult(id, struct {}{}),
        .http => return self.respondHTTP(null),
    }
}

pub fn replyError(self: *BiDi, reply: Reply, code: []const u8, message: []const u8) !void {
    switch (reply) {
        .bidi => |id| return self.sendError(id, code, message),
        .http => return self.respondHTTPStatus(http.webDriverErrorStatus(code), .{
            .@"error" = code,
            .message = message,
            .stacktrace = "",
        }),
    }
}

// {"value": value} to the HTTP request parked on our worker.
pub fn respondHTTP(self: *BiDi, value: anytype) !void {
    return self.respondHTTPStatus(.ok, value);
}

fn respondHTTPStatus(self: *BiDi, status: std.http.Status, value: anytype) !void {
    const worker = switch (self.mode) {
        .http => |worker| worker,
        .bidi_only => return, // an .http reply only comes from onHttpCommand, which checks
    };
    const arena = try self.app.arena_pool.acquire(.small, "http response");
    errdefer arena.release();
    const bytes = try http.webDriverResponse(arena, status, value);
    worker.respond(.{ .arena = arena, .bytes = bytes });
}

// Without a link there's nobody to tell: an HTTP session between
// connections drops events and late results (a navigate that completes
// after the client went away).
fn sendJSON(self: *BiDi, message: anytype) !void {
    const l = self.link orelse return;
    return l.sendJSON(message, .{});
}