Skip to content

Lightpanda: the headless browser designed for AI and automation

12.7 KB
Raw Blame
// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
//
// Francis Bouvier <[email protected]>
// Pierre Tachoire <[email protected]>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

const std = @import("std");

const CDP = @import("CDP.zig");
const Inbox = @import("../../Inbox.zig");
const Driver = @import("../Driver.zig");
const sys_net = @import("../../sys/net.zig");

pub const base = @import("../../testing.zig");

const json = std.json;
const posix = std.posix;

pub const io = base.io;
pub const allocator = base.allocator;
pub const expectJson = base.expectJson;
pub const expect = std.testing.expect;
pub const expectEqual = base.expectEqual;
pub const expectError = base.expectError;
pub const expectEqualSlices = base.expectEqualSlices;
pub const pageTest = base.pageTest;
pub const newString = base.newString;
pub const expectLog = base.expectLog;
pub const silenceLog = base.silenceLog;
pub const tmpPath = base.tmpPath;
pub const arena_allocator = base.arena_allocator;

pub const TestContext = struct {
    read_at: usize = 0,
    read_buf: [1024 * 32]u8 = undefined,
    cdp_: CDP = undefined,
    cdp_initialized: bool = false,
    inbox: Inbox = .{},
    driver: Driver = undefined,
    cdp_socket: posix.socket_t,
    socket: posix.socket_t,
    received: std.ArrayList(json.Value) = .empty,
    received_raw: std.ArrayList([]const u8) = .empty,

    pub fn deinit(self: *TestContext) void {
        if (self.cdp_initialized) {
            self.driver.detach();
            self.cdp_.deinit();
        }
        self.inbox.deinit();
        _ = std.c.close(self.socket);
        _ = std.c.close(self.cdp_socket);
        base.reset();
    }

    pub fn cdp(self: *TestContext) *CDP {
        if (!self.cdp_initialized) {
            self.cdp_.init(base.test_app, self.cdp_socket, &self.inbox) catch |err| @panic(@errorName(err));
            self.cdp_initialized = true;
            self.driver = .init(.{ .cdp = &self.cdp_ }, &self.inbox);
            self.driver.attach();
        }
        return &self.cdp_;
    }

    const BrowserContextOpts = struct {
        id: ?[]const u8 = null,
        target_id: ?[14]u8 = null,
        session_id: ?[]const u8 = null,
        url: ?[:0]const u8 = null,
    };
    pub fn loadBrowserContext(self: *TestContext, opts: BrowserContextOpts) !*CDP.BrowserContext {
        var c = self.cdp();
        c.disposeBrowserContext();

        _ = try c.createBrowserContext();
        var bc = &c.browser_context.?;

        if (opts.id) |id| {
            bc.id = id;
        }

        if (opts.target_id) |tid| {
            bc.target_id = tid;
        }

        if (opts.session_id) |sid| {
            _ = try bc.attachPrimarySession(sid);
        }

        if (opts.url) |url| {
            if (bc.session_id == null) {
                _ = try bc.attachPrimarySession("SID-X");
            }
            if (bc.target_id == null) {
                bc.target_id = "TID-000000000Z".*;
            }
            const page = try bc.session.createPage();
            const full_url = try base.arena_allocator.printSentinel(
                "http://127.0.0.1:9582/src/browser/tests/{s}",
                .{url},
                0,
            );
            try page.navigate(full_url, .{});
            try waitForPage(bc);
        }
        return bc;
    }

    pub fn processMessage(self: *TestContext, msg: anytype) !void {
        const json_message: []const u8 = blk: {
            if (@typeInfo(@TypeOf(msg)) != .pointer) {
                break :blk try std.json.Stringify.valueAlloc(base.arena_allocator, msg, .{});
            }
            // assume this is a string we want to send as-is, if it isn't, we'll
            // get a compile error, so no big deal.
            break :blk msg;
        };
        return self.cdp().processMessage(json_message);
    }

    pub fn expectSentCount(self: *TestContext, expected: usize) !void {
        try self.read();
        try expectEqual(expected, self.received.items.len);
    }

    const ExpectResultOpts = struct {
        id: ?usize = null,
        index: ?usize = null,
        session_id: ?[]const u8 = null,
    };
    pub fn expectSentResult(self: *TestContext, expected: anytype, opts: ExpectResultOpts) !void {
        const expected_result = .{
            .id = opts.id,
            .result = if (comptime @typeInfo(@TypeOf(expected)) == .null) struct {}{} else expected,
            .sessionId = opts.session_id,
        };

        try self.expectSent(expected_result, .{ .index = opts.index });
    }

    const ExpectEventOpts = struct {
        index: ?usize = null,
        session_id: ?[]const u8 = null,
    };
    pub fn expectSentEvent(self: *TestContext, method: []const u8, params: anytype, opts: ExpectEventOpts) !void {
        const expected_event = .{
            .method = method,
            .params = if (comptime @typeInfo(@TypeOf(params)) == .null) struct {}{} else params,
            .sessionId = opts.session_id,
        };

        try self.expectSent(expected_event, .{ .index = opts.index });
    }

    const ExpectErrorOpts = struct {
        id: ?usize = null,
        index: ?usize = null,
    };
    pub fn expectSentError(self: *TestContext, code: i32, message: []const u8, opts: ExpectErrorOpts) !void {
        const expected_message = .{
            .id = opts.id,
            .@"error" = .{ .code = code, .message = message },
        };
        try self.expectSent(expected_message, .{ .index = opts.index });
    }

    const SentOpts = struct {
        index: ?usize = null,
    };
    pub fn expectSent(self: *TestContext, expected: anytype, opts: SentOpts) !void {
        const expected_json = blk: {
            // Zig makes this hard. When sendJSON is called, we're sending an anytype.
            // We can't record that in an ArrayList(???), so we serialize it to JSON.
            // Now, ideally, we could just take our expected structure, serialize it to
            // json and check if the two are equal.
            // Except serializing to JSON isn't deterministic.
            // So we serialize the JSON then we deserialize to json.Value. And then we can
            // compare our anytype expectation with the json.Value that we captured

            const serialized = try json.Stringify.valueAlloc(base.arena_allocator, expected, .{
                .whitespace = .indent_2,
                .emit_null_optional_fields = false,
            });

            break :blk try std.json.parseFromSliceLeaky(json.Value, base.arena_allocator, serialized, .{});
        };

        try self.read();
        const start: std.Io.Timestamp = .now(io, .awake);
        while (true) {
            for (self.received.items, 0..) |received, i| {
                if (try base.isEqualJson(expected_json, received) == false) {
                    continue;
                }

                if (opts.index) |expected_index| {
                    if (expected_index != i) {
                        std.debug.print("Expected message at index: {d}, was at index: {d}\n", .{ expected_index, i });
                        self.dumpReceived();
                        return error.ErrorAtWrongIndex;
                    }
                }
                return;
            }

            if (start.durationTo(.now(io, .awake)).toMilliseconds() >= 2000) {
                break;
            }
            try self.tick();
            try self.read();
        }
        self.dumpReceived();
        return error.ErrorNotFound;
    }

    fn dumpReceived(self: *const TestContext) void {
        std.debug.print("CDP Message Received ({d})\n", .{self.received_raw.items.len});
        for (self.received_raw.items, 0..) |received, i| {
            std.debug.print("===Message: {d}===\n{s}\n\n", .{ i, received });
        }
    }

    pub fn getSentMessage(self: *TestContext, index: usize) !?json.Value {
        if (index >= self.received.items.len) {
            try self.read();
        }
        if (index < self.received.items.len) {
            return self.received.items[index];
        }
        return null;
    }

    /// Advances the page so asynchronous events get a chance to be sent.
    fn tick(self: *TestContext) !void {
        if (self.cdp_.browser_context) |*bc| {
            if (bc.session.hasPage()) {
                var runner = bc.session.runner(.{});
                switch (try runner.tickForFrame(bc.page_handle.?.frame_id, 20, .{ .until = .done })) {
                    .ok => return,
                    .done => {},
                }
            }
        }
        io.sleep(.fromMilliseconds(1), .awake) catch {};
    }

    fn read(self: *TestContext) !void {
        while (true) {
            const n = posix.read(self.socket, self.read_buf[self.read_at..]) catch |err| switch (err) {
                error.WouldBlock => return,
                else => return err,
            };

            if (n == 0) {
                return;
            }

            self.read_at += n;

            // Try to parse complete WebSocket frames
            var pos: usize = 0;
            while (pos < self.read_at) {
                // Need at least 2 bytes for header
                if (self.read_at - pos < 2) break;

                const opcode = self.read_buf[pos] & 0x0F;
                const payload_len_byte = self.read_buf[pos + 1] & 0x7F;

                var header_size: usize = 2;
                var payload_len: usize = payload_len_byte;

                if (payload_len_byte == 126) {
                    if (self.read_at - pos < 4) break;
                    payload_len = std.mem.readInt(u16, self.read_buf[pos + 2 ..][0..2], .big);
                    header_size = 4;
                }
                // Skip 8-byte length case (127) - not needed

                const frame_size = header_size + payload_len;
                if (self.read_at - pos < frame_size) break;

                // We have a complete frame - process text (1) or binary (2), skip others
                if (opcode == 1 or opcode == 2) {
                    const payload = self.read_buf[pos + header_size ..][0..payload_len];
                    const parsed = try std.json.parseFromSliceLeaky(json.Value, base.arena_allocator, payload, .{});
                    try self.received.append(base.arena_allocator, parsed);
                    try self.received_raw.append(base.arena_allocator, try base.arena_allocator.dupe(u8, payload));
                }

                pos += frame_size;
            }

            // Move remaining partial data to beginning of buffer
            if (pos > 0 and pos < self.read_at) {
                std.mem.copyForwards(u8, &self.read_buf, self.read_buf[pos..self.read_at]);
                self.read_at -= pos;
            } else if (pos == self.read_at) {
                self.read_at = 0;
            }
        }
    }
};

pub fn context() !TestContext {
    var pair: [2]posix.socket_t = undefined;
    const rc = std.c.socketpair(posix.AF.LOCAL, posix.SOCK.STREAM, 0, &pair);
    if (rc != 0) {
        return error.SocketPairFailed;
    }

    errdefer {
        _ = std.c.close(pair[0]);
        _ = std.c.close(pair[1]);
    }

    // Replies are written synchronously, so reads never need to wait for data.
    const flags = try sys_net.fcntl(pair[0], posix.F.GETFL, 0);
    _ = try sys_net.fcntl(pair[0], posix.F.SETFL, flags | @as(u32, @bitCast(posix.O{ .NONBLOCK = true })));

    const timeout = std.mem.toBytes(posix.timeval{ .sec = 0, .usec = 5_000 });
    try posix.setsockopt(pair[1], posix.SOL.SOCKET, posix.SO.RCVTIMEO, &timeout);
    try posix.setsockopt(pair[1], posix.SOL.SOCKET, posix.SO.SNDTIMEO, &timeout);

    try posix.setsockopt(pair[0], posix.SOL.SOCKET, posix.SO.RCVBUF, &std.mem.toBytes(@as(c_int, 32_768)));
    try posix.setsockopt(pair[0], posix.SOL.SOCKET, posix.SO.SNDBUF, &std.mem.toBytes(@as(c_int, 32_768)));
    try posix.setsockopt(pair[1], posix.SOL.SOCKET, posix.SO.RCVBUF, &std.mem.toBytes(@as(c_int, 32_768)));
    try posix.setsockopt(pair[1], posix.SOL.SOCKET, posix.SO.SNDBUF, &std.mem.toBytes(@as(c_int, 32_768)));

    return .{
        .cdp_socket = pair[1],
        .socket = pair[0],
    };
}

pub fn waitForPage(bc: *CDP.BrowserContext) !void {
    var runner = bc.session.runner(.{});
    try runner.waitForFrame(bc.page_handle.?.frame_id, 2000, .{ .until = .done });
}