Skip to content

Lightpanda: the headless browser designed for AI and automation

18.9 KB
Raw Blame
// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
//
// Francis Bouvier <[email protected]>
// Pierre Tachoire <[email protected]>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

//! Reference partials for the docs site (lightpanda.io/docs), written by
//! `zig build docs`. Each is an MDX fragment with no frontmatter that a
//! handwritten docs page imports, rendered from the registries the runtime
//! serves (`lp.tools`, the MCP tool list, the REPL slash commands, the
//! PandaScript builtins) so the reference can't drift from the code.

const std = @import("std");
const lp = @import("lightpanda");
const Schema = lp.Schema;
const SlashCommand = @import("agent/SlashCommand.zig");
const mcp_tools = @import("mcp/tools.zig");
const mcp_resources = @import("mcp/resources.zig");
const protocol = @import("mcp/protocol.zig");

const Tool = lp.tools.Tool;
const Writer = std.Io.Writer;

pub const Partial = struct {
    name: []const u8,
    write: *const fn (w: *Writer) Writer.Error!void,
};

pub const partials = [_]Partial{
    .{ .name = "browser-tools.mdx", .write = writeBrowserTools },
    .{ .name = "mcp-tools.mdx", .write = writeMcpTools },
    .{ .name = "agent-commands.mdx", .write = writeAgentCommands },
    .{ .name = "pandascript-builtins.mdx", .write = writePandascriptBuiltins },
};

const banner = "{/* Generated by `zig build docs` from the lightpanda browser source. Do not edit. */}\n\n";
const browser_tools_page = "/reference/browser-tools";

/// Every browser tool, grouped, with its full description and parameters.
/// The MCP and agent partials link here instead of repeating it.
fn writeBrowserTools(w: *Writer) Writer.Error!void {
    try w.writeAll(banner);
    for (std.enums.values(Tool.Group)) |g| {
        try writeGroupHeading(w, g, "tools", "| Tool | Summary |\n|---|---|\n");
        for (Schema.all()) |*s| {
            if (s.tool.group() != g) continue;
            try w.print("| [`{s}`]({f}) | {f} |\n", .{ s.tool_name, Anchor{ .name = s.tool_name }, Cell{ .s = s.summary } });
        }
        try w.writeAll("\n");

        for (Schema.all()) |*s| {
            if (s.tool.group() != g) continue;
            try w.print("### {s}\n\n", .{s.tool_name});
            try writeText(w, s.description);
            try w.writeAll("\n\n");
            try writeParams(w, s.parameters);
            const traits = [_]Flag{
                .{ s.tool.isRecorded(), "Recorded in saved scripts." },
                .{ s.tool.navigatesToUrl(), "Navigates first when given `url`." },
                .{ s.tool.reportsPageState(), "Reports the resulting page state (`url`, `httpStatus`, `title`)." },
            };
            if (try writeFlags(w, &traits, " ")) try w.writeAll("\n\n");
        }
    }
}

/// The MCP view: titles and hints per browser tool, plus the MCP-only tools
/// and resources in full.
fn writeMcpTools(w: *Writer) Writer.Error!void {
    try w.writeAll(banner);
    for (std.enums.values(Tool.Group)) |g| {
        try writeGroupHeading(w, g, "tools", "| Tool | Title | Hints |\n|---|---|---|\n");
        for (std.enums.values(Tool), mcp_tools.browser_tool_list) |t, mt| {
            if (t.group() != g) continue;
            try w.print("| [`{s}`]({f}) | {f} | ", .{ mt.name, Anchor{ .page = browser_tools_page, .name = mt.name }, Cell{ .s = mt.title orelse "" } });
            try writeHints(w, mt);
            try w.writeAll(" |\n");
        }
        try w.writeAll("\n");
    }

    var arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
    defer arena.deinit();
    try w.writeAll(
        \\## MCP-only tools
        \\
        \\These tools exist only in the MCP server, not in the agent or in scripts. Over the [HTTP transport](/usage/mcp#http-transport), every tool call runs in the session the `Mcp-Session-Id` header names, falling back to the always-present `default` session; the `session_*` tools create, list and close those sessions.
        \\
        \\
    );
    for (mcp_tools.extra_tools) |mt| {
        try w.print("### {s}\n\n", .{mt.name});
        if (mt.title) |title| {
            try w.writeAll("**");
            try writeText(w, title);
            try w.writeAll(".** ");
        }
        try writeText(w, mt.description orelse "");
        try w.writeAll("\n\n");
        const schema = std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), mt.inputSchema, .{}) catch |err| {
            std.debug.panic("invalid input schema for MCP tool '{s}': {s}", .{ mt.name, @errorName(err) });
        };
        try writeParams(w, schema);
        try w.writeAll("Hints: ");
        try writeHints(w, mt);
        try w.writeAll("\n\n");
    }

    try w.writeAll(
        \\## Resources
        \\
        \\The host application reads these with `resources/read`. The page resources need a loaded page; the skill resources don't.
        \\
        \\| URI | MIME type | Description |
        \\|---|---|---|
        \\
    );
    for (mcp_resources.resource_list) |r| {
        try w.print("| `{s}` | `{s}` | {f} |\n", .{ r.uri, r.mimeType orelse "", Cell{ .s = r.description orelse r.name } });
    }
    try w.writeAll(
        \\
        \\```json copy
        \\{"jsonrpc":"2.0","id":2,"method":"resources/read",
        \\ "params":{"uri":"mcp://page/markdown"}}
        \\```
        \\
        \\The `markdown` tool and the `mcp://page/markdown` resource return the same content; the difference is who initiates. The agent calls **tools** during its workflow, while the host application reads **resources**, e.g. an IDE displaying page state in the background.
        \\
    );
}

/// The agent REPL's slash commands: meta, LLM-prompt, and one per browser
/// tool in the `key=value` syntax the REPL parses.
fn writeAgentCommands(w: *Writer) Writer.Error!void {
    try w.writeAll(banner);
    try w.writeAll("### Meta commands\n\nThese commands configure the REPL and never reach the model.\n\n| Command | Description |\n|---|---|\n");
    for (SlashCommand.meta_commands) |m| {
        try w.print("| `/{s}", .{m.name});
        if (m.hint.len > 0) try w.print(" {f}", .{Cell{ .s = m.hint, .in_code = true }});
        try w.print("` | {f} |\n", .{Cell{ .s = m.description }});
    }

    try w.writeAll("\n### LLM commands\n\nThese commands send a prebuilt prompt to the model, so they need a provider.\n\n| Command | Description |\n|---|---|\n");
    for (SlashCommand.llm_commands) |row| {
        try w.print("| `/{s}` | {f} |\n", .{ row.name, Cell{ .s = row.description } });
    }

    try w.writeAll("\n### Browser commands\n\nGiven its fields, every browser tool runs directly as a slash command, with no model call. A lone required argument can be passed positionally; the rest take `key=value` or a JSON object.\n\n");
    try w.writeAll("These take only `key=value`, and also accept a prompt in place of their fields:");
    var first = true;
    for (Schema.all()) |*s| {
        if (!Schema.takesPrompt(s.tool)) continue;
        try w.print("{s} `/{s}`", .{ if (first) "" else ",", s.tool_name });
        first = false;
    }
    try w.writeAll(". `/click the sign in button` asks the model for the selector, and the call it makes is what `/save` records.\n\n| Command | Description |\n|---|---|\n");
    for (std.enums.values(Tool.Group)) |g| {
        for (Schema.all()) |*s| {
            if (s.tool.group() != g) continue;
            try w.print("| [`/{s}", .{s.tool_name});
            for (s.hints) |h| try w.print(" {f}", .{Cell{ .s = h.fragment, .in_code = true }});
            try w.print("`]({f}) | {f} |\n", .{ Anchor{ .page = browser_tools_page, .name = s.tool_name }, Cell{ .s = s.summary } });
        }
    }
}

/// The methods `new Page()` carries in a PandaScript, i.e. the recorded tools
/// plus the runtime's own page methods, and the `console` global.
fn writePandascriptBuiltins(w: *Writer) Writer.Error!void {
    const table_head = "| Method | Call | Summary |\n|---|---|---|\n";
    try w.writeAll(banner);
    for (std.enums.values(Tool.Group)) |g| {
        if (!hasRecorded(g)) continue;
        try writeGroupHeading(w, g, "methods", table_head);
        for (Schema.all()) |*s| {
            if (s.tool.group() != g or !s.tool.isRecorded()) continue;
            try w.print("| [`{s}`]({f}) | `", .{ s.tool_name, Anchor{ .page = browser_tools_page, .name = s.tool_name } });
            try lp.skill.writeSignature(w, s);
            try w.print("` | {f} |\n", .{Cell{ .s = s.summary }});
        }
        try w.writeAll("\n");
    }

    try w.writeAll("## Page lifecycle\n\n" ++ table_head);
    for (lp.Runtime.page_methods) |m| {
        try w.print("| `{s}` | `page.{s}()` | {f} |\n", .{ m.name, m.name, Cell{ .s = m.summary } });
    }

    try w.writeAll("\n## Console\n\n| Method | Writes to |\n|---|---|\n");
    for (std.enums.values(lp.Runtime.ConsoleMethod)) |m| {
        try w.print("| `console.{s}` | {s} |\n", .{ @tagName(m), if (m.writesStderr()) "stderr" else "stdout" });
    }
}

fn hasRecorded(g: Tool.Group) bool {
    for (std.enums.values(Tool)) |t| {
        if (t.group() == g and t.isRecorded()) return true;
    }
    return false;
}

fn writeGroupHeading(w: *Writer, g: Tool.Group, noun: []const u8, table_head: []const u8) Writer.Error!void {
    const title, const scope = switch (g) {
        .navigation => .{ "Navigation and search", "bring a page into the browser, or save a URL to a file" },
        .reading => .{ "Reading the page", "read the loaded page without modifying it" },
        .extraction => .{ "Data extraction and scripting", "return structured results from the loaded page" },
        .interaction => .{ "Interacting with the page", "dispatch real DOM events on the page" },
        .waiting => .{ "Waiting", "block until the page reaches a condition" },
        .state => .{ "State and debugging", "inspect browser state outside the DOM" },
    };
    try w.print("## {s}\n\nThese {s} {s}.\n\n{s}", .{ title, noun, scope, table_head });
}

/// An `Argument | Type | Required | Description` table for a JSON Schema
/// object.
fn writeParams(w: *Writer, schema: std.json.Value) Writer.Error!void {
    const props = schema.object.get("properties") orelse return w.writeAll("No arguments.\n\n");
    if (props.object.count() == 0) return w.writeAll("No arguments.\n\n");
    const required: []const std.json.Value = if (schema.object.get("required")) |r| r.array.items else &.{};

    try w.writeAll("| Argument | Type | Required | Description |\n|---|---|---|---|\n");
    var it = props.object.iterator();
    while (it.next()) |entry| {
        const name = entry.key_ptr.*;
        const prop = entry.value_ptr.*.object;
        try w.print("| `{s}` | ", .{name});
        try writeType(w, prop);
        try w.print(" | {s} | ", .{if (isRequired(required, name)) "yes" else "no"});

        var sep: []const u8 = "";
        if (prop.get("description")) |d| {
            try w.print("{f}", .{Cell{ .s = d.string }});
            sep = " ";
        }
        if (prop.get("enum")) |values| {
            try w.print("{s}One of ", .{sep});
            for (values.array.items, 0..) |v, i| {
                if (i > 0) try w.writeAll(", ");
                try w.print("`{s}`", .{v.string});
            }
            try w.writeAll(".");
            sep = " ";
        }
        if (prop.get("default")) |d| {
            try w.print("{s}Defaults to `", .{sep});
            try std.json.Stringify.value(d, .{}, w);
            try w.writeAll("`.");
            sep = " ";
        }
        if (prop.get("properties")) |sub| {
            try w.print("{s}Keys:", .{sep});
            var sub_it = sub.object.iterator();
            var first = true;
            while (sub_it.next()) |s| : (first = false) {
                try w.print("{s} `{s}`", .{ if (first) "" else ",", s.key_ptr.* });
            }
            try w.writeAll(".");
        }
        try w.writeAll(" |\n");
    }
    try w.writeAll("\n");
}

fn writeType(w: *Writer, prop: std.json.ObjectMap) Writer.Error!void {
    const ty = prop.get("type") orelse return w.writeAll("any");
    if (std.mem.eql(u8, ty.string, "array")) {
        const item_ty = if (prop.get("items")) |items| if (items.object.get("type")) |t| t.string else "any" else "any";
        return w.print("{s}[]", .{item_ty});
    }
    try w.writeAll(ty.string);
}

fn isRequired(required: []const std.json.Value, name: []const u8) bool {
    for (required) |r| {
        if (std.mem.eql(u8, r.string, name)) return true;
    }
    return false;
}

fn writeHints(w: *Writer, mt: protocol.Tool) Writer.Error!void {
    const a = mt.annotations orelse protocol.ToolAnnotations{};
    const hints = [_]Flag{
        .{ a.readOnlyHint, "read-only" },
        .{ a.destructiveHint, "destructive" },
        .{ a.idempotentHint, "idempotent" },
        .{ a.openWorldHint, "open-world" },
        .{ mt.outputSchema != null, "page state" },
    };
    if (!try writeFlags(w, &hints, ", ")) try w.writeAll("—");
}

const Flag = struct { bool, []const u8 };

/// Writes the labels of the set flags, separated by `sep`. Returns whether
/// any was set.
fn writeFlags(w: *Writer, flags: []const Flag, sep: []const u8) Writer.Error!bool {
    var wrote = false;
    for (flags) |flag| {
        if (!flag[0]) continue;
        if (wrote) try w.writeAll(sep);
        try w.writeAll(flag[1]);
        wrote = true;
    }
    return wrote;
}

/// A link target to a tool's heading. Nextra slugs headings in lowercase, so
/// `### nodeDetails` is `#nodedetails`.
const Anchor = struct {
    page: []const u8 = "",
    name: []const u8,

    pub fn format(self: Anchor, w: *Writer) Writer.Error!void {
        try w.print("{s}#", .{self.page});
        for (self.name) |c| try w.writeByte(std.ascii.toLower(c));
    }
};

/// Text for a GFM table cell: `|` ends the cell even inside a code span, and
/// a newline ends the row. `in_code` is for text already inside a code span.
const Cell = struct {
    s: []const u8,
    in_code: bool = false,

    pub fn format(self: Cell, w: *Writer) Writer.Error!void {
        if (self.in_code) return writeCode(w, self.s, true);
        try writeEscaped(w, self.s, true);
    }
};

/// Prose for MDX: outside code spans and fences, `<` would open JSX and `{`
/// an expression.
fn writeText(w: *Writer, s: []const u8) Writer.Error!void {
    try writeEscaped(w, s, false);
}

fn writeCode(w: *Writer, s: []const u8, cell: bool) Writer.Error!void {
    if (!cell) return w.writeAll(s);
    for (s) |c| switch (c) {
        '|' => try w.writeAll("\\|"),
        else => try w.writeByte(c),
    };
}

fn writeEscaped(w: *Writer, s: []const u8, cell: bool) Writer.Error!void {
    var i: usize = 0;
    while (i < s.len) {
        if (s[i] == '`') {
            const run_end = std.mem.findNonePos(u8, s, i, "`") orelse s.len;
            const fence = s[i..run_end];
            try w.writeAll(fence);
            i = run_end;
            if (std.mem.findPos(u8, s, run_end, fence)) |close| {
                try writeCode(w, s[run_end..close], cell);
                try w.writeAll(fence);
                i = close + fence.len;
            }
            continue;
        }
        switch (s[i]) {
            '<' => try w.writeAll("&lt;"),
            '{' => try w.writeAll("\\{"),
            '}' => try w.writeAll("\\}"),
            '|' => try w.writeAll(if (cell) "\\|" else "|"),
            '\n' => try w.writeAll(if (cell) " " else "\n"),
            else => try w.writeByte(s[i]),
        }
        i += 1;
    }
}

const testing = std.testing;

fn render(write: *const fn (w: *Writer) Writer.Error!void) ![]u8 {
    var aw: Writer.Allocating = .init(testing.allocator);
    errdefer aw.deinit();
    try write(&aw.writer);
    return aw.toOwnedSlice();
}

test "docs: browser-tools documents every tool exactly once" {
    const out = try render(writeBrowserTools);
    defer testing.allocator.free(out);
    for (std.enums.values(Tool)) |t| {
        var buf: [64]u8 = undefined;
        const heading = try std.mem.print(&buf, "\n### {s}\n", .{@tagName(t)});
        try testing.expectEqual(1, std.mem.count(u8, out, heading));
    }
}

test "docs: mcp-tools lists every MCP tool and resource" {
    const out = try render(writeMcpTools);
    defer testing.allocator.free(out);
    for (mcp_tools.browser_tool_list ++ mcp_tools.extra_tools) |mt| {
        var row_buf: [64]u8 = undefined;
        var heading_buf: [64]u8 = undefined;
        const row = try std.mem.print(&row_buf, "| [`{s}`]", .{mt.name});
        const heading = try std.mem.print(&heading_buf, "\n### {s}\n", .{mt.name});
        try testing.expectEqual(1, std.mem.count(u8, out, row) + std.mem.count(u8, out, heading));
    }
    for (mcp_resources.resource_list) |r| {
        try testing.expect(std.mem.find(u8, out, r.uri) != null);
    }
}

test "docs: agent-commands covers every slash command" {
    const out = try render(writeAgentCommands);
    defer testing.allocator.free(out);
    for (SlashCommand.all_names) |name| {
        var buf: [64]u8 = undefined;
        const cmd = try std.mem.print(&buf, "`/{s}", .{name});
        try testing.expect(std.mem.find(u8, out, cmd) != null);
    }
    try testing.expect(std.mem.find(u8, out, "[`/goto <url> [timeout=…]") != null);
}

test "docs: pandascript-builtins lists exactly the recorded tools" {
    const out = try render(writePandascriptBuiltins);
    defer testing.allocator.free(out);
    for (std.enums.values(Tool)) |t| {
        var buf: [64]u8 = undefined;
        const call = try std.mem.print(&buf, "page.{s}(", .{@tagName(t)});
        try testing.expectEqual(@intFromBool(t.isRecorded()), std.mem.count(u8, out, call));
    }
    try testing.expect(std.mem.find(u8, out, "`page.extract(schema)`") != null);
    try testing.expect(std.mem.find(u8, out, "backendNodeId") == null);
    for (lp.Runtime.page_methods) |m| {
        var buf: [64]u8 = undefined;
        try testing.expect(std.mem.find(u8, out, try std.mem.print(&buf, "`page.{s}()`", .{m.name})) != null);
    }
    try testing.expect(std.mem.find(u8, out, "| `console.warn` | stderr |") != null);
}

test "docs: escaping leaves code spans alone" {
    var aw: Writer.Allocating = .init(testing.allocator);
    defer aw.deinit();
    try aw.writer.print("{f}", .{Cell{ .s = "a <b> {c} `x|<y>{}` d|e\nf `open" }});
    try testing.expectEqualStrings("a &lt;b> \\{c\\} `x\\|<y>{}` d\\|e f `open", aw.written());
}

test "docs: prose keeps fenced blocks verbatim" {
    var aw: Writer.Allocating = .init(testing.allocator);
    defer aw.deinit();
    try writeText(&aw.writer, "Shapes:\n```text\n\"<sel>\" → {a}\n```\nAfter <x>.");
    try testing.expectEqualStrings("Shapes:\n```text\n\"<sel>\" → {a}\n```\nAfter &lt;x>.", aw.written());
}