Lightpanda: the headless browser designed for AI and automation
docs: simplify the reference generator
Adrià Arrufat6c50c000cb3b
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 192bd1fdc..83c9eeb89 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -65,7 +65,8 @@ jobs:# Reference partials for the docs site, pulled daily by its# browser-reference workflow. A debug build: the output doesn't depend- # on the optimize mode and the self-hosted backend compiles it fastest.+ # on the optimize mode, it compiles far faster than ReleaseFast, and its+ # flags match the snapshot step's so the C deps come from the cache.- name: docs referenceif: matrix.arch == 'x86_64'run: |diff --git a/build.zig b/build.zigindex f123a4939..7579c5cac 100644--- a/build.zig+++ b/build.zig@@ -189,35 +189,8 @@ pub fn build(b: *Build) !void {run_step.dependOn(&run_cmd.step);}- {- // skills generator- const exe = addExe(b, exe_config, "lightpanda-skills", "skills_check", "src/main_skills.zig");-- const run_cmd = b.addRunArtifact(exe);- const out_dir = run_cmd.addOutputDirectoryArg2("skills", .{});- const install = b.addInstallDirectory(.{- .source_dir = out_dir,- .install_dir = .prefix,- .install_subdir = "skills",- });- const skills_step = b.step("skills", "Generate LLM skill docs (zig-out/skills/<name>/SKILL.md)");- skills_step.dependOn(&install.step);- }-- {- // docs-site reference partials- const exe = addExe(b, exe_config, "lightpanda-docs", "docs_check", "src/main_docs.zig");-- const run_cmd = b.addRunArtifact(exe);- const out_dir = run_cmd.addOutputDirectoryArg2("docs", .{});- const install = b.addInstallDirectory(.{- .source_dir = out_dir,- .install_dir = .prefix,- .install_subdir = "docs",- });- const docs_step = b.step("docs", "Generate reference partials for the docs site (zig-out/docs/*.mdx)");- docs_step.dependOn(&install.step);- }+ addGenerator(b, exe_config, "skills", "src/main_skills.zig", "Generate LLM skill docs (zig-out/skills/<name>/SKILL.md)");+ addGenerator(b, exe_config, "docs", "src/main_docs.zig", "Generate reference partials for the docs site (zig-out/docs/*.mdx)");{// test@@ -248,6 +221,20 @@ const ExeConfig = struct {sanitize_thread: bool,};+/// A `zig build <name>` step that runs `root_source_file` with an output+/// directory and installs what it writes to `zig-out/<name>`.+fn addGenerator(b: *Build, config: ExeConfig, comptime name: []const u8, root_source_file: []const u8, description: []const u8) void {+ const exe = addExe(b, config, "lightpanda-" ++ name, name ++ "_check", root_source_file);+ const run_cmd = b.addRunArtifact(exe);+ const out_dir = run_cmd.addOutputDirectoryArg2(name, .{});+ const install = b.addInstallDirectory(.{+ .source_dir = out_dir,+ .install_dir = .prefix,+ .install_subdir = name,+ });+ b.step(name, description).dependOn(&install.step);+}+fn addExe(b: *Build, config: ExeConfig, name: []const u8, check_name: []const u8, root_source_file: []const u8) *Build.Step.Compile {const exe = b.addExecutable(.{.name = name,diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 054cd78e5..d6f0ecbde 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -396,28 +396,6 @@ pub const Tool = enum {interaction,waiting,state,-- pub fn title(self: Group) []const u8 {- return switch (self) {- .navigation => "Navigation and search",- .reading => "Reading the page",- .extraction => "Data extraction and scripting",- .interaction => "Interacting with the page",- .waiting => "Waiting",- .state => "State and debugging",- };- }-- pub fn intro(self: Group) []const u8 {- return switch (self) {- .navigation => "These tools bring a page into the browser.",- .reading => "These tools read the loaded page without modifying it.",- .extraction => "These tools return structured results from the loaded page.",- .interaction => "These tools dispatch real DOM events on the page.",- .waiting => "These tools block until the page reaches a condition.",- .state => "These tools inspect browser state outside the DOM.",- };- }};/// Per-tool LLM-facing metadata. Tool identity (name + predicates) livesdiff --git a/src/docs.zig b/src/docs.zigindex 552db4cdf..8cce99a1d 100644--- a/src/docs.zig+++ b/src/docs.zig@@ -24,7 +24,7 @@const std = @import("std");const lp = @import("lightpanda");-const Schema = @import("script/Schema.zig");+const Schema = lp.Schema;const SlashCommand = @import("agent/SlashCommand.zig");const mcp_tools = @import("mcp/tools.zig");const mcp_resources = @import("mcp/resources.zig");@@ -49,53 +49,49 @@ const browser_tools_page = "/reference/browser-tools";/// Every browser tool, grouped, with its full description and parameters./// The MCP and agent partials link here instead of repeating it.-pub fn writeBrowserTools(w: *Writer) Writer.Error!void {+fn writeBrowserTools(w: *Writer) Writer.Error!void {try w.writeAll(banner);for (std.enums.values(Tool.Group)) |g| {- try w.print("## {s}\n\n{s}\n\n| Tool | Summary |\n|---|---|\n", .{ g.title(), g.intro() });- for (std.enums.values(Tool)) |t| {- if (t.group() != g) continue;- try w.print("| [`{s}`](", .{@tagName(t)});- try writeAnchor(w, "", @tagName(t));- try w.writeAll(") | ");- try writeCell(w, t.definition().summary);- try w.writeAll(" |\n");+ try writeGroupHeading(w, g, "| Tool | Summary |\n|---|---|\n");+ for (Schema.all()) |*s| {+ if (s.tool.group() != g) continue;+ try w.print("| [`{s}`]({f}) | {f} |\n", .{ s.tool_name, Anchor{ .name = s.tool_name }, Cell{ .s = s.summary } });}try w.writeAll("\n");- for (std.enums.values(Tool)) |t| {- if (t.group() != g) continue;- const def = t.definition();- try w.print("### {s}\n\n", .{@tagName(t)});- try writeText(w, def.description);+ for (Schema.all()) |*s| {+ if (s.tool.group() != g) continue;+ try w.print("### {s}\n\n", .{s.tool_name});+ try writeText(w, s.description);try w.writeAll("\n\n");- try writeParams(w, def.input_schema);- try writeTraits(w, t);+ try writeParams(w, s.parameters);+ const traits = [_]Flag{+ .{ s.tool.isRecorded(), "Recorded in saved scripts." },+ .{ s.tool.navigatesToUrl(), "Navigates first when given `url`." },+ .{ s.tool.reportsPageState(), "Reports the resulting page state (`url`, `httpStatus`, `title`)." },+ };+ if (try writeFlags(w, &traits, " ")) try w.writeAll("\n\n");}}}/// The MCP view: titles and hints per browser tool, plus the MCP-only tools/// and resources in full.-pub fn writeMcpTools(w: *Writer) Writer.Error!void {+fn writeMcpTools(w: *Writer) Writer.Error!void {try w.writeAll(banner);- const browser_list = mcp_tools.all_tools[0 .. mcp_tools.all_tools.len - mcp_tools.extra_tools.len];for (std.enums.values(Tool.Group)) |g| {- try w.print("## {s}\n\n{s}\n\n| Tool | Title | Hints |\n|---|---|---|\n", .{ g.title(), g.intro() });- for (std.enums.values(Tool), browser_list) |t, mt| {+ try writeGroupHeading(w, g, "| Tool | Title | Hints |\n|---|---|---|\n");+ for (std.enums.values(Tool), mcp_tools.browser_tool_list) |t, mt| {if (t.group() != g) continue;- std.debug.assert(std.mem.eql(u8, @tagName(t), mt.name));- try w.print("| [`{s}`](", .{mt.name});- try writeAnchor(w, browser_tools_page, mt.name);- try w.writeAll(") | ");- try writeCell(w, mt.title orelse "");- try w.writeAll(" | ");+ try w.print("| [`{s}`]({f}) | {f} | ", .{ mt.name, Anchor{ .page = browser_tools_page, .name = mt.name }, Cell{ .s = mt.title orelse "" } });try writeHints(w, mt);try w.writeAll(" |\n");}try w.writeAll("\n");}+ var arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);+ defer arena.deinit();try w.writeAll("## MCP-only tools\n\nThese tools exist only in the MCP server, not in the agent or in scripts.\n\n");for (mcp_tools.extra_tools) |mt| {try w.print("### {s}\n\n", .{mt.name});@@ -106,7 +102,10 @@ pub fn writeMcpTools(w: *Writer) Writer.Error!void {}try writeText(w, mt.description orelse "");try w.writeAll("\n\n");- try writeParams(w, mt.inputSchema);+ const schema = std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), mt.inputSchema, .{}) catch |err| {+ std.debug.panic("invalid input schema for MCP tool '{s}': {s}", .{ mt.name, @errorName(err) });+ };+ try writeParams(w, schema);try w.writeAll("Hints: ");try writeHints(w, mt);try w.writeAll("\n\n");@@ -114,63 +113,52 @@ pub fn writeMcpTools(w: *Writer) Writer.Error!void {try w.writeAll("## Resources\n\nRead with `resources/read`.\n\n| URI | MIME type | Description |\n|---|---|---|\n");for (mcp_resources.resource_list) |r| {- try w.print("| `{s}` | `{s}` | ", .{ r.uri, r.mimeType orelse "" });- try writeCell(w, r.description orelse r.name);- try w.writeAll(" |\n");+ try w.print("| `{s}` | `{s}` | {f} |\n", .{ r.uri, r.mimeType orelse "", Cell{ .s = r.description orelse r.name } });}}/// The agent REPL's slash commands: meta, LLM-prompt, and one per browser/// tool in the `key=value` syntax the REPL parses.-pub fn writeAgentCommands(w: *Writer) Writer.Error!void {+fn writeAgentCommands(w: *Writer) Writer.Error!void {try w.writeAll(banner);try w.writeAll("### Meta commands\n\nThese commands configure the REPL and never reach the model.\n\n| Command | Description |\n|---|---|\n");for (SlashCommand.meta_commands) |m| {try w.print("| `/{s}", .{m.name});- if (m.hint.len > 0) {- try w.writeAll(" ");- try writeCellRaw(w, m.hint);- }- try w.writeAll("` | ");- try writeCell(w, m.description);- try w.writeAll(" |\n");+ if (m.hint.len > 0) try w.print(" {f}", .{Cell{ .s = m.hint, .in_code = true }});+ try w.print("` | {f} |\n", .{Cell{ .s = m.description }});}try w.writeAll("\n### LLM commands\n\nThese commands send a prebuilt prompt to the model, so they need a provider.\n\n| Command | Description |\n|---|---|\n");for (SlashCommand.llm_commands) |row| {- try w.print("| `/{s}` | ", .{row.name});- try writeCell(w, row.description);- try w.writeAll(" |\n");+ try w.print("| `/{s}` | {f} |\n", .{ row.name, Cell{ .s = row.description } });}try w.writeAll("\n### Browser commands\n\nEvery browser tool runs directly as a slash command, with no model call. A lone required argument can be passed positionally; the rest take `key=value` or a JSON object.\n\n| Command | Description |\n|---|---|\n");- const schemas = Schema.all();for (std.enums.values(Tool.Group)) |g| {- for (schemas) |*s| {+ for (Schema.all()) |*s| {if (s.tool.group() != g) continue;try w.print("| [`/{s}", .{s.tool_name});- for (s.hints) |h| {- try w.writeAll(" ");- try writeCellRaw(w, h.fragment);- }- try w.writeAll("`](");- try writeAnchor(w, browser_tools_page, s.tool_name);- try w.writeAll(") | ");- try writeCell(w, s.summary);- try w.writeAll(" |\n");+ for (s.hints) |h| try w.print(" {f}", .{Cell{ .s = h.fragment, .in_code = true }});+ try w.print("`]({f}) | {f} |\n", .{ Anchor{ .page = browser_tools_page, .name = s.tool_name }, Cell{ .s = s.summary } });}}}-/// An `Argument | Type | Required | Description` table for a JSON Schema-/// object. Panics on malformed JSON: the schemas are comptime constants.-fn writeParams(w: *Writer, input_schema: []const u8) Writer.Error!void {- var arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);- defer arena.deinit();- const schema = std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), input_schema, .{}) catch |err| {- std.debug.panic("invalid tool input schema: {s}", .{@errorName(err)});+fn writeGroupHeading(w: *Writer, g: Tool.Group, table_head: []const u8) Writer.Error!void {+ const title, const scope = switch (g) {+ .navigation => .{ "Navigation and search", "bring a page into the browser" },+ .reading => .{ "Reading the page", "read the loaded page without modifying it" },+ .extraction => .{ "Data extraction and scripting", "return structured results from the loaded page" },+ .interaction => .{ "Interacting with the page", "dispatch real DOM events on the page" },+ .waiting => .{ "Waiting", "block until the page reaches a condition" },+ .state => .{ "State and debugging", "inspect browser state outside the DOM" },};+ try w.print("## {s}\n\nThese tools {s}.\n\n{s}", .{ title, scope, table_head });+}+/// An `Argument | Type | Required | Description` table for a JSON Schema+/// object.+fn writeParams(w: *Writer, schema: std.json.Value) Writer.Error!void {const props = schema.object.get("properties") orelse return w.writeAll("No arguments.\n\n");if (props.object.count() == 0) return w.writeAll("No arguments.\n\n");const required: []const std.json.Value = if (schema.object.get("required")) |r| r.array.items else &.{};@@ -186,7 +174,7 @@ fn writeParams(w: *Writer, input_schema: []const u8) Writer.Error!void {var sep: []const u8 = "";if (prop.get("description")) |d| {- try writeCell(w, d.string);+ try w.print("{f}", .{Cell{ .s = d.string }});sep = " ";}if (prop.get("enum")) |values| {@@ -218,13 +206,6 @@ fn writeParams(w: *Writer, input_schema: []const u8) Writer.Error!void {try w.writeAll("\n");}-/// Nextra slugs headings in lowercase, so `### nodeDetails` is-/// `#nodedetails`.-fn writeAnchor(w: *Writer, page: []const u8, name: []const u8) Writer.Error!void {- try w.print("{s}#", .{page});- for (name) |c| try w.writeByte(std.ascii.toLower(c));-}-fn writeType(w: *Writer, prop: std.json.ObjectMap) Writer.Error!void {const ty = prop.get("type") orelse return w.writeAll("any");if (std.mem.eql(u8, ty.string, "array")) {@@ -241,41 +222,57 @@ fn isRequired(required: []const std.json.Value, name: []const u8) bool {return false;}-fn writeTraits(w: *Writer, t: Tool) Writer.Error!void {- const traits = [_]struct { bool, []const u8 }{- .{ t.isRecorded(), "Recorded in saved scripts." },- .{ t.navigatesToUrl(), "Navigates first when given `url`." },- .{ t.reportsPageState(), "Reports the resulting page state (`url`, `httpStatus`, `title`)." },- };- var first = true;- for (traits) |trait| {- if (!trait[0]) continue;- if (!first) try w.writeAll(" ");- try w.writeAll(trait[1]);- first = false;- }- if (!first) try w.writeAll("\n\n");-}-fn writeHints(w: *Writer, mt: protocol.Tool) Writer.Error!void {const a = mt.annotations orelse protocol.ToolAnnotations{};- const hints = [_]struct { bool, []const u8 }{+ const hints = [_]Flag{.{ a.readOnlyHint, "read-only" },- .{ !a.readOnlyHint and a.destructiveHint, "destructive" },+ .{ a.destructiveHint, "destructive" },.{ a.idempotentHint, "idempotent" },.{ a.openWorldHint, "open-world" },.{ mt.outputSchema != null, "page state" },};- var first = true;- for (hints) |hint| {- if (!hint[0]) continue;- if (!first) try w.writeAll(", ");- try w.writeAll(hint[1]);- first = false;+ if (!try writeFlags(w, &hints, ", ")) try w.writeAll("—");+}++const Flag = struct { bool, []const u8 };++/// Writes the labels of the set flags, separated by `sep`. Returns whether+/// any was set.+fn writeFlags(w: *Writer, flags: []const Flag, sep: []const u8) Writer.Error!bool {+ var wrote = false;+ for (flags) |flag| {+ if (!flag[0]) continue;+ if (wrote) try w.writeAll(sep);+ try w.writeAll(flag[1]);+ wrote = true;}- if (first) try w.writeAll("—");+ return wrote;}+/// A link target to a tool's heading. Nextra slugs headings in lowercase, so+/// `### nodeDetails` is `#nodedetails`.+const Anchor = struct {+ page: []const u8 = "",+ name: []const u8,++ pub fn format(self: Anchor, w: *Writer) Writer.Error!void {+ try w.print("{s}#", .{self.page});+ for (self.name) |c| try w.writeByte(std.ascii.toLower(c));+ }+};++/// Text for a GFM table cell: `|` ends the cell even inside a code span, and+/// a newline ends the row. `in_code` is for text already inside a code span.+const Cell = struct {+ s: []const u8,+ in_code: bool = false,++ pub fn format(self: Cell, w: *Writer) Writer.Error!void {+ if (self.in_code) return writeCode(w, self.s, true);+ try writeEscaped(w, self.s, true);+ }+};+/// Prose for MDX: outside code spans, `<` would open JSX and `{` an/// expression. Tool descriptions are written for models, so a run of/// indented lines after a line ending in `:` is an aligned listing (extract's@@ -305,14 +302,8 @@ fn writeText(w: *Writer, s: []const u8) Writer.Error!void {if (in_fence) try w.writeAll("\n```");}-/// Like `writeText`, for a GFM table cell: `|` ends the cell even inside a-/// code span, and a newline ends the row.-fn writeCell(w: *Writer, s: []const u8) Writer.Error!void {- try writeEscaped(w, s, true);-}--/// A table cell already inside a code span.-fn writeCellRaw(w: *Writer, s: []const u8) Writer.Error!void {+fn writeCode(w: *Writer, s: []const u8, cell: bool) Writer.Error!void {+ if (!cell) return w.writeAll(s);for (s) |c| switch (c) {'|' => try w.writeAll("\\|"),else => try w.writeByte(c),@@ -325,19 +316,13 @@ fn writeEscaped(w: *Writer, s: []const u8, cell: bool) Writer.Error!void {if (s[i] == '`') {const run_end = std.mem.findNonePos(u8, s, i, "`") orelse s.len;const fence = s[i..run_end];+ try w.writeAll(fence);+ i = run_end;if (std.mem.findPos(u8, s, run_end, fence)) |close| {- try w.writeAll(fence);- if (cell) {- try writeCellRaw(w, s[run_end..close]);- } else {- try w.writeAll(s[run_end..close]);- }+ try writeCode(w, s[run_end..close], cell);try w.writeAll(fence);i = close + fence.len;- continue;}- try w.writeAll(fence);- i = run_end;continue;}switch (s[i]) {@@ -354,15 +339,15 @@ fn writeEscaped(w: *Writer, s: []const u8, cell: bool) Writer.Error!void {const testing = std.testing;-fn render(partial: Partial) ![]u8 {+fn render(write: *const fn (w: *Writer) Writer.Error!void) ![]u8 {var aw: Writer.Allocating = .init(testing.allocator);errdefer aw.deinit();- try partial.write(&aw.writer);+ try write(&aw.writer);return aw.toOwnedSlice();}test "docs: browser-tools documents every tool exactly once" {- const out = try render(partials[0]);+ const out = try render(writeBrowserTools);defer testing.allocator.free(out);for (std.enums.values(Tool)) |t| {var buf: [64]u8 = undefined;@@ -372,9 +357,9 @@ test "docs: browser-tools documents every tool exactly once" {}test "docs: mcp-tools lists every MCP tool and resource" {- const out = try render(partials[1]);+ const out = try render(writeMcpTools);defer testing.allocator.free(out);- for (mcp_tools.all_tools) |mt| {+ for (mcp_tools.browser_tool_list ++ mcp_tools.extra_tools) |mt| {var row_buf: [64]u8 = undefined;var heading_buf: [64]u8 = undefined;const row = try std.mem.print(&row_buf, "| [`{s}`]", .{mt.name});@@ -387,7 +372,7 @@ test "docs: mcp-tools lists every MCP tool and resource" {}test "docs: agent-commands covers every slash command" {- const out = try render(partials[2]);+ const out = try render(writeAgentCommands);defer testing.allocator.free(out);for (SlashCommand.all_names) |name| {var buf: [64]u8 = undefined;@@ -400,7 +385,7 @@ test "docs: agent-commands covers every slash command" {test "docs: escaping leaves code spans alone" {var aw: Writer.Allocating = .init(testing.allocator);defer aw.deinit();- try writeCell(&aw.writer, "a <b> {c} `x|<y>{}` d|e\nf `open");+ try aw.writer.print("{f}", .{Cell{ .s = "a <b> {c} `x|<y>{}` d|e\nf `open" }});try testing.expectEqualStrings("a <b> \\{c\\} `x\\|<y>{}` d\\|e f `open", aw.written());}diff --git a/src/main_docs.zig b/src/main_docs.zigindex 900449c17..461cd06ff 100644--- a/src/main_docs.zig+++ b/src/main_docs.zig@@ -23,7 +23,6 @@ const std = @import("std");const lp = @import("lightpanda");pub fn main(init: std.process.Init) !void {- // usage: docs <outdir>var args = std.process.Args.Iterator.init(init.minimal.args);_ = args.next(); // executable nameconst out_path = args.next() orelse {diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex 0b43b75bc..c19714073 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -12,7 +12,7 @@ const McpTool = protocol.Tool;/// Convert browser tool_defs to MCP wire-protocol tools (comptime)./// Tool identity comes from the `BrowserTool` tag — `tool_defs` only/// carries the LLM-facing description and JSON schema.-const browser_tool_list = blk: {+pub const browser_tool_list = blk: {const field_names = @typeInfo(BrowserTool).@"enum".field_names;var tools: [field_names.len]McpTool = undefined;for (browser_tools.tool_defs, field_names, 0..) |td, field_name, i| {@@ -124,7 +124,7 @@ pub const extra_tools = [_]McpTool{},};-pub const all_tools = browser_tool_list ++ extra_tools;+const all_tools = browser_tool_list ++ extra_tools;/// Tools that bypass the browser-tool dispatch and have their own handlers./// Values are telemetry wire ids, kept clear of `browser_tools.Tool`'s.