Lightpanda: the headless browser designed for AI and automation
markdown: follow text/plain alternates, say which source was used
Jay Hemnanif39164724166
diff --git a/src/browser/markdown.zig b/src/browser/markdown.zigindex 0ad457b3d..b64379b62 100644--- a/src/browser/markdown.zig+++ b/src/browser/markdown.zig@@ -35,6 +35,10 @@ const repeat = @import("../string.zig").repeat;pub const Opts = struct {max_bytes: ?u32 = null,+ /// The document is a followed `<link rel="alternate" type="text/markdown">`.+ /// The link already says the file is markdown, and many servers send `.md`+ /// files as text/plain, so a text/plain alternate is written as-is too.+ alternate: bool = false,};const truncation_marker = LimitedWriter.truncation_marker;@@ -481,10 +485,14 @@ const Context = struct {/// A text/markdown response is loaded as a text document, its body a single/// `<pre>`. Converting that would fence the whole file, so the document's-/// text, which is the markdown the server sent, is written as-is.-fn markdownSource(node: *Node) ?*Node {+/// text, which is the markdown the server sent, is written as-is. A text/plain+/// document counts only as a followed alternate: plain text loaded directly+/// isn't necessarily markdown and stays fenced.+fn markdownSource(node: *Node, alternate: bool) ?*Node {const doc = node.is(Document) orelse return null;- if (!std.mem.eql(u8, doc.getContentType(), "text/markdown")) return null;+ const content_type = doc.getContentType();+ if (!std.mem.eql(u8, content_type, "text/markdown") and+ !(alternate and std.mem.eql(u8, content_type, "text/plain"))) return null;const root = doc.getDocumentElement() orelse return null;return root.asNode();}@@ -513,17 +521,17 @@ pub fn alternateUrl(arena: std.mem.Allocator, frame: *Frame) !?[:0]const u8 {}/// Whether a followed alternate can stand in for the page: a 2xx-/// text/markdown response.+/// text/markdown or text/plain response.pub fn isAlternateLoaded(frame: *Frame) bool {if (frame._last_navigate_error != null) return false;const status = frame._http_status orelse return false;if (status < 200 or status >= 300) return false;- return markdownSource(frame.document.asNode()) != null;+ return markdownSource(frame.document.asNode(), true) != null;}pub fn dump(state: RenderTree.State, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {const node = state.root;- if (markdownSource(node)) |source| {+ if (markdownSource(node, opts.alternate)) |source| {return writeSource(source, opts, writer);}if (opts.max_bytes) |limit| {@@ -1018,6 +1026,23 @@ test "browser.markdown: a text/markdown document is written as-is" {try testing.expectString("# Title" ++ truncation_marker, capped.written());}+test "browser.markdown: text/plain is written as-is only as a followed alternate" {+ var page = try testing.pageTest("markdown_source.txt", .{});+ defer page.close();+ const frame = page.frame().?;+ const source = @embedFile("tests/markdown_source.txt");++ // Plain text loaded directly isn't necessarily markdown: it stays fenced.+ var direct: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ try dump(.{ .root = frame.window._document.asNode() }, .{}, &direct.writer, frame);+ try testing.expect(std.mem.find(u8, direct.written(), "```") != null);+ try testing.expect(!std.mem.eql(u8, direct.written(), source));++ var followed: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ try dump(.{ .root = frame.window._document.asNode() }, .{ .alternate = true }, &followed.writer, frame);+ try testing.expectString(source, followed.written());+}+test "browser.markdown: alternateUrl" {{var page = try testing.pageTest("markdown_alternate.html", .{});diff --git a/src/browser/tests/markdown_alternate_plain.html b/src/browser/tests/markdown_alternate_plain.htmlnew file mode 100644index 000000000..9e993b96b--- /dev/null+++ b/src/browser/tests/markdown_alternate_plain.html@@ -0,0 +1,5 @@+<!DOCTYPE html>+<head>+<link rel="alternate" type="text/markdown" href="markdown_source.txt">+</head>+<p>Rendered page, plain alternate</p>diff --git a/src/browser/tests/markdown_source.txt b/src/browser/tests/markdown_source.txtnew file mode 100644index 000000000..69a67e088--- /dev/null+++ b/src/browser/tests/markdown_source.txt@@ -0,0 +1,7 @@+# Title++Some *emphasis*, a [link](/x) & a <tag>.++```zig+const x = 1;+```diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex b1721971c..6ed311bbc 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -456,7 +456,7 @@ pub const Tool = enum {\\ "maxBytes": { "type": "integer", "description": "Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap." },\\ "strip": { "type": "object", "description": "Optional. Omit element groups from the output; same groups as the html tool's strip. `shell` (page chrome by markup) and `clutter` (keep only the main content, in the manner of reader modes) are the ones that matter for reading; `ui` also drops images.", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" }, "shell": { "type": "boolean" }, "clutter": { "type": "boolean" } } },\\ "url": { "type": "string", "description": "Optional URL to navigate to before rendering. An empty string is treated as omitted." },- \\ "alternate": { "type": "boolean", "description": "Optional. When the page links a markdown version of itself (<link rel=\"alternate\" type=\"text/markdown\">), return that file instead of converting the page. It is the author's file and can differ from the rendered page. Falls back to the page when there is no such link or it does not load as markdown. Ignored with selector or backendNodeId. Defaults to false." },+ \\ "alternate": { "type": "boolean", "description": "Optional. When the page links a markdown version of itself (<link rel=\"alternate\" type=\"text/markdown\">), return that file instead of converting the page. It is the author's file and can differ from the rendered page; the output then starts with a comment naming its URL. Falls back to the page when there is no such link or it does not load as markdown. Cannot be combined with selector or backendNodeId. Defaults to false." },\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }\\ }\\}@@ -1652,7 +1652,9 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRconst args = try parseArgsOrDefault(Params, arena, arguments);const page = try ensurePage(session, registry, args.url, args.timeout);- if (args.alternate and args.selector == null and args.backendNodeId == null) {+ if (args.alternate) {+ // The alternate stands for the whole page, like `fetch --markdown-alternate`.+ if (args.selector != null or args.backendNodeId != null) return ToolError.InvalidParams;if (try readMarkdownAlternate(arena, session, page, args.maxBytes, args.timeout)) |text| return text;}@@ -1680,8 +1682,10 @@ fn readMarkdownAlternate(arena: std.mem.Allocator, session: *lp.Session, frame:const alternate = page.frame() orelse return null;if (!lp.markdown.isAlternateLoaded(alternate)) return null;+ // Say which source this is: the alternate can differ from the rendered page.var aw: std.Io.Writer.Allocating = .init(arena);- lp.markdown.dump(.{ .root = alternate.document.asNode() }, .{ .max_bytes = max_bytes }, &aw.writer, alternate) catch+ aw.writer.print("<!-- markdown alternate: {s} -->\n", .{alternate.url}) catch return ToolError.OutOfMemory;+ lp.markdown.dump(.{ .root = alternate.document.asNode() }, .{ .max_bytes = max_bytes, .alternate = true }, &aw.writer, alternate) catchreturn ToolError.InternalError;return aw.written();}@@ -3545,7 +3549,7 @@ test "markdown: alternate reads the page's text/markdown link" {const followed = try call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa,\\{"alternate":true}, .{}), .{ .source = .user });- try std.testing.expectEqualStrings(source, followed.text);+ try std.testing.expectEqualStrings("<!-- markdown alternate: " ++ base ++ "markdown_source.md -->\n" ++ source, followed.text);// Read in a page of its own: the current page stays put.try std.testing.expect(!followed.navigated);try std.testing.expectEqual(1, session.pages.items.len);@@ -3554,12 +3558,16 @@ test "markdown: alternate reads the page's text/markdown link" {const capped = try call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa,\\{"alternate":true,"maxBytes":7}, .{}), .{ .source = .user });- try std.testing.expect(std.mem.startsWith(u8, capped.text, "# Title\n\n[truncated]"));+ try std.testing.expect(std.mem.find(u8, capped.text, "-->\n# Title\n\n[truncated]") != null);- const scoped = try call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa,+ // The alternate stands for the whole page: no scoping, as in `fetch`.+ try std.testing.expectError(ToolError.InvalidParams, call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa,\\{"alternate":true,"selector":"p"}- , .{}), .{ .source = .user });- try std.testing.expect(std.mem.find(u8, scoped.text, "Rendered page") != null);+ , .{}), .{ .source = .user }));++ // Many servers send .md files as text/plain; followed, it is written as-is.+ const plain = try call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa, "{\"alternate\":true,\"url\":\"" ++ base ++ "markdown_alternate_plain.html\"}", .{}), .{ .source = .user });+ try std.testing.expectEqualStrings("<!-- markdown alternate: " ++ base ++ "markdown_source.txt -->\n" ++ source, plain.text);// An alternate that doesn't load as markdown falls back to the page.const fallback = try call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa, "{\"alternate\":true,\"url\":\"" ++ base ++ "markdown_alternate_html.html\"}", .{}), .{ .source = .user });diff --git a/src/help.zon b/src/help.zonindex 3c4032efc..349e25caf 100644--- a/src/help.zon+++ b/src/help.zon@@ -142,8 +142,10 @@\\ file instead of converting the page. The file is the author's\\ and can differ from the rendered page. Falls back to converting\\ the page when there is no such link or it does not load as- \\ markdown. With --json, "url" is where the markdown came from.- \\ Defaults to false.+ \\ markdown (text/markdown, or text/plain since many servers send+ \\ .md files that way). The alternate's URL is logged; with+ \\ --json, "url" is where the markdown came from. The alternate+ \\ loads within what is left of --wait-ms. Defaults to false.\\ --metrics\\ Write metrics (Prometheus text format) to stdout on exit, after\\ any --dump output.diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 44c97df18..52183792d 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -349,9 +349,12 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet}}+ // Pages swapped for their markdown alternate.+ const alternates = try session.arena.allocator().alloc(bool, pages.items.len);+ @memset(alternates, false);if (opts.markdown_alternate) {const remaining = opts.wait_ms -| @as(u32, @intCast(timer.untilNow(io, .boot).toMilliseconds()));- try followMarkdownAlternates(session, pages.items, errors, remaining);+ try followMarkdownAlternates(session, pages.items, errors, remaining, alternates);}var http_error = false;@@ -382,7 +385,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet}}- try writeResults(app, opts, pages.items, errors);+ try writeResults(app, opts, pages.items, errors, alternates);var failed = false;for (urls, pages.items, errors) |url, page, err| {@@ -410,8 +413,9 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet/// Loads each page's markdown alternate in a page of its own (same session, so/// same cookies, robots and proxy) and swaps it in for the page when it loads-/// as markdown. Otherwise the page is kept and converted as usual.-fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, errors: []const ?anyerror, timeout_ms: u32) !void {+/// as markdown, marking it in `followed`. Otherwise the page is kept and+/// converted as usual.+fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, errors: []const ?anyerror, timeout_ms: u32, followed: []bool) !void {const arena = session.arena.allocator();const alternates = try arena.alloc(?Session.PageHandle, pages.len);var conditions: std.ArrayList(Session.Runner.WaitCondition) = .empty;@@ -431,22 +435,27 @@ fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, erroif (conditions.items.len == 0) return;var runner = session.runner(.{});- _ = try runner.waitResult(timeout_ms, conditions.items);+ const result = try runner.waitResult(timeout_ms, conditions.items);- for (pages, alternates) |*page, alternate| {+ for (pages, alternates, followed) |*page, alternate, *was_followed| {const alt = alternate orelse continue;if (alt.frame()) |frame| {if (markdown.isAlternateLoaded(frame)) {+ log.info(.app, "markdown alternate", .{ .page = page.frame().?.url, .alternate = frame.url });page.close();page.* = alt;+ was_followed.* = true;continue;}+ if (result == .timeout) {+ log.warn(.app, "markdown alternate skipped", .{ .alternate = frame.url, .reason = "wait budget used up" });+ }}alt.close();}}-fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, errors: []?anyerror) !void {+fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, errors: []?anyerror, alternates: []const bool) !void {const writer = opts.writer orelse return;if (opts.json) {@@ -459,7 +468,7 @@ fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, eif (wrap) {try writer.writeAll("{\"results\":[");}- for (pages, errors, 0..) |page, *err, i| {+ for (pages, errors, alternates, 0..) |page, *err, alternate, i| {if (i != 0) {try writer.writeByte(',');}@@ -484,7 +493,7 @@ fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, edefer aw.deinit();if (opts.dump_mode) |mode| {- if (frame) |f| dumpContent(app, mode, opts, f, &aw.writer) catch |e| {+ if (frame) |f| dumpContent(app, mode, opts, f, alternate, &aw.writer) catch |e| {aw.clearRetainingCapacity();if (err.* == null) err.* = e;};@@ -503,7 +512,7 @@ fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, etry writer.writeAll("Frame closed. Please open a bug report including the URL\n");break :blk;};- try dumpContent(app, mode, opts, frame, writer);+ try dumpContent(app, mode, opts, frame, alternates[0], writer);}}try writer.flush();@@ -542,13 +551,13 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {return (element orelse return error.SelectorNotFound).asNode();}-fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {+fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, alternate: bool, writer: *std.Io.Writer) !void {var arena: std.heap.ArenaAllocator = .init(app.allocator);defer arena.deinit();const state = try RenderTree.resolve(arena.allocator(), try dumpRoot(frame, opts.selector), opts.dump.strip, frame);switch (mode) {.html => try dump.render(state, opts.dump, writer, frame),- .markdown => try markdown.dump(state, .{ .max_bytes = opts.dump.max_bytes }, writer, frame),+ .markdown => try markdown.dump(state, .{ .max_bytes = opts.dump.max_bytes, .alternate = alternate }, writer, frame),.png => {_ = try screenshot.png(arena.allocator(), state, pngOpts(frame), writer, frame);},@@ -779,7 +788,7 @@ test "dumpContent: structured_data" {defer page.close();var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);- try dumpContent(testing.test_app, .structured_data, .{ .dump = .{} }, page.frame().?, &aw.writer);+ try dumpContent(testing.test_app, .structured_data, .{ .dump = .{} }, page.frame().?, false, &aw.writer);try testing.expectJson(.{.jsonLd = &.{"{\"@type\":\"Product\",\"name\":\"Blue Kettle\"}"},.openGraph = .{ .type = "product" },@@ -807,15 +816,28 @@ test "fetch: followMarkdownAlternates swaps in a page's markdown alternate" {try testing.pageTest("markdown_alternate.html", .{ .wait_until_done = true }),try testing.pageTest("markdown_alternate_html.html", .{ .wait_until_done = true }),try testing.pageTest("markdown_flex.html", .{ .wait_until_done = true }),+ try testing.pageTest("markdown_alternate_plain.html", .{ .wait_until_done = true }),};- var errors = [_]?anyerror{ null, null, null };- try followMarkdownAlternates(session, &pages, &errors, 2000);+ var errors = [_]?anyerror{ null, null, null, null };+ var followed = [_]bool{ false, false, false, false };+ try followMarkdownAlternates(session, &pages, &errors, 2000, &followed);const base = "http://127.0.0.1:9582/src/browser/tests/";try testing.expectString(base ++ "markdown_source.md", pages[0].frame().?.url);try testing.expectString(base ++ "markdown_alternate_html.html", pages[1].frame().?.url);try testing.expectString(base ++ "markdown_flex.html", pages[2].frame().?.url);- try testing.expectEqual(3, session.pages.items.len);+ // Many servers send .md files as text/plain; the link already says it's markdown.+ try testing.expectString(base ++ "markdown_source.txt", pages[3].frame().?.url);+ try testing.expectEqual(true, followed[0]);+ try testing.expectEqual(false, followed[1]);+ try testing.expectEqual(false, followed[2]);+ try testing.expectEqual(true, followed[3]);+ try testing.expectEqual(4, session.pages.items.len);++ // A followed text/plain alternate is written as-is, not fenced.+ var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ try dumpContent(testing.test_app, .markdown, .{ .dump = .{} }, pages[3].frame().?, true, &aw.writer);+ try testing.expectString(@embedFile("browser/tests/markdown_source.txt"), aw.written());}test {