Lightpanda: the headless browser designed for AI and automation
markdown: don't swap a failed page for its alternate, keep text documents byte-faithful
Jay Hemnani16fc3852f716
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex e6c9226be..91e80fdf1 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1953,7 +1953,9 @@ fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {},.application_json, .text_javascript, .text_css, .text_plain, .text_markdown => {var arr: std.ArrayList(u8) = .empty;- try arr.appendSlice(self.page_arena, "<html><head><meta charset=\"utf-8\"></head><body><pre>");+ // The parser drops a newline right after <pre>; this one+ // keeps a body that starts with a newline intact.+ try arr.appendSlice(self.page_arena, "<html><head><meta charset=\"utf-8\"></head><body><pre>\n");self._parse_state = .{ .text = arr };},.image_jpeg, .image_gif, .image_png, .image_webp => {diff --git a/src/browser/markdown.zig b/src/browser/markdown.zigindex b64379b62..1df50da86 100644--- a/src/browser/markdown.zig+++ b/src/browser/markdown.zig@@ -520,6 +520,15 @@ pub fn alternateUrl(arena: std.mem.Allocator, frame: *Frame) !?[:0]const u8 {return url;}+/// Whether a page may be swapped for its alternate: only one that loaded+/// fine. The alternate of a 404 or a bot challenge page would hide the+/// failure behind its own 200.+pub fn canFollowAlternate(frame: *Frame) bool {+ if (frame._last_navigate_error != null or frame._bot_challenge != null) return false;+ const status = frame._http_status orelse return true;+ return status < 400;+}+/// Whether a followed alternate can stand in for the page: a 2xx/// text/markdown or text/plain response.pub fn isAlternateLoaded(frame: *Frame) bool {@@ -1026,6 +1035,16 @@ test "browser.markdown: a text/markdown document is written as-is" {try testing.expectString("# Title" ++ truncation_marker, capped.written());}+test "browser.markdown: a text/markdown document keeps its leading newlines" {+ var page = try testing.pageTest("markdown_leading_blank.md", .{});+ defer page.close();+ const frame = page.frame().?;++ var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ try dump(.{ .root = frame.window._document.asNode() }, .{}, &aw.writer, frame);+ try testing.expectString(@embedFile("tests/markdown_leading_blank.md"), aw.written());+}+test "browser.markdown: text/plain is written as-is only as a followed alternate" {var page = try testing.pageTest("markdown_source.txt", .{});defer page.close();diff --git a/src/browser/tests/markdown_leading_blank.md b/src/browser/tests/markdown_leading_blank.mdnew file mode 100644index 000000000..9b5d04941--- /dev/null+++ b/src/browser/tests/markdown_leading_blank.md@@ -0,0 +1,5 @@+++# Title++Text.diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 6ed311bbc..773708cd9 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -456,7 +456,7 @@ pub const Tool = enum {\\ "maxBytes": { "type": "integer", "description": "Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap." },\\ "strip": { "type": "object", "description": "Optional. Omit element groups from the output; same groups as the html tool's strip. `shell` (page chrome by markup) and `clutter` (keep only the main content, in the manner of reader modes) are the ones that matter for reading; `ui` also drops images.", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" }, "shell": { "type": "boolean" }, "clutter": { "type": "boolean" } } },\\ "url": { "type": "string", "description": "Optional URL to navigate to before rendering. An empty string is treated as omitted." },- \\ "alternate": { "type": "boolean", "description": "Optional. When the page links a markdown version of itself (<link rel=\"alternate\" type=\"text/markdown\">), return that file instead of converting the page. It is the author's file and can differ from the rendered page; the output then starts with a comment naming its URL. Falls back to the page when there is no such link or it does not load as markdown. Cannot be combined with selector or backendNodeId. Defaults to false." },+ \\ "alternate": { "type": "boolean", "description": "Optional. When the page links a markdown version of itself (<link rel=\"alternate\" type=\"text/markdown\">), return that file instead of converting the page. It is the author's file and can differ from the rendered page; the output then starts with a comment naming its URL, which maxBytes does not count. Falls back to the page when there is no such link, the page itself failed to load (HTTP error status or bot challenge), or the alternate does not load as markdown within what is left of the timeout. Cannot be combined with selector or backendNodeId. Defaults to false." },\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }\\ }\\}@@ -1650,12 +1650,18 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRtimeout: ?u32 = null,};const args = try parseArgsOrDefault(Params, arena, arguments);+ const timer: std.Io.Timestamp = .now(lp.io, .boot);const page = try ensurePage(session, registry, args.url, args.timeout);if (args.alternate) {// The alternate stands for the whole page, like `fetch --markdown-alternate`.if (args.selector != null or args.backendNodeId != null) return ToolError.InvalidParams;- if (try readMarkdownAlternate(arena, session, page, args.maxBytes, args.timeout)) |text| return text;+ if (lp.markdown.canFollowAlternate(page)) {+ // One timeout for the call: the alternate gets what loading the page left.+ const elapsed: u32 = @intCast(timer.untilNow(lp.io, .boot).toMilliseconds());+ const remaining = (args.timeout orelse default_nav_timeout_ms) -| elapsed;+ if (try readMarkdownAlternate(arena, session, page, args.maxBytes, remaining)) |text| return text;+ }}const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);@@ -1668,16 +1674,20 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeR/// The page's `<link rel="alternate" type="text/markdown">`, loaded in a page/// of its own so the current page stays where it is. Null when there is no-/// such link or it doesn't load as markdown.-fn readMarkdownAlternate(arena: std.mem.Allocator, session: *lp.Session, frame: *lp.Frame, max_bytes: ?u32, timeout: ?u32) ToolError!?[]const u8 {+/// such link or it doesn't load as markdown within `timeout_ms`.+fn readMarkdownAlternate(arena: std.mem.Allocator, session: *lp.Session, frame: *lp.Frame, max_bytes: ?u32, timeout_ms: u32) ToolError!?[]const u8 {const url = (lp.markdown.alternateUrl(arena, frame) catch return ToolError.InternalError) orelse return null;const page = openPage(session, url) catch return null;defer page.close();var runner = session.runner(.{});var conditions = [_]lp.Session.Runner.WaitCondition{.{ .frame_id = page.frame_id, .until = default_nav_wait }};- _ = runner.waitResult(timeout orelse default_nav_timeout_ms, &conditions) catch |err| {- return if (err == error.Cancelled) ToolError.Cancelled else null;+ // A timeout leaves the alternate unloaded, so the check below falls back+ // to the page; a failed wait is an error.+ _ = runner.waitResult(timeout_ms, &conditions) catch |err| return switch (err) {+ error.Cancelled => ToolError.Cancelled,+ error.OutOfMemory => ToolError.OutOfMemory,+ else => ToolError.InternalError,};const alternate = page.frame() orelse return null;@@ -3573,6 +3583,15 @@ test "markdown: alternate reads the page's text/markdown link" {const fallback = try call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa, "{\"alternate\":true,\"url\":\"" ++ base ++ "markdown_alternate_html.html\"}", .{}), .{ .source = .user });try std.testing.expect(std.mem.find(u8, fallback.text, "Alternate is HTML") != null);try std.testing.expectEqual(1, session.pages.items.len);++ // A page that failed keeps its own content: the alternate's 200 would hide it.+ _ = try call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa, "{\"url\":\"" ++ base ++ "markdown_alternate.html\"}", .{}), .{ .source = .user });+ session.currentFrame().?._http_status = 404;+ const failed = try call(aa, session, ®istry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa,+ \\{"alternate":true}+ , .{}), .{ .source = .user });+ try std.testing.expect(std.mem.find(u8, failed.text, "Rendered page") != null);+ try std.testing.expect(std.mem.find(u8, failed.text, "markdown alternate") == null);}test "markdown: a same-url page whose navigation failed is navigated again" {diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 52183792d..0b9a8a266 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -349,14 +349,6 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet}}- // Pages swapped for their markdown alternate.- const alternates = try session.arena.allocator().alloc(bool, pages.items.len);- @memset(alternates, false);- if (opts.markdown_alternate) {- const remaining = opts.wait_ms -| @as(u32, @intCast(timer.untilNow(io, .boot).toMilliseconds()));- try followMarkdownAlternates(session, pages.items, errors, remaining, alternates);- }-var http_error = false;for (pages.items, errors) |page, *err| {const frame = page.frame() orelse {@@ -385,6 +377,16 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet}}+ // Pages swapped for their markdown alternate. This runs after the errors+ // and HTTP statuses above are recorded, and skips pages that failed, so an+ // alternate's 200 can't hide a failed page.+ const alternates = try session.arena.allocator().alloc(bool, pages.items.len);+ @memset(alternates, false);+ if (opts.markdown_alternate) {+ const remaining = opts.wait_ms -| @as(u32, @intCast(timer.untilNow(io, .boot).toMilliseconds()));+ try followMarkdownAlternates(session, pages.items, errors, remaining, alternates);+ }+try writeResults(app, opts, pages.items, errors, alternates);var failed = false;@@ -414,7 +416,8 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet/// Loads each page's markdown alternate in a page of its own (same session, so/// same cookies, robots and proxy) and swaps it in for the page when it loads/// as markdown, marking it in `followed`. Otherwise the page is kept and-/// converted as usual.+/// converted as usual. Pages with an error, a bot challenge or an HTTP error+/// status are kept as they are.fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, errors: []const ?anyerror, timeout_ms: u32, followed: []bool) !void {const arena = session.arena.allocator();const alternates = try arena.alloc(?Session.PageHandle, pages.len);@@ -423,6 +426,7 @@ fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, erroalternate.* = null;if (err != null) continue;const frame = page.frame() orelse continue;+ if (!markdown.canFollowAlternate(frame)) continue;const url = try markdown.alternateUrl(arena, frame) orelse continue;const alt = try session.createPage();alt.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } }) catch {@@ -840,6 +844,30 @@ test "fetch: followMarkdownAlternates swaps in a page's markdown alternate" {try testing.expectString(@embedFile("browser/tests/markdown_source.txt"), aw.written());}+test "fetch: followMarkdownAlternates keeps a page that failed" {+ const session = testing.test_session;+ defer session.closeAllPages();++ var pages = [_]Session.PageHandle{+ try testing.pageTest("markdown_alternate.html", .{ .wait_until_done = true }),+ try testing.pageTest("markdown_alternate.html", .{ .wait_until_done = true }),+ };+ // A 404 page with a markdown link: its alternate's 200 would hide the 404+ // from --fail-on-http-error.+ pages[0].frame().?._http_status = 404;+ pages[1].frame().?._bot_challenge = .cloudflare;+ var errors = [_]?anyerror{ null, null };+ var followed = [_]bool{ false, false };+ try followMarkdownAlternates(session, &pages, &errors, 2000, &followed);++ const page_url = "http://127.0.0.1:9582/src/browser/tests/markdown_alternate.html";+ try testing.expectString(page_url, pages[0].frame().?.url);+ try testing.expectString(page_url, pages[1].frame().?.url);+ try testing.expectEqual(false, followed[0]);+ try testing.expectEqual(false, followed[1]);+ try testing.expectEqual(2, session.pages.items.len);+}+test {std.testing.refAllDecls(@This());}