Lightpanda: the headless browser designed for AI and automation
markdown: share opening the alternate, write a followed alternate directly
Adrià Arrufat2b924d796237
diff --git a/src/browser/URL.zig b/src/browser/URL.zigindex eea555915..8ffe302fc 100644--- a/src/browser/URL.zig+++ b/src/browser/URL.zig@@ -242,6 +242,10 @@ pub fn getProtocol(raw: []const u8) []const u8 {return raw[0 .. pos + 1];}+pub fn isRemoteScheme(raw: []const u8) bool {+ return std.ascii.startsWithIgnoreCase(raw, "http:") or std.ascii.startsWithIgnoreCase(raw, "https:");+}+pub fn isSecure(raw: []const u8) bool {return std.mem.startsWith(u8, raw, "https:") or std.mem.startsWith(u8, raw, "wss:");}diff --git a/src/browser/frame/preload.zig b/src/browser/frame/preload.zigindex 43d17f210..c9ea9ebdb 100644--- a/src/browser/frame/preload.zig+++ b/src/browser/frame/preload.zig@@ -41,7 +41,7 @@ pub fn scriptHint(frame: *Frame, element: ?*Element.Html, href: []const u8) booldefer arena.release();const resolved = URL.resolve(arena.allocator(), frame.base(), href, .{ .encoding = frame.charset }) catch return false;- if (!isRemoteScheme(resolved)) {+ if (!URL.isRemoteScheme(resolved)) {return false;}return frame._script_manager.preloadScript(element, resolved) catch false;@@ -61,7 +61,7 @@ pub fn moduleHint(frame: *Frame, element: ?*Element.Html, href: []const u8) bool// The url becomes the imported_modules key, which must outlive the fetch// so it lives on the frame arenaconst resolved = URL.resolve(frame.page_arena, frame.base(), href, .{ .encoding = frame.charset }) catch return false;- if (!isRemoteScheme(resolved)) {+ if (!URL.isRemoteScheme(resolved)) {return false;}@@ -106,7 +106,7 @@ const Prescan = struct {return;}const resolved = URL.resolve(self.arena, self.base, href, .{ .encoding = frame.charset }) catch return;- if (isRemoteScheme(resolved) == false) {+ if (URL.isRemoteScheme(resolved) == false) {return;}_ = frame._script_manager.preloadScript(null, resolved) catch {};@@ -118,7 +118,7 @@ const Prescan = struct {// The url becomes the imported_modules key, which must// outlive the fetch so it lives on the frame arena.const resolved = URL.resolve(frame.page_arena, self.base, href, .{ .encoding = frame.charset }) catch return;- if (isRemoteScheme(resolved) == false) {+ if (URL.isRemoteScheme(resolved) == false) {return;}_ = frame._script_manager.base.preloadModuleHint(null, resolved, frame.url) catch {};@@ -128,15 +128,11 @@ const Prescan = struct {}};-fn isRemoteScheme(url: []const u8) bool {- return std.ascii.startsWithIgnoreCase(url, "http:") or std.ascii.startsWithIgnoreCase(url, "https:");-}-// Non-http(s) scheme (e.g. data:, blob:) never resolve to a remote URL. We// detect this upfront to prevent uncessary URL.resolves that would dupe the// (potentially very lage) data.fn hasNonRemoteScheme(href: []const u8) bool {- if (isRemoteScheme(href)) {+ if (URL.isRemoteScheme(href)) {return false;}if (href.len == 0 or !std.ascii.isAlphabetic(href[0])) {diff --git a/src/browser/markdown.zig b/src/browser/markdown.zigindex 1df50da86..12ae78f76 100644--- a/src/browser/markdown.zig+++ b/src/browser/markdown.zig@@ -20,6 +20,7 @@ const std = @import("std");const URL = @import("URL.zig");const Frame = @import("Frame.zig");+const Session = @import("Session.zig");const Selector = @import("webapi/selector/Selector.zig");const RenderTree = @import("RenderTree.zig");const StyleManager = @import("StyleManager.zig");@@ -35,10 +36,6 @@ const repeat = @import("../string.zig").repeat;pub const Opts = struct {max_bytes: ?u32 = null,- /// The document is a followed `<link rel="alternate" type="text/markdown">`.- /// The link already says the file is markdown, and many servers send `.md`- /// files as text/plain, so a text/plain alternate is written as-is too.- alternate: bool = false,};const truncation_marker = LimitedWriter.truncation_marker;@@ -485,20 +482,16 @@ const Context = struct {/// A text/markdown response is loaded as a text document, its body a single/// `<pre>`. Converting that would fence the whole file, so the document's-/// text, which is the markdown the server sent, is written as-is. A text/plain-/// document counts only as a followed alternate: plain text loaded directly-/// isn't necessarily markdown and stays fenced.-fn markdownSource(node: *Node, alternate: bool) ?*Node {+/// text, which is the markdown the server sent, is written as-is.+fn markdownSource(node: *Node) ?*Node {const doc = node.is(Document) orelse return null;- const content_type = doc.getContentType();- if (!std.mem.eql(u8, content_type, "text/markdown") and- !(alternate and std.mem.eql(u8, content_type, "text/plain"))) return null;+ if (!std.mem.eql(u8, doc.getContentType(), "text/markdown")) return null;const root = doc.getDocumentElement() orelse return null;return root.asNode();}-fn writeSource(source: *Node, opts: Opts, writer: *std.Io.Writer) !void {- var lw = LimitedWriter.init(writer, opts.max_bytes);+fn writeSource(source: *Node, max_bytes: ?u32, writer: *std.Io.Writer) !void {+ var lw = LimitedWriter.init(writer, max_bytes);source.getTextContent(&lw.writer) catch |err| {if (!lw.truncated) return err;try writer.writeAll(truncation_marker);@@ -515,33 +508,49 @@ pub fn alternateUrl(arena: std.mem.Allocator, frame: *Frame) !?[:0]const u8 {error.TypeError => return null,else => return err,};- const protocol = URL.getProtocol(url);- if (!std.mem.eql(u8, protocol, "http:") and !std.mem.eql(u8, protocol, "https:")) return null;+ if (!URL.isRemoteScheme(url)) return null;return url;}-/// Whether a page may be swapped for its alternate: only one that loaded-/// fine. The alternate of a 404 or a bot challenge page would hide the-/// failure behind its own 200.-pub fn canFollowAlternate(frame: *Frame) bool {- if (frame._last_navigate_error != null or frame._bot_challenge != null) return false;- const status = frame._http_status orelse return true;- return status < 400;+/// Starts loading the page's markdown alternate in a new page of the same+/// session. Null when the page has no alternate, or failed itself: the+/// alternate of a 404 or a bot challenge page would hide the failure behind+/// its own 200.+pub fn openAlternate(arena: std.mem.Allocator, session: *Session, frame: *Frame) !?Session.PageHandle {+ if (frame._last_navigate_error != null or frame._bot_challenge != null) return null;+ if (frame._http_status) |status| {+ if (status >= 400) return null;+ }+ const url = try alternateUrl(arena, frame) orelse return null;+ const page = try session.createPage();+ page.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } }) catch {+ page.close();+ return null;+ };+ return page;}-/// Whether a followed alternate can stand in for the page: a 2xx-/// text/markdown or text/plain response.+/// Whether a loaded alternate can stand in for the page: a 2xx text/markdown+/// or text/plain response. The link already says the file is markdown, and+/// many servers send `.md` files as text/plain.pub fn isAlternateLoaded(frame: *Frame) bool {if (frame._last_navigate_error != null) return false;const status = frame._http_status orelse return false;if (status < 200 or status >= 300) return false;- return markdownSource(frame.document.asNode(), true) != null;+ const content_type = frame.document.getContentType();+ return std.mem.eql(u8, content_type, "text/markdown") or std.mem.eql(u8, content_type, "text/plain");+}++/// Writes an alternate that passed `isAlternateLoaded` as-is.+pub fn dumpAlternate(frame: *Frame, max_bytes: ?u32, writer: *std.Io.Writer) !void {+ const root = frame.document.getDocumentElement() orelse return;+ return writeSource(root.asNode(), max_bytes, writer);}pub fn dump(state: RenderTree.State, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {const node = state.root;- if (markdownSource(node, opts.alternate)) |source| {- return writeSource(source, opts, writer);+ if (markdownSource(node)) |source| {+ return writeSource(source, opts.max_bytes, writer);}if (opts.max_bytes) |limit| {var lw = LimitedWriter.init(writer, limit);@@ -1058,7 +1067,7 @@ test "browser.markdown: text/plain is written as-is only as a followed alternatetry testing.expect(!std.mem.eql(u8, direct.written(), source));var followed: std.Io.Writer.Allocating = .init(testing.arena_allocator);- try dump(.{ .root = frame.window._document.asNode() }, .{ .alternate = true }, &followed.writer, frame);+ try dumpAlternate(frame, null, &followed.writer);try testing.expectString(source, followed.written());}diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 773708cd9..f909f8659 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -1656,12 +1656,9 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRif (args.alternate) {// The alternate stands for the whole page, like `fetch --markdown-alternate`.if (args.selector != null or args.backendNodeId != null) return ToolError.InvalidParams;- if (lp.markdown.canFollowAlternate(page)) {- // One timeout for the call: the alternate gets what loading the page left.- const elapsed: u32 = @intCast(timer.untilNow(lp.io, .boot).toMilliseconds());- const remaining = (args.timeout orelse default_nav_timeout_ms) -| elapsed;- if (try readMarkdownAlternate(arena, session, page, args.maxBytes, remaining)) |text| return text;- }+ const elapsed: u32 = @intCast(timer.untilNow(lp.io, .boot).toMilliseconds());+ const remaining = (args.timeout orelse default_nav_timeout_ms) -| elapsed;+ if (try readMarkdownAlternate(arena, session, page, args.maxBytes, remaining)) |text| return text;}const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);@@ -1674,10 +1671,13 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeR/// The page's `<link rel="alternate" type="text/markdown">`, loaded in a page/// of its own so the current page stays where it is. Null when there is no-/// such link or it doesn't load as markdown within `timeout_ms`.+/// such link, the page failed, or the alternate doesn't load as markdown+/// within `timeout_ms`.fn readMarkdownAlternate(arena: std.mem.Allocator, session: *lp.Session, frame: *lp.Frame, max_bytes: ?u32, timeout_ms: u32) ToolError!?[]const u8 {- const url = (lp.markdown.alternateUrl(arena, frame) catch return ToolError.InternalError) orelse return null;- const page = openPage(session, url) catch return null;+ const page = (lp.markdown.openAlternate(arena, session, frame) catch |err| return switch (err) {+ error.OutOfMemory => ToolError.OutOfMemory,+ else => ToolError.InternalError,+ }) orelse return null;defer page.close();var runner = session.runner(.{});@@ -1695,8 +1695,7 @@ fn readMarkdownAlternate(arena: std.mem.Allocator, session: *lp.Session, frame:// Say which source this is: the alternate can differ from the rendered page.var aw: std.Io.Writer.Allocating = .init(arena);aw.writer.print("<!-- markdown alternate: {s} -->\n", .{alternate.url}) catch return ToolError.OutOfMemory;- lp.markdown.dump(.{ .root = alternate.document.asNode() }, .{ .max_bytes = max_bytes, .alternate = true }, &aw.writer, alternate) catch- return ToolError.InternalError;+ lp.markdown.dumpAlternate(alternate, max_bytes, &aw.writer) catch return ToolError.InternalError;return aw.written();}diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 0b9a8a266..7509c14c0 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -377,9 +377,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet}}- // Pages swapped for their markdown alternate. This runs after the errors- // and HTTP statuses above are recorded, and skips pages that failed, so an- // alternate's 200 can't hide a failed page.+ // After the errors above are recorded, so a failed page keeps them.const alternates = try session.arena.allocator().alloc(bool, pages.items.len);@memset(alternates, false);if (opts.markdown_alternate) {@@ -416,8 +414,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet/// Loads each page's markdown alternate in a page of its own (same session, so/// same cookies, robots and proxy) and swaps it in for the page when it loads/// as markdown, marking it in `followed`. Otherwise the page is kept and-/// converted as usual. Pages with an error, a bot challenge or an HTTP error-/// status are kept as they are.+/// converted as usual.fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, errors: []const ?anyerror, timeout_ms: u32, followed: []bool) !void {const arena = session.arena.allocator();const alternates = try arena.alloc(?Session.PageHandle, pages.len);@@ -426,13 +423,7 @@ fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, erroalternate.* = null;if (err != null) continue;const frame = page.frame() orelse continue;- if (!markdown.canFollowAlternate(frame)) continue;- const url = try markdown.alternateUrl(arena, frame) orelse continue;- const alt = try session.createPage();- alt.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } }) catch {- alt.close();- continue;- };+ const alt = try markdown.openAlternate(arena, session, frame) orelse continue;alternate.* = alt;try conditions.append(arena, .{ .frame_id = alt.frame_id, .until = .load });}@@ -556,12 +547,15 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {}fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, alternate: bool, writer: *std.Io.Writer) !void {+ if (alternate) {+ return markdown.dumpAlternate(frame, opts.dump.max_bytes, writer);+ }var arena: std.heap.ArenaAllocator = .init(app.allocator);defer arena.deinit();const state = try RenderTree.resolve(arena.allocator(), try dumpRoot(frame, opts.selector), opts.dump.strip, frame);switch (mode) {.html => try dump.render(state, opts.dump, writer, frame),- .markdown => try markdown.dump(state, .{ .max_bytes = opts.dump.max_bytes, .alternate = alternate }, writer, frame),+ .markdown => try markdown.dump(state, .{ .max_bytes = opts.dump.max_bytes }, writer, frame),.png => {_ = try screenshot.png(arena.allocator(), state, pngOpts(frame), writer, frame);},@@ -832,10 +826,7 @@ test "fetch: followMarkdownAlternates swaps in a page's markdown alternate" {try testing.expectString(base ++ "markdown_flex.html", pages[2].frame().?.url);// Many servers send .md files as text/plain; the link already says it's markdown.try testing.expectString(base ++ "markdown_source.txt", pages[3].frame().?.url);- try testing.expectEqual(true, followed[0]);- try testing.expectEqual(false, followed[1]);- try testing.expectEqual(false, followed[2]);- try testing.expectEqual(true, followed[3]);+ try testing.expectEqualSlices(bool, &.{ true, false, false, true }, &followed);try testing.expectEqual(4, session.pages.items.len);// A followed text/plain alternate is written as-is, not fenced.@@ -863,8 +854,7 @@ test "fetch: followMarkdownAlternates keeps a page that failed" {const page_url = "http://127.0.0.1:9582/src/browser/tests/markdown_alternate.html";try testing.expectString(page_url, pages[0].frame().?.url);try testing.expectString(page_url, pages[1].frame().?.url);- try testing.expectEqual(false, followed[0]);- try testing.expectEqual(false, followed[1]);+ try testing.expectEqualSlices(bool, &.{ false, false }, &followed);try testing.expectEqual(2, session.pages.items.len);}