Skip to content

Lightpanda: the headless browser designed for AI and automation

fetch, tools: opt-in follow of a page's text/markdown alternate

Jay Hemnani40290f329058
diff --git a/src/Config.zig b/src/Config.zig
index 12bad7047..8c3114e92 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -433,6 +433,7 @@ pub const Commands = cli.Builder(.{
.{ .name = "dump_selector", .type = ?[:0]const u8 },
.{ .name = "dump_max_bytes", .type = ?u32 },
.{ .name = "fail_on_http_error", .type = bool },
+ .{ .name = "markdown_alternate", .type = bool },
.{ .name = "wait_ms", .type = u32, .default = 5_000 },
.{ .name = "wait_until", .type = ?WaitUntil },
.{
diff --git a/src/browser/markdown.zig b/src/browser/markdown.zig
index d4ab05239..0ad457b3d 100644
--- a/src/browser/markdown.zig
+++ b/src/browser/markdown.zig
@@ -20,6 +20,7 @@ const std = @import("std");
const URL = @import("URL.zig");
const Frame = @import("Frame.zig");
+const Selector = @import("webapi/selector/Selector.zig");
const RenderTree = @import("RenderTree.zig");
const StyleManager = @import("StyleManager.zig");
@@ -496,6 +497,30 @@ fn writeSource(source: *Node, opts: Opts, writer: *std.Io.Writer) !void {
};
}
+/// The http(s) URL of the page's `<link rel="alternate" type="text/markdown">`:
+/// the author's own markdown version of the page.
+pub fn alternateUrl(arena: std.mem.Allocator, frame: *Frame) !?[:0]const u8 {
+ const selector = "link[rel~=alternate i][type=\"text/markdown\" i][href]";
+ const link = try Selector.querySelector(frame.document.asNode(), selector, frame) orelse return null;
+ const href = link.getAttributeInterned("href") orelse return null;
+ const url = URL.resolve(arena, frame.base(), href, .{ .encoding = frame.charset }) catch |err| switch (err) {
+ error.TypeError => return null,
+ else => return err,
+ };
+ const protocol = URL.getProtocol(url);
+ if (!std.mem.eql(u8, protocol, "http:") and !std.mem.eql(u8, protocol, "https:")) return null;
+ return url;
+}
+
+/// Whether a followed alternate can stand in for the page: a 2xx
+/// text/markdown response.
+pub fn isAlternateLoaded(frame: *Frame) bool {
+ if (frame._last_navigate_error != null) return false;
+ const status = frame._http_status orelse return false;
+ if (status < 200 or status >= 300) return false;
+ return markdownSource(frame.document.asNode()) != null;
+}
+
pub fn dump(state: RenderTree.State, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
const node = state.root;
if (markdownSource(node)) |source| {
@@ -993,6 +1018,26 @@ test "browser.markdown: a text/markdown document is written as-is" {
try testing.expectString("# Title" ++ truncation_marker, capped.written());
}
+test "browser.markdown: alternateUrl" {
+ {
+ var page = try testing.pageTest("markdown_alternate.html", .{});
+ defer page.close();
+ const url = try alternateUrl(testing.arena_allocator, page.frame().?);
+ try testing.expectString("http://127.0.0.1:9582/src/browser/tests/markdown_source.md", url.?);
+ }
+ {
+ // Only http(s): a page can't point the reader at a local file.
+ var page = try testing.pageTest("markdown_alternate_file.html", .{});
+ defer page.close();
+ try testing.expect(try alternateUrl(testing.arena_allocator, page.frame().?) == null);
+ }
+ {
+ var page = try testing.pageTest("markdown_flex.html", .{});
+ defer page.close();
+ try testing.expect(try alternateUrl(testing.arena_allocator, page.frame().?) == null);
+ }
+}
+
// Builds a shadow host <div>, populates its light DOM with `light` and its
// (open) shadow tree with `shadow`, then dumps the host. Declarative shadow DOM
// parsing isn't implemented, so the shadow tree is attached imperatively.
diff --git a/src/browser/tests/markdown_alternate.html b/src/browser/tests/markdown_alternate.html
new file mode 100644
index 000000000..3906a6228
--- /dev/null
+++ b/src/browser/tests/markdown_alternate.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<head>
+<link rel="alternate" type="application/rss+xml" href="feed.xml">
+<link rel="Alternate" type="Text/Markdown" href="markdown_source.md">
+</head>
+<p>Rendered page</p>
diff --git a/src/browser/tests/markdown_alternate_file.html b/src/browser/tests/markdown_alternate_file.html
new file mode 100644
index 000000000..38575ba65
--- /dev/null
+++ b/src/browser/tests/markdown_alternate_file.html
@@ -0,0 +1,5 @@
+<!DOCTYPE html>
+<head>
+<link rel="alternate" type="text/markdown" href="file:///etc/hostname">
+</head>
+<p>Alternate is a local file</p>
diff --git a/src/browser/tests/markdown_alternate_html.html b/src/browser/tests/markdown_alternate_html.html
new file mode 100644
index 000000000..5d5c8c087
--- /dev/null
+++ b/src/browser/tests/markdown_alternate_html.html
@@ -0,0 +1,5 @@
+<!DOCTYPE html>
+<head>
+<link rel="alternate" type="text/markdown" href="markdown_alternate.html">
+</head>
+<p>Alternate is HTML</p>
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index ef6ab038e..b1721971c 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -456,6 +456,7 @@ pub const Tool = enum {
\\ "maxBytes": { "type": "integer", "description": "Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap." },
\\ "strip": { "type": "object", "description": "Optional. Omit element groups from the output; same groups as the html tool's strip. `shell` (page chrome by markup) and `clutter` (keep only the main content, in the manner of reader modes) are the ones that matter for reading; `ui` also drops images.", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" }, "shell": { "type": "boolean" }, "clutter": { "type": "boolean" } } },
\\ "url": { "type": "string", "description": "Optional URL to navigate to before rendering. An empty string is treated as omitted." },
+ \\ "alternate": { "type": "boolean", "description": "Optional. When the page links a markdown version of itself (<link rel=\"alternate\" type=\"text/markdown\">), return that file instead of converting the page. It is the author's file and can differ from the rendered page. Falls back to the page when there is no such link or it does not load as markdown. Ignored with selector or backendNodeId. Defaults to false." },
\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }
\\ }
\\}
@@ -1645,11 +1646,16 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeR
maxBytes: ?u32 = null,
strip: lp.dump.Opts.Strip = .{},
url: ?[:0]const u8 = null,
+ alternate: bool = false,
timeout: ?u32 = null,
};
const args = try parseArgsOrDefault(Params, arena, arguments);
const page = try ensurePage(session, registry, args.url, args.timeout);
+ if (args.alternate and args.selector == null and args.backendNodeId == null) {
+ if (try readMarkdownAlternate(arena, session, page, args.maxBytes, args.timeout)) |text| return text;
+ }
+
const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);
const state = lp.RenderTree.resolve(arena, node, args.strip, page) catch return ToolError.OutOfMemory;
@@ -1658,6 +1664,28 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeR
return aw.written();
}
+/// The page's `<link rel="alternate" type="text/markdown">`, loaded in a page
+/// of its own so the current page stays where it is. Null when there is no
+/// such link or it doesn't load as markdown.
+fn readMarkdownAlternate(arena: std.mem.Allocator, session: *lp.Session, frame: *lp.Frame, max_bytes: ?u32, timeout: ?u32) ToolError!?[]const u8 {
+ const url = (lp.markdown.alternateUrl(arena, frame) catch return ToolError.InternalError) orelse return null;
+ const page = openPage(session, url) catch return null;
+ defer page.close();
+
+ var runner = session.runner(.{});
+ var conditions = [_]lp.Session.Runner.WaitCondition{.{ .frame_id = page.frame_id, .until = default_nav_wait }};
+ _ = runner.waitResult(timeout orelse default_nav_timeout_ms, &conditions) catch |err| {
+ return if (err == error.Cancelled) ToolError.Cancelled else null;
+ };
+
+ const alternate = page.frame() orelse return null;
+ if (!lp.markdown.isAlternateLoaded(alternate)) return null;
+ var aw: std.Io.Writer.Allocating = .init(arena);
+ lp.markdown.dump(.{ .root = alternate.document.asNode() }, .{ .max_bytes = max_bytes }, &aw.writer, alternate) catch
+ return ToolError.InternalError;
+ return aw.written();
+}
+
/// The node a read tool works on: the selector match, the registry node, or
/// the whole document. All three live in the current frame.
fn resolveScope(session: *lp.Session, registry: *NodeRegistry, page: *lp.Frame, selector: ?[]const u8, node_id: ?NodeRegistry.Id) ToolError!*DOMNode {
@@ -3500,6 +3528,45 @@ test "isPathSafe: absolute paths and traversal are rejected" {
try std.testing.expect(!isPathSafe(".."));
}
+test "markdown: alternate reads the page's text/markdown link" {
+ var registry: NodeRegistry = .init(std.testing.allocator);
+ defer registry.deinit();
+
+ const session = testing.test_session;
+ defer if (session.primaryPage()) |page| page.close();
+
+ const aa = testing.arena_allocator;
+ const base = "http://localhost:9582/src/browser/tests/";
+ const source = @embedFile("tests/markdown_source.md");
+
+ const page = try call(aa, session, &registry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa, "{\"url\":\"" ++ base ++ "markdown_alternate.html\"}", .{}), .{ .source = .user });
+ try std.testing.expect(std.mem.find(u8, page.text, "Rendered page") != null);
+
+ const followed = try call(aa, session, &registry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa,
+ \\{"alternate":true}
+ , .{}), .{ .source = .user });
+ try std.testing.expectEqualStrings(source, followed.text);
+ // Read in a page of its own: the current page stays put.
+ try std.testing.expect(!followed.navigated);
+ try std.testing.expectEqual(1, session.pages.items.len);
+ try std.testing.expectEqualStrings(base ++ "markdown_alternate.html", session.currentFrame().?.url);
+
+ const capped = try call(aa, session, &registry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa,
+ \\{"alternate":true,"maxBytes":7}
+ , .{}), .{ .source = .user });
+ try std.testing.expect(std.mem.startsWith(u8, capped.text, "# Title\n\n[truncated]"));
+
+ const scoped = try call(aa, session, &registry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa,
+ \\{"alternate":true,"selector":"p"}
+ , .{}), .{ .source = .user });
+ try std.testing.expect(std.mem.find(u8, scoped.text, "Rendered page") != null);
+
+ // An alternate that doesn't load as markdown falls back to the page.
+ const fallback = try call(aa, session, &registry, "markdown", try std.json.parseFromSliceLeaky(std.json.Value, aa, "{\"alternate\":true,\"url\":\"" ++ base ++ "markdown_alternate_html.html\"}", .{}), .{ .source = .user });
+ try std.testing.expect(std.mem.find(u8, fallback.text, "Alternate is HTML") != null);
+ try std.testing.expectEqual(1, session.pages.items.len);
+}
+
test "markdown: a same-url page whose navigation failed is navigated again" {
var registry: NodeRegistry = .init(std.testing.allocator);
defer registry.deinit();
diff --git a/src/help.zon b/src/help.zon
index 31c347f8a..3c4032efc 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -136,6 +136,14 @@
\\ "error" field: null, or the name of the navigation, wait or
\\ dump failure for that URL. One failed URL does not stop the
\\ others; the process exits 1 if any URL failed.
+ \\ --markdown-alternate
+ \\ With --dump markdown, when a page links a markdown version of
+ \\ itself (<link rel="alternate" type="text/markdown">), dump that
+ \\ file instead of converting the page. The file is the author's
+ \\ and can differ from the rendered page. Falls back to converting
+ \\ the page when there is no such link or it does not load as
+ \\ markdown. With --json, "url" is where the markdown came from.
+ \\ Defaults to false.
\\ --metrics
\\ Write metrics (Prometheus text format) to stdout on exit, after
\\ any --dump output.
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index f9d4b2fcf..44c97df18 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -205,6 +205,9 @@ pub const FetchOpts = struct {
selector: ?[:0]const u8 = null,
/// Any page with an HTTP status >= 400 fails the fetch with `error.HttpError`.
fail_on_http_error: bool = false,
+ /// Dump a page's `<link rel="alternate" type="text/markdown">` instead of
+ /// converting the page, when that link loads as markdown.
+ markdown_alternate: bool = false,
writer: ?*std.Io.Writer = null,
json: bool = false,
};
@@ -346,6 +349,11 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
}
}
+ if (opts.markdown_alternate) {
+ const remaining = opts.wait_ms -| @as(u32, @intCast(timer.untilNow(io, .boot).toMilliseconds()));
+ try followMarkdownAlternates(session, pages.items, errors, remaining);
+ }
+
var http_error = false;
for (pages.items, errors) |page, *err| {
const frame = page.frame() orelse {
@@ -400,6 +408,44 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
}
}
+/// Loads each page's markdown alternate in a page of its own (same session, so
+/// same cookies, robots and proxy) and swaps it in for the page when it loads
+/// as markdown. Otherwise the page is kept and converted as usual.
+fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, errors: []const ?anyerror, timeout_ms: u32) !void {
+ const arena = session.arena.allocator();
+ const alternates = try arena.alloc(?Session.PageHandle, pages.len);
+ var conditions: std.ArrayList(Session.Runner.WaitCondition) = .empty;
+ for (pages, errors, alternates) |page, err, *alternate| {
+ alternate.* = null;
+ if (err != null) continue;
+ const frame = page.frame() orelse continue;
+ const url = try markdown.alternateUrl(arena, frame) orelse continue;
+ const alt = try session.createPage();
+ alt.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } }) catch {
+ alt.close();
+ continue;
+ };
+ alternate.* = alt;
+ try conditions.append(arena, .{ .frame_id = alt.frame_id, .until = .load });
+ }
+ if (conditions.items.len == 0) return;
+
+ var runner = session.runner(.{});
+ _ = try runner.waitResult(timeout_ms, conditions.items);
+
+ for (pages, alternates) |*page, alternate| {
+ const alt = alternate orelse continue;
+ if (alt.frame()) |frame| {
+ if (markdown.isAlternateLoaded(frame)) {
+ page.close();
+ page.* = alt;
+ continue;
+ }
+ }
+ alt.close();
+ }
+}
+
fn writeResults(app: *App, opts: FetchOpts, pages: []const Session.PageHandle, errors: []?anyerror) !void {
const writer = opts.writer orelse return;
@@ -753,6 +799,25 @@ test "fetch: resolveWaitUntil" {
);
}
+test "fetch: followMarkdownAlternates swaps in a page's markdown alternate" {
+ const session = testing.test_session;
+ defer session.closeAllPages();
+
+ var pages = [_]Session.PageHandle{
+ try testing.pageTest("markdown_alternate.html", .{ .wait_until_done = true }),
+ try testing.pageTest("markdown_alternate_html.html", .{ .wait_until_done = true }),
+ try testing.pageTest("markdown_flex.html", .{ .wait_until_done = true }),
+ };
+ var errors = [_]?anyerror{ null, null, null };
+ try followMarkdownAlternates(session, &pages, &errors, 2000);
+
+ const base = "http://127.0.0.1:9582/src/browser/tests/";
+ try testing.expectString(base ++ "markdown_source.md", pages[0].frame().?.url);
+ try testing.expectString(base ++ "markdown_alternate_html.html", pages[1].frame().?.url);
+ try testing.expectString(base ++ "markdown_flex.html", pages[2].frame().?.url);
+ try testing.expectEqual(3, session.pages.items.len);
+}
+
test {
std.testing.refAllDecls(@This());
}
diff --git a/src/main.zig b/src/main.zig
index 32a22945e..946086a75 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -190,6 +190,14 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args)
log.fatal(.app, "--dump-max-bytes needs text", .{ .dump = opts.dump, .allowed = "html, markdown" });
return error.InvalidArgument;
}
+ if (opts.markdown_alternate and opts.dump != .markdown) {
+ log.fatal(.app, "invalid --markdown-alternate", .{ .needs = "--dump markdown" });
+ return error.InvalidArgument;
+ }
+ if (opts.markdown_alternate and opts.dump_selector != null) {
+ log.fatal(.app, "invalid --markdown-alternate", .{ .conflicts_with = "--dump-selector" });
+ return error.InvalidArgument;
+ }
var fetch_opts = lp.FetchOpts{
.wait_ms = opts.wait_ms,
@@ -200,6 +208,7 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args)
.dump_mode = opts.dump,
.selector = opts.dump_selector,
.fail_on_http_error = opts.fail_on_http_error,
+ .markdown_alternate = opts.markdown_alternate,
.dump = .{
.strip = opts.strip_mode,
.with_base = opts.with_base,