Skip to content

Lightpanda: the headless browser designed for AI and automation

markdown: share opening the alternate, write a followed alternate directly

Adrià Arrufat2b924d796237
diff --git a/src/browser/URL.zig b/src/browser/URL.zig
index eea555915..8ffe302fc 100644
--- a/src/browser/URL.zig
+++ b/src/browser/URL.zig
@@ -242,6 +242,10 @@ pub fn getProtocol(raw: []const u8) []const u8 {
return raw[0 .. pos + 1];
}
+pub fn isRemoteScheme(raw: []const u8) bool {
+ return std.ascii.startsWithIgnoreCase(raw, "http:") or std.ascii.startsWithIgnoreCase(raw, "https:");
+}
+
pub fn isSecure(raw: []const u8) bool {
return std.mem.startsWith(u8, raw, "https:") or std.mem.startsWith(u8, raw, "wss:");
}
diff --git a/src/browser/frame/preload.zig b/src/browser/frame/preload.zig
index 43d17f210..c9ea9ebdb 100644
--- a/src/browser/frame/preload.zig
+++ b/src/browser/frame/preload.zig
@@ -41,7 +41,7 @@ pub fn scriptHint(frame: *Frame, element: ?*Element.Html, href: []const u8) bool
defer arena.release();
const resolved = URL.resolve(arena.allocator(), frame.base(), href, .{ .encoding = frame.charset }) catch return false;
- if (!isRemoteScheme(resolved)) {
+ if (!URL.isRemoteScheme(resolved)) {
return false;
}
return frame._script_manager.preloadScript(element, resolved) catch false;
@@ -61,7 +61,7 @@ pub fn moduleHint(frame: *Frame, element: ?*Element.Html, href: []const u8) bool
// The url becomes the imported_modules key, which must outlive the fetch
// so it lives on the frame arena
const resolved = URL.resolve(frame.page_arena, frame.base(), href, .{ .encoding = frame.charset }) catch return false;
- if (!isRemoteScheme(resolved)) {
+ if (!URL.isRemoteScheme(resolved)) {
return false;
}
@@ -106,7 +106,7 @@ const Prescan = struct {
return;
}
const resolved = URL.resolve(self.arena, self.base, href, .{ .encoding = frame.charset }) catch return;
- if (isRemoteScheme(resolved) == false) {
+ if (URL.isRemoteScheme(resolved) == false) {
return;
}
_ = frame._script_manager.preloadScript(null, resolved) catch {};
@@ -118,7 +118,7 @@ const Prescan = struct {
// The url becomes the imported_modules key, which must
// outlive the fetch so it lives on the frame arena.
const resolved = URL.resolve(frame.page_arena, self.base, href, .{ .encoding = frame.charset }) catch return;
- if (isRemoteScheme(resolved) == false) {
+ if (URL.isRemoteScheme(resolved) == false) {
return;
}
_ = frame._script_manager.base.preloadModuleHint(null, resolved, frame.url) catch {};
@@ -128,15 +128,11 @@ const Prescan = struct {
}
};
-fn isRemoteScheme(url: []const u8) bool {
- return std.ascii.startsWithIgnoreCase(url, "http:") or std.ascii.startsWithIgnoreCase(url, "https:");
-}
-
// Non-http(s) scheme (e.g. data:, blob:) never resolve to a remote URL. We
// detect this upfront to prevent uncessary URL.resolves that would dupe the
// (potentially very lage) data.
fn hasNonRemoteScheme(href: []const u8) bool {
- if (isRemoteScheme(href)) {
+ if (URL.isRemoteScheme(href)) {
return false;
}
if (href.len == 0 or !std.ascii.isAlphabetic(href[0])) {
diff --git a/src/browser/markdown.zig b/src/browser/markdown.zig
index 1df50da86..12ae78f76 100644
--- a/src/browser/markdown.zig
+++ b/src/browser/markdown.zig
@@ -20,6 +20,7 @@ const std = @import("std");
const URL = @import("URL.zig");
const Frame = @import("Frame.zig");
+const Session = @import("Session.zig");
const Selector = @import("webapi/selector/Selector.zig");
const RenderTree = @import("RenderTree.zig");
const StyleManager = @import("StyleManager.zig");
@@ -35,10 +36,6 @@ const repeat = @import("../string.zig").repeat;
pub const Opts = struct {
max_bytes: ?u32 = null,
- /// The document is a followed `<link rel="alternate" type="text/markdown">`.
- /// The link already says the file is markdown, and many servers send `.md`
- /// files as text/plain, so a text/plain alternate is written as-is too.
- alternate: bool = false,
};
const truncation_marker = LimitedWriter.truncation_marker;
@@ -485,20 +482,16 @@ const Context = struct {
/// A text/markdown response is loaded as a text document, its body a single
/// `<pre>`. Converting that would fence the whole file, so the document's
-/// text, which is the markdown the server sent, is written as-is. A text/plain
-/// document counts only as a followed alternate: plain text loaded directly
-/// isn't necessarily markdown and stays fenced.
-fn markdownSource(node: *Node, alternate: bool) ?*Node {
+/// text, which is the markdown the server sent, is written as-is.
+fn markdownSource(node: *Node) ?*Node {
const doc = node.is(Document) orelse return null;
- const content_type = doc.getContentType();
- if (!std.mem.eql(u8, content_type, "text/markdown") and
- !(alternate and std.mem.eql(u8, content_type, "text/plain"))) return null;
+ if (!std.mem.eql(u8, doc.getContentType(), "text/markdown")) return null;
const root = doc.getDocumentElement() orelse return null;
return root.asNode();
}
-fn writeSource(source: *Node, opts: Opts, writer: *std.Io.Writer) !void {
- var lw = LimitedWriter.init(writer, opts.max_bytes);
+fn writeSource(source: *Node, max_bytes: ?u32, writer: *std.Io.Writer) !void {
+ var lw = LimitedWriter.init(writer, max_bytes);
source.getTextContent(&lw.writer) catch |err| {
if (!lw.truncated) return err;
try writer.writeAll(truncation_marker);
@@ -515,33 +508,49 @@ pub fn alternateUrl(arena: std.mem.Allocator, frame: *Frame) !?[:0]const u8 {
error.TypeError => return null,
else => return err,
};
- const protocol = URL.getProtocol(url);
- if (!std.mem.eql(u8, protocol, "http:") and !std.mem.eql(u8, protocol, "https:")) return null;
+ if (!URL.isRemoteScheme(url)) return null;
return url;
}
-/// Whether a page may be swapped for its alternate: only one that loaded
-/// fine. The alternate of a 404 or a bot challenge page would hide the
-/// failure behind its own 200.
-pub fn canFollowAlternate(frame: *Frame) bool {
- if (frame._last_navigate_error != null or frame._bot_challenge != null) return false;
- const status = frame._http_status orelse return true;
- return status < 400;
+/// Starts loading the page's markdown alternate in a new page of the same
+/// session. Null when the page has no alternate, or failed itself: the
+/// alternate of a 404 or a bot challenge page would hide the failure behind
+/// its own 200.
+pub fn openAlternate(arena: std.mem.Allocator, session: *Session, frame: *Frame) !?Session.PageHandle {
+ if (frame._last_navigate_error != null or frame._bot_challenge != null) return null;
+ if (frame._http_status) |status| {
+ if (status >= 400) return null;
+ }
+ const url = try alternateUrl(arena, frame) orelse return null;
+ const page = try session.createPage();
+ page.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } }) catch {
+ page.close();
+ return null;
+ };
+ return page;
}
-/// Whether a followed alternate can stand in for the page: a 2xx
-/// text/markdown or text/plain response.
+/// Whether a loaded alternate can stand in for the page: a 2xx text/markdown
+/// or text/plain response. The link already says the file is markdown, and
+/// many servers send `.md` files as text/plain.
pub fn isAlternateLoaded(frame: *Frame) bool {
if (frame._last_navigate_error != null) return false;
const status = frame._http_status orelse return false;
if (status < 200 or status >= 300) return false;
- return markdownSource(frame.document.asNode(), true) != null;
+ const content_type = frame.document.getContentType();
+ return std.mem.eql(u8, content_type, "text/markdown") or std.mem.eql(u8, content_type, "text/plain");
+}
+
+/// Writes an alternate that passed `isAlternateLoaded` as-is.
+pub fn dumpAlternate(frame: *Frame, max_bytes: ?u32, writer: *std.Io.Writer) !void {
+ const root = frame.document.getDocumentElement() orelse return;
+ return writeSource(root.asNode(), max_bytes, writer);
}
pub fn dump(state: RenderTree.State, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
const node = state.root;
- if (markdownSource(node, opts.alternate)) |source| {
- return writeSource(source, opts, writer);
+ if (markdownSource(node)) |source| {
+ return writeSource(source, opts.max_bytes, writer);
}
if (opts.max_bytes) |limit| {
var lw = LimitedWriter.init(writer, limit);
@@ -1058,7 +1067,7 @@ test "browser.markdown: text/plain is written as-is only as a followed alternate
try testing.expect(!std.mem.eql(u8, direct.written(), source));
var followed: std.Io.Writer.Allocating = .init(testing.arena_allocator);
- try dump(.{ .root = frame.window._document.asNode() }, .{ .alternate = true }, &followed.writer, frame);
+ try dumpAlternate(frame, null, &followed.writer);
try testing.expectString(source, followed.written());
}
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 773708cd9..f909f8659 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -1656,12 +1656,9 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeR
if (args.alternate) {
// The alternate stands for the whole page, like `fetch --markdown-alternate`.
if (args.selector != null or args.backendNodeId != null) return ToolError.InvalidParams;
- if (lp.markdown.canFollowAlternate(page)) {
- // One timeout for the call: the alternate gets what loading the page left.
- const elapsed: u32 = @intCast(timer.untilNow(lp.io, .boot).toMilliseconds());
- const remaining = (args.timeout orelse default_nav_timeout_ms) -| elapsed;
- if (try readMarkdownAlternate(arena, session, page, args.maxBytes, remaining)) |text| return text;
- }
+ const elapsed: u32 = @intCast(timer.untilNow(lp.io, .boot).toMilliseconds());
+ const remaining = (args.timeout orelse default_nav_timeout_ms) -| elapsed;
+ if (try readMarkdownAlternate(arena, session, page, args.maxBytes, remaining)) |text| return text;
}
const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);
@@ -1674,10 +1671,13 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeR
/// The page's `<link rel="alternate" type="text/markdown">`, loaded in a page
/// of its own so the current page stays where it is. Null when there is no
-/// such link or it doesn't load as markdown within `timeout_ms`.
+/// such link, the page failed, or the alternate doesn't load as markdown
+/// within `timeout_ms`.
fn readMarkdownAlternate(arena: std.mem.Allocator, session: *lp.Session, frame: *lp.Frame, max_bytes: ?u32, timeout_ms: u32) ToolError!?[]const u8 {
- const url = (lp.markdown.alternateUrl(arena, frame) catch return ToolError.InternalError) orelse return null;
- const page = openPage(session, url) catch return null;
+ const page = (lp.markdown.openAlternate(arena, session, frame) catch |err| return switch (err) {
+ error.OutOfMemory => ToolError.OutOfMemory,
+ else => ToolError.InternalError,
+ }) orelse return null;
defer page.close();
var runner = session.runner(.{});
@@ -1695,8 +1695,7 @@ fn readMarkdownAlternate(arena: std.mem.Allocator, session: *lp.Session, frame:
// Say which source this is: the alternate can differ from the rendered page.
var aw: std.Io.Writer.Allocating = .init(arena);
aw.writer.print("<!-- markdown alternate: {s} -->\n", .{alternate.url}) catch return ToolError.OutOfMemory;
- lp.markdown.dump(.{ .root = alternate.document.asNode() }, .{ .max_bytes = max_bytes, .alternate = true }, &aw.writer, alternate) catch
- return ToolError.InternalError;
+ lp.markdown.dumpAlternate(alternate, max_bytes, &aw.writer) catch return ToolError.InternalError;
return aw.written();
}
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index 0b9a8a266..7509c14c0 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -377,9 +377,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
}
}
- // Pages swapped for their markdown alternate. This runs after the errors
- // and HTTP statuses above are recorded, and skips pages that failed, so an
- // alternate's 200 can't hide a failed page.
+ // After the errors above are recorded, so a failed page keeps them.
const alternates = try session.arena.allocator().alloc(bool, pages.items.len);
@memset(alternates, false);
if (opts.markdown_alternate) {
@@ -416,8 +414,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
/// Loads each page's markdown alternate in a page of its own (same session, so
/// same cookies, robots and proxy) and swaps it in for the page when it loads
/// as markdown, marking it in `followed`. Otherwise the page is kept and
-/// converted as usual. Pages with an error, a bot challenge or an HTTP error
-/// status are kept as they are.
+/// converted as usual.
fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, errors: []const ?anyerror, timeout_ms: u32, followed: []bool) !void {
const arena = session.arena.allocator();
const alternates = try arena.alloc(?Session.PageHandle, pages.len);
@@ -426,13 +423,7 @@ fn followMarkdownAlternates(session: *Session, pages: []Session.PageHandle, erro
alternate.* = null;
if (err != null) continue;
const frame = page.frame() orelse continue;
- if (!markdown.canFollowAlternate(frame)) continue;
- const url = try markdown.alternateUrl(arena, frame) orelse continue;
- const alt = try session.createPage();
- alt.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } }) catch {
- alt.close();
- continue;
- };
+ const alt = try markdown.openAlternate(arena, session, frame) orelse continue;
alternate.* = alt;
try conditions.append(arena, .{ .frame_id = alt.frame_id, .until = .load });
}
@@ -556,12 +547,15 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
}
fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, alternate: bool, writer: *std.Io.Writer) !void {
+ if (alternate) {
+ return markdown.dumpAlternate(frame, opts.dump.max_bytes, writer);
+ }
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
const state = try RenderTree.resolve(arena.allocator(), try dumpRoot(frame, opts.selector), opts.dump.strip, frame);
switch (mode) {
.html => try dump.render(state, opts.dump, writer, frame),
- .markdown => try markdown.dump(state, .{ .max_bytes = opts.dump.max_bytes, .alternate = alternate }, writer, frame),
+ .markdown => try markdown.dump(state, .{ .max_bytes = opts.dump.max_bytes }, writer, frame),
.png => {
_ = try screenshot.png(arena.allocator(), state, pngOpts(frame), writer, frame);
},
@@ -832,10 +826,7 @@ test "fetch: followMarkdownAlternates swaps in a page's markdown alternate" {
try testing.expectString(base ++ "markdown_flex.html", pages[2].frame().?.url);
// Many servers send .md files as text/plain; the link already says it's markdown.
try testing.expectString(base ++ "markdown_source.txt", pages[3].frame().?.url);
- try testing.expectEqual(true, followed[0]);
- try testing.expectEqual(false, followed[1]);
- try testing.expectEqual(false, followed[2]);
- try testing.expectEqual(true, followed[3]);
+ try testing.expectEqualSlices(bool, &.{ true, false, false, true }, &followed);
try testing.expectEqual(4, session.pages.items.len);
// A followed text/plain alternate is written as-is, not fenced.
@@ -863,8 +854,7 @@ test "fetch: followMarkdownAlternates keeps a page that failed" {
const page_url = "http://127.0.0.1:9582/src/browser/tests/markdown_alternate.html";
try testing.expectString(page_url, pages[0].frame().?.url);
try testing.expectString(page_url, pages[1].frame().?.url);
- try testing.expectEqual(false, followed[0]);
- try testing.expectEqual(false, followed[1]);
+ try testing.expectEqualSlices(bool, &.{ false, false }, &followed);
try testing.expectEqual(2, session.pages.items.len);
}