Skip to content

Lightpanda: the headless browser designed for AI and automation

delay meta refersh until document "load"

Karl Seguin5aec8ea943c6
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 9c821fa04..b75cdfe05 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -275,6 +275,12 @@ _notified_network_almost_idle: IdleNotification = .init,
// next tick.
_queued_navigation: ?*QueuedNavigation = null,
+// Is there a <meta http-equiv=refresh> we need to fire on "load"?
+// Exactly which we need to fire can change, so rather than keeping it in sync
+// it's easier to scan for it on "load", but this is hint to avoid that scan
+// for the very common case where we're sure there isn't any.
+_maybe_meta_refresh: bool = false,
+
// The URL of the current frame
url: [:0]const u8 = "about:blank",
@@ -1232,6 +1238,11 @@ pub fn documentIsComplete(self: *Frame) void {
else => log.err(.frame, "document is complete", .{ .err = err, .type = self._type, .url = self.url }),
};
+ if (self._maybe_meta_refresh) {
+ self._maybe_meta_refresh = false;
+ self.metaRefreshOnLoad();
+ }
+
if (self.parent == null) {
self._session.browser.reportJsHeap();
}
@@ -1288,6 +1299,28 @@ fn notifyParentLoadComplete(self: *Frame) void {
parent.iframeCompletedLoading(self.iframe.?, self._delays_parent_load);
}
+fn metaRefreshOnLoad(self: *Frame) void {
+ const root = self.document.getDocumentElement() orelse return;
+ var tw = TreeWalker.Full.init(root.asNode(), .{});
+ while (tw.next()) |node| {
+ const meta = node.is(Element.Html.Meta) orelse continue;
+ const target = meta.refreshTarget() orelse continue;
+ return self.metaRefresh(target) catch |err| {
+ log.err(.frame, "meta refresh", .{ .err = err, .type = self._type, .url = self.url });
+ };
+ }
+}
+
+pub fn metaRefresh(self: *Frame, target: []const u8) !void {
+ if (self.isGoingAway()) {
+ return;
+ }
+ return self.scheduleNavigation(target, .{
+ .reason = .script,
+ .kind = .{ .replace = null },
+ }, .{ .script = self });
+}
+
fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.HeaderResult {
var self: *Frame = @ptrCast(@alignCast(transfer.req.ctx));
@@ -3738,24 +3771,78 @@ test "Frame: static immediate meta refresh navigates" {
const page = try testing.pageTest("fixtures/meta_refresh.html", .{});
defer page.close();
- try testing.expect(std.mem.endsWith(u8, page.frame().?.url, "/fixtures/meta_refresh_target.html"));
+ const frame = page.frame().?;
+ try testing.expect(std.mem.endsWith(u8, frame.url, "/fixtures/meta_refresh_target.html"));
+ // the rest of the page, including scripts after the meta, still ran
+ try testing.expectString("ran=1", (try frame.getTitle()).?);
+}
+
+test "Frame: meta refresh in a detached tree is ignored" {
+ // a <template> child and a clone are both created with a full set of
+ // attributes, but neither is ever in the document
+ const page = try testing.pageTest("fixtures/meta_refresh_ignored.html", .{});
+ defer page.close();
+
+ const frame = page.frame().?;
+ try testing.expect(std.mem.endsWith(u8, frame.url, "/fixtures/meta_refresh_ignored.html"));
+}
+
+test "Frame: dynamic immediate meta refresh waits for the load event" {
+ const frame = try testing.createFrame();
+ defer testing.test_session.closeAllPages();
+ frame.url = "https://example.com/source";
+ _ = try appendMetaRefresh(frame, "0; /target");
+
+ // still loading, so the navigation is held
+ try testing.expectEqual(null, frame._queued_navigation);
+ try testing.expect(frame._maybe_meta_refresh);
+
+ frame.documentIsComplete();
+ try testing.expectString("https://example.com/target", frame._queued_navigation.?.url);
}
-test "Frame: dynamic immediate meta refresh schedules navigation" {
+test "Frame: meta refresh added after the load event navigates" {
const frame = try testing.createFrame();
defer testing.test_session.closeAllPages();
frame.url = "https://example.com/source";
+ frame.documentIsComplete();
+
+ _ = try appendMetaRefresh(frame, "0; /target");
+ try testing.expectString("https://example.com/target", frame._queued_navigation.?.url);
+}
+
+test "Frame: meta refresh edited into something inert before the load event" {
+ const frame = try testing.createFrame();
+ defer testing.test_session.closeAllPages();
+ frame.url = "https://example.com/source";
+
+ const first = try appendMetaRefresh(frame, "0; /first");
+ _ = try appendMetaRefresh(frame, "0; /second");
+
+ // no longer an immediate refresh, so the next one in the document wins
+ try first.setAttribute(comptime .wrap("content"), comptime .wrap("10; /first"), frame);
+
+ frame.documentIsComplete();
+ try testing.expectString("https://example.com/second", frame._queued_navigation.?.url);
+}
+
+fn appendMetaRefresh(frame: *Frame, content: []const u8) !*Element {
+ const head = blk: {
+ if (frame.document.getDocumentElement()) |html| {
+ break :blk html.asNode().firstChild().?.as(Element);
+ }
+ const html = try frame.document.createElement("html", null, frame);
+ _ = try frame.document.asNode().appendChild(html.asNode(), frame);
+ const head = try frame.document.createElement("head", null, frame);
+ _ = try html.asNode().appendChild(head.asNode(), frame);
+ break :blk head;
+ };
- const html = try frame.document.createElement("html", null, frame);
- _ = try frame.document.asNode().appendChild(html.asNode(), frame);
- const head = try frame.document.createElement("head", null, frame);
- _ = try html.asNode().appendChild(head.asNode(), frame);
const meta = try frame.document.createElement("meta", null, frame);
try meta.setAttribute(comptime .wrap("http-equiv"), comptime .wrap("refresh"), frame);
- try meta.setAttribute(comptime .wrap("content"), comptime .wrap("0; /target"), frame);
+ try meta.setAttribute(comptime .wrap("content"), .wrap(content), frame);
_ = try head.asNode().appendChild(meta.asNode(), frame);
-
- try testing.expectString("https://example.com/target", frame._queued_navigation.?.url);
+ return meta;
}
test "Frame: httpMetadata after navigation" {
diff --git a/src/browser/tests/fixtures/meta_refresh.html b/src/browser/tests/fixtures/meta_refresh.html
index 7c12e17d4..15baa988d 100644
--- a/src/browser/tests/fixtures/meta_refresh.html
+++ b/src/browser/tests/fixtures/meta_refresh.html
@@ -1,3 +1,4 @@
<!doctype html>
<meta http-equiv="refresh" content="0; meta_refresh_target.html">
<title>source</title>
+<script>sessionStorage.setItem("meta_refresh_ran", "1");</script>
diff --git a/src/browser/tests/fixtures/meta_refresh_ignored.html b/src/browser/tests/fixtures/meta_refresh_ignored.html
new file mode 100644
index 000000000..6218d6a2e
--- /dev/null
+++ b/src/browser/tests/fixtures/meta_refresh_ignored.html
@@ -0,0 +1,7 @@
+<!doctype html>
+<title>ignored</title>
+<template><meta http-equiv="refresh" content="0; meta_refresh_target.html"></template>
+<script>
+ const meta = document.querySelector("template").content.querySelector("meta");
+ document.createElement("div").appendChild(meta.cloneNode());
+</script>
diff --git a/src/browser/tests/fixtures/meta_refresh_target.html b/src/browser/tests/fixtures/meta_refresh_target.html
index 296ed8c3a..923fb3a6b 100644
--- a/src/browser/tests/fixtures/meta_refresh_target.html
+++ b/src/browser/tests/fixtures/meta_refresh_target.html
@@ -1,3 +1,4 @@
<!doctype html>
<title>target</title>
+<script>document.title = "ran=" + (sessionStorage.getItem("meta_refresh_ran") ?? "0");</script>
<h1>target</h1>
diff --git a/src/browser/webapi/element/html/Meta.zig b/src/browser/webapi/element/html/Meta.zig
index 2ae016fa3..0a129551c 100644
--- a/src/browser/webapi/element/html/Meta.zig
+++ b/src/browser/webapi/element/html/Meta.zig
@@ -86,44 +86,90 @@ pub fn setScheme(self: *Meta, value: []const u8, frame: *Frame) !void {
}
pub fn processRefresh(self: *Meta, frame: *Frame) !void {
- if (!self.asNode().isConnected()) return;
- try self.scheduleRefresh(frame);
-}
+ if (!std.ascii.eqlIgnoreCase(self.getHttpEquiv(), "refresh")) {
+ return;
+ }
-fn scheduleRefresh(self: *Meta, frame: *Frame) !void {
- if (!std.ascii.eqlIgnoreCase(self.getHttpEquiv(), "refresh")) return;
+ if (frame._load_state != .complete) {
+ // meta refresh doesn't fire until document and, by that point, this
+ // meta tag might change. We signal the frame that MAYBE there's a meta
+ // refresh that it needs to execute. For the common case where a page
+ // never has a meta refresh, the frame's load can skip finding one.
+ frame._maybe_meta_refresh = true;
+ return;
+ }
+ // Already loaded, so this navigates as soon as it's a valid refresh.
+ if (!self.asNode().isConnected()) {
+ return;
+ }
const target = immediateRefreshTarget(self.getContent()) orelse return;
- try frame.scheduleNavigation(target, .{
- .reason = .script,
- .kind = .{ .replace = null },
- }, .{ .script = frame });
+ return frame.metaRefresh(target);
}
+// Where this meta wants to navigate, or null if it doesn't. Only for a meta
+// that's in the document: the caller's tree walk is what establishes that.
+pub fn refreshTarget(self: *Meta) ?[]const u8 {
+ if (!std.ascii.eqlIgnoreCase(self.getHttpEquiv(), "refresh")) {
+ return null;
+ }
+ return immediateRefreshTarget(self.getContent());
+}
+
+// Extracts the URL out of a <meta http-equiv=refresh> `content`, but only for
+// a 0 second delay. A longer delay is deliberately ignored: those are almost
+// always interstitial placeholders, where the page we already have is the one
+// worth keeping.
+// The grammar is the one browsers actually accept (Chromium's ParseHTTPRefresh
+// and the HTML spec's shared declarative refresh steps):
+// "5" a bare delay is a reload of the current page: no URL
+// "0; /a" "0,/a" "0 /a"
+// "0; url=/a" "0;url/a" "0; URL = '/a' junk"
fn immediateRefreshTarget(content: []const u8) ?[]const u8 {
const trimmed = std.mem.trim(u8, content, &std.ascii.whitespace);
- const separator = std.mem.indexOfAny(u8, trimmed, ";,") orelse return null;
- const delay = std.mem.trim(u8, trimmed[0..separator], &std.ascii.whitespace);
- const seconds = std.fmt.parseFloat(f64, delay) catch return null;
- if (!std.math.isFinite(seconds) or seconds != 0) return null;
-
- var target = std.mem.trim(u8, trimmed[separator + 1 ..], &std.ascii.whitespace);
- if (target.len >= 3 and std.ascii.eqlIgnoreCase(target[0..3], "url")) {
- target = std.mem.trimStart(u8, target[3..], &std.ascii.whitespace);
- if (target.len == 0 or target[0] != '=') return null;
- target = std.mem.trim(u8, target[1..], &std.ascii.whitespace);
+
+ // The delay ends at the first separator. Without one, there's no URL.
+ const separators = std.ascii.whitespace ++ [_]u8{ ';', ',' };
+ const separator = std.mem.indexOfAny(u8, trimmed, &separators) orelse return null;
+ const seconds = std.fmt.parseFloat(f64, trimmed[0..separator]) catch return null;
+ if (seconds != 0) {
+ // For now, we skip any meta refresh where the delay isn't 0. It isn't
+ // clear if there's a "best" option in all cases for this.
+ return null;
+ }
+
+ var rest = std.mem.trimStart(u8, trimmed[separator..], &std.ascii.whitespace);
+ if (rest.len > 0 and (rest[0] == ';' or rest[0] == ',')) {
+ rest = std.mem.trimStart(u8, rest[1..], &std.ascii.whitespace);
}
- if (target.len >= 2 and ((target[0] == '"' and target[target.len - 1] == '"') or (target[0] == '\'' and target[target.len - 1] == '\''))) {
- target = std.mem.trim(u8, target[1 .. target.len - 1], &std.ascii.whitespace);
+
+ // An optional "url" prefix. Without the "=", it isn't a prefix at all, the
+ // value is the URL itself (i.e. "0; url.html").
+ if (rest.len >= 3 and std.ascii.eqlIgnoreCase(rest[0..3], "url")) {
+ const after_url = std.mem.trimStart(u8, rest[3..], &std.ascii.whitespace);
+ if (after_url.len > 0 and after_url[0] == '=') {
+ rest = std.mem.trimStart(u8, after_url[1..], &std.ascii.whitespace);
+ }
+ }
+
+ // A quoted URL ends at its matching quote; whatever follows is junk.
+ if (rest.len > 0 and (rest[0] == '"' or rest[0] == '\'')) {
+ const quote = rest[0];
+ const quoted = rest[1..];
+ rest = quoted[0 .. std.mem.indexOfScalar(u8, quoted, quote) orelse quoted.len];
}
+
+ const target = std.mem.trim(u8, rest, &std.ascii.whitespace);
+ // an empty URL is a reload of the current page
return if (target.len == 0) null else target;
}
pub const Build = struct {
- // <meta name=referrer> sets the document's referrer policy.
pub fn created(node: *Node, frame: *Frame) !void {
const self = node.as(Meta);
const el = self.asElement();
+
+ // <meta name=referrer> sets the document's referrer policy.
if (el.getAttributeSafe(comptime .wrap("name"))) |name| {
if (std.ascii.eqlIgnoreCase(name, "referrer")) {
if (el.getAttributeSafe(comptime .wrap("content"))) |content| {
@@ -134,14 +180,19 @@ pub const Build = struct {
}
}
- if (frame._parse_mode != .fragment) {
- try self.scheduleRefresh(frame);
+ if (frame._parse_mode == .fragment) {
+ // innerHTML and DOMParser: these nodes are detached, they'll go
+ // through nodeIsReady if they're ever inserted.
+ return;
}
+ return self.processRefresh(frame);
}
pub fn attributeChange(element: *Element, name: String, _: String, frame: *Frame) !void {
- if (!name.eql(comptime .wrap("http-equiv")) and !name.eql(comptime .wrap("content"))) return;
- try element.as(Meta).processRefresh(frame);
+ if (!name.eql(comptime .wrap("http-equiv")) and !name.eql(comptime .wrap("content"))) {
+ return;
+ }
+ return element.as(Meta).processRefresh(element.asNode().ownerFrame(frame));
}
};
@@ -164,8 +215,21 @@ pub const JsApi = struct {
const testing = @import("../../../../testing.zig");
test "WebApi: Meta immediate refresh target" {
try testing.expectString("/target", immediateRefreshTarget("0; /target").?);
+ try testing.expectString("/target", immediateRefreshTarget("0,/target").?);
+ try testing.expectString("/target", immediateRefreshTarget(" 0 /target ").?);
try testing.expectString("/target", immediateRefreshTarget("0, URL = '/target'").?);
+ try testing.expectString("/target", immediateRefreshTarget("0;url=/target").?);
+ try testing.expectString("/target", immediateRefreshTarget("0; \"/target\" junk").?);
+ try testing.expectString("url.html", immediateRefreshTarget("0; url.html").?);
+ try testing.expectString("urlencoded.html", immediateRefreshTarget("0;urlencoded.html").?);
try testing.expectString("https://example.com/", immediateRefreshTarget("0.0; https://example.com/").?);
+
+ try testing.expectEqual(null, immediateRefreshTarget(""));
+ try testing.expectEqual(null, immediateRefreshTarget("0"));
+ try testing.expectEqual(null, immediateRefreshTarget("0;"));
+ try testing.expectEqual(null, immediateRefreshTarget("0; url="));
try testing.expectEqual(null, immediateRefreshTarget("1; /target"));
+ try testing.expectEqual(null, immediateRefreshTarget("0.5; /target"));
try testing.expectEqual(null, immediateRefreshTarget("invalid; /target"));
+ try testing.expectEqual(null, immediateRefreshTarget("0abc; /target"));
}