Lightpanda: the headless browser designed for AI and automation
delay meta refersh until document "load"
Karl Seguin5aec8ea943c6
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 9c821fa04..b75cdfe05 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -275,6 +275,12 @@ _notified_network_almost_idle: IdleNotification = .init,// next tick._queued_navigation: ?*QueuedNavigation = null,+// Is there a <meta http-equiv=refresh> we need to fire on "load"?+// Exactly which we need to fire can change, so rather than keeping it in sync+// it's easier to scan for it on "load", but this is hint to avoid that scan+// for the very common case where we're sure there isn't any.+_maybe_meta_refresh: bool = false,+// The URL of the current frameurl: [:0]const u8 = "about:blank",@@ -1232,6 +1238,11 @@ pub fn documentIsComplete(self: *Frame) void {else => log.err(.frame, "document is complete", .{ .err = err, .type = self._type, .url = self.url }),};+ if (self._maybe_meta_refresh) {+ self._maybe_meta_refresh = false;+ self.metaRefreshOnLoad();+ }+if (self.parent == null) {self._session.browser.reportJsHeap();}@@ -1288,6 +1299,28 @@ fn notifyParentLoadComplete(self: *Frame) void {parent.iframeCompletedLoading(self.iframe.?, self._delays_parent_load);}+fn metaRefreshOnLoad(self: *Frame) void {+ const root = self.document.getDocumentElement() orelse return;+ var tw = TreeWalker.Full.init(root.asNode(), .{});+ while (tw.next()) |node| {+ const meta = node.is(Element.Html.Meta) orelse continue;+ const target = meta.refreshTarget() orelse continue;+ return self.metaRefresh(target) catch |err| {+ log.err(.frame, "meta refresh", .{ .err = err, .type = self._type, .url = self.url });+ };+ }+}++pub fn metaRefresh(self: *Frame, target: []const u8) !void {+ if (self.isGoingAway()) {+ return;+ }+ return self.scheduleNavigation(target, .{+ .reason = .script,+ .kind = .{ .replace = null },+ }, .{ .script = self });+}+fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.HeaderResult {var self: *Frame = @ptrCast(@alignCast(transfer.req.ctx));@@ -3738,24 +3771,78 @@ test "Frame: static immediate meta refresh navigates" {const page = try testing.pageTest("fixtures/meta_refresh.html", .{});defer page.close();- try testing.expect(std.mem.endsWith(u8, page.frame().?.url, "/fixtures/meta_refresh_target.html"));+ const frame = page.frame().?;+ try testing.expect(std.mem.endsWith(u8, frame.url, "/fixtures/meta_refresh_target.html"));+ // the rest of the page, including scripts after the meta, still ran+ try testing.expectString("ran=1", (try frame.getTitle()).?);+}++test "Frame: meta refresh in a detached tree is ignored" {+ // a <template> child and a clone are both created with a full set of+ // attributes, but neither is ever in the document+ const page = try testing.pageTest("fixtures/meta_refresh_ignored.html", .{});+ defer page.close();++ const frame = page.frame().?;+ try testing.expect(std.mem.endsWith(u8, frame.url, "/fixtures/meta_refresh_ignored.html"));+}++test "Frame: dynamic immediate meta refresh waits for the load event" {+ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();+ frame.url = "https://example.com/source";+ _ = try appendMetaRefresh(frame, "0; /target");++ // still loading, so the navigation is held+ try testing.expectEqual(null, frame._queued_navigation);+ try testing.expect(frame._maybe_meta_refresh);++ frame.documentIsComplete();+ try testing.expectString("https://example.com/target", frame._queued_navigation.?.url);}-test "Frame: dynamic immediate meta refresh schedules navigation" {+test "Frame: meta refresh added after the load event navigates" {const frame = try testing.createFrame();defer testing.test_session.closeAllPages();frame.url = "https://example.com/source";+ frame.documentIsComplete();++ _ = try appendMetaRefresh(frame, "0; /target");+ try testing.expectString("https://example.com/target", frame._queued_navigation.?.url);+}++test "Frame: meta refresh edited into something inert before the load event" {+ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();+ frame.url = "https://example.com/source";++ const first = try appendMetaRefresh(frame, "0; /first");+ _ = try appendMetaRefresh(frame, "0; /second");++ // no longer an immediate refresh, so the next one in the document wins+ try first.setAttribute(comptime .wrap("content"), comptime .wrap("10; /first"), frame);++ frame.documentIsComplete();+ try testing.expectString("https://example.com/second", frame._queued_navigation.?.url);+}++fn appendMetaRefresh(frame: *Frame, content: []const u8) !*Element {+ const head = blk: {+ if (frame.document.getDocumentElement()) |html| {+ break :blk html.asNode().firstChild().?.as(Element);+ }+ const html = try frame.document.createElement("html", null, frame);+ _ = try frame.document.asNode().appendChild(html.asNode(), frame);+ const head = try frame.document.createElement("head", null, frame);+ _ = try html.asNode().appendChild(head.asNode(), frame);+ break :blk head;+ };- const html = try frame.document.createElement("html", null, frame);- _ = try frame.document.asNode().appendChild(html.asNode(), frame);- const head = try frame.document.createElement("head", null, frame);- _ = try html.asNode().appendChild(head.asNode(), frame);const meta = try frame.document.createElement("meta", null, frame);try meta.setAttribute(comptime .wrap("http-equiv"), comptime .wrap("refresh"), frame);- try meta.setAttribute(comptime .wrap("content"), comptime .wrap("0; /target"), frame);+ try meta.setAttribute(comptime .wrap("content"), .wrap(content), frame);_ = try head.asNode().appendChild(meta.asNode(), frame);-- try testing.expectString("https://example.com/target", frame._queued_navigation.?.url);+ return meta;}test "Frame: httpMetadata after navigation" {diff --git a/src/browser/tests/fixtures/meta_refresh.html b/src/browser/tests/fixtures/meta_refresh.htmlindex 7c12e17d4..15baa988d 100644--- a/src/browser/tests/fixtures/meta_refresh.html+++ b/src/browser/tests/fixtures/meta_refresh.html@@ -1,3 +1,4 @@<!doctype html><meta http-equiv="refresh" content="0; meta_refresh_target.html"><title>source</title>+<script>sessionStorage.setItem("meta_refresh_ran", "1");</script>diff --git a/src/browser/tests/fixtures/meta_refresh_ignored.html b/src/browser/tests/fixtures/meta_refresh_ignored.htmlnew file mode 100644index 000000000..6218d6a2e--- /dev/null+++ b/src/browser/tests/fixtures/meta_refresh_ignored.html@@ -0,0 +1,7 @@+<!doctype html>+<title>ignored</title>+<template><meta http-equiv="refresh" content="0; meta_refresh_target.html"></template>+<script>+ const meta = document.querySelector("template").content.querySelector("meta");+ document.createElement("div").appendChild(meta.cloneNode());+</script>diff --git a/src/browser/tests/fixtures/meta_refresh_target.html b/src/browser/tests/fixtures/meta_refresh_target.htmlindex 296ed8c3a..923fb3a6b 100644--- a/src/browser/tests/fixtures/meta_refresh_target.html+++ b/src/browser/tests/fixtures/meta_refresh_target.html@@ -1,3 +1,4 @@<!doctype html><title>target</title>+<script>document.title = "ran=" + (sessionStorage.getItem("meta_refresh_ran") ?? "0");</script><h1>target</h1>diff --git a/src/browser/webapi/element/html/Meta.zig b/src/browser/webapi/element/html/Meta.zigindex 2ae016fa3..0a129551c 100644--- a/src/browser/webapi/element/html/Meta.zig+++ b/src/browser/webapi/element/html/Meta.zig@@ -86,44 +86,90 @@ pub fn setScheme(self: *Meta, value: []const u8, frame: *Frame) !void {}pub fn processRefresh(self: *Meta, frame: *Frame) !void {- if (!self.asNode().isConnected()) return;- try self.scheduleRefresh(frame);-}+ if (!std.ascii.eqlIgnoreCase(self.getHttpEquiv(), "refresh")) {+ return;+ }-fn scheduleRefresh(self: *Meta, frame: *Frame) !void {- if (!std.ascii.eqlIgnoreCase(self.getHttpEquiv(), "refresh")) return;+ if (frame._load_state != .complete) {+ // meta refresh doesn't fire until document and, by that point, this+ // meta tag might change. We signal the frame that MAYBE there's a meta+ // refresh that it needs to execute. For the common case where a page+ // never has a meta refresh, the frame's load can skip finding one.+ frame._maybe_meta_refresh = true;+ return;+ }+ // Already loaded, so this navigates as soon as it's a valid refresh.+ if (!self.asNode().isConnected()) {+ return;+ }const target = immediateRefreshTarget(self.getContent()) orelse return;- try frame.scheduleNavigation(target, .{- .reason = .script,- .kind = .{ .replace = null },- }, .{ .script = frame });+ return frame.metaRefresh(target);}+// Where this meta wants to navigate, or null if it doesn't. Only for a meta+// that's in the document: the caller's tree walk is what establishes that.+pub fn refreshTarget(self: *Meta) ?[]const u8 {+ if (!std.ascii.eqlIgnoreCase(self.getHttpEquiv(), "refresh")) {+ return null;+ }+ return immediateRefreshTarget(self.getContent());+}++// Extracts the URL out of a <meta http-equiv=refresh> `content`, but only for+// a 0 second delay. A longer delay is deliberately ignored: those are almost+// always interstitial placeholders, where the page we already have is the one+// worth keeping.+// The grammar is the one browsers actually accept (Chromium's ParseHTTPRefresh+// and the HTML spec's shared declarative refresh steps):+// "5" a bare delay is a reload of the current page: no URL+// "0; /a" "0,/a" "0 /a"+// "0; url=/a" "0;url/a" "0; URL = '/a' junk"fn immediateRefreshTarget(content: []const u8) ?[]const u8 {const trimmed = std.mem.trim(u8, content, &std.ascii.whitespace);- const separator = std.mem.indexOfAny(u8, trimmed, ";,") orelse return null;- const delay = std.mem.trim(u8, trimmed[0..separator], &std.ascii.whitespace);- const seconds = std.fmt.parseFloat(f64, delay) catch return null;- if (!std.math.isFinite(seconds) or seconds != 0) return null;-- var target = std.mem.trim(u8, trimmed[separator + 1 ..], &std.ascii.whitespace);- if (target.len >= 3 and std.ascii.eqlIgnoreCase(target[0..3], "url")) {- target = std.mem.trimStart(u8, target[3..], &std.ascii.whitespace);- if (target.len == 0 or target[0] != '=') return null;- target = std.mem.trim(u8, target[1..], &std.ascii.whitespace);++ // The delay ends at the first separator. Without one, there's no URL.+ const separators = std.ascii.whitespace ++ [_]u8{ ';', ',' };+ const separator = std.mem.indexOfAny(u8, trimmed, &separators) orelse return null;+ const seconds = std.fmt.parseFloat(f64, trimmed[0..separator]) catch return null;+ if (seconds != 0) {+ // For now, we skip any meta refresh where the delay isn't 0. It isn't+ // clear if there's a "best" option in all cases for this.+ return null;+ }++ var rest = std.mem.trimStart(u8, trimmed[separator..], &std.ascii.whitespace);+ if (rest.len > 0 and (rest[0] == ';' or rest[0] == ',')) {+ rest = std.mem.trimStart(u8, rest[1..], &std.ascii.whitespace);}- if (target.len >= 2 and ((target[0] == '"' and target[target.len - 1] == '"') or (target[0] == '\'' and target[target.len - 1] == '\''))) {- target = std.mem.trim(u8, target[1 .. target.len - 1], &std.ascii.whitespace);++ // An optional "url" prefix. Without the "=", it isn't a prefix at all, the+ // value is the URL itself (i.e. "0; url.html").+ if (rest.len >= 3 and std.ascii.eqlIgnoreCase(rest[0..3], "url")) {+ const after_url = std.mem.trimStart(u8, rest[3..], &std.ascii.whitespace);+ if (after_url.len > 0 and after_url[0] == '=') {+ rest = std.mem.trimStart(u8, after_url[1..], &std.ascii.whitespace);+ }+ }++ // A quoted URL ends at its matching quote; whatever follows is junk.+ if (rest.len > 0 and (rest[0] == '"' or rest[0] == '\'')) {+ const quote = rest[0];+ const quoted = rest[1..];+ rest = quoted[0 .. std.mem.indexOfScalar(u8, quoted, quote) orelse quoted.len];}++ const target = std.mem.trim(u8, rest, &std.ascii.whitespace);+ // an empty URL is a reload of the current pagereturn if (target.len == 0) null else target;}pub const Build = struct {- // <meta name=referrer> sets the document's referrer policy.pub fn created(node: *Node, frame: *Frame) !void {const self = node.as(Meta);const el = self.asElement();++ // <meta name=referrer> sets the document's referrer policy.if (el.getAttributeSafe(comptime .wrap("name"))) |name| {if (std.ascii.eqlIgnoreCase(name, "referrer")) {if (el.getAttributeSafe(comptime .wrap("content"))) |content| {@@ -134,14 +180,19 @@ pub const Build = struct {}}- if (frame._parse_mode != .fragment) {- try self.scheduleRefresh(frame);+ if (frame._parse_mode == .fragment) {+ // innerHTML and DOMParser: these nodes are detached, they'll go+ // through nodeIsReady if they're ever inserted.+ return;}+ return self.processRefresh(frame);}pub fn attributeChange(element: *Element, name: String, _: String, frame: *Frame) !void {- if (!name.eql(comptime .wrap("http-equiv")) and !name.eql(comptime .wrap("content"))) return;- try element.as(Meta).processRefresh(frame);+ if (!name.eql(comptime .wrap("http-equiv")) and !name.eql(comptime .wrap("content"))) {+ return;+ }+ return element.as(Meta).processRefresh(element.asNode().ownerFrame(frame));}};@@ -164,8 +215,21 @@ pub const JsApi = struct {const testing = @import("../../../../testing.zig");test "WebApi: Meta immediate refresh target" {try testing.expectString("/target", immediateRefreshTarget("0; /target").?);+ try testing.expectString("/target", immediateRefreshTarget("0,/target").?);+ try testing.expectString("/target", immediateRefreshTarget(" 0 /target ").?);try testing.expectString("/target", immediateRefreshTarget("0, URL = '/target'").?);+ try testing.expectString("/target", immediateRefreshTarget("0;url=/target").?);+ try testing.expectString("/target", immediateRefreshTarget("0; \"/target\" junk").?);+ try testing.expectString("url.html", immediateRefreshTarget("0; url.html").?);+ try testing.expectString("urlencoded.html", immediateRefreshTarget("0;urlencoded.html").?);try testing.expectString("https://example.com/", immediateRefreshTarget("0.0; https://example.com/").?);++ try testing.expectEqual(null, immediateRefreshTarget(""));+ try testing.expectEqual(null, immediateRefreshTarget("0"));+ try testing.expectEqual(null, immediateRefreshTarget("0;"));+ try testing.expectEqual(null, immediateRefreshTarget("0; url="));try testing.expectEqual(null, immediateRefreshTarget("1; /target"));+ try testing.expectEqual(null, immediateRefreshTarget("0.5; /target"));try testing.expectEqual(null, immediateRefreshTarget("invalid; /target"));+ try testing.expectEqual(null, immediateRefreshTarget("0abc; /target"));}