Lightpanda: the headless browser designed for AI and automation
http: recalculate Referer header on each redirect
Karl Seguinabc6f1ef76f4
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex c046fc98b..538802d64 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -601,6 +601,7 @@ pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {const arena = transfer.arena.allocator();if (try referrer.compute(arena, self.referrer_policy, self.url, transfer.req.url)) |ref| {try transfer.addHeader("Referer", ref, .{});+ transfer.req.referrer_policy = self.referrer_policy;}}@@ -803,6 +804,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !voif (opts.referer) |ref| {try transfer.addHeader("Referer", ref, .{});self._referrer = try self.arena.dupe(u8, ref);+ transfer.req.referrer_policy = opts.referrer_policy;}}@@ -967,6 +969,7 @@ fn scheduleNavigationWithArena(originator: *Frame, arena: *lp.Arena, request_urlif (std.mem.startsWith(u8, originator.url, "http")) {if (nav_opts.referer == null) {nav_opts.referer = try referrer.compute(arena.allocator(), originator.referrer_policy, originator.url, resolved_url);+ nav_opts.referrer_policy = originator.referrer_policy;}if (nav_opts.initiator_url == null) {nav_opts.initiator_url = try arena.dupeZ(u8, originator.url);@@ -1255,6 +1258,13 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.no.body = null;no.header = null;}++ // The Referer may have been recomputed at each hop; document.referrer+ // reports what the final request actually sent.+ self._referrer = if (transfer.findRequestHeader("referer")) |ref|+ try self.arena.dupe(u8, ref)+ else+ null;}// Init new location.@@ -1861,6 +1871,7 @@ pub fn iframeAddedCallback(self: *Frame, iframe: *IFrame) !void {new_frame.navigate(url, .{.reason = .initialFrameNavigation,.referer = try referrer.compute(self.call_arena, self.referrer_policy, self.url, url),+ .referrer_policy = self.referrer_policy,.initiator_url = parent_url,.initiator_origin = self.origin,}) catch |err| {@@ -3117,6 +3128,10 @@ pub const NavigateOpts = struct {// anchor click / form submit / location.href navigations carry a Referer.// null on CDP Page.navigate (address-bar) and Page.reload — matches Chrome.referer: ?[]const u8 = null,+ // The originating frame's policy, paired with `referer` so redirect hops+ // can recompute the header. null (e.g. a CDP-supplied referrer) leaves+ // the Referer untouched across redirects.+ referrer_policy: ?referrer.Policy = null,// The URL of the document that initiated this navigation, used as the// "site for cookies" when computing SameSite. Distinct from `referer`// because a Referrer-Policy can suppress the Referer header withoutdiff --git a/src/browser/frame/user_input.zig b/src/browser/frame/user_input.zigindex 153d8c0db..0dbcd9297 100644--- a/src/browser/frame/user_input.zig+++ b/src/browser/frame/user_input.zig@@ -204,7 +204,14 @@ fn deltaToScroll(d: f64) i32 {// implements.fn hasClickActivationBehavior(node: *Node) bool {const element = node.is(Element) orelse return false;- const html_element = element.is(Element.Html) orelse return false;++ const html_element = element.is(Element.Html) orelse {+ if (element.is(Element.Svg.Graphics.A) != null) {+ return svgAnchorHref(element) != null;+ }+ return false;+ };+return switch (html_element._type) {.anchor => element.getAttributeSafe(comptime .wrap("href")) != null,.input, .button, .select, .textarea, .label => true,@@ -213,6 +220,11 @@ fn hasClickActivationBehavior(node: *Node) bool {};}+// SVG 2 <a> links via `href`; xlink:href is the deprecated SVG 1.1 spelling.+fn svgAnchorHref(element: *Element) ?[]const u8 {+ return element.getAttributeSafe(comptime .wrap("href")) orelse element.getAttributeSafe(comptime .wrap("xlink:href"));+}+// Clicks on editable content are for editing: they don't activate the// element or any enclosing link.// "contenteditable" is 15 bytes — past the comptime SSO limit — so the@@ -325,44 +337,20 @@ const JavascriptUrlTask = struct {pub fn handleClick(frame: *Frame, target: *Node) !void {// TODO: Also support <area> elements when implementconst element = target.is(Element) orelse return;++ if (element.is(Element.Svg.Graphics.A) != null) {+ const href = svgAnchorHref(element) orelse return;+ const target_name = element.getAttributeSafe(comptime .wrap("target")) orelse "";+ return followLink(frame, target, element, href, target_name);+ }+const html_element = element.is(Element.Html) orelse return;switch (html_element._type) {.anchor => {const anchor = html_element.subtype(Element.Html.Anchor);const href = element.getAttributeSafe(comptime .wrap("href")) orelse return;- if (href.len == 0) {- return;- }-- if (std.mem.startsWith(u8, href, "javascript:")) {- // Navigating to a javascript: URL evaluates the script in the- // node's frame as a queued task. (A string completion value- // would replace the document; we ignore results.)- return runJavascriptUrl(target.ownerFrame(frame), href["javascript:".len..]);- }-- if (try element.hasAttribute(comptime .wrap("download"), frame)) {- log.warn(.browser, "a.download", .{ .type = frame._type, .url = frame.url });- return;- }-- const target_frame = blk: {- const target_name = anchor.getTarget();- if (target_name.len == 0) {- break :blk target.ownerFrame(frame);- }- break :blk frame.resolveTargetFrame(target_name) orelse {- log.warn(.not_implemented, "target", .{ .type = frame._type, .url = frame.url, .target = target_name });- return;- };- };-- try element.focus(frame);- try frame.scheduleNavigation(href, .{- .reason = .script,- .kind = .{ .push = null },- }, .{ .anchor = target_frame });+ return followLink(frame, target, element, href, anchor.getTarget());},.input => {const input = html_element.subtype(Element.Html.Input);@@ -414,6 +402,41 @@ pub fn handleClick(frame: *Frame, target: *Node) !void {}}+// Follow a link on activation. Shared by HTML <a> and SVG <a>.+fn followLink(frame: *Frame, target: *Node, element: *Element, href: []const u8, target_name: []const u8) !void {+ if (href.len == 0) {+ return;+ }++ if (std.mem.startsWith(u8, href, "javascript:")) {+ // Navigating to a javascript: URL evaluates the script in the+ // node's frame as a queued task. (A string completion value+ // would replace the document; we ignore results.)+ return runJavascriptUrl(target.ownerFrame(frame), href["javascript:".len..]);+ }++ if (try element.hasAttribute(comptime .wrap("download"), frame)) {+ log.warn(.browser, "a.download", .{ .type = frame._type, .url = frame.url });+ return;+ }++ const target_frame = blk: {+ if (target_name.len == 0) {+ break :blk target.ownerFrame(frame);+ }+ break :blk frame.resolveTargetFrame(target_name) orelse {+ log.warn(.not_implemented, "target", .{ .type = frame._type, .url = frame.url, .target = target_name });+ return;+ };+ };++ try element.focus(frame);+ try frame.scheduleNavigation(href, .{+ .reason = .script,+ .kind = .{ .push = null },+ }, .{ .anchor = target_frame });+}+pub fn triggerKeyboard(frame: *Frame, keyboard_event: *KeyboardEvent) !void {const event = keyboard_event.asEvent();// Dispatch to the effective active element. When nothing is explicitlydiff --git a/src/browser/tests/frames/target.html b/src/browser/tests/frames/target.htmlindex 924e0e590..a44a47cb3 100644--- a/src/browser/tests/frames/target.html+++ b/src/browser/tests/frames/target.html@@ -126,3 +126,44 @@});}</script>++<!-- SVG <a> has the same link activation behavior as HTML <a>. SVGElement has+ no click(), so activate via a dispatched MouseEvent, bubbling up from the+ <text> child like a real click on the rendered link. -->+<iframe name=frame7 id=f7></iframe>+<svg xmlns="http://www.w3.org/2000/svg">+ <a href="support/page.html" target=frame7><text id=svgtext y=10>svg link</text></a>+</svg>++<script id=svg_anchor type=module>+ {+ const state = await testing.async();+ $('#svgtext').dispatchEvent(new MouseEvent('click', {bubbles: true, cancelable: true}));+ $('#f7').onload = () => {+ state.resolve();+ };++ await state.done(() => {+ testing.expectEqual('a-page\n', $('#f7').contentDocument.body.textContent);+ });+ }+</script>++<iframe name=frame8 id=f8></iframe>+<svg xmlns="http://www.w3.org/2000/svg">+ <a id=svglink2 xlink:href="support/page.html" target=frame8><text y=10>svg 1.1 link</text></a>+</svg>++<script id=svg_anchor_xlink type=module>+ {+ const state = await testing.async();+ $('#svglink2').dispatchEvent(new MouseEvent('click', {bubbles: true, cancelable: true}));+ $('#f8').onload = () => {+ state.resolve();+ };++ await state.done(() => {+ testing.expectEqual('a-page\n', $('#f8').contentDocument.body.textContent);+ });+ }+</script>diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.htmlindex fdd13dbb2..ef676af35 100644--- a/src/browser/tests/net/fetch.html+++ b/src/browser/tests/net/fetch.html@@ -451,3 +451,20 @@});}</script>++<!-- Referer across redirects: a same-origin hop keeps the full referrer; a+ cross-origin hop (localhost alias) recomputes it at the redirect, which+ the default policy (strict-origin-when-cross-origin) strips to origin. -->+<script id=fetch_redirect_referer type=module>+ {+ const state = await testing.async();+ const same = await (await fetch('/redirect_same_echo_referer')).text();+ const cross = await (await fetch('/redirect_cross_echo_referer')).text();+ state.resolve();++ await state.done(() => {+ testing.expectEqual(`<html><body>referer=${location.href}</body></html>`, same);+ testing.expectEqual('<html><body>referer=http://127.0.0.1:9582/</body></html>', cross);+ });+ }+</script>diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex bdf981bc8..90a2c4eab 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -26,6 +26,7 @@ const Notification = @import("../Notification.zig");const CDP = @import("../cdp/CDP.zig");const Watchdog = @import("../Watchdog.zig");const URL = @import("../browser/URL.zig");+const referrer = @import("../browser/referrer.zig");const WebSocket = @import("../browser/webapi/net/WebSocket.zig");const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;@@ -1464,8 +1465,8 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (isRedirectStatus(status)) {if (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {.follow => {- try transfer.handleRedirect(location.value);transfer.restoreInterceptHeaders();+ try transfer.handleRedirect(location.value);if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {log.warn(.http, "blocked url", .{ .url = transfer.req.url });@@ -1657,6 +1658,7 @@ pub const Request = struct {cookie_origin: [:0]const u8,resource_type: ResourceType,redirect: RedirectMode = .follow,+ referrer_policy: ?referrer.Policy = null,credentials: ?[:0]const u8 = null,notification: *Notification,timeout_ms: u32 = 0,@@ -2604,6 +2606,21 @@ pub const Transfer = struct {}}+ // A Referrer-Policy header on a redirect response applies to the+ // remaining hops.+ if (req.referrer_policy != null) {+ // referrer-policy is re-applied on every hop.+ var i: usize = 0;+ while (conn.getResponseHeader("referrer-policy", i)) |hdr| : (i += 1) {+ if (referrer.parseHeader(hdr.value)) |policy| {+ req.referrer_policy = policy;+ }+ if (i >= hdr.amount) {+ break;+ }+ }+ }+// base_url and location are owned by curl; applyRedirectTarget resolves a// fresh arena-owned copy that gets stored in transfer.req.url.const base_url = try conn.getEffectiveUrl();@@ -2651,6 +2668,21 @@ pub const Transfer = struct {req.method = .GET;req.body = null;}++ if (req.referrer_policy) |policy| {+ // Referer header was applied based on the original target. It+ // needs to be updated based on the redirect target. A redirect can+ // only strip it (full -> origin -> none), so we can use whatever+ // value we have now as the base+ if (transfer.findRequestHeader("referer")) |current| {+ const alloc = arena.allocator();+ if (try referrer.compute(alloc, policy, try alloc.dupeZ(u8, current), req.url)) |value| {+ try transfer.setHeader("Referer", value, .{});+ } else {+ transfer.removeHeader("Referer");+ }+ }+ }}fn detectAuthChallenge(transfer: *Transfer, conn: *const http.Connection) void {@@ -2702,6 +2734,26 @@ pub const Transfer = struct {});}+ pub fn findRequestHeader(self: *const Transfer, name: []const u8) ?[]const u8 {+ for (self.req_headers.items) |hdr| {+ if (std.ascii.eqlIgnoreCase(hdr.name, name)) {+ return hdr.value;+ }+ }+ return null;+ }++ fn removeHeader(self: *Transfer, name: []const u8) void {+ var i: usize = 0;+ while (i < self.req_headers.items.len) {+ if (std.ascii.eqlIgnoreCase(self.req_headers.items[i].name, name)) {+ _ = self.req_headers.orderedRemove(i);+ continue;+ }+ i += 1;+ }+ }+// Adds, replacing every existing header with the same case-insensitive namepub fn setHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {var found = false;diff --git a/src/testing.zig b/src/testing.zigindex ce3b15460..b62a5a327 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -1003,6 +1003,28 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {});}+ if (std.mem.eql(u8, path, "/redirect_same_echo_referer")) {+ // Same-origin 302 to /echo_referer: the full Referer must survive the hop.+ return req.respond("", .{+ .status = .found,+ .extra_headers = &.{+ .{ .name = "Location", .value = "/echo_referer" },+ },+ });+ }++ if (std.mem.eql(u8, path, "/redirect_cross_echo_referer")) {+ // 302 to /echo_referer on the localhost alias — a cross-origin hop, so+ // the Referer must be recomputed at the redirect (stripped to origin+ // under the default policy) rather than re-sent in full.+ return req.respond("", .{+ .status = .found,+ .extra_headers = &.{+ .{ .name = "Location", .value = "http://localhost:9582/echo_referer" },+ },+ });+ }+if (std.mem.eql(u8, path, "/redirect_to_echo")) {// 302 to /echo_method. Used by the Page.reload-after-redirect test to// confirm a POST→302→GET chain doesn't replay POST on reload.