Skip to content

Lightpanda: the headless browser designed for AI and automation

http: recalculate Referer header on each redirect

Karl Seguinabc6f1ef76f4
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index c046fc98b..538802d64 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -601,6 +601,7 @@ pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {
const arena = transfer.arena.allocator();
if (try referrer.compute(arena, self.referrer_policy, self.url, transfer.req.url)) |ref| {
try transfer.addHeader("Referer", ref, .{});
+ transfer.req.referrer_policy = self.referrer_policy;
}
}
@@ -803,6 +804,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
if (opts.referer) |ref| {
try transfer.addHeader("Referer", ref, .{});
self._referrer = try self.arena.dupe(u8, ref);
+ transfer.req.referrer_policy = opts.referrer_policy;
}
}
@@ -967,6 +969,7 @@ fn scheduleNavigationWithArena(originator: *Frame, arena: *lp.Arena, request_url
if (std.mem.startsWith(u8, originator.url, "http")) {
if (nav_opts.referer == null) {
nav_opts.referer = try referrer.compute(arena.allocator(), originator.referrer_policy, originator.url, resolved_url);
+ nav_opts.referrer_policy = originator.referrer_policy;
}
if (nav_opts.initiator_url == null) {
nav_opts.initiator_url = try arena.dupeZ(u8, originator.url);
@@ -1255,6 +1258,13 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.
no.body = null;
no.header = null;
}
+
+ // The Referer may have been recomputed at each hop; document.referrer
+ // reports what the final request actually sent.
+ self._referrer = if (transfer.findRequestHeader("referer")) |ref|
+ try self.arena.dupe(u8, ref)
+ else
+ null;
}
// Init new location.
@@ -1861,6 +1871,7 @@ pub fn iframeAddedCallback(self: *Frame, iframe: *IFrame) !void {
new_frame.navigate(url, .{
.reason = .initialFrameNavigation,
.referer = try referrer.compute(self.call_arena, self.referrer_policy, self.url, url),
+ .referrer_policy = self.referrer_policy,
.initiator_url = parent_url,
.initiator_origin = self.origin,
}) catch |err| {
@@ -3117,6 +3128,10 @@ pub const NavigateOpts = struct {
// anchor click / form submit / location.href navigations carry a Referer.
// null on CDP Page.navigate (address-bar) and Page.reload — matches Chrome.
referer: ?[]const u8 = null,
+ // The originating frame's policy, paired with `referer` so redirect hops
+ // can recompute the header. null (e.g. a CDP-supplied referrer) leaves
+ // the Referer untouched across redirects.
+ referrer_policy: ?referrer.Policy = null,
// The URL of the document that initiated this navigation, used as the
// "site for cookies" when computing SameSite. Distinct from `referer`
// because a Referrer-Policy can suppress the Referer header without
diff --git a/src/browser/frame/user_input.zig b/src/browser/frame/user_input.zig
index 153d8c0db..0dbcd9297 100644
--- a/src/browser/frame/user_input.zig
+++ b/src/browser/frame/user_input.zig
@@ -204,7 +204,14 @@ fn deltaToScroll(d: f64) i32 {
// implements.
fn hasClickActivationBehavior(node: *Node) bool {
const element = node.is(Element) orelse return false;
- const html_element = element.is(Element.Html) orelse return false;
+
+ const html_element = element.is(Element.Html) orelse {
+ if (element.is(Element.Svg.Graphics.A) != null) {
+ return svgAnchorHref(element) != null;
+ }
+ return false;
+ };
+
return switch (html_element._type) {
.anchor => element.getAttributeSafe(comptime .wrap("href")) != null,
.input, .button, .select, .textarea, .label => true,
@@ -213,6 +220,11 @@ fn hasClickActivationBehavior(node: *Node) bool {
};
}
+// SVG 2 <a> links via `href`; xlink:href is the deprecated SVG 1.1 spelling.
+fn svgAnchorHref(element: *Element) ?[]const u8 {
+ return element.getAttributeSafe(comptime .wrap("href")) orelse element.getAttributeSafe(comptime .wrap("xlink:href"));
+}
+
// Clicks on editable content are for editing: they don't activate the
// element or any enclosing link.
// "contenteditable" is 15 bytes — past the comptime SSO limit — so the
@@ -325,44 +337,20 @@ const JavascriptUrlTask = struct {
pub fn handleClick(frame: *Frame, target: *Node) !void {
// TODO: Also support <area> elements when implement
const element = target.is(Element) orelse return;
+
+ if (element.is(Element.Svg.Graphics.A) != null) {
+ const href = svgAnchorHref(element) orelse return;
+ const target_name = element.getAttributeSafe(comptime .wrap("target")) orelse "";
+ return followLink(frame, target, element, href, target_name);
+ }
+
const html_element = element.is(Element.Html) orelse return;
switch (html_element._type) {
.anchor => {
const anchor = html_element.subtype(Element.Html.Anchor);
const href = element.getAttributeSafe(comptime .wrap("href")) orelse return;
- if (href.len == 0) {
- return;
- }
-
- if (std.mem.startsWith(u8, href, "javascript:")) {
- // Navigating to a javascript: URL evaluates the script in the
- // node's frame as a queued task. (A string completion value
- // would replace the document; we ignore results.)
- return runJavascriptUrl(target.ownerFrame(frame), href["javascript:".len..]);
- }
-
- if (try element.hasAttribute(comptime .wrap("download"), frame)) {
- log.warn(.browser, "a.download", .{ .type = frame._type, .url = frame.url });
- return;
- }
-
- const target_frame = blk: {
- const target_name = anchor.getTarget();
- if (target_name.len == 0) {
- break :blk target.ownerFrame(frame);
- }
- break :blk frame.resolveTargetFrame(target_name) orelse {
- log.warn(.not_implemented, "target", .{ .type = frame._type, .url = frame.url, .target = target_name });
- return;
- };
- };
-
- try element.focus(frame);
- try frame.scheduleNavigation(href, .{
- .reason = .script,
- .kind = .{ .push = null },
- }, .{ .anchor = target_frame });
+ return followLink(frame, target, element, href, anchor.getTarget());
},
.input => {
const input = html_element.subtype(Element.Html.Input);
@@ -414,6 +402,41 @@ pub fn handleClick(frame: *Frame, target: *Node) !void {
}
}
+// Follow a link on activation. Shared by HTML <a> and SVG <a>.
+fn followLink(frame: *Frame, target: *Node, element: *Element, href: []const u8, target_name: []const u8) !void {
+ if (href.len == 0) {
+ return;
+ }
+
+ if (std.mem.startsWith(u8, href, "javascript:")) {
+ // Navigating to a javascript: URL evaluates the script in the
+ // node's frame as a queued task. (A string completion value
+ // would replace the document; we ignore results.)
+ return runJavascriptUrl(target.ownerFrame(frame), href["javascript:".len..]);
+ }
+
+ if (try element.hasAttribute(comptime .wrap("download"), frame)) {
+ log.warn(.browser, "a.download", .{ .type = frame._type, .url = frame.url });
+ return;
+ }
+
+ const target_frame = blk: {
+ if (target_name.len == 0) {
+ break :blk target.ownerFrame(frame);
+ }
+ break :blk frame.resolveTargetFrame(target_name) orelse {
+ log.warn(.not_implemented, "target", .{ .type = frame._type, .url = frame.url, .target = target_name });
+ return;
+ };
+ };
+
+ try element.focus(frame);
+ try frame.scheduleNavigation(href, .{
+ .reason = .script,
+ .kind = .{ .push = null },
+ }, .{ .anchor = target_frame });
+}
+
pub fn triggerKeyboard(frame: *Frame, keyboard_event: *KeyboardEvent) !void {
const event = keyboard_event.asEvent();
// Dispatch to the effective active element. When nothing is explicitly
diff --git a/src/browser/tests/frames/target.html b/src/browser/tests/frames/target.html
index 924e0e590..a44a47cb3 100644
--- a/src/browser/tests/frames/target.html
+++ b/src/browser/tests/frames/target.html
@@ -126,3 +126,44 @@
});
}
</script>
+
+<!-- SVG <a> has the same link activation behavior as HTML <a>. SVGElement has
+ no click(), so activate via a dispatched MouseEvent, bubbling up from the
+ <text> child like a real click on the rendered link. -->
+<iframe name=frame7 id=f7></iframe>
+<svg xmlns="http://www.w3.org/2000/svg">
+ <a href="support/page.html" target=frame7><text id=svgtext y=10>svg link</text></a>
+</svg>
+
+<script id=svg_anchor type=module>
+ {
+ const state = await testing.async();
+ $('#svgtext').dispatchEvent(new MouseEvent('click', {bubbles: true, cancelable: true}));
+ $('#f7').onload = () => {
+ state.resolve();
+ };
+
+ await state.done(() => {
+ testing.expectEqual('a-page\n', $('#f7').contentDocument.body.textContent);
+ });
+ }
+</script>
+
+<iframe name=frame8 id=f8></iframe>
+<svg xmlns="http://www.w3.org/2000/svg">
+ <a id=svglink2 xlink:href="support/page.html" target=frame8><text y=10>svg 1.1 link</text></a>
+</svg>
+
+<script id=svg_anchor_xlink type=module>
+ {
+ const state = await testing.async();
+ $('#svglink2').dispatchEvent(new MouseEvent('click', {bubbles: true, cancelable: true}));
+ $('#f8').onload = () => {
+ state.resolve();
+ };
+
+ await state.done(() => {
+ testing.expectEqual('a-page\n', $('#f8').contentDocument.body.textContent);
+ });
+ }
+</script>
diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.html
index fdd13dbb2..ef676af35 100644
--- a/src/browser/tests/net/fetch.html
+++ b/src/browser/tests/net/fetch.html
@@ -451,3 +451,20 @@
});
}
</script>
+
+<!-- Referer across redirects: a same-origin hop keeps the full referrer; a
+ cross-origin hop (localhost alias) recomputes it at the redirect, which
+ the default policy (strict-origin-when-cross-origin) strips to origin. -->
+<script id=fetch_redirect_referer type=module>
+ {
+ const state = await testing.async();
+ const same = await (await fetch('/redirect_same_echo_referer')).text();
+ const cross = await (await fetch('/redirect_cross_echo_referer')).text();
+ state.resolve();
+
+ await state.done(() => {
+ testing.expectEqual(`<html><body>referer=${location.href}</body></html>`, same);
+ testing.expectEqual('<html><body>referer=http://127.0.0.1:9582/</body></html>', cross);
+ });
+ }
+</script>
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index bdf981bc8..90a2c4eab 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -26,6 +26,7 @@ const Notification = @import("../Notification.zig");
const CDP = @import("../cdp/CDP.zig");
const Watchdog = @import("../Watchdog.zig");
const URL = @import("../browser/URL.zig");
+const referrer = @import("../browser/referrer.zig");
const WebSocket = @import("../browser/webapi/net/WebSocket.zig");
const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
@@ -1464,8 +1465,8 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (isRedirectStatus(status)) {
if (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {
.follow => {
- try transfer.handleRedirect(location.value);
transfer.restoreInterceptHeaders();
+ try transfer.handleRedirect(location.value);
if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
log.warn(.http, "blocked url", .{ .url = transfer.req.url });
@@ -1657,6 +1658,7 @@ pub const Request = struct {
cookie_origin: [:0]const u8,
resource_type: ResourceType,
redirect: RedirectMode = .follow,
+ referrer_policy: ?referrer.Policy = null,
credentials: ?[:0]const u8 = null,
notification: *Notification,
timeout_ms: u32 = 0,
@@ -2604,6 +2606,21 @@ pub const Transfer = struct {
}
}
+ // A Referrer-Policy header on a redirect response applies to the
+ // remaining hops.
+ if (req.referrer_policy != null) {
+ // referrer-policy is re-applied on every hop.
+ var i: usize = 0;
+ while (conn.getResponseHeader("referrer-policy", i)) |hdr| : (i += 1) {
+ if (referrer.parseHeader(hdr.value)) |policy| {
+ req.referrer_policy = policy;
+ }
+ if (i >= hdr.amount) {
+ break;
+ }
+ }
+ }
+
// base_url and location are owned by curl; applyRedirectTarget resolves a
// fresh arena-owned copy that gets stored in transfer.req.url.
const base_url = try conn.getEffectiveUrl();
@@ -2651,6 +2668,21 @@ pub const Transfer = struct {
req.method = .GET;
req.body = null;
}
+
+ if (req.referrer_policy) |policy| {
+ // Referer header was applied based on the original target. It
+ // needs to be updated based on the redirect target. A redirect can
+ // only strip it (full -> origin -> none), so we can use whatever
+ // value we have now as the base
+ if (transfer.findRequestHeader("referer")) |current| {
+ const alloc = arena.allocator();
+ if (try referrer.compute(alloc, policy, try alloc.dupeZ(u8, current), req.url)) |value| {
+ try transfer.setHeader("Referer", value, .{});
+ } else {
+ transfer.removeHeader("Referer");
+ }
+ }
+ }
}
fn detectAuthChallenge(transfer: *Transfer, conn: *const http.Connection) void {
@@ -2702,6 +2734,26 @@ pub const Transfer = struct {
});
}
+ pub fn findRequestHeader(self: *const Transfer, name: []const u8) ?[]const u8 {
+ for (self.req_headers.items) |hdr| {
+ if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
+ return hdr.value;
+ }
+ }
+ return null;
+ }
+
+ fn removeHeader(self: *Transfer, name: []const u8) void {
+ var i: usize = 0;
+ while (i < self.req_headers.items.len) {
+ if (std.ascii.eqlIgnoreCase(self.req_headers.items[i].name, name)) {
+ _ = self.req_headers.orderedRemove(i);
+ continue;
+ }
+ i += 1;
+ }
+ }
+
// Adds, replacing every existing header with the same case-insensitive name
pub fn setHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
var found = false;
diff --git a/src/testing.zig b/src/testing.zig
index ce3b15460..b62a5a327 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -1003,6 +1003,28 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
+ if (std.mem.eql(u8, path, "/redirect_same_echo_referer")) {
+ // Same-origin 302 to /echo_referer: the full Referer must survive the hop.
+ return req.respond("", .{
+ .status = .found,
+ .extra_headers = &.{
+ .{ .name = "Location", .value = "/echo_referer" },
+ },
+ });
+ }
+
+ if (std.mem.eql(u8, path, "/redirect_cross_echo_referer")) {
+ // 302 to /echo_referer on the localhost alias — a cross-origin hop, so
+ // the Referer must be recomputed at the redirect (stripped to origin
+ // under the default policy) rather than re-sent in full.
+ return req.respond("", .{
+ .status = .found,
+ .extra_headers = &.{
+ .{ .name = "Location", .value = "http://localhost:9582/echo_referer" },
+ },
+ });
+ }
+
if (std.mem.eql(u8, path, "/redirect_to_echo")) {
// 302 to /echo_method. Used by the Page.reload-after-redirect test to
// confirm a POST→302→GET chain doesn't replay POST on reload.