Skip to content

Lightpanda: the headless browser designed for AI and automation

http: start implementation correct referrer policy

Karl Seguina71bf101d693
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index ea0651489..d9f2042b9 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -34,6 +34,7 @@ const h5e = @import("parser/html5ever.zig");
const CustomElementReactions = @import("CustomElementReactions.zig");
const URL = @import("URL.zig");
+const referrer = @import("referrer.zig");
const Blob = @import("webapi/Blob.zig");
const FileList = @import("webapi/FileList.zig");
const Node = @import("webapi/Node.zig");
@@ -331,6 +332,9 @@ _navigated_options: ?NavigatedOpts = null,
_http_status: ?u16 = null,
_http_headers: std.ArrayList(HttpHeader) = .empty,
+_referrer: ?[]const u8 = null,
+referrer_policy: referrer.Policy = .default,
+
pub const HttpHeader = struct {
name: []const u8,
value: []const u8,
@@ -593,8 +597,9 @@ pub fn httpMetadata(self: *const Frame) HttpMetadata {
// Add common headers for a request:
// * referer
pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {
- if (std.mem.startsWith(u8, self.url, "http")) {
- try transfer.addHeader("Referer", self.url, .{});
+ const arena = transfer.arena.allocator();
+ if (try referrer.compute(arena, self.referrer_policy, self.url, transfer.req.url)) |ref| {
+ try transfer.addHeader("Referer", ref, .{});
}
}
@@ -742,6 +747,9 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
self._http_status = null;
self._http_headers = .empty;
+ self._referrer = null;
+ self.referrer_policy = .default;
+
self.url = blk: {
if (URL.isCompleteHTTPUrl(request_url)) {
break :blk try self.arena.dupeZ(u8, request_url);
@@ -793,6 +801,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
}
if (opts.referer) |ref| {
try transfer.addHeader("Referer", ref, .{});
+ self._referrer = try self.arena.dupe(u8, ref);
}
}
@@ -951,20 +960,15 @@ fn scheduleNavigationWithArena(originator: *Frame, arena: *lp.Arena, request_url
// Capture the originating frame's URL as the Referer for this
// navigation. The originator's frame may be torn down before navigate()
- // runs (processRootQueuedNavigation rebuilds the Page in-place), so dup
- // into the QueuedNavigation arena which outlives that tear-down.
+ // runs (processRootQueuedNavigation rebuilds the Page in-place), so
+ // allocate from the QueuedNavigation arena which outlives that tear-down.
var nav_opts = opts;
if (std.mem.startsWith(u8, originator.url, "http")) {
- // The same dup feeds two purposes: Referer header (subject to
- // Referrer-Policy in the future) and SameSite computation (which
- // must use the real initiator regardless of policy). We share the
- // same allocation for both.
- const dup = try arena.dupeZ(u8, originator.url);
if (nav_opts.referer == null) {
- nav_opts.referer = dup;
+ nav_opts.referer = try referrer.compute(arena.allocator(), originator.referrer_policy, originator.url, resolved_url);
}
if (nav_opts.initiator_url == null) {
- nav_opts.initiator_url = dup;
+ nav_opts.initiator_url = try arena.dupeZ(u8, originator.url);
}
}
if (nav_opts.initiator_origin == null) {
@@ -1274,6 +1278,11 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.
.name = try self.arena.dupe(u8, hdr.name),
.value = try self.arena.dupe(u8, hdr.value),
});
+ if (std.ascii.eqlIgnoreCase(hdr.name, "referrer-policy")) {
+ if (referrer.parseHeader(hdr.value)) |rp| {
+ self.referrer_policy = rp;
+ }
+ }
}
if (self._navigated_options) |no| {
@@ -1843,13 +1852,14 @@ pub fn iframeAddedCallback(self: *Frame, iframe: *IFrame) !void {
const was_sorted = self.child_frames_sorted;
self.child_frames_sorted = false;
- // Iframe's initial src request carries the parent's URL as Referer and
- // as the SameSite initiator. Parent frame outlives this navigate()
- // call, so the slice is safe.
+ // Iframe's initial src request carries the parent's URL as Referer
+ // (subject to the parent's Referrer-Policy) and as the SameSite
+ // initiator. Parent frame outlives this navigate() call, so the slice
+ // is safe; navigate dupes what it keeps.
const parent_url: ?[:0]const u8 = if (std.mem.startsWith(u8, self.url, "http")) self.url else null;
new_frame.navigate(url, .{
.reason = .initialFrameNavigation,
- .referer = parent_url,
+ .referer = try referrer.compute(self.call_arena, self.referrer_policy, self.url, url),
.initiator_url = parent_url,
.initiator_origin = self.origin,
}) catch |err| {
diff --git a/src/browser/referrer.zig b/src/browser/referrer.zig
new file mode 100644
index 000000000..0e3250441
--- /dev/null
+++ b/src/browser/referrer.zig
@@ -0,0 +1,222 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// Referrer Policy: https://www.w3.org/TR/referrer-policy/
+const std = @import("std");
+const URL = @import("URL.zig");
+
+const Allocator = std.mem.Allocator;
+
+pub const Policy = enum {
+ no_referrer,
+ no_referrer_when_downgrade,
+ origin,
+ origin_when_cross_origin,
+ same_origin,
+ strict_origin,
+ strict_origin_when_cross_origin,
+ unsafe_url,
+
+ pub const default: Policy = .strict_origin_when_cross_origin;
+};
+
+pub fn parse(value: []const u8) ?Policy {
+ const map = std.StaticStringMapWithEql(Policy, staticStringMapEqlAsciiIgnoreCase).initComptime(.{
+ .{ "no-referrer", Policy.no_referrer },
+ .{ "no-referrer-when-downgrade", Policy.no_referrer_when_downgrade },
+ .{ "origin", Policy.origin },
+ .{ "origin-when-cross-origin", Policy.origin_when_cross_origin },
+ .{ "same-origin", Policy.same_origin },
+ .{ "strict-origin", Policy.strict_origin },
+ .{ "strict-origin-when-cross-origin", Policy.strict_origin_when_cross_origin },
+ .{ "unsafe-url", Policy.unsafe_url },
+ });
+ return map.get(value);
+}
+
+pub fn parseHeader(value: []const u8) ?Policy {
+ var policy: ?Policy = null;
+ var it = std.mem.splitScalar(u8, value, ',');
+ while (it.next()) |token| {
+ if (parse(std.mem.trim(u8, token, " \t"))) |p| {
+ policy = p;
+ }
+ }
+ return policy;
+}
+
+pub fn parseMeta(value: []const u8) ?Policy {
+ if (parse(value)) |p| {
+ return p;
+ }
+ // legacy values
+ const map = std.StaticStringMapWithEql(Policy, staticStringMapEqlAsciiIgnoreCase).initComptime(.{
+ .{ "never", Policy.no_referrer },
+ .{ "always", Policy.unsafe_url },
+ .{ "origin-when-crossorigin", Policy.origin_when_cross_origin },
+ .{ "default", Policy.default },
+ });
+ return map.get(value);
+}
+
+// returns the value to send as the Referrer header based on the target_url
+// and the given policy
+pub fn compute(arena: Allocator, policy: Policy, referrer_url: [:0]const u8, target_url: [:0]const u8) !?[]const u8 {
+ if (policy == .no_referrer) {
+ return null;
+ }
+
+ const referrer_origin = (try URL.getOrigin(arena, referrer_url)) orelse {
+ // blob:, data: ... don't set get a referrer
+ return null;
+ };
+
+ const same_origin = blk: {
+ const target_origin = (try URL.getOrigin(arena, target_url)) orelse break :blk false;
+ break :blk std.ascii.eqlIgnoreCase(referrer_origin, target_origin);
+ };
+ const downgrade = URL.isSecure(referrer_url) and !URL.isSecure(target_url);
+
+ const full = switch (policy) {
+ .no_referrer => unreachable,
+ .unsafe_url => true,
+ .origin => false,
+ .no_referrer_when_downgrade => if (downgrade) return null else true,
+ .same_origin => if (same_origin) true else return null,
+ .origin_when_cross_origin => same_origin,
+ .strict_origin => if (downgrade) return null else false,
+ .strict_origin_when_cross_origin => if (same_origin) true else if (downgrade) return null else false,
+ };
+
+ if (full) {
+ // Serializing through origin + path + query strips credentials and
+ // the fragment, and normalizes away default ports.
+ const value = try std.fmt.allocPrint(arena, "{s}{s}{s}", .{
+ referrer_origin,
+ URL.getPathname(referrer_url),
+ URL.getSearch(referrer_url),
+ });
+
+ if (value.len <= 4096) {
+ // spec limit, if it's more than this, we falllback to the origin
+ return value;
+ }
+ }
+ return try std.fmt.allocPrint(arena, "{s}/", .{referrer_origin});
+}
+
+fn staticStringMapEqlAsciiIgnoreCase(a: []const u8, b: []const u8) bool {
+ for (a, b) |a_c, b_c| {
+ if (std.ascii.toLower(a_c) != std.ascii.toLower(b_c)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+const testing = @import("../testing.zig");
+test "referrer: parse" {
+ try testing.expectEqual(Policy.no_referrer, parse("no-referrer"));
+ try testing.expectEqual(Policy.unsafe_url, parse("Unsafe-URL"));
+ try testing.expectEqual(null, parse(""));
+ try testing.expectEqual(null, parse("never"));
+ try testing.expectEqual(null, parse("no-referrer "));
+
+ try testing.expectEqual(null, parseHeader(""));
+ try testing.expectEqual(null, parseHeader("nope"));
+ try testing.expectEqual(Policy.origin, parseHeader("origin"));
+ try testing.expectEqual(Policy.same_origin, parseHeader("origin, same-origin"));
+ try testing.expectEqual(Policy.origin, parseHeader("origin, garbage"));
+ try testing.expectEqual(Policy.same_origin, parseHeader(" origin ,\tsame-origin "));
+
+ try testing.expectEqual(Policy.no_referrer, parseMeta("never"));
+ try testing.expectEqual(Policy.unsafe_url, parseMeta("always"));
+ try testing.expectEqual(Policy.origin_when_cross_origin, parseMeta("origin-when-crossorigin"));
+ try testing.expectEqual(Policy.strict_origin_when_cross_origin, parseMeta("default"));
+ try testing.expectEqual(Policy.origin, parseMeta("origin"));
+ try testing.expectEqual(null, parseMeta("garbage"));
+}
+
+test "referrer: compute" {
+ const Case = struct {
+ policy: Policy,
+ referrer: [:0]const u8,
+ target: [:0]const u8,
+ expected: ?[]const u8,
+ };
+
+ const cases = [_]Case{
+ .{ .policy = .no_referrer, .referrer = "http://a.com/p", .target = "http://a.com/x", .expected = null },
+
+ .{ .policy = .unsafe_url, .referrer = "http://a.com/p?q=1#frag", .target = "https://b.com/", .expected = "http://a.com/p?q=1" },
+ .{ .policy = .unsafe_url, .referrer = "https://a.com/p", .target = "http://b.com/", .expected = "https://a.com/p" },
+ .{ .policy = .unsafe_url, .referrer = "https://user:[email protected]/p", .target = "http://b.com/", .expected = "https://a.com/p" },
+ .{ .policy = .unsafe_url, .referrer = "https://a.com:443/p", .target = "http://b.com/", .expected = "https://a.com/p" },
+ .{ .policy = .unsafe_url, .referrer = "http://a.com", .target = "http://b.com/", .expected = "http://a.com/" },
+
+ .{ .policy = .origin, .referrer = "http://a.com:8000/p?q=1", .target = "http://a.com:8000/x", .expected = "http://a.com:8000/" },
+
+ .{ .policy = .same_origin, .referrer = "http://a.com/p", .target = "http://a.com/x", .expected = "http://a.com/p" },
+ .{ .policy = .same_origin, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = null },
+ .{ .policy = .same_origin, .referrer = "http://a.com/p", .target = "https://a.com/x", .expected = null },
+
+ .{ .policy = .origin_when_cross_origin, .referrer = "http://a.com/p", .target = "http://a.com/x", .expected = "http://a.com/p" },
+ .{ .policy = .origin_when_cross_origin, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = "http://a.com/" },
+
+ .{ .policy = .strict_origin, .referrer = "https://a.com/p", .target = "http://a.com/x", .expected = null },
+ .{ .policy = .strict_origin, .referrer = "https://a.com/p", .target = "https://b.com/x", .expected = "https://a.com/" },
+ .{ .policy = .strict_origin, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = "http://a.com/" },
+
+ .{ .policy = .no_referrer_when_downgrade, .referrer = "https://a.com/p", .target = "http://b.com/x", .expected = null },
+ .{ .policy = .no_referrer_when_downgrade, .referrer = "https://a.com/p", .target = "https://b.com/x", .expected = "https://a.com/p" },
+ .{ .policy = .no_referrer_when_downgrade, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = "http://a.com/p" },
+
+ .{ .policy = .strict_origin_when_cross_origin, .referrer = "http://a.com/p?q=1", .target = "http://a.com/x", .expected = "http://a.com/p?q=1" },
+ .{ .policy = .strict_origin_when_cross_origin, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = "http://a.com/" },
+ .{ .policy = .strict_origin_when_cross_origin, .referrer = "https://a.com/p", .target = "http://b.com/x", .expected = null },
+ .{ .policy = .strict_origin_when_cross_origin, .referrer = "https://a.com/p", .target = "http://a.com/x", .expected = null },
+
+ // no referrer from non-http(s) documents
+ .{ .policy = .unsafe_url, .referrer = "about:blank", .target = "http://b.com/x", .expected = null },
+ .{ .policy = .unsafe_url, .referrer = "data:text/html,x", .target = "http://b.com/x", .expected = null },
+ };
+
+ for (cases) |case| {
+ const actual = try compute(testing.arena_allocator, case.policy, case.referrer, case.target);
+ if (case.expected) |expected| {
+ try testing.expectEqual(expected, actual orelse return error.UnexpectedNull);
+ } else {
+ try testing.expectEqual(null, actual);
+ }
+ }
+}
+
+test "referrer: compute caps at 4096 bytes" {
+ const path = "/" ++ ("a" ** 4096);
+ const url = "http://a.com" ++ path;
+ // over the cap: falls back to the origin form
+ try testing.expectEqual("http://a.com/", (try compute(testing.arena_allocator, .unsafe_url, url, "http://b.com/x")).?);
+ try testing.expectEqual("http://a.com/", (try compute(testing.arena_allocator, .no_referrer_when_downgrade, url, "http://a.com/x")).?);
+
+ // exactly at the cap: sent in full
+ const at_cap = "http://a.com/" ++ ("a" ** (4096 - "http://a.com/".len));
+ try testing.expectEqual(at_cap, (try compute(testing.arena_allocator, .unsafe_url, at_cap, "http://b.com/x")).?);
+
+ // origin-only policies are unaffected by the referrer's length
+ try testing.expectEqual("http://a.com/", (try compute(testing.arena_allocator, .origin, url, "http://b.com/x")).?);
+}
diff --git a/src/browser/webapi/Document.zig b/src/browser/webapi/Document.zig
index 78e2a3f83..e5e0b5e6e 100644
--- a/src/browser/webapi/Document.zig
+++ b/src/browser/webapi/Document.zig
@@ -212,6 +212,11 @@ pub fn getLastModified(self: *const Document, frame: *Frame) ![]const u8 {
});
}
+pub fn getReferrer(self: *const Document) []const u8 {
+ const frame = self._frame orelse return "";
+ return frame._referrer orelse "";
+}
+
pub fn getCharset(self: *const Document) []const u8 {
if (self._charset) |charset| {
return charset;
@@ -1588,15 +1593,12 @@ pub const JsApi = struct {
pub const hasFocus = bridge.function(Document.hasFocus, .{});
pub const prerendering = bridge.property(false, .{ .template = false });
- pub const characterSet = bridge.accessor(getCharacterSet, null, .{});
- pub const charset = bridge.accessor(getCharacterSet, null, .{});
- pub const inputEncoding = bridge.accessor(getCharacterSet, null, .{});
+ pub const characterSet = bridge.accessor(Document.getCharset, null, .{});
+ pub const charset = bridge.accessor(Document.getCharset, null, .{});
+ pub const inputEncoding = bridge.accessor(Document.getCharset, null, .{});
pub const compatMode = bridge.accessor(Document.getCompatMode, null, .{});
pub const lastModified = bridge.accessor(Document.getLastModified, null, .{});
- fn getCharacterSet(self: *const Document) []const u8 {
- return self.getCharset();
- }
- pub const referrer = bridge.property("", .{ .template = false });
+ pub const referrer = bridge.accessor(Document.getReferrer, null, .{});
// Generates a getter/setter pair backed by the frame's attribute-listener
// map, like onclick above, for other document event handler properties.
diff --git a/src/browser/webapi/element/html/Meta.zig b/src/browser/webapi/element/html/Meta.zig
index ea5baee1a..7c869c290 100644
--- a/src/browser/webapi/element/html/Meta.zig
+++ b/src/browser/webapi/element/html/Meta.zig
@@ -16,11 +16,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+const std = @import("std");
+
const js = @import("../../../js/js.zig");
const Frame = @import("../../../Frame.zig");
const Node = @import("../../Node.zig");
const Element = @import("../../Element.zig");
const HtmlElement = @import("../Html.zig");
+const referrer = @import("../../../referrer.zig");
const Meta = @This();
@@ -78,6 +81,21 @@ pub fn setScheme(self: *Meta, value: []const u8, frame: *Frame) !void {
try self.asElement().setAttributeSafe(comptime .wrap("scheme"), .wrap(value), frame);
}
+pub const Build = struct {
+ // <meta name=referrer> sets the document's referrer policy.
+ pub fn created(node: *Node, frame: *Frame) !void {
+ const el = node.as(Element);
+ const name = el.getAttributeSafe(comptime .wrap("name")) orelse return;
+ if (std.ascii.eqlIgnoreCase(name, "referrer") == false) {
+ return;
+ }
+ const content = el.getAttributeSafe(comptime .wrap("content")) orelse return;
+ if (referrer.parseMeta(content)) |rp| {
+ frame.referrer_policy = rp;
+ }
+ }
+};
+
pub const JsApi = struct {
pub const bridge = js.Bridge(MetaElement);