Lightpanda: the headless browser designed for AI and automation
http: start implementation correct referrer policy
Karl Seguina71bf101d693
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex ea0651489..d9f2042b9 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -34,6 +34,7 @@ const h5e = @import("parser/html5ever.zig");const CustomElementReactions = @import("CustomElementReactions.zig");const URL = @import("URL.zig");+const referrer = @import("referrer.zig");const Blob = @import("webapi/Blob.zig");const FileList = @import("webapi/FileList.zig");const Node = @import("webapi/Node.zig");@@ -331,6 +332,9 @@ _navigated_options: ?NavigatedOpts = null,_http_status: ?u16 = null,_http_headers: std.ArrayList(HttpHeader) = .empty,+_referrer: ?[]const u8 = null,+referrer_policy: referrer.Policy = .default,+pub const HttpHeader = struct {name: []const u8,value: []const u8,@@ -593,8 +597,9 @@ pub fn httpMetadata(self: *const Frame) HttpMetadata {// Add common headers for a request:// * refererpub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {- if (std.mem.startsWith(u8, self.url, "http")) {- try transfer.addHeader("Referer", self.url, .{});+ const arena = transfer.arena.allocator();+ if (try referrer.compute(arena, self.referrer_policy, self.url, transfer.req.url)) |ref| {+ try transfer.addHeader("Referer", ref, .{});}}@@ -742,6 +747,9 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !voself._http_status = null;self._http_headers = .empty;+ self._referrer = null;+ self.referrer_policy = .default;+self.url = blk: {if (URL.isCompleteHTTPUrl(request_url)) {break :blk try self.arena.dupeZ(u8, request_url);@@ -793,6 +801,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo}if (opts.referer) |ref| {try transfer.addHeader("Referer", ref, .{});+ self._referrer = try self.arena.dupe(u8, ref);}}@@ -951,20 +960,15 @@ fn scheduleNavigationWithArena(originator: *Frame, arena: *lp.Arena, request_url// Capture the originating frame's URL as the Referer for this// navigation. The originator's frame may be torn down before navigate()- // runs (processRootQueuedNavigation rebuilds the Page in-place), so dup- // into the QueuedNavigation arena which outlives that tear-down.+ // runs (processRootQueuedNavigation rebuilds the Page in-place), so+ // allocate from the QueuedNavigation arena which outlives that tear-down.var nav_opts = opts;if (std.mem.startsWith(u8, originator.url, "http")) {- // The same dup feeds two purposes: Referer header (subject to- // Referrer-Policy in the future) and SameSite computation (which- // must use the real initiator regardless of policy). We share the- // same allocation for both.- const dup = try arena.dupeZ(u8, originator.url);if (nav_opts.referer == null) {- nav_opts.referer = dup;+ nav_opts.referer = try referrer.compute(arena.allocator(), originator.referrer_policy, originator.url, resolved_url);}if (nav_opts.initiator_url == null) {- nav_opts.initiator_url = dup;+ nav_opts.initiator_url = try arena.dupeZ(u8, originator.url);}}if (nav_opts.initiator_origin == null) {@@ -1274,6 +1278,11 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer..name = try self.arena.dupe(u8, hdr.name),.value = try self.arena.dupe(u8, hdr.value),});+ if (std.ascii.eqlIgnoreCase(hdr.name, "referrer-policy")) {+ if (referrer.parseHeader(hdr.value)) |rp| {+ self.referrer_policy = rp;+ }+ }}if (self._navigated_options) |no| {@@ -1843,13 +1852,14 @@ pub fn iframeAddedCallback(self: *Frame, iframe: *IFrame) !void {const was_sorted = self.child_frames_sorted;self.child_frames_sorted = false;- // Iframe's initial src request carries the parent's URL as Referer and- // as the SameSite initiator. Parent frame outlives this navigate()- // call, so the slice is safe.+ // Iframe's initial src request carries the parent's URL as Referer+ // (subject to the parent's Referrer-Policy) and as the SameSite+ // initiator. Parent frame outlives this navigate() call, so the slice+ // is safe; navigate dupes what it keeps.const parent_url: ?[:0]const u8 = if (std.mem.startsWith(u8, self.url, "http")) self.url else null;new_frame.navigate(url, .{.reason = .initialFrameNavigation,- .referer = parent_url,+ .referer = try referrer.compute(self.call_arena, self.referrer_policy, self.url, url),.initiator_url = parent_url,.initiator_origin = self.origin,}) catch |err| {diff --git a/src/browser/referrer.zig b/src/browser/referrer.zignew file mode 100644index 000000000..0e3250441--- /dev/null+++ b/src/browser/referrer.zig@@ -0,0 +1,222 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++// Referrer Policy: https://www.w3.org/TR/referrer-policy/+const std = @import("std");+const URL = @import("URL.zig");++const Allocator = std.mem.Allocator;++pub const Policy = enum {+ no_referrer,+ no_referrer_when_downgrade,+ origin,+ origin_when_cross_origin,+ same_origin,+ strict_origin,+ strict_origin_when_cross_origin,+ unsafe_url,++ pub const default: Policy = .strict_origin_when_cross_origin;+};++pub fn parse(value: []const u8) ?Policy {+ const map = std.StaticStringMapWithEql(Policy, staticStringMapEqlAsciiIgnoreCase).initComptime(.{+ .{ "no-referrer", Policy.no_referrer },+ .{ "no-referrer-when-downgrade", Policy.no_referrer_when_downgrade },+ .{ "origin", Policy.origin },+ .{ "origin-when-cross-origin", Policy.origin_when_cross_origin },+ .{ "same-origin", Policy.same_origin },+ .{ "strict-origin", Policy.strict_origin },+ .{ "strict-origin-when-cross-origin", Policy.strict_origin_when_cross_origin },+ .{ "unsafe-url", Policy.unsafe_url },+ });+ return map.get(value);+}++pub fn parseHeader(value: []const u8) ?Policy {+ var policy: ?Policy = null;+ var it = std.mem.splitScalar(u8, value, ',');+ while (it.next()) |token| {+ if (parse(std.mem.trim(u8, token, " \t"))) |p| {+ policy = p;+ }+ }+ return policy;+}++pub fn parseMeta(value: []const u8) ?Policy {+ if (parse(value)) |p| {+ return p;+ }+ // legacy values+ const map = std.StaticStringMapWithEql(Policy, staticStringMapEqlAsciiIgnoreCase).initComptime(.{+ .{ "never", Policy.no_referrer },+ .{ "always", Policy.unsafe_url },+ .{ "origin-when-crossorigin", Policy.origin_when_cross_origin },+ .{ "default", Policy.default },+ });+ return map.get(value);+}++// returns the value to send as the Referrer header based on the target_url+// and the given policy+pub fn compute(arena: Allocator, policy: Policy, referrer_url: [:0]const u8, target_url: [:0]const u8) !?[]const u8 {+ if (policy == .no_referrer) {+ return null;+ }++ const referrer_origin = (try URL.getOrigin(arena, referrer_url)) orelse {+ // blob:, data: ... don't set get a referrer+ return null;+ };++ const same_origin = blk: {+ const target_origin = (try URL.getOrigin(arena, target_url)) orelse break :blk false;+ break :blk std.ascii.eqlIgnoreCase(referrer_origin, target_origin);+ };+ const downgrade = URL.isSecure(referrer_url) and !URL.isSecure(target_url);++ const full = switch (policy) {+ .no_referrer => unreachable,+ .unsafe_url => true,+ .origin => false,+ .no_referrer_when_downgrade => if (downgrade) return null else true,+ .same_origin => if (same_origin) true else return null,+ .origin_when_cross_origin => same_origin,+ .strict_origin => if (downgrade) return null else false,+ .strict_origin_when_cross_origin => if (same_origin) true else if (downgrade) return null else false,+ };++ if (full) {+ // Serializing through origin + path + query strips credentials and+ // the fragment, and normalizes away default ports.+ const value = try std.fmt.allocPrint(arena, "{s}{s}{s}", .{+ referrer_origin,+ URL.getPathname(referrer_url),+ URL.getSearch(referrer_url),+ });++ if (value.len <= 4096) {+ // spec limit, if it's more than this, we falllback to the origin+ return value;+ }+ }+ return try std.fmt.allocPrint(arena, "{s}/", .{referrer_origin});+}++fn staticStringMapEqlAsciiIgnoreCase(a: []const u8, b: []const u8) bool {+ for (a, b) |a_c, b_c| {+ if (std.ascii.toLower(a_c) != std.ascii.toLower(b_c)) {+ return false;+ }+ }+ return true;+}++const testing = @import("../testing.zig");+test "referrer: parse" {+ try testing.expectEqual(Policy.no_referrer, parse("no-referrer"));+ try testing.expectEqual(Policy.unsafe_url, parse("Unsafe-URL"));+ try testing.expectEqual(null, parse(""));+ try testing.expectEqual(null, parse("never"));+ try testing.expectEqual(null, parse("no-referrer "));++ try testing.expectEqual(null, parseHeader(""));+ try testing.expectEqual(null, parseHeader("nope"));+ try testing.expectEqual(Policy.origin, parseHeader("origin"));+ try testing.expectEqual(Policy.same_origin, parseHeader("origin, same-origin"));+ try testing.expectEqual(Policy.origin, parseHeader("origin, garbage"));+ try testing.expectEqual(Policy.same_origin, parseHeader(" origin ,\tsame-origin "));++ try testing.expectEqual(Policy.no_referrer, parseMeta("never"));+ try testing.expectEqual(Policy.unsafe_url, parseMeta("always"));+ try testing.expectEqual(Policy.origin_when_cross_origin, parseMeta("origin-when-crossorigin"));+ try testing.expectEqual(Policy.strict_origin_when_cross_origin, parseMeta("default"));+ try testing.expectEqual(Policy.origin, parseMeta("origin"));+ try testing.expectEqual(null, parseMeta("garbage"));+}++test "referrer: compute" {+ const Case = struct {+ policy: Policy,+ referrer: [:0]const u8,+ target: [:0]const u8,+ expected: ?[]const u8,+ };++ const cases = [_]Case{+ .{ .policy = .no_referrer, .referrer = "http://a.com/p", .target = "http://a.com/x", .expected = null },++ .{ .policy = .unsafe_url, .referrer = "http://a.com/p?q=1#frag", .target = "https://b.com/", .expected = "http://a.com/p?q=1" },+ .{ .policy = .unsafe_url, .referrer = "https://a.com/p", .target = "http://b.com/", .expected = "https://a.com/p" },+ .{ .policy = .unsafe_url, .referrer = "https://user:[email protected]/p", .target = "http://b.com/", .expected = "https://a.com/p" },+ .{ .policy = .unsafe_url, .referrer = "https://a.com:443/p", .target = "http://b.com/", .expected = "https://a.com/p" },+ .{ .policy = .unsafe_url, .referrer = "http://a.com", .target = "http://b.com/", .expected = "http://a.com/" },++ .{ .policy = .origin, .referrer = "http://a.com:8000/p?q=1", .target = "http://a.com:8000/x", .expected = "http://a.com:8000/" },++ .{ .policy = .same_origin, .referrer = "http://a.com/p", .target = "http://a.com/x", .expected = "http://a.com/p" },+ .{ .policy = .same_origin, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = null },+ .{ .policy = .same_origin, .referrer = "http://a.com/p", .target = "https://a.com/x", .expected = null },++ .{ .policy = .origin_when_cross_origin, .referrer = "http://a.com/p", .target = "http://a.com/x", .expected = "http://a.com/p" },+ .{ .policy = .origin_when_cross_origin, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = "http://a.com/" },++ .{ .policy = .strict_origin, .referrer = "https://a.com/p", .target = "http://a.com/x", .expected = null },+ .{ .policy = .strict_origin, .referrer = "https://a.com/p", .target = "https://b.com/x", .expected = "https://a.com/" },+ .{ .policy = .strict_origin, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = "http://a.com/" },++ .{ .policy = .no_referrer_when_downgrade, .referrer = "https://a.com/p", .target = "http://b.com/x", .expected = null },+ .{ .policy = .no_referrer_when_downgrade, .referrer = "https://a.com/p", .target = "https://b.com/x", .expected = "https://a.com/p" },+ .{ .policy = .no_referrer_when_downgrade, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = "http://a.com/p" },++ .{ .policy = .strict_origin_when_cross_origin, .referrer = "http://a.com/p?q=1", .target = "http://a.com/x", .expected = "http://a.com/p?q=1" },+ .{ .policy = .strict_origin_when_cross_origin, .referrer = "http://a.com/p", .target = "http://b.com/x", .expected = "http://a.com/" },+ .{ .policy = .strict_origin_when_cross_origin, .referrer = "https://a.com/p", .target = "http://b.com/x", .expected = null },+ .{ .policy = .strict_origin_when_cross_origin, .referrer = "https://a.com/p", .target = "http://a.com/x", .expected = null },++ // no referrer from non-http(s) documents+ .{ .policy = .unsafe_url, .referrer = "about:blank", .target = "http://b.com/x", .expected = null },+ .{ .policy = .unsafe_url, .referrer = "data:text/html,x", .target = "http://b.com/x", .expected = null },+ };++ for (cases) |case| {+ const actual = try compute(testing.arena_allocator, case.policy, case.referrer, case.target);+ if (case.expected) |expected| {+ try testing.expectEqual(expected, actual orelse return error.UnexpectedNull);+ } else {+ try testing.expectEqual(null, actual);+ }+ }+}++test "referrer: compute caps at 4096 bytes" {+ const path = "/" ++ ("a" ** 4096);+ const url = "http://a.com" ++ path;+ // over the cap: falls back to the origin form+ try testing.expectEqual("http://a.com/", (try compute(testing.arena_allocator, .unsafe_url, url, "http://b.com/x")).?);+ try testing.expectEqual("http://a.com/", (try compute(testing.arena_allocator, .no_referrer_when_downgrade, url, "http://a.com/x")).?);++ // exactly at the cap: sent in full+ const at_cap = "http://a.com/" ++ ("a" ** (4096 - "http://a.com/".len));+ try testing.expectEqual(at_cap, (try compute(testing.arena_allocator, .unsafe_url, at_cap, "http://b.com/x")).?);++ // origin-only policies are unaffected by the referrer's length+ try testing.expectEqual("http://a.com/", (try compute(testing.arena_allocator, .origin, url, "http://b.com/x")).?);+}diff --git a/src/browser/webapi/Document.zig b/src/browser/webapi/Document.zigindex 78e2a3f83..e5e0b5e6e 100644--- a/src/browser/webapi/Document.zig+++ b/src/browser/webapi/Document.zig@@ -212,6 +212,11 @@ pub fn getLastModified(self: *const Document, frame: *Frame) ![]const u8 {});}+pub fn getReferrer(self: *const Document) []const u8 {+ const frame = self._frame orelse return "";+ return frame._referrer orelse "";+}+pub fn getCharset(self: *const Document) []const u8 {if (self._charset) |charset| {return charset;@@ -1588,15 +1593,12 @@ pub const JsApi = struct {pub const hasFocus = bridge.function(Document.hasFocus, .{});pub const prerendering = bridge.property(false, .{ .template = false });- pub const characterSet = bridge.accessor(getCharacterSet, null, .{});- pub const charset = bridge.accessor(getCharacterSet, null, .{});- pub const inputEncoding = bridge.accessor(getCharacterSet, null, .{});+ pub const characterSet = bridge.accessor(Document.getCharset, null, .{});+ pub const charset = bridge.accessor(Document.getCharset, null, .{});+ pub const inputEncoding = bridge.accessor(Document.getCharset, null, .{});pub const compatMode = bridge.accessor(Document.getCompatMode, null, .{});pub const lastModified = bridge.accessor(Document.getLastModified, null, .{});- fn getCharacterSet(self: *const Document) []const u8 {- return self.getCharset();- }- pub const referrer = bridge.property("", .{ .template = false });+ pub const referrer = bridge.accessor(Document.getReferrer, null, .{});// Generates a getter/setter pair backed by the frame's attribute-listener// map, like onclick above, for other document event handler properties.diff --git a/src/browser/webapi/element/html/Meta.zig b/src/browser/webapi/element/html/Meta.zigindex ea5baee1a..7c869c290 100644--- a/src/browser/webapi/element/html/Meta.zig+++ b/src/browser/webapi/element/html/Meta.zig@@ -16,11 +16,14 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+const std = @import("std");+const js = @import("../../../js/js.zig");const Frame = @import("../../../Frame.zig");const Node = @import("../../Node.zig");const Element = @import("../../Element.zig");const HtmlElement = @import("../Html.zig");+const referrer = @import("../../../referrer.zig");const Meta = @This();@@ -78,6 +81,21 @@ pub fn setScheme(self: *Meta, value: []const u8, frame: *Frame) !void {try self.asElement().setAttributeSafe(comptime .wrap("scheme"), .wrap(value), frame);}+pub const Build = struct {+ // <meta name=referrer> sets the document's referrer policy.+ pub fn created(node: *Node, frame: *Frame) !void {+ const el = node.as(Element);+ const name = el.getAttributeSafe(comptime .wrap("name")) orelse return;+ if (std.ascii.eqlIgnoreCase(name, "referrer") == false) {+ return;+ }+ const content = el.getAttributeSafe(comptime .wrap("content")) orelse return;+ if (referrer.parseMeta(content)) |rp| {+ frame.referrer_policy = rp;+ }+ }+};+pub const JsApi = struct {pub const bridge = js.Bridge(MetaElement);