Skip to content

Lightpanda: the headless browser designed for AI and automation

add per-request referrer policy to fetch

Muki Kiboigo0c94eba7f919
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 4ae6e1938..acba85464 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -627,7 +627,7 @@ pub fn base(self: *const Frame) [:0]const u8 {
return self.base_url orelse self.url;
}
-fn referrerSource(self: *const Frame) [:0]const u8 {
+pub fn referrerSource(self: *const Frame) [:0]const u8 {
var frame = self;
while (std.mem.startsWith(u8, frame.url, "about:")) {
// about:blank and about:srcdoc documents aren't valid referrer sources,
diff --git a/src/browser/global_scope.zig b/src/browser/global_scope.zig
index 458af8594..71cc25ab0 100644
--- a/src/browser/global_scope.zig
+++ b/src/browser/global_scope.zig
@@ -91,6 +91,13 @@ pub const GlobalScope = union(enum) {
};
}
+ pub fn referrerSource(self: GlobalScope) [:0]const u8 {
+ return switch (self) {
+ .frame => |frame| frame.referrerSource(),
+ .worker => |worker| worker.url,
+ };
+ }
+
pub fn isSameOrigin(self: GlobalScope, target: [:0]const u8) bool {
return switch (self) {
inline else => |g| g.isSameOrigin(target),
diff --git a/src/browser/js/Execution.zig b/src/browser/js/Execution.zig
index 3313555f8..16066896b 100644
--- a/src/browser/js/Execution.zig
+++ b/src/browser/js/Execution.zig
@@ -92,6 +92,10 @@ pub fn headersForRequest(self: *const Execution, transfer: *HttpClient.Transfer)
return self.js.global.headersForRequest(transfer);
}
+pub fn referrerSource(self: *const Execution) [:0]const u8 {
+ return self.js.global.referrerSource();
+}
+
pub fn isSameOrigin(self: *const Execution, url: [:0]const u8) bool {
return self.js.global.isSameOrigin(url);
}
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index 46f3cfea6..aa9ed4f9f 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -33,6 +33,8 @@ const log = lp.log;
const Execution = js.Execution;
const Transfer = HttpClient.Transfer;
+const referrer = @import("../../referrer.zig");
+
const Fetch = @This();
_exec: *const Execution,
@@ -141,7 +143,28 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis
if (request._headers) |h| {
try h.populateRequestHeaders(transfer);
}
- try exec.headersForRequest(transfer);
+
+ switch (request._referrer) {
+ .none => {},
+ .client => {
+ if (request._referrer_policy) |policy| {
+ const source = exec.referrerSource();
+ if (try referrer.compute(transfer.arena.allocator(), policy, source, transfer.req.url)) |ref| {
+ try transfer.setHeader("Referer", ref, .{});
+ }
+ transfer.req.referrer_policy = policy;
+ } else {
+ try exec.headersForRequest(transfer);
+ }
+ },
+ .url => |source| {
+ const policy = request._referrer_policy orelse referrer.Policy.default;
+ if (try referrer.compute(transfer.arena.allocator(), policy, source, transfer.req.url)) |ref| {
+ try transfer.setHeader("Referer", ref, .{});
+ }
+ transfer.req.referrer_policy = policy;
+ },
+ }
}
// Held for Response.deinit's abort; the error, shutdown and done
diff --git a/src/browser/webapi/net/Request.zig b/src/browser/webapi/net/Request.zig
index 4a2357ff2..064b32a69 100644
--- a/src/browser/webapi/net/Request.zig
+++ b/src/browser/webapi/net/Request.zig
@@ -32,6 +32,7 @@ const Headers = @import("Headers.zig");
const FormData = @import("FormData.zig");
const body_init = @import("body_init.zig");
const BodyInit = body_init.BodyInit;
+const referrer = @import("../../referrer.zig");
const Execution = js.Execution;
@@ -49,6 +50,8 @@ _credentials: Credentials,
_redirect: Redirect,
_mode: Mode,
_signal: ?*AbortSignal,
+_referrer: ReferrerValue,
+_referrer_policy: ?referrer.Policy,
_body_used: bool = false,
pub const Input = union(enum) {
@@ -65,9 +68,17 @@ pub const InitOpts = struct {
mode: Mode = .cors,
priority: ?[]const u8 = null,
redirect: Redirect = .follow,
+ referrer: ?[]const u8 = null,
+ referrerPolicy: ?[]const u8 = null,
signal: ?*AbortSignal = null,
};
+pub const ReferrerValue = union(enum) {
+ client,
+ none,
+ url: [:0]const u8,
+};
+
const Priority = enum { high, low, auto };
const Redirect = enum {
@@ -173,6 +184,24 @@ pub fn init(input: Input, opts_: ?InitOpts, exec: *const Execution) !*Request {
.request => |r| r._signal,
};
+ const referrer_value: ReferrerValue = if (opts.referrer) |r| blk: {
+ if (r.len == 0) break :blk .none;
+ if (std.mem.eql(u8, r, "about:client")) break :blk .client;
+ break :blk .{ .url = try URL.resolve(arena.allocator(), exec.base(), r, .{ .encoding = exec.charset.* }) };
+ } else switch (input) {
+ .url => .client,
+ .request => |r| r._referrer,
+ };
+
+ // Per spec, an unrecognized policy string is ignored
+ // referrer.parse already returns null for that case.
+ const referrer_policy: ?referrer.Policy = if (opts.referrerPolicy) |rp|
+ referrer.parse(rp)
+ else switch (input) {
+ .url => null,
+ .request => |r| r._referrer_policy,
+ };
+
const self = try arena.create(Request);
self.* = .{
._url = url,
@@ -186,6 +215,8 @@ pub fn init(input: Input, opts_: ?InitOpts, exec: *const Execution) !*Request {
._body = body,
._body_stream = body_stream,
._signal = signal,
+ ._referrer = referrer_value,
+ ._referrer_policy = referrer_policy,
};
arena.report();
return self;
@@ -372,6 +403,8 @@ pub fn clone(self: *Request, exec: *const Execution) !*Request {
._mode = self._mode,
._body = if (body) |b| try arena.dupe(u8, b) else null,
._signal = self._signal,
+ ._referrer = self._referrer,
+ ._referrer_policy = self._referrer_policy,
};
arena.report();
return request;