Skip to content

Lightpanda: the headless browser designed for AI and automation

add option to headersForRequest

Muki Kiboigob776bad19066
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index acba85464..baab3846d 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -665,7 +665,9 @@ pub fn httpMetadata(self: *const Frame) HttpMetadata {
// Add common headers for a request:
// * referer
-pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {
+pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer, opts: JS.Execution.HeadersForRequestOptions) !void {
+ if (!opts.referer) return;
+
const arena = transfer.arena.allocator();
if (try referrer.compute(arena, self.referrer_policy, self.referrerSource(), transfer.req.url)) |ref| {
try transfer.setHeader("Referer", ref, .{});
@@ -1141,7 +1143,7 @@ pub fn makeRequest(self: *Frame, req: HttpClient.Request) !void {
const transfer = try self._session.browser.http_client.newRequest(req, &self._http_owner);
{
errdefer transfer.deinit();
- try self.headersForRequest(transfer);
+ try self.headersForRequest(transfer, .{});
}
transfer.submit() catch {};
}
@@ -2423,7 +2425,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
{
errdefer transfer.deinit();
try transfer.setHeader("Accept", "text/css,*/*;q=0.1", .{});
- try self.headersForRequest(transfer);
+ try self.headersForRequest(transfer, .{});
}
// Set the script-manager `is_evaluating` flag for the same reason
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig
index 3f3e71400..a0437c64a 100644
--- a/src/browser/ScriptManager.zig
+++ b/src/browser/ScriptManager.zig
@@ -392,7 +392,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
}, &frame._http_owner);
{
errdefer transfer.deinit();
- try frame.headersForRequest(transfer);
+ try frame.headersForRequest(transfer, .{});
}
const response = try transfer.submitSync();
@@ -448,7 +448,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
.shutdown_callback = HttpClient.noopShutdown,
});
errdefer transfer.deinit();
- try frame.headersForRequest(transfer);
+ try frame.headersForRequest(transfer, .{});
break :blk transfer;
};
diff --git a/src/browser/frame/resource_load.zig b/src/browser/frame/resource_load.zig
index b66febc2a..3edae3cfa 100644
--- a/src/browser/frame/resource_load.zig
+++ b/src/browser/frame/resource_load.zig
@@ -98,7 +98,7 @@ pub fn image(frame: *Frame, img: *Element.Html.Image, src: []const u8) !void {
// (or that turn away clients which don't look like browsers) key off
// exactly this header.
try transfer.setHeader("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", .{});
- try frame.headersForRequest(transfer);
+ try frame.headersForRequest(transfer, .{});
}
// From here the transfer owns `load`. `submit` either succeeds or has
diff --git a/src/browser/global_scope.zig b/src/browser/global_scope.zig
index f8980af83..b54e7af66 100644
--- a/src/browser/global_scope.zig
+++ b/src/browser/global_scope.zig
@@ -30,6 +30,7 @@ const EventTarget = @import("webapi/EventTarget.zig");
const Performance = @import("webapi/Performance.zig");
const WorkerGlobalScope = @import("webapi/WorkerGlobalScope.zig");
+const JS = @import("js/js.zig");
const Frame = @import("Frame.zig");
const Session = @import("Session.zig");
const referrer = @import("referrer.zig");
@@ -86,9 +87,9 @@ pub const GlobalScope = union(enum) {
};
}
- pub fn headersForRequest(self: GlobalScope, transfer: *HttpClient.Transfer) !void {
+ pub fn headersForRequest(self: GlobalScope, transfer: *HttpClient.Transfer, opts: JS.Execution.HeadersForRequestOptions) !void {
return switch (self) {
- inline else => |g| g.headersForRequest(transfer),
+ inline else => |g| g.headersForRequest(transfer, opts),
};
}
diff --git a/src/browser/js/Execution.zig b/src/browser/js/Execution.zig
index 9d15660a1..2011f8fe1 100644
--- a/src/browser/js/Execution.zig
+++ b/src/browser/js/Execution.zig
@@ -89,8 +89,14 @@ pub fn base(self: *const Execution) [:0]const u8 {
return self.js.global.base();
}
-pub fn headersForRequest(self: *const Execution, transfer: *HttpClient.Transfer) !void {
- return self.js.global.headersForRequest(transfer);
+pub const HeadersForRequestOptions = struct {
+ // Callers that compute their own Referer (e.g. fetch()'s referrer/referrerPolicy
+ // options) set this to false so we don't set a header here just to override it.
+ referer: bool = true,
+};
+
+pub fn headersForRequest(self: *const Execution, transfer: *HttpClient.Transfer, opts: HeadersForRequestOptions) !void {
+ return self.js.global.headersForRequest(transfer, opts);
}
pub fn referrerSource(self: *const Execution) [:0]const u8 {
diff --git a/src/browser/webapi/WorkerGlobalScope.zig b/src/browser/webapi/WorkerGlobalScope.zig
index 5a060e792..1d04344b7 100644
--- a/src/browser/webapi/WorkerGlobalScope.zig
+++ b/src/browser/webapi/WorkerGlobalScope.zig
@@ -257,8 +257,8 @@ pub fn hasDirectListeners(self: *WorkerGlobalScope, target: *EventTarget, typ: [
// Workers don't have their own Referer; per spec, dedicated worker requests
// use the parent document's URL. Delegate to the owning frame.
-pub fn headersForRequest(self: *WorkerGlobalScope, transfer: *HttpClient.Transfer) !void {
- return self._frame.headersForRequest(transfer);
+pub fn headersForRequest(self: *WorkerGlobalScope, transfer: *HttpClient.Transfer, opts: JS.Execution.HeadersForRequestOptions) !void {
+ return self._frame.headersForRequest(transfer, opts);
}
pub fn isSameOrigin(self: *const WorkerGlobalScope, url: [:0]const u8) bool {
@@ -270,7 +270,7 @@ pub fn makeRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !void {
const transfer = try self._session.browser.http_client.newRequest(req, &self._http_owner);
{
errdefer transfer.deinit();
- try self.headersForRequest(transfer);
+ try self.headersForRequest(transfer, .{});
}
transfer.submit() catch {};
}
@@ -430,7 +430,7 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !
};
{
errdefer transfer.deinit();
- try self.headersForRequest(transfer);
+ try self.headersForRequest(transfer, .{});
}
var response = transfer.submitSync() catch |err| {
diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zig
index faf6b5341..989204f4d 100644
--- a/src/browser/webapi/net/EventSource.zig
+++ b/src/browser/webapi/net/EventSource.zig
@@ -201,7 +201,7 @@ fn connect(self: *EventSource) !void {
// document's origin ("null" for opaque origins, like Chrome).
try transfer.setHeader("Origin", exec.origin() orelse "null", .{});
}
- try exec.headersForRequest(transfer);
+ try exec.headersForRequest(transfer, .{});
}
self._transfer = transfer;
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index c8815de00..14fab632d 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -144,7 +144,9 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis
try h.populateRequestHeaders(transfer);
}
- try exec.headersForRequest(transfer);
+ // fetch() computes its own Referer below, from request._referrer /
+ // request._referrer_policy, so skip the default one here.
+ try exec.headersForRequest(transfer, .{ .referer = false });
const source: ?[:0]const u8 = switch (request._referrer) {
.none => null,
diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig
index 8e5cdb630..3f5899df1 100644
--- a/src/browser/webapi/net/XMLHttpRequest.zig
+++ b/src/browser/webapi/net/XMLHttpRequest.zig
@@ -360,7 +360,7 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
try self._request_headers.populateRequestHeaders(transfer);
if (transfer.req.credentialsAllowed()) {
- try exec.headersForRequest(transfer);
+ try exec.headersForRequest(transfer, .{});
}
}