Lightpanda: the headless browser designed for AI and automation
add option to headersForRequest
Muki Kiboigob776bad19066
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex acba85464..baab3846d 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -665,7 +665,9 @@ pub fn httpMetadata(self: *const Frame) HttpMetadata {// Add common headers for a request:// * referer-pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {+pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer, opts: JS.Execution.HeadersForRequestOptions) !void {+ if (!opts.referer) return;+const arena = transfer.arena.allocator();if (try referrer.compute(arena, self.referrer_policy, self.referrerSource(), transfer.req.url)) |ref| {try transfer.setHeader("Referer", ref, .{});@@ -1141,7 +1143,7 @@ pub fn makeRequest(self: *Frame, req: HttpClient.Request) !void {const transfer = try self._session.browser.http_client.newRequest(req, &self._http_owner);{errdefer transfer.deinit();- try self.headersForRequest(transfer);+ try self.headersForRequest(transfer, .{});}transfer.submit() catch {};}@@ -2423,7 +2425,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co{errdefer transfer.deinit();try transfer.setHeader("Accept", "text/css,*/*;q=0.1", .{});- try self.headersForRequest(transfer);+ try self.headersForRequest(transfer, .{});}// Set the script-manager `is_evaluating` flag for the same reasondiff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zigindex 3f3e71400..a0437c64a 100644--- a/src/browser/ScriptManager.zig+++ b/src/browser/ScriptManager.zig@@ -392,7 +392,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e}, &frame._http_owner);{errdefer transfer.deinit();- try frame.headersForRequest(transfer);+ try frame.headersForRequest(transfer, .{});}const response = try transfer.submitSync();@@ -448,7 +448,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e.shutdown_callback = HttpClient.noopShutdown,});errdefer transfer.deinit();- try frame.headersForRequest(transfer);+ try frame.headersForRequest(transfer, .{});break :blk transfer;};diff --git a/src/browser/frame/resource_load.zig b/src/browser/frame/resource_load.zigindex b66febc2a..3edae3cfa 100644--- a/src/browser/frame/resource_load.zig+++ b/src/browser/frame/resource_load.zig@@ -98,7 +98,7 @@ pub fn image(frame: *Frame, img: *Element.Html.Image, src: []const u8) !void {// (or that turn away clients which don't look like browsers) key off// exactly this header.try transfer.setHeader("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", .{});- try frame.headersForRequest(transfer);+ try frame.headersForRequest(transfer, .{});}// From here the transfer owns `load`. `submit` either succeeds or hasdiff --git a/src/browser/global_scope.zig b/src/browser/global_scope.zigindex f8980af83..b54e7af66 100644--- a/src/browser/global_scope.zig+++ b/src/browser/global_scope.zig@@ -30,6 +30,7 @@ const EventTarget = @import("webapi/EventTarget.zig");const Performance = @import("webapi/Performance.zig");const WorkerGlobalScope = @import("webapi/WorkerGlobalScope.zig");+const JS = @import("js/js.zig");const Frame = @import("Frame.zig");const Session = @import("Session.zig");const referrer = @import("referrer.zig");@@ -86,9 +87,9 @@ pub const GlobalScope = union(enum) {};}- pub fn headersForRequest(self: GlobalScope, transfer: *HttpClient.Transfer) !void {+ pub fn headersForRequest(self: GlobalScope, transfer: *HttpClient.Transfer, opts: JS.Execution.HeadersForRequestOptions) !void {return switch (self) {- inline else => |g| g.headersForRequest(transfer),+ inline else => |g| g.headersForRequest(transfer, opts),};}diff --git a/src/browser/js/Execution.zig b/src/browser/js/Execution.zigindex 9d15660a1..2011f8fe1 100644--- a/src/browser/js/Execution.zig+++ b/src/browser/js/Execution.zig@@ -89,8 +89,14 @@ pub fn base(self: *const Execution) [:0]const u8 {return self.js.global.base();}-pub fn headersForRequest(self: *const Execution, transfer: *HttpClient.Transfer) !void {- return self.js.global.headersForRequest(transfer);+pub const HeadersForRequestOptions = struct {+ // Callers that compute their own Referer (e.g. fetch()'s referrer/referrerPolicy+ // options) set this to false so we don't set a header here just to override it.+ referer: bool = true,+};++pub fn headersForRequest(self: *const Execution, transfer: *HttpClient.Transfer, opts: HeadersForRequestOptions) !void {+ return self.js.global.headersForRequest(transfer, opts);}pub fn referrerSource(self: *const Execution) [:0]const u8 {diff --git a/src/browser/webapi/WorkerGlobalScope.zig b/src/browser/webapi/WorkerGlobalScope.zigindex 5a060e792..1d04344b7 100644--- a/src/browser/webapi/WorkerGlobalScope.zig+++ b/src/browser/webapi/WorkerGlobalScope.zig@@ -257,8 +257,8 @@ pub fn hasDirectListeners(self: *WorkerGlobalScope, target: *EventTarget, typ: [// Workers don't have their own Referer; per spec, dedicated worker requests// use the parent document's URL. Delegate to the owning frame.-pub fn headersForRequest(self: *WorkerGlobalScope, transfer: *HttpClient.Transfer) !void {- return self._frame.headersForRequest(transfer);+pub fn headersForRequest(self: *WorkerGlobalScope, transfer: *HttpClient.Transfer, opts: JS.Execution.HeadersForRequestOptions) !void {+ return self._frame.headersForRequest(transfer, opts);}pub fn isSameOrigin(self: *const WorkerGlobalScope, url: [:0]const u8) bool {@@ -270,7 +270,7 @@ pub fn makeRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !void {const transfer = try self._session.browser.http_client.newRequest(req, &self._http_owner);{errdefer transfer.deinit();- try self.headersForRequest(transfer);+ try self.headersForRequest(transfer, .{});}transfer.submit() catch {};}@@ -430,7 +430,7 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !};{errdefer transfer.deinit();- try self.headersForRequest(transfer);+ try self.headersForRequest(transfer, .{});}var response = transfer.submitSync() catch |err| {diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zigindex faf6b5341..989204f4d 100644--- a/src/browser/webapi/net/EventSource.zig+++ b/src/browser/webapi/net/EventSource.zig@@ -201,7 +201,7 @@ fn connect(self: *EventSource) !void {// document's origin ("null" for opaque origins, like Chrome).try transfer.setHeader("Origin", exec.origin() orelse "null", .{});}- try exec.headersForRequest(transfer);+ try exec.headersForRequest(transfer, .{});}self._transfer = transfer;diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex c8815de00..14fab632d 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -144,7 +144,9 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promistry h.populateRequestHeaders(transfer);}- try exec.headersForRequest(transfer);+ // fetch() computes its own Referer below, from request._referrer /+ // request._referrer_policy, so skip the default one here.+ try exec.headersForRequest(transfer, .{ .referer = false });const source: ?[:0]const u8 = switch (request._referrer) {.none => null,diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zigindex 8e5cdb630..3f5899df1 100644--- a/src/browser/webapi/net/XMLHttpRequest.zig+++ b/src/browser/webapi/net/XMLHttpRequest.zig@@ -360,7 +360,7 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !vtry self._request_headers.populateRequestHeaders(transfer);if (transfer.req.credentialsAllowed()) {- try exec.headersForRequest(transfer);+ try exec.headersForRequest(transfer, .{});}}