Lightpanda: the headless browser designed for AI and automation
htttp: add Transfer.appendHeader
Pierre Tachoiredd833000959b
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex aa0c2ece0..33e29ab74 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -623,7 +623,7 @@ pub fn httpMetadata(self: *const Frame) HttpMetadata {pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {const arena = transfer.arena.allocator();if (try referrer.compute(arena, self.referrer_policy, self.referrerSource(), transfer.req.url)) |ref| {- try transfer.addHeader("Referer", ref, .{});+ try transfer.setHeader("Referer", ref, .{});transfer.req.referrer_policy = self.referrer_policy;}}@@ -851,15 +851,15 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo{// Ours until submit; clean up if header setup fails.errdefer transfer.deinit();- try transfer.addHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});+ try transfer.setHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});if (opts.header) |hdr| {// Arrives pre-joined ("Name: Value"), e.g. from the CLI.if (HttpClient.Header.parse(hdr)) |parsed| {- try transfer.addHeader(parsed.name, parsed.value, .{});+ try transfer.setHeader(parsed.name, parsed.value, .{});}}if (opts.referer) |ref| {- try transfer.addHeader("Referer", ref, .{});+ try transfer.setHeader("Referer", ref, .{});self._referrer = try self.arena.dupe(u8, ref);transfer.req.referrer_policy = opts.referrer_policy;}@@ -2315,7 +2315,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co};{errdefer transfer.deinit();- try transfer.addHeader("Accept", "text/css,*/*;q=0.1", .{});+ try transfer.setHeader("Accept", "text/css,*/*;q=0.1", .{});try self.headersForRequest(transfer);}diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zigindex fadd69401..0fdc14c08 100644--- a/src/browser/webapi/net/EventSource.zig+++ b/src/browser/webapi/net/EventSource.zig@@ -196,15 +196,15 @@ fn connect(self: *EventSource) !void {{errdefer transfer.deinit();- try transfer.addHeader("Accept", "text/event-stream", .{});- try transfer.addHeader("Cache-Control", "no-cache", .{});+ try transfer.setHeader("Accept", "text/event-stream", .{});+ try transfer.setHeader("Cache-Control", "no-cache", .{});if (self._last_event_id.items.len > 0) {- try transfer.addHeader("Last-Event-ID", self._last_event_id.items, .{});+ try transfer.setHeader("Last-Event-ID", self._last_event_id.items, .{});}if (!same_origin) {// EventSource is a CORS request: cross-origin fetches carry the// document's origin ("null" for opaque origins, like Chrome).- try transfer.addHeader("Origin", exec.origin() orelse "null", .{});+ try transfer.setHeader("Origin", exec.origin() orelse "null", .{});}try exec.headersForRequest(transfer);}diff --git a/src/browser/webapi/net/Headers.zig b/src/browser/webapi/net/Headers.zigindex 7d3915d3a..5436aac2d 100644--- a/src/browser/webapi/net/Headers.zig+++ b/src/browser/webapi/net/Headers.zig@@ -101,7 +101,7 @@ pub fn forEach(self: *Headers, cb_: js.Function, js_this_: ?js.Object) !void {const HttpClient = @import("../../../network/HttpClient.zig");pub fn populateRequestHeaders(self: *Headers, transfer: *HttpClient.Transfer) !void {for (self._list._entries.items) |entry| {- try transfer.addHeader(entry.name.str(), entry.value.str(), .{ .source = .author });+ try transfer.appendHeader(entry.name.str(), entry.value.str(), .{ .source = .author });}}diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 04b03a451..a7ae331a8 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -931,10 +931,10 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {.last_modified = cached.last_modified,});if (cached.etag) |etag| {- try transfer.addHeader("If-None-Match", etag, .{});+ try transfer.setHeader("If-None-Match", etag, .{});}if (cached.last_modified) |lm| {- try transfer.addHeader("If-Modified-Since", lm, .{});+ try transfer.setHeader("If-Modified-Since", lm, .{});}transfer._cache_intent = .{ .revalidate = cached };return false;@@ -2737,7 +2737,7 @@ pub const Transfer = struct {source: HeaderSource = .user_agent,};- pub fn addHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {+ fn addHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {const arena = self.arena.allocator();try self.req_headers.append(arena, .{.name = try arena.dupe(u8, name),@@ -2797,6 +2797,30 @@ pub const Transfer = struct {}}+ // Adds, appending every existing header with the first same case-insensitive name.+ // appendHeader doesn't deduplicate any existing headers.+ pub fn appendHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {+ var i: usize = 0;+ while (i < self.req_headers.items.len) {+ const hdr = &self.req_headers.items[i];+ if (std.ascii.eqlIgnoreCase(hdr.name, name) == false) {+ i += 1;+ continue;+ }++ // Don't override any fixed header.+ if (hdr.source == .fixed) {+ log.info(.http, "ignore overriding fixed header", .{ .header = hdr.name });+ return;+ }++ hdr.value = try std.fmt.allocPrint(self.arena.allocator(), "{s}, {s}", .{ hdr.value, value });+ hdr.source = opts.source;+ return;+ }+ try self.addHeader(name, value, opts);+ }+// The client's baseline headers, added to every request at creation.fn seedHeaders(self: *Transfer) !void {for (self.client.baselineHeaders()) |hdr| {@@ -3470,9 +3494,15 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {.start_time = 0,};- try transfer.addHeader("User-Agent", "Lightpanda/1.0", .{});- try transfer.addHeader("X-Twice", "a", .{});- try transfer.addHeader("x-twice", "b", .{});+ try transfer.setHeader("User-Agent", "Lightpanda/1.0", .{});+ try transfer.setHeader("X-Twice", "a", .{});++ // force a duplicated value manually+ try transfer.req_headers.append(transfer.arena.allocator(), .{+ .name = "x-twice",+ .value = "b",+ .source = .user_agent,+ });// replaces in place, collapsing duplicatestry transfer.setHeader("user-agent", "Custom/1.0", .{});@@ -3491,6 +3521,46 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {try testing.expectEqual("yes", headers[2].value);}+test "HttpClient: Transfer.appendHeader joins values by case-insensitive name" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ const arena = try pool.acquire(.small, "test");+ defer arena.release();++ var transfer = Transfer{+ .arena = arena,+ .owner = null,+ .req = .{+ .frame_id = 0,+ .loader_id = 0,+ .method = .GET,+ .url = "http://example.com/",+ .cookie_jar = null,+ .cookie_origin = "",+ .resource_type = .document,+ .notification = undefined,+ .shutdown_callback = noopShutdown,+ },+ .client = undefined,+ .id = 1,+ .start_time = 0,+ };++ // no match: appends a new header+ try transfer.appendHeader("Cookie", "a=1", .{});+ // match by case-insensitive name: joins onto the existing value+ try transfer.appendHeader("COOKIE", "b=2", .{ .source = .author });++ {+ const headers = transfer.req_headers.items;+ try testing.expectEqual(1, headers.len);+ try testing.expectEqual("Cookie", headers[0].name);+ try testing.expectEqual("a=1, b=2", headers[0].value);+ try testing.expectEqual(.author, headers[0].source);+ }+}+test "HttpClient: Fetch header overrides restore after one hop" {const original = [_]Transfer.RequestHeader{.{ .name = "User-Agent", .value = "original" },diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zigindex aae9d1df1..e5ef72afd 100644--- a/src/network/WebBotAuth.zig+++ b/src/network/WebBotAuth.zig@@ -121,9 +121,9 @@ pub fn signRequest(const encoded = try arena.alloc(u8, encoded_len);_ = std.base64.standard.Encoder.encode(encoded, &sig);- try transfer.addHeader("Signature-Agent", try std.fmt.allocPrint(arena, "\"{s}\"", .{self.directory_url}), .{});- try transfer.addHeader("Signature-Input", try std.fmt.allocPrint(arena, "sig1={s}", .{sig_input_value}), .{});- try transfer.addHeader("Signature", try std.fmt.allocPrint(arena, "sig1=:{s}:", .{encoded}), .{});+ try transfer.setHeader("Signature-Agent", try std.fmt.allocPrint(arena, "\"{s}\"", .{self.directory_url}), .{});+ try transfer.setHeader("Signature-Input", try std.fmt.allocPrint(arena, "sig1={s}", .{sig_input_value}), .{});+ try transfer.setHeader("Signature", try std.fmt.allocPrint(arena, "sig1=:{s}:", .{encoded}), .{});}pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {