Skip to content

Lightpanda: the headless browser designed for AI and automation

htttp: add Transfer.appendHeader

Pierre Tachoiredd833000959b
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index aa0c2ece0..33e29ab74 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -623,7 +623,7 @@ pub fn httpMetadata(self: *const Frame) HttpMetadata {
pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {
const arena = transfer.arena.allocator();
if (try referrer.compute(arena, self.referrer_policy, self.referrerSource(), transfer.req.url)) |ref| {
- try transfer.addHeader("Referer", ref, .{});
+ try transfer.setHeader("Referer", ref, .{});
transfer.req.referrer_policy = self.referrer_policy;
}
}
@@ -851,15 +851,15 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
{
// Ours until submit; clean up if header setup fails.
errdefer transfer.deinit();
- try transfer.addHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});
+ try transfer.setHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});
if (opts.header) |hdr| {
// Arrives pre-joined ("Name: Value"), e.g. from the CLI.
if (HttpClient.Header.parse(hdr)) |parsed| {
- try transfer.addHeader(parsed.name, parsed.value, .{});
+ try transfer.setHeader(parsed.name, parsed.value, .{});
}
}
if (opts.referer) |ref| {
- try transfer.addHeader("Referer", ref, .{});
+ try transfer.setHeader("Referer", ref, .{});
self._referrer = try self.arena.dupe(u8, ref);
transfer.req.referrer_policy = opts.referrer_policy;
}
@@ -2315,7 +2315,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
};
{
errdefer transfer.deinit();
- try transfer.addHeader("Accept", "text/css,*/*;q=0.1", .{});
+ try transfer.setHeader("Accept", "text/css,*/*;q=0.1", .{});
try self.headersForRequest(transfer);
}
diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zig
index fadd69401..0fdc14c08 100644
--- a/src/browser/webapi/net/EventSource.zig
+++ b/src/browser/webapi/net/EventSource.zig
@@ -196,15 +196,15 @@ fn connect(self: *EventSource) !void {
{
errdefer transfer.deinit();
- try transfer.addHeader("Accept", "text/event-stream", .{});
- try transfer.addHeader("Cache-Control", "no-cache", .{});
+ try transfer.setHeader("Accept", "text/event-stream", .{});
+ try transfer.setHeader("Cache-Control", "no-cache", .{});
if (self._last_event_id.items.len > 0) {
- try transfer.addHeader("Last-Event-ID", self._last_event_id.items, .{});
+ try transfer.setHeader("Last-Event-ID", self._last_event_id.items, .{});
}
if (!same_origin) {
// EventSource is a CORS request: cross-origin fetches carry the
// document's origin ("null" for opaque origins, like Chrome).
- try transfer.addHeader("Origin", exec.origin() orelse "null", .{});
+ try transfer.setHeader("Origin", exec.origin() orelse "null", .{});
}
try exec.headersForRequest(transfer);
}
diff --git a/src/browser/webapi/net/Headers.zig b/src/browser/webapi/net/Headers.zig
index 7d3915d3a..5436aac2d 100644
--- a/src/browser/webapi/net/Headers.zig
+++ b/src/browser/webapi/net/Headers.zig
@@ -101,7 +101,7 @@ pub fn forEach(self: *Headers, cb_: js.Function, js_this_: ?js.Object) !void {
const HttpClient = @import("../../../network/HttpClient.zig");
pub fn populateRequestHeaders(self: *Headers, transfer: *HttpClient.Transfer) !void {
for (self._list._entries.items) |entry| {
- try transfer.addHeader(entry.name.str(), entry.value.str(), .{ .source = .author });
+ try transfer.appendHeader(entry.name.str(), entry.value.str(), .{ .source = .author });
}
}
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 04b03a451..a7ae331a8 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -931,10 +931,10 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
.last_modified = cached.last_modified,
});
if (cached.etag) |etag| {
- try transfer.addHeader("If-None-Match", etag, .{});
+ try transfer.setHeader("If-None-Match", etag, .{});
}
if (cached.last_modified) |lm| {
- try transfer.addHeader("If-Modified-Since", lm, .{});
+ try transfer.setHeader("If-Modified-Since", lm, .{});
}
transfer._cache_intent = .{ .revalidate = cached };
return false;
@@ -2737,7 +2737,7 @@ pub const Transfer = struct {
source: HeaderSource = .user_agent,
};
- pub fn addHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
+ fn addHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
const arena = self.arena.allocator();
try self.req_headers.append(arena, .{
.name = try arena.dupe(u8, name),
@@ -2797,6 +2797,30 @@ pub const Transfer = struct {
}
}
+ // Adds, appending every existing header with the first same case-insensitive name.
+ // appendHeader doesn't deduplicate any existing headers.
+ pub fn appendHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
+ var i: usize = 0;
+ while (i < self.req_headers.items.len) {
+ const hdr = &self.req_headers.items[i];
+ if (std.ascii.eqlIgnoreCase(hdr.name, name) == false) {
+ i += 1;
+ continue;
+ }
+
+ // Don't override any fixed header.
+ if (hdr.source == .fixed) {
+ log.info(.http, "ignore overriding fixed header", .{ .header = hdr.name });
+ return;
+ }
+
+ hdr.value = try std.fmt.allocPrint(self.arena.allocator(), "{s}, {s}", .{ hdr.value, value });
+ hdr.source = opts.source;
+ return;
+ }
+ try self.addHeader(name, value, opts);
+ }
+
// The client's baseline headers, added to every request at creation.
fn seedHeaders(self: *Transfer) !void {
for (self.client.baselineHeaders()) |hdr| {
@@ -3470,9 +3494,15 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
.start_time = 0,
};
- try transfer.addHeader("User-Agent", "Lightpanda/1.0", .{});
- try transfer.addHeader("X-Twice", "a", .{});
- try transfer.addHeader("x-twice", "b", .{});
+ try transfer.setHeader("User-Agent", "Lightpanda/1.0", .{});
+ try transfer.setHeader("X-Twice", "a", .{});
+
+ // force a duplicated value manually
+ try transfer.req_headers.append(transfer.arena.allocator(), .{
+ .name = "x-twice",
+ .value = "b",
+ .source = .user_agent,
+ });
// replaces in place, collapsing duplicates
try transfer.setHeader("user-agent", "Custom/1.0", .{});
@@ -3491,6 +3521,46 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
try testing.expectEqual("yes", headers[2].value);
}
+test "HttpClient: Transfer.appendHeader joins values by case-insensitive name" {
+ var pool = ArenaPool.init(testing.allocator, .{});
+ defer pool.deinit();
+
+ const arena = try pool.acquire(.small, "test");
+ defer arena.release();
+
+ var transfer = Transfer{
+ .arena = arena,
+ .owner = null,
+ .req = .{
+ .frame_id = 0,
+ .loader_id = 0,
+ .method = .GET,
+ .url = "http://example.com/",
+ .cookie_jar = null,
+ .cookie_origin = "",
+ .resource_type = .document,
+ .notification = undefined,
+ .shutdown_callback = noopShutdown,
+ },
+ .client = undefined,
+ .id = 1,
+ .start_time = 0,
+ };
+
+ // no match: appends a new header
+ try transfer.appendHeader("Cookie", "a=1", .{});
+ // match by case-insensitive name: joins onto the existing value
+ try transfer.appendHeader("COOKIE", "b=2", .{ .source = .author });
+
+ {
+ const headers = transfer.req_headers.items;
+ try testing.expectEqual(1, headers.len);
+ try testing.expectEqual("Cookie", headers[0].name);
+ try testing.expectEqual("a=1, b=2", headers[0].value);
+ try testing.expectEqual(.author, headers[0].source);
+ }
+}
+
test "HttpClient: Fetch header overrides restore after one hop" {
const original = [_]Transfer.RequestHeader{
.{ .name = "User-Agent", .value = "original" },
diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig
index aae9d1df1..e5ef72afd 100644
--- a/src/network/WebBotAuth.zig
+++ b/src/network/WebBotAuth.zig
@@ -121,9 +121,9 @@ pub fn signRequest(
const encoded = try arena.alloc(u8, encoded_len);
_ = std.base64.standard.Encoder.encode(encoded, &sig);
- try transfer.addHeader("Signature-Agent", try std.fmt.allocPrint(arena, "\"{s}\"", .{self.directory_url}), .{});
- try transfer.addHeader("Signature-Input", try std.fmt.allocPrint(arena, "sig1={s}", .{sig_input_value}), .{});
- try transfer.addHeader("Signature", try std.fmt.allocPrint(arena, "sig1=:{s}:", .{encoded}), .{});
+ try transfer.setHeader("Signature-Agent", try std.fmt.allocPrint(arena, "\"{s}\"", .{self.directory_url}), .{});
+ try transfer.setHeader("Signature-Input", try std.fmt.allocPrint(arena, "sig1={s}", .{sig_input_value}), .{});
+ try transfer.setHeader("Signature", try std.fmt.allocPrint(arena, "sig1=:{s}:", .{encoded}), .{});
}
pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {