Skip to content

Lightpanda: the headless browser designed for AI and automation

chore: remove http.HeaderIterator

Karl Seguina60058ad9a3d
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index b2f1d4ce2..11d70286c 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -1518,8 +1518,7 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.
self._http_status = transfer.responseStatus();
self._bot_challenge = transfer.botChallenge();
- var it = transfer.responseHeaderIterator();
- while (it.next()) |hdr| {
+ for (transfer.responseHeaders()) |hdr| {
try self._http_headers.append(self.arena, .{
.name = try self.arena.dupe(u8, hdr.name),
.value = try self.arena.dupe(u8, hdr.value),
@@ -1568,8 +1567,7 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {
}
const disposition: HttpClient.Header = blk: {
- var it = transfer.responseHeaderIterator();
- while (it.next()) |hdr| {
+ for (transfer.responseHeaders()) |hdr| {
if (std.mem.eql(u8, hdr.name, "content-disposition")) {
break :blk hdr;
}
diff --git a/src/browser/frame/framing.zig b/src/browser/frame/framing.zig
index 9db52e2d2..f06b6bc6a 100644
--- a/src/browser/frame/framing.zig
+++ b/src/browser/frame/framing.zig
@@ -24,8 +24,7 @@ const HttpClient = @import("../../network/HttpClient.zig");
// https://html.spec.whatwg.org/multipage/document-lifecycle.html#the-x-frame-options-header
pub fn allowed(frame: *const Frame, transfer: *HttpClient.Transfer) bool {
var options: XFrameOptions = .{};
- var it = transfer.responseHeaderIterator();
- while (it.next()) |hdr| {
+ for (transfer.responseHeaders()) |hdr| {
if (std.mem.eql(u8, hdr.name, "content-security-policy")) {
if (hasFrameAncestors(hdr.value)) {
// has priority over any x-frame-options
diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zig
index b74257353..f1f68a905 100644
--- a/src/browser/webapi/net/EventSource.zig
+++ b/src/browser/webapi/net/EventSource.zig
@@ -348,8 +348,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
fn corsAllowed(self: *const EventSource, transfer: *Transfer) bool {
var allow_origin: ?[]const u8 = null;
var allow_credentials: ?[]const u8 = null;
- var it = transfer.responseHeaderIterator();
- while (it.next()) |hdr| {
+ for (transfer.responseHeaders()) |hdr| {
if (std.mem.eql(u8, hdr.name, "access-control-allow-origin")) {
allow_origin = hdr.value;
} else if (std.mem.eql(u8, hdr.name, "access-control-allow-credentials")) {
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index be9e9c12b..685adac56 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -282,8 +282,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
res._type = .basic;
}
- var it = transfer.responseHeaderIterator();
- while (it.next()) |hdr| {
+ for (transfer.responseHeaders()) |hdr| {
try res._headers.append(hdr.name, hdr.value, exec);
}
diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig
index a7fa1bc55..2b6c186e8 100644
--- a/src/browser/webapi/net/XMLHttpRequest.zig
+++ b/src/browser/webapi/net/XMLHttpRequest.zig
@@ -417,7 +417,7 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
break;
}
}
- try self.applyResponseHeaders(.{ .list = .{ .list = resp.headers } });
+ try self.applyResponseHeaders(resp.headers);
try self._response_data.appendSlice(self._arena.allocator(), resp.body.items);
@@ -654,9 +654,8 @@ fn applyContentType(self: *XMLHttpRequest, content_type: []const u8) !void {
self._response_mime_raw = try self._arena.dupe(u8, std.mem.trim(u8, content_type, &std.ascii.whitespace));
}
-fn applyResponseHeaders(self: *XMLHttpRequest, headers: http.HeaderIterator) !void {
- var it = headers;
- while (it.next()) |hdr| {
+fn applyResponseHeaders(self: *XMLHttpRequest, headers: []const http.Header) !void {
+ for (headers) |hdr| {
if (Headers.isForbiddenResponseHeaderName(hdr.name)) {
continue;
}
@@ -687,7 +686,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
};
}
- try self.applyResponseHeaders(transfer.responseHeaderIterator());
+ try self.applyResponseHeaders(transfer.responseHeaders());
self._response_status = transfer.responseStatus().?;
if (transfer.statusText()) |st| {
diff --git a/src/network/CorsGate.zig b/src/network/CorsGate.zig
index b6ab991b8..8cc3577bc 100644
--- a/src/network/CorsGate.zig
+++ b/src/network/CorsGate.zig
@@ -491,8 +491,7 @@ const CorsPreflightContext = struct {
var acac: ?[]const u8 = null;
var acma: ?[]const u8 = null;
- var iter = transfer.responseHeaderIterator();
- while (iter.next()) |hdr| {
+ for (transfer.responseHeaders()) |hdr| {
if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_ORIGIN)) {
acao = hdr.value;
} else if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_METHODS)) {
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 2f6f7e92d..760f482cc 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -47,7 +47,6 @@ const Allocator = std.mem.Allocator;
pub const Method = http.Method;
pub const Header = http.Header;
-const HeaderIterator = http.HeaderIterator;
// This is loosely tied to a browser Frame. Loading all the <scripts>, doing
// XHR requests, and loading imports all happens through here. Sine the app
@@ -1466,8 +1465,7 @@ const SyncContext = struct {
}
if (self.options.copy_headers) {
- var it = transfer.responseHeaderIterator();
- while (it.next()) |hdr| {
+ for (transfer.responseHeaders()) |hdr| {
try self.headers.append(allocator, .{
.name = try allocator.dupe(u8, hdr.name),
.value = try allocator.dupe(u8, hdr.value),
@@ -3259,7 +3257,7 @@ pub const Transfer = struct {
.decoded_body_size = decoded_body_size,
.status = status,
.content_type = content_type,
- .content_encoding = findHeader(self.res.headers, "content-encoding") orelse "",
+ .content_encoding = self.responseHeader("content-encoding") orelse "",
.from_cache = t.cache != .none,
.timing_allow = timing_allow,
}) catch |err| {
@@ -3396,9 +3394,7 @@ pub const Transfer = struct {
// in the arena so it survives the conn release.
self.res.header.?.url = (try arena.dupeZ(u8, std.mem.span(self.res.header.?.url))).ptr;
- var it = HeaderIterator{ .curl = .{ .conn = conn } };
- const headers = try it.collect(arena.allocator());
- self.setResponseHeaders(headers.items);
+ self.setResponseHeaders(try conn.collectResponseHeaders(arena.allocator()));
if (self.req.credentialsAllowed()) {
if (self.cookie_jar) |jar| {
@@ -3954,7 +3950,12 @@ pub const Transfer = struct {
}
if (transfer.req.partial == null) {
- if (transfer.getContentLength()) |cl| {
+ // Headers aren't materialized yet; read them off the conn.
+ const content_length: ?usize = blk: {
+ const hdr = conn.getResponseHeader("content-length", 0) orelse break :blk null;
+ break :blk std.fmt.parseInt(usize, hdr.value, 10) catch null;
+ };
+ if (content_length) |cl| {
if (cl > transfer.client.max_response_size) {
res.callback_error = error.ResponseTooLarge;
return http.writefunc_error;
@@ -4088,13 +4089,10 @@ pub const Transfer = struct {
return rh.redirect_count;
}
- pub fn responseHeaderIterator(self: *Transfer) HeaderIterator {
- if (self.res.headers.len > 0 or self._conn == null) {
- return .{ .list = .{ .list = self.res.headers } };
- }
- // Mid-stream (pump time): headers aren't materialized yet, read
- // them off the live conn.
- return .{ .curl = .{ .conn = self._conn.? } };
+ // Response headers are only available once materialized (i.e. from
+ // dispatch). Before that, at pump time, read them off the conn.
+ pub fn responseHeaders(self: *const Transfer) []const http.Header {
+ return self.res.headers;
}
pub fn getContentLength(self: *const Transfer) ?usize {
@@ -4110,22 +4108,10 @@ pub const Transfer = struct {
return self._body_len;
}
- // First value of a response header. `name` must be lowercase.
- pub fn responseHeader(self: *const Transfer, name: [:0]const u8) ?[]const u8 {
- // Materialized headers (dispatch time, any source).
- for (self.res.headers) |hdr| {
- if (std.mem.eql(u8, hdr.name, name)) {
- return hdr.value;
- }
- }
-
- // Mid-stream (curl's write callback): read from the live conn.
- if (self._conn) |c| {
- const value = c.getResponseHeader(name, 0) orelse return null;
- return value.value;
- }
-
- return null;
+ // First value of a response header. `name` must be lowercase. Same
+ // materialization caveat as responseHeaders.
+ pub fn responseHeader(self: *const Transfer, name: []const u8) ?[]const u8 {
+ return findHeader(self.res.headers, name);
}
pub fn botChallenge(self: *const Transfer) ?BotChallenge {
@@ -4344,7 +4330,7 @@ const ResourceTiming = struct {
// Reset on every retry (auth retry, redirect) via Transfer.reset — only
// the cross-retry counters (_auth_challenge, _redirect_count) live on
// Transfer itself. Consumers read it through the accessors above
-// (status / contentType / responseHeaderIterator / getContentLength).
+// (status / contentType / responseHeaders / getContentLength).
const Response = struct {
header: ?http.ResponseHead = null,
@@ -4955,6 +4941,41 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
try testing.expectEqual("yes", headers[2].value);
}
+test "HttpClient: Transfer.botChallenge" {
+ var pool = ArenaPool.init(testing.allocator, .{});
+ defer pool.deinit();
+
+ const arena = try pool.acquire(.small, "test");
+ defer arena.release();
+
+ var transfer = testTransfer(arena);
+ const Case = struct {
+ status: u16,
+ headers: []const http.Header,
+ expected: ?BotChallenge,
+ };
+ const cases = [_]Case{
+ .{ .status = 403, .headers = &.{ .{ .name = "server", .value = "cloudflare" }, .{ .name = "cf-mitigated", .value = "challenge" } }, .expected = .cloudflare },
+ // the header alone isn't enough, the status gates the lookup
+ .{ .status = 200, .headers = &.{.{ .name = "cf-mitigated", .value = "challenge" }}, .expected = null },
+ .{ .status = 403, .headers = &.{.{ .name = "server", .value = "cloudflare" }}, .expected = null },
+ .{ .status = 429, .headers = &.{.{ .name = "x-vercel-mitigated", .value = "challenge" }}, .expected = .vercel },
+ .{ .status = 405, .headers = &.{.{ .name = "x-amzn-waf-action", .value = "captcha" }}, .expected = .aws_waf },
+ .{ .status = 202, .headers = &.{.{ .name = "x-amzn-waf-action", .value = "challenge" }}, .expected = .aws_waf },
+ // a mitigation other than a challenge
+ .{ .status = 405, .headers = &.{.{ .name = "x-amzn-waf-action", .value = "block" }}, .expected = null },
+ };
+
+ // no response yet
+ try testing.expectEqual(null, transfer.botChallenge());
+
+ for (cases) |case| {
+ transfer.setResponseHead(case.status, null);
+ transfer.setResponseHeaders(case.headers);
+ try testing.expectEqual(case.expected, transfer.botChallenge());
+ }
+}
+
test "HttpClient: Transfer.appendHeader combines same-source values" {
var pool = ArenaPool.init(testing.allocator, .{});
defer pool.deinit();
diff --git a/src/network/http.zig b/src/network/http.zig
index 7bd5486f6..b66d3f793 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -136,61 +136,6 @@ pub const Header = struct {
};
};
-// In normal cases, the header iterator comes from the curl connection.
-// But it's also possible to inject a response, via `transfer.fulfill`. In that
-// case, the response headers are a list, []const Http.Header.
-// This union, is an iterator that exposes the same API for either case.
-pub const HeaderIterator = union(enum) {
- curl: CurlHeaderIterator,
- list: ListHeaderIterator,
-
- pub fn next(self: *HeaderIterator) ?Header {
- switch (self.*) {
- inline else => |*it| return it.next(),
- }
- }
-
- pub fn collect(self: *HeaderIterator, allocator: std.mem.Allocator) !std.ArrayList(Header) {
- var list: std.ArrayList(Header) = .empty;
-
- while (self.next()) |hdr| {
- try list.append(allocator, try hdr.normalize(allocator));
- }
-
- return list;
- }
-
- const CurlHeaderIterator = struct {
- conn: *const Connection,
- prev: ?*libcurl.CurlHeader = null,
-
- pub fn next(self: *CurlHeaderIterator) ?Header {
- const h = libcurl.curl_easy_nextheader(self.conn._easy, .header, -1, self.prev) orelse return null;
- self.prev = h;
-
- const header = h.*;
- return .{
- .name = std.mem.span(header.name),
- .value = std.mem.span(header.value),
- };
- }
- };
-
- const ListHeaderIterator = struct {
- index: usize = 0,
- list: []const Header,
-
- pub fn next(self: *ListHeaderIterator) ?Header {
- const idx = self.index;
- if (idx == self.list.len) {
- return null;
- }
- self.index = idx + 1;
- return self.list[idx];
- }
- };
-};
-
const HeaderValue = struct {
value: []const u8,
amount: usize,
@@ -709,6 +654,21 @@ pub const Connection = struct {
};
}
+ // Copies the response headers, names lowercased, into `allocator`.
+ pub fn collectResponseHeaders(self: *const Connection, allocator: std.mem.Allocator) ![]const Header {
+ var list: std.ArrayList(Header) = .empty;
+ var prev: ?*libcurl.CurlHeader = null;
+ while (libcurl.curl_easy_nextheader(self._easy, .header, -1, prev)) |h| {
+ prev = h;
+ const hdr: Header = .{
+ .name = std.mem.span(h.name),
+ .value = std.mem.span(h.value),
+ };
+ try list.append(allocator, try hdr.normalize(allocator));
+ }
+ return list.items;
+ }
+
pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {
var hdr: ?*libcurl.CurlHeader = null;
libcurl.curl_easy_header(self._easy, name, index, .header, -1, &hdr) catch |err| {
diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zig
index d7f1b54d7..7e61e4f3a 100644
--- a/src/server/cdp/domains/network.zig
+++ b/src/server/cdp/domains/network.zig
@@ -679,9 +679,8 @@ const ResponseWriter = struct {
// common to get these from a server (e.g. for Cache-Control), but
// Chrome joins these. So we have to too.
const arena = self.arena;
- var it = transfer.responseHeaderIterator();
var map: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
- while (it.next()) |hdr| {
+ for (transfer.responseHeaders()) |hdr| {
const gop = try map.getOrPut(arena, hdr.name);
if (gop.found_existing) {
// yes, chrome joins multi-value headers with a \n