Lightpanda: the headless browser designed for AI and automation
chore: remove http.HeaderIterator
Karl Seguina60058ad9a3d
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex b2f1d4ce2..11d70286c 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1518,8 +1518,7 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.self._http_status = transfer.responseStatus();self._bot_challenge = transfer.botChallenge();- var it = transfer.responseHeaderIterator();- while (it.next()) |hdr| {+ for (transfer.responseHeaders()) |hdr| {try self._http_headers.append(self.arena, .{.name = try self.arena.dupe(u8, hdr.name),.value = try self.arena.dupe(u8, hdr.value),@@ -1568,8 +1567,7 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {}const disposition: HttpClient.Header = blk: {- var it = transfer.responseHeaderIterator();- while (it.next()) |hdr| {+ for (transfer.responseHeaders()) |hdr| {if (std.mem.eql(u8, hdr.name, "content-disposition")) {break :blk hdr;}diff --git a/src/browser/frame/framing.zig b/src/browser/frame/framing.zigindex 9db52e2d2..f06b6bc6a 100644--- a/src/browser/frame/framing.zig+++ b/src/browser/frame/framing.zig@@ -24,8 +24,7 @@ const HttpClient = @import("../../network/HttpClient.zig");// https://html.spec.whatwg.org/multipage/document-lifecycle.html#the-x-frame-options-headerpub fn allowed(frame: *const Frame, transfer: *HttpClient.Transfer) bool {var options: XFrameOptions = .{};- var it = transfer.responseHeaderIterator();- while (it.next()) |hdr| {+ for (transfer.responseHeaders()) |hdr| {if (std.mem.eql(u8, hdr.name, "content-security-policy")) {if (hasFrameAncestors(hdr.value)) {// has priority over any x-frame-optionsdiff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zigindex b74257353..f1f68a905 100644--- a/src/browser/webapi/net/EventSource.zig+++ b/src/browser/webapi/net/EventSource.zig@@ -348,8 +348,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {fn corsAllowed(self: *const EventSource, transfer: *Transfer) bool {var allow_origin: ?[]const u8 = null;var allow_credentials: ?[]const u8 = null;- var it = transfer.responseHeaderIterator();- while (it.next()) |hdr| {+ for (transfer.responseHeaders()) |hdr| {if (std.mem.eql(u8, hdr.name, "access-control-allow-origin")) {allow_origin = hdr.value;} else if (std.mem.eql(u8, hdr.name, "access-control-allow-credentials")) {diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex be9e9c12b..685adac56 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -282,8 +282,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {res._type = .basic;}- var it = transfer.responseHeaderIterator();- while (it.next()) |hdr| {+ for (transfer.responseHeaders()) |hdr| {try res._headers.append(hdr.name, hdr.value, exec);}diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zigindex a7fa1bc55..2b6c186e8 100644--- a/src/browser/webapi/net/XMLHttpRequest.zig+++ b/src/browser/webapi/net/XMLHttpRequest.zig@@ -417,7 +417,7 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !vbreak;}}- try self.applyResponseHeaders(.{ .list = .{ .list = resp.headers } });+ try self.applyResponseHeaders(resp.headers);try self._response_data.appendSlice(self._arena.allocator(), resp.body.items);@@ -654,9 +654,8 @@ fn applyContentType(self: *XMLHttpRequest, content_type: []const u8) !void {self._response_mime_raw = try self._arena.dupe(u8, std.mem.trim(u8, content_type, &std.ascii.whitespace));}-fn applyResponseHeaders(self: *XMLHttpRequest, headers: http.HeaderIterator) !void {- var it = headers;- while (it.next()) |hdr| {+fn applyResponseHeaders(self: *XMLHttpRequest, headers: []const http.Header) !void {+ for (headers) |hdr| {if (Headers.isForbiddenResponseHeaderName(hdr.name)) {continue;}@@ -687,7 +686,7 @@ fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {};}- try self.applyResponseHeaders(transfer.responseHeaderIterator());+ try self.applyResponseHeaders(transfer.responseHeaders());self._response_status = transfer.responseStatus().?;if (transfer.statusText()) |st| {diff --git a/src/network/CorsGate.zig b/src/network/CorsGate.zigindex b6ab991b8..8cc3577bc 100644--- a/src/network/CorsGate.zig+++ b/src/network/CorsGate.zig@@ -491,8 +491,7 @@ const CorsPreflightContext = struct {var acac: ?[]const u8 = null;var acma: ?[]const u8 = null;- var iter = transfer.responseHeaderIterator();- while (iter.next()) |hdr| {+ for (transfer.responseHeaders()) |hdr| {if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_ORIGIN)) {acao = hdr.value;} else if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_METHODS)) {diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 2f6f7e92d..760f482cc 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -47,7 +47,6 @@ const Allocator = std.mem.Allocator;pub const Method = http.Method;pub const Header = http.Header;-const HeaderIterator = http.HeaderIterator;// This is loosely tied to a browser Frame. Loading all the <scripts>, doing// XHR requests, and loading imports all happens through here. Sine the app@@ -1466,8 +1465,7 @@ const SyncContext = struct {}if (self.options.copy_headers) {- var it = transfer.responseHeaderIterator();- while (it.next()) |hdr| {+ for (transfer.responseHeaders()) |hdr| {try self.headers.append(allocator, .{.name = try allocator.dupe(u8, hdr.name),.value = try allocator.dupe(u8, hdr.value),@@ -3259,7 +3257,7 @@ pub const Transfer = struct {.decoded_body_size = decoded_body_size,.status = status,.content_type = content_type,- .content_encoding = findHeader(self.res.headers, "content-encoding") orelse "",+ .content_encoding = self.responseHeader("content-encoding") orelse "",.from_cache = t.cache != .none,.timing_allow = timing_allow,}) catch |err| {@@ -3396,9 +3394,7 @@ pub const Transfer = struct {// in the arena so it survives the conn release.self.res.header.?.url = (try arena.dupeZ(u8, std.mem.span(self.res.header.?.url))).ptr;- var it = HeaderIterator{ .curl = .{ .conn = conn } };- const headers = try it.collect(arena.allocator());- self.setResponseHeaders(headers.items);+ self.setResponseHeaders(try conn.collectResponseHeaders(arena.allocator()));if (self.req.credentialsAllowed()) {if (self.cookie_jar) |jar| {@@ -3954,7 +3950,12 @@ pub const Transfer = struct {}if (transfer.req.partial == null) {- if (transfer.getContentLength()) |cl| {+ // Headers aren't materialized yet; read them off the conn.+ const content_length: ?usize = blk: {+ const hdr = conn.getResponseHeader("content-length", 0) orelse break :blk null;+ break :blk std.fmt.parseInt(usize, hdr.value, 10) catch null;+ };+ if (content_length) |cl| {if (cl > transfer.client.max_response_size) {res.callback_error = error.ResponseTooLarge;return http.writefunc_error;@@ -4088,13 +4089,10 @@ pub const Transfer = struct {return rh.redirect_count;}- pub fn responseHeaderIterator(self: *Transfer) HeaderIterator {- if (self.res.headers.len > 0 or self._conn == null) {- return .{ .list = .{ .list = self.res.headers } };- }- // Mid-stream (pump time): headers aren't materialized yet, read- // them off the live conn.- return .{ .curl = .{ .conn = self._conn.? } };+ // Response headers are only available once materialized (i.e. from+ // dispatch). Before that, at pump time, read them off the conn.+ pub fn responseHeaders(self: *const Transfer) []const http.Header {+ return self.res.headers;}pub fn getContentLength(self: *const Transfer) ?usize {@@ -4110,22 +4108,10 @@ pub const Transfer = struct {return self._body_len;}- // First value of a response header. `name` must be lowercase.- pub fn responseHeader(self: *const Transfer, name: [:0]const u8) ?[]const u8 {- // Materialized headers (dispatch time, any source).- for (self.res.headers) |hdr| {- if (std.mem.eql(u8, hdr.name, name)) {- return hdr.value;- }- }-- // Mid-stream (curl's write callback): read from the live conn.- if (self._conn) |c| {- const value = c.getResponseHeader(name, 0) orelse return null;- return value.value;- }-- return null;+ // First value of a response header. `name` must be lowercase. Same+ // materialization caveat as responseHeaders.+ pub fn responseHeader(self: *const Transfer, name: []const u8) ?[]const u8 {+ return findHeader(self.res.headers, name);}pub fn botChallenge(self: *const Transfer) ?BotChallenge {@@ -4344,7 +4330,7 @@ const ResourceTiming = struct {// Reset on every retry (auth retry, redirect) via Transfer.reset — only// the cross-retry counters (_auth_challenge, _redirect_count) live on// Transfer itself. Consumers read it through the accessors above-// (status / contentType / responseHeaderIterator / getContentLength).+// (status / contentType / responseHeaders / getContentLength).const Response = struct {header: ?http.ResponseHead = null,@@ -4955,6 +4941,41 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {try testing.expectEqual("yes", headers[2].value);}+test "HttpClient: Transfer.botChallenge" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ const arena = try pool.acquire(.small, "test");+ defer arena.release();++ var transfer = testTransfer(arena);+ const Case = struct {+ status: u16,+ headers: []const http.Header,+ expected: ?BotChallenge,+ };+ const cases = [_]Case{+ .{ .status = 403, .headers = &.{ .{ .name = "server", .value = "cloudflare" }, .{ .name = "cf-mitigated", .value = "challenge" } }, .expected = .cloudflare },+ // the header alone isn't enough, the status gates the lookup+ .{ .status = 200, .headers = &.{.{ .name = "cf-mitigated", .value = "challenge" }}, .expected = null },+ .{ .status = 403, .headers = &.{.{ .name = "server", .value = "cloudflare" }}, .expected = null },+ .{ .status = 429, .headers = &.{.{ .name = "x-vercel-mitigated", .value = "challenge" }}, .expected = .vercel },+ .{ .status = 405, .headers = &.{.{ .name = "x-amzn-waf-action", .value = "captcha" }}, .expected = .aws_waf },+ .{ .status = 202, .headers = &.{.{ .name = "x-amzn-waf-action", .value = "challenge" }}, .expected = .aws_waf },+ // a mitigation other than a challenge+ .{ .status = 405, .headers = &.{.{ .name = "x-amzn-waf-action", .value = "block" }}, .expected = null },+ };++ // no response yet+ try testing.expectEqual(null, transfer.botChallenge());++ for (cases) |case| {+ transfer.setResponseHead(case.status, null);+ transfer.setResponseHeaders(case.headers);+ try testing.expectEqual(case.expected, transfer.botChallenge());+ }+}+test "HttpClient: Transfer.appendHeader combines same-source values" {var pool = ArenaPool.init(testing.allocator, .{});defer pool.deinit();diff --git a/src/network/http.zig b/src/network/http.zigindex 7bd5486f6..b66d3f793 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -136,61 +136,6 @@ pub const Header = struct {};};-// In normal cases, the header iterator comes from the curl connection.-// But it's also possible to inject a response, via `transfer.fulfill`. In that-// case, the response headers are a list, []const Http.Header.-// This union, is an iterator that exposes the same API for either case.-pub const HeaderIterator = union(enum) {- curl: CurlHeaderIterator,- list: ListHeaderIterator,-- pub fn next(self: *HeaderIterator) ?Header {- switch (self.*) {- inline else => |*it| return it.next(),- }- }-- pub fn collect(self: *HeaderIterator, allocator: std.mem.Allocator) !std.ArrayList(Header) {- var list: std.ArrayList(Header) = .empty;-- while (self.next()) |hdr| {- try list.append(allocator, try hdr.normalize(allocator));- }-- return list;- }-- const CurlHeaderIterator = struct {- conn: *const Connection,- prev: ?*libcurl.CurlHeader = null,-- pub fn next(self: *CurlHeaderIterator) ?Header {- const h = libcurl.curl_easy_nextheader(self.conn._easy, .header, -1, self.prev) orelse return null;- self.prev = h;-- const header = h.*;- return .{- .name = std.mem.span(header.name),- .value = std.mem.span(header.value),- };- }- };-- const ListHeaderIterator = struct {- index: usize = 0,- list: []const Header,-- pub fn next(self: *ListHeaderIterator) ?Header {- const idx = self.index;- if (idx == self.list.len) {- return null;- }- self.index = idx + 1;- return self.list[idx];- }- };-};-const HeaderValue = struct {value: []const u8,amount: usize,@@ -709,6 +654,21 @@ pub const Connection = struct {};}+ // Copies the response headers, names lowercased, into `allocator`.+ pub fn collectResponseHeaders(self: *const Connection, allocator: std.mem.Allocator) ![]const Header {+ var list: std.ArrayList(Header) = .empty;+ var prev: ?*libcurl.CurlHeader = null;+ while (libcurl.curl_easy_nextheader(self._easy, .header, -1, prev)) |h| {+ prev = h;+ const hdr: Header = .{+ .name = std.mem.span(h.name),+ .value = std.mem.span(h.value),+ };+ try list.append(allocator, try hdr.normalize(allocator));+ }+ return list.items;+ }+pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {var hdr: ?*libcurl.CurlHeader = null;libcurl.curl_easy_header(self._easy, name, index, .header, -1, &hdr) catch |err| {diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zigindex d7f1b54d7..7e61e4f3a 100644--- a/src/server/cdp/domains/network.zig+++ b/src/server/cdp/domains/network.zig@@ -679,9 +679,8 @@ const ResponseWriter = struct {// common to get these from a server (e.g. for Cache-Control), but// Chrome joins these. So we have to too.const arena = self.arena;- var it = transfer.responseHeaderIterator();var map: std.StringArrayHashMapUnmanaged([]const u8) = .empty;- while (it.next()) |hdr| {+ for (transfer.responseHeaders()) |hdr| {const gop = try map.getOrPut(arena, hdr.name);if (gop.found_existing) {// yes, chrome joins multi-value headers with a \n