Lightpanda: the headless browser designed for AI and automation
http: lowercase response headers
Karl Seguine5ecdf1fe3d5
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 92c6945a0..de1e0947e 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1499,7 +1499,7 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer..name = try self.arena.dupe(u8, hdr.name),.value = try self.arena.dupe(u8, hdr.value),});- if (std.ascii.eqlIgnoreCase(hdr.name, "referrer-policy")) {+ if (std.mem.eql(u8, hdr.name, "referrer-policy")) {if (referrer.parseHeader(hdr.value)) |rp| {self.referrer_policy = rp;}@@ -1545,7 +1545,7 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {const disposition: HttpClient.Header = blk: {var it = transfer.responseHeaderIterator();while (it.next()) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "content-disposition")) {+ if (std.mem.eql(u8, hdr.name, "content-disposition")) {break :blk hdr;}}diff --git a/src/browser/frame/framing.zig b/src/browser/frame/framing.zigindex f16f3ad82..9db52e2d2 100644--- a/src/browser/frame/framing.zig+++ b/src/browser/frame/framing.zig@@ -26,12 +26,12 @@ pub fn allowed(frame: *const Frame, transfer: *HttpClient.Transfer) bool {var options: XFrameOptions = .{};var it = transfer.responseHeaderIterator();while (it.next()) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "content-security-policy")) {+ if (std.mem.eql(u8, hdr.name, "content-security-policy")) {if (hasFrameAncestors(hdr.value)) {// has priority over any x-frame-optionsreturn true;}- } else if (std.ascii.eqlIgnoreCase(hdr.name, "x-frame-options")) {+ } else if (std.mem.eql(u8, hdr.name, "x-frame-options")) {options.add(hdr.value);}}diff --git a/src/browser/structured_data.zig b/src/browser/structured_data.zigindex f3f29ce25..b89763261 100644--- a/src/browser/structured_data.zig+++ b/src/browser/structured_data.zig@@ -236,7 +236,7 @@ fn collectLinkHeaders(const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" };for (frame._http_headers.items) |header| {- if (!std.ascii.eqlIgnoreCase(header.name, "link")) {+ if (std.mem.eql(u8, header.name, "link") == false) {continue;}@@ -705,7 +705,7 @@ test "structured_data: link headers from response" {defer testing.test_session.closeAllPages();// Stand in for what frameHeaderDoneCallback records from the navigation.- try frame._http_headers.append(frame.arena, .{ .name = "Link", .value =+ try frame._http_headers.append(frame.arena, .{ .name = "link", .value =\\<https://docs.example.com/>; rel="service-doc"});try frame._http_headers.append(frame.arena, .{ .name = "link", .value =diff --git a/src/browser/tests/net/xhr.html b/src/browser/tests/net/xhr.htmlindex 6e944c946..0974098cd 100644--- a/src/browser/tests/net/xhr.html+++ b/src/browser/tests/net/xhr.html@@ -36,7 +36,7 @@testing.expectEqual(200, req.status);testing.expectEqual('OK', req.statusText);testing.expectEqual('text/html; charset=utf-8', req.getResponseHeader('Content-Type'));- testing.expectEqual('content-length: 100\r\nContent-Type: text/html; charset=utf-8\r\n', req.getAllResponseHeaders());+ testing.expectEqual('content-length: 100\r\ncontent-type: text/html; charset=utf-8\r\n', req.getAllResponseHeaders());testing.expectEqual(100, req.responseText.length);testing.expectEqual(req.responseText.length, req.response.length);testing.expectEqual('http://127.0.0.1:9582/xhr', req.responseURL);diff --git a/src/browser/webapi/Document.zig b/src/browser/webapi/Document.zigindex 226add2d6..cd770dd39 100644--- a/src/browser/webapi/Document.zig+++ b/src/browser/webapi/Document.zig@@ -197,7 +197,7 @@ fn getLastModified(self: *const Document, frame: *Frame) ![]const u8 {const timestamp = blk: {if (self._frame) |owner| {for (owner._http_headers.items) |header| {- if (std.ascii.eqlIgnoreCase(header.name, "last-modified")) {+ if (std.mem.eql(u8, header.name, "last-modified")) {if (dt.DateTime.parse(header.value, .rfc822)) |parsed| {break :blk parsed.unix(.seconds);} else |_| {}diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zigindex 989204f4d..14371d1d7 100644--- a/src/browser/webapi/net/EventSource.zig+++ b/src/browser/webapi/net/EventSource.zig@@ -350,9 +350,9 @@ fn corsAllowed(self: *const EventSource, transfer: *Transfer) bool {var allow_credentials: ?[]const u8 = null;var it = transfer.responseHeaderIterator();while (it.next()) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-origin")) {+ if (std.mem.eql(u8, hdr.name, "access-control-allow-origin")) {allow_origin = hdr.value;- } else if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-credentials")) {+ } else if (std.mem.eql(u8, hdr.name, "access-control-allow-credentials")) {allow_credentials = hdr.value;}}diff --git a/src/network/CorsGate.zig b/src/network/CorsGate.zigindex 7302ab3b9..8efbe0f45 100644--- a/src/network/CorsGate.zig+++ b/src/network/CorsGate.zig@@ -396,13 +396,13 @@ const CorsPreflightContext = struct {var iter = transfer.responseHeaderIterator();while (iter.next()) |hdr| {- if (std.ascii.eqlIgnoreCase(ACCESS_CONTROL_ALLOW_ORIGIN, hdr.name)) {+ if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_ORIGIN)) {acao = hdr.value;- } else if (std.ascii.eqlIgnoreCase(ACCESS_CONTROL_ALLOW_METHODS, hdr.name)) {+ } else if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_METHODS)) {acam = hdr.value;- } else if (std.ascii.eqlIgnoreCase(ACCESS_CONTROL_ALLOW_HEADERS, hdr.name)) {+ } else if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_HEADERS)) {acah = hdr.value;- } else if (std.ascii.eqlIgnoreCase(ACCESS_CONTROL_ALLOW_CREDENTIALS, hdr.name)) {+ } else if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_CREDENTIALS)) {acac = hdr.value;}}diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 45e155065..695209ff8 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1207,9 +1207,18 @@ pub fn resumeAfterCors(self: *Client, transfer: *Transfer) !void {return self.pipeline(transfer, .after_cors);}+// `headers` are response headers (lowercased names); `name` must be lowercase.pub fn findHeader(headers: []const http.Header, name: []const u8) ?[]const u8 {+ if (comptime lp.IS_DEBUG) {+ for (name) |c| {+ if (std.ascii.isUpper(c)) {+ log.fatal(.bug, "non-lowercase header", .{ .name = name });+ @panic("non-lowercase-header");+ }+ }+ }for (headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, name)) {+ if (std.mem.eql(u8, hdr.name, name)) {return hdr.value;}}@@ -2118,12 +2127,19 @@ pub fn fulfillIntercepted(transfer.unpark();+ // Copied into the transfer arena with lowercased names, like headers+ // materialized from curl.+ const owned = transfer.dupeHeadersLower(headers) catch |err| {+ transfer.abortPipelineError(err);+ return err;+ };+const followed = blk: {if (isRedirectStatus(status) == false) {break :blk false;}- const location = findHeader(headers, "location") orelse break :blk false;- try self.fulfillRedirect(transfer, status, headers, location);+ const location = findHeader(owned, "location") orelse break :blk false;+ try self.fulfillRedirect(transfer, status, owned, location);break :blk true;};@@ -2131,7 +2147,7 @@ pub fn fulfillIntercepted(return;}- transfer.bufferFulfilled(status, headers, body) catch |err| {+ transfer.bufferFulfilled(status, owned, body) catch |err| {transfer.abortPipelineError(err);return err;};@@ -2150,7 +2166,7 @@ fn fulfillRedirect(if (transfer.req.credentialsAllowed()) {if (transfer.cookie_jar) |jar| {for (headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {+ if (std.mem.eql(u8, hdr.name, "set-cookie")) {try jar.populateFromResponse(transfer.req.url, hdr.value);}}@@ -3064,7 +3080,7 @@ pub const Transfer = struct {return true;}for (headers) |hdr| {- if (!std.ascii.eqlIgnoreCase(hdr.name, "timing-allow-origin")) {+ if (std.mem.eql(u8, hdr.name, "timing-allow-origin") == false) {continue;}var it = std.mem.splitScalar(u8, hdr.value, ',');@@ -3192,6 +3208,18 @@ pub const Transfer = struct {self.scheduleDispatch();}+ // Response header names are lowercase, so lookups use std.mem.eql.+ fn setResponseHeaders(self: *Transfer, headers: []const http.Header) void {+ if (comptime lp.IS_DEBUG) {+ for (headers) |hdr| {+ for (hdr.name) |c| {+ std.debug.assert(std.ascii.isUpper(c) == false);+ }+ }+ }+ self.res.headers = headers;+ }+fn setResponseHead(self: *Transfer, status: u16, content_type: ?[]const u8) void {self.res.header = .{.url = self.req.url.ptr,@@ -3214,7 +3242,7 @@ pub const Transfer = struct {};self.setResponseHead(cached.status, cached.content_type);- self.res.headers = cached.headers;+ self.setResponseHeaders(cached.headers);self._from_cache = true;self._timing.cache = cache_state;self._content_length = body.len;@@ -3222,27 +3250,29 @@ pub const Transfer = struct {}// Materialize an interceptor-supplied response (CDP fulfillRequest).- // `headers` and `body` are caller-owned; copy everything that must- // survive until dispatch.+ fn dupeHeadersLower(self: *Transfer, headers: []const http.Header) ![]const http.Header {+ const allocator = self.arena.allocator();+ const owned = try allocator.alloc(http.Header, headers.len);+ for (headers, owned) |hdr, *o| {+ o.* = try hdr.normalize(allocator);+ }+ return owned;+ }++ // our arena already owns `header` but not `body`. Why so complicated?fn bufferFulfilled(self: *Transfer, status: u16, headers: []const http.Header, body: ?[]const u8) !void {const arena = self.arena;-- const owned = try arena.alloc(http.Header, headers.len);var content_type: ?[]const u8 = null;- for (headers, 0..) |hdr, i| {- owned[i] = .{- .name = try arena.dupe(u8, hdr.name),- .value = try arena.dupe(u8, hdr.value),- };- if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) {- content_type = owned[i].value;+ for (headers) |hdr| {+ if (std.mem.eql(u8, hdr.name, "content-type")) {+ content_type = hdr.value;}}const owned_body: []const u8 = if (body) |b| try arena.dupe(u8, b) else "";self.setResponseHead(status, content_type);- self.res.headers = owned;+ self.setResponseHeaders(headers);self._content_length = owned_body.len;try self.bufferEvents(owned_body);}@@ -3278,12 +3308,12 @@ pub const Transfer = struct {var it = HeaderIterator{ .curl = .{ .conn = conn } };const headers = try it.collect(arena.allocator());- self.res.headers = headers.items;+ self.setResponseHeaders(headers.items);if (self.req.credentialsAllowed()) {if (self.cookie_jar) |jar| {for (self.res.headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {+ if (std.mem.eql(u8, hdr.name, "set-cookie")) {jar.populateFromResponse(self.req.url, hdr.value) catch |err| {log.err(.http, "set cookie", .{ .err = err, .req = self });return err;@@ -3973,7 +4003,7 @@ pub const Transfer = struct {fn getContentLengthRawValue(self: *const Transfer) ?[]const u8 {// Materialized headers (dispatch time, any source).for (self.res.headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "content-length")) {+ if (std.mem.eql(u8, hdr.name, "content-length")) {return hdr.value;}}@@ -4178,7 +4208,8 @@ const Response = struct {header: ?http.ResponseHead = null,// Full response headers, materialized into the transfer arena at- // completion (or set directly by cache / synthetic / fulfill).+ // completion (or set directly by cache / synthetic / fulfill). Names are+ // lowercased.headers: []const http.Header = &.{},// total bytes received in the response, including the response status@@ -4285,7 +4316,7 @@ const Synthetic = struct {if (content_type.len > 0) {const h = try arena.alloc(http.Header, 1);h[0] = .{ .name = "content-type", .value = content_type };- transfer.res.headers = h;+ transfer.setResponseHeaders(h);}transfer._content_length = body.len;try transfer.bufferEvents(body);diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zigindex 9ae32bfd7..9a667fde3 100644--- a/src/network/cache/Cache.zig+++ b/src/network/cache/Cache.zig@@ -320,33 +320,33 @@ const ResponseHeaders = struct {for (headers) |h| {switch (h.name.len) {- 3 => if (std.ascii.eqlIgnoreCase(h.name, "Age")) {+ 3 => if (std.mem.eql(u8, h.name, "age")) {self.age = h.value;},4 => {- if (std.ascii.eqlIgnoreCase(h.name, "Date")) {+ if (std.mem.eql(u8, h.name, "date")) {self.date = h.value;- } else if (std.ascii.eqlIgnoreCase(h.name, "ETag")) {+ } else if (std.mem.eql(u8, h.name, "etag")) {self.etag = h.value;- } else if (std.ascii.eqlIgnoreCase(h.name, "Vary")) {+ } else if (std.mem.eql(u8, h.name, "vary")) {self.vary = h.value;}},- 7 => if (std.ascii.eqlIgnoreCase(h.name, "Expires")) {+ 7 => if (std.mem.eql(u8, h.name, "expires")) {self.expires = h.value;},- 10 => if (std.ascii.eqlIgnoreCase(h.name, "Set-Cookie")) {+ 10 => if (std.mem.eql(u8, h.name, "set-cookie")) {self.has_set_cookie = true;},- 12 => if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {+ 12 => if (std.mem.eql(u8, h.name, "content-type")) {self.content_type = h.value;},13 => {- if (std.ascii.eqlIgnoreCase(h.name, "Cache-Control")) {+ if (std.mem.eql(u8, h.name, "cache-control")) {self.directives = .parse(h.value);- } else if (std.ascii.eqlIgnoreCase(h.name, "Last-Modified")) {+ } else if (std.mem.eql(u8, h.name, "last-modified")) {self.last_modified = h.value;- } else if (std.ascii.eqlIgnoreCase(h.name, "Authorization")) {+ } else if (std.mem.eql(u8, h.name, "authorization")) {self.has_authorization = true;}},@@ -576,11 +576,11 @@ const TestResponse = struct {fn run(self: TestResponse, arena: std.mem.Allocator) !?CachePutRequest {var headers: std.ArrayList(Http.Header) = .empty;inline for (.{- .{ "Cache-Control", self.cache_control },- .{ "Expires", self.expires },- .{ "Date", self.date },- .{ "ETag", self.etag },- .{ "Last-Modified", self.last_modified },+ .{ "cache-control", self.cache_control },+ .{ "expires", self.expires },+ .{ "date", self.date },+ .{ "etag", self.etag },+ .{ "last-modified", self.last_modified },}) |field| {if (field[1]) |value| {try headers.append(arena, .{ .name = field[0], .value = value });@@ -721,8 +721,8 @@ test "Cache: tryCache vary headers" {.status = 200,.content_type = "text/html",.headers = &.{- .{ .name = "Cache-Control", .value = "max-age=300" },- .{ .name = "Vary", .value = "accept-encoding, accept-language" },+ .{ .name = "cache-control", .value = "max-age=300" },+ .{ .name = "vary", .value = "accept-encoding, accept-language" },},.request_headers = &request_headers,});@@ -738,8 +738,8 @@ test "Cache: tryCache vary headers" {.status = 200,.content_type = "text/html",.headers = &.{- .{ .name = "Cache-Control", .value = "max-age=300" },- .{ .name = "Vary", .value = "*" },+ .{ .name = "cache-control", .value = "max-age=300" },+ .{ .name = "vary", .value = "*" },},.request_headers = &request_headers,});diff --git a/src/network/http.zig b/src/network/http.zigindex 4a1418df0..43ba5bef0 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -71,6 +71,13 @@ pub const Header = struct {value: []const u8,};+ pub fn normalize(self: Header, allocator: std.mem.Allocator) !Header {+ return .{+ .name = try std.ascii.allocLowerString(allocator, self.name),+ .value = try allocator.dupe(u8, self.value),+ };+ }+pub fn parse(header_str: []const u8) ?Header {const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;@@ -147,10 +154,7 @@ pub const HeaderIterator = union(enum) {var list: std.ArrayList(Header) = .empty;while (self.next()) |hdr| {- try list.append(allocator, .{- .name = try allocator.dupe(u8, hdr.name),- .value = try allocator.dupe(u8, hdr.value),- });+ try list.append(allocator, try hdr.normalize(allocator));}return list;