Skip to content

Lightpanda: the headless browser designed for AI and automation

http: lowercase response headers

Karl Seguine5ecdf1fe3d5
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 92c6945a0..de1e0947e 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -1499,7 +1499,7 @@ fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.
.name = try self.arena.dupe(u8, hdr.name),
.value = try self.arena.dupe(u8, hdr.value),
});
- if (std.ascii.eqlIgnoreCase(hdr.name, "referrer-policy")) {
+ if (std.mem.eql(u8, hdr.name, "referrer-policy")) {
if (referrer.parseHeader(hdr.value)) |rp| {
self.referrer_policy = rp;
}
@@ -1545,7 +1545,7 @@ fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {
const disposition: HttpClient.Header = blk: {
var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "content-disposition")) {
+ if (std.mem.eql(u8, hdr.name, "content-disposition")) {
break :blk hdr;
}
}
diff --git a/src/browser/frame/framing.zig b/src/browser/frame/framing.zig
index f16f3ad82..9db52e2d2 100644
--- a/src/browser/frame/framing.zig
+++ b/src/browser/frame/framing.zig
@@ -26,12 +26,12 @@ pub fn allowed(frame: *const Frame, transfer: *HttpClient.Transfer) bool {
var options: XFrameOptions = .{};
var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "content-security-policy")) {
+ if (std.mem.eql(u8, hdr.name, "content-security-policy")) {
if (hasFrameAncestors(hdr.value)) {
// has priority over any x-frame-options
return true;
}
- } else if (std.ascii.eqlIgnoreCase(hdr.name, "x-frame-options")) {
+ } else if (std.mem.eql(u8, hdr.name, "x-frame-options")) {
options.add(hdr.value);
}
}
diff --git a/src/browser/structured_data.zig b/src/browser/structured_data.zig
index f3f29ce25..b89763261 100644
--- a/src/browser/structured_data.zig
+++ b/src/browser/structured_data.zig
@@ -236,7 +236,7 @@ fn collectLinkHeaders(
const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" };
for (frame._http_headers.items) |header| {
- if (!std.ascii.eqlIgnoreCase(header.name, "link")) {
+ if (std.mem.eql(u8, header.name, "link") == false) {
continue;
}
@@ -705,7 +705,7 @@ test "structured_data: link headers from response" {
defer testing.test_session.closeAllPages();
// Stand in for what frameHeaderDoneCallback records from the navigation.
- try frame._http_headers.append(frame.arena, .{ .name = "Link", .value =
+ try frame._http_headers.append(frame.arena, .{ .name = "link", .value =
\\<https://docs.example.com/>; rel="service-doc"
});
try frame._http_headers.append(frame.arena, .{ .name = "link", .value =
diff --git a/src/browser/tests/net/xhr.html b/src/browser/tests/net/xhr.html
index 6e944c946..0974098cd 100644
--- a/src/browser/tests/net/xhr.html
+++ b/src/browser/tests/net/xhr.html
@@ -36,7 +36,7 @@
testing.expectEqual(200, req.status);
testing.expectEqual('OK', req.statusText);
testing.expectEqual('text/html; charset=utf-8', req.getResponseHeader('Content-Type'));
- testing.expectEqual('content-length: 100\r\nContent-Type: text/html; charset=utf-8\r\n', req.getAllResponseHeaders());
+ testing.expectEqual('content-length: 100\r\ncontent-type: text/html; charset=utf-8\r\n', req.getAllResponseHeaders());
testing.expectEqual(100, req.responseText.length);
testing.expectEqual(req.responseText.length, req.response.length);
testing.expectEqual('http://127.0.0.1:9582/xhr', req.responseURL);
diff --git a/src/browser/webapi/Document.zig b/src/browser/webapi/Document.zig
index 226add2d6..cd770dd39 100644
--- a/src/browser/webapi/Document.zig
+++ b/src/browser/webapi/Document.zig
@@ -197,7 +197,7 @@ fn getLastModified(self: *const Document, frame: *Frame) ![]const u8 {
const timestamp = blk: {
if (self._frame) |owner| {
for (owner._http_headers.items) |header| {
- if (std.ascii.eqlIgnoreCase(header.name, "last-modified")) {
+ if (std.mem.eql(u8, header.name, "last-modified")) {
if (dt.DateTime.parse(header.value, .rfc822)) |parsed| {
break :blk parsed.unix(.seconds);
} else |_| {}
diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zig
index 989204f4d..14371d1d7 100644
--- a/src/browser/webapi/net/EventSource.zig
+++ b/src/browser/webapi/net/EventSource.zig
@@ -350,9 +350,9 @@ fn corsAllowed(self: *const EventSource, transfer: *Transfer) bool {
var allow_credentials: ?[]const u8 = null;
var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-origin")) {
+ if (std.mem.eql(u8, hdr.name, "access-control-allow-origin")) {
allow_origin = hdr.value;
- } else if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-credentials")) {
+ } else if (std.mem.eql(u8, hdr.name, "access-control-allow-credentials")) {
allow_credentials = hdr.value;
}
}
diff --git a/src/network/CorsGate.zig b/src/network/CorsGate.zig
index 7302ab3b9..8efbe0f45 100644
--- a/src/network/CorsGate.zig
+++ b/src/network/CorsGate.zig
@@ -396,13 +396,13 @@ const CorsPreflightContext = struct {
var iter = transfer.responseHeaderIterator();
while (iter.next()) |hdr| {
- if (std.ascii.eqlIgnoreCase(ACCESS_CONTROL_ALLOW_ORIGIN, hdr.name)) {
+ if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_ORIGIN)) {
acao = hdr.value;
- } else if (std.ascii.eqlIgnoreCase(ACCESS_CONTROL_ALLOW_METHODS, hdr.name)) {
+ } else if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_METHODS)) {
acam = hdr.value;
- } else if (std.ascii.eqlIgnoreCase(ACCESS_CONTROL_ALLOW_HEADERS, hdr.name)) {
+ } else if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_HEADERS)) {
acah = hdr.value;
- } else if (std.ascii.eqlIgnoreCase(ACCESS_CONTROL_ALLOW_CREDENTIALS, hdr.name)) {
+ } else if (std.mem.eql(u8, hdr.name, ACCESS_CONTROL_ALLOW_CREDENTIALS)) {
acac = hdr.value;
}
}
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 45e155065..695209ff8 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1207,9 +1207,18 @@ pub fn resumeAfterCors(self: *Client, transfer: *Transfer) !void {
return self.pipeline(transfer, .after_cors);
}
+// `headers` are response headers (lowercased names); `name` must be lowercase.
pub fn findHeader(headers: []const http.Header, name: []const u8) ?[]const u8 {
+ if (comptime lp.IS_DEBUG) {
+ for (name) |c| {
+ if (std.ascii.isUpper(c)) {
+ log.fatal(.bug, "non-lowercase header", .{ .name = name });
+ @panic("non-lowercase-header");
+ }
+ }
+ }
for (headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
+ if (std.mem.eql(u8, hdr.name, name)) {
return hdr.value;
}
}
@@ -2118,12 +2127,19 @@ pub fn fulfillIntercepted(
transfer.unpark();
+ // Copied into the transfer arena with lowercased names, like headers
+ // materialized from curl.
+ const owned = transfer.dupeHeadersLower(headers) catch |err| {
+ transfer.abortPipelineError(err);
+ return err;
+ };
+
const followed = blk: {
if (isRedirectStatus(status) == false) {
break :blk false;
}
- const location = findHeader(headers, "location") orelse break :blk false;
- try self.fulfillRedirect(transfer, status, headers, location);
+ const location = findHeader(owned, "location") orelse break :blk false;
+ try self.fulfillRedirect(transfer, status, owned, location);
break :blk true;
};
@@ -2131,7 +2147,7 @@ pub fn fulfillIntercepted(
return;
}
- transfer.bufferFulfilled(status, headers, body) catch |err| {
+ transfer.bufferFulfilled(status, owned, body) catch |err| {
transfer.abortPipelineError(err);
return err;
};
@@ -2150,7 +2166,7 @@ fn fulfillRedirect(
if (transfer.req.credentialsAllowed()) {
if (transfer.cookie_jar) |jar| {
for (headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {
+ if (std.mem.eql(u8, hdr.name, "set-cookie")) {
try jar.populateFromResponse(transfer.req.url, hdr.value);
}
}
@@ -3064,7 +3080,7 @@ pub const Transfer = struct {
return true;
}
for (headers) |hdr| {
- if (!std.ascii.eqlIgnoreCase(hdr.name, "timing-allow-origin")) {
+ if (std.mem.eql(u8, hdr.name, "timing-allow-origin") == false) {
continue;
}
var it = std.mem.splitScalar(u8, hdr.value, ',');
@@ -3192,6 +3208,18 @@ pub const Transfer = struct {
self.scheduleDispatch();
}
+ // Response header names are lowercase, so lookups use std.mem.eql.
+ fn setResponseHeaders(self: *Transfer, headers: []const http.Header) void {
+ if (comptime lp.IS_DEBUG) {
+ for (headers) |hdr| {
+ for (hdr.name) |c| {
+ std.debug.assert(std.ascii.isUpper(c) == false);
+ }
+ }
+ }
+ self.res.headers = headers;
+ }
+
fn setResponseHead(self: *Transfer, status: u16, content_type: ?[]const u8) void {
self.res.header = .{
.url = self.req.url.ptr,
@@ -3214,7 +3242,7 @@ pub const Transfer = struct {
};
self.setResponseHead(cached.status, cached.content_type);
- self.res.headers = cached.headers;
+ self.setResponseHeaders(cached.headers);
self._from_cache = true;
self._timing.cache = cache_state;
self._content_length = body.len;
@@ -3222,27 +3250,29 @@ pub const Transfer = struct {
}
// Materialize an interceptor-supplied response (CDP fulfillRequest).
- // `headers` and `body` are caller-owned; copy everything that must
- // survive until dispatch.
+ fn dupeHeadersLower(self: *Transfer, headers: []const http.Header) ![]const http.Header {
+ const allocator = self.arena.allocator();
+ const owned = try allocator.alloc(http.Header, headers.len);
+ for (headers, owned) |hdr, *o| {
+ o.* = try hdr.normalize(allocator);
+ }
+ return owned;
+ }
+
+ // our arena already owns `header` but not `body`. Why so complicated?
fn bufferFulfilled(self: *Transfer, status: u16, headers: []const http.Header, body: ?[]const u8) !void {
const arena = self.arena;
-
- const owned = try arena.alloc(http.Header, headers.len);
var content_type: ?[]const u8 = null;
- for (headers, 0..) |hdr, i| {
- owned[i] = .{
- .name = try arena.dupe(u8, hdr.name),
- .value = try arena.dupe(u8, hdr.value),
- };
- if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) {
- content_type = owned[i].value;
+ for (headers) |hdr| {
+ if (std.mem.eql(u8, hdr.name, "content-type")) {
+ content_type = hdr.value;
}
}
const owned_body: []const u8 = if (body) |b| try arena.dupe(u8, b) else "";
self.setResponseHead(status, content_type);
- self.res.headers = owned;
+ self.setResponseHeaders(headers);
self._content_length = owned_body.len;
try self.bufferEvents(owned_body);
}
@@ -3278,12 +3308,12 @@ pub const Transfer = struct {
var it = HeaderIterator{ .curl = .{ .conn = conn } };
const headers = try it.collect(arena.allocator());
- self.res.headers = headers.items;
+ self.setResponseHeaders(headers.items);
if (self.req.credentialsAllowed()) {
if (self.cookie_jar) |jar| {
for (self.res.headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {
+ if (std.mem.eql(u8, hdr.name, "set-cookie")) {
jar.populateFromResponse(self.req.url, hdr.value) catch |err| {
log.err(.http, "set cookie", .{ .err = err, .req = self });
return err;
@@ -3973,7 +4003,7 @@ pub const Transfer = struct {
fn getContentLengthRawValue(self: *const Transfer) ?[]const u8 {
// Materialized headers (dispatch time, any source).
for (self.res.headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "content-length")) {
+ if (std.mem.eql(u8, hdr.name, "content-length")) {
return hdr.value;
}
}
@@ -4178,7 +4208,8 @@ const Response = struct {
header: ?http.ResponseHead = null,
// Full response headers, materialized into the transfer arena at
- // completion (or set directly by cache / synthetic / fulfill).
+ // completion (or set directly by cache / synthetic / fulfill). Names are
+ // lowercased.
headers: []const http.Header = &.{},
// total bytes received in the response, including the response status
@@ -4285,7 +4316,7 @@ const Synthetic = struct {
if (content_type.len > 0) {
const h = try arena.alloc(http.Header, 1);
h[0] = .{ .name = "content-type", .value = content_type };
- transfer.res.headers = h;
+ transfer.setResponseHeaders(h);
}
transfer._content_length = body.len;
try transfer.bufferEvents(body);
diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig
index 9ae32bfd7..9a667fde3 100644
--- a/src/network/cache/Cache.zig
+++ b/src/network/cache/Cache.zig
@@ -320,33 +320,33 @@ const ResponseHeaders = struct {
for (headers) |h| {
switch (h.name.len) {
- 3 => if (std.ascii.eqlIgnoreCase(h.name, "Age")) {
+ 3 => if (std.mem.eql(u8, h.name, "age")) {
self.age = h.value;
},
4 => {
- if (std.ascii.eqlIgnoreCase(h.name, "Date")) {
+ if (std.mem.eql(u8, h.name, "date")) {
self.date = h.value;
- } else if (std.ascii.eqlIgnoreCase(h.name, "ETag")) {
+ } else if (std.mem.eql(u8, h.name, "etag")) {
self.etag = h.value;
- } else if (std.ascii.eqlIgnoreCase(h.name, "Vary")) {
+ } else if (std.mem.eql(u8, h.name, "vary")) {
self.vary = h.value;
}
},
- 7 => if (std.ascii.eqlIgnoreCase(h.name, "Expires")) {
+ 7 => if (std.mem.eql(u8, h.name, "expires")) {
self.expires = h.value;
},
- 10 => if (std.ascii.eqlIgnoreCase(h.name, "Set-Cookie")) {
+ 10 => if (std.mem.eql(u8, h.name, "set-cookie")) {
self.has_set_cookie = true;
},
- 12 => if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {
+ 12 => if (std.mem.eql(u8, h.name, "content-type")) {
self.content_type = h.value;
},
13 => {
- if (std.ascii.eqlIgnoreCase(h.name, "Cache-Control")) {
+ if (std.mem.eql(u8, h.name, "cache-control")) {
self.directives = .parse(h.value);
- } else if (std.ascii.eqlIgnoreCase(h.name, "Last-Modified")) {
+ } else if (std.mem.eql(u8, h.name, "last-modified")) {
self.last_modified = h.value;
- } else if (std.ascii.eqlIgnoreCase(h.name, "Authorization")) {
+ } else if (std.mem.eql(u8, h.name, "authorization")) {
self.has_authorization = true;
}
},
@@ -576,11 +576,11 @@ const TestResponse = struct {
fn run(self: TestResponse, arena: std.mem.Allocator) !?CachePutRequest {
var headers: std.ArrayList(Http.Header) = .empty;
inline for (.{
- .{ "Cache-Control", self.cache_control },
- .{ "Expires", self.expires },
- .{ "Date", self.date },
- .{ "ETag", self.etag },
- .{ "Last-Modified", self.last_modified },
+ .{ "cache-control", self.cache_control },
+ .{ "expires", self.expires },
+ .{ "date", self.date },
+ .{ "etag", self.etag },
+ .{ "last-modified", self.last_modified },
}) |field| {
if (field[1]) |value| {
try headers.append(arena, .{ .name = field[0], .value = value });
@@ -721,8 +721,8 @@ test "Cache: tryCache vary headers" {
.status = 200,
.content_type = "text/html",
.headers = &.{
- .{ .name = "Cache-Control", .value = "max-age=300" },
- .{ .name = "Vary", .value = "accept-encoding, accept-language" },
+ .{ .name = "cache-control", .value = "max-age=300" },
+ .{ .name = "vary", .value = "accept-encoding, accept-language" },
},
.request_headers = &request_headers,
});
@@ -738,8 +738,8 @@ test "Cache: tryCache vary headers" {
.status = 200,
.content_type = "text/html",
.headers = &.{
- .{ .name = "Cache-Control", .value = "max-age=300" },
- .{ .name = "Vary", .value = "*" },
+ .{ .name = "cache-control", .value = "max-age=300" },
+ .{ .name = "vary", .value = "*" },
},
.request_headers = &request_headers,
});
diff --git a/src/network/http.zig b/src/network/http.zig
index 4a1418df0..43ba5bef0 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -71,6 +71,13 @@ pub const Header = struct {
value: []const u8,
};
+ pub fn normalize(self: Header, allocator: std.mem.Allocator) !Header {
+ return .{
+ .name = try std.ascii.allocLowerString(allocator, self.name),
+ .value = try allocator.dupe(u8, self.value),
+ };
+ }
+
pub fn parse(header_str: []const u8) ?Header {
const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;
@@ -147,10 +154,7 @@ pub const HeaderIterator = union(enum) {
var list: std.ArrayList(Header) = .empty;
while (self.next()) |hdr| {
- try list.append(allocator, .{
- .name = try allocator.dupe(u8, hdr.name),
- .value = try allocator.dupe(u8, hdr.value),
- });
+ try list.append(allocator, try hdr.normalize(allocator));
}
return list;