Skip to content

Lightpanda: the headless browser designed for AI and automation

http: improve header overwrite/enforcement

Karl Seguin737f69ee4468
diff --git a/src/Config.zig b/src/Config.zig
index 097911bc4..3e95bec21 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -1067,8 +1067,8 @@ test "Config: validateUserAgent" {
fn userAgentValidator(allocator: Allocator, args: *std.process.Args.Iterator, ua: *?[]const u8) !void {
const str = args.next() orelse return error.MissingArgument;
- validateUserAgent(str) catch {
- log.fatal(.app, "invalid user-agent", .{ .hint = "User agent can't contain Mozilla" });
+ validateUserAgent(str) catch |err| {
+ log.fatal(.app, "invalid user-agent", .{ .err = err, .hint = "must be printable ASCII and can't contain Mozilla" });
return error.InvalidArgument;
};
diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.html
index ef676af35..c559092e1 100644
--- a/src/browser/tests/net/fetch.html
+++ b/src/browser/tests/net/fetch.html
@@ -452,9 +452,38 @@
}
</script>
-<!-- Referer across redirects: a same-origin hop keeps the full referrer; a
- cross-origin hop (localhost alias) recomputes it at the redirect, which
- the default policy (strict-origin-when-cross-origin) strips to origin. -->
+<script id=fetch_header_layers type=module>
+ {
+ const state = await testing.async();
+ const headers = new Headers();
+ headers.set('Accept-Language', 'fr');
+ headers.append('X-Multi', 'a');
+ headers.append('X-Multi', 'b');
+ headers.append('Cookie', 'a=1');
+ headers.append('Cookie', 'b=2');
+ headers.set('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64)');
+ headers.set('Sec-Ch-Ua', '"Chromium";v="140"');
+ const response = await fetch('http://127.0.0.1:9582/echo_headers', { headers });
+ const text = await response.text();
+ state.resolve();
+
+ await state.done(() => {
+ const got = {};
+ for (const line of text.trim().split('\n')) {
+ const sep = line.indexOf(': ');
+ got[line.slice(0, sep).toLowerCase()] = line.slice(sep + 2);
+ }
+ testing.expectEqual('fr', got['accept-language']);
+ testing.expectEqual('a, b', got['x-multi']);
+ testing.expectEqual('a=1; b=2', got['cookie']);
+ testing.expectEqual(true, got['user-agent'].includes('Lightpanda'));
+ testing.expectEqual(false, got['user-agent'].includes('Mozilla'));
+ testing.expectEqual(true, got['sec-ch-ua'].includes('Lightpanda'));
+ testing.expectEqual(false, got['sec-ch-ua'].includes('Chromium'));
+ });
+ }
+</script>
+
<script id=fetch_redirect_referer type=module>
{
const state = await testing.async();
diff --git a/src/browser/tests/net/xhr.html b/src/browser/tests/net/xhr.html
index 0025e8966..5b57dc926 100644
--- a/src/browser/tests/net/xhr.html
+++ b/src/browser/tests/net/xhr.html
@@ -563,3 +563,32 @@
await state2.done(() => testing.expectEqual(200, req.status));
}
</script>
+
+<script id=xhr_request_headers type=module>
+ {
+ const state = await testing.async();
+ const req = new XMLHttpRequest();
+ req.onload = () => state.resolve();
+ req.open('GET', 'http://127.0.0.1:9582/echo_headers');
+ req.setRequestHeader('Accept-Language', 'fr');
+ req.setRequestHeader('X-Multi', 'a');
+ req.setRequestHeader('X-Multi', 'b');
+ req.setRequestHeader('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64)');
+ req.setRequestHeader('Sec-Ch-Ua', '"Chromium";v="140"');
+ req.send();
+
+ await state.done(() => {
+ const got = {};
+ for (const line of req.responseText.trim().split('\n')) {
+ const sep = line.indexOf(': ');
+ got[line.slice(0, sep).toLowerCase()] = line.slice(sep + 2);
+ }
+ testing.expectEqual('fr', got['accept-language']);
+ testing.expectEqual('a, b', got['x-multi']);
+ testing.expectEqual(true, got['user-agent'].includes('Lightpanda'));
+ testing.expectEqual(false, got['user-agent'].includes('Mozilla'));
+ testing.expectEqual(true, got['sec-ch-ua'].includes('Lightpanda'));
+ testing.expectEqual(false, got['sec-ch-ua'].includes('Chromium'));
+ });
+ }
+</script>
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index 0e58aef76..944a1d2db 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -292,6 +292,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {
const testing = @import("../../../testing.zig");
test "WebApi: fetch" {
+ testing.expectLog(&.{ .http, .http });
try testing.htmlRunner("net/fetch.html", .{});
try testing.htmlRunner("net/fetch_hash_route.html", .{});
}
diff --git a/src/browser/webapi/net/Headers.zig b/src/browser/webapi/net/Headers.zig
index 3998b4efe..5436aac2d 100644
--- a/src/browser/webapi/net/Headers.zig
+++ b/src/browser/webapi/net/Headers.zig
@@ -101,15 +101,6 @@ pub fn forEach(self: *Headers, cb_: js.Function, js_this_: ?js.Object) !void {
const HttpClient = @import("../../../network/HttpClient.zig");
pub fn populateRequestHeaders(self: *Headers, transfer: *HttpClient.Transfer) !void {
for (self._list._entries.items) |entry| {
- if (std.ascii.eqlIgnoreCase(entry.name.str(), "user-agent")) {
- lp.Config.validateUserAgent(entry.value.str()) catch |err| {
- log.info(.js, "populateRequestHeaders", .{ .info = "user-agent override", .err = err });
- continue;
- };
- try transfer.setHeader(entry.name.str(), entry.value.str(), .{ .source = .author });
- continue;
- }
-
try transfer.appendHeader(entry.name.str(), entry.value.str(), .{ .source = .author });
}
}
diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig
index b55e925d9..b4aa23a43 100644
--- a/src/browser/webapi/net/XMLHttpRequest.zig
+++ b/src/browser/webapi/net/XMLHttpRequest.zig
@@ -732,6 +732,7 @@ pub const JsApi = struct {
const testing = @import("../../../testing.zig");
test "WebApi: XHR" {
+ testing.expectLog(&.{ .http, .http });
try testing.htmlRunner("net/xhr.html", .{});
}
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index 1983f661b..d27686f64 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -369,11 +369,11 @@ pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const N
const frame_id = req.document_frame_id orelse req.frame_id;
const frame = bc.session.findFrameByFrameId(frame_id) orelse return;
- // Modify request with extra CDP headers. Use setHeader (replace by name)
- // so a caller-supplied header overrides a built-in default of the same
- // name (e.g. User-Agent) instead of producing a duplicate.
+ // Modify request with extra CDP headers: the .cdp layer overrides both
+ // built-in defaults and script-set headers of the same name (Chrome
+ // behavior), but never a .fixed header.
for (bc.extra_headers.items) |extra| {
- try transfer.setHeader(extra.name, extra.value, .{});
+ try transfer.setHeader(extra.name, extra.value, .{ .source = .cdp });
}
// We're missing a bunch of fields, but, for now, this eems like enough
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index a7ae331a8..2e089b278 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -533,7 +533,7 @@ pub fn request(self: *Client, req: Request, owner: ?*Owner) anyerror!void {
// const transfer = try client.newRequest(.{...}, owner);
// {
// errdefer transfer.deinit();
-// try transfer.addHeader("Blah", "x", .{});
+// try transfer.setHeader("Blah", "x", .{});
// }
// try transfer.submit();
//
@@ -898,6 +898,16 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
return false;
}
+ // A request already carrying its own validators (e.g. a script-set
+ // If-None-Match) should reach the server as-is. The script may be doing its
+ // own caching, and if we inject our own headers and our own interpretation,
+ // we can cause issues for the script (e.g. we respond with a 200 from our
+ // cache, the server repaints the page because it didn't get the correct 304
+ // that the upstream would have returned).
+ if (transfer.findRequestHeader("If-None-Match") != null or transfer.findRequestHeader("If-Modified-Since") != null) {
+ return false;
+ }
+
// Redirects rewrite req.url; the entry must be stored/renewed under the
// URL this lookup ran against, not the final hop. req.url is arena-owned,
// so the captured slice outlives any redirect rewrite.
@@ -2728,10 +2738,11 @@ pub const Transfer = struct {
source: HeaderSource = .user_agent,
};
- // Who put the header on the request. CORS cares: only non-safelisted
- // author (i.e. script-set) headers trigger a preflight.
- // fixed is hardcoded and can't be changed.
- pub const HeaderSource = enum { user_agent, author, fixed };
+ // Who put the header on the request, ordered lowest priority first:
+ // setHeader/appendHeader let a source overwrite headers from its own or
+ // a lower layer, never a higher one. .fixed is hardcoded and can't be
+ // changed (Sec-Ch-Ua). For CORS, only script-set headers cause a preflight.
+ pub const HeaderSource = enum { user_agent, author, cdp, fixed };
pub const HeaderOpts = struct {
source: HeaderSource = .user_agent,
@@ -2766,8 +2777,21 @@ pub const Transfer = struct {
}
}
- // Adds, replacing every existing header with the same case-insensitive name
+ // Add or replace or noops the header based on the source layering priority
pub fn setHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
+ return self.putHeader(name, value, opts.source, .set);
+ }
+
+ // Add or append or noops the header based on the source layering priority
+ pub fn appendHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
+ return self.putHeader(name, value, opts.source, .append);
+ }
+
+ fn putHeader(self: *Transfer, name: []const u8, value: []const u8, source: HeaderSource, mode: enum { set, append }) !void {
+ if (verifyHeader(name, value) == false) {
+ return;
+ }
+
var found = false;
var i: usize = 0;
while (i < self.req_headers.items.len) {
@@ -2776,49 +2800,42 @@ pub const Transfer = struct {
i += 1;
continue;
}
-
- // Don't override any fixed header.
- if (hdr.source == .fixed) {
- log.info(.http, "ignore overriding fixed header", .{ .header = hdr.name });
- return;
- }
-
if (found) {
_ = self.req_headers.orderedRemove(i);
continue;
}
found = true;
+
+ if (@intFromEnum(hdr.source) > @intFromEnum(source)) {
+ if (hdr.source == .fixed) {
+ log.warn(.http, "ignore overriding fixed header", .{ .header = hdr.name });
+ }
+ return;
+ }
+ if (mode == .append and hdr.source == source) {
+ const sep = if (std.ascii.eqlIgnoreCase(name, "cookie")) "; " else ", ";
+ hdr.value = try std.fmt.allocPrint(self.arena.allocator(), "{s}{s}{s}", .{ hdr.value, sep, value });
+ return;
+ }
hdr.value = try self.arena.allocator().dupe(u8, value);
- hdr.source = opts.source;
+ hdr.source = source;
i += 1;
}
if (!found) {
- try self.addHeader(name, value, opts);
+ try self.addHeader(name, value, .{ .source = source });
}
}
- // Adds, appending every existing header with the first same case-insensitive name.
- // appendHeader doesn't deduplicate any existing headers.
- pub fn appendHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
- var i: usize = 0;
- while (i < self.req_headers.items.len) {
- const hdr = &self.req_headers.items[i];
- if (std.ascii.eqlIgnoreCase(hdr.name, name) == false) {
- i += 1;
- continue;
- }
-
- // Don't override any fixed header.
- if (hdr.source == .fixed) {
- log.info(.http, "ignore overriding fixed header", .{ .header = hdr.name });
- return;
- }
-
- hdr.value = try std.fmt.allocPrint(self.arena.allocator(), "{s}, {s}", .{ hdr.value, value });
- hdr.source = opts.source;
- return;
+ // Central gate for every header entering req_headers; name-keyed
+ // restrictions live here so entry points don't need their own checks.
+ fn verifyHeader(name: []const u8, value: []const u8) bool {
+ if (std.ascii.eqlIgnoreCase(name, "user-agent")) {
+ lp.Config.validateUserAgent(value) catch |err| {
+ log.warn(.http, "invalid header dropped", .{ .name = name, .err = err });
+ return false;
+ };
}
- try self.addHeader(name, value, opts);
+ return true;
}
// The client's baseline headers, added to every request at creation.
@@ -2836,13 +2853,7 @@ pub const Transfer = struct {
self.req_headers.clearRetainingCapacity();
try self.seedHeaders();
for (headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "user-agent")) {
- lp.Config.validateUserAgent(hdr.value) catch |err| {
- log.info(.http, "ignored request header", .{ .intercepted = self.client.intercepted, .name = hdr.name, .err = err });
- continue;
- };
- }
- try self.setHeader(hdr.name, hdr.value, .{});
+ try self.setHeader(hdr.name, hdr.value, .{ .source = .cdp });
}
}
@@ -3468,14 +3479,8 @@ test "HttpClient: URL blocking exempts internal transfers" {
try testing.expect(!client.isUrlBlocked("https://example.test/robots.txt", true));
}
-test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
- var pool = ArenaPool.init(testing.allocator, .{});
- defer pool.deinit();
-
- const arena = try pool.acquire(.small, "test");
- defer arena.release();
-
- var transfer = Transfer{
+fn testTransfer(arena: *lp.Arena) Transfer {
+ return .{
.arena = arena,
.owner = null,
.req = .{
@@ -3493,6 +3498,16 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
.id = 1,
.start_time = 0,
};
+}
+
+test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
+ var pool = ArenaPool.init(testing.allocator, .{});
+ defer pool.deinit();
+
+ const arena = try pool.acquire(.small, "test");
+ defer arena.release();
+
+ var transfer = testTransfer(arena);
try transfer.setHeader("User-Agent", "Lightpanda/1.0", .{});
try transfer.setHeader("X-Twice", "a", .{});
@@ -3521,44 +3536,66 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
try testing.expectEqual("yes", headers[2].value);
}
-test "HttpClient: Transfer.appendHeader joins values by case-insensitive name" {
+test "HttpClient: Transfer.appendHeader combines same-source values" {
var pool = ArenaPool.init(testing.allocator, .{});
defer pool.deinit();
const arena = try pool.acquire(.small, "test");
defer arena.release();
- var transfer = Transfer{
- .arena = arena,
- .owner = null,
- .req = .{
- .frame_id = 0,
- .loader_id = 0,
- .method = .GET,
- .url = "http://example.com/",
- .cookie_jar = null,
- .cookie_origin = "",
- .resource_type = .document,
- .notification = undefined,
- .shutdown_callback = noopShutdown,
- },
- .client = undefined,
- .id = 1,
- .start_time = 0,
- };
+ var transfer = testTransfer(arena);
// no match: appends a new header
- try transfer.appendHeader("Cookie", "a=1", .{});
- // match by case-insensitive name: joins onto the existing value
+ try transfer.appendHeader("Accept-Language", "en-US", .{});
+ // a higher layer replaces rather than joins
+ try transfer.appendHeader("ACCEPT-LANGUAGE", "fr", .{ .source = .author });
+ // a same-source repeat joins onto the existing value
+ try transfer.appendHeader("accept-language", "de", .{ .source = .author });
+
+ // Cookie joins with its own separator
+ try transfer.appendHeader("Cookie", "a=1", .{ .source = .author });
try transfer.appendHeader("COOKIE", "b=2", .{ .source = .author });
- {
- const headers = transfer.req_headers.items;
- try testing.expectEqual(1, headers.len);
- try testing.expectEqual("Cookie", headers[0].name);
- try testing.expectEqual("a=1, b=2", headers[0].value);
- try testing.expectEqual(.author, headers[0].source);
- }
+ const headers = transfer.req_headers.items;
+ try testing.expectEqual(2, headers.len);
+ try testing.expectEqual("Accept-Language", headers[0].name);
+ try testing.expectEqual("fr, de", headers[0].value);
+ try testing.expectEqual(.author, headers[0].source);
+ try testing.expectEqual("Cookie", headers[1].name);
+ try testing.expectEqual("a=1; b=2", headers[1].value);
+}
+
+test "HttpClient: Transfer header layering" {
+ var pool = ArenaPool.init(testing.allocator, .{});
+ defer pool.deinit();
+
+ const arena = try pool.acquire(.small, "test");
+ defer arena.release();
+
+ var transfer = testTransfer(arena);
+
+ try transfer.setHeader("User-Agent", "Lightpanda/1.0", .{});
+ try transfer.setHeader("Sec-Ch-Ua", "\"Lightpanda\";v=\"1\"", .{ .source = .fixed });
+ try transfer.setHeader("If-None-Match", "\"author-etag\"", .{ .source = .author });
+
+ // a lower layer never takes back a header a higher one owns
+ try transfer.setHeader("If-None-Match", "\"cache-etag\"", .{});
+ try testing.expectEqual("\"author-etag\"", transfer.findRequestHeader("if-none-match").?);
+
+ // nothing overrides a fixed header, whatever the layer or mode
+ testing.expectLog(&.{ .http, .http });
+ try transfer.setHeader("sec-ch-ua", "\"Chromium\";v=\"140\"", .{ .source = .cdp });
+ try transfer.appendHeader("SEC-CH-UA", "\"Chromium\";v=\"140\"", .{ .source = .author });
+ try testing.expectEqual("\"Lightpanda\";v=\"1\"", transfer.findRequestHeader("sec-ch-ua").?);
+
+ // an invalid User-Agent never enters the list
+ testing.expectLog(&.{.http});
+ try transfer.setHeader("user-agent", "Mozilla/5.0", .{ .source = .author });
+ try testing.expectEqual("Lightpanda/1.0", transfer.findRequestHeader("user-agent").?);
+
+ // a valid author User-Agent replaces the default
+ try transfer.setHeader("User-Agent", "MyBot/2.0", .{ .source = .author });
+ try testing.expectEqual("MyBot/2.0", transfer.findRequestHeader("user-agent").?);
}
test "HttpClient: Fetch header overrides restore after one hop" {
diff --git a/src/testing.zig b/src/testing.zig
index daac8eded..cbf6098f9 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -1028,6 +1028,23 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
+ if (std.mem.eql(u8, path, "/echo_headers")) {
+ // Echo every request header back as "name: value" lines, so tests
+ // can assert on the headers a request actually sent.
+ var buf: [8192]u8 = undefined;
+ var pos: usize = 0;
+ var it = req.iterateHeaders();
+ while (it.next()) |header| {
+ const line = try std.fmt.bufPrint(buf[pos..], "{s}: {s}\n", .{ header.name, header.value });
+ pos += line.len;
+ }
+ return req.respond(buf[0..pos], .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/plain; charset=utf-8" },
+ },
+ });
+ }
+
if (std.mem.eql(u8, path, "/redirect_same_echo_referer")) {
// Same-origin 302 to /echo_referer: the full Referer must survive the hop.
return req.respond("", .{