Lightpanda: the headless browser designed for AI and automation
http: improve header overwrite/enforcement
Karl Seguin737f69ee4468
diff --git a/src/Config.zig b/src/Config.zigindex 097911bc4..3e95bec21 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -1067,8 +1067,8 @@ test "Config: validateUserAgent" {fn userAgentValidator(allocator: Allocator, args: *std.process.Args.Iterator, ua: *?[]const u8) !void {const str = args.next() orelse return error.MissingArgument;- validateUserAgent(str) catch {- log.fatal(.app, "invalid user-agent", .{ .hint = "User agent can't contain Mozilla" });+ validateUserAgent(str) catch |err| {+ log.fatal(.app, "invalid user-agent", .{ .err = err, .hint = "must be printable ASCII and can't contain Mozilla" });return error.InvalidArgument;};diff --git a/src/browser/tests/net/fetch.html b/src/browser/tests/net/fetch.htmlindex ef676af35..c559092e1 100644--- a/src/browser/tests/net/fetch.html+++ b/src/browser/tests/net/fetch.html@@ -452,9 +452,38 @@}</script>-<!-- Referer across redirects: a same-origin hop keeps the full referrer; a- cross-origin hop (localhost alias) recomputes it at the redirect, which- the default policy (strict-origin-when-cross-origin) strips to origin. -->+<script id=fetch_header_layers type=module>+ {+ const state = await testing.async();+ const headers = new Headers();+ headers.set('Accept-Language', 'fr');+ headers.append('X-Multi', 'a');+ headers.append('X-Multi', 'b');+ headers.append('Cookie', 'a=1');+ headers.append('Cookie', 'b=2');+ headers.set('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64)');+ headers.set('Sec-Ch-Ua', '"Chromium";v="140"');+ const response = await fetch('http://127.0.0.1:9582/echo_headers', { headers });+ const text = await response.text();+ state.resolve();++ await state.done(() => {+ const got = {};+ for (const line of text.trim().split('\n')) {+ const sep = line.indexOf(': ');+ got[line.slice(0, sep).toLowerCase()] = line.slice(sep + 2);+ }+ testing.expectEqual('fr', got['accept-language']);+ testing.expectEqual('a, b', got['x-multi']);+ testing.expectEqual('a=1; b=2', got['cookie']);+ testing.expectEqual(true, got['user-agent'].includes('Lightpanda'));+ testing.expectEqual(false, got['user-agent'].includes('Mozilla'));+ testing.expectEqual(true, got['sec-ch-ua'].includes('Lightpanda'));+ testing.expectEqual(false, got['sec-ch-ua'].includes('Chromium'));+ });+ }+</script>+<script id=fetch_redirect_referer type=module>{const state = await testing.async();diff --git a/src/browser/tests/net/xhr.html b/src/browser/tests/net/xhr.htmlindex 0025e8966..5b57dc926 100644--- a/src/browser/tests/net/xhr.html+++ b/src/browser/tests/net/xhr.html@@ -563,3 +563,32 @@await state2.done(() => testing.expectEqual(200, req.status));}</script>++<script id=xhr_request_headers type=module>+ {+ const state = await testing.async();+ const req = new XMLHttpRequest();+ req.onload = () => state.resolve();+ req.open('GET', 'http://127.0.0.1:9582/echo_headers');+ req.setRequestHeader('Accept-Language', 'fr');+ req.setRequestHeader('X-Multi', 'a');+ req.setRequestHeader('X-Multi', 'b');+ req.setRequestHeader('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64)');+ req.setRequestHeader('Sec-Ch-Ua', '"Chromium";v="140"');+ req.send();++ await state.done(() => {+ const got = {};+ for (const line of req.responseText.trim().split('\n')) {+ const sep = line.indexOf(': ');+ got[line.slice(0, sep).toLowerCase()] = line.slice(sep + 2);+ }+ testing.expectEqual('fr', got['accept-language']);+ testing.expectEqual('a, b', got['x-multi']);+ testing.expectEqual(true, got['user-agent'].includes('Lightpanda'));+ testing.expectEqual(false, got['user-agent'].includes('Mozilla'));+ testing.expectEqual(true, got['sec-ch-ua'].includes('Lightpanda'));+ testing.expectEqual(false, got['sec-ch-ua'].includes('Chromium'));+ });+ }+</script>diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex 0e58aef76..944a1d2db 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -292,6 +292,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {const testing = @import("../../../testing.zig");test "WebApi: fetch" {+ testing.expectLog(&.{ .http, .http });try testing.htmlRunner("net/fetch.html", .{});try testing.htmlRunner("net/fetch_hash_route.html", .{});}diff --git a/src/browser/webapi/net/Headers.zig b/src/browser/webapi/net/Headers.zigindex 3998b4efe..5436aac2d 100644--- a/src/browser/webapi/net/Headers.zig+++ b/src/browser/webapi/net/Headers.zig@@ -101,15 +101,6 @@ pub fn forEach(self: *Headers, cb_: js.Function, js_this_: ?js.Object) !void {const HttpClient = @import("../../../network/HttpClient.zig");pub fn populateRequestHeaders(self: *Headers, transfer: *HttpClient.Transfer) !void {for (self._list._entries.items) |entry| {- if (std.ascii.eqlIgnoreCase(entry.name.str(), "user-agent")) {- lp.Config.validateUserAgent(entry.value.str()) catch |err| {- log.info(.js, "populateRequestHeaders", .{ .info = "user-agent override", .err = err });- continue;- };- try transfer.setHeader(entry.name.str(), entry.value.str(), .{ .source = .author });- continue;- }-try transfer.appendHeader(entry.name.str(), entry.value.str(), .{ .source = .author });}}diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zigindex b55e925d9..b4aa23a43 100644--- a/src/browser/webapi/net/XMLHttpRequest.zig+++ b/src/browser/webapi/net/XMLHttpRequest.zig@@ -732,6 +732,7 @@ pub const JsApi = struct {const testing = @import("../../../testing.zig");test "WebApi: XHR" {+ testing.expectLog(&.{ .http, .http });try testing.htmlRunner("net/xhr.html", .{});}diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex 1983f661b..d27686f64 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -369,11 +369,11 @@ pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const Nconst frame_id = req.document_frame_id orelse req.frame_id;const frame = bc.session.findFrameByFrameId(frame_id) orelse return;- // Modify request with extra CDP headers. Use setHeader (replace by name)- // so a caller-supplied header overrides a built-in default of the same- // name (e.g. User-Agent) instead of producing a duplicate.+ // Modify request with extra CDP headers: the .cdp layer overrides both+ // built-in defaults and script-set headers of the same name (Chrome+ // behavior), but never a .fixed header.for (bc.extra_headers.items) |extra| {- try transfer.setHeader(extra.name, extra.value, .{});+ try transfer.setHeader(extra.name, extra.value, .{ .source = .cdp });}// We're missing a bunch of fields, but, for now, this eems like enoughdiff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex a7ae331a8..2e089b278 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -533,7 +533,7 @@ pub fn request(self: *Client, req: Request, owner: ?*Owner) anyerror!void {// const transfer = try client.newRequest(.{...}, owner);// {// errdefer transfer.deinit();-// try transfer.addHeader("Blah", "x", .{});+// try transfer.setHeader("Blah", "x", .{});// }// try transfer.submit();//@@ -898,6 +898,16 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {return false;}+ // A request already carrying its own validators (e.g. a script-set+ // If-None-Match) should reach the server as-is. The script may be doing its+ // own caching, and if we inject our own headers and our own interpretation,+ // we can cause issues for the script (e.g. we respond with a 200 from our+ // cache, the server repaints the page because it didn't get the correct 304+ // that the upstream would have returned).+ if (transfer.findRequestHeader("If-None-Match") != null or transfer.findRequestHeader("If-Modified-Since") != null) {+ return false;+ }+// Redirects rewrite req.url; the entry must be stored/renewed under the// URL this lookup ran against, not the final hop. req.url is arena-owned,// so the captured slice outlives any redirect rewrite.@@ -2728,10 +2738,11 @@ pub const Transfer = struct {source: HeaderSource = .user_agent,};- // Who put the header on the request. CORS cares: only non-safelisted- // author (i.e. script-set) headers trigger a preflight.- // fixed is hardcoded and can't be changed.- pub const HeaderSource = enum { user_agent, author, fixed };+ // Who put the header on the request, ordered lowest priority first:+ // setHeader/appendHeader let a source overwrite headers from its own or+ // a lower layer, never a higher one. .fixed is hardcoded and can't be+ // changed (Sec-Ch-Ua). For CORS, only script-set headers cause a preflight.+ pub const HeaderSource = enum { user_agent, author, cdp, fixed };pub const HeaderOpts = struct {source: HeaderSource = .user_agent,@@ -2766,8 +2777,21 @@ pub const Transfer = struct {}}- // Adds, replacing every existing header with the same case-insensitive name+ // Add or replace or noops the header based on the source layering prioritypub fn setHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {+ return self.putHeader(name, value, opts.source, .set);+ }++ // Add or append or noops the header based on the source layering priority+ pub fn appendHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {+ return self.putHeader(name, value, opts.source, .append);+ }++ fn putHeader(self: *Transfer, name: []const u8, value: []const u8, source: HeaderSource, mode: enum { set, append }) !void {+ if (verifyHeader(name, value) == false) {+ return;+ }+var found = false;var i: usize = 0;while (i < self.req_headers.items.len) {@@ -2776,49 +2800,42 @@ pub const Transfer = struct {i += 1;continue;}-- // Don't override any fixed header.- if (hdr.source == .fixed) {- log.info(.http, "ignore overriding fixed header", .{ .header = hdr.name });- return;- }-if (found) {_ = self.req_headers.orderedRemove(i);continue;}found = true;++ if (@intFromEnum(hdr.source) > @intFromEnum(source)) {+ if (hdr.source == .fixed) {+ log.warn(.http, "ignore overriding fixed header", .{ .header = hdr.name });+ }+ return;+ }+ if (mode == .append and hdr.source == source) {+ const sep = if (std.ascii.eqlIgnoreCase(name, "cookie")) "; " else ", ";+ hdr.value = try std.fmt.allocPrint(self.arena.allocator(), "{s}{s}{s}", .{ hdr.value, sep, value });+ return;+ }hdr.value = try self.arena.allocator().dupe(u8, value);- hdr.source = opts.source;+ hdr.source = source;i += 1;}if (!found) {- try self.addHeader(name, value, opts);+ try self.addHeader(name, value, .{ .source = source });}}- // Adds, appending every existing header with the first same case-insensitive name.- // appendHeader doesn't deduplicate any existing headers.- pub fn appendHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {- var i: usize = 0;- while (i < self.req_headers.items.len) {- const hdr = &self.req_headers.items[i];- if (std.ascii.eqlIgnoreCase(hdr.name, name) == false) {- i += 1;- continue;- }-- // Don't override any fixed header.- if (hdr.source == .fixed) {- log.info(.http, "ignore overriding fixed header", .{ .header = hdr.name });- return;- }-- hdr.value = try std.fmt.allocPrint(self.arena.allocator(), "{s}, {s}", .{ hdr.value, value });- hdr.source = opts.source;- return;+ // Central gate for every header entering req_headers; name-keyed+ // restrictions live here so entry points don't need their own checks.+ fn verifyHeader(name: []const u8, value: []const u8) bool {+ if (std.ascii.eqlIgnoreCase(name, "user-agent")) {+ lp.Config.validateUserAgent(value) catch |err| {+ log.warn(.http, "invalid header dropped", .{ .name = name, .err = err });+ return false;+ };}- try self.addHeader(name, value, opts);+ return true;}// The client's baseline headers, added to every request at creation.@@ -2836,13 +2853,7 @@ pub const Transfer = struct {self.req_headers.clearRetainingCapacity();try self.seedHeaders();for (headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "user-agent")) {- lp.Config.validateUserAgent(hdr.value) catch |err| {- log.info(.http, "ignored request header", .{ .intercepted = self.client.intercepted, .name = hdr.name, .err = err });- continue;- };- }- try self.setHeader(hdr.name, hdr.value, .{});+ try self.setHeader(hdr.name, hdr.value, .{ .source = .cdp });}}@@ -3468,14 +3479,8 @@ test "HttpClient: URL blocking exempts internal transfers" {try testing.expect(!client.isUrlBlocked("https://example.test/robots.txt", true));}-test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {- var pool = ArenaPool.init(testing.allocator, .{});- defer pool.deinit();-- const arena = try pool.acquire(.small, "test");- defer arena.release();-- var transfer = Transfer{+fn testTransfer(arena: *lp.Arena) Transfer {+ return .{.arena = arena,.owner = null,.req = .{@@ -3493,6 +3498,16 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {.id = 1,.start_time = 0,};+}++test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ const arena = try pool.acquire(.small, "test");+ defer arena.release();++ var transfer = testTransfer(arena);try transfer.setHeader("User-Agent", "Lightpanda/1.0", .{});try transfer.setHeader("X-Twice", "a", .{});@@ -3521,44 +3536,66 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {try testing.expectEqual("yes", headers[2].value);}-test "HttpClient: Transfer.appendHeader joins values by case-insensitive name" {+test "HttpClient: Transfer.appendHeader combines same-source values" {var pool = ArenaPool.init(testing.allocator, .{});defer pool.deinit();const arena = try pool.acquire(.small, "test");defer arena.release();- var transfer = Transfer{- .arena = arena,- .owner = null,- .req = .{- .frame_id = 0,- .loader_id = 0,- .method = .GET,- .url = "http://example.com/",- .cookie_jar = null,- .cookie_origin = "",- .resource_type = .document,- .notification = undefined,- .shutdown_callback = noopShutdown,- },- .client = undefined,- .id = 1,- .start_time = 0,- };+ var transfer = testTransfer(arena);// no match: appends a new header- try transfer.appendHeader("Cookie", "a=1", .{});- // match by case-insensitive name: joins onto the existing value+ try transfer.appendHeader("Accept-Language", "en-US", .{});+ // a higher layer replaces rather than joins+ try transfer.appendHeader("ACCEPT-LANGUAGE", "fr", .{ .source = .author });+ // a same-source repeat joins onto the existing value+ try transfer.appendHeader("accept-language", "de", .{ .source = .author });++ // Cookie joins with its own separator+ try transfer.appendHeader("Cookie", "a=1", .{ .source = .author });try transfer.appendHeader("COOKIE", "b=2", .{ .source = .author });- {- const headers = transfer.req_headers.items;- try testing.expectEqual(1, headers.len);- try testing.expectEqual("Cookie", headers[0].name);- try testing.expectEqual("a=1, b=2", headers[0].value);- try testing.expectEqual(.author, headers[0].source);- }+ const headers = transfer.req_headers.items;+ try testing.expectEqual(2, headers.len);+ try testing.expectEqual("Accept-Language", headers[0].name);+ try testing.expectEqual("fr, de", headers[0].value);+ try testing.expectEqual(.author, headers[0].source);+ try testing.expectEqual("Cookie", headers[1].name);+ try testing.expectEqual("a=1; b=2", headers[1].value);+}++test "HttpClient: Transfer header layering" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ const arena = try pool.acquire(.small, "test");+ defer arena.release();++ var transfer = testTransfer(arena);++ try transfer.setHeader("User-Agent", "Lightpanda/1.0", .{});+ try transfer.setHeader("Sec-Ch-Ua", "\"Lightpanda\";v=\"1\"", .{ .source = .fixed });+ try transfer.setHeader("If-None-Match", "\"author-etag\"", .{ .source = .author });++ // a lower layer never takes back a header a higher one owns+ try transfer.setHeader("If-None-Match", "\"cache-etag\"", .{});+ try testing.expectEqual("\"author-etag\"", transfer.findRequestHeader("if-none-match").?);++ // nothing overrides a fixed header, whatever the layer or mode+ testing.expectLog(&.{ .http, .http });+ try transfer.setHeader("sec-ch-ua", "\"Chromium\";v=\"140\"", .{ .source = .cdp });+ try transfer.appendHeader("SEC-CH-UA", "\"Chromium\";v=\"140\"", .{ .source = .author });+ try testing.expectEqual("\"Lightpanda\";v=\"1\"", transfer.findRequestHeader("sec-ch-ua").?);++ // an invalid User-Agent never enters the list+ testing.expectLog(&.{.http});+ try transfer.setHeader("user-agent", "Mozilla/5.0", .{ .source = .author });+ try testing.expectEqual("Lightpanda/1.0", transfer.findRequestHeader("user-agent").?);++ // a valid author User-Agent replaces the default+ try transfer.setHeader("User-Agent", "MyBot/2.0", .{ .source = .author });+ try testing.expectEqual("MyBot/2.0", transfer.findRequestHeader("user-agent").?);}test "HttpClient: Fetch header overrides restore after one hop" {diff --git a/src/testing.zig b/src/testing.zigindex daac8eded..cbf6098f9 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -1028,6 +1028,23 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {});}+ if (std.mem.eql(u8, path, "/echo_headers")) {+ // Echo every request header back as "name: value" lines, so tests+ // can assert on the headers a request actually sent.+ var buf: [8192]u8 = undefined;+ var pos: usize = 0;+ var it = req.iterateHeaders();+ while (it.next()) |header| {+ const line = try std.fmt.bufPrint(buf[pos..], "{s}: {s}\n", .{ header.name, header.value });+ pos += line.len;+ }+ return req.respond(buf[0..pos], .{+ .extra_headers = &.{+ .{ .name = "Content-Type", .value = "text/plain; charset=utf-8" },+ },+ });+ }+if (std.mem.eql(u8, path, "/redirect_same_echo_referer")) {// Same-origin 302 to /echo_referer: the full Referer must survive the hop.return req.respond("", .{