Lightpanda: the headless browser designed for AI and automation
add --http-header option to send headers on every HTTP requests
Pierre Tachoirea6afdce444c6
diff --git a/src/Config.zig b/src/Config.zigindex 3e95bec21..94d797e5d 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -24,6 +24,7 @@ const cli = @import("cli.zig");const dump = @import("browser/dump.zig");const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;+const HttpHeader = @import("network/http.zig").Header;const log = lp.log;const crypto = @import("sys/libcrypto.zig");@@ -85,6 +86,29 @@ fn logLevelValidator(_: Allocator, args: *std.process.Args.Iterator, target: *?l};}+fn httpHeaderValidator(allocator: Allocator, args: *std.process.Args.Iterator, list: *std.ArrayList(HttpHeader)) !void {+ const str = args.next() orelse return error.MissingArgument;+ const header = HttpHeader.parse(str) orelse {+ log.fatal(.app, "invalid option value", .{ .arg = "--http-header", .value = str, .hint = "expected \"Name: Value\"" });+ return error.InvalidArgument;+ };++ if (std.ascii.eqlIgnoreCase(header.name, "User-Agent")) {+ log.fatal(.app, "invalid option value", .{ .arg = "--http-header", .value = str, .hint = "Use --user-agent instead" });+ return error.InvalidArgument;+ }++ if (std.ascii.eqlIgnoreCase(header.name, "Sec-Ch-Ua")) {+ log.fatal(.app, "invalid option value", .{ .arg = "--http-header", .value = str, .hint = "Sec-Ch-Ua is not oevrridable" });+ return error.InvalidArgument;+ }++ try list.append(allocator, .{+ .name = try allocator.dupe(u8, header.name),+ .value = try allocator.dupe(u8, header.value),+ });+}+const Cert = struct {/// On successful CLI argument parsing phase, ownership of this transferred/// to `Network`. Consider it as invalid.@@ -183,6 +207,7 @@ const CommonOptions = .{.{ .name = "http_max_host_open", .type = ?u8 },.{ .name = "http_timeout", .type = ?u31 },.{ .name = "http_connect_timeout", .type = ?u31 },+ .{ .name = "http_header", .type = HttpHeader, .multiple = true, .validator = httpHeaderValidator },.{ .name = "http_max_response_size", .type = ?usize },.{ .name = "ws_max_concurrent", .type = ?u8 },.{ .name = "insecure_disable_tls_host_verification", .type = bool },@@ -507,6 +532,13 @@ pub fn httpProxy(self: *const Config) ?[:0]const u8 {};}+pub fn httpHeaders(self: *const Config) []const HttpHeader {+ return switch (self.mode) {+ inline .serve, .fetch, .mcp, .agent => |opts| opts.http_header.items,+ else => &.{},+ };+}+pub fn proxyBearerToken(self: *const Config) ?[:0]const u8 {return switch (self.mode) {inline .serve, .fetch, .mcp, .agent => |opts| opts.proxy_bearer_token,@@ -1065,6 +1097,29 @@ test "Config: validateUserAgent" {try std.testing.expectError(error.NonPrintable, validateUserAgent("bad\x01ua"));}+test "Config: httpHeaders accessor" {+ {+ var config = try Config.init(std.testing.allocator, "test", .{ .serve = .{} });+ defer config.deinit(std.testing.allocator);+ try std.testing.expectEqual(0, config.httpHeaders().len);+ }+ {+ var list: std.ArrayList(HttpHeader) = .empty;+ defer list.deinit(std.testing.allocator);+ try list.append(std.testing.allocator, .{ .name = "X-Extra", .value = "1" });++ var config = try Config.init(std.testing.allocator, "test", .{ .serve = .{+ .http_header = list,+ } });+ defer config.deinit(std.testing.allocator);++ const headers = config.httpHeaders();+ try std.testing.expectEqual(1, headers.len);+ try std.testing.expectEqualStrings("X-Extra", headers[0].name);+ try std.testing.expectEqualStrings("1", headers[0].value);+ }+}+fn userAgentValidator(allocator: Allocator, args: *std.process.Args.Iterator, ua: *?[]const u8) !void {const str = args.next() orelse return error.MissingArgument;validateUserAgent(str) catch |err| {diff --git a/src/browser/webapi/net/WebSocket.zig b/src/browser/webapi/net/WebSocket.zigindex aa594ad6d..997df6a4d 100644--- a/src/browser/webapi/net/WebSocket.zig+++ b/src/browser/webapi/net/WebSocket.zig@@ -240,7 +240,6 @@ fn connect(self: *WebSocket, protocols: [][]const u8) !void {for (http_client.baselineHeaders()) |hdr| {try conn.addHeader(allocator, hdr.name, hdr.value);}-if (protocols.len > 0) {try conn.addHeader(allocator, "Sec-WebSocket-Protocol", try std.mem.join(allocator, ", ", protocols));}diff --git a/src/help.zon b/src/help.zonindex 99dd019fa..62710367d 100644--- a/src/help.zon+++ b/src/help.zon@@ -354,6 +354,13 @@\\ Time in ms to establish an HTTP connection before timing out. 0 means\\ never.\\ Defaults to 0.+ \\ --http-header <HEADER>+ \\ Extra custom header added to every outgoing HTTP request, including redirections,+ \\ in "Name: Value" form. Can be passed multiple times. The first name is used.+ \\ Note that if you add a custom header that has the same name as one of the+ \\ internal ones Lightpanda would use, your set header is used instead of the+ \\ internal one. You should not replace internally set headers without knowing+ \\ perfectly well what you are doing.\\ --http-max-concurrent <INT>\\ Maximum number of concurrent HTTP requests.\\ Defaults to 40.diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 2e089b278..3000785bb 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -2742,7 +2742,7 @@ pub const Transfer = struct {// setHeader/appendHeader let a source overwrite headers from its own or// a lower layer, never a higher one. .fixed is hardcoded and can't be// changed (Sec-Ch-Ua). For CORS, only script-set headers cause a preflight.- pub const HeaderSource = enum { user_agent, author, cdp, fixed };+ pub const HeaderSource = enum { user_agent, author, cdp, cli, fixed };pub const HeaderOpts = struct {source: HeaderSource = .user_agent,@@ -2843,6 +2843,12 @@ pub const Transfer = struct {for (self.client.baselineHeaders()) |hdr| {try self.addHeader(hdr.name, hdr.value, .{ .source = hdr.source });}++ // --http-header extras; setHeader so a same-name header (e.g.+ // Accept-Language) overrides the baseline instead of duplicating.+ for (self.client.network.config.httpHeaders()) |hdr| {+ try self.setHeader(hdr.name, hdr.value, .{ .source = .cli });+ }}// CDP Fetch.continueRequest: the intercepting client supplies thediff --git a/src/network/http.zig b/src/network/http.zigindex b2631dd04..926b47784 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -62,11 +62,25 @@ pub const Header = struct {};pub fn parse(header_str: []const u8) ?Header {+ // ignore headers containing "\r\n".+ if (std.mem.indexOfAny(u8, header_str, "\r\n") != null) {+ return null;+ }+ // ignore headers w/o colon.const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");+ // no empty name.+ if (name.len == 0) {+ return null;+ }+ // no name incl. space.+ if (std.mem.indexOfAny(u8, name, " ") != null) {+ return null;+ }+return .{ .name = name, .value = value };}@@ -900,6 +914,50 @@ test "isBadPort" {}}+test "Header.parse" {+ {+ const h = Header.parse("Content-Type: text/html; charset=utf-8").?;+ try testing.expectEqualSlices(u8, "Content-Type", h.name);+ try testing.expectEqualSlices(u8, "text/html; charset=utf-8", h.value);+ }+ {+ // no space after the colon+ const h = Header.parse("X-Custom:value").?;+ try testing.expectEqualSlices(u8, "X-Custom", h.name);+ try testing.expectEqualSlices(u8, "value", h.value);+ }+ {+ // name and value are trimmed of spaces and tabs+ const h = Header.parse(" \tAccept \t: \tapplication/json \t").?;+ try testing.expectEqualSlices(u8, "Accept", h.name);+ try testing.expectEqualSlices(u8, "application/json", h.value);+ }+ {+ // only the first colon splits; later colons stay in the value+ const h = Header.parse("Referer: http://example.com:8080/").?;+ try testing.expectEqualSlices(u8, "Referer", h.name);+ try testing.expectEqualSlices(u8, "http://example.com:8080/", h.value);+ }+ {+ // empty value+ const h = Header.parse("X-Empty:").?;+ try testing.expectEqualSlices(u8, "X-Empty", h.name);+ try testing.expectEqualSlices(u8, "", h.value);+ }++ // empty name+ try testing.expect(Header.parse(": value") == null);+ // bad name+ try testing.expect(Header.parse("Foo Bar: value") == null);+ // no colon+ try testing.expect(Header.parse("not-a-header") == null);+ try testing.expect(Header.parse("") == null);+ // CR or LF anywhere rejects the header (injection guard)+ try testing.expect(Header.parse("X-A: b\r\nX-B: c") == null);+ try testing.expect(Header.parse("X-A: b\n") == null);+ try testing.expect(Header.parse("\rX-A: b") == null);+}+test "Header.firstValue" {try testing.expectEqualSlices(u8, "attachment", (Header{ .name = "Content-Disposition", .value = "attachment" }).firstValue());// firstValue trims but preserves case (callers compare case-insensitively).