Skip to content

Lightpanda: the headless browser designed for AI and automation

add --http-header option to send headers on every HTTP requests

Pierre Tachoirea6afdce444c6
diff --git a/src/Config.zig b/src/Config.zig
index 3e95bec21..94d797e5d 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -24,6 +24,7 @@ const cli = @import("cli.zig");
const dump = @import("browser/dump.zig");
const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;
+const HttpHeader = @import("network/http.zig").Header;
const log = lp.log;
const crypto = @import("sys/libcrypto.zig");
@@ -85,6 +86,29 @@ fn logLevelValidator(_: Allocator, args: *std.process.Args.Iterator, target: *?l
};
}
+fn httpHeaderValidator(allocator: Allocator, args: *std.process.Args.Iterator, list: *std.ArrayList(HttpHeader)) !void {
+ const str = args.next() orelse return error.MissingArgument;
+ const header = HttpHeader.parse(str) orelse {
+ log.fatal(.app, "invalid option value", .{ .arg = "--http-header", .value = str, .hint = "expected \"Name: Value\"" });
+ return error.InvalidArgument;
+ };
+
+ if (std.ascii.eqlIgnoreCase(header.name, "User-Agent")) {
+ log.fatal(.app, "invalid option value", .{ .arg = "--http-header", .value = str, .hint = "Use --user-agent instead" });
+ return error.InvalidArgument;
+ }
+
+ if (std.ascii.eqlIgnoreCase(header.name, "Sec-Ch-Ua")) {
+ log.fatal(.app, "invalid option value", .{ .arg = "--http-header", .value = str, .hint = "Sec-Ch-Ua is not oevrridable" });
+ return error.InvalidArgument;
+ }
+
+ try list.append(allocator, .{
+ .name = try allocator.dupe(u8, header.name),
+ .value = try allocator.dupe(u8, header.value),
+ });
+}
+
const Cert = struct {
/// On successful CLI argument parsing phase, ownership of this transferred
/// to `Network`. Consider it as invalid.
@@ -183,6 +207,7 @@ const CommonOptions = .{
.{ .name = "http_max_host_open", .type = ?u8 },
.{ .name = "http_timeout", .type = ?u31 },
.{ .name = "http_connect_timeout", .type = ?u31 },
+ .{ .name = "http_header", .type = HttpHeader, .multiple = true, .validator = httpHeaderValidator },
.{ .name = "http_max_response_size", .type = ?usize },
.{ .name = "ws_max_concurrent", .type = ?u8 },
.{ .name = "insecure_disable_tls_host_verification", .type = bool },
@@ -507,6 +532,13 @@ pub fn httpProxy(self: *const Config) ?[:0]const u8 {
};
}
+pub fn httpHeaders(self: *const Config) []const HttpHeader {
+ return switch (self.mode) {
+ inline .serve, .fetch, .mcp, .agent => |opts| opts.http_header.items,
+ else => &.{},
+ };
+}
+
pub fn proxyBearerToken(self: *const Config) ?[:0]const u8 {
return switch (self.mode) {
inline .serve, .fetch, .mcp, .agent => |opts| opts.proxy_bearer_token,
@@ -1065,6 +1097,29 @@ test "Config: validateUserAgent" {
try std.testing.expectError(error.NonPrintable, validateUserAgent("bad\x01ua"));
}
+test "Config: httpHeaders accessor" {
+ {
+ var config = try Config.init(std.testing.allocator, "test", .{ .serve = .{} });
+ defer config.deinit(std.testing.allocator);
+ try std.testing.expectEqual(0, config.httpHeaders().len);
+ }
+ {
+ var list: std.ArrayList(HttpHeader) = .empty;
+ defer list.deinit(std.testing.allocator);
+ try list.append(std.testing.allocator, .{ .name = "X-Extra", .value = "1" });
+
+ var config = try Config.init(std.testing.allocator, "test", .{ .serve = .{
+ .http_header = list,
+ } });
+ defer config.deinit(std.testing.allocator);
+
+ const headers = config.httpHeaders();
+ try std.testing.expectEqual(1, headers.len);
+ try std.testing.expectEqualStrings("X-Extra", headers[0].name);
+ try std.testing.expectEqualStrings("1", headers[0].value);
+ }
+}
+
fn userAgentValidator(allocator: Allocator, args: *std.process.Args.Iterator, ua: *?[]const u8) !void {
const str = args.next() orelse return error.MissingArgument;
validateUserAgent(str) catch |err| {
diff --git a/src/browser/webapi/net/WebSocket.zig b/src/browser/webapi/net/WebSocket.zig
index aa594ad6d..997df6a4d 100644
--- a/src/browser/webapi/net/WebSocket.zig
+++ b/src/browser/webapi/net/WebSocket.zig
@@ -240,7 +240,6 @@ fn connect(self: *WebSocket, protocols: [][]const u8) !void {
for (http_client.baselineHeaders()) |hdr| {
try conn.addHeader(allocator, hdr.name, hdr.value);
}
-
if (protocols.len > 0) {
try conn.addHeader(allocator, "Sec-WebSocket-Protocol", try std.mem.join(allocator, ", ", protocols));
}
diff --git a/src/help.zon b/src/help.zon
index 99dd019fa..62710367d 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -354,6 +354,13 @@
\\ Time in ms to establish an HTTP connection before timing out. 0 means
\\ never.
\\ Defaults to 0.
+ \\ --http-header <HEADER>
+ \\ Extra custom header added to every outgoing HTTP request, including redirections,
+ \\ in "Name: Value" form. Can be passed multiple times. The first name is used.
+ \\ Note that if you add a custom header that has the same name as one of the
+ \\ internal ones Lightpanda would use, your set header is used instead of the
+ \\ internal one. You should not replace internally set headers without knowing
+ \\ perfectly well what you are doing.
\\ --http-max-concurrent <INT>
\\ Maximum number of concurrent HTTP requests.
\\ Defaults to 40.
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 2e089b278..3000785bb 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -2742,7 +2742,7 @@ pub const Transfer = struct {
// setHeader/appendHeader let a source overwrite headers from its own or
// a lower layer, never a higher one. .fixed is hardcoded and can't be
// changed (Sec-Ch-Ua). For CORS, only script-set headers cause a preflight.
- pub const HeaderSource = enum { user_agent, author, cdp, fixed };
+ pub const HeaderSource = enum { user_agent, author, cdp, cli, fixed };
pub const HeaderOpts = struct {
source: HeaderSource = .user_agent,
@@ -2843,6 +2843,12 @@ pub const Transfer = struct {
for (self.client.baselineHeaders()) |hdr| {
try self.addHeader(hdr.name, hdr.value, .{ .source = hdr.source });
}
+
+ // --http-header extras; setHeader so a same-name header (e.g.
+ // Accept-Language) overrides the baseline instead of duplicating.
+ for (self.client.network.config.httpHeaders()) |hdr| {
+ try self.setHeader(hdr.name, hdr.value, .{ .source = .cli });
+ }
}
// CDP Fetch.continueRequest: the intercepting client supplies the
diff --git a/src/network/http.zig b/src/network/http.zig
index b2631dd04..926b47784 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -62,11 +62,25 @@ pub const Header = struct {
};
pub fn parse(header_str: []const u8) ?Header {
+ // ignore headers containing "\r\n".
+ if (std.mem.indexOfAny(u8, header_str, "\r\n") != null) {
+ return null;
+ }
+ // ignore headers w/o colon.
const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;
const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");
+ // no empty name.
+ if (name.len == 0) {
+ return null;
+ }
+ // no name incl. space.
+ if (std.mem.indexOfAny(u8, name, " ") != null) {
+ return null;
+ }
+
return .{ .name = name, .value = value };
}
@@ -900,6 +914,50 @@ test "isBadPort" {
}
}
+test "Header.parse" {
+ {
+ const h = Header.parse("Content-Type: text/html; charset=utf-8").?;
+ try testing.expectEqualSlices(u8, "Content-Type", h.name);
+ try testing.expectEqualSlices(u8, "text/html; charset=utf-8", h.value);
+ }
+ {
+ // no space after the colon
+ const h = Header.parse("X-Custom:value").?;
+ try testing.expectEqualSlices(u8, "X-Custom", h.name);
+ try testing.expectEqualSlices(u8, "value", h.value);
+ }
+ {
+ // name and value are trimmed of spaces and tabs
+ const h = Header.parse(" \tAccept \t: \tapplication/json \t").?;
+ try testing.expectEqualSlices(u8, "Accept", h.name);
+ try testing.expectEqualSlices(u8, "application/json", h.value);
+ }
+ {
+ // only the first colon splits; later colons stay in the value
+ const h = Header.parse("Referer: http://example.com:8080/").?;
+ try testing.expectEqualSlices(u8, "Referer", h.name);
+ try testing.expectEqualSlices(u8, "http://example.com:8080/", h.value);
+ }
+ {
+ // empty value
+ const h = Header.parse("X-Empty:").?;
+ try testing.expectEqualSlices(u8, "X-Empty", h.name);
+ try testing.expectEqualSlices(u8, "", h.value);
+ }
+
+ // empty name
+ try testing.expect(Header.parse(": value") == null);
+ // bad name
+ try testing.expect(Header.parse("Foo Bar: value") == null);
+ // no colon
+ try testing.expect(Header.parse("not-a-header") == null);
+ try testing.expect(Header.parse("") == null);
+ // CR or LF anywhere rejects the header (injection guard)
+ try testing.expect(Header.parse("X-A: b\r\nX-B: c") == null);
+ try testing.expect(Header.parse("X-A: b\n") == null);
+ try testing.expect(Header.parse("\rX-A: b") == null);
+}
+
test "Header.firstValue" {
try testing.expectEqualSlices(u8, "attachment", (Header{ .name = "Content-Disposition", .value = "attachment" }).firstValue());
// firstValue trims but preserves case (callers compare case-insensitively).