Lightpanda: the headless browser designed for AI and automation
cli: one Accept-Language parser for the header and navigator.languages
Adrià Arrufat37bf7b68c151
diff --git a/src/Config.zig b/src/Config.zigindex 9db44ba42..fbbb4c223 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -276,8 +276,8 @@ const CommonOptions = .{.{ .name = "web_bot_auth_keyid", .type = ?[]const u8 },.{ .name = "web_bot_auth_domain", .type = ?[]const u8 },.{ .name = "user_agent", .type = ?[]const u8, .validator = userAgentValidator },- .{ .name = "locale", .type = []const u8, .default = HttpHeaders.default_locale, .validator = localeValidator },- .{ .name = "timezone", .type = ?[]const u8, .validator = timezoneValidator },+ .{ .name = "locale", .type = [:0]const u8, .default = HttpHeaders.default_locale, .validator = localeValidator },+ .{ .name = "timezone", .type = ?[:0]const u8, .validator = timezoneValidator },.{ .name = "block_private_networks", .type = bool },.{ .name = "block_cidrs", .type = ?[]const u8, .validator = accumulateValidator },.{ .name = "block_urls", .type = ?[]const u8, .validator = accumulateValidator },@@ -729,14 +729,14 @@ pub fn userAgent(self: *const Config) ?[]const u8 {};}-pub fn locale(self: *const Config) []const u8 {+pub fn locale(self: *const Config) [:0]const u8 {return switch (self.mode) {inline .serve, .fetch, .mcp, .agent => |opts| opts.locale,else => HttpHeaders.default_locale,};}-pub fn timezone(self: *const Config) ?[]const u8 {+pub fn timezone(self: *const Config) ?[:0]const u8 {return switch (self.mode) {inline .serve, .fetch, .mcp, .agent => |opts| opts.timezone,else => null,@@ -983,12 +983,10 @@ pub const HttpHeaders = struct {break :blk out;};- // Some bot-protection frontends (e.g. Akamai on canada.ca) RST the HTTP/2- // stream when a client sends Accept-Encoding without Accept-Language,- // treating it as a bot signal. Ship a neutral default so we look like a- // normal client.- pub const default_locale: []const u8 = "en-US";- pub const accept_language_default: [:0]const u8 = "en-US,en;q=0.9";+ // The neutral default: some bot-protection frontends (e.g. Akamai on+ // canada.ca) RST the HTTP/2 stream when a client sends Accept-Encoding+ // without Accept-Language.+ pub const default_locale: [:0]const u8 = "en-US";// Document-navigation Accept value Chrome sends.pub const navigation_accept: [:0]const u8 = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";@@ -997,11 +995,42 @@ pub const HttpHeaders = struct {proxy_bearer_header: ?[:0]const u8,- // Derived from --locale: the header and the navigator.languages list are- // the same information in two shapes, so they are built together.- accept_language: [:0]const u8,- languages: [2][]const u8,- languages_len: u8,+ accept_language: AcceptLanguage,++ /// An Accept-Language header and the tags it lists, which is what+ /// navigator.languages reports (Chrome keeps the two in step).+ pub const AcceptLanguage = struct {+ header: [:0]const u8,+ // Sub-slices of `header`.+ languages: []const []const u8,++ pub fn init(allocator: Allocator, value: []const u8) !AcceptLanguage {+ const header = try allocator.dupeZ(u8, value);+ errdefer allocator.free(header);++ var languages: std.ArrayList([]const u8) = .empty;+ errdefer languages.deinit(allocator);++ var it = std.mem.splitScalar(u8, header, ',');+ while (it.next()) |item| {+ const end = std.mem.indexOfScalar(u8, item, ';') orelse item.len;+ const tag = std.mem.trim(u8, item[0..end], " \t");+ if (tag.len > 0) {+ try languages.append(allocator, tag);+ }+ }++ return .{+ .header = header,+ .languages = try languages.toOwnedSlice(allocator),+ };+ }++ pub fn deinit(self: *const AcceptLanguage, allocator: Allocator) void {+ allocator.free(self.languages);+ allocator.free(self.header);+ }+ };pub fn init(allocator: Allocator, config: *const Config) !HttpHeaders {const user_agent: [:0]const u8 = if (config.userAgent()) |ua|@@ -1018,16 +1047,13 @@ pub const HttpHeaders = struct {null;errdefer if (proxy_bearer_header) |hdr| allocator.free(hdr);- const tag = config.locale();- const primary = primarySubtag(tag);- const accept_language = try acceptLanguageFor(allocator, tag);+ var buf: [64]u8 = undefined;+ const accept_language: AcceptLanguage = try .init(allocator, acceptLanguageFor(&buf, config.locale()));return .{.user_agent = user_agent,.proxy_bearer_header = proxy_bearer_header,.accept_language = accept_language,- .languages = .{ tag, primary },- .languages_len = if (primary.len == tag.len) 1 else 2,};}@@ -1038,36 +1064,25 @@ pub const HttpHeaders = struct {if (self.user_agent.ptr != user_agent_base.ptr) {allocator.free(self.user_agent);}- if (self.accept_language.ptr != accept_language_default.ptr) {- allocator.free(self.accept_language);- }+ self.accept_language.deinit(allocator);}- pub fn primarySubtag(tag: []const u8) []const u8 {- const end = std.mem.indexOfScalar(u8, tag, '-') orelse tag.len;- return tag[0..end];- }-- /// Chrome's shape: the tag first, then its language alone, then English- /// as a last resort, with descending q values. The default returns the- /// comptime constant so callers can tell it apart by pointer.- pub fn acceptLanguageFor(allocator: Allocator, tag: []const u8) ![:0]const u8 {- if (std.mem.eql(u8, tag, default_locale)) {- return accept_language_default;- }- const primary = primarySubtag(tag);- const has_region = primary.len != tag.len;- const is_english = std.ascii.eqlIgnoreCase(primary, "en");- if (has_region and !is_english) {- return std.fmt.allocPrintSentinel(allocator, "{s},{s};q=0.9,en;q=0.8", .{ tag, primary }, 0);+ /// Chrome's shape: the tag, then its language alone, then English as a+ /// last resort, with descending q values. `buf` must hold the longest+ /// output for a tag that passed validateLocale (35 + 24 bytes).+ pub fn acceptLanguageFor(buf: *[64]u8, tag: []const u8) []const u8 {+ const primary = tag[0 .. std.mem.indexOfScalar(u8, tag, '-') orelse tag.len];+ var w: std.Io.Writer = .fixed(buf);+ w.writeAll(tag) catch unreachable;+ var q: u8 = 9;+ if (primary.len != tag.len) {+ w.print(",{s};q=0.{d}", .{ primary, q }) catch unreachable;+ q -= 1;}- if (has_region) {- return std.fmt.allocPrintSentinel(allocator, "{s},{s};q=0.9", .{ tag, primary }, 0);+ if (!std.ascii.eqlIgnoreCase(primary, "en")) {+ w.print(",en;q=0.{d}", .{q}) catch unreachable;}- if (!is_english) {- return std.fmt.allocPrintSentinel(allocator, "{s},en;q=0.9", .{tag}, 0);- }- return allocator.dupeZ(u8, tag);+ return w.buffered();}};@@ -1332,7 +1347,6 @@ test "Config: validateTimezone" {}test "Config: HttpHeaders.acceptLanguageFor" {- const allocator = std.testing.allocator;const cases = [_]struct { tag: []const u8, expected: []const u8 }{.{ .tag = "en-US", .expected = "en-US,en;q=0.9" },.{ .tag = "en-GB", .expected = "en-GB,en;q=0.9" },@@ -1342,12 +1356,29 @@ test "Config: HttpHeaders.acceptLanguageFor" {.{ .tag = "zh-Hant-TW", .expected = "zh-Hant-TW,zh;q=0.9,en;q=0.8" },};for (cases) |case| {- const value = try HttpHeaders.acceptLanguageFor(allocator, case.tag);- defer if (value.ptr != HttpHeaders.accept_language_default.ptr) allocator.free(value);- try std.testing.expectEqualStrings(case.expected, value);+ var buf: [64]u8 = undefined;+ try std.testing.expectEqualStrings(case.expected, HttpHeaders.acceptLanguageFor(&buf, case.tag));+ }+}++test "Config: HttpHeaders.AcceptLanguage lists the header's tags" {+ const allocator = std.testing.allocator;+ const cases = [_]struct { header: []const u8, tags: []const []const u8 }{+ .{ .header = "en-US,en;q=0.9", .tags = &.{ "en-US", "en" } },+ .{ .header = "de-DE,de;q=0.9, en;q=0.8", .tags = &.{ "de-DE", "de", "en" } },+ .{ .header = "fr-FR", .tags = &.{"fr-FR"} },+ .{ .header = " , ;q=0.5,", .tags = &.{} },+ .{ .header = "", .tags = &.{} },+ };+ for (cases) |case| {+ const al: HttpHeaders.AcceptLanguage = try .init(allocator, case.header);+ defer al.deinit(allocator);+ try std.testing.expectEqualStrings(case.header, al.header);+ try std.testing.expectEqual(case.tags.len, al.languages.len);+ for (case.tags, al.languages) |expected, actual| {+ try std.testing.expectEqualStrings(expected, actual);+ }}- const default = try HttpHeaders.acceptLanguageFor(allocator, "en-US");- try std.testing.expectEqual(HttpHeaders.accept_language_default.ptr, default.ptr);}test "Config: locale drives http_headers" {@@ -1355,17 +1386,16 @@ test "Config: locale drives http_headers" {{var config = try Config.init(allocator, "test", .{ .serve = .{ .host = "127.0.0.1" } });defer config.deinit(allocator);- try std.testing.expectEqualStrings("en-US,en;q=0.9", config.http_headers.accept_language);- try std.testing.expectEqual(2, config.http_headers.languages_len);- try std.testing.expectEqualStrings("en-US", config.http_headers.languages[0]);- try std.testing.expectEqualStrings("en", config.http_headers.languages[1]);+ try std.testing.expectEqualStrings("en-US,en;q=0.9", config.http_headers.accept_language.header);+ try std.testing.expectEqual(2, config.http_headers.accept_language.languages.len);+ try std.testing.expectEqualStrings("en", config.http_headers.accept_language.languages[1]);}{var config = try Config.init(allocator, "test", .{ .serve = .{ .host = "127.0.0.1", .locale = "fr" } });defer config.deinit(allocator);- try std.testing.expectEqualStrings("fr,en;q=0.9", config.http_headers.accept_language);- try std.testing.expectEqual(1, config.http_headers.languages_len);- try std.testing.expectEqualStrings("fr", config.http_headers.languages[0]);+ try std.testing.expectEqualStrings("fr,en;q=0.9", config.http_headers.accept_language.header);+ try std.testing.expectEqual(2, config.http_headers.accept_language.languages.len);+ try std.testing.expectEqualStrings("fr", config.http_headers.accept_language.languages[0]);}}@@ -1514,22 +1544,22 @@ pub fn validateUserAgent(ua: []const u8) !void {}}-fn localeValidator(allocator: Allocator, args: *std.process.Args.Iterator, field: *[]const u8) !void {+fn localeValidator(allocator: Allocator, args: *std.process.Args.Iterator, field: *[:0]const u8) !void {const str = args.next() orelse return error.MissingArgument;validateLocale(str) catch |err| {log.fatal(.app, "invalid option value", .{ .arg = "--locale", .value = str, .err = err, .hint = "must be a BCP 47 tag such as en-US, de or zh-Hant-TW" });return error.InvalidArgument;};- field.* = try allocator.dupe(u8, str);+ field.* = try allocator.dupeZ(u8, str);}-fn timezoneValidator(allocator: Allocator, args: *std.process.Args.Iterator, field: *?[]const u8) !void {+fn timezoneValidator(allocator: Allocator, args: *std.process.Args.Iterator, field: *?[:0]const u8) !void {const str = args.next() orelse return error.MissingArgument;validateTimezone(str) catch |err| {log.fatal(.app, "invalid option value", .{ .arg = "--timezone", .value = str, .err = err, .hint = "must be an IANA time zone such as Europe/Paris or UTC" });return error.InvalidArgument;};- field.* = try allocator.dupe(u8, str);+ field.* = try allocator.dupeZ(u8, str);}/// A BCP 47 tag restricted to what ICU and the Accept-Language derivationdiff --git a/src/browser/js/Platform.zig b/src/browser/js/Platform.zigindex 942b84150..ca58de079 100644--- a/src/browser/js/Platform.zig+++ b/src/browser/js/Platform.zig@@ -16,7 +16,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-const std = @import("std");const js = @import("js.zig");const v8 = js.v8;@@ -26,27 +25,25 @@ handle: *v8.Platform,pub const Options = struct {v8_flags: ?[]const u8 = null,// BCP 47 tag; becomes ICU's default locale (Intl, toLocaleString).- locale: ?[]const u8 = null,+ locale: ?[:0]const u8 = null,// IANA id; becomes ICU's default time zone. Null keeps the host zone.- timezone: ?[]const u8 = null,+ timezone: ?[:0]const u8 = null,};/// ICU reads LC_ALL and TZ lazily on first use, so the environment must be/// set here, before InitializeICU and before the platform starts its thread-/// pool (setenv is not safe once other threads may call getenv).+/// pool (setenv is not safe once other threads may call getenv). ICU+/// canonicalizes a BCP 47 tag itself, script subtag included.pub fn init(opts: Options) !Platform {if (opts.v8_flags) |flags| {v8.v8__V8__SetFlagsFromString(flags.ptr, flags.len);}if (opts.locale) |tag| {- var buf: [32]u8 = undefined;- _ = setenv("LC_ALL", posixLocaleId(&buf, tag), 1);+ _ = setenv("LC_ALL", tag, 1);}if (opts.timezone) |id| {- var buf: [128]u8 = undefined;- const value = std.fmt.bufPrintZ(&buf, "{s}", .{id}) catch return error.TimezoneTooLong;- _ = setenv("TZ", value, 1);+ _ = setenv("TZ", id, 1);}if (v8.v8__V8__InitializeICU() == false) {@@ -66,40 +63,4 @@ pub fn deinit(self: Platform) void {v8.v8__Platform__DELETE(self.handle);}-/// `language[_REGION].UTF-8`, the POSIX id ICU parses from LC_ALL. The region-/// is the first 2-letter or 3-digit subtag, so a script subtag is skipped.-/// The tag is assumed valid per Config.validateLocale (language <= 3 bytes).-pub fn posixLocaleId(buf: *[32]u8, tag: []const u8) [:0]const u8 {- var it = std.mem.splitScalar(u8, tag, '-');- var language_buf: [3]u8 = undefined;- const language = std.ascii.lowerString(&language_buf, it.next().?);-- while (it.next()) |subtag| {- const is_alpha2 = subtag.len == 2 and std.ascii.isAlphabetic(subtag[0]) and std.ascii.isAlphabetic(subtag[1]);- const is_digit3 = subtag.len == 3 and std.ascii.isDigit(subtag[0]) and std.ascii.isDigit(subtag[1]) and std.ascii.isDigit(subtag[2]);- if (is_alpha2 or is_digit3) {- var region_buf: [3]u8 = undefined;- const region = std.ascii.upperString(®ion_buf, subtag);- return std.fmt.bufPrintZ(buf, "{s}_{s}.UTF-8", .{ language, region }) catch unreachable;- }- }- return std.fmt.bufPrintZ(buf, "{s}.UTF-8", .{language}) catch unreachable;-}-extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, override: c_int) c_int;--test "Platform: posixLocaleId" {- const cases = [_]struct { tag: []const u8, expected: []const u8 }{- .{ .tag = "en-US", .expected = "en_US.UTF-8" },- .{ .tag = "de-DE", .expected = "de_DE.UTF-8" },- .{ .tag = "en", .expected = "en.UTF-8" },- .{ .tag = "zh-Hant-TW", .expected = "zh_TW.UTF-8" },- .{ .tag = "es-419", .expected = "es_419.UTF-8" },- .{ .tag = "PT-br", .expected = "pt_BR.UTF-8" },- .{ .tag = "de-DE-1996", .expected = "de_DE.UTF-8" },- };- for (cases) |case| {- var buf: [32]u8 = undefined;- try std.testing.expectEqualStrings(case.expected, posixLocaleId(&buf, case.tag));- }-}diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 12ff36afe..b03270465 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -155,7 +155,7 @@ user_agent_override: ?[:0]const u8 = null,// Accept-Language override set via CDP Emulation.setUserAgentOverride.// Drives both the request header and navigator.languages.-accept_language_override: ?AcceptLanguageOverride = null,+accept_language_override: ?lp.Config.HttpHeaders.AcceptLanguage = null,// The driver (CDP / BiDi) attached to us. If there's a driver, then there's// an inbox for us to process (and there's someone to wake us up from a poll)@@ -308,48 +308,15 @@ pub fn clearUserAgentOverride(self: *Client) void {}}-const AcceptLanguageOverride = struct {- // The header as given; `languages` are sub-slices of it.- header: [:0]const u8,- languages: []const []const u8,-- const max_languages = 8;-};--/// Set an Accept-Language override, allocated from self.allocator. The-/// navigator.languages list is the header's tags with weights dropped.+// Set an Accept-Language override, allocated from self.allocator.pub fn setAcceptLanguageOverride(self: *Client, value: []const u8) !void {self.clearAcceptLanguageOverride();-- const header = try self.allocator.dupeZ(u8, value);- errdefer self.allocator.free(header);-- var languages: std.ArrayList([]const u8) = .empty;- errdefer languages.deinit(self.allocator);-- var it = std.mem.splitScalar(u8, header, ',');- while (it.next()) |item| {- if (languages.items.len == AcceptLanguageOverride.max_languages) {- break;- }- const end = std.mem.indexOfScalar(u8, item, ';') orelse item.len;- const tag = std.mem.trim(u8, item[0..end], " \t");- if (tag.len == 0) {- continue;- }- try languages.append(self.allocator, tag);- }-- self.accept_language_override = .{- .header = header,- .languages = try languages.toOwnedSlice(self.allocator),- };+ self.accept_language_override = try .init(self.allocator, value);}pub fn clearAcceptLanguageOverride(self: *Client) void {if (self.accept_language_override) |override| {- self.allocator.free(override.languages);- self.allocator.free(override.header);+ override.deinit(self.allocator);self.accept_language_override = null;}}@@ -480,20 +447,11 @@ pub fn getUserAgent(self: *const Client) [:0]const u8 {}pub fn getAcceptLanguage(self: *const Client) [:0]const u8 {- if (self.accept_language_override) |override| {- return override.header;- }- return self.network.config.http_headers.accept_language;+ return (self.accept_language_override orelse self.network.config.http_headers.accept_language).header;}-/// What navigator.languages reports: the override's tags when set, else the-/// list derived from --locale.pub fn getLanguages(self: *const Client) []const []const u8 {- if (self.accept_language_override) |override| {- return override.languages;- }- const headers = &self.network.config.http_headers;- return headers.languages[0..headers.languages_len];+ return (self.accept_language_override orelse self.network.config.http_headers.accept_language).languages;}// Headers _all_ requests include.@@ -4300,7 +4258,7 @@ test "HttpClient: setBlockedUrls owns, replaces, and clears patterns" {try testing.expectEqual(null, client.url_blocklist);}-test "HttpClient: setAcceptLanguageOverride owns, parses, replaces, and clears" {+test "HttpClient: setAcceptLanguageOverride owns, replaces, and clears" {var pool = ArenaPool.init(testing.allocator, .{});defer pool.deinit();@@ -4308,24 +4266,15 @@ test "HttpClient: setAcceptLanguageOverride owns, parses, replaces, and clears"initTestClient(&client, &pool);defer client.clearAcceptLanguageOverride();- var first = "de-DE,de;q=0.9, en;q=0.8".*;+ var first = "de-DE,de;q=0.9".*;try client.setAcceptLanguageOverride(&first);@memset(&first, 'x');-- try std.testing.expectEqualStrings("de-DE,de;q=0.9, en;q=0.8", client.getAcceptLanguage());- try testing.expectEqual(3, client.getLanguages().len);- try std.testing.expectEqualStrings("de-DE", client.getLanguages()[0]);- try std.testing.expectEqualStrings("de", client.getLanguages()[1]);- try std.testing.expectEqualStrings("en", client.getLanguages()[2]);+ try std.testing.expectEqualStrings("de-DE,de;q=0.9", client.getAcceptLanguage());+ try testing.expectEqual(2, client.getLanguages().len);try client.setAcceptLanguageOverride("fr-FR");- try testing.expectEqual(1, client.getLanguages().len);try std.testing.expectEqualStrings("fr-FR", client.getLanguages()[0]);- try client.setAcceptLanguageOverride("");- try std.testing.expectEqualStrings("", client.getAcceptLanguage());- try testing.expectEqual(0, client.getLanguages().len);-client.clearAcceptLanguageOverride();try testing.expectEqual(null, client.accept_language_override);}diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex e2be5d3f1..06ea118e6 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -478,7 +478,6 @@ pub const BrowserContext = struct {http_proxy_changed: bool = false,user_agent_changed: bool = false,- accept_language_changed: bool = false,// Extra headers to add to all requests.extra_headers: std.ArrayList(http.Header) = .empty,@@ -628,9 +627,7 @@ pub const BrowserContext = struct {if (self.user_agent_changed) {browser.http_client.clearUserAgentOverride();}- if (self.accept_language_changed) {- browser.http_client.clearAcceptLanguageOverride();- }+ browser.http_client.clearAcceptLanguageOverride();self.intercept_state.deinit();}diff --git a/src/server/cdp/domains/emulation.zig b/src/server/cdp/domains/emulation.zigindex a841023a6..a92a095f9 100644--- a/src/server/cdp/domains/emulation.zig+++ b/src/server/cdp/domains/emulation.zig@@ -163,9 +163,9 @@ pub fn setUserAgentOverride(cmd: *CDP.Command) !void {}const ua = params.userAgent;- Config.validateUserAgent(ua) catch |err| switch (err) {+ const reserved = if (Config.validateUserAgent(ua)) false else |err| switch (err) {error.NonPrintable => return cmd.sendError(-32602, "User agent contains non-printable characters", .{}),- error.Reserved => {},+ error.Reserved => true,};const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;@@ -178,10 +178,9 @@ pub fn setUserAgentOverride(cmd: *CDP.Command) !void {return cmd.sendError(-32602, "Accept-Language contains CR, LF or NUL", .{});}try http_client.setAcceptLanguageOverride(accept_language);- bc.accept_language_changed = true;}- if (std.ascii.indexOfIgnoreCase(ua, "mozilla") != null) {+ if (reserved) {log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "userAgent", .value = ua, .info = "User agent must not contain Mozilla" });return cmd.sendResult(null, .{});}@@ -414,7 +413,6 @@ test "cdp.Emulation: setUserAgentOverride acceptLanguage drives navigator.languatry ctx.expectSentResult(null, .{ .id = 1 });try expectJs(frame, "navigator.language === 'de-DE' && navigator.languages.join() === 'de-DE,de,en'");try std.testing.expectEqualStrings("de-DE,de;q=0.9, en;q=0.8", ctx.cdp().browser.http_client.getAcceptLanguage());- try testing.expect(bc.accept_language_changed);// A Mozilla user agent is refused, the language still applies.try ctx.processMessage(.{