Lightpanda: the headless browser designed for AI and automation
cli: add --locale and --timezone, derive language signals from one value
Adrià Arrufatc55afc1df9b3
diff --git a/src/App.zig b/src/App.zigindex 484b67984..fe5a0effa 100644--- a/src/App.zig+++ b/src/App.zig@@ -44,7 +44,11 @@ arena_pool: ArenaPool,app_dir_path: ?[]const u8,pub fn init(allocator: Allocator, config: *const Config) !*App {- const platform = try Platform.init(config.v8Flags());+ const platform = try Platform.init(.{+ .v8_flags = config.v8Flags(),+ .locale = config.locale(),+ .timezone = config.timezone(),+ });errdefer platform.deinit();const snapshot = try Snapshot.load();diff --git a/src/Config.zig b/src/Config.zigindex 031becf6b..9db44ba42 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -276,6 +276,8 @@ const CommonOptions = .{.{ .name = "web_bot_auth_keyid", .type = ?[]const u8 },.{ .name = "web_bot_auth_domain", .type = ?[]const u8 },.{ .name = "user_agent", .type = ?[]const u8, .validator = userAgentValidator },+ .{ .name = "locale", .type = []const u8, .default = HttpHeaders.default_locale, .validator = localeValidator },+ .{ .name = "timezone", .type = ?[]const u8, .validator = timezoneValidator },.{ .name = "block_private_networks", .type = bool },.{ .name = "block_cidrs", .type = ?[]const u8, .validator = accumulateValidator },.{ .name = "block_urls", .type = ?[]const u8, .validator = accumulateValidator },@@ -727,6 +729,20 @@ pub fn userAgent(self: *const Config) ?[]const u8 {};}+pub fn locale(self: *const Config) []const u8 {+ return switch (self.mode) {+ inline .serve, .fetch, .mcp, .agent => |opts| opts.locale,+ else => HttpHeaders.default_locale,+ };+}++pub fn timezone(self: *const Config) ?[]const u8 {+ return switch (self.mode) {+ inline .serve, .fetch, .mcp, .agent => |opts| opts.timezone,+ else => null,+ };+}+pub fn httpCacheDir(self: *const Config) ?[]const u8 {return switch (self.mode) {inline .serve, .fetch, .mcp, .agent => |opts| opts.http_cache_dir,@@ -971,7 +987,8 @@ pub const HttpHeaders = struct {// stream when a client sends Accept-Encoding without Accept-Language,// treating it as a bot signal. Ship a neutral default so we look like a// normal client.- pub const accept_language: [:0]const u8 = "en-US,en;q=0.9";+ pub const default_locale: []const u8 = "en-US";+ pub const accept_language_default: [:0]const u8 = "en-US,en;q=0.9";// Document-navigation Accept value Chrome sends.pub const navigation_accept: [:0]const u8 = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";@@ -980,6 +997,12 @@ pub const HttpHeaders = struct {proxy_bearer_header: ?[:0]const u8,+ // Derived from --locale: the header and the navigator.languages list are+ // the same information in two shapes, so they are built together.+ accept_language: [:0]const u8,+ languages: [2][]const u8,+ languages_len: u8,+pub fn init(allocator: Allocator, config: *const Config) !HttpHeaders {const user_agent: [:0]const u8 = if (config.userAgent()) |ua|try allocator.dupeZ(u8, ua)@@ -993,10 +1016,18 @@ pub const HttpHeaders = struct {try std.fmt.allocPrintSentinel(allocator, "Proxy-Authorization: Bearer {s}", .{token}, 0)elsenull;+ errdefer if (proxy_bearer_header) |hdr| allocator.free(hdr);++ const tag = config.locale();+ const primary = primarySubtag(tag);+ const accept_language = try acceptLanguageFor(allocator, tag);return .{.user_agent = user_agent,.proxy_bearer_header = proxy_bearer_header,+ .accept_language = accept_language,+ .languages = .{ tag, primary },+ .languages_len = if (primary.len == tag.len) 1 else 2,};}@@ -1007,6 +1038,36 @@ pub const HttpHeaders = struct {if (self.user_agent.ptr != user_agent_base.ptr) {allocator.free(self.user_agent);}+ if (self.accept_language.ptr != accept_language_default.ptr) {+ allocator.free(self.accept_language);+ }+ }++ pub fn primarySubtag(tag: []const u8) []const u8 {+ const end = std.mem.indexOfScalar(u8, tag, '-') orelse tag.len;+ return tag[0..end];+ }++ /// Chrome's shape: the tag first, then its language alone, then English+ /// as a last resort, with descending q values. The default returns the+ /// comptime constant so callers can tell it apart by pointer.+ pub fn acceptLanguageFor(allocator: Allocator, tag: []const u8) ![:0]const u8 {+ if (std.mem.eql(u8, tag, default_locale)) {+ return accept_language_default;+ }+ const primary = primarySubtag(tag);+ const has_region = primary.len != tag.len;+ const is_english = std.ascii.eqlIgnoreCase(primary, "en");+ if (has_region and !is_english) {+ return std.fmt.allocPrintSentinel(allocator, "{s},{s};q=0.9,en;q=0.8", .{ tag, primary }, 0);+ }+ if (has_region) {+ return std.fmt.allocPrintSentinel(allocator, "{s},{s};q=0.9", .{ tag, primary }, 0);+ }+ if (!is_english) {+ return std.fmt.allocPrintSentinel(allocator, "{s},en;q=0.9", .{tag}, 0);+ }+ return allocator.dupeZ(u8, tag);}};@@ -1245,6 +1306,84 @@ test "Config: validateUserAgent" {try std.testing.expectError(error.NonPrintable, validateUserAgent("bad\x01ua"));}+test "Config: validateLocale" {+ try validateLocale("en");+ try validateLocale("en-US");+ try validateLocale("zh-Hant-TW");+ try validateLocale("es-419");+ try std.testing.expectError(error.InvalidLanguage, validateLocale(""));+ try std.testing.expectError(error.InvalidLanguage, validateLocale("e"));+ try std.testing.expectError(error.InvalidLanguage, validateLocale("en_US"));+ try std.testing.expectError(error.InvalidSubtag, validateLocale("en-"));+ try std.testing.expectError(error.InvalidSubtag, validateLocale("en-U"));+ try std.testing.expectError(error.InvalidSubtag, validateLocale("en-US-x-toolongsub"));+ try std.testing.expectError(error.InvalidSubtag, validateLocale("en-U$"));+ try std.testing.expectError(error.TooLong, validateLocale("en-" ++ "a" ** 40));+}++test "Config: validateTimezone" {+ try validateTimezone("UTC");+ try validateTimezone("Europe/Paris");+ try validateTimezone("America/Argentina/Buenos_Aires");+ try std.testing.expectError(error.Empty, validateTimezone(""));+ try std.testing.expectError(error.InvalidCharacter, validateTimezone("Europe/ Paris"));+ try std.testing.expectError(error.InvalidCharacter, validateTimezone("UTC\n"));+ try std.testing.expectError(error.TooLong, validateTimezone("a" ** 65));+}++test "Config: HttpHeaders.acceptLanguageFor" {+ const allocator = std.testing.allocator;+ const cases = [_]struct { tag: []const u8, expected: []const u8 }{+ .{ .tag = "en-US", .expected = "en-US,en;q=0.9" },+ .{ .tag = "en-GB", .expected = "en-GB,en;q=0.9" },+ .{ .tag = "de-DE", .expected = "de-DE,de;q=0.9,en;q=0.8" },+ .{ .tag = "de", .expected = "de,en;q=0.9" },+ .{ .tag = "en", .expected = "en" },+ .{ .tag = "zh-Hant-TW", .expected = "zh-Hant-TW,zh;q=0.9,en;q=0.8" },+ };+ for (cases) |case| {+ const value = try HttpHeaders.acceptLanguageFor(allocator, case.tag);+ defer if (value.ptr != HttpHeaders.accept_language_default.ptr) allocator.free(value);+ try std.testing.expectEqualStrings(case.expected, value);+ }+ const default = try HttpHeaders.acceptLanguageFor(allocator, "en-US");+ try std.testing.expectEqual(HttpHeaders.accept_language_default.ptr, default.ptr);+}++test "Config: locale drives http_headers" {+ const allocator = std.testing.allocator;+ {+ var config = try Config.init(allocator, "test", .{ .serve = .{ .host = "127.0.0.1" } });+ defer config.deinit(allocator);+ try std.testing.expectEqualStrings("en-US,en;q=0.9", config.http_headers.accept_language);+ try std.testing.expectEqual(2, config.http_headers.languages_len);+ try std.testing.expectEqualStrings("en-US", config.http_headers.languages[0]);+ try std.testing.expectEqualStrings("en", config.http_headers.languages[1]);+ }+ {+ var config = try Config.init(allocator, "test", .{ .serve = .{ .host = "127.0.0.1", .locale = "fr" } });+ defer config.deinit(allocator);+ try std.testing.expectEqualStrings("fr,en;q=0.9", config.http_headers.accept_language);+ try std.testing.expectEqual(1, config.http_headers.languages_len);+ try std.testing.expectEqualStrings("fr", config.http_headers.languages[0]);+ }+}++test "Config: parseArgs refuses an invalid --locale and --timezone" {+ {+ log.expectLog(&.{.app});+ const argv = [_][*:0]const u8{ "lightpanda", "fetch", "--locale", "en_US" };+ const proc_args: std.process.Args = .{ .vector = &argv };+ try std.testing.expectError(error.InvalidArgument, parseArgs(std.testing.allocator, proc_args));+ }+ {+ log.expectLog(&.{.app});+ const argv = [_][*:0]const u8{ "lightpanda", "fetch", "--timezone", "Europe/ Paris" };+ const proc_args: std.process.Args = .{ .vector = &argv };+ try std.testing.expectError(error.InvalidArgument, parseArgs(std.testing.allocator, proc_args));+ }+}+test "Config: parseArgs refuses an invalid --http-header" {const invalid = [_][*:0]const u8{// no colon to split on@@ -1375,6 +1514,68 @@ pub fn validateUserAgent(ua: []const u8) !void {}}+fn localeValidator(allocator: Allocator, args: *std.process.Args.Iterator, field: *[]const u8) !void {+ const str = args.next() orelse return error.MissingArgument;+ validateLocale(str) catch |err| {+ log.fatal(.app, "invalid option value", .{ .arg = "--locale", .value = str, .err = err, .hint = "must be a BCP 47 tag such as en-US, de or zh-Hant-TW" });+ return error.InvalidArgument;+ };+ field.* = try allocator.dupe(u8, str);+}++fn timezoneValidator(allocator: Allocator, args: *std.process.Args.Iterator, field: *?[]const u8) !void {+ const str = args.next() orelse return error.MissingArgument;+ validateTimezone(str) catch |err| {+ log.fatal(.app, "invalid option value", .{ .arg = "--timezone", .value = str, .err = err, .hint = "must be an IANA time zone such as Europe/Paris or UTC" });+ return error.InvalidArgument;+ };+ field.* = try allocator.dupe(u8, str);+}++/// A BCP 47 tag restricted to what ICU and the Accept-Language derivation+/// need: a 2-3 letter language followed by 2-8 character alphanumeric subtags.+pub fn validateLocale(tag: []const u8) !void {+ if (tag.len > 35) {+ return error.TooLong;+ }+ var it = std.mem.splitScalar(u8, tag, '-');+ const language = it.next().?;+ if (language.len < 2 or language.len > 3) {+ return error.InvalidLanguage;+ }+ for (language) |c| {+ if (!std.ascii.isAlphabetic(c)) {+ return error.InvalidLanguage;+ }+ }+ while (it.next()) |subtag| {+ if (subtag.len < 2 or subtag.len > 8) {+ return error.InvalidSubtag;+ }+ for (subtag) |c| {+ if (!std.ascii.isAlphanumeric(c)) {+ return error.InvalidSubtag;+ }+ }+ }+}++/// Only the shape is checked; ICU resolves the id itself and falls back to+/// GMT for names it does not know.+pub fn validateTimezone(id: []const u8) !void {+ if (id.len == 0) {+ return error.Empty;+ }+ if (id.len > 64) {+ return error.TooLong;+ }+ for (id) |c| {+ if (c <= ' ' or c == 0x7f) {+ return error.InvalidCharacter;+ }+ }+}+/// Tag names of a Zig enum, so a command's allowed values can't drift from the/// enum it sets.pub const tagNames = cli.tagNames;diff --git a/src/browser/js/Platform.zig b/src/browser/js/Platform.zigindex ab9a60c08..942b84150 100644--- a/src/browser/js/Platform.zig+++ b/src/browser/js/Platform.zig@@ -16,17 +16,39 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+const std = @import("std");const js = @import("js.zig");const v8 = js.v8;const Platform = @This();handle: *v8.Platform,-pub fn init(v8_flags: ?[]const u8) !Platform {- if (v8_flags) |flags| {+pub const Options = struct {+ v8_flags: ?[]const u8 = null,+ // BCP 47 tag; becomes ICU's default locale (Intl, toLocaleString).+ locale: ?[]const u8 = null,+ // IANA id; becomes ICU's default time zone. Null keeps the host zone.+ timezone: ?[]const u8 = null,+};++/// ICU reads LC_ALL and TZ lazily on first use, so the environment must be+/// set here, before InitializeICU and before the platform starts its thread+/// pool (setenv is not safe once other threads may call getenv).+pub fn init(opts: Options) !Platform {+ if (opts.v8_flags) |flags| {v8.v8__V8__SetFlagsFromString(flags.ptr, flags.len);}+ if (opts.locale) |tag| {+ var buf: [32]u8 = undefined;+ _ = setenv("LC_ALL", posixLocaleId(&buf, tag), 1);+ }+ if (opts.timezone) |id| {+ var buf: [128]u8 = undefined;+ const value = std.fmt.bufPrintZ(&buf, "{s}", .{id}) catch return error.TimezoneTooLong;+ _ = setenv("TZ", value, 1);+ }+if (v8.v8__V8__InitializeICU() == false) {return error.FailedToInitializeICU;}@@ -43,3 +65,41 @@ pub fn deinit(self: Platform) void {v8.v8__V8__DisposePlatform();v8.v8__Platform__DELETE(self.handle);}++/// `language[_REGION].UTF-8`, the POSIX id ICU parses from LC_ALL. The region+/// is the first 2-letter or 3-digit subtag, so a script subtag is skipped.+/// The tag is assumed valid per Config.validateLocale (language <= 3 bytes).+pub fn posixLocaleId(buf: *[32]u8, tag: []const u8) [:0]const u8 {+ var it = std.mem.splitScalar(u8, tag, '-');+ var language_buf: [3]u8 = undefined;+ const language = std.ascii.lowerString(&language_buf, it.next().?);++ while (it.next()) |subtag| {+ const is_alpha2 = subtag.len == 2 and std.ascii.isAlphabetic(subtag[0]) and std.ascii.isAlphabetic(subtag[1]);+ const is_digit3 = subtag.len == 3 and std.ascii.isDigit(subtag[0]) and std.ascii.isDigit(subtag[1]) and std.ascii.isDigit(subtag[2]);+ if (is_alpha2 or is_digit3) {+ var region_buf: [3]u8 = undefined;+ const region = std.ascii.upperString(®ion_buf, subtag);+ return std.fmt.bufPrintZ(buf, "{s}_{s}.UTF-8", .{ language, region }) catch unreachable;+ }+ }+ return std.fmt.bufPrintZ(buf, "{s}.UTF-8", .{language}) catch unreachable;+}++extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, override: c_int) c_int;++test "Platform: posixLocaleId" {+ const cases = [_]struct { tag: []const u8, expected: []const u8 }{+ .{ .tag = "en-US", .expected = "en_US.UTF-8" },+ .{ .tag = "de-DE", .expected = "de_DE.UTF-8" },+ .{ .tag = "en", .expected = "en.UTF-8" },+ .{ .tag = "zh-Hant-TW", .expected = "zh_TW.UTF-8" },+ .{ .tag = "es-419", .expected = "es_419.UTF-8" },+ .{ .tag = "PT-br", .expected = "pt_BR.UTF-8" },+ .{ .tag = "de-DE-1996", .expected = "de_DE.UTF-8" },+ };+ for (cases) |case| {+ var buf: [32]u8 = undefined;+ try std.testing.expectEqualStrings(case.expected, posixLocaleId(&buf, case.tag));+ }+}diff --git a/src/browser/webapi/Navigator.zig b/src/browser/webapi/Navigator.zigindex 4beeae7dc..65fd19d86 100644--- a/src/browser/webapi/Navigator.zig+++ b/src/browser/webapi/Navigator.zig@@ -53,8 +53,8 @@ pub fn getUserAgent(_: *const Navigator, exec: *const Execution) []const u8 {return exec.session.browser.http_client.getUserAgent();}-pub fn getLanguages(_: *const Navigator) [2][]const u8 {- return .{ "en-US", "en" };+pub fn getLanguages(_: *const Navigator, exec: *const Execution) []const []const u8 {+ return exec.session.browser.http_client.getLanguages();}pub fn getDoNotTrack(_: *const Navigator) ?[]const u8 {@@ -73,8 +73,9 @@ pub fn getAppVersion(_: *const Navigator) []const u8 {return "1.0";}-pub fn getLanguage(_: *const Navigator) []const u8 {- return "en-US";+pub fn getLanguage(self: *const Navigator, exec: *const Execution) []const u8 {+ const languages = self.getLanguages(exec);+ return if (languages.len == 0) "" else languages[0];}pub fn getOnLine(_: *const Navigator) bool {diff --git a/src/browser/webapi/WorkerNavigator.zig b/src/browser/webapi/WorkerNavigator.zigindex cced8ae70..7a03cedac 100644--- a/src/browser/webapi/WorkerNavigator.zig+++ b/src/browser/webapi/WorkerNavigator.zig@@ -46,8 +46,8 @@ pub fn getUserAgent(_: *const WorkerNavigator, exec: *const Execution) []const ureturn Navigator.getUserAgent(&Navigator.init, exec);}-pub fn getLanguages(_: *const WorkerNavigator) [2][]const u8 {- return Navigator.getLanguages(&Navigator.init);+pub fn getLanguages(_: *const WorkerNavigator, exec: *const Execution) []const []const u8 {+ return Navigator.getLanguages(&Navigator.init, exec);}pub fn getAppName(_: *const WorkerNavigator) []const u8 {@@ -62,8 +62,8 @@ pub fn getAppVersion(_: *const WorkerNavigator) []const u8 {return Navigator.getAppVersion(&Navigator.init);}-pub fn getLanguage(_: *const WorkerNavigator) []const u8 {- return Navigator.getLanguage(&Navigator.init);+pub fn getLanguage(_: *const WorkerNavigator, exec: *const Execution) []const u8 {+ return Navigator.getLanguage(&Navigator.init, exec);}pub fn getOnLine(_: *const WorkerNavigator) bool {diff --git a/src/help.zon b/src/help.zonindex dbb92e69f..809efa3b0 100644--- a/src/help.zon+++ b/src/help.zon@@ -459,6 +459,10 @@\\ The log level.\\ Defaults to {1s}.\\ Allowed values: "debug", "info", "warn", "error", "fatal".+ \\ --locale <TAG>+ \\ BCP 47 language tag driving navigator.language, the+ \\ Accept-Language header and the default locale of Intl and+ \\ toLocaleString. Defaults to en-US.\\ --obey-robots\\ Fetches and obeys robots.txt of the target page.\\ Defaults to false.@@ -472,6 +476,9 @@\\ --storage-sqlite-path <PATH>\\ Path to the SQLite database file for persistent storage.\\ Use ":memory:" for in-memory storage.+ \\ --timezone <IANA>+ \\ Time zone used by Date and Intl, e.g. Europe/Paris or UTC.+ \\ Defaults to the host time zone.\\ --user-agent <STRING>\\ Override the User-Agent header entirely. Must not impersonate other\\ browsers; any value containing "Mozilla" is forbidden. The browserdiff --git a/src/main_snapshot_creator.zig b/src/main_snapshot_creator.zigindex a5b3736d8..03378b6ad 100644--- a/src/main_snapshot_creator.zig+++ b/src/main_snapshot_creator.zig@@ -39,7 +39,7 @@ pub fn main(init: std.process.Init) !void {}}- var platform = try lp.js.Platform.init(v8_flags);+ var platform = try lp.js.Platform.init(.{ .v8_flags = v8_flags });defer platform.deinit();const snapshot = try lp.js.Snapshot.create();diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 2ff673156..12ff36afe 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -153,6 +153,10 @@ test_fail_submit: if (lp.IS_TEST) ?anyerror else void = if (lp.IS_TEST) null els// Allocated from self.allocator when set, null otherwise.user_agent_override: ?[:0]const u8 = null,+// Accept-Language override set via CDP Emulation.setUserAgentOverride.+// Drives both the request header and navigator.languages.+accept_language_override: ?AcceptLanguageOverride = null,+// The driver (CDP / BiDi) attached to us. If there's a driver, then there's// an inbox for us to process (and there's someone to wake us up from a poll)driver: ?Driver = null,@@ -263,6 +267,7 @@ pub fn deinit(self: *Client) void {self.handles.deinit();self.clearUserAgentOverride();+ self.clearAcceptLanguageOverride();if (self.http_proxy_owned) |owned| {self.allocator.free(owned);}@@ -303,6 +308,52 @@ pub fn clearUserAgentOverride(self: *Client) void {}}+const AcceptLanguageOverride = struct {+ // The header as given; `languages` are sub-slices of it.+ header: [:0]const u8,+ languages: []const []const u8,++ const max_languages = 8;+};++/// Set an Accept-Language override, allocated from self.allocator. The+/// navigator.languages list is the header's tags with weights dropped.+pub fn setAcceptLanguageOverride(self: *Client, value: []const u8) !void {+ self.clearAcceptLanguageOverride();++ const header = try self.allocator.dupeZ(u8, value);+ errdefer self.allocator.free(header);++ var languages: std.ArrayList([]const u8) = .empty;+ errdefer languages.deinit(self.allocator);++ var it = std.mem.splitScalar(u8, header, ',');+ while (it.next()) |item| {+ if (languages.items.len == AcceptLanguageOverride.max_languages) {+ break;+ }+ const end = std.mem.indexOfScalar(u8, item, ';') orelse item.len;+ const tag = std.mem.trim(u8, item[0..end], " \t");+ if (tag.len == 0) {+ continue;+ }+ try languages.append(self.allocator, tag);+ }++ self.accept_language_override = .{+ .header = header,+ .languages = try languages.toOwnedSlice(self.allocator),+ };+}++pub fn clearAcceptLanguageOverride(self: *Client) void {+ if (self.accept_language_override) |override| {+ self.allocator.free(override.languages);+ self.allocator.free(override.header);+ self.accept_language_override = null;+ }+}+// Enable TLS verification on all connections.pub fn setTlsVerify(self: *Client, verify: bool) !void {// Remove inflight connections check on enable TLS b/c chromiumoxide calls@@ -428,6 +479,23 @@ pub fn getUserAgent(self: *const Client) [:0]const u8 {return self.user_agent_override orelse self.network.config.http_headers.user_agent;}+pub fn getAcceptLanguage(self: *const Client) [:0]const u8 {+ if (self.accept_language_override) |override| {+ return override.header;+ }+ return self.network.config.http_headers.accept_language;+}++/// What navigator.languages reports: the override's tags when set, else the+/// list derived from --locale.+pub fn getLanguages(self: *const Client) []const []const u8 {+ if (self.accept_language_override) |override| {+ return override.languages;+ }+ const headers = &self.network.config.http_headers;+ return headers.languages[0..headers.languages_len];+}+// Headers _all_ requests include.pub fn baselineHeaders(self: *const Client) [4]Transfer.RequestHeader {return .{@@ -436,7 +504,7 @@ pub fn baselineHeaders(self: *const Client) [4]Transfer.RequestHeader {.{ .name = "Sec-Ch-Ua-Full-Version-List", .value = lp.Config.HttpHeaders.sec_ch_ua_full_version_list, .source = .fixed },// Omitting Accept-Language triggers bot-protection on some CDNs// (Akamai) when Accept-Encoding is present.- .{ .name = "Accept-Language", .value = lp.Config.HttpHeaders.accept_language },+ .{ .name = "Accept-Language", .value = self.getAcceptLanguage() },};}@@ -4203,6 +4271,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {.single_flight = .init(testing.allocator),};client.url_blocklist = null;+ client.accept_language_override = null;client.test_fail_submit = null;// isUrlBlocked reaches through here for the adblocker; tests that want// one assign it to `client.network` after this returns.@@ -4231,6 +4300,36 @@ test "HttpClient: setBlockedUrls owns, replaces, and clears patterns" {try testing.expectEqual(null, client.url_blocklist);}+test "HttpClient: setAcceptLanguageOverride owns, parses, replaces, and clears" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ var client: Client = undefined;+ initTestClient(&client, &pool);+ defer client.clearAcceptLanguageOverride();++ var first = "de-DE,de;q=0.9, en;q=0.8".*;+ try client.setAcceptLanguageOverride(&first);+ @memset(&first, 'x');++ try std.testing.expectEqualStrings("de-DE,de;q=0.9, en;q=0.8", client.getAcceptLanguage());+ try testing.expectEqual(3, client.getLanguages().len);+ try std.testing.expectEqualStrings("de-DE", client.getLanguages()[0]);+ try std.testing.expectEqualStrings("de", client.getLanguages()[1]);+ try std.testing.expectEqualStrings("en", client.getLanguages()[2]);++ try client.setAcceptLanguageOverride("fr-FR");+ try testing.expectEqual(1, client.getLanguages().len);+ try std.testing.expectEqualStrings("fr-FR", client.getLanguages()[0]);++ try client.setAcceptLanguageOverride("");+ try std.testing.expectEqualStrings("", client.getAcceptLanguage());+ try testing.expectEqual(0, client.getLanguages().len);++ client.clearAcceptLanguageOverride();+ try testing.expectEqual(null, client.accept_language_override);+}+const TestRequest = struct {url: [:0]const u8,document: [:0]const u8 = "",diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex 4785f787a..e2be5d3f1 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -478,6 +478,7 @@ pub const BrowserContext = struct {http_proxy_changed: bool = false,user_agent_changed: bool = false,+ accept_language_changed: bool = false,// Extra headers to add to all requests.extra_headers: std.ArrayList(http.Header) = .empty,@@ -627,6 +628,9 @@ pub const BrowserContext = struct {if (self.user_agent_changed) {browser.http_client.clearUserAgentOverride();}+ if (self.accept_language_changed) {+ browser.http_client.clearAcceptLanguageOverride();+ }self.intercept_state.deinit();}diff --git a/src/server/cdp/domains/emulation.zig b/src/server/cdp/domains/emulation.zigindex 37f3aae93..a841023a6 100644--- a/src/server/cdp/domains/emulation.zig+++ b/src/server/cdp/domains/emulation.zig@@ -21,6 +21,7 @@ const lp = @import("lightpanda");const CDP = @import("../CDP.zig");const Config = @import("../../../Config.zig");+const Mime = @import("../../../browser/Mime.zig");const js = @import("../../../browser/js/js.zig");const log = lp.log;@@ -157,9 +158,6 @@ pub fn setUserAgentOverride(cmd: *CDP.Command) !void {platform: ?[]const u8 = null,})) orelse return error.InvalidParams;- if (params.acceptLanguage) |v| {- log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "acceptLanguage", .value = v });- }if (params.platform) |v| {log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "platform", .value = v });}@@ -167,14 +165,27 @@ pub fn setUserAgentOverride(cmd: *CDP.Command) !void {const ua = params.userAgent;Config.validateUserAgent(ua) catch |err| switch (err) {error.NonPrintable => return cmd.sendError(-32602, "User agent contains non-printable characters", .{}),- error.Reserved => {- log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "userAgent", .value = ua, .info = "User agent must not contain Mozilla" });- return cmd.sendResult(null, .{});- },+ error.Reserved => {},};const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;const http_client = &cmd.cdp.browser.http_client;++ // Applied even when the user agent is refused below: Playwright always+ // sends a Mozilla user agent alongside the locale it was asked for.+ if (params.acceptLanguage) |accept_language| {+ if (!Mime.isHttpHeaderValue(accept_language)) {+ return cmd.sendError(-32602, "Accept-Language contains CR, LF or NUL", .{});+ }+ try http_client.setAcceptLanguageOverride(accept_language);+ bc.accept_language_changed = true;+ }++ if (std.ascii.indexOfIgnoreCase(ua, "mozilla") != null) {+ log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "userAgent", .value = ua, .info = "User agent must not contain Mozilla" });+ return cmd.sendResult(null, .{});+ }+try http_client.setUserAgentOverride(ua);bc.user_agent_changed = true;@@ -211,8 +222,10 @@ fn clearGeolocationOverride(cmd: *CDP.Command) !void {return cmd.sendResult(null, .{});}-// Accepted so drivers can finish context setup; Intl/Date keep the host's-// locale and timezone.+// Accepted so drivers can finish context setup; Intl/Date keep the process+// locale and timezone (--locale, --timezone). Changing them at runtime needs+// zig-v8-fork bindings for Isolate::DateTimeConfigurationChangeNotification+// and the ICU default locale.fn setLocaleOverride(cmd: *CDP.Command) !void {const Params = struct { locale: ?[]const u8 = null };const params = (try cmd.params(Params)) orelse Params{};@@ -381,6 +394,62 @@ test "cdp.Emulation: setUserAgentOverride with optional params" {try ctx.expectSentResult(null, .{ .id = 5 });}+test "cdp.Emulation: setUserAgentOverride acceptLanguage drives navigator.languages" {+ testing.silenceLog(&.{.not_implemented});++ var ctx = try testing.context();+ defer ctx.deinit();+ var bc = try ctx.loadBrowserContext(.{ .id = "BID-AL", .url = "hi.html", .target_id = "FID-00000000AL".* });+ const frame = bc.mainFrame() orelse unreachable;++ // The default locale reaches both navigator and ICU.+ try expectJs(frame, "navigator.language === 'en-US' && navigator.languages.join() === 'en-US,en'");+ try expectJs(frame, "Intl.DateTimeFormat().resolvedOptions().locale === navigator.language");++ try ctx.processMessage(.{+ .id = 1,+ .method = "Emulation.setUserAgentOverride",+ .params = .{ .userAgent = "CustomBot/2.0", .acceptLanguage = "de-DE,de;q=0.9, en;q=0.8" },+ });+ try ctx.expectSentResult(null, .{ .id = 1 });+ try expectJs(frame, "navigator.language === 'de-DE' && navigator.languages.join() === 'de-DE,de,en'");+ try std.testing.expectEqualStrings("de-DE,de;q=0.9, en;q=0.8", ctx.cdp().browser.http_client.getAcceptLanguage());+ try testing.expect(bc.accept_language_changed);++ // A Mozilla user agent is refused, the language still applies.+ try ctx.processMessage(.{+ .id = 2,+ .method = "Emulation.setUserAgentOverride",+ .params = .{ .userAgent = "Mozilla/5.0", .acceptLanguage = "fr-FR" },+ });+ try ctx.expectSentResult(null, .{ .id = 2 });+ try expectJs(frame, "navigator.language === 'fr-FR' && navigator.languages.length === 1");+ try std.testing.expectEqualStrings("CustomBot/2.0", ctx.cdp().browser.http_client.getUserAgent());++ try ctx.processMessage(.{+ .id = 3,+ .method = "Emulation.setUserAgentOverride",+ .params = .{ .userAgent = "CustomBot/2.0", .acceptLanguage = "" },+ });+ try ctx.expectSentResult(null, .{ .id = 3 });+ try expectJs(frame, "navigator.language === '' && navigator.languages.length === 0");++ try ctx.processMessage(.{+ .id = 4,+ .method = "Emulation.setUserAgentOverride",+ .params = .{ .userAgent = "CustomBot/2.0", .acceptLanguage = "en\r\nX-Injected: 1" },+ });+ try ctx.expectSentError(-32602, "Accept-Language contains CR, LF or NUL", .{ .id = 4 });+}++fn expectJs(frame: *lp.Frame, expression: [:0]const u8) !void {+ var ls: js.Local.Scope = undefined;+ frame.js.localScope(&ls);+ defer ls.deinit();+ const value = try ls.local.exec(expression, null);+ try testing.expect(value.toBool());+}+test "cdp.Emulation: setUserAgentOverride can be called multiple times" {var ctx = try testing.context();defer ctx.deinit();