Lightpanda: the headless browser designed for AI and automation
Store a script's resolved encoding and let V8 validate UTF-8
Adrià Arrufatb65c6caa06ba
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zigindex f60a10604..e29736d47 100644--- a/src/browser/ScriptManager.zig+++ b/src/browser/ScriptManager.zig@@ -376,7 +376,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e// There was a preloaded script, we borrow it's source and statusconsumed_preload = pre;script.source = pre.source;- script.charset = pre.charset;+ script.encoding = pre.encoding;script.status = pre.status;script.complete = true;} else {@@ -402,7 +402,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e// has to outlive this call, up to script.deinit().script.source_arena = response.arena;script.source = .{ .remote = response.body };- try script.setContentType(HttpClient.findHeader(response.headers, "content-type"));+ script.setContentType(HttpClient.findHeader(response.headers, "content-type"));script.status = response.status;script.complete = true;}diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zigindex ea827b448..af51f457c 100644--- a/src/browser/ScriptManagerBase.zig+++ b/src/browser/ScriptManagerBase.zig@@ -604,8 +604,8 @@ pub const Script = struct {// limited .large arena poolsource_arena: ?*lp.Arena = null,- // The Content-Type charset of a remote script, in `arena`.- charset: ?[]const u8 = null,+ // The encoding a remote script's Content-Type charset names.+ encoding: ?h5e.EncodingInfo = null,extra: Extra,node: std.DoublyLinkedList.Node,@@ -695,11 +695,9 @@ pub const Script = struct {return (self.source_arena orelse self.arena).allocator();}- pub fn setContentType(self: *Script, content_type: ?[]const u8) !void {+ pub fn setContentType(self: *Script, content_type: ?[]const u8) void {const mime = Mime.parse(content_type orelse return) catch return;- if (mime.explicitCharset()) |charset| {- self.charset = try self.arena.allocator().dupe(u8, charset);- }+ self.encoding = .forLabel(mime.explicitCharset());}/// A classic script's text. Remote bytes are decoded with the@@ -707,15 +705,20 @@ pub const Script = struct {/// document's encoding; a BOM overrides all three./// https://html.spec.whatwg.org/#fetch-a-classic-scriptfn classicText(self: *Script, fe: Extra.FrameExtra) ![]const u8 {- const bytes = self.source.content();if (self.source != .remote) {+ return self.source.content();+ }+ const bytes = self.source.content();+ const encoding = self.encoding orelse+ h5e.EncodingInfo.forLabel(fe.script_element.asElement().getAttributeSafe(comptime .wrap("charset"))) orelse+ h5e.EncodingInfo.forLabel(fe.frame.charset) orelse+ h5e.EncodingInfo.utf8();+ // V8 reads UTF-8 itself, replacing malformed sequences as decoding+ // would, so only a UTF-16 BOM needs the decoder.+ if (encoding.isUtf8() and !std.mem.startsWith(u8, bytes, "\xFE\xFF") and !std.mem.startsWith(u8, bytes, "\xFF\xFE")) {return bytes;}- const encoding = h5e.encodingFor(self.charset) orelse- h5e.encodingFor(fe.script_element.asElement().getAttributeSafe(comptime .wrap("charset"))) orelse- h5e.encodingFor(fe.frame.charset) orelse- h5e.utf8();- return h5e.asUtf8(self.sourceAllocator(), encoding, bytes);+ return h5e.decodeToUtf8(self.sourceAllocator(), encoding.handle.?, bytes);}pub fn startCallback(transfer: *HttpClient.Transfer) !void {@@ -775,7 +778,7 @@ pub const Script = struct {self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;}- try self.setContentType(transfer.contentType());+ self.setContentType(transfer.contentType());lp.assert(self.source.remote.capacity == 0, "ScriptManagerBase.Header buffer", .{ .capacity = self.source.remote.capacity });diff --git a/src/browser/css/charset.zig b/src/browser/css/charset.zigindex 0bd2d44f8..86a529452 100644--- a/src/browser/css/charset.zig+++ b/src/browser/css/charset.zig@@ -24,20 +24,21 @@ const std = @import("std");const h5e = @import("../parser/html5ever.zig");const Allocator = std.mem.Allocator;+const EncodingInfo = h5e.EncodingInfo;/// Decodes `bytes` to UTF-8. A BOM wins, then `protocol_charset` (the/// Content-Type charset), then a leading `@charset` rule, then/// `environment_charset` (the document's encoding). Malformed sequences/// become U+FFFD. Returns `bytes` itself when it is already valid UTF-8.pub fn decode(allocator: Allocator, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) ![]const u8 {- return h5e.asUtf8(allocator, fallbackEncoding(bytes, protocol_charset, environment_charset), bytes);+ return h5e.validUtf8(allocator, fallbackEncoding(bytes, protocol_charset, environment_charset), bytes);}/// https://drafts.csswg.org/css-syntax/#determine-the-fallback-encoding-fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) h5e.EncodingInfo {- if (h5e.encodingFor(protocol_charset)) |info| return info;- if (h5e.encodingFor(charsetRuleLabel(bytes))) |info| return if (info.isUtf16()) h5e.utf8() else info;- return h5e.encodingFor(environment_charset) orelse h5e.utf8();+fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) EncodingInfo {+ if (EncodingInfo.forLabel(protocol_charset)) |info| return info;+ if (EncodingInfo.forLabel(charsetRuleLabel(bytes))) |info| return if (info.isUtf16()) .utf8() else info;+ return EncodingInfo.forLabel(environment_charset) orelse .utf8();}/// The label of a `@charset "<label>";` rule opening the first 1024 bytes,diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zigindex 456501d9c..73e23449c 100644--- a/src/browser/parser/html5ever.zig+++ b/src/browser/parser/html5ever.zig@@ -242,28 +242,32 @@ pub const EncodingInfo = extern struct {return self.name_ptr[0..self.name_len];}+ /// The encoding `label` names, if it names one.+ pub fn forLabel(label: ?[]const u8) ?EncodingInfo {+ const l = label orelse return null;+ const info = encoding_for_label(l.ptr, l.len);+ return if (info.isValid()) info else null;+ }++ pub fn utf8() EncodingInfo {+ return forLabel("utf-8").?;+ }++ pub fn isUtf8(self: *const EncodingInfo) bool {+ return std.mem.eql(u8, self.name(), "UTF-8");+ }+pub fn isUtf16(self: *const EncodingInfo) bool {const n = self.name();return std.mem.eql(u8, n, "UTF-16LE") or std.mem.eql(u8, n, "UTF-16BE");}};-/// The encoding `label` names, if it names one.-pub fn encodingFor(label: ?[]const u8) ?EncodingInfo {- const l = label orelse return null;- const info = encoding_for_label(l.ptr, l.len);- return if (info.isValid()) info else null;-}--pub fn utf8() EncodingInfo {- return encodingFor("utf-8").?;-}--/// `decodeToUtf8`, except that `input` is returned as is when it is already-/// valid UTF-8 without a BOM and `encoding` is UTF-8.-pub fn asUtf8(allocator: std.mem.Allocator, encoding: EncodingInfo, input: []const u8) ![]const u8 {+/// `input` as valid UTF-8 without a BOM: `input` itself when it already is+/// and `encoding` is UTF-8, else decoded with `decodeToUtf8`.+pub fn validUtf8(allocator: std.mem.Allocator, encoding: EncodingInfo, input: []const u8) ![]const u8 {// A UTF-16 BOM is never valid UTF-8, so only the UTF-8 one needs a check.- if (std.mem.eql(u8, encoding.name(), "UTF-8") and+ if (encoding.isUtf8() and!std.mem.startsWith(u8, input, "\xEF\xBB\xBF") andstd.unicode.utf8ValidateSlice(input)){diff --git a/src/testing.zig b/src/testing.zigindex 8d25c7cb1..4b6b1bce3 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -1200,9 +1200,10 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {});}- if (std.mem.startsWith(u8, path, "/scripts/latin1/") or std.mem.startsWith(u8, path, "/scripts/latin1-header/")) {+ const declares_charset = std.mem.startsWith(u8, path, "/scripts/latin1-header/");+ if (declares_charset or std.mem.startsWith(u8, path, "/scripts/latin1/")) {const body = try std.fmt.allocPrint(req_allocator, "window['{s}'] = 'caf\xE9';", .{std.fs.path.stem(path)});- const content_type = if (std.mem.startsWith(u8, path, "/scripts/latin1/")) "text/javascript" else "text/javascript; charset=iso-8859-1";+ const content_type = if (declares_charset) "text/javascript; charset=iso-8859-1" else "text/javascript";return req.respond(body, .{.extra_headers = &.{.{ .name = "Content-Type", .value = content_type },