Skip to content

Lightpanda: the headless browser designed for AI and automation

Store a script's resolved encoding and let V8 validate UTF-8

Adrià Arrufatb65c6caa06ba
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig
index f60a10604..e29736d47 100644
--- a/src/browser/ScriptManager.zig
+++ b/src/browser/ScriptManager.zig
@@ -376,7 +376,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
// There was a preloaded script, we borrow it's source and status
consumed_preload = pre;
script.source = pre.source;
- script.charset = pre.charset;
+ script.encoding = pre.encoding;
script.status = pre.status;
script.complete = true;
} else {
@@ -402,7 +402,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
// has to outlive this call, up to script.deinit().
script.source_arena = response.arena;
script.source = .{ .remote = response.body };
- try script.setContentType(HttpClient.findHeader(response.headers, "content-type"));
+ script.setContentType(HttpClient.findHeader(response.headers, "content-type"));
script.status = response.status;
script.complete = true;
}
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zig
index ea827b448..af51f457c 100644
--- a/src/browser/ScriptManagerBase.zig
+++ b/src/browser/ScriptManagerBase.zig
@@ -604,8 +604,8 @@ pub const Script = struct {
// limited .large arena pool
source_arena: ?*lp.Arena = null,
- // The Content-Type charset of a remote script, in `arena`.
- charset: ?[]const u8 = null,
+ // The encoding a remote script's Content-Type charset names.
+ encoding: ?h5e.EncodingInfo = null,
extra: Extra,
node: std.DoublyLinkedList.Node,
@@ -695,11 +695,9 @@ pub const Script = struct {
return (self.source_arena orelse self.arena).allocator();
}
- pub fn setContentType(self: *Script, content_type: ?[]const u8) !void {
+ pub fn setContentType(self: *Script, content_type: ?[]const u8) void {
const mime = Mime.parse(content_type orelse return) catch return;
- if (mime.explicitCharset()) |charset| {
- self.charset = try self.arena.allocator().dupe(u8, charset);
- }
+ self.encoding = .forLabel(mime.explicitCharset());
}
/// A classic script's text. Remote bytes are decoded with the
@@ -707,15 +705,20 @@ pub const Script = struct {
/// document's encoding; a BOM overrides all three.
/// https://html.spec.whatwg.org/#fetch-a-classic-script
fn classicText(self: *Script, fe: Extra.FrameExtra) ![]const u8 {
- const bytes = self.source.content();
if (self.source != .remote) {
+ return self.source.content();
+ }
+ const bytes = self.source.content();
+ const encoding = self.encoding orelse
+ h5e.EncodingInfo.forLabel(fe.script_element.asElement().getAttributeSafe(comptime .wrap("charset"))) orelse
+ h5e.EncodingInfo.forLabel(fe.frame.charset) orelse
+ h5e.EncodingInfo.utf8();
+ // V8 reads UTF-8 itself, replacing malformed sequences as decoding
+ // would, so only a UTF-16 BOM needs the decoder.
+ if (encoding.isUtf8() and !std.mem.startsWith(u8, bytes, "\xFE\xFF") and !std.mem.startsWith(u8, bytes, "\xFF\xFE")) {
return bytes;
}
- const encoding = h5e.encodingFor(self.charset) orelse
- h5e.encodingFor(fe.script_element.asElement().getAttributeSafe(comptime .wrap("charset"))) orelse
- h5e.encodingFor(fe.frame.charset) orelse
- h5e.utf8();
- return h5e.asUtf8(self.sourceAllocator(), encoding, bytes);
+ return h5e.decodeToUtf8(self.sourceAllocator(), encoding.handle.?, bytes);
}
pub fn startCallback(transfer: *HttpClient.Transfer) !void {
@@ -775,7 +778,7 @@ pub const Script = struct {
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
}
- try self.setContentType(transfer.contentType());
+ self.setContentType(transfer.contentType());
lp.assert(self.source.remote.capacity == 0, "ScriptManagerBase.Header buffer", .{ .capacity = self.source.remote.capacity });
diff --git a/src/browser/css/charset.zig b/src/browser/css/charset.zig
index 0bd2d44f8..86a529452 100644
--- a/src/browser/css/charset.zig
+++ b/src/browser/css/charset.zig
@@ -24,20 +24,21 @@ const std = @import("std");
const h5e = @import("../parser/html5ever.zig");
const Allocator = std.mem.Allocator;
+const EncodingInfo = h5e.EncodingInfo;
/// Decodes `bytes` to UTF-8. A BOM wins, then `protocol_charset` (the
/// Content-Type charset), then a leading `@charset` rule, then
/// `environment_charset` (the document's encoding). Malformed sequences
/// become U+FFFD. Returns `bytes` itself when it is already valid UTF-8.
pub fn decode(allocator: Allocator, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) ![]const u8 {
- return h5e.asUtf8(allocator, fallbackEncoding(bytes, protocol_charset, environment_charset), bytes);
+ return h5e.validUtf8(allocator, fallbackEncoding(bytes, protocol_charset, environment_charset), bytes);
}
/// https://drafts.csswg.org/css-syntax/#determine-the-fallback-encoding
-fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) h5e.EncodingInfo {
- if (h5e.encodingFor(protocol_charset)) |info| return info;
- if (h5e.encodingFor(charsetRuleLabel(bytes))) |info| return if (info.isUtf16()) h5e.utf8() else info;
- return h5e.encodingFor(environment_charset) orelse h5e.utf8();
+fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) EncodingInfo {
+ if (EncodingInfo.forLabel(protocol_charset)) |info| return info;
+ if (EncodingInfo.forLabel(charsetRuleLabel(bytes))) |info| return if (info.isUtf16()) .utf8() else info;
+ return EncodingInfo.forLabel(environment_charset) orelse .utf8();
}
/// The label of a `@charset "<label>";` rule opening the first 1024 bytes,
diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zig
index 456501d9c..73e23449c 100644
--- a/src/browser/parser/html5ever.zig
+++ b/src/browser/parser/html5ever.zig
@@ -242,28 +242,32 @@ pub const EncodingInfo = extern struct {
return self.name_ptr[0..self.name_len];
}
+ /// The encoding `label` names, if it names one.
+ pub fn forLabel(label: ?[]const u8) ?EncodingInfo {
+ const l = label orelse return null;
+ const info = encoding_for_label(l.ptr, l.len);
+ return if (info.isValid()) info else null;
+ }
+
+ pub fn utf8() EncodingInfo {
+ return forLabel("utf-8").?;
+ }
+
+ pub fn isUtf8(self: *const EncodingInfo) bool {
+ return std.mem.eql(u8, self.name(), "UTF-8");
+ }
+
pub fn isUtf16(self: *const EncodingInfo) bool {
const n = self.name();
return std.mem.eql(u8, n, "UTF-16LE") or std.mem.eql(u8, n, "UTF-16BE");
}
};
-/// The encoding `label` names, if it names one.
-pub fn encodingFor(label: ?[]const u8) ?EncodingInfo {
- const l = label orelse return null;
- const info = encoding_for_label(l.ptr, l.len);
- return if (info.isValid()) info else null;
-}
-
-pub fn utf8() EncodingInfo {
- return encodingFor("utf-8").?;
-}
-
-/// `decodeToUtf8`, except that `input` is returned as is when it is already
-/// valid UTF-8 without a BOM and `encoding` is UTF-8.
-pub fn asUtf8(allocator: std.mem.Allocator, encoding: EncodingInfo, input: []const u8) ![]const u8 {
+/// `input` as valid UTF-8 without a BOM: `input` itself when it already is
+/// and `encoding` is UTF-8, else decoded with `decodeToUtf8`.
+pub fn validUtf8(allocator: std.mem.Allocator, encoding: EncodingInfo, input: []const u8) ![]const u8 {
// A UTF-16 BOM is never valid UTF-8, so only the UTF-8 one needs a check.
- if (std.mem.eql(u8, encoding.name(), "UTF-8") and
+ if (encoding.isUtf8() and
!std.mem.startsWith(u8, input, "\xEF\xBB\xBF") and
std.unicode.utf8ValidateSlice(input))
{
diff --git a/src/testing.zig b/src/testing.zig
index 8d25c7cb1..4b6b1bce3 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -1200,9 +1200,10 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
- if (std.mem.startsWith(u8, path, "/scripts/latin1/") or std.mem.startsWith(u8, path, "/scripts/latin1-header/")) {
+ const declares_charset = std.mem.startsWith(u8, path, "/scripts/latin1-header/");
+ if (declares_charset or std.mem.startsWith(u8, path, "/scripts/latin1/")) {
const body = try std.fmt.allocPrint(req_allocator, "window['{s}'] = 'caf\xE9';", .{std.fs.path.stem(path)});
- const content_type = if (std.mem.startsWith(u8, path, "/scripts/latin1/")) "text/javascript" else "text/javascript; charset=iso-8859-1";
+ const content_type = if (declares_charset) "text/javascript; charset=iso-8859-1" else "text/javascript";
return req.respond(body, .{
.extra_headers = &.{
.{ .name = "Content-Type", .value = content_type },