Lightpanda: the headless browser designed for AI and automation
Share the UTF-8 decode step between css/charset and FileReader
Adrià Arrufat46fadeeac9df
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 08bdd2848..94913f187 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1770,10 +1770,6 @@ fn urlBasename(arena: Allocator, url: []const u8) !?[]const u8 {return try arena.dupe(u8, name);}-fn isUtf16Encoding(charset: []const u8) bool {- return std.mem.eql(u8, charset, "UTF-16LE") or std.mem.eql(u8, charset, "UTF-16BE");-}-fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {var self: *Frame = @ptrCast(@alignCast(transfer.req.ctx));@@ -1835,7 +1831,7 @@ fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {const info = h5e.encoding_for_label(charset_str.ptr, charset_str.len);if (info.isValid()) {const name = info.name();- self.charset = if (html_prescan_found_charset and isUtf16Encoding(name)) "UTF-8" else name;+ self.charset = if (html_prescan_found_charset and info.isUtf16()) "UTF-8" else name;}self._parse_state = .{ .html = .{.buffer = .empty,@@ -2705,12 +2701,11 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co// scratch-list pattern in `CSSStyleSheet.replaceSync`. Keeping// `_href` consistent with what the sheet actually contains is the// minimum.- const content_type: ?Mime = for (response.headers) |header| {- if (std.ascii.eqlIgnoreCase(header.name, "content-type")) {- break Mime.parse(header.value) catch null;- }- } else null;- const protocol_charset = if (content_type) |*mime| (if (mime.is_default_charset) null else mime.charsetString()) else null;+ const content_type: ?Mime = if (HttpClient.findHeader(response.headers, "content-type")) |value|+ Mime.parse(value) catch null+ else+ null;+ const protocol_charset = if (content_type) |*mime| mime.explicitCharset() else null;const text = try css_charset.decode(arena.allocator(), response.body.items, protocol_charset, self.charset);sheet.replaceSync(text, self) catch |err| {log.debug(.http, "external stylesheet parse", .{ .err = err, .url = resolved });diff --git a/src/browser/Mime.zig b/src/browser/Mime.zigindex 772b49b02..c9f6c8422 100644--- a/src/browser/Mime.zig+++ b/src/browser/Mime.zig@@ -176,6 +176,11 @@ pub fn charsetString(mime: *const Mime) []const u8 {return mime.charset[0..mime.charset_len];}+/// The charset parameter, if the input gave one.+pub fn explicitCharset(mime: *const Mime) ?[]const u8 {+ return if (mime.is_default_charset) null else mime.charsetString();+}+pub fn parse(input: []const u8) !Mime {if (input.len > 255) {return error.TooBig;diff --git a/src/browser/css/Tokenizer.zig b/src/browser/css/Tokenizer.zigindex 73d1721dd..17aaabe97 100644--- a/src/browser/css/Tokenizer.zig+++ b/src/browser/css/Tokenizer.zig@@ -333,7 +333,7 @@ fn byteToDecimalDigit(b: u8) ?u32 {return if (std.ascii.isDigit(b)) b - '0' else null;}-// Up to 6 hex digits.+/// Up to 6 hex digits.fn consumeHexDigits(self: *Tokenizer) void {var digits: u32 = 0;while (digits < 6 and !self.isEof() and std.ascii.isHex(self.nextByteUnchecked())) {diff --git a/src/browser/css/charset.zig b/src/browser/css/charset.zigindex 5e1108082..58f6a70ba 100644--- a/src/browser/css/charset.zig+++ b/src/browser/css/charset.zig@@ -32,18 +32,15 @@ const Allocator = std.mem.Allocator;pub fn decode(allocator: Allocator, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) ![]const u8 {const encoding = fallbackEncoding(bytes, protocol_charset, environment_charset);- // The decoder sniffs the BOM itself, but the fast path must not keep one.- const has_bom = std.mem.startsWith(u8, bytes, "\xEF\xBB\xBF") or- std.mem.startsWith(u8, bytes, "\xFE\xFF") or- std.mem.startsWith(u8, bytes, "\xFF\xFE");- if (!has_bom and std.mem.eql(u8, encoding.name(), "UTF-8") and std.unicode.utf8ValidateSlice(bytes)) {+ // A UTF-16 BOM is never valid UTF-8, so only the UTF-8 one can slip+ // through the fast path; the decoder drops it.+ if (std.mem.eql(u8, encoding.name(), "UTF-8") and+ !std.mem.startsWith(u8, bytes, "\xEF\xBB\xBF") and+ std.unicode.utf8ValidateSlice(bytes))+ {return bytes;}-- const handle = encoding.handle.?;- const out = try allocator.alloc(u8, h5e.encoding_max_utf8_buffer_length(handle, bytes.len));- const result = h5e.encoding_decode(handle, bytes.ptr, bytes.len, out.ptr, out.len, 1);- return out[0..result.bytes_written];+ return h5e.decodeToUtf8(allocator, encoding.handle.?, bytes);}/// https://drafts.csswg.org/css-syntax/#determine-the-fallback-encoding@@ -55,11 +52,7 @@ fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environmenif (charsetRuleLabel(bytes)) |label| {const info = h5e.encoding_for_label(label.ptr, label.len);- if (info.isValid()) {- const name = info.name();- if (!std.mem.eql(u8, name, "UTF-16BE") and !std.mem.eql(u8, name, "UTF-16LE")) return info;- return utf8();- }+ if (info.isValid()) return if (info.isUtf16()) utf8() else info;}const info = h5e.encoding_for_label(environment_charset.ptr, environment_charset.len);diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zigindex f4773a933..a03bacdda 100644--- a/src/browser/parser/html5ever.zig+++ b/src/browser/parser/html5ever.zig@@ -16,6 +16,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+const std = @import("std");const ParsedNode = @import("Parser.zig").ParsedNode;pub extern "c" fn html5ever_parse_document(@@ -240,8 +241,24 @@ pub const EncodingInfo = extern struct {}return self.name_ptr[0..self.name_len];}++ pub fn isUtf16(self: *const EncodingInfo) bool {+ const n = self.name();+ return std.mem.eql(u8, n, "UTF-16LE") or std.mem.eql(u8, n, "UTF-16BE");+ }};+/// Decodes `input` from the encoding `handle` names into UTF-8. A BOM+/// switches to the encoding it marks and is dropped.+pub fn decodeToUtf8(allocator: std.mem.Allocator, handle: *anyopaque, input: []const u8) ![]const u8 {+ if (input.len == 0) {+ return "";+ }+ const out = try allocator.alloc(u8, encoding_max_utf8_buffer_length(handle, input.len));+ const result = encoding_decode(handle, input.ptr, input.len, out.ptr, out.len, 1);+ return out[0..result.bytes_written];+}+const DecodeResult = extern struct {had_errors: u8,bytes_read: usize,diff --git a/src/browser/webapi/FileReader.zig b/src/browser/webapi/FileReader.zigindex e03dbf273..8f1146f14 100644--- a/src/browser/webapi/FileReader.zig+++ b/src/browser/webapi/FileReader.zig@@ -342,29 +342,14 @@ pub fn decodeText(arena: Allocator, input: []const u8, label_: ?[]const u8, mimeif (handle == null and mime_type.len > 0) {if (Mime.parse(mime_type)) |mime| {- if (mime.is_default_charset == false) {- handle = encodingHandle(mime.charsetString());+ if (mime.explicitCharset()) |charset| {+ handle = encodingHandle(charset);}} else |_| {}}const encoding = handle orelse encodingHandle("utf-8") orelse return arena.dupe(u8, data);-- const max_out = html5ever.encoding_max_utf8_buffer_length(encoding, if (data.len == 0) 4 else data.len);- if (max_out == 0) {- return "";- }-- const out = try arena.alloc(u8, max_out);- const result = html5ever.encoding_decode(- encoding,- if (data.len == 0) null else data.ptr,- data.len,- out.ptr,- out.len,- 1, // is_last- );- return out[0..result.bytes_written];+ return html5ever.decodeToUtf8(arena, encoding, data);}pub fn abort(self: *FileReader) !void {