Skip to content

Lightpanda: the headless browser designed for AI and automation

Share the UTF-8 decode step between css/charset and FileReader

Adrià Arrufat46fadeeac9df
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 08bdd2848..94913f187 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -1770,10 +1770,6 @@ fn urlBasename(arena: Allocator, url: []const u8) !?[]const u8 {
return try arena.dupe(u8, name);
}
-fn isUtf16Encoding(charset: []const u8) bool {
- return std.mem.eql(u8, charset, "UTF-16LE") or std.mem.eql(u8, charset, "UTF-16BE");
-}
-
fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
var self: *Frame = @ptrCast(@alignCast(transfer.req.ctx));
@@ -1835,7 +1831,7 @@ fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
const info = h5e.encoding_for_label(charset_str.ptr, charset_str.len);
if (info.isValid()) {
const name = info.name();
- self.charset = if (html_prescan_found_charset and isUtf16Encoding(name)) "UTF-8" else name;
+ self.charset = if (html_prescan_found_charset and info.isUtf16()) "UTF-8" else name;
}
self._parse_state = .{ .html = .{
.buffer = .empty,
@@ -2705,12 +2701,11 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
// scratch-list pattern in `CSSStyleSheet.replaceSync`. Keeping
// `_href` consistent with what the sheet actually contains is the
// minimum.
- const content_type: ?Mime = for (response.headers) |header| {
- if (std.ascii.eqlIgnoreCase(header.name, "content-type")) {
- break Mime.parse(header.value) catch null;
- }
- } else null;
- const protocol_charset = if (content_type) |*mime| (if (mime.is_default_charset) null else mime.charsetString()) else null;
+ const content_type: ?Mime = if (HttpClient.findHeader(response.headers, "content-type")) |value|
+ Mime.parse(value) catch null
+ else
+ null;
+ const protocol_charset = if (content_type) |*mime| mime.explicitCharset() else null;
const text = try css_charset.decode(arena.allocator(), response.body.items, protocol_charset, self.charset);
sheet.replaceSync(text, self) catch |err| {
log.debug(.http, "external stylesheet parse", .{ .err = err, .url = resolved });
diff --git a/src/browser/Mime.zig b/src/browser/Mime.zig
index 772b49b02..c9f6c8422 100644
--- a/src/browser/Mime.zig
+++ b/src/browser/Mime.zig
@@ -176,6 +176,11 @@ pub fn charsetString(mime: *const Mime) []const u8 {
return mime.charset[0..mime.charset_len];
}
+/// The charset parameter, if the input gave one.
+pub fn explicitCharset(mime: *const Mime) ?[]const u8 {
+ return if (mime.is_default_charset) null else mime.charsetString();
+}
+
pub fn parse(input: []const u8) !Mime {
if (input.len > 255) {
return error.TooBig;
diff --git a/src/browser/css/Tokenizer.zig b/src/browser/css/Tokenizer.zig
index 73d1721dd..17aaabe97 100644
--- a/src/browser/css/Tokenizer.zig
+++ b/src/browser/css/Tokenizer.zig
@@ -333,7 +333,7 @@ fn byteToDecimalDigit(b: u8) ?u32 {
return if (std.ascii.isDigit(b)) b - '0' else null;
}
-// Up to 6 hex digits.
+/// Up to 6 hex digits.
fn consumeHexDigits(self: *Tokenizer) void {
var digits: u32 = 0;
while (digits < 6 and !self.isEof() and std.ascii.isHex(self.nextByteUnchecked())) {
diff --git a/src/browser/css/charset.zig b/src/browser/css/charset.zig
index 5e1108082..58f6a70ba 100644
--- a/src/browser/css/charset.zig
+++ b/src/browser/css/charset.zig
@@ -32,18 +32,15 @@ const Allocator = std.mem.Allocator;
pub fn decode(allocator: Allocator, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) ![]const u8 {
const encoding = fallbackEncoding(bytes, protocol_charset, environment_charset);
- // The decoder sniffs the BOM itself, but the fast path must not keep one.
- const has_bom = std.mem.startsWith(u8, bytes, "\xEF\xBB\xBF") or
- std.mem.startsWith(u8, bytes, "\xFE\xFF") or
- std.mem.startsWith(u8, bytes, "\xFF\xFE");
- if (!has_bom and std.mem.eql(u8, encoding.name(), "UTF-8") and std.unicode.utf8ValidateSlice(bytes)) {
+ // A UTF-16 BOM is never valid UTF-8, so only the UTF-8 one can slip
+ // through the fast path; the decoder drops it.
+ if (std.mem.eql(u8, encoding.name(), "UTF-8") and
+ !std.mem.startsWith(u8, bytes, "\xEF\xBB\xBF") and
+ std.unicode.utf8ValidateSlice(bytes))
+ {
return bytes;
}
-
- const handle = encoding.handle.?;
- const out = try allocator.alloc(u8, h5e.encoding_max_utf8_buffer_length(handle, bytes.len));
- const result = h5e.encoding_decode(handle, bytes.ptr, bytes.len, out.ptr, out.len, 1);
- return out[0..result.bytes_written];
+ return h5e.decodeToUtf8(allocator, encoding.handle.?, bytes);
}
/// https://drafts.csswg.org/css-syntax/#determine-the-fallback-encoding
@@ -55,11 +52,7 @@ fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environmen
if (charsetRuleLabel(bytes)) |label| {
const info = h5e.encoding_for_label(label.ptr, label.len);
- if (info.isValid()) {
- const name = info.name();
- if (!std.mem.eql(u8, name, "UTF-16BE") and !std.mem.eql(u8, name, "UTF-16LE")) return info;
- return utf8();
- }
+ if (info.isValid()) return if (info.isUtf16()) utf8() else info;
}
const info = h5e.encoding_for_label(environment_charset.ptr, environment_charset.len);
diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zig
index f4773a933..a03bacdda 100644
--- a/src/browser/parser/html5ever.zig
+++ b/src/browser/parser/html5ever.zig
@@ -16,6 +16,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+const std = @import("std");
const ParsedNode = @import("Parser.zig").ParsedNode;
pub extern "c" fn html5ever_parse_document(
@@ -240,8 +241,24 @@ pub const EncodingInfo = extern struct {
}
return self.name_ptr[0..self.name_len];
}
+
+ pub fn isUtf16(self: *const EncodingInfo) bool {
+ const n = self.name();
+ return std.mem.eql(u8, n, "UTF-16LE") or std.mem.eql(u8, n, "UTF-16BE");
+ }
};
+/// Decodes `input` from the encoding `handle` names into UTF-8. A BOM
+/// switches to the encoding it marks and is dropped.
+pub fn decodeToUtf8(allocator: std.mem.Allocator, handle: *anyopaque, input: []const u8) ![]const u8 {
+ if (input.len == 0) {
+ return "";
+ }
+ const out = try allocator.alloc(u8, encoding_max_utf8_buffer_length(handle, input.len));
+ const result = encoding_decode(handle, input.ptr, input.len, out.ptr, out.len, 1);
+ return out[0..result.bytes_written];
+}
+
const DecodeResult = extern struct {
had_errors: u8,
bytes_read: usize,
diff --git a/src/browser/webapi/FileReader.zig b/src/browser/webapi/FileReader.zig
index e03dbf273..8f1146f14 100644
--- a/src/browser/webapi/FileReader.zig
+++ b/src/browser/webapi/FileReader.zig
@@ -342,29 +342,14 @@ pub fn decodeText(arena: Allocator, input: []const u8, label_: ?[]const u8, mime
if (handle == null and mime_type.len > 0) {
if (Mime.parse(mime_type)) |mime| {
- if (mime.is_default_charset == false) {
- handle = encodingHandle(mime.charsetString());
+ if (mime.explicitCharset()) |charset| {
+ handle = encodingHandle(charset);
}
} else |_| {}
}
const encoding = handle orelse encodingHandle("utf-8") orelse return arena.dupe(u8, data);
-
- const max_out = html5ever.encoding_max_utf8_buffer_length(encoding, if (data.len == 0) 4 else data.len);
- if (max_out == 0) {
- return "";
- }
-
- const out = try arena.alloc(u8, max_out);
- const result = html5ever.encoding_decode(
- encoding,
- if (data.len == 0) null else data.ptr,
- data.len,
- out.ptr,
- out.len,
- 1, // is_last
- );
- return out[0..result.bytes_written];
+ return html5ever.decodeToUtf8(arena, encoding, data);
}
pub fn abort(self: *FileReader) !void {