Lightpanda: the headless browser designed for AI and automation
Move encoding selection helpers to html5ever.zig
Adrià Arrufat775a8533fa6f
diff --git a/src/browser/css/charset.zig b/src/browser/css/charset.zigindex 58f6a70ba..0bd2d44f8 100644--- a/src/browser/css/charset.zig+++ b/src/browser/css/charset.zig@@ -30,39 +30,14 @@ const Allocator = std.mem.Allocator;/// `environment_charset` (the document's encoding). Malformed sequences/// become U+FFFD. Returns `bytes` itself when it is already valid UTF-8.pub fn decode(allocator: Allocator, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) ![]const u8 {- const encoding = fallbackEncoding(bytes, protocol_charset, environment_charset);-- // A UTF-16 BOM is never valid UTF-8, so only the UTF-8 one can slip- // through the fast path; the decoder drops it.- if (std.mem.eql(u8, encoding.name(), "UTF-8") and- !std.mem.startsWith(u8, bytes, "\xEF\xBB\xBF") and- std.unicode.utf8ValidateSlice(bytes))- {- return bytes;- }- return h5e.decodeToUtf8(allocator, encoding.handle.?, bytes);+ return h5e.asUtf8(allocator, fallbackEncoding(bytes, protocol_charset, environment_charset), bytes);}/// https://drafts.csswg.org/css-syntax/#determine-the-fallback-encodingfn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) h5e.EncodingInfo {- if (protocol_charset) |label| {- const info = h5e.encoding_for_label(label.ptr, label.len);- if (info.isValid()) return info;- }-- if (charsetRuleLabel(bytes)) |label| {- const info = h5e.encoding_for_label(label.ptr, label.len);- if (info.isValid()) return if (info.isUtf16()) utf8() else info;- }-- const info = h5e.encoding_for_label(environment_charset.ptr, environment_charset.len);- if (info.isValid()) return info;-- return utf8();-}--fn utf8() h5e.EncodingInfo {- return h5e.encoding_for_label("utf-8", 5);+ if (h5e.encodingFor(protocol_charset)) |info| return info;+ if (h5e.encodingFor(charsetRuleLabel(bytes))) |info| return if (info.isUtf16()) h5e.utf8() else info;+ return h5e.encodingFor(environment_charset) orelse h5e.utf8();}/// The label of a `@charset "<label>";` rule opening the first 1024 bytes,diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zigindex a03bacdda..456501d9c 100644--- a/src/browser/parser/html5ever.zig+++ b/src/browser/parser/html5ever.zig@@ -248,6 +248,30 @@ pub const EncodingInfo = extern struct {}};+/// The encoding `label` names, if it names one.+pub fn encodingFor(label: ?[]const u8) ?EncodingInfo {+ const l = label orelse return null;+ const info = encoding_for_label(l.ptr, l.len);+ return if (info.isValid()) info else null;+}++pub fn utf8() EncodingInfo {+ return encodingFor("utf-8").?;+}++/// `decodeToUtf8`, except that `input` is returned as is when it is already+/// valid UTF-8 without a BOM and `encoding` is UTF-8.+pub fn asUtf8(allocator: std.mem.Allocator, encoding: EncodingInfo, input: []const u8) ![]const u8 {+ // A UTF-16 BOM is never valid UTF-8, so only the UTF-8 one needs a check.+ if (std.mem.eql(u8, encoding.name(), "UTF-8") and+ !std.mem.startsWith(u8, input, "\xEF\xBB\xBF") and+ std.unicode.utf8ValidateSlice(input))+ {+ return input;+ }+ return decodeToUtf8(allocator, encoding.handle.?, input);+}+/// Decodes `input` from the encoding `handle` names into UTF-8. A BOM/// switches to the encoding it marks and is dropped.pub fn decodeToUtf8(allocator: std.mem.Allocator, handle: *anyopaque, input: []const u8) ![]const u8 {