Skip to content

Lightpanda: the headless browser designed for AI and automation

Move encoding selection helpers to html5ever.zig

Adrià Arrufat775a8533fa6f
diff --git a/src/browser/css/charset.zig b/src/browser/css/charset.zig
index 58f6a70ba..0bd2d44f8 100644
--- a/src/browser/css/charset.zig
+++ b/src/browser/css/charset.zig
@@ -30,39 +30,14 @@ const Allocator = std.mem.Allocator;
/// `environment_charset` (the document's encoding). Malformed sequences
/// become U+FFFD. Returns `bytes` itself when it is already valid UTF-8.
pub fn decode(allocator: Allocator, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) ![]const u8 {
- const encoding = fallbackEncoding(bytes, protocol_charset, environment_charset);
-
- // A UTF-16 BOM is never valid UTF-8, so only the UTF-8 one can slip
- // through the fast path; the decoder drops it.
- if (std.mem.eql(u8, encoding.name(), "UTF-8") and
- !std.mem.startsWith(u8, bytes, "\xEF\xBB\xBF") and
- std.unicode.utf8ValidateSlice(bytes))
- {
- return bytes;
- }
- return h5e.decodeToUtf8(allocator, encoding.handle.?, bytes);
+ return h5e.asUtf8(allocator, fallbackEncoding(bytes, protocol_charset, environment_charset), bytes);
}
/// https://drafts.csswg.org/css-syntax/#determine-the-fallback-encoding
fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) h5e.EncodingInfo {
- if (protocol_charset) |label| {
- const info = h5e.encoding_for_label(label.ptr, label.len);
- if (info.isValid()) return info;
- }
-
- if (charsetRuleLabel(bytes)) |label| {
- const info = h5e.encoding_for_label(label.ptr, label.len);
- if (info.isValid()) return if (info.isUtf16()) utf8() else info;
- }
-
- const info = h5e.encoding_for_label(environment_charset.ptr, environment_charset.len);
- if (info.isValid()) return info;
-
- return utf8();
-}
-
-fn utf8() h5e.EncodingInfo {
- return h5e.encoding_for_label("utf-8", 5);
+ if (h5e.encodingFor(protocol_charset)) |info| return info;
+ if (h5e.encodingFor(charsetRuleLabel(bytes))) |info| return if (info.isUtf16()) h5e.utf8() else info;
+ return h5e.encodingFor(environment_charset) orelse h5e.utf8();
}
/// The label of a `@charset "<label>";` rule opening the first 1024 bytes,
diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zig
index a03bacdda..456501d9c 100644
--- a/src/browser/parser/html5ever.zig
+++ b/src/browser/parser/html5ever.zig
@@ -248,6 +248,30 @@ pub const EncodingInfo = extern struct {
}
};
+/// The encoding `label` names, if it names one.
+pub fn encodingFor(label: ?[]const u8) ?EncodingInfo {
+ const l = label orelse return null;
+ const info = encoding_for_label(l.ptr, l.len);
+ return if (info.isValid()) info else null;
+}
+
+pub fn utf8() EncodingInfo {
+ return encodingFor("utf-8").?;
+}
+
+/// `decodeToUtf8`, except that `input` is returned as is when it is already
+/// valid UTF-8 without a BOM and `encoding` is UTF-8.
+pub fn asUtf8(allocator: std.mem.Allocator, encoding: EncodingInfo, input: []const u8) ![]const u8 {
+ // A UTF-16 BOM is never valid UTF-8, so only the UTF-8 one needs a check.
+ if (std.mem.eql(u8, encoding.name(), "UTF-8") and
+ !std.mem.startsWith(u8, input, "\xEF\xBB\xBF") and
+ std.unicode.utf8ValidateSlice(input))
+ {
+ return input;
+ }
+ return decodeToUtf8(allocator, encoding.handle.?, input);
+}
+
/// Decodes `input` from the encoding `handle` names into UTF-8. A BOM
/// switches to the encoding it marks and is dropped.
pub fn decodeToUtf8(allocator: std.mem.Allocator, handle: *anyopaque, input: []const u8) ![]const u8 {