Skip to content

Lightpanda: the headless browser designed for AI and automation

css: decode fetched stylesheets before tokenizing

Adrià Arrufatde6917d08e8d
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index bffd3f428..08bdd2848 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -35,6 +35,7 @@ const h5e = @import("parser/html5ever.zig");
const CustomElementReactions = @import("CustomElementReactions.zig");
const URL = @import("URL.zig");
+const css_charset = @import("css/charset.zig");
const referrer = @import("referrer.zig");
const Blob = @import("webapi/Blob.zig");
const FileList = @import("webapi/FileList.zig");
@@ -2660,7 +2661,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
sm.is_evaluating = true;
defer sm.endEvaluationWindow(was_evaluating);
- var response = transfer.submitSync(.{}) catch |err| {
+ var response = transfer.submitSync(.{ .copy_headers = true }) catch |err| {
log.debug(.http, "external stylesheet fetch", .{ .err = err, .url = resolved });
return self.fireElementEvent(element, comptime .wrap("error"));
};
@@ -2704,7 +2705,14 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
// scratch-list pattern in `CSSStyleSheet.replaceSync`. Keeping
// `_href` consistent with what the sheet actually contains is the
// minimum.
- sheet.replaceSync(response.body.items, self) catch |err| {
+ const content_type: ?Mime = for (response.headers) |header| {
+ if (std.ascii.eqlIgnoreCase(header.name, "content-type")) {
+ break Mime.parse(header.value) catch null;
+ }
+ } else null;
+ const protocol_charset = if (content_type) |*mime| (if (mime.is_default_charset) null else mime.charsetString()) else null;
+ const text = try css_charset.decode(arena.allocator(), response.body.items, protocol_charset, self.charset);
+ sheet.replaceSync(text, self) catch |err| {
log.debug(.http, "external stylesheet parse", .{ .err = err, .url = resolved });
return self.fireElementEvent(element, comptime .wrap("error"));
};
diff --git a/src/browser/css/charset.zig b/src/browser/css/charset.zig
new file mode 100644
index 000000000..5e1108082
--- /dev/null
+++ b/src/browser/css/charset.zig
@@ -0,0 +1,132 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Decoding of fetched stylesheet bytes into the UTF-8 the tokenizer requires.
+//!
+//! See https://drafts.csswg.org/css-syntax/#input-byte-stream
+
+const std = @import("std");
+const h5e = @import("../parser/html5ever.zig");
+
+const Allocator = std.mem.Allocator;
+
+/// Decodes `bytes` to UTF-8. A BOM wins, then `protocol_charset` (the
+/// Content-Type charset), then a leading `@charset` rule, then
+/// `environment_charset` (the document's encoding). Malformed sequences
+/// become U+FFFD. Returns `bytes` itself when it is already valid UTF-8.
+pub fn decode(allocator: Allocator, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) ![]const u8 {
+ const encoding = fallbackEncoding(bytes, protocol_charset, environment_charset);
+
+ // The decoder sniffs the BOM itself, but the fast path must not keep one.
+ const has_bom = std.mem.startsWith(u8, bytes, "\xEF\xBB\xBF") or
+ std.mem.startsWith(u8, bytes, "\xFE\xFF") or
+ std.mem.startsWith(u8, bytes, "\xFF\xFE");
+ if (!has_bom and std.mem.eql(u8, encoding.name(), "UTF-8") and std.unicode.utf8ValidateSlice(bytes)) {
+ return bytes;
+ }
+
+ const handle = encoding.handle.?;
+ const out = try allocator.alloc(u8, h5e.encoding_max_utf8_buffer_length(handle, bytes.len));
+ const result = h5e.encoding_decode(handle, bytes.ptr, bytes.len, out.ptr, out.len, 1);
+ return out[0..result.bytes_written];
+}
+
+/// https://drafts.csswg.org/css-syntax/#determine-the-fallback-encoding
+fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) h5e.EncodingInfo {
+ if (protocol_charset) |label| {
+ const info = h5e.encoding_for_label(label.ptr, label.len);
+ if (info.isValid()) return info;
+ }
+
+ if (charsetRuleLabel(bytes)) |label| {
+ const info = h5e.encoding_for_label(label.ptr, label.len);
+ if (info.isValid()) {
+ const name = info.name();
+ if (!std.mem.eql(u8, name, "UTF-16BE") and !std.mem.eql(u8, name, "UTF-16LE")) return info;
+ return utf8();
+ }
+ }
+
+ const info = h5e.encoding_for_label(environment_charset.ptr, environment_charset.len);
+ if (info.isValid()) return info;
+
+ return utf8();
+}
+
+fn utf8() h5e.EncodingInfo {
+ return h5e.encoding_for_label("utf-8", 5);
+}
+
+/// The label of a `@charset "<label>";` rule opening the first 1024 bytes,
+/// matched byte for byte as the spec requires.
+fn charsetRuleLabel(bytes: []const u8) ?[]const u8 {
+ const prefix = "@charset \"";
+ const head = bytes[0..@min(bytes.len, 1024)];
+ if (!std.mem.startsWith(u8, head, prefix)) return null;
+
+ const rest = head[prefix.len..];
+ const end = std.mem.findAny(u8, rest, "\";") orelse return null;
+ if (!std.mem.startsWith(u8, rest[end..], "\";")) return null;
+ return rest[0..end];
+}
+
+const testing = std.testing;
+
+fn expectDecoded(expected: []const u8, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) !void {
+ var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+ defer arena.deinit();
+ try testing.expectEqualStrings(expected, try decode(arena.allocator(), bytes, protocol_charset, environment_charset));
+}
+
+test "css charset: valid UTF-8 is returned as is" {
+ const bytes = "a{content:\"é\"}";
+ try testing.expectEqual(bytes.ptr, (try decode(testing.allocator, bytes, null, "UTF-8")).ptr);
+}
+
+test "css charset: invalid UTF-8 is replaced" {
+ try expectDecoded("a{content:\"\u{FFFD}\";color:red}", "a{content:\"\xE9\";color:red}", null, "UTF-8");
+ try expectDecoded("a{}\u{FFFD}", "a{}\xE2", null, "UTF-8");
+}
+
+test "css charset: protocol charset" {
+ try expectDecoded("/* café */", "/* caf\xE9 */", "iso-8859-1", "UTF-8");
+}
+
+test "css charset: protocol charset wins over @charset" {
+ try expectDecoded("@charset \"utf-8\";é", "@charset \"utf-8\";\xE9", "windows-1252", "UTF-8");
+}
+
+test "css charset: @charset" {
+ try expectDecoded("@charset \"windows-1252\";é", "@charset \"windows-1252\";\xE9", null, "UTF-8");
+ try expectDecoded("@charset \"utf-16le\";\u{FFFD}", "@charset \"utf-16le\";\xE9", null, "windows-1252");
+ // Not byte-exact: single quotes, or a missing space.
+ try expectDecoded("@charset 'windows-1252';\u{FFFD}", "@charset 'windows-1252';\xE9", null, "UTF-8");
+ try expectDecoded("@charset\"windows-1252\";\u{FFFD}", "@charset\"windows-1252\";\xE9", null, "UTF-8");
+ // Unknown label falls through to the environment.
+ try expectDecoded("@charset \"bogus\";é", "@charset \"bogus\";\xE9", null, "windows-1252");
+}
+
+test "css charset: environment charset" {
+ try expectDecoded("é", "\xE9", null, "windows-1252");
+}
+
+test "css charset: BOM wins and is stripped" {
+ try expectDecoded("é", "\xEF\xBB\xBF\xC3\xA9", "windows-1252", "windows-1252");
+ try expectDecoded("a", "\xFF\xFEa\x00", null, "UTF-8");
+ try expectDecoded("a", "\xEF\xBB\xBFa", null, "UTF-8");
+}
diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zig
index 797faf0e1..f4773a933 100644
--- a/src/browser/parser/html5ever.zig
+++ b/src/browser/parser/html5ever.zig
@@ -224,7 +224,7 @@ pub extern "c" fn xml5ever_parse_document(
) void;
// General encoding api
-const EncodingInfo = extern struct {
+pub const EncodingInfo = extern struct {
found: u8,
handle: ?*anyopaque,
name_len: usize,
diff --git a/src/browser/tests/css/stylesheet_charset.html b/src/browser/tests/css/stylesheet_charset.html
new file mode 100644
index 000000000..45b7bd801
--- /dev/null
+++ b/src/browser/tests/css/stylesheet_charset.html
@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<head>
+ <meta charset="utf-8">
+ <script src="../testing.js"></script>
+ <link rel="stylesheet" href="/styles/latin1-header.css">
+ <link rel="stylesheet" href="/styles/latin1-at-charset.css">
+ <link rel="stylesheet" href="/styles/truncated-utf8.css">
+</head>
+<body>
+ <div id="by-header" class="by-header">hidden</div>
+ <div id="by-at-charset" class="by-at-charset">hidden</div>
+ <div id="before-truncated" class="before-truncated">hidden</div>
+
+ <script id="content_type_charset">
+ {
+ const el = document.getElementById('by-header');
+ testing.expectEqual('none', getComputedStyle(el).display);
+ testing.expectEqual('"café"', document.styleSheets[0].cssRules[0].style.getPropertyValue('--name'));
+ }
+ </script>
+
+ <script id="at_charset">
+ testing.expectEqual('none', getComputedStyle(document.getElementById('by-at-charset')).display);
+ </script>
+
+ <script id="truncated_utf8">
+ testing.expectEqual('none', getComputedStyle(document.getElementById('before-truncated')).display);
+ </script>
+</body>
diff --git a/src/browser/webapi/element/html/Link.zig b/src/browser/webapi/element/html/Link.zig
index 684c03c33..1af7bb563 100644
--- a/src/browser/webapi/element/html/Link.zig
+++ b/src/browser/webapi/element/html/Link.zig
@@ -212,6 +212,10 @@ test "WebApi: HTML.Link external stylesheet" {
try testing.htmlRunner("css/external_stylesheet.html", .{ .load_resources = .{ .stylesheet = true } });
}
+test "WebApi: HTML.Link external stylesheet charset" {
+ try testing.htmlRunner("css/stylesheet_charset.html", .{ .load_resources = .{ .stylesheet = true } });
+}
+
// Regression: a synchronous external-stylesheet fetch must not strand the
// completion of an in-flight <script defer> (held back by the blocking-
// request gate during the sync window). Otherwise the deferred-script queue
diff --git a/src/testing.zig b/src/testing.zig
index b48c4e68e..d83f9108f 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -1163,6 +1163,32 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
+ // Latin-1 sheets for css/stylesheet_charset.html: undecoded, the 0xE9
+ // swallows the bytes after it and the rule that follows is lost.
+ if (std.mem.eql(u8, path, "/styles/latin1-header.css")) {
+ return req.respond("/* caf\xE9 */ .by-header { display: none; --name: \"caf\xE9\"; }", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/css; charset=iso-8859-1" },
+ },
+ });
+ }
+
+ if (std.mem.eql(u8, path, "/styles/latin1-at-charset.css")) {
+ return req.respond("@charset \"windows-1252\"; /* caf\xE9 */ .by-at-charset { display: none; }", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/css" },
+ },
+ });
+ }
+
+ if (std.mem.eql(u8, path, "/styles/truncated-utf8.css")) {
+ return req.respond(".before-truncated { display: none; } \xE2", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/css" },
+ },
+ });
+ }
+
if (std.mem.eql(u8, path, "/styles/404.css")) {
return req.respond("/* unused */", .{
.status = .not_found,