Lightpanda: the headless browser designed for AI and automation
css: decode fetched stylesheets before tokenizing
Adrià Arrufatde6917d08e8d
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex bffd3f428..08bdd2848 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -35,6 +35,7 @@ const h5e = @import("parser/html5ever.zig");const CustomElementReactions = @import("CustomElementReactions.zig");const URL = @import("URL.zig");+const css_charset = @import("css/charset.zig");const referrer = @import("referrer.zig");const Blob = @import("webapi/Blob.zig");const FileList = @import("webapi/FileList.zig");@@ -2660,7 +2661,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []cosm.is_evaluating = true;defer sm.endEvaluationWindow(was_evaluating);- var response = transfer.submitSync(.{}) catch |err| {+ var response = transfer.submitSync(.{ .copy_headers = true }) catch |err| {log.debug(.http, "external stylesheet fetch", .{ .err = err, .url = resolved });return self.fireElementEvent(element, comptime .wrap("error"));};@@ -2704,7 +2705,14 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co// scratch-list pattern in `CSSStyleSheet.replaceSync`. Keeping// `_href` consistent with what the sheet actually contains is the// minimum.- sheet.replaceSync(response.body.items, self) catch |err| {+ const content_type: ?Mime = for (response.headers) |header| {+ if (std.ascii.eqlIgnoreCase(header.name, "content-type")) {+ break Mime.parse(header.value) catch null;+ }+ } else null;+ const protocol_charset = if (content_type) |*mime| (if (mime.is_default_charset) null else mime.charsetString()) else null;+ const text = try css_charset.decode(arena.allocator(), response.body.items, protocol_charset, self.charset);+ sheet.replaceSync(text, self) catch |err| {log.debug(.http, "external stylesheet parse", .{ .err = err, .url = resolved });return self.fireElementEvent(element, comptime .wrap("error"));};diff --git a/src/browser/css/charset.zig b/src/browser/css/charset.zignew file mode 100644index 000000000..5e1108082--- /dev/null+++ b/src/browser/css/charset.zig@@ -0,0 +1,132 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! Decoding of fetched stylesheet bytes into the UTF-8 the tokenizer requires.+//!+//! See https://drafts.csswg.org/css-syntax/#input-byte-stream++const std = @import("std");+const h5e = @import("../parser/html5ever.zig");++const Allocator = std.mem.Allocator;++/// Decodes `bytes` to UTF-8. A BOM wins, then `protocol_charset` (the+/// Content-Type charset), then a leading `@charset` rule, then+/// `environment_charset` (the document's encoding). Malformed sequences+/// become U+FFFD. Returns `bytes` itself when it is already valid UTF-8.+pub fn decode(allocator: Allocator, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) ![]const u8 {+ const encoding = fallbackEncoding(bytes, protocol_charset, environment_charset);++ // The decoder sniffs the BOM itself, but the fast path must not keep one.+ const has_bom = std.mem.startsWith(u8, bytes, "\xEF\xBB\xBF") or+ std.mem.startsWith(u8, bytes, "\xFE\xFF") or+ std.mem.startsWith(u8, bytes, "\xFF\xFE");+ if (!has_bom and std.mem.eql(u8, encoding.name(), "UTF-8") and std.unicode.utf8ValidateSlice(bytes)) {+ return bytes;+ }++ const handle = encoding.handle.?;+ const out = try allocator.alloc(u8, h5e.encoding_max_utf8_buffer_length(handle, bytes.len));+ const result = h5e.encoding_decode(handle, bytes.ptr, bytes.len, out.ptr, out.len, 1);+ return out[0..result.bytes_written];+}++/// https://drafts.csswg.org/css-syntax/#determine-the-fallback-encoding+fn fallbackEncoding(bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) h5e.EncodingInfo {+ if (protocol_charset) |label| {+ const info = h5e.encoding_for_label(label.ptr, label.len);+ if (info.isValid()) return info;+ }++ if (charsetRuleLabel(bytes)) |label| {+ const info = h5e.encoding_for_label(label.ptr, label.len);+ if (info.isValid()) {+ const name = info.name();+ if (!std.mem.eql(u8, name, "UTF-16BE") and !std.mem.eql(u8, name, "UTF-16LE")) return info;+ return utf8();+ }+ }++ const info = h5e.encoding_for_label(environment_charset.ptr, environment_charset.len);+ if (info.isValid()) return info;++ return utf8();+}++fn utf8() h5e.EncodingInfo {+ return h5e.encoding_for_label("utf-8", 5);+}++/// The label of a `@charset "<label>";` rule opening the first 1024 bytes,+/// matched byte for byte as the spec requires.+fn charsetRuleLabel(bytes: []const u8) ?[]const u8 {+ const prefix = "@charset \"";+ const head = bytes[0..@min(bytes.len, 1024)];+ if (!std.mem.startsWith(u8, head, prefix)) return null;++ const rest = head[prefix.len..];+ const end = std.mem.findAny(u8, rest, "\";") orelse return null;+ if (!std.mem.startsWith(u8, rest[end..], "\";")) return null;+ return rest[0..end];+}++const testing = std.testing;++fn expectDecoded(expected: []const u8, bytes: []const u8, protocol_charset: ?[]const u8, environment_charset: []const u8) !void {+ var arena: std.heap.ArenaAllocator = .init(testing.allocator);+ defer arena.deinit();+ try testing.expectEqualStrings(expected, try decode(arena.allocator(), bytes, protocol_charset, environment_charset));+}++test "css charset: valid UTF-8 is returned as is" {+ const bytes = "a{content:\"é\"}";+ try testing.expectEqual(bytes.ptr, (try decode(testing.allocator, bytes, null, "UTF-8")).ptr);+}++test "css charset: invalid UTF-8 is replaced" {+ try expectDecoded("a{content:\"\u{FFFD}\";color:red}", "a{content:\"\xE9\";color:red}", null, "UTF-8");+ try expectDecoded("a{}\u{FFFD}", "a{}\xE2", null, "UTF-8");+}++test "css charset: protocol charset" {+ try expectDecoded("/* café */", "/* caf\xE9 */", "iso-8859-1", "UTF-8");+}++test "css charset: protocol charset wins over @charset" {+ try expectDecoded("@charset \"utf-8\";é", "@charset \"utf-8\";\xE9", "windows-1252", "UTF-8");+}++test "css charset: @charset" {+ try expectDecoded("@charset \"windows-1252\";é", "@charset \"windows-1252\";\xE9", null, "UTF-8");+ try expectDecoded("@charset \"utf-16le\";\u{FFFD}", "@charset \"utf-16le\";\xE9", null, "windows-1252");+ // Not byte-exact: single quotes, or a missing space.+ try expectDecoded("@charset 'windows-1252';\u{FFFD}", "@charset 'windows-1252';\xE9", null, "UTF-8");+ try expectDecoded("@charset\"windows-1252\";\u{FFFD}", "@charset\"windows-1252\";\xE9", null, "UTF-8");+ // Unknown label falls through to the environment.+ try expectDecoded("@charset \"bogus\";é", "@charset \"bogus\";\xE9", null, "windows-1252");+}++test "css charset: environment charset" {+ try expectDecoded("é", "\xE9", null, "windows-1252");+}++test "css charset: BOM wins and is stripped" {+ try expectDecoded("é", "\xEF\xBB\xBF\xC3\xA9", "windows-1252", "windows-1252");+ try expectDecoded("a", "\xFF\xFEa\x00", null, "UTF-8");+ try expectDecoded("a", "\xEF\xBB\xBFa", null, "UTF-8");+}diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zigindex 797faf0e1..f4773a933 100644--- a/src/browser/parser/html5ever.zig+++ b/src/browser/parser/html5ever.zig@@ -224,7 +224,7 @@ pub extern "c" fn xml5ever_parse_document() void;// General encoding api-const EncodingInfo = extern struct {+pub const EncodingInfo = extern struct {found: u8,handle: ?*anyopaque,name_len: usize,diff --git a/src/browser/tests/css/stylesheet_charset.html b/src/browser/tests/css/stylesheet_charset.htmlnew file mode 100644index 000000000..45b7bd801--- /dev/null+++ b/src/browser/tests/css/stylesheet_charset.html@@ -0,0 +1,29 @@+<!DOCTYPE html>+<head>+ <meta charset="utf-8">+ <script src="../testing.js"></script>+ <link rel="stylesheet" href="/styles/latin1-header.css">+ <link rel="stylesheet" href="/styles/latin1-at-charset.css">+ <link rel="stylesheet" href="/styles/truncated-utf8.css">+</head>+<body>+ <div id="by-header" class="by-header">hidden</div>+ <div id="by-at-charset" class="by-at-charset">hidden</div>+ <div id="before-truncated" class="before-truncated">hidden</div>++ <script id="content_type_charset">+ {+ const el = document.getElementById('by-header');+ testing.expectEqual('none', getComputedStyle(el).display);+ testing.expectEqual('"café"', document.styleSheets[0].cssRules[0].style.getPropertyValue('--name'));+ }+ </script>++ <script id="at_charset">+ testing.expectEqual('none', getComputedStyle(document.getElementById('by-at-charset')).display);+ </script>++ <script id="truncated_utf8">+ testing.expectEqual('none', getComputedStyle(document.getElementById('before-truncated')).display);+ </script>+</body>diff --git a/src/browser/webapi/element/html/Link.zig b/src/browser/webapi/element/html/Link.zigindex 684c03c33..1af7bb563 100644--- a/src/browser/webapi/element/html/Link.zig+++ b/src/browser/webapi/element/html/Link.zig@@ -212,6 +212,10 @@ test "WebApi: HTML.Link external stylesheet" {try testing.htmlRunner("css/external_stylesheet.html", .{ .load_resources = .{ .stylesheet = true } });}+test "WebApi: HTML.Link external stylesheet charset" {+ try testing.htmlRunner("css/stylesheet_charset.html", .{ .load_resources = .{ .stylesheet = true } });+}+// Regression: a synchronous external-stylesheet fetch must not strand the// completion of an in-flight <script defer> (held back by the blocking-// request gate during the sync window). Otherwise the deferred-script queuediff --git a/src/testing.zig b/src/testing.zigindex b48c4e68e..d83f9108f 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -1163,6 +1163,32 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {});}+ // Latin-1 sheets for css/stylesheet_charset.html: undecoded, the 0xE9+ // swallows the bytes after it and the rule that follows is lost.+ if (std.mem.eql(u8, path, "/styles/latin1-header.css")) {+ return req.respond("/* caf\xE9 */ .by-header { display: none; --name: \"caf\xE9\"; }", .{+ .extra_headers = &.{+ .{ .name = "Content-Type", .value = "text/css; charset=iso-8859-1" },+ },+ });+ }++ if (std.mem.eql(u8, path, "/styles/latin1-at-charset.css")) {+ return req.respond("@charset \"windows-1252\"; /* caf\xE9 */ .by-at-charset { display: none; }", .{+ .extra_headers = &.{+ .{ .name = "Content-Type", .value = "text/css" },+ },+ });+ }++ if (std.mem.eql(u8, path, "/styles/truncated-utf8.css")) {+ return req.respond(".before-truncated { display: none; } \xE2", .{+ .extra_headers = &.{+ .{ .name = "Content-Type", .value = "text/css" },+ },+ });+ }+if (std.mem.eql(u8, path, "/styles/404.css")) {return req.respond("/* unused */", .{.status = .not_found,