Lightpanda: the headless browser designed for AI and automation
Fix charset unicode HTML prescan fallback
Dustin Persekf8934f575b8e
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 7cde1c6c8..51f6f869c 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1340,6 +1340,10 @@ fn urlBasename(arena: Allocator, url: []const u8) !?[]const u8 {return try arena.dupe(u8, name);}+fn isUtf16Encoding(charset: []const u8) bool {+ return std.mem.eql(u8, charset, "UTF-16LE") or std.mem.eql(u8, charset, "UTF-16BE");+}+fn frameDataCallback(response: HttpClient.Response, data: []const u8) !void {var self: *Frame = @ptrCast(@alignCast(response.ctx));@@ -1355,8 +1359,10 @@ fn frameDataCallback(response: HttpClient.Response, data: []const u8) !void {// If the HTTP Content-Type header didn't specify a charset and this is HTML,// prescan the first 1024 bytes for a <meta charset> declaration.+ var html_prescan_found_charset = false;if (mime.content_type == .text_html and mime.is_default_charset) {if (Mime.prescanCharset(data)) |charset| {+ html_prescan_found_charset = true;if (charset.len <= 40) {@memcpy(mime.charset[0..charset.len], charset);mime.charset[charset.len] = 0;@@ -1380,7 +1386,8 @@ fn frameDataCallback(response: HttpClient.Response, data: []const u8) !void {const charset_str = mime.charsetString();const info = h5e.encoding_for_label(charset_str.ptr, charset_str.len);if (info.isValid()) {- self.charset = info.name();+ const name = info.name();+ self.charset = if (html_prescan_found_charset and isUtf16Encoding(name)) "UTF-8" else name;}self._parse_state = .{ .html = .{.buffer = .empty,diff --git a/src/browser/tests/page/encoding.html b/src/browser/tests/page/encoding.htmlindex a07869329..1d86abad7 100644--- a/src/browser/tests/page/encoding.html+++ b/src/browser/tests/page/encoding.html@@ -69,6 +69,30 @@}</script>+<script id="unicode_label_falls_back_to_utf8" type=module>+ {+ const state = await testing.async();++ // The legacy "unicode" label maps to UTF-16LE, but HTML prescan maps+ // UTF-16 labels back to UTF-8 so ASCII pages stay readable.+ const iframe = document.createElement('iframe');+ document.body.appendChild(iframe);+ iframe.src = 'encoding/unicode_label.html';+ iframe.onload = () => { state.resolve(); }++ await state.done(() => {+ testing.expectEqual('Hello world', iframe.contentDocument.getElementById('title').textContent);+ testing.expectEqual(+ 'This page is plain ASCII but declares charset=unicode.',+ iframe.contentDocument.getElementById('body').textContent,+ );+ testing.expectEqual('UTF-8', iframe.contentDocument.characterSet);+ testing.expectEqual('UTF-8', iframe.contentDocument.charset);+ testing.expectEqual('UTF-8', iframe.contentDocument.inputEncoding);+ });+ }+</script>+<script id="no_charset_fallback" type=module>{const state = await testing.async();diff --git a/src/browser/tests/page/encoding/unicode_label.html b/src/browser/tests/page/encoding/unicode_label.htmlnew file mode 100644index 000000000..7e93a9358--- /dev/null+++ b/src/browser/tests/page/encoding/unicode_label.html@@ -0,0 +1,5 @@+<!DOCTYPE html>+<META http-equiv=Content-Type content="text/html; charset=unicode">+<title>Hello</title>+<h1 id="title">Hello world</h1>+<p id="body">This page is plain ASCII but declares charset=unicode.</p>