Skip to content

Lightpanda: the headless browser designed for AI and automation

Fix charset unicode HTML prescan fallback

Dustin Persekf8934f575b8e
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 7cde1c6c8..51f6f869c 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -1340,6 +1340,10 @@ fn urlBasename(arena: Allocator, url: []const u8) !?[]const u8 {
return try arena.dupe(u8, name);
}
+fn isUtf16Encoding(charset: []const u8) bool {
+ return std.mem.eql(u8, charset, "UTF-16LE") or std.mem.eql(u8, charset, "UTF-16BE");
+}
+
fn frameDataCallback(response: HttpClient.Response, data: []const u8) !void {
var self: *Frame = @ptrCast(@alignCast(response.ctx));
@@ -1355,8 +1359,10 @@ fn frameDataCallback(response: HttpClient.Response, data: []const u8) !void {
// If the HTTP Content-Type header didn't specify a charset and this is HTML,
// prescan the first 1024 bytes for a <meta charset> declaration.
+ var html_prescan_found_charset = false;
if (mime.content_type == .text_html and mime.is_default_charset) {
if (Mime.prescanCharset(data)) |charset| {
+ html_prescan_found_charset = true;
if (charset.len <= 40) {
@memcpy(mime.charset[0..charset.len], charset);
mime.charset[charset.len] = 0;
@@ -1380,7 +1386,8 @@ fn frameDataCallback(response: HttpClient.Response, data: []const u8) !void {
const charset_str = mime.charsetString();
const info = h5e.encoding_for_label(charset_str.ptr, charset_str.len);
if (info.isValid()) {
- self.charset = info.name();
+ const name = info.name();
+ self.charset = if (html_prescan_found_charset and isUtf16Encoding(name)) "UTF-8" else name;
}
self._parse_state = .{ .html = .{
.buffer = .empty,
diff --git a/src/browser/tests/page/encoding.html b/src/browser/tests/page/encoding.html
index a07869329..1d86abad7 100644
--- a/src/browser/tests/page/encoding.html
+++ b/src/browser/tests/page/encoding.html
@@ -69,6 +69,30 @@
}
</script>
+<script id="unicode_label_falls_back_to_utf8" type=module>
+ {
+ const state = await testing.async();
+
+ // The legacy "unicode" label maps to UTF-16LE, but HTML prescan maps
+ // UTF-16 labels back to UTF-8 so ASCII pages stay readable.
+ const iframe = document.createElement('iframe');
+ document.body.appendChild(iframe);
+ iframe.src = 'encoding/unicode_label.html';
+ iframe.onload = () => { state.resolve(); }
+
+ await state.done(() => {
+ testing.expectEqual('Hello world', iframe.contentDocument.getElementById('title').textContent);
+ testing.expectEqual(
+ 'This page is plain ASCII but declares charset=unicode.',
+ iframe.contentDocument.getElementById('body').textContent,
+ );
+ testing.expectEqual('UTF-8', iframe.contentDocument.characterSet);
+ testing.expectEqual('UTF-8', iframe.contentDocument.charset);
+ testing.expectEqual('UTF-8', iframe.contentDocument.inputEncoding);
+ });
+ }
+</script>
+
<script id="no_charset_fallback" type=module>
{
const state = await testing.async();
diff --git a/src/browser/tests/page/encoding/unicode_label.html b/src/browser/tests/page/encoding/unicode_label.html
new file mode 100644
index 000000000..7e93a9358
--- /dev/null
+++ b/src/browser/tests/page/encoding/unicode_label.html
@@ -0,0 +1,5 @@
+<!DOCTYPE html>
+<META http-equiv=Content-Type content="text/html; charset=unicode">
+<title>Hello</title>
+<h1 id="title">Hello world</h1>
+<p id="body">This page is plain ASCII but declares charset=unicode.</p>