Skip to content

Lightpanda: the headless browser designed for AI and automation

Decode classic scripts with their charset

Adrià Arrufat697331a3b81b
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig
index a492cc730..f60a10604 100644
--- a/src/browser/ScriptManager.zig
+++ b/src/browser/ScriptManager.zig
@@ -376,6 +376,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
// There was a preloaded script, we borrow it's source and status
consumed_preload = pre;
script.source = pre.source;
+ script.charset = pre.charset;
script.status = pre.status;
script.complete = true;
} else {
@@ -395,12 +396,13 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
try frame.headersForRequest(transfer, .{});
}
- const response = try transfer.submitSync(.{});
+ const response = try transfer.submitSync(.{ .copy_headers = true });
// Take the body's arena rather than releasing it: `source`
// has to outlive this call, up to script.deinit().
script.source_arena = response.arena;
script.source = .{ .remote = response.body };
+ try script.setContentType(HttpClient.findHeader(response.headers, "content-type"));
script.status = response.status;
script.complete = true;
}
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zig
index 61266818e..ea827b448 100644
--- a/src/browser/ScriptManagerBase.zig
+++ b/src/browser/ScriptManagerBase.zig
@@ -23,7 +23,9 @@ const HttpClient = @import("../network/HttpClient.zig");
const js = @import("js/js.zig");
const Frame = @import("Frame.zig");
+const Mime = @import("Mime.zig");
const ImportMap = @import("ImportMap.zig");
+const h5e = @import("parser/html5ever.zig");
const Element = @import("webapi/Element.zig");
const WorkerGlobalScope = @import("webapi/WorkerGlobalScope.zig");
@@ -602,6 +604,9 @@ pub const Script = struct {
// limited .large arena pool
source_arena: ?*lp.Arena = null,
+ // The Content-Type charset of a remote script, in `arena`.
+ charset: ?[]const u8 = null,
+
extra: Extra,
node: std.DoublyLinkedList.Node,
manager: *ScriptManagerBase,
@@ -690,6 +695,29 @@ pub const Script = struct {
return (self.source_arena orelse self.arena).allocator();
}
+ pub fn setContentType(self: *Script, content_type: ?[]const u8) !void {
+ const mime = Mime.parse(content_type orelse return) catch return;
+ if (mime.explicitCharset()) |charset| {
+ self.charset = try self.arena.allocator().dupe(u8, charset);
+ }
+ }
+
+ /// A classic script's text. Remote bytes are decoded with the
+ /// Content-Type charset, else the element's charset attribute, else the
+ /// document's encoding; a BOM overrides all three.
+ /// https://html.spec.whatwg.org/#fetch-a-classic-script
+ fn classicText(self: *Script, fe: Extra.FrameExtra) ![]const u8 {
+ const bytes = self.source.content();
+ if (self.source != .remote) {
+ return bytes;
+ }
+ const encoding = h5e.encodingFor(self.charset) orelse
+ h5e.encodingFor(fe.script_element.asElement().getAttributeSafe(comptime .wrap("charset"))) orelse
+ h5e.encodingFor(fe.frame.charset) orelse
+ h5e.utf8();
+ return h5e.asUtf8(self.sourceAllocator(), encoding, bytes);
+ }
+
pub fn startCallback(transfer: *HttpClient.Transfer) !void {
log.debug(.http, "script fetch start", .{ .req = transfer });
}
@@ -747,6 +775,8 @@ pub const Script = struct {
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
}
+ try self.setContentType(transfer.contentType());
+
lp.assert(self.source.remote.capacity == 0, "ScriptManagerBase.Header buffer", .{ .capacity = self.source.remote.capacity });
const body_len = transfer.bodyLen();
@@ -947,12 +977,14 @@ pub const Script = struct {
defer frame._ce_reactions.popAndInvoke(ce_checkpoint, frame);
const success = blk: {
- const content = self.source.content();
switch (fe.kind) {
- .javascript => _ = local.eval(content, url) catch break :blk false,
+ .javascript => {
+ const text = self.classicText(fe) catch break :blk false;
+ _ = local.eval(text, url) catch break :blk false;
+ },
.module => {
// We don't care about waiting for the evaluation here.
- frame.js.module(false, local, content, url, cacheable) catch break :blk false;
+ frame.js.module(false, local, self.source.content(), url, cacheable) catch break :blk false;
},
.importmap => unreachable, // handled before the try/catch.
}
diff --git a/src/browser/tests/page/encoding/script_document.html b/src/browser/tests/page/encoding/script_document.html
new file mode 100644
index 000000000..e5b75b660
--- /dev/null
+++ b/src/browser/tests/page/encoding/script_document.html
@@ -0,0 +1,6 @@
+<html>
+<head><meta charset="windows-1252"></head>
+<body>
+<script src="/scripts/latin1/byDocument.js"></script>
+</body>
+</html>
diff --git a/src/browser/tests/page/script_charset.html b/src/browser/tests/page/script_charset.html
new file mode 100644
index 000000000..b5c4f4624
--- /dev/null
+++ b/src/browser/tests/page/script_charset.html
@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<head>
+ <meta charset="utf-8">
+ <script src="../testing.js"></script>
+ <link rel="preload" as="script" href="/scripts/latin1-header/byPreload.js">
+</head>
+<body>
+<script src="/scripts/latin1-header/byHeader.js"></script>
+<script src="/scripts/latin1-header/byPreload.js"></script>
+<script src="/scripts/bom-over-header.js"></script>
+<script src="/scripts/latin1/byAttribute.js" charset="windows-1252"></script>
+<script defer src="/scripts/latin1/byAttributeDefer.js" charset="windows-1252"></script>
+<script src="/scripts/latin1/byDocumentUtf8.js"></script>
+<script type="module" src="/scripts/latin1/byModule.js" charset="windows-1252"></script>
+
+<script id="content_type_charset">
+ testing.expectEqual('café', window.byHeader);
+ testing.expectEqual('café', window.byPreload);
+</script>
+
+<script id="bom_wins">
+ testing.expectEqual('café', window.byBom);
+</script>
+
+<script id="charset_attribute">
+ testing.expectEqual('café', window.byAttribute);
+</script>
+
+<script id="document_encoding_utf8">
+ testing.expectEqual('caf�', window.byDocumentUtf8);
+</script>
+
+<script id="defer_module_and_document_encoding" type="module">
+ {
+ const state = await testing.async();
+ const iframe = document.createElement('iframe');
+ iframe.src = 'encoding/script_document.html';
+ iframe.onload = () => state.resolve();
+ document.body.appendChild(iframe);
+
+ await state.done(() => {
+ testing.expectEqual('café', window.byAttributeDefer);
+ // Module scripts are always UTF-8, whatever the charset attribute says.
+ testing.expectEqual('caf�', window.byModule);
+ testing.expectEqual('café', iframe.contentWindow.byDocument);
+ });
+ }
+</script>
+</body>
diff --git a/src/testing.zig b/src/testing.zig
index d83f9108f..8d25c7cb1 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -1189,6 +1189,27 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
+ // Latin-1 scripts for page/script_charset.html. /scripts/latin1/<name>.js
+ // and /scripts/latin1-header/<name>.js set window[<name>], the latter
+ // declaring its charset.
+ if (std.mem.eql(u8, path, "/scripts/bom-over-header.js")) {
+ return req.respond("\xEF\xBB\xBFwindow.byBom = 'caf\xC3\xA9';", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/javascript; charset=iso-8859-1" },
+ },
+ });
+ }
+
+ if (std.mem.startsWith(u8, path, "/scripts/latin1/") or std.mem.startsWith(u8, path, "/scripts/latin1-header/")) {
+ const body = try std.fmt.allocPrint(req_allocator, "window['{s}'] = 'caf\xE9';", .{std.fs.path.stem(path)});
+ const content_type = if (std.mem.startsWith(u8, path, "/scripts/latin1/")) "text/javascript" else "text/javascript; charset=iso-8859-1";
+ return req.respond(body, .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = content_type },
+ },
+ });
+ }
+
if (std.mem.eql(u8, path, "/styles/404.css")) {
return req.respond("/* unused */", .{
.status = .not_found,