Lightpanda: the headless browser designed for AI and automation
Decode classic scripts with their charset
Adrià Arrufat697331a3b81b
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zigindex a492cc730..f60a10604 100644--- a/src/browser/ScriptManager.zig+++ b/src/browser/ScriptManager.zig@@ -376,6 +376,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e// There was a preloaded script, we borrow it's source and statusconsumed_preload = pre;script.source = pre.source;+ script.charset = pre.charset;script.status = pre.status;script.complete = true;} else {@@ -395,12 +396,13 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_etry frame.headersForRequest(transfer, .{});}- const response = try transfer.submitSync(.{});+ const response = try transfer.submitSync(.{ .copy_headers = true });// Take the body's arena rather than releasing it: `source`// has to outlive this call, up to script.deinit().script.source_arena = response.arena;script.source = .{ .remote = response.body };+ try script.setContentType(HttpClient.findHeader(response.headers, "content-type"));script.status = response.status;script.complete = true;}diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zigindex 61266818e..ea827b448 100644--- a/src/browser/ScriptManagerBase.zig+++ b/src/browser/ScriptManagerBase.zig@@ -23,7 +23,9 @@ const HttpClient = @import("../network/HttpClient.zig");const js = @import("js/js.zig");const Frame = @import("Frame.zig");+const Mime = @import("Mime.zig");const ImportMap = @import("ImportMap.zig");+const h5e = @import("parser/html5ever.zig");const Element = @import("webapi/Element.zig");const WorkerGlobalScope = @import("webapi/WorkerGlobalScope.zig");@@ -602,6 +604,9 @@ pub const Script = struct {// limited .large arena poolsource_arena: ?*lp.Arena = null,+ // The Content-Type charset of a remote script, in `arena`.+ charset: ?[]const u8 = null,+extra: Extra,node: std.DoublyLinkedList.Node,manager: *ScriptManagerBase,@@ -690,6 +695,29 @@ pub const Script = struct {return (self.source_arena orelse self.arena).allocator();}+ pub fn setContentType(self: *Script, content_type: ?[]const u8) !void {+ const mime = Mime.parse(content_type orelse return) catch return;+ if (mime.explicitCharset()) |charset| {+ self.charset = try self.arena.allocator().dupe(u8, charset);+ }+ }++ /// A classic script's text. Remote bytes are decoded with the+ /// Content-Type charset, else the element's charset attribute, else the+ /// document's encoding; a BOM overrides all three.+ /// https://html.spec.whatwg.org/#fetch-a-classic-script+ fn classicText(self: *Script, fe: Extra.FrameExtra) ![]const u8 {+ const bytes = self.source.content();+ if (self.source != .remote) {+ return bytes;+ }+ const encoding = h5e.encodingFor(self.charset) orelse+ h5e.encodingFor(fe.script_element.asElement().getAttributeSafe(comptime .wrap("charset"))) orelse+ h5e.encodingFor(fe.frame.charset) orelse+ h5e.utf8();+ return h5e.asUtf8(self.sourceAllocator(), encoding, bytes);+ }+pub fn startCallback(transfer: *HttpClient.Transfer) !void {log.debug(.http, "script fetch start", .{ .req = transfer });}@@ -747,6 +775,8 @@ pub const Script = struct {self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;}+ try self.setContentType(transfer.contentType());+lp.assert(self.source.remote.capacity == 0, "ScriptManagerBase.Header buffer", .{ .capacity = self.source.remote.capacity });const body_len = transfer.bodyLen();@@ -947,12 +977,14 @@ pub const Script = struct {defer frame._ce_reactions.popAndInvoke(ce_checkpoint, frame);const success = blk: {- const content = self.source.content();switch (fe.kind) {- .javascript => _ = local.eval(content, url) catch break :blk false,+ .javascript => {+ const text = self.classicText(fe) catch break :blk false;+ _ = local.eval(text, url) catch break :blk false;+ },.module => {// We don't care about waiting for the evaluation here.- frame.js.module(false, local, content, url, cacheable) catch break :blk false;+ frame.js.module(false, local, self.source.content(), url, cacheable) catch break :blk false;},.importmap => unreachable, // handled before the try/catch.}diff --git a/src/browser/tests/page/encoding/script_document.html b/src/browser/tests/page/encoding/script_document.htmlnew file mode 100644index 000000000..e5b75b660--- /dev/null+++ b/src/browser/tests/page/encoding/script_document.html@@ -0,0 +1,6 @@+<html>+<head><meta charset="windows-1252"></head>+<body>+<script src="/scripts/latin1/byDocument.js"></script>+</body>+</html>diff --git a/src/browser/tests/page/script_charset.html b/src/browser/tests/page/script_charset.htmlnew file mode 100644index 000000000..b5c4f4624--- /dev/null+++ b/src/browser/tests/page/script_charset.html@@ -0,0 +1,49 @@+<!DOCTYPE html>+<head>+ <meta charset="utf-8">+ <script src="../testing.js"></script>+ <link rel="preload" as="script" href="/scripts/latin1-header/byPreload.js">+</head>+<body>+<script src="/scripts/latin1-header/byHeader.js"></script>+<script src="/scripts/latin1-header/byPreload.js"></script>+<script src="/scripts/bom-over-header.js"></script>+<script src="/scripts/latin1/byAttribute.js" charset="windows-1252"></script>+<script defer src="/scripts/latin1/byAttributeDefer.js" charset="windows-1252"></script>+<script src="/scripts/latin1/byDocumentUtf8.js"></script>+<script type="module" src="/scripts/latin1/byModule.js" charset="windows-1252"></script>++<script id="content_type_charset">+ testing.expectEqual('café', window.byHeader);+ testing.expectEqual('café', window.byPreload);+</script>++<script id="bom_wins">+ testing.expectEqual('café', window.byBom);+</script>++<script id="charset_attribute">+ testing.expectEqual('café', window.byAttribute);+</script>++<script id="document_encoding_utf8">+ testing.expectEqual('caf�', window.byDocumentUtf8);+</script>++<script id="defer_module_and_document_encoding" type="module">+ {+ const state = await testing.async();+ const iframe = document.createElement('iframe');+ iframe.src = 'encoding/script_document.html';+ iframe.onload = () => state.resolve();+ document.body.appendChild(iframe);++ await state.done(() => {+ testing.expectEqual('café', window.byAttributeDefer);+ // Module scripts are always UTF-8, whatever the charset attribute says.+ testing.expectEqual('caf�', window.byModule);+ testing.expectEqual('café', iframe.contentWindow.byDocument);+ });+ }+</script>+</body>diff --git a/src/testing.zig b/src/testing.zigindex d83f9108f..8d25c7cb1 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -1189,6 +1189,27 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {});}+ // Latin-1 scripts for page/script_charset.html. /scripts/latin1/<name>.js+ // and /scripts/latin1-header/<name>.js set window[<name>], the latter+ // declaring its charset.+ if (std.mem.eql(u8, path, "/scripts/bom-over-header.js")) {+ return req.respond("\xEF\xBB\xBFwindow.byBom = 'caf\xC3\xA9';", .{+ .extra_headers = &.{+ .{ .name = "Content-Type", .value = "text/javascript; charset=iso-8859-1" },+ },+ });+ }++ if (std.mem.startsWith(u8, path, "/scripts/latin1/") or std.mem.startsWith(u8, path, "/scripts/latin1-header/")) {+ const body = try std.fmt.allocPrint(req_allocator, "window['{s}'] = 'caf\xE9';", .{std.fs.path.stem(path)});+ const content_type = if (std.mem.startsWith(u8, path, "/scripts/latin1/")) "text/javascript" else "text/javascript; charset=iso-8859-1";+ return req.respond(body, .{+ .extra_headers = &.{+ .{ .name = "Content-Type", .value = content_type },+ },+ });+ }+if (std.mem.eql(u8, path, "/styles/404.css")) {return req.respond("/* unused */", .{.status = .not_found,