Lightpanda: the headless browser designed for AI and automation
cleanup: Enhance preload scripts
Karl Seguina18886b1a93f
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zigindex 0d3d2de84..95ec7334c 100644--- a/src/browser/ScriptManager.zig+++ b/src/browser/ScriptManager.zig@@ -46,7 +46,8 @@ frame: *Frame,// "load" event).frame_notified_of_completion: bool,-// scripts loaded based on a <link rel=preload as=script href=...> found during parsing+// Scripts loaded based on a <link rel=preload as=script href=...> found during+// parsing, keyed by resolved URL.preloaded_scripts: std.StringHashMapUnmanaged(PreloadedScript),pub fn init(allocator: Allocator, http_client: *HttpClient, frame: *Frame) ScriptManager {@@ -225,54 +226,28 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_ereturn;};- var handover = false;const frame = self.frame;+ const base_url = frame.base();- // A consumed preload (waitForPreload below) is owned by us: its buffer is- // borrowed by `script`, so it must outlive eval.- var consumed_preload: ?*Script = null;- defer if (consumed_preload) |p| {- p.deinit();+ const src = element.getAttributeSafe(comptime .wrap("src")) orelse {+ return self.addInlineScript(script_element, kind);};+ // The script is remote (even data: and blob: are synthesized via HttpClient)++ // Set once arena ownership is resolved — transferred to `script`, or+ // released early on the adoption path — so the errdefer can't double-free.+ var handover = false;+const arena = try frame.getArena(.large, "SM.addFromElement");- errdefer if (!handover) {+ errdefer if (handover == false) {frame.releaseArena(arena);};- var source: Script.Source = undefined;- var remote_url: ?[:0]const u8 = null;- const base_url = frame.base();- if (element.getAttributeSafe(comptime .wrap("src"))) |src| {- // data: and blob: srcs flow through the normal request path; HttpClient- // synthesizes the response. Execution mode (blocking vs async/defer) is- // attribute-driven, the same as any other src.- remote_url = try URL.resolve(arena, base_url, src, .{ .encoding = frame.charset });- source = .{ .remote = .{} };- } else {- var buf = std.Io.Writer.Allocating.init(arena);- try element.asNode().getChildTextContent(&buf.writer);- try buf.writer.writeByte(0);- const data = buf.written();- const inline_source: [:0]const u8 = data[0 .. data.len - 1 :0];- if (inline_source.len == 0) {- // we haven't set script_element._executed = true yet, which is good.- // If content is appended to the script, we will execute it then.- frame.releaseArena(arena);- return;- }- source = .{ .@"inline" = inline_source };- }-- // Only set _executed (already-started) when we actually have content to execute+ const remote_url = try URL.resolve(arena, base_url, src, .{ .encoding = frame.charset });script_element._executed = true;- const is_inline = source == .@"inline";const mode: Script.Extra.FrameExtra.Mode = blk: {- if (source == .@"inline") {- break :blk if (kind == .module) .@"defer" else .normal;- }-if (element.getAttributeSafe(comptime .wrap("async")) != null) {break :blk .async;}@@ -294,54 +269,86 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_ebreak :blk .normal;};+ if (comptime IS_DEBUG) {+ var ls: js.Local.Scope = undefined;+ frame.js.localScope(&ls);+ defer ls.deinit();++ log.debug(.http, "script queue", .{+ .ctx = ctx,+ .url = remote_url,+ .element = element,+ .stack = ls.local.stackTrace() catch "???",+ });+ }++ const frame_extra: Script.Extra = .{ .frame = .{+ .kind = kind,+ .mode = mode,+ .script_element = script_element,+ .frame = frame,+ } };++ if (mode != .normal) {+ if (self.takePreload(remote_url)) |pre| {+ // A <link rel=preload as=script> already fetched (or is still+ // fetching) this URL. Adopt the preloaded Script, zero-copy — the+ // classic-script mirror of getAsyncImport's module-hint adoption.+ // The transfer's still-registered PreloadedScript callbacks see+ // the rewritten extra and route to the normal Script callbacks.+ // (Blocking scripts consume preloads via waitForPreload below.)+ if (comptime IS_DEBUG) {+ log.debug(.http, "script adopt", .{ .url = remote_url, .ctx = ctx, .state = if (pre.complete) "done" else "loading" });+ }+ pre.extra = frame_extra;++ // The adopted Script has its own arena; ours held only the URL+ // resolution, which nothing below needs.+ handover = true;+ frame.releaseArena(arena);++ if (pre.complete and mode == .async) {+ // The fetch already finished, so no doneCallback will move it+ // to ready_scripts; queue it there directly.+ self.base.ready_scripts.append(&pre.node);+ } else {+ self.base.scriptList(pre).append(&pre.node);+ }+ if (pre.complete) {+ // ...and no doneCallback will trigger evaluation.+ self.base.evaluate();+ }+ return;+ }+ }+const script = try arena.create(Script);script.* = .{.node = .{},.arena = arena,.manager = &self.base,- .source = source,- .complete = is_inline,- .status = if (is_inline) 200 else 0,- .url = remote_url orelse base_url,- .extra = .{ .frame = .{- .kind = kind,- .mode = mode,- .script_element = script_element,- .frame = frame,- } },+ .source = .{ .remote = .{} },+ .complete = false,+ .url = remote_url,+ .extra = frame_extra,};- const is_blocking = mode == .normal;-- // Once parsing is done, the deferred-script batch has already drained and- // won't run again, so a non-blocking script inserted afterwards would go- // unprocessed. Run it immediately instead. Remote scripts still need to- // be queued so they execute when their fetch completes.- const run_immediately = is_blocking or (self.base.static_scripts_done and remote_url == null);- if (run_immediately == false) {- self.base.scriptList(script).append(&script.node);- }+ if (mode == .normal) {+ // Blocking: fetch synchronously and evaluate before the parser resumes.- if (remote_url) |url| {- if (comptime IS_DEBUG) {- var ls: js.Local.Scope = undefined;- frame.js.localScope(&ls);- defer ls.deinit();-- log.debug(.http, "script queue", .{- .ctx = ctx,- .url = remote_url.?,- .element = element,- .stack = ls.local.stackTrace() catch "???",- });- }+ // A consumed preload (waitForPreload below) is owned by us: its buffer+ // is borrowed by `script`, so it must outlive eval.+ var consumed_preload: ?*Script = null;+ defer if (consumed_preload) |p| {+ p.deinit();+ };- const was_evaluating = self.base.is_evaluating;- self.base.is_evaluating = true;- defer self.base.endEvaluationWindow(was_evaluating);+ {+ const was_evaluating = self.base.is_evaluating;+ self.base.is_evaluating = true;+ defer self.base.endEvaluationWindow(was_evaluating);- if (is_blocking) {- if (self.waitForPreload(url)) |pre| {+ if (self.waitForPreload(remote_url)) |pre| {// There was a preloaded script, we borrow it's source and statusconsumed_preload = pre;script.source = pre.source;@@ -349,7 +356,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_escript.complete = true;} else {const response = try self.base.client.syncRequest(arena, .{- .url = url,+ .url = remote_url,.method = .GET,.frame_id = frame._frame_id,.loader_id = frame._loader_id,@@ -365,48 +372,107 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_escript.status = response.status;script.complete = true;}- } else {- errdefer {- self.base.scriptList(script).remove(&script.node);- // Let the outer errdefer handle releasing the arena if client.request fails- }+ handover = true;+ }- try frame.makeRequest(.{- .ctx = script,- .url = url,- .method = .GET,- .frame_id = frame._frame_id,- .loader_id = frame._loader_id,- .headers = try self.getHeaders(),- .cookie_jar = &frame._session.cookie_jar,- .cookie_origin = frame.url,- .resource_type = .script,- .notification = frame._session.notification,- .start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,- .header_callback = Script.headerCallback,- .data_callback = Script.dataCallback,- .done_callback = Script.doneCallback,- .error_callback = Script.errorCallback,- // Nothing holds the transfer; teardown cleanup runs through- // the manager's script lists.- .shutdown_callback = HttpClient.noopShutdown,- });+ if (script.status < 200 or script.status > 299) {+ log.info(.http, "script load error", .{ .status = script.status });+ script.executeCallback(comptime .wrap("error"));+ script.deinit();+ return;}- handover = true;+ return self.evalNow(script);}- if (run_immediately == false) {+ // async/defer: queue the script and fetch in the background; doneCallback+ // routes it through the ready_scripts / defer_scripts draining.+ self.base.scriptList(script).append(&script.node);++ const was_evaluating = self.base.is_evaluating;+ self.base.is_evaluating = true;+ defer self.base.endEvaluationWindow(was_evaluating);++ errdefer self.base.scriptList(script).remove(&script.node);+ try frame.makeRequest(.{+ .ctx = script,+ .url = remote_url,+ .method = .GET,+ .frame_id = frame._frame_id,+ .loader_id = frame._loader_id,+ .headers = try self.getHeaders(),+ .cookie_jar = &frame._session.cookie_jar,+ .cookie_origin = frame.url,+ .resource_type = .script,+ .notification = frame._session.notification,+ .start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,+ .header_callback = Script.headerCallback,+ .data_callback = Script.dataCallback,+ .done_callback = Script.doneCallback,+ .error_callback = Script.errorCallback,+ // Nothing holds the transfer; teardown cleanup runs through+ // the manager's script lists.+ .shutdown_callback = HttpClient.noopShutdown,+ });+ handover = true;+}++// A <script> with no src. Runs synchronously right now, except an inline+// module during parsing, which waits its turn in defer_scripts.+fn addInlineScript(self: *ScriptManager, script_element: *Element.Html.Script, kind: Script.Extra.FrameExtra.Kind) !void {+ const node = script_element.asElement().asNode();++ const source_len = node.childTextContentLen();+ if (source_len == 0) {+ // if content is appended later, we'll execute it then since+ // script_element._executed is still falsereturn;}- if (script.status < 200 or script.status > 299) {- log.info(.http, "script load error", .{ .status = script.status });- script.executeCallback(comptime .wrap("error"));- script.deinit();+ const frame = self.frame;+ const arena = try frame.getArena(source_len + @sizeOf(Script) + 1, "SM.addInlineScript");+ errdefer frame.releaseArena(arena);++ const source = blk: {+ const buf = try arena.alloc(u8, source_len + 1);+ var writer: std.Io.Writer = .fixed(buf);+ try node.getChildTextContent(&writer);+ buf[source_len] = 0;+ break :blk buf[0..source_len :0];+ };++ script_element._executed = true;++ const mode: Script.Extra.FrameExtra.Mode = if (kind == .module) .@"defer" else .normal;+ const script = try arena.create(Script);+ script.* = .{+ .node = .{},+ .arena = arena,+ .manager = &self.base,+ .source = .{ .@"inline" = source },+ .complete = true,+ .status = 200,+ .url = frame.base(),+ .extra = .{ .frame = .{+ .kind = kind,+ .mode = mode,+ .script_element = script_element,+ .frame = frame,+ } },+ };++ // An inline module found during parsing waits its turn in document order.+ // Once parsing is done, the deferred batch has already drained and won't+ // run again, so run it immediately instead.+ if (mode == .@"defer" and self.base.static_scripts_done == false) {+ self.base.scriptList(script).append(&script.node);return;}+ self.evalNow(script);+}++fn evalNow(self: *ScriptManager, script: *Script) void {// could have already been evaluating if this is dynamically addedconst was_evaluating = self.base.is_evaluating;self.base.is_evaluating = true;@@ -422,6 +488,14 @@ pub fn staticScriptsDone(self: *ScriptManager) void {self.base.staticScriptsDone();}+// Removes and returns the preload entry for `url` in whatever state it's in.+fn takePreload(self: *ScriptManager, url: [:0]const u8) ?*Script {+ const kv = self.preloaded_scripts.fetchRemove(url) orelse return null;+ return switch (kv.value.state) {+ inline else => |script| script,+ };+}+const PreloadedScript = struct {state: State,@@ -438,6 +512,11 @@ const PreloadedScript = struct {fn doneCallback(ctx: *anyopaque) !void {const script: *Script = @ptrCast(@alignCast(ctx));+ if (script.extra != .preload) {+ // Adopted by a real <script> element (addFromElement) while the+ // fetch was in flight; complete it as a normal frame script.+ return Script.doneCallback(ctx);+ }script.complete = true;if (comptime IS_DEBUG) {log.debug(.http, "script fetch complete", .{ .req = script.url });@@ -450,6 +529,11 @@ const PreloadedScript = struct {fn errorCallback(ctx: *anyopaque, err: anyerror) void {const script: *Script = @ptrCast(@alignCast(ctx));+ if (script.extra != .preload) {+ // Adopted mid-flight; fail it as a normal frame script (list+ // unlink, error event on the element and the hint link).+ return Script.errorCallback(ctx, err);+ }if (script.status == 404) {log.info(.http, "script 404", .{ .req = script.url, .extra = "preload" });} else {@@ -469,6 +553,11 @@ const PreloadedScript = struct {// errorCallback, since the owner is being torn down.fn shutdownCallback(ctx: *anyopaque) void {const script: *Script = @ptrCast(@alignCast(ctx));+ if (script.extra != .preload) {+ // Adopted: the Script lives in the manager's script lists and is+ // reaped by reset(), same as any async script (noopShutdown).+ return;+ }const self: *ScriptManager = @fieldParentPtr("base", script.manager);_ = self.preloaded_scripts.remove(script.url);script.deinit();diff --git a/src/browser/tests/element/html/script/preload_nonblocking.html b/src/browser/tests/element/html/script/preload_nonblocking.htmlnew file mode 100644index 000000000..04bc18c8d--- /dev/null+++ b/src/browser/tests/element/html/script/preload_nonblocking.html@@ -0,0 +1,56 @@+<!DOCTYPE html>+<script src="../../../testing.js"></script>++<!--+ Non-blocking consumers of <link rel=preload as=script>: async, defer, and+ dynamically-inserted scripts adopt the preloaded Script (takePreload in+ addFromElement) instead of fetching the URL a second time.++ Each /serve-count/ URL is served by the test server with a body of+ `window.__serve_count_<name> = N;` where N counts serves of that URL — so the+ body the consumer executes records whether the preload was reused (1) or a+ duplicate fetch happened (2). The responses are Cache-Control: no-store, so+ the HTTP cache can't mask a duplicate fetch.+-->++<!--+ Adopted by a defer script. The parser doesn't pump HTTP between the link and+ the script tag, so the entry is still in flight at adoption: this exercises+ loading-adoption, where the transfer completes after the Script has been+ rewritten into a .frame defer script (PreloadedScript.doneCallback delegates+ to Script.doneCallback).+-->+<link rel="preload" as="script" href="/serve-count/defer.js" onload="window.link_defer_load = true">+<script defer src="/serve-count/defer.js"></script>++<!-- Same, adopted by an async script. -->+<link rel="preload" as="script" href="/serve-count/async.js" onload="window.link_async_load = true">+<script async src="/serve-count/async.js"></script>++<!--+ Adopted by a dynamically-inserted script (defaults to async). The blocking+ empty.js fetch below pumps the HTTP client, so by insertion time the preload+ may be done (done-adoption: queued straight onto ready_scripts) or still in+ flight (loading-adoption) — both must produce the same result.+-->+<link rel="preload" as="script" href="/serve-count/dynamic.js" onload="window.link_dynamic_load = true">+<script src="empty.js"></script>+<script>+ const s = document.createElement('script');+ s.src = '/serve-count/dynamic.js';+ document.head.appendChild(s);+</script>++<script id="checks">+ testing.onload(() => {+ // Every consumer executed the first (and only) serve of its URL.+ testing.expectEqual(1, window.__serve_count_defer);+ testing.expectEqual(1, window.__serve_count_async);+ testing.expectEqual(1, window.__serve_count_dynamic);+ // Adoption must not lose the hint links' load events (hint_element rides+ // on the Script, outside extra).+ testing.expectEqual(true, window.link_defer_load);+ testing.expectEqual(true, window.link_async_load);+ testing.expectEqual(true, window.link_dynamic_load);+ });+</script>diff --git a/src/browser/webapi/Node.zig b/src/browser/webapi/Node.zigindex 2151e1d14..c9f4603c5 100644--- a/src/browser/webapi/Node.zig+++ b/src/browser/webapi/Node.zig@@ -306,6 +306,19 @@ pub fn getChildTextContent(self: *Node, writer: *std.Io.Writer) error{WriteFaile}}+// The byte length of what getChildTextContent writes; lets callers size a+// buffer (or arena) before extracting.+pub fn childTextContentLen(self: *Node) usize {+ var len: usize = 0;+ var it = self.childrenIterator();+ while (it.next()) |child| {+ if (child.is(CData.Text)) |text| {+ len += text._proto._data.str().len;+ }+ }+ return len;+}+pub fn setTextContent(self: *Node, data: []const u8, frame: *Frame) !void {switch (self._type) {.element => |el| {diff --git a/src/testing.zig b/src/testing.zigindex cc3a191da..05f7b8150 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -605,6 +605,11 @@ fn serveCDP(wg: *std.Thread.WaitGroup) !void {test_app.network.run();}+// /serve-count/ counters; only ever touched from the test HTTP server thread.+var serve_count_defer: u32 = 0;+var serve_count_async: u32 = 0;+var serve_count_dynamic: u32 = 0;+fn testHTTPHandler(req: *std.http.Server.Request) !void {const path = req.head.target;@@ -732,6 +737,30 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {});}+ if (std.mem.startsWith(u8, path, "/serve-count/") and std.mem.endsWith(u8, path, ".js")) {+ // Serves `window.__serve_count_<name> = N;` where N counts how many+ // times this URL has been served. Lets a fixture assert a preloaded+ // script was fetched exactly once: the body the consumer executes+ // carries N == 1, while a duplicate fetch would execute N == 2.+ // no-store so the second fetch can't be satisfied by the HTTP cache.+ const name = path["/serve-count/".len .. path.len - ".js".len];+ const slot: *u32 = blk: {+ if (std.mem.eql(u8, name, "defer")) break :blk &serve_count_defer;+ if (std.mem.eql(u8, name, "async")) break :blk &serve_count_async;+ if (std.mem.eql(u8, name, "dynamic")) break :blk &serve_count_dynamic;+ return req.respond("unknown counter", .{ .status = .not_found });+ };+ slot.* += 1;+ var buf: [64]u8 = undefined;+ const body = try std.fmt.bufPrint(&buf, "window.__serve_count_{s} = {d};", .{ name, slot.* });+ return req.respond(body, .{+ .extra_headers = &.{+ .{ .name = "Content-Type", .value = "application/javascript" },+ .{ .name = "Cache-Control", .value = "no-store" },+ },+ });+ }+if (std.mem.eql(u8, path, "/xhr/500")) {return req.respond("Internal Server Error", .{.status = .internal_server_error,