Lightpanda: the headless browser designed for AI and automation
perf, http: Preload modules
Karl Seguina70c3cefb885
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex bbe669fc2..a9a437f5e 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -1714,6 +1714,23 @@ pub fn preloadScriptHint(self: *Frame, href: []const u8) void {self._script_manager.preloadScript(resolved) catch {};}+// start prefetching <link rel="modulepreload" href=...>+pub fn preloadModuleHint(self: *Frame, href: []const u8) void {+ if (self.isGoingAway() or self._parse_mode == .fragment) {+ return;+ }++ // The url becomes the imported_modules key, which must outlive the fetch+ // so it lives on the frame arena+ const resolved = URL.resolve(self.arena, self.base(), href, .{ .encoding = self.charset }) catch return;+ if (!std.ascii.startsWithIgnoreCase(resolved, "http:") and !std.ascii.startsWithIgnoreCase(resolved, "https:")) {+ // data:/blob: are synthesized locally — no round-trip to hide.+ return;+ }++ self._script_manager.base.preloadModuleHint(resolved, self.url) catch {};+}+// Synchronously fetch and parse an external `<link rel=stylesheet>`.// href is passed in as an optimization since the [currently] only callsite has// it, so why look it up again?diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zigindex f917ced60..a1969501f 100644--- a/src/browser/ScriptManagerBase.zig+++ b/src/browser/ScriptManagerBase.zig@@ -217,10 +217,19 @@ pub fn resolveSpecifier(self: *ScriptManagerBase, arena: Allocator, base: [:0]coreturn error.SpecifierResolutionFailed;}-pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []const u8) !void {+const PreloadOpts = struct {+ hint: bool = false,+};+pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []const u8, opts: PreloadOpts) !void {const gop = try self.imported_modules.getOrPut(self.allocator, url);if (gop.found_existing) {- gop.value_ptr.waiters += 1;+ if (gop.value_ptr.hint) {+ // A <link rel=modulepreload> never calls waitForImport, so the+ // first real import adopts its waiter slot rather than adding one.+ gop.value_ptr.hint = false;+ } else {+ gop.value_ptr.waiters += 1;+ }return;}errdefer _ = self.imported_modules.remove(url);@@ -239,7 +248,7 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co.extra = .import,};- gop.value_ptr.* = ImportedModule{};+ gop.value_ptr.* = .{ .state = .{ .loading = script }, .hint = opts.hint };if (comptime IS_DEBUG) {var ls: js.Local.Scope = undefined;@@ -284,6 +293,15 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co};}+// <link rel=modulepreload href=...> — start fetching a module before import+// resolution discovers it.+pub fn preloadModuleHint(self: *ScriptManagerBase, url: [:0]const u8, referrer: []const u8) !void {+ if (self.imported_modules.contains(url)) {+ return;+ }+ try self.preloadImport(url, referrer, .{ .hint = true });+}+pub fn waitForImport(self: *ScriptManagerBase, url: [:0]const u8) !ModuleSource {const entry = self.imported_modules.getEntry(url) orelse {// It shouldn't be possible for v8 to ask for a module that we didn't@@ -324,7 +342,57 @@ pub fn waitForImport(self: *ScriptManagerBase, url: [:0]const u8) !ModuleSource}}+pub fn releaseImport(self: *ScriptManagerBase, url: [:0]const u8) void {+ const entry = self.imported_modules.getEntry(url) orelse {+ return;+ };+ if (entry.value_ptr.waiters > 1) {+ entry.value_ptr.waiters -= 1;+ return;+ }+ switch (entry.value_ptr.state) {+ .done => |script| script.deinit(),+ .loading, .err => return,+ }+ self.imported_modules.removeByPtr(entry.key_ptr);+}+pub fn getAsyncImport(self: *ScriptManagerBase, url: [:0]const u8, cb: ImportAsync.Callback, cb_data: *anyopaque, referrer: []const u8) !void {+ // A <link rel=modulepreload> hint may already be fetching/fetched this module+ if (self.imported_modules.getEntry(url)) |entry| {+ if (entry.value_ptr.hint) {+ switch (entry.value_ptr.state) {+ .loading => |script| {+ // fetch is in flight, take the script and turn it into+ // our normal getAsyncImport flow (e.g. what we do at the+ // end of this file as-if imported_modules didn't have this script)+ if (comptime IS_DEBUG) {+ log.debug(.http, "script adopt", .{ .url = url, .ctx = "dynamic module", .state = "loading" });+ }+ script.extra = .{ .import_async = .{ .callback = cb, .data = cb_data } };+ self.imported_modules.removeByPtr(entry.key_ptr);+ return;+ },+ .done => |script| {+ // fetch is complete; deliver through the normal+ // ready_scripts flow. evaluate() runs it now, or — if an+ // evaluation window is open — evaluate_pending runs it+ // when that window closes.+ if (comptime IS_DEBUG) {+ log.debug(.http, "script adopt", .{ .url = url, .ctx = "dynamic module", .state = "done" });+ }+ script.extra = .{ .import_async = .{ .callback = cb, .data = cb_data } };+ self.imported_modules.removeByPtr(entry.key_ptr);+ self.ready_scripts.append(&script.node);+ self.evaluate();+ return;+ },+ // The hint's fetch failed; give the import its own attempt.+ .err => {},+ }+ }+ }+const arena = try self.acquireArena(.large, "SM.getAsyncImport");errdefer self.releaseArena(arena);@@ -890,12 +958,17 @@ pub const ModuleSource = struct {pub const ImportedModule = struct {waiters: u16 = 1,- state: State = .loading,+ // Created by a <link rel=modulepreload> hint and not yet claimed by a real+ // import. While set, the single waiter slot belongs to the hint, which+ // will never collect it (see preloadModuleHint). A dynamic import may+ // adopt a hint entry outright (see getAsyncImport).+ hint: bool = false,+ state: State,buffer: std.ArrayList(u8) = .{},pub const State = union(enum) {err,- loading,+ loading: *Script,done: *Script,};};diff --git a/src/browser/js/Context.zig b/src/browser/js/Context.zigindex af39fe0a8..572e12741 100644--- a/src/browser/js/Context.zig+++ b/src/browser/js/Context.zig@@ -516,7 +516,7 @@ fn postCompileModule(self: *Context, mod: js.Module, url: [:0]const u8, local: *const owned_specifier = try self.arena.dupeZ(u8, normalized_specifier);nested_gop.key_ptr.* = owned_specifier;nested_gop.value_ptr.* = .{};- try script_manager.preloadImport(owned_specifier, url);+ try script_manager.preloadImport(owned_specifier, url, .{});} else if (nested_gop.value_ptr.module == null) {// Entry exists but module failed to compile previously.// The imported_modules entry may have been consumed, so@@ -524,7 +524,7 @@ fn postCompileModule(self: *Context, mod: js.Module, url: [:0]const u8, local: *// Key was stored via dupeZ so it has a sentinel in memory.const key = nested_gop.key_ptr.*;const key_z: [:0]const u8 = key.ptr[0..key.len :0];- try script_manager.preloadImport(key_z, url);+ try script_manager.preloadImport(key_z, url, .{});}}}@@ -733,6 +733,11 @@ fn _resolveModuleCallback(self: *Context, referrer: js.Module, specifier: [:0]coconst entry = self.module_cache.getPtr(normalized_specifier).?;if (entry.module) |m| {+ // This import registered a waiter via preloadImport when it was discovered+ // but the compiled module is already cached so we don't have to call+ // waitForImport. Release our waiter so we no longer hold on waiter on+ // the resource.+ self.script_manager.releaseImport(normalized_specifier);return local.toLocal(m).handle;}@@ -740,7 +745,7 @@ fn _resolveModuleCallback(self: *Context, referrer: js.Module, specifier: [:0]coerror.UnknownModule => blk: {// Module is in cache but was consumed from imported_modules// (e.g., by a previous failed resolution). Re-preload and retry.- try self.script_manager.preloadImport(normalized_specifier, referrer_path);+ try self.script_manager.preloadImport(normalized_specifier, referrer_path, .{});break :blk try self.script_manager.waitForImport(normalized_specifier);},else => return err,diff --git a/src/browser/tests/element/html/link.html b/src/browser/tests/element/html/link.htmlindex d3f2faa0f..754fe27e7 100644--- a/src/browser/tests/element/html/link.html+++ b/src/browser/tests/element/html/link.html@@ -120,11 +120,15 @@<script id="refs">{const rels = ['stylesheet', 'preload', 'modulepreload'];+ // stylesheet fetching is off by default and a bare preload has no `as`, so+ // neither touches the network. modulepreload actually fetches, so it needs+ // a real file — the load event still fires synthetically either way.+ const hrefs = ['/nope', '/nope', 'script/modulepreload.js'];const results = rels.map(() => false);rels.forEach((rel, i) => {let link = document.createElement('link')link.rel = rel;- link.href = '/nope';+ link.href = hrefs[i];link.onload = () => results[i] = true;document.documentElement.appendChild(link);});diff --git a/src/browser/tests/element/html/script/modulepreload.html b/src/browser/tests/element/html/script/modulepreload.htmlnew file mode 100644index 000000000..72b19892a--- /dev/null+++ b/src/browser/tests/element/html/script/modulepreload.html@@ -0,0 +1,57 @@+<!DOCTYPE html>+<script src="../../../testing.js"></script>++<!--+ The modulepreload hint is declared before the module that imports it, so by+ the time import resolution asks for modulepreload.js the fetch is already in+ flight (or done). The hint registers no waiter of its own: the first real+ preloadImport adopts its slot, so waitForImport transfers buffer ownership+ exactly as if the hint never existed. Both halves of that handover run under+ the leak detector here.+-->+<link rel="modulepreload" href="modulepreload.js">++<!--+ A hinted module that nothing imports is fetched but never evaluated. It+ lingers as a `.done` imported_modules entry until reset() frees it —+ exercises the map cleanup under the leak detector — and it must not run on+ its own, nor delay the load event.+-->+<link rel="modulepreload" href="modulepreload_unused.js">++<!-- A duplicate hint must not double-fetch or double-register a waiter. -->+<link rel="modulepreload" href="modulepreload.js">++<!--+ The Vite pattern: a runtime-style hint immediately followed by a dynamic+ import(). The hint's load event is synthetic, so the import fires while the+ hint's fetch is still in flight — getAsyncImport adopts the loading Script+ (converts it to .import_async) instead of fetching the module a second time.+-->+<link rel="modulepreload" href="modulepreload_dynamic.js">+<script id="dynamic_adopt_loading">+ window.dynamic_val = '';+ import('./modulepreload_dynamic.js').then((m) => { window.dynamic_val = m.dval; });+ testing.onload(() => testing.expectEqual('dynamic-preloaded', window.dynamic_val));+</script>++<link rel="modulepreload" href="modulepreload_dynamic2.js">++<script id="consume" type="module">+ import { val } from "./modulepreload.js";+ testing.expectEqual('preloaded-module', val, {script_id: 'consume'});+</script>++<script id="dynamic_adopt_done" type="module">+ // Deferred module: by the time this evaluates, the hint above has usually+ // finished fetching — the import() adopts a .done entry and the body is+ // delivered inline. (If the fetch is still in flight it exercises the+ // loading-adoption path instead; both must produce the same result.)+ const m = await import("./modulepreload_dynamic2.js");+ testing.expectEqual('dynamic-preloaded-2', m.dval, {script_id: 'dynamic_adopt_done'});+</script>++<script id="unused_check">+ // A hint that nothing imports must never evaluate.+ testing.expectEqual(undefined, window.modulepreload_unused_ran);+</script>diff --git a/src/browser/tests/element/html/script/modulepreload.js b/src/browser/tests/element/html/script/modulepreload.jsnew file mode 100644index 000000000..fed413f5b--- /dev/null+++ b/src/browser/tests/element/html/script/modulepreload.js@@ -0,0 +1 @@+export const val = 'preloaded-module';diff --git a/src/browser/tests/element/html/script/modulepreload_dynamic.js b/src/browser/tests/element/html/script/modulepreload_dynamic.jsnew file mode 100644index 000000000..d8c3e9858--- /dev/null+++ b/src/browser/tests/element/html/script/modulepreload_dynamic.js@@ -0,0 +1 @@+export const dval = 'dynamic-preloaded';diff --git a/src/browser/tests/element/html/script/modulepreload_dynamic2.js b/src/browser/tests/element/html/script/modulepreload_dynamic2.jsnew file mode 100644index 000000000..e9983ba8f--- /dev/null+++ b/src/browser/tests/element/html/script/modulepreload_dynamic2.js@@ -0,0 +1 @@+export const dval = 'dynamic-preloaded-2';diff --git a/src/browser/tests/element/html/script/modulepreload_unused.js b/src/browser/tests/element/html/script/modulepreload_unused.jsnew file mode 100644index 000000000..3535633b2--- /dev/null+++ b/src/browser/tests/element/html/script/modulepreload_unused.js@@ -0,0 +1,5 @@+// Nothing imports this module, so it must never be evaluated. If it runs, the+// flag trips the assertion in modulepreload.html (and this fail() fires+// directly).+window.modulepreload_unused_ran = true;+testing.fail('an unconsumed <link rel=modulepreload> must not evaluate');diff --git a/src/browser/tests/page/module.html b/src/browser/tests/page/module.htmlindex bb820332b..bba90ebe5 100644--- a/src/browser/tests/page/module.html+++ b/src/browser/tests/page/module.html@@ -58,6 +58,18 @@testing.expectEqual('a', getFromA());</script>+<script id=diamond-imports type=module>+ // a and b both statically import diamond-shared.js: two waiter+ // registrations on one imported_modules entry, but only one waitForImport —+ // the other resolution is satisfied from module_cache and must release its+ // slot (Context._resolveModuleCallback) so the source buffer is freed at+ // resolution, not at reset. Runs under the leak detector.+ import { aShared } from "./modules/diamond-a.js";+ import { bShared } from "./modules/diamond-b.js";+ testing.expectEqual('diamond-shared', aShared);+ testing.expectEqual('diamond-shared', bShared);+</script>+<script id=basic-async type=module>import { "val1" as val1 } from "./mod1.js";const m = await import("./mod1.js");diff --git a/src/browser/tests/page/modules/diamond-a.js b/src/browser/tests/page/modules/diamond-a.jsnew file mode 100644index 000000000..c8f5b7ea1--- /dev/null+++ b/src/browser/tests/page/modules/diamond-a.js@@ -0,0 +1 @@+export { sharedValue as aShared } from './diamond-shared.js';diff --git a/src/browser/tests/page/modules/diamond-b.js b/src/browser/tests/page/modules/diamond-b.jsnew file mode 100644index 000000000..a6cdccf5a--- /dev/null+++ b/src/browser/tests/page/modules/diamond-b.js@@ -0,0 +1 @@+export { sharedValue as bShared } from './diamond-shared.js';diff --git a/src/browser/tests/page/modules/diamond-shared.js b/src/browser/tests/page/modules/diamond-shared.jsnew file mode 100644index 000000000..884c85534--- /dev/null+++ b/src/browser/tests/page/modules/diamond-shared.js@@ -0,0 +1 @@+export const sharedValue = 'diamond-shared';diff --git a/src/browser/webapi/element/html/Link.zig b/src/browser/webapi/element/html/Link.zigindex 999bac1a1..0e8e1d424 100644--- a/src/browser/webapi/element/html/Link.zig+++ b/src/browser/webapi/element/html/Link.zig@@ -203,33 +203,27 @@ pub fn linkAddedCallback(self: *Link, frame: *Frame) !void {const rel = element.getAttributeSafe(comptime .wrap("rel")) orelse return;// Opt-in fetch for `rel="stylesheet"` — drives `frame.loadExternalStylesheet`,- // which fires the load/error event itself. Other rels (preload,- // modulepreload) and the disabled case keep the rendering-free stub that- // fires a synthetic `load` event without touching the network.+ // which fires the load/error event itself. preload/modulepreload start a+ // background fetch (a hint, never evaluated on its own) and fire a+ // synthetic `load` event regardless of how the fetch goes.if (std.mem.eql(u8, rel, "stylesheet")) {return frame.loadExternalStylesheet(self, href);}- var queue_load = false;if (std.mem.eql(u8, rel, "preload")) {const as = element.getAttributeSafe(comptime .wrap("as")) orelse "";if (std.ascii.eqlIgnoreCase(as, "script")) {frame.preloadScriptHint(href);}- queue_load = true;- }-- {- // this block just means we don't need to re-check rel for a type we- // already processed, e.g. "preload"- const loadable_rels = std.StaticStringMap(void).initComptime(.{- .{ "stylesheet", {} },- .{ "modulepreload", {} },- });-- if (queue_load == false and loadable_rels.has(rel) == false) {- return;+ } else if (std.mem.eql(u8, rel, "modulepreload")) {+ // "as" default to script in this case+ const as = element.getAttributeSafe(comptime .wrap("as")) orelse "";+ if (as.len == 0 or std.ascii.eqlIgnoreCase(as, "script")) {+ frame.preloadModuleHint(href);}+ } else {+ // remaining rels (icon, canonical, ...): no fetch, no load event+ return;}try frame.queueLoad(self._proto);