Skip to content

Lightpanda: the headless browser designed for AI and automation

perf, http: Preload modules

Karl Seguina70c3cefb885
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index bbe669fc2..a9a437f5e 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -1714,6 +1714,23 @@ pub fn preloadScriptHint(self: *Frame, href: []const u8) void {
self._script_manager.preloadScript(resolved) catch {};
}
+// start prefetching <link rel="modulepreload" href=...>
+pub fn preloadModuleHint(self: *Frame, href: []const u8) void {
+ if (self.isGoingAway() or self._parse_mode == .fragment) {
+ return;
+ }
+
+ // The url becomes the imported_modules key, which must outlive the fetch
+ // so it lives on the frame arena
+ const resolved = URL.resolve(self.arena, self.base(), href, .{ .encoding = self.charset }) catch return;
+ if (!std.ascii.startsWithIgnoreCase(resolved, "http:") and !std.ascii.startsWithIgnoreCase(resolved, "https:")) {
+ // data:/blob: are synthesized locally — no round-trip to hide.
+ return;
+ }
+
+ self._script_manager.base.preloadModuleHint(resolved, self.url) catch {};
+}
+
// Synchronously fetch and parse an external `<link rel=stylesheet>`.
// href is passed in as an optimization since the [currently] only callsite has
// it, so why look it up again?
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zig
index f917ced60..a1969501f 100644
--- a/src/browser/ScriptManagerBase.zig
+++ b/src/browser/ScriptManagerBase.zig
@@ -217,10 +217,19 @@ pub fn resolveSpecifier(self: *ScriptManagerBase, arena: Allocator, base: [:0]co
return error.SpecifierResolutionFailed;
}
-pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []const u8) !void {
+const PreloadOpts = struct {
+ hint: bool = false,
+};
+pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []const u8, opts: PreloadOpts) !void {
const gop = try self.imported_modules.getOrPut(self.allocator, url);
if (gop.found_existing) {
- gop.value_ptr.waiters += 1;
+ if (gop.value_ptr.hint) {
+ // A <link rel=modulepreload> never calls waitForImport, so the
+ // first real import adopts its waiter slot rather than adding one.
+ gop.value_ptr.hint = false;
+ } else {
+ gop.value_ptr.waiters += 1;
+ }
return;
}
errdefer _ = self.imported_modules.remove(url);
@@ -239,7 +248,7 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co
.extra = .import,
};
- gop.value_ptr.* = ImportedModule{};
+ gop.value_ptr.* = .{ .state = .{ .loading = script }, .hint = opts.hint };
if (comptime IS_DEBUG) {
var ls: js.Local.Scope = undefined;
@@ -284,6 +293,15 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co
};
}
+// <link rel=modulepreload href=...> — start fetching a module before import
+// resolution discovers it.
+pub fn preloadModuleHint(self: *ScriptManagerBase, url: [:0]const u8, referrer: []const u8) !void {
+ if (self.imported_modules.contains(url)) {
+ return;
+ }
+ try self.preloadImport(url, referrer, .{ .hint = true });
+}
+
pub fn waitForImport(self: *ScriptManagerBase, url: [:0]const u8) !ModuleSource {
const entry = self.imported_modules.getEntry(url) orelse {
// It shouldn't be possible for v8 to ask for a module that we didn't
@@ -324,7 +342,57 @@ pub fn waitForImport(self: *ScriptManagerBase, url: [:0]const u8) !ModuleSource
}
}
+pub fn releaseImport(self: *ScriptManagerBase, url: [:0]const u8) void {
+ const entry = self.imported_modules.getEntry(url) orelse {
+ return;
+ };
+ if (entry.value_ptr.waiters > 1) {
+ entry.value_ptr.waiters -= 1;
+ return;
+ }
+ switch (entry.value_ptr.state) {
+ .done => |script| script.deinit(),
+ .loading, .err => return,
+ }
+ self.imported_modules.removeByPtr(entry.key_ptr);
+}
+
pub fn getAsyncImport(self: *ScriptManagerBase, url: [:0]const u8, cb: ImportAsync.Callback, cb_data: *anyopaque, referrer: []const u8) !void {
+ // A <link rel=modulepreload> hint may already be fetching/fetched this module
+ if (self.imported_modules.getEntry(url)) |entry| {
+ if (entry.value_ptr.hint) {
+ switch (entry.value_ptr.state) {
+ .loading => |script| {
+ // fetch is in flight, take the script and turn it into
+ // our normal getAsyncImport flow (e.g. what we do at the
+ // end of this file as-if imported_modules didn't have this script)
+ if (comptime IS_DEBUG) {
+ log.debug(.http, "script adopt", .{ .url = url, .ctx = "dynamic module", .state = "loading" });
+ }
+ script.extra = .{ .import_async = .{ .callback = cb, .data = cb_data } };
+ self.imported_modules.removeByPtr(entry.key_ptr);
+ return;
+ },
+ .done => |script| {
+ // fetch is complete; deliver through the normal
+ // ready_scripts flow. evaluate() runs it now, or — if an
+ // evaluation window is open — evaluate_pending runs it
+ // when that window closes.
+ if (comptime IS_DEBUG) {
+ log.debug(.http, "script adopt", .{ .url = url, .ctx = "dynamic module", .state = "done" });
+ }
+ script.extra = .{ .import_async = .{ .callback = cb, .data = cb_data } };
+ self.imported_modules.removeByPtr(entry.key_ptr);
+ self.ready_scripts.append(&script.node);
+ self.evaluate();
+ return;
+ },
+ // The hint's fetch failed; give the import its own attempt.
+ .err => {},
+ }
+ }
+ }
+
const arena = try self.acquireArena(.large, "SM.getAsyncImport");
errdefer self.releaseArena(arena);
@@ -890,12 +958,17 @@ pub const ModuleSource = struct {
pub const ImportedModule = struct {
waiters: u16 = 1,
- state: State = .loading,
+ // Created by a <link rel=modulepreload> hint and not yet claimed by a real
+ // import. While set, the single waiter slot belongs to the hint, which
+ // will never collect it (see preloadModuleHint). A dynamic import may
+ // adopt a hint entry outright (see getAsyncImport).
+ hint: bool = false,
+ state: State,
buffer: std.ArrayList(u8) = .{},
pub const State = union(enum) {
err,
- loading,
+ loading: *Script,
done: *Script,
};
};
diff --git a/src/browser/js/Context.zig b/src/browser/js/Context.zig
index af39fe0a8..572e12741 100644
--- a/src/browser/js/Context.zig
+++ b/src/browser/js/Context.zig
@@ -516,7 +516,7 @@ fn postCompileModule(self: *Context, mod: js.Module, url: [:0]const u8, local: *
const owned_specifier = try self.arena.dupeZ(u8, normalized_specifier);
nested_gop.key_ptr.* = owned_specifier;
nested_gop.value_ptr.* = .{};
- try script_manager.preloadImport(owned_specifier, url);
+ try script_manager.preloadImport(owned_specifier, url, .{});
} else if (nested_gop.value_ptr.module == null) {
// Entry exists but module failed to compile previously.
// The imported_modules entry may have been consumed, so
@@ -524,7 +524,7 @@ fn postCompileModule(self: *Context, mod: js.Module, url: [:0]const u8, local: *
// Key was stored via dupeZ so it has a sentinel in memory.
const key = nested_gop.key_ptr.*;
const key_z: [:0]const u8 = key.ptr[0..key.len :0];
- try script_manager.preloadImport(key_z, url);
+ try script_manager.preloadImport(key_z, url, .{});
}
}
}
@@ -733,6 +733,11 @@ fn _resolveModuleCallback(self: *Context, referrer: js.Module, specifier: [:0]co
const entry = self.module_cache.getPtr(normalized_specifier).?;
if (entry.module) |m| {
+ // This import registered a waiter via preloadImport when it was discovered
+ // but the compiled module is already cached so we don't have to call
+ // waitForImport. Release our waiter so we no longer hold on waiter on
+ // the resource.
+ self.script_manager.releaseImport(normalized_specifier);
return local.toLocal(m).handle;
}
@@ -740,7 +745,7 @@ fn _resolveModuleCallback(self: *Context, referrer: js.Module, specifier: [:0]co
error.UnknownModule => blk: {
// Module is in cache but was consumed from imported_modules
// (e.g., by a previous failed resolution). Re-preload and retry.
- try self.script_manager.preloadImport(normalized_specifier, referrer_path);
+ try self.script_manager.preloadImport(normalized_specifier, referrer_path, .{});
break :blk try self.script_manager.waitForImport(normalized_specifier);
},
else => return err,
diff --git a/src/browser/tests/element/html/link.html b/src/browser/tests/element/html/link.html
index d3f2faa0f..754fe27e7 100644
--- a/src/browser/tests/element/html/link.html
+++ b/src/browser/tests/element/html/link.html
@@ -120,11 +120,15 @@
<script id="refs">
{
const rels = ['stylesheet', 'preload', 'modulepreload'];
+ // stylesheet fetching is off by default and a bare preload has no `as`, so
+ // neither touches the network. modulepreload actually fetches, so it needs
+ // a real file — the load event still fires synthetically either way.
+ const hrefs = ['/nope', '/nope', 'script/modulepreload.js'];
const results = rels.map(() => false);
rels.forEach((rel, i) => {
let link = document.createElement('link')
link.rel = rel;
- link.href = '/nope';
+ link.href = hrefs[i];
link.onload = () => results[i] = true;
document.documentElement.appendChild(link);
});
diff --git a/src/browser/tests/element/html/script/modulepreload.html b/src/browser/tests/element/html/script/modulepreload.html
new file mode 100644
index 000000000..72b19892a
--- /dev/null
+++ b/src/browser/tests/element/html/script/modulepreload.html
@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+<script src="../../../testing.js"></script>
+
+<!--
+ The modulepreload hint is declared before the module that imports it, so by
+ the time import resolution asks for modulepreload.js the fetch is already in
+ flight (or done). The hint registers no waiter of its own: the first real
+ preloadImport adopts its slot, so waitForImport transfers buffer ownership
+ exactly as if the hint never existed. Both halves of that handover run under
+ the leak detector here.
+-->
+<link rel="modulepreload" href="modulepreload.js">
+
+<!--
+ A hinted module that nothing imports is fetched but never evaluated. It
+ lingers as a `.done` imported_modules entry until reset() frees it —
+ exercises the map cleanup under the leak detector — and it must not run on
+ its own, nor delay the load event.
+-->
+<link rel="modulepreload" href="modulepreload_unused.js">
+
+<!-- A duplicate hint must not double-fetch or double-register a waiter. -->
+<link rel="modulepreload" href="modulepreload.js">
+
+<!--
+ The Vite pattern: a runtime-style hint immediately followed by a dynamic
+ import(). The hint's load event is synthetic, so the import fires while the
+ hint's fetch is still in flight — getAsyncImport adopts the loading Script
+ (converts it to .import_async) instead of fetching the module a second time.
+-->
+<link rel="modulepreload" href="modulepreload_dynamic.js">
+<script id="dynamic_adopt_loading">
+ window.dynamic_val = '';
+ import('./modulepreload_dynamic.js').then((m) => { window.dynamic_val = m.dval; });
+ testing.onload(() => testing.expectEqual('dynamic-preloaded', window.dynamic_val));
+</script>
+
+<link rel="modulepreload" href="modulepreload_dynamic2.js">
+
+<script id="consume" type="module">
+ import { val } from "./modulepreload.js";
+ testing.expectEqual('preloaded-module', val, {script_id: 'consume'});
+</script>
+
+<script id="dynamic_adopt_done" type="module">
+ // Deferred module: by the time this evaluates, the hint above has usually
+ // finished fetching — the import() adopts a .done entry and the body is
+ // delivered inline. (If the fetch is still in flight it exercises the
+ // loading-adoption path instead; both must produce the same result.)
+ const m = await import("./modulepreload_dynamic2.js");
+ testing.expectEqual('dynamic-preloaded-2', m.dval, {script_id: 'dynamic_adopt_done'});
+</script>
+
+<script id="unused_check">
+ // A hint that nothing imports must never evaluate.
+ testing.expectEqual(undefined, window.modulepreload_unused_ran);
+</script>
diff --git a/src/browser/tests/element/html/script/modulepreload.js b/src/browser/tests/element/html/script/modulepreload.js
new file mode 100644
index 000000000..fed413f5b
--- /dev/null
+++ b/src/browser/tests/element/html/script/modulepreload.js
@@ -0,0 +1 @@
+export const val = 'preloaded-module';
diff --git a/src/browser/tests/element/html/script/modulepreload_dynamic.js b/src/browser/tests/element/html/script/modulepreload_dynamic.js
new file mode 100644
index 000000000..d8c3e9858
--- /dev/null
+++ b/src/browser/tests/element/html/script/modulepreload_dynamic.js
@@ -0,0 +1 @@
+export const dval = 'dynamic-preloaded';
diff --git a/src/browser/tests/element/html/script/modulepreload_dynamic2.js b/src/browser/tests/element/html/script/modulepreload_dynamic2.js
new file mode 100644
index 000000000..e9983ba8f
--- /dev/null
+++ b/src/browser/tests/element/html/script/modulepreload_dynamic2.js
@@ -0,0 +1 @@
+export const dval = 'dynamic-preloaded-2';
diff --git a/src/browser/tests/element/html/script/modulepreload_unused.js b/src/browser/tests/element/html/script/modulepreload_unused.js
new file mode 100644
index 000000000..3535633b2
--- /dev/null
+++ b/src/browser/tests/element/html/script/modulepreload_unused.js
@@ -0,0 +1,5 @@
+// Nothing imports this module, so it must never be evaluated. If it runs, the
+// flag trips the assertion in modulepreload.html (and this fail() fires
+// directly).
+window.modulepreload_unused_ran = true;
+testing.fail('an unconsumed <link rel=modulepreload> must not evaluate');
diff --git a/src/browser/tests/page/module.html b/src/browser/tests/page/module.html
index bb820332b..bba90ebe5 100644
--- a/src/browser/tests/page/module.html
+++ b/src/browser/tests/page/module.html
@@ -58,6 +58,18 @@
testing.expectEqual('a', getFromA());
</script>
+<script id=diamond-imports type=module>
+ // a and b both statically import diamond-shared.js: two waiter
+ // registrations on one imported_modules entry, but only one waitForImport —
+ // the other resolution is satisfied from module_cache and must release its
+ // slot (Context._resolveModuleCallback) so the source buffer is freed at
+ // resolution, not at reset. Runs under the leak detector.
+ import { aShared } from "./modules/diamond-a.js";
+ import { bShared } from "./modules/diamond-b.js";
+ testing.expectEqual('diamond-shared', aShared);
+ testing.expectEqual('diamond-shared', bShared);
+</script>
+
<script id=basic-async type=module>
import { "val1" as val1 } from "./mod1.js";
const m = await import("./mod1.js");
diff --git a/src/browser/tests/page/modules/diamond-a.js b/src/browser/tests/page/modules/diamond-a.js
new file mode 100644
index 000000000..c8f5b7ea1
--- /dev/null
+++ b/src/browser/tests/page/modules/diamond-a.js
@@ -0,0 +1 @@
+export { sharedValue as aShared } from './diamond-shared.js';
diff --git a/src/browser/tests/page/modules/diamond-b.js b/src/browser/tests/page/modules/diamond-b.js
new file mode 100644
index 000000000..a6cdccf5a
--- /dev/null
+++ b/src/browser/tests/page/modules/diamond-b.js
@@ -0,0 +1 @@
+export { sharedValue as bShared } from './diamond-shared.js';
diff --git a/src/browser/tests/page/modules/diamond-shared.js b/src/browser/tests/page/modules/diamond-shared.js
new file mode 100644
index 000000000..884c85534
--- /dev/null
+++ b/src/browser/tests/page/modules/diamond-shared.js
@@ -0,0 +1 @@
+export const sharedValue = 'diamond-shared';
diff --git a/src/browser/webapi/element/html/Link.zig b/src/browser/webapi/element/html/Link.zig
index 999bac1a1..0e8e1d424 100644
--- a/src/browser/webapi/element/html/Link.zig
+++ b/src/browser/webapi/element/html/Link.zig
@@ -203,33 +203,27 @@ pub fn linkAddedCallback(self: *Link, frame: *Frame) !void {
const rel = element.getAttributeSafe(comptime .wrap("rel")) orelse return;
// Opt-in fetch for `rel="stylesheet"` — drives `frame.loadExternalStylesheet`,
- // which fires the load/error event itself. Other rels (preload,
- // modulepreload) and the disabled case keep the rendering-free stub that
- // fires a synthetic `load` event without touching the network.
+ // which fires the load/error event itself. preload/modulepreload start a
+ // background fetch (a hint, never evaluated on its own) and fire a
+ // synthetic `load` event regardless of how the fetch goes.
if (std.mem.eql(u8, rel, "stylesheet")) {
return frame.loadExternalStylesheet(self, href);
}
- var queue_load = false;
if (std.mem.eql(u8, rel, "preload")) {
const as = element.getAttributeSafe(comptime .wrap("as")) orelse "";
if (std.ascii.eqlIgnoreCase(as, "script")) {
frame.preloadScriptHint(href);
}
- queue_load = true;
- }
-
- {
- // this block just means we don't need to re-check rel for a type we
- // already processed, e.g. "preload"
- const loadable_rels = std.StaticStringMap(void).initComptime(.{
- .{ "stylesheet", {} },
- .{ "modulepreload", {} },
- });
-
- if (queue_load == false and loadable_rels.has(rel) == false) {
- return;
+ } else if (std.mem.eql(u8, rel, "modulepreload")) {
+ // "as" default to script in this case
+ const as = element.getAttributeSafe(comptime .wrap("as")) orelse "";
+ if (as.len == 0 or std.ascii.eqlIgnoreCase(as, "script")) {
+ frame.preloadModuleHint(href);
}
+ } else {
+ // remaining rels (icon, canonical, ...): no fetch, no load event
+ return;
}
try frame.queueLoad(self._proto);