Skip to content

Lightpanda: the headless browser designed for AI and automation

perf: Pre-parse HTML to find and preload scripts

Karl Seguin2edc2eb1313c
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 4e7e5b34f..99277d936 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -69,6 +69,7 @@ const milliTimestamp = @import("../datetime.zig").milliTimestamp;
const GlobalEventHandlersLookup = @import("webapi/global_event_handlers.zig").Lookup;
+pub const preload = @import("frame/preload.zig");
pub const observers = @import("frame/observers.zig");
pub const user_input = @import("frame/user_input.zig");
pub const node_factory = @import("frame/node_factory.zig");
@@ -1516,6 +1517,8 @@ fn frameDoneCallback(ctx: *anyopaque) !void {
const raw_html = html.buffer.items;
+ preload.prescan(self, raw_html);
+
if (std.mem.eql(u8, self.charset, "UTF-8")) {
parser.parse(raw_html);
} else {
@@ -2054,40 +2057,6 @@ pub fn queueHashChange(self: *Frame, old_url: []const u8, new_url: []const u8) !
// splitting by route anyway).
const MAX_STYLESHEET_BYTES: usize = 2 * 1024 * 1024;
-// start prefetching <link rel="preload" as="script" href=...>`
-pub fn preloadScriptHint(self: *Frame, element: *Element.Html, href: []const u8) bool {
- if (self.isGoingAway() or self._parse_mode == .fragment) {
- return false;
- }
-
- const arena = self.getArena(.small, "Frame.preloadScriptHint") catch return false;
- defer self.releaseArena(arena);
-
- const resolved = URL.resolve(arena, self.base(), href, .{ .encoding = self.charset }) catch return false;
- if (!std.ascii.startsWithIgnoreCase(resolved, "http:") and !std.ascii.startsWithIgnoreCase(resolved, "https:")) {
- // data:/blob: are synthesized locally — no round-trip to hide.
- return false;
- }
- return self._script_manager.preloadScript(element, resolved) catch false;
-}
-
-// start prefetching <link rel="modulepreload" href=...>
-pub fn preloadModuleHint(self: *Frame, element: *Element.Html, href: []const u8) bool {
- if (self.isGoingAway() or self._parse_mode == .fragment) {
- return false;
- }
-
- // The url becomes the imported_modules key, which must outlive the fetch
- // so it lives on the frame arena
- const resolved = URL.resolve(self.arena, self.base(), href, .{ .encoding = self.charset }) catch return false;
- if (!std.ascii.startsWithIgnoreCase(resolved, "http:") and !std.ascii.startsWithIgnoreCase(resolved, "https:")) {
- // data:/blob: are synthesized locally — no round-trip to hide.
- return false;
- }
-
- return self._script_manager.base.preloadModuleHint(element, resolved, self.url) catch false;
-}
-
// Synchronously fetch and parse an external `<link rel=stylesheet>`.
// href is passed in as an optimization since the [currently] only callsite has
// it, so why look it up again?
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig
index 95ec7334c..26c90f0cb 100644
--- a/src/browser/ScriptManager.zig
+++ b/src/browser/ScriptManager.zig
@@ -105,7 +105,8 @@ fn getHeaders(self: *ScriptManager) !HttpClient.Headers {
// Returns true when a fetch was started: the link's load/error event fires
// when the fetch settles. false (duplicate hint) = no event will fire.
-pub fn preloadScript(self: *ScriptManager, element: *Element.Html, url: []const u8) !bool {
+// element is null when the hint came from the prescan rather than a <link>.
+pub fn preloadScript(self: *ScriptManager, element: ?*Element.Html, url: []const u8) !bool {
if (self.preloaded_scripts.contains(url)) {
return false;
}
@@ -290,13 +291,11 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
} };
if (mode != .normal) {
- if (self.takePreload(remote_url)) |pre| {
- // A <link rel=preload as=script> already fetched (or is still
- // fetching) this URL. Adopt the preloaded Script, zero-copy — the
- // classic-script mirror of getAsyncImport's module-hint adoption.
- // The transfer's still-registered PreloadedScript callbacks see
- // the rewritten extra and route to the normal Script callbacks.
- // (Blocking scripts consume preloads via waitForPreload below.)
+ var preloaded = self.takePreload(remote_url);
+ if (preloaded == null and kind == .module) {
+ preloaded = self.base.takeModuleHint(remote_url);
+ }
+ if (preloaded) |pre| {
if (comptime IS_DEBUG) {
log.debug(.http, "script adopt", .{ .url = remote_url, .ctx = ctx, .state = if (pre.complete) "done" else "loading" });
}
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zig
index d2ff6e4f9..c2e5f6104 100644
--- a/src/browser/ScriptManagerBase.zig
+++ b/src/browser/ScriptManagerBase.zig
@@ -216,7 +216,12 @@ pub fn resolveSpecifier(self: *ScriptManagerBase, arena: Allocator, base: [:0]co
}
const PreloadOpts = struct {
- // set when the preload comes from a <link rel=modulepreload> hint
+ // true when the preload is a hint (modulepreload link or prescan) and can
+ // be taken by anyone.
+ hint: bool = false,
+
+ // the <link rel=modulepreload> element to fire load/error on, when the
+ // hint came from one.
hint_element: ?*Element.Html = null,
};
pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []const u8, opts: PreloadOpts) !void {
@@ -248,7 +253,7 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co
.hint_element = opts.hint_element,
};
- gop.value_ptr.* = .{ .state = .{ .loading = script }, .hint = opts.hint_element != null };
+ gop.value_ptr.* = .{ .state = .{ .loading = script }, .hint = opts.hint };
if (comptime IS_DEBUG) {
var ls: js.Local.Scope = undefined;
@@ -294,17 +299,45 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co
};
}
-// <link rel=modulepreload href=...> — start fetching a module before import
-// resolution discovers it. Returns false when no fetch was started (the
-// module is already fetched/fetching): no event will fire on the link.
-pub fn preloadModuleHint(self: *ScriptManagerBase, element: *Element.Html, url: [:0]const u8, referrer: []const u8) !bool {
+// <link rel=modulepreload href=...> (element set) or the prescan finding a
+// <script type=module src=...> (element null) — start fetching a module
+// before import resolution discovers it. Returns false when no fetch was
+// started (the module is already fetched/fetching): no event will fire on
+// the link.
+pub fn preloadModuleHint(self: *ScriptManagerBase, element: ?*Element.Html, url: [:0]const u8, referrer: []const u8) !bool {
if (self.imported_modules.contains(url)) {
return false;
}
- try self.preloadImport(url, referrer, .{ .hint_element = element });
+ try self.preloadImport(url, referrer, .{ .hint = true, .hint_element = element });
return true;
}
+// A <script type=module src=...> whose URL was hinted (modulepreload link or
+// prescan)
+pub fn takeModuleHint(self: *ScriptManagerBase, url: [:0]const u8) ?*Script {
+ const entry = self.imported_modules.getEntry(url) orelse return null;
+ if (entry.value_ptr.hint == false) {
+ // The script was preloaded, but not because of a hint. It came from v8
+ // telling us to preload the module. We cannot take it here because we know
+ // v8 is going to come back and wait for it.
+ return null;
+ }
+
+ const script = switch (entry.value_ptr.state) {
+ .loading => |script| blk: {
+ self.async_scripts.remove(&script.node);
+ break :blk script;
+ },
+ .done => |script| script,
+ // The hint's fetch failed; give the script its own attempt.
+ // I'm not sure if this is the right behavior. Why would a preload fail
+ // but the "real" load work? But it's definetly safer.
+ .err => return null,
+ };
+ self.imported_modules.removeByPtr(entry.key_ptr);
+ return script;
+}
+
pub fn waitForImport(self: *ScriptManagerBase, url: [:0]const u8) !ModuleSource {
const was_evaluating = self.is_evaluating;
self.is_evaluating = true;
diff --git a/src/browser/frame/preload.zig b/src/browser/frame/preload.zig
new file mode 100644
index 000000000..08071f003
--- /dev/null
+++ b/src/browser/frame/preload.zig
@@ -0,0 +1,196 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// Speculative script fetching: <link rel=preload/modulepreload> hints and the
+// pre-parse scan of the raw HTML. All of it only starts downloads early —
+// consumption happens in ScriptManager / ScriptManagerBase when the real
+// <script> element or import shows up.
+
+const std = @import("std");
+
+const URL = @import("../URL.zig");
+const Frame = @import("../Frame.zig");
+const Parser = @import("../parser/Parser.zig");
+const Element = @import("../webapi/Element.zig");
+
+const Allocator = std.mem.Allocator;
+
+// start prefetching <link rel="preload" as="script" href=...>`. element is the
+// hint <link> to fire load/error on, null when the hint came from the prescan.
+pub fn scriptHint(frame: *Frame, element: ?*Element.Html, href: []const u8) bool {
+ if (frame.isGoingAway() or frame._parse_mode == .fragment) {
+ return false;
+ }
+
+ const arena = frame.getArena(.small, "preload.scriptHint") catch return false;
+ defer frame.releaseArena(arena);
+
+ const resolved = URL.resolve(arena, frame.base(), href, .{ .encoding = frame.charset }) catch return false;
+ if (!isRemoteScheme(resolved)) {
+ return false;
+ }
+ return frame._script_manager.preloadScript(element, resolved) catch false;
+}
+
+// start prefetching <link rel="modulepreload" href=...>. element is the hint
+// <link> to fire load/error on, null when the hint came from the prescan.
+pub fn moduleHint(frame: *Frame, element: ?*Element.Html, href: []const u8) bool {
+ if (frame.isGoingAway() or frame._parse_mode == .fragment) {
+ return false;
+ }
+
+ if (hasNonRemoteScheme(href)) {
+ return false;
+ }
+
+ // The url becomes the imported_modules key, which must outlive the fetch
+ // so it lives on the frame arena
+ const resolved = URL.resolve(frame.arena, frame.base(), href, .{ .encoding = frame.charset }) catch return false;
+ if (!isRemoteScheme(resolved)) {
+ return false;
+ }
+
+ return frame._script_manager.base.preloadModuleHint(element, resolved, frame.url) catch false;
+}
+
+// Scan the HTML for <script src=...> before parsing so that we can start
+// fetching them ASAP. The downside is we might accidentally fetch more than we
+// should, but the upside can be a pretty significant performance improvement.
+// Without this, N large blocking <script src=...> tags are downloaded serially.
+// This essentially does the same thing as the <link preload/preloadModule> but
+// without needing anything special from the HTML.
+pub fn prescan(frame: *Frame, html: []const u8) void {
+ if (frame.isGoingAway() or frame._parse_mode == .fragment) {
+ return;
+ }
+ const arena = frame.getArena(.small, "preload.prescan") catch return;
+ defer frame.releaseArena(arena);
+
+ var scan = Prescan{ .frame = frame, .base = frame.base(), .arena = arena };
+ Parser.prescan(html, frame.charset, &scan, Prescan.callback);
+}
+
+const Prescan = struct {
+ frame: *Frame,
+ arena: Allocator,
+ base: [:0]const u8,
+
+ fn callback(ctx: *anyopaque, kind: Parser.PrescanResource, url_ptr: [*c]const u8, url_len: usize) callconv(.c) void {
+ const self: *Prescan = @ptrCast(@alignCast(ctx));
+ if (url_len == 0) {
+ return;
+ }
+ const href = url_ptr[0..url_len];
+ const frame = self.frame;
+ switch (kind) {
+ .base => {
+ self.base = URL.resolve(self.arena, self.base, href, .{ .encoding = frame.charset }) catch return;
+ },
+ .script => {
+ if (hasNonRemoteScheme(href)) {
+ return;
+ }
+ const resolved = URL.resolve(self.arena, self.base, href, .{ .encoding = frame.charset }) catch return;
+ if (isRemoteScheme(resolved) == false) {
+ return;
+ }
+ _ = frame._script_manager.preloadScript(null, resolved) catch {};
+ },
+ .module => {
+ if (hasNonRemoteScheme(href)) {
+ return;
+ }
+ // The url becomes the imported_modules key, which must
+ // outlive the fetch so it lives on the frame arena.
+ const resolved = URL.resolve(frame.arena, self.base, href, .{ .encoding = frame.charset }) catch return;
+ if (isRemoteScheme(resolved) == false) {
+ return;
+ }
+ _ = frame._script_manager.base.preloadModuleHint(null, resolved, frame.url) catch {};
+ },
+ _ => {},
+ }
+ }
+};
+
+fn isRemoteScheme(url: []const u8) bool {
+ return std.ascii.startsWithIgnoreCase(url, "http:") or std.ascii.startsWithIgnoreCase(url, "https:");
+}
+
+// Non-http(s) scheme (e.g. data:, blob:) never resolve to a remote URL. We
+// detect this upfront to prevent uncessary URL.resolves that would dupe the
+// (potentially very lage) data.
+fn hasNonRemoteScheme(href: []const u8) bool {
+ if (isRemoteScheme(href)) {
+ return false;
+ }
+ if (href.len == 0 or !std.ascii.isAlphabetic(href[0])) {
+ return false;
+ }
+ for (href[1..]) |c| {
+ switch (c) {
+ ':' => return true,
+ 'a'...'z', 'A'...'Z', '0'...'9', '+', '-', '.' => {},
+ else => return false,
+ }
+ }
+ return false;
+}
+
+const testing = @import("../../testing.zig");
+
+test "preload: prescan" {
+ defer testing.reset();
+
+ const page = try testing.pageTest("mcp_nav.html", .{});
+ defer page.close();
+ const frame = page.frame().?;
+
+ // The fetches this starts stay in flight (nothing ticks the client before
+ // page.close aborts them), so the maps hold exactly what the scan found.
+ // All srcs sit under /serve-count/ because the test server panics on
+ // unknown paths elsewhere; unknown names there get a plain 404.
+ prescan(frame,
+ \\<html><head>
+ \\<script src="/serve-count/unit_a.js"></script>
+ \\<script src="/serve-count/unit_a.js"></script>
+ \\<script type="module" src="/serve-count/unit_m.js"></script>
+ \\<script src="/serve-count/unit_skipped.js" nomodule></script>
+ \\<script type="application/json" src="/serve-count/unit_data.js"></script>
+ \\<script src="data:text/javascript,window.unit_data_url = 1;"></script>
+ \\<script type="module" src="data:text/javascript,window.unit_data_module = 1;"></script>
+ \\<script>var trap = "</scr" + "ipt><script src=/serve-count/unit_inline.js>";</script>
+ \\<!-- <script src="/serve-count/unit_commented.js"></script> -->
+ \\<template><script src="/serve-count/unit_templated.js"></script></template>
+ \\<noscript><script src="/serve-count/unit_noscripted.js"></script></noscript>
+ \\<style>p:before { content: "<script src=/serve-count/unit_styled.js></script>" }</style>
+ \\<base href="/serve-count/sub/">
+ \\<base href="/serve-count/ignored/">
+ \\<script src="based.js"></script>
+ \\</head><body></body></html>
+ );
+
+ const sm = &frame._script_manager;
+ try testing.expectEqual(2, sm.preloaded_scripts.count());
+ try testing.expectEqual(true, sm.preloaded_scripts.contains("http://127.0.0.1:9582/serve-count/unit_a.js"));
+ try testing.expectEqual(true, sm.preloaded_scripts.contains("http://127.0.0.1:9582/serve-count/sub/based.js"));
+
+ try testing.expectEqual(1, sm.base.imported_modules.count());
+ const module = sm.base.imported_modules.get("http://127.0.0.1:9582/serve-count/unit_m.js") orelse return error.MissingModule;
+ try testing.expectEqual(true, module.hint);
+}
diff --git a/src/browser/parser/Parser.zig b/src/browser/parser/Parser.zig
index 159dbe372..d252aa9db 100644
--- a/src/browser/parser/Parser.zig
+++ b/src/browser/parser/Parser.zig
@@ -171,6 +171,16 @@ const Error = struct {
};
};
+pub const PrescanResource = h5e.PrescanResource;
+pub const PrescanCallback = h5e.PrescanCallback;
+
+// Preload scanner: a tokenizer-only pass over a buffered document, reporting
+// fetchable script resources (and the first <base href>) through `callback`.
+// Builds no tree; purely a hint source.
+pub fn prescan(html: []const u8, charset: []const u8, ctx: *anyopaque, callback: PrescanCallback) void {
+ h5e.html5ever_prescan(html.ptr, html.len, charset.ptr, charset.len, ctx, callback);
+}
+
pub fn parse(self: *Parser, html: []const u8) void {
h5e.html5ever_parse_document(
html.ptr,
diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zig
index 3c5905302..b6d1c7ff3 100644
--- a/src/browser/parser/html5ever.zig
+++ b/src/browser/parser/html5ever.zig
@@ -315,3 +315,23 @@ pub extern "c" fn encoding_max_encode_buffer_length(
handle: *anyopaque,
input_len: usize,
) usize;
+
+// Preload scanner (see prescan.rs). Kinds must stay in sync with the
+// KIND_* constants there.
+pub const PrescanResource = enum(u32) {
+ script = 0,
+ module = 1,
+ base = 2,
+ _,
+};
+
+pub const PrescanCallback = *const fn (ctx: *anyopaque, kind: PrescanResource, url: [*c]const u8, url_len: usize) callconv(.c) void;
+
+pub extern "c" fn html5ever_prescan(
+ html: [*c]const u8,
+ len: usize,
+ charset: [*c]const u8,
+ charset_len: usize,
+ ctx: *anyopaque,
+ callback: PrescanCallback,
+) void;
diff --git a/src/browser/tests/element/html/script/prescan.html b/src/browser/tests/element/html/script/prescan.html
new file mode 100644
index 000000000..61989552e
--- /dev/null
+++ b/src/browser/tests/element/html/script/prescan.html
@@ -0,0 +1,38 @@
+<!DOCTYPE html>
+<script src="../../../testing.js"></script>
+
+<!--
+ No <link> hints anywhere here: the preload scanner (Frame.prescanScripts)
+ runs over the raw buffered HTML before parsing and starts every script
+ download up front. Each consumer below must then reuse that fetch rather
+ than issue its own.
+
+ /serve-count/ bodies record how many times the URL was served (see
+ testing.zig); the body a consumer executes carries 1 when the prescan's
+ fetch was reused, 2 when a duplicate fetch happened.
+-->
+
+<!-- Blocking: waitForPreload consumes the prescan's in-flight fetch. -->
+<script src="/serve-count/prescan_blocking.js"></script>
+
+<!-- Defer: takePreload adopts the prescan's Script. -->
+<script defer src="/serve-count/prescan_defer.js"></script>
+
+<!--
+ Module: the prescan parks module hints in imported_modules; the script
+ element's own fetch adopts it via takeModuleHint.
+-->
+<script type="module" src="/serve-count/prescan_module.js"></script>
+
+<script id="order">
+ // The blocking script above executed in document order, before us.
+ testing.expectEqual(1, window.__serve_count_prescan_blocking);
+</script>
+
+<script id="checks">
+ testing.onload(() => {
+ testing.expectEqual(1, window.__serve_count_prescan_blocking);
+ testing.expectEqual(1, window.__serve_count_prescan_defer);
+ testing.expectEqual(1, window.__serve_count_prescan_module);
+ });
+</script>
diff --git a/src/browser/webapi/element/html/Link.zig b/src/browser/webapi/element/html/Link.zig
index 7eb2190f5..5c766c9db 100644
--- a/src/browser/webapi/element/html/Link.zig
+++ b/src/browser/webapi/element/html/Link.zig
@@ -211,7 +211,7 @@ pub fn linkAddedCallback(self: *Link, frame: *Frame) !void {
if (std.mem.eql(u8, rel, "preload")) {
const as = element.getAttributeSafe(comptime .wrap("as")) orelse "";
if (std.ascii.eqlIgnoreCase(as, "script")) {
- if (frame.preloadScriptHint(self._proto, href)) {
+ if (Frame.preload.scriptHint(frame, self._proto, href)) {
// load/error fires when the fetch settles
return;
}
@@ -224,7 +224,7 @@ pub fn linkAddedCallback(self: *Link, frame: *Frame) !void {
// "as" defaults to script in this case
const as = element.getAttributeSafe(comptime .wrap("as")) orelse "";
if (as.len == 0 or std.ascii.eqlIgnoreCase(as, "script")) {
- if (frame.preloadModuleHint(self._proto, href)) {
+ if (Frame.preload.moduleHint(frame, self._proto, href)) {
// load/error fires when the fetch settles
return;
}
diff --git a/src/html5ever/lib.rs b/src/html5ever/lib.rs
index f778dbeb7..8504da071 100644
--- a/src/html5ever/lib.rs
+++ b/src/html5ever/lib.rs
@@ -16,6 +16,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+mod prescan;
mod sink;
mod types;
mod url;
diff --git a/src/html5ever/prescan.rs b/src/html5ever/prescan.rs
new file mode 100644
index 000000000..e5702f22f
--- /dev/null
+++ b/src/html5ever/prescan.rs
@@ -0,0 +1,202 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// A preload scanner: a lightweight tokenizer-only pass over a document that
+// reports fetchable script resources, so downloads can start before the tree
+// builder (which stalls on every blocking <script src>) reaches them. Purely
+// a hint source — it builds no tree and a wrong guess only costs a fetch.
+// Same idea as Servo's dom/servoparser/prefetch.rs.
+
+use std::cell::Cell;
+use std::os::raw::{c_uchar, c_void};
+
+use encoding_rs::Encoding;
+use html5ever::tendril::StrTendril;
+use html5ever::tokenizer::states::RawKind;
+use html5ever::tokenizer::{
+ BufferQueue, Tag, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts,
+};
+use html5ever::{local_name, ns, Attribute, LocalName};
+
+// Keep in sync with PrescanResource in src/browser/parser/html5ever.zig.
+const KIND_SCRIPT: u32 = 0;
+const KIND_MODULE: u32 = 1;
+const KIND_BASE: u32 = 2;
+
+pub type PrescanCallback =
+ extern "C" fn(ctx: *mut c_void, kind: u32, url: *const c_uchar, url_len: usize);
+
+struct PrescanSink {
+ ctx: *mut c_void,
+ callback: PrescanCallback,
+ // Only the first <base href> counts, per spec.
+ base_seen: Cell<bool>,
+ // Scripts inside <template> are inert until cloned; don't fetch them.
+ template_depth: Cell<u32>,
+}
+
+impl PrescanSink {
+ fn attr<'a>(tag: &'a Tag, name: &LocalName) -> Option<&'a Attribute> {
+ tag.attrs
+ .iter()
+ .find(|a| a.name.ns == ns!() && a.name.local == *name)
+ }
+
+ fn emit(&self, kind: u32, url: &str) {
+ (self.callback)(self.ctx, kind, url.as_ptr(), url.len());
+ }
+
+ fn scan_script(&self, tag: &Tag) {
+ // Mirrors ScriptManager.addFromElement: nomodule scripts are skipped,
+ // and only classic javascript and module types are fetched.
+ if Self::attr(tag, &local_name!("nomodule")).is_some() {
+ return;
+ }
+ let Some(src) = Self::attr(tag, &local_name!("src")) else {
+ return;
+ };
+ if src.value.is_empty() {
+ return;
+ }
+ let kind = match Self::attr(tag, &local_name!("type")) {
+ None => KIND_SCRIPT,
+ Some(attr) => {
+ let t: &str = &attr.value;
+ if t.is_empty()
+ || t.eq_ignore_ascii_case("application/javascript")
+ || t.eq_ignore_ascii_case("text/javascript")
+ {
+ KIND_SCRIPT
+ } else if t.eq_ignore_ascii_case("module") {
+ KIND_MODULE
+ } else {
+ return;
+ }
+ }
+ };
+ self.emit(kind, &src.value);
+ }
+}
+
+impl TokenSink for PrescanSink {
+ type Handle = ();
+
+ fn process_token(&self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
+ let Token::TagToken(tag) = token else {
+ return TokenSinkResult::Continue;
+ };
+
+ if tag.kind == TagKind::EndTag {
+ if tag.name == local_name!("template") {
+ let depth = self.template_depth.get();
+ self.template_depth.set(depth.saturating_sub(1));
+ }
+ return TokenSinkResult::Continue;
+ }
+
+ if tag.name == local_name!("script") {
+ if self.template_depth.get() == 0 {
+ self.scan_script(&tag);
+ }
+ // The tokenizer has no tree builder giving it state feedback, so
+ // the sink must request the script-data state itself or the
+ // script body would be tokenized as markup.
+ return TokenSinkResult::RawData(RawKind::ScriptData);
+ }
+ if tag.name == local_name!("base") {
+ if !self.base_seen.get() {
+ if let Some(href) = Self::attr(&tag, &local_name!("href")) {
+ self.base_seen.set(true);
+ self.emit(KIND_BASE, &href.value);
+ }
+ }
+ return TokenSinkResult::Continue;
+ }
+ if tag.name == local_name!("template") {
+ self.template_depth.set(self.template_depth.get() + 1);
+ return TokenSinkResult::Continue;
+ }
+ // The remaining raw-text/rcdata elements: their content must not be
+ // tokenized as markup, or text mentioning "<script src=…>" (a style
+ // rule, a <noscript> fallback, an <iframe> srcdoc-ish body) would
+ // produce phantom fetches. noscript is raw text because scripting is
+ // enabled in this browser.
+ if tag.name == local_name!("style")
+ || tag.name == local_name!("noscript")
+ || tag.name == local_name!("iframe")
+ || tag.name == local_name!("xmp")
+ || tag.name == local_name!("noembed")
+ || tag.name == local_name!("noframes")
+ {
+ return TokenSinkResult::RawData(RawKind::Rawtext);
+ }
+ if tag.name == local_name!("title") || tag.name == local_name!("textarea") {
+ return TokenSinkResult::RawData(RawKind::Rcdata);
+ }
+ if tag.name == local_name!("plaintext") {
+ return TokenSinkResult::Plaintext;
+ }
+ TokenSinkResult::Continue
+ }
+}
+
+/// Tokenize the (fully buffered) document and report each fetchable script
+/// resource through `callback`: raw (unresolved) src values for classic and
+/// module scripts, plus the first <base href> so the caller can resolve
+/// subsequent URLs the way the real parse will.
+#[no_mangle]
+pub extern "C" fn html5ever_prescan(
+ html: *const c_uchar,
+ len: usize,
+ charset: *const c_uchar,
+ charset_len: usize,
+ ctx: *mut c_void,
+ callback: PrescanCallback,
+) {
+ if html.is_null() || len == 0 {
+ return;
+ }
+ let input = unsafe { std::slice::from_raw_parts(html, len) };
+ let charset_bytes: &[u8] = if charset.is_null() {
+ &[]
+ } else {
+ unsafe { std::slice::from_raw_parts(charset, charset_len) }
+ };
+
+ // No allocation when the content is already valid UTF-8.
+ let encoding = Encoding::for_label(charset_bytes).unwrap_or(encoding_rs::UTF_8);
+ let (decoded, _, _) = encoding.decode(input);
+
+ // A panic must not unwind across the FFI boundary; hints are best-effort,
+ // so just stop scanning.
+ let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ let sink = PrescanSink {
+ ctx,
+ callback,
+ base_seen: Cell::new(false),
+ template_depth: Cell::new(0),
+ };
+ let tokenizer = Tokenizer::new(sink, TokenizerOpts::default());
+ let queue = BufferQueue::default();
+ queue.push_back(StrTendril::from(decoded.as_ref()));
+ // The sink never returns TokenSinkResult::Script, so one feed
+ // consumes the whole buffer.
+ let _ = tokenizer.feed(&queue);
+ tokenizer.end();
+ }));
+}
diff --git a/src/testing.zig b/src/testing.zig
index 05f7b8150..21b3afa6f 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -606,9 +606,14 @@ fn serveCDP(wg: *std.Thread.WaitGroup) !void {
}
// /serve-count/ counters; only ever touched from the test HTTP server thread.
-var serve_count_defer: u32 = 0;
-var serve_count_async: u32 = 0;
-var serve_count_dynamic: u32 = 0;
+var serve_counts = [_]struct { name: []const u8, count: u32 = 0 }{
+ .{ .name = "defer" },
+ .{ .name = "async" },
+ .{ .name = "dynamic" },
+ .{ .name = "prescan_blocking" },
+ .{ .name = "prescan_defer" },
+ .{ .name = "prescan_module" },
+};
fn testHTTPHandler(req: *std.http.Server.Request) !void {
const path = req.head.target;
@@ -745,9 +750,9 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
// no-store so the second fetch can't be satisfied by the HTTP cache.
const name = path["/serve-count/".len .. path.len - ".js".len];
const slot: *u32 = blk: {
- if (std.mem.eql(u8, name, "defer")) break :blk &serve_count_defer;
- if (std.mem.eql(u8, name, "async")) break :blk &serve_count_async;
- if (std.mem.eql(u8, name, "dynamic")) break :blk &serve_count_dynamic;
+ for (&serve_counts) |*sc| {
+ if (std.mem.eql(u8, sc.name, name)) break :blk &sc.count;
+ }
return req.respond("unknown counter", .{ .status = .not_found });
};
slot.* += 1;