Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor: extract HTML parser input stack
ldm0a142d011c0eb
diff --git a/moli-parser/src/html_input.rs b/moli-parser/src/html_input.rsnew file mode 100644index 000000000..684324874--- /dev/null+++ b/moli-parser/src/html_input.rs@@ -0,0 +1,203 @@+//! Tokenizer-owned HTML input and parser insertion frames.+//!+//! `html5ever` consumes one [`BufferQueue`] at a time, but `document.write()`+//! can insert a new source in front of input that the tokenizer has not+//! consumed yet. That unconsumed tail must not be discarded or appended to the+//! inserted source. It becomes the parent frame and is restored after the+//! inserted frame is empty:+//!+//! ```text+//! before insertion: [ current tail ]+//! during insertion: [ inserted input ] [ saved current tail ]+//! nested insertion: [ inner input ] [ outer input ] [ original tail ]+//! ```+//!+//! This module owns only input that has already been admitted to the tokenizer.+//! It does not decide whether an arriving chunk is parser-inserted or belongs+//! at the end of the document stream, and it does not decide whether an empty+//! parser should finish or wait for more input. Those decisions require runtime+//! provenance and parser lifetime, respectively, and remain with their owners.+//! The frames are explicit rather than represented by a stack-scoped guard+//! because parser execution can yield for an external script and resume on a+//! later runtime turn.++use html5ever::{tendril::StrTendril, tokenizer::BufferQueue};++/// The tokenizer's current input plus suspended parent insertion frames.+///+/// `parents` is stored from outermost to innermost. Consequently, pending input+/// is consumed from `current`, followed by `parents` in reverse order.+#[derive(Default)]+pub(super) struct InputStack {+ current: BufferQueue,+ parents: Vec<BufferQueue>,+}++impl InputStack {+ /// Append ordinary input to the tokenizer's current queue.+ pub(super) fn push_back(&mut self, input: StrTendril) {+ self.current.push_back(input);+ }++ /// Start a parser-inserted frame before the current unconsumed input.+ ///+ /// Even an empty current queue is saved. Its presence records that there is+ /// an active insertion point to which a later write may belong.+ pub(super) fn begin_inserted(&mut self, input: StrTendril) {+ if input.is_empty() {+ return;+ }+ let parent = std::mem::take(&mut self.current);+ self.parents.push(parent);+ self.current.push_back(input);+ }++ /// Append after the unconsumed tail of the active inserted frame.+ ///+ /// This is different from starting another inserted frame: a nested frame+ /// would run before the current tail, while this input must run after it.+ /// Empty input is accepted as a no-op; non-empty input returns `false` when+ /// no parser insertion frame is active.+ pub(super) fn append_to_current_inserted(&mut self, input: StrTendril) -> bool {+ if input.is_empty() {+ return true;+ }+ if self.parents.is_empty() {+ return false;+ }+ self.current.push_back(input);+ true+ }++ /// Return the queue that `html5ever` may consume in the current parser step.+ pub(super) fn current(&self) -> &BufferQueue {+ &self.current+ }++ /// Restore one parent after the current frame has been fully consumed.+ ///+ /// At most one frame is restored per tokenizer boundary. The caller feeds+ /// the restored frame on its next step, preserving the same incremental+ /// parser behavior for every insertion depth.+ pub(super) fn restore_parent_if_current_empty(&mut self) -> bool {+ if !self.current.is_empty() {+ return false;+ }+ let Some(parent) = self.parents.pop() else {+ return false;+ };+ self.current = parent;+ true+ }++ /// Return whether any frame still contains characters to tokenize.+ ///+ /// This deliberately says nothing about whether an insertion point exists:+ /// an active frame and all of its parents may legitimately be empty.+ pub(super) fn has_input(&self) -> bool {+ self.queues_in_consumption_order()+ .any(|input| !input.is_empty())+ }++ pub(super) fn len(&self) -> usize {+ self.queues_in_consumption_order()+ .fold(0usize, |total, input| {+ total.saturating_add(queue_len(input))+ })+ }++ /// Snapshot pending characters in the order the tokenizer will see them.+ pub(super) fn snapshot(&self) -> String {+ let mut pending = String::new();+ for input in self.queues_in_consumption_order() {+ append_queue_snapshot(input, &mut pending);+ }+ pending+ }++ /// Collapse every insertion frame into one queue for parser finalization.+ ///+ /// The innermost current frame stays first, followed by each parent from+ /// nearest to farthest. No input is reparsed or otherwise transformed.+ pub(super) fn into_buffer(mut self) -> BufferQueue {+ while let Some(parent) = self.parents.pop() {+ append_queue(parent, &self.current);+ }+ self.current+ }++ fn queues_in_consumption_order(&self) -> impl Iterator<Item = &BufferQueue> {+ std::iter::once(&self.current).chain(self.parents.iter().rev())+ }+}++fn queue_len(input: &BufferQueue) -> usize {+ let input = input.clone();+ let mut len = 0usize;+ while let Some(chunk) = input.pop_front() {+ len = len.saturating_add(chunk.len());+ }+ len+}++fn append_queue_snapshot(input: &BufferQueue, snapshot: &mut String) {+ let input = input.clone();+ while let Some(chunk) = input.pop_front() {+ snapshot.push_str(&chunk);+ }+}++fn append_queue(source: BufferQueue, destination: &BufferQueue) {+ while let Some(chunk) = source.pop_front() {+ destination.push_back(chunk);+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn tendril(input: &str) -> StrTendril {+ StrTendril::from(input)+ }++ #[test]+ fn nested_frames_are_snapshotted_and_flattened_in_consumption_order() {+ let mut input = InputStack::default();+ input.push_back(tendril("e")); // Original parser tail.+ input.begin_inserted(tendril("r")); // Tail behind a nested external script.+ assert!(input.append_to_current_inserted(tendril("k"))); // Same inserted frame.+ input.begin_inserted(tendril("wo")); // Writes made by that external script.++ assert_eq!(input.snapshot(), "worke");+ assert_eq!(input.len(), 5);++ let flattened = input.into_buffer();+ let mut snapshot = String::new();+ append_queue_snapshot(&flattened, &mut snapshot);+ assert_eq!(snapshot, "worke");+ }++ #[test]+ fn an_empty_boundary_restores_exactly_one_parent() {+ let mut input = InputStack::default();+ input.push_back(tendril("outer"));+ input.begin_inserted(tendril("middle"));+ input.begin_inserted(tendril("inner"));++ while input.current.pop_front().is_some() {}+ assert!(input.restore_parent_if_current_empty());+ assert_eq!(input.snapshot(), "middleouter");+ assert!(!input.restore_parent_if_current_empty());+ }++ #[test]+ fn appending_to_an_inserted_frame_requires_an_active_insertion_point() {+ let mut input = InputStack::default();+ assert!(!input.append_to_current_inserted(tendril("outside")));++ input.begin_inserted(tendril("inside"));+ assert!(input.append_to_current_inserted(tendril(" tail")));+ assert_eq!(input.snapshot(), "inside tail");+ }+}diff --git a/moli-parser/src/lib.rs b/moli-parser/src/lib.rsindex 5d8b8795c..16668096e 100644--- a/moli-parser/src/lib.rs+++ b/moli-parser/src/lib.rs@@ -4,6 +4,7 @@//! discovery outputs without depending on the renderer implementation.mod html;+mod html_input;mod live_target;mod script_planning;mod session;diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rsindex 9deec4060..7e15581e0 100644--- a/moli-parser/src/session.rs+++ b/moli-parser/src/session.rs@@ -13,6 +13,7 @@ use moli_dom::native::NativeNodeId;use super::{html::{DocumentSink, ParseHandle, ParserFinishDiscoverySignals, ParserInputQueue},+ html_input::InputStack,live_target::ParserStreamHtmlTreeSinkTarget,};@@ -23,8 +24,7 @@ pub(super) struct HtmlTreeSinkSession {pub(super) struct HtmlParserSession {tokenizer: Tokenizer<EmbedderPausingTreeBuilder>,- input_buffer: BufferQueue,- suspended_input_buffers: Vec<BufferQueue>,+ input: InputStack,}pub(super) enum HtmlParserSessionResult {@@ -137,8 +137,7 @@ impl HtmlParserSession {let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);Self {tokenizer: Tokenizer::new(tree_builder, opts.tokenizer),- input_buffer: BufferQueue::default(),- suspended_input_buffers: Vec::new(),+ input: InputStack::default(),}}@@ -160,15 +159,14 @@ impl HtmlParserSession {};Self {tokenizer: Tokenizer::new(tree_builder, tokenizer_options),- input_buffer: BufferQueue::default(),- suspended_input_buffers: Vec::new(),+ input: InputStack::default(),}}pub(super) fn process(&mut self, input: StrTendril) {- self.input_buffer.push_back(input);+ self.input.push_back(input);while let HtmlParserSessionResult::Script(_) =- feed_with_definitive_encoding(&self.tokenizer, &self.input_buffer)+ feed_with_definitive_encoding(&self.tokenizer, self.input.current()){// Non-pump callers intentionally parse through embedder pauses. They have no// runtime owner to notify, so parser-side custom-element handoffs and@@ -178,7 +176,7 @@ impl HtmlParserSession {}pub(super) fn push_back(&mut self, input: StrTendril) {- self.input_buffer.push_back(input);+ self.input.push_back(input);}pub(super) fn begin_inserted_input(&mut self, input: StrTendril) {@@ -190,63 +188,31 @@ impl HtmlParserSession {// restored. Prefer permanent unknown locations over reporting// plausible but incorrect document lines.self.tokenizer.sink.sink().mark_source_positions_unknown();- let parent = std::mem::take(&mut self.input_buffer);- self.suspended_input_buffers.push(parent);- self.input_buffer.push_back(input);+ self.input.begin_inserted(input);}pub(super) fn append_to_current_inserted_input(&mut self, input: StrTendril) -> bool {- if input.is_empty() {- return true;- }- if self.suspended_input_buffers.is_empty() {- return false;- }- self.input_buffer.push_back(input);- true+ self.input.append_to_current_inserted(input)}pub(super) fn has_buffered_input(&self) -> bool {- !self.input_buffer.is_empty()- || self- .suspended_input_buffers- .iter()- .any(|input| !input.is_empty())+ self.input.has_input()}pub(super) fn buffered_input_len(&self) -> usize {- let mut len = 0usize;- for input in- std::iter::once(&self.input_buffer).chain(self.suspended_input_buffers.iter().rev())- {- let input = input.clone();- while let Some(chunk) = input.pop_front() {- len = len.saturating_add(chunk.len());- }- }- len+ self.input.len()}pub(super) fn snapshot_buffered_input(&self) -> String {- let mut buffered = String::new();- for input in- std::iter::once(&self.input_buffer).chain(self.suspended_input_buffers.iter().rev())- {- let input = input.clone();- while let Some(chunk) = input.pop_front() {- buffered.push_str(&chunk);- }- }- buffered+ self.input.snapshot()}pub(super) fn feed(&mut self) -> HtmlParserSessionResult {- let result = feed_with_definitive_encoding(&self.tokenizer, &self.input_buffer);- if matches!(result, HtmlParserSessionResult::InputDrained)- && self.input_buffer.is_empty()- && let Some(parent) = self.suspended_input_buffers.pop()- {- self.input_buffer = parent;+ let result = feed_with_definitive_encoding(&self.tokenizer, self.input.current());+ if matches!(result, HtmlParserSessionResult::InputDrained) {+ // The restored parent is intentionally consumed by the next parser+ // step so each insertion depth keeps an explicit input boundary.+ self.input.restore_parent_if_current_empty();}result}@@ -256,12 +222,8 @@ impl HtmlParserSession {}pub(super) fn finish(self) -> ParserStreamHtmlTreeSinkTarget {- let Self {- tokenizer,- mut input_buffer,- mut suspended_input_buffers,- } = self;- restore_all_suspended_input(&mut input_buffer, &mut suspended_input_buffers);+ let Self { tokenizer, input } = self;+ let input_buffer = input.into_buffer();while let HtmlParserSessionResult::Script(_) =feed_with_definitive_encoding(&tokenizer, &input_buffer){@@ -281,12 +243,8 @@ impl HtmlParserSession {}pub(super) fn finish_live_runtime_dom_sink_parser(self) -> ParserFinishDiscoverySignals {- let Self {- tokenizer,- mut input_buffer,- mut suspended_input_buffers,- } = self;- restore_all_suspended_input(&mut input_buffer, &mut suspended_input_buffers);+ let Self { tokenizer, input } = self;+ let input_buffer = input.into_buffer();while let HtmlParserSessionResult::Script(_) =feed_with_definitive_encoding(&tokenizer, &input_buffer){@@ -346,17 +304,6 @@ fn feed_with_definitive_encoding(}}-fn restore_all_suspended_input(- input_buffer: &mut BufferQueue,- suspended_input_buffers: &mut Vec<BufferQueue>,-) {- while let Some(parent) = suspended_input_buffers.pop() {- while let Some(chunk) = parent.pop_front() {- input_buffer.push_back(chunk);- }- }-}-pub(super) fn new_html_tree_sink_session(target: ParserStreamHtmlTreeSinkTarget,) -> HtmlTreeSinkSession {