Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

refactor: extract HTML parser input stack

ldm0a142d011c0eb
diff --git a/moli-parser/src/html_input.rs b/moli-parser/src/html_input.rs
new file mode 100644
index 000000000..684324874
--- /dev/null
+++ b/moli-parser/src/html_input.rs
@@ -0,0 +1,203 @@
+//! Tokenizer-owned HTML input and parser insertion frames.
+//!
+//! `html5ever` consumes one [`BufferQueue`] at a time, but `document.write()`
+//! can insert a new source in front of input that the tokenizer has not
+//! consumed yet. That unconsumed tail must not be discarded or appended to the
+//! inserted source. It becomes the parent frame and is restored after the
+//! inserted frame is empty:
+//!
+//! ```text
+//! before insertion: [ current tail ]
+//! during insertion: [ inserted input ] [ saved current tail ]
+//! nested insertion: [ inner input ] [ outer input ] [ original tail ]
+//! ```
+//!
+//! This module owns only input that has already been admitted to the tokenizer.
+//! It does not decide whether an arriving chunk is parser-inserted or belongs
+//! at the end of the document stream, and it does not decide whether an empty
+//! parser should finish or wait for more input. Those decisions require runtime
+//! provenance and parser lifetime, respectively, and remain with their owners.
+//! The frames are explicit rather than represented by a stack-scoped guard
+//! because parser execution can yield for an external script and resume on a
+//! later runtime turn.
+
+use html5ever::{tendril::StrTendril, tokenizer::BufferQueue};
+
+/// The tokenizer's current input plus suspended parent insertion frames.
+///
+/// `parents` is stored from outermost to innermost. Consequently, pending input
+/// is consumed from `current`, followed by `parents` in reverse order.
+#[derive(Default)]
+pub(super) struct InputStack {
+ current: BufferQueue,
+ parents: Vec<BufferQueue>,
+}
+
+impl InputStack {
+ /// Append ordinary input to the tokenizer's current queue.
+ pub(super) fn push_back(&mut self, input: StrTendril) {
+ self.current.push_back(input);
+ }
+
+ /// Start a parser-inserted frame before the current unconsumed input.
+ ///
+ /// Even an empty current queue is saved. Its presence records that there is
+ /// an active insertion point to which a later write may belong.
+ pub(super) fn begin_inserted(&mut self, input: StrTendril) {
+ if input.is_empty() {
+ return;
+ }
+ let parent = std::mem::take(&mut self.current);
+ self.parents.push(parent);
+ self.current.push_back(input);
+ }
+
+ /// Append after the unconsumed tail of the active inserted frame.
+ ///
+ /// This is different from starting another inserted frame: a nested frame
+ /// would run before the current tail, while this input must run after it.
+ /// Empty input is accepted as a no-op; non-empty input returns `false` when
+ /// no parser insertion frame is active.
+ pub(super) fn append_to_current_inserted(&mut self, input: StrTendril) -> bool {
+ if input.is_empty() {
+ return true;
+ }
+ if self.parents.is_empty() {
+ return false;
+ }
+ self.current.push_back(input);
+ true
+ }
+
+ /// Return the queue that `html5ever` may consume in the current parser step.
+ pub(super) fn current(&self) -> &BufferQueue {
+ &self.current
+ }
+
+ /// Restore one parent after the current frame has been fully consumed.
+ ///
+ /// At most one frame is restored per tokenizer boundary. The caller feeds
+ /// the restored frame on its next step, preserving the same incremental
+ /// parser behavior for every insertion depth.
+ pub(super) fn restore_parent_if_current_empty(&mut self) -> bool {
+ if !self.current.is_empty() {
+ return false;
+ }
+ let Some(parent) = self.parents.pop() else {
+ return false;
+ };
+ self.current = parent;
+ true
+ }
+
+ /// Return whether any frame still contains characters to tokenize.
+ ///
+ /// This deliberately says nothing about whether an insertion point exists:
+ /// an active frame and all of its parents may legitimately be empty.
+ pub(super) fn has_input(&self) -> bool {
+ self.queues_in_consumption_order()
+ .any(|input| !input.is_empty())
+ }
+
+ pub(super) fn len(&self) -> usize {
+ self.queues_in_consumption_order()
+ .fold(0usize, |total, input| {
+ total.saturating_add(queue_len(input))
+ })
+ }
+
+ /// Snapshot pending characters in the order the tokenizer will see them.
+ pub(super) fn snapshot(&self) -> String {
+ let mut pending = String::new();
+ for input in self.queues_in_consumption_order() {
+ append_queue_snapshot(input, &mut pending);
+ }
+ pending
+ }
+
+ /// Collapse every insertion frame into one queue for parser finalization.
+ ///
+ /// The innermost current frame stays first, followed by each parent from
+ /// nearest to farthest. No input is reparsed or otherwise transformed.
+ pub(super) fn into_buffer(mut self) -> BufferQueue {
+ while let Some(parent) = self.parents.pop() {
+ append_queue(parent, &self.current);
+ }
+ self.current
+ }
+
+ fn queues_in_consumption_order(&self) -> impl Iterator<Item = &BufferQueue> {
+ std::iter::once(&self.current).chain(self.parents.iter().rev())
+ }
+}
+
+fn queue_len(input: &BufferQueue) -> usize {
+ let input = input.clone();
+ let mut len = 0usize;
+ while let Some(chunk) = input.pop_front() {
+ len = len.saturating_add(chunk.len());
+ }
+ len
+}
+
+fn append_queue_snapshot(input: &BufferQueue, snapshot: &mut String) {
+ let input = input.clone();
+ while let Some(chunk) = input.pop_front() {
+ snapshot.push_str(&chunk);
+ }
+}
+
+fn append_queue(source: BufferQueue, destination: &BufferQueue) {
+ while let Some(chunk) = source.pop_front() {
+ destination.push_back(chunk);
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn tendril(input: &str) -> StrTendril {
+ StrTendril::from(input)
+ }
+
+ #[test]
+ fn nested_frames_are_snapshotted_and_flattened_in_consumption_order() {
+ let mut input = InputStack::default();
+ input.push_back(tendril("e")); // Original parser tail.
+ input.begin_inserted(tendril("r")); // Tail behind a nested external script.
+ assert!(input.append_to_current_inserted(tendril("k"))); // Same inserted frame.
+ input.begin_inserted(tendril("wo")); // Writes made by that external script.
+
+ assert_eq!(input.snapshot(), "worke");
+ assert_eq!(input.len(), 5);
+
+ let flattened = input.into_buffer();
+ let mut snapshot = String::new();
+ append_queue_snapshot(&flattened, &mut snapshot);
+ assert_eq!(snapshot, "worke");
+ }
+
+ #[test]
+ fn an_empty_boundary_restores_exactly_one_parent() {
+ let mut input = InputStack::default();
+ input.push_back(tendril("outer"));
+ input.begin_inserted(tendril("middle"));
+ input.begin_inserted(tendril("inner"));
+
+ while input.current.pop_front().is_some() {}
+ assert!(input.restore_parent_if_current_empty());
+ assert_eq!(input.snapshot(), "middleouter");
+ assert!(!input.restore_parent_if_current_empty());
+ }
+
+ #[test]
+ fn appending_to_an_inserted_frame_requires_an_active_insertion_point() {
+ let mut input = InputStack::default();
+ assert!(!input.append_to_current_inserted(tendril("outside")));
+
+ input.begin_inserted(tendril("inside"));
+ assert!(input.append_to_current_inserted(tendril(" tail")));
+ assert_eq!(input.snapshot(), "inside tail");
+ }
+}
diff --git a/moli-parser/src/lib.rs b/moli-parser/src/lib.rs
index 5d8b8795c..16668096e 100644
--- a/moli-parser/src/lib.rs
+++ b/moli-parser/src/lib.rs
@@ -4,6 +4,7 @@
//! discovery outputs without depending on the renderer implementation.
mod html;
+mod html_input;
mod live_target;
mod script_planning;
mod session;
diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rs
index 9deec4060..7e15581e0 100644
--- a/moli-parser/src/session.rs
+++ b/moli-parser/src/session.rs
@@ -13,6 +13,7 @@ use moli_dom::native::NativeNodeId;
use super::{
html::{DocumentSink, ParseHandle, ParserFinishDiscoverySignals, ParserInputQueue},
+ html_input::InputStack,
live_target::ParserStreamHtmlTreeSinkTarget,
};
@@ -23,8 +24,7 @@ pub(super) struct HtmlTreeSinkSession {
pub(super) struct HtmlParserSession {
tokenizer: Tokenizer<EmbedderPausingTreeBuilder>,
- input_buffer: BufferQueue,
- suspended_input_buffers: Vec<BufferQueue>,
+ input: InputStack,
}
pub(super) enum HtmlParserSessionResult {
@@ -137,8 +137,7 @@ impl HtmlParserSession {
let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);
Self {
tokenizer: Tokenizer::new(tree_builder, opts.tokenizer),
- input_buffer: BufferQueue::default(),
- suspended_input_buffers: Vec::new(),
+ input: InputStack::default(),
}
}
@@ -160,15 +159,14 @@ impl HtmlParserSession {
};
Self {
tokenizer: Tokenizer::new(tree_builder, tokenizer_options),
- input_buffer: BufferQueue::default(),
- suspended_input_buffers: Vec::new(),
+ input: InputStack::default(),
}
}
pub(super) fn process(&mut self, input: StrTendril) {
- self.input_buffer.push_back(input);
+ self.input.push_back(input);
while let HtmlParserSessionResult::Script(_) =
- feed_with_definitive_encoding(&self.tokenizer, &self.input_buffer)
+ feed_with_definitive_encoding(&self.tokenizer, self.input.current())
{
// Non-pump callers intentionally parse through embedder pauses. They have no
// runtime owner to notify, so parser-side custom-element handoffs and
@@ -178,7 +176,7 @@ impl HtmlParserSession {
}
pub(super) fn push_back(&mut self, input: StrTendril) {
- self.input_buffer.push_back(input);
+ self.input.push_back(input);
}
pub(super) fn begin_inserted_input(&mut self, input: StrTendril) {
@@ -190,63 +188,31 @@ impl HtmlParserSession {
// restored. Prefer permanent unknown locations over reporting
// plausible but incorrect document lines.
self.tokenizer.sink.sink().mark_source_positions_unknown();
- let parent = std::mem::take(&mut self.input_buffer);
- self.suspended_input_buffers.push(parent);
- self.input_buffer.push_back(input);
+ self.input.begin_inserted(input);
}
pub(super) fn append_to_current_inserted_input(&mut self, input: StrTendril) -> bool {
- if input.is_empty() {
- return true;
- }
- if self.suspended_input_buffers.is_empty() {
- return false;
- }
- self.input_buffer.push_back(input);
- true
+ self.input.append_to_current_inserted(input)
}
pub(super) fn has_buffered_input(&self) -> bool {
- !self.input_buffer.is_empty()
- || self
- .suspended_input_buffers
- .iter()
- .any(|input| !input.is_empty())
+ self.input.has_input()
}
pub(super) fn buffered_input_len(&self) -> usize {
- let mut len = 0usize;
- for input in
- std::iter::once(&self.input_buffer).chain(self.suspended_input_buffers.iter().rev())
- {
- let input = input.clone();
- while let Some(chunk) = input.pop_front() {
- len = len.saturating_add(chunk.len());
- }
- }
- len
+ self.input.len()
}
pub(super) fn snapshot_buffered_input(&self) -> String {
- let mut buffered = String::new();
- for input in
- std::iter::once(&self.input_buffer).chain(self.suspended_input_buffers.iter().rev())
- {
- let input = input.clone();
- while let Some(chunk) = input.pop_front() {
- buffered.push_str(&chunk);
- }
- }
- buffered
+ self.input.snapshot()
}
pub(super) fn feed(&mut self) -> HtmlParserSessionResult {
- let result = feed_with_definitive_encoding(&self.tokenizer, &self.input_buffer);
- if matches!(result, HtmlParserSessionResult::InputDrained)
- && self.input_buffer.is_empty()
- && let Some(parent) = self.suspended_input_buffers.pop()
- {
- self.input_buffer = parent;
+ let result = feed_with_definitive_encoding(&self.tokenizer, self.input.current());
+ if matches!(result, HtmlParserSessionResult::InputDrained) {
+ // The restored parent is intentionally consumed by the next parser
+ // step so each insertion depth keeps an explicit input boundary.
+ self.input.restore_parent_if_current_empty();
}
result
}
@@ -256,12 +222,8 @@ impl HtmlParserSession {
}
pub(super) fn finish(self) -> ParserStreamHtmlTreeSinkTarget {
- let Self {
- tokenizer,
- mut input_buffer,
- mut suspended_input_buffers,
- } = self;
- restore_all_suspended_input(&mut input_buffer, &mut suspended_input_buffers);
+ let Self { tokenizer, input } = self;
+ let input_buffer = input.into_buffer();
while let HtmlParserSessionResult::Script(_) =
feed_with_definitive_encoding(&tokenizer, &input_buffer)
{
@@ -281,12 +243,8 @@ impl HtmlParserSession {
}
pub(super) fn finish_live_runtime_dom_sink_parser(self) -> ParserFinishDiscoverySignals {
- let Self {
- tokenizer,
- mut input_buffer,
- mut suspended_input_buffers,
- } = self;
- restore_all_suspended_input(&mut input_buffer, &mut suspended_input_buffers);
+ let Self { tokenizer, input } = self;
+ let input_buffer = input.into_buffer();
while let HtmlParserSessionResult::Script(_) =
feed_with_definitive_encoding(&tokenizer, &input_buffer)
{
@@ -346,17 +304,6 @@ fn feed_with_definitive_encoding(
}
}
-fn restore_all_suspended_input(
- input_buffer: &mut BufferQueue,
- suspended_input_buffers: &mut Vec<BufferQueue>,
-) {
- while let Some(parent) = suspended_input_buffers.pop() {
- while let Some(chunk) = parent.pop_front() {
- input_buffer.push_back(chunk);
- }
- }
-}
-
pub(super) fn new_html_tree_sink_session(
target: ParserStreamHtmlTreeSinkTarget,
) -> HtmlTreeSinkSession {