Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor(markdown): unify output ownership and table state
ldm0a14ec9788cf2
diff --git a/moli-html2md/README.md b/moli-html2md/README.mdindex 67584234b..cd6414ec5 100644--- a/moli-html2md/README.md+++ b/moli-html2md/README.md@@ -38,11 +38,11 @@ their children as ordinary content. Comments and doctypes produce no output.tasks. Raw code text uses the same iterative traversal with formatting disabled.- Tables inspect their direct sections, rows and cells before conversion. Nestedcontent uses the task stack; eligibility checks never rescan cell descendants.-- Finished table cells form an owned fragment tree. Fallback joins and writes to- parent buffers move fragment roots without copying descendant text; the final- output is flattened in one iterative pass. Fragment destruction is iterative- too. Quotes, headings and list items still materialize their captured text- when rewriting its lines.+- Finished blocks and table cells form an owned fragment tree. Fallback joins+ and writes to parent buffers move fragment roots without copying descendant+ text; the final output is flattened in one iterative pass. Fragment destruction+ is iterative too. Quotes, headings and list items still materialize their+ captured text when rewriting its lines.- List spacing follows block boundaries recorded during conversion. Nestedlists keep their own spacing; list items need no preliminary DOM scan.- Ordered lists use `start` followed by sequential numbers, like Turndown core.@@ -102,8 +102,9 @@ Targeted regressions also cover empty elements, links, attributes in headings antables, code whitespace and language names, list numbering, and loose lists.An operation-count regression checks nested table fallback at 64, 128 and 256levels with 256 text bytes per level, both with and without headers. It bounds-bytes copied during fragment materialization and writes to parent buffers by-output size, so a repeated flatten/copy regression fails without timing thresholds.+bytes copied during fragment materialization by output size; completed blocks+move into parent buffers without copying their text. A repeated flatten/copy+regression fails without timing thresholds.Dropping 20,000 nested fragments is also tested on a 64 KiB thread stack.Emphasis tests cover adjacent links that can use Markdown delimiters andintraword link boundaries that still require inline HTML.diff --git a/moli-html2md/src/copy_tests.rs b/moli-html2md/src/copy_tests.rsindex e7fed121b..9439eecea 100644--- a/moli-html2md/src/copy_tests.rs+++ b/moli-html2md/src/copy_tests.rs@@ -30,8 +30,9 @@ fn nested_table_fallback_copies_text_linearly() {support::rendered_html(&actual).matches("<table>").count(),1);- // Count actual bytes copied when materializing fragments or writing- // strings back to a parent, not elapsed time or recursion depth.+ // Count actual bytes copied when materializing fragments; completed+ // blocks move into parents without copying their text. This measures+ // work rather than elapsed time or recursion depth.assert!(copied <= 2 * actual.len(),"depth={depth}, output={}, copied={copied}",diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rsindex 903f042b5..d6afabd4c 100644--- a/moli-html2md/src/machine.rs+++ b/moli-html2md/src/machine.rs@@ -82,14 +82,14 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {quote.push_str(line);}}- self.writer().block("e, 2, 2);+ self.writer().block(quote.into(), 2, 2);}Task::EndHeading(level) => {let content = self.take_writer().into_string();if !content.is_empty() {let heading =format!("{} {}", "#".repeat(level), content.replace('\n', " "));- self.writer().block(&heading, 2, 2);+ self.writer().block(heading.into(), 2, 2);} else {self.writer().boundary(2);}@@ -98,7 +98,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {self.lists.pop();let content = self.take_writer();let has_blocks = self.writer().has_blocks;- self.writer().block_output(content, before, 2);+ self.writer().block(content, before, 2);if before == 1 {// A nested list alone does not make its parent item loose.self.writer().has_blocks = has_blocks;@@ -127,7 +127,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {let list = self.lists.last_mut().expect("list item belongs to a list");let separation = if loose || list.loose { 2 } else { 1 };list.loose |= loose;- self.writer().block(&item, separation, separation);+ self.writer().block(item.into(), separation, separation);}Task::RawChildren(Some(node), depth, inline) => self.raw_node(node, depth, inline),Task::EndCode => {@@ -140,8 +140,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {Task::EndTableCell => {let content = self.take_writer();let table = self.tables.last_mut().expect("cell belongs to a table");- table.content.push(content);- table.in_cell = false;+ table.finish_cell(content);}}}@@ -193,26 +192,8 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {NodeKind::Other => return,NodeKind::Element(tag) => tag,};- if let Some(table) = self.tables.last_mut()- && table.in_cell- && matches!(- tag,- "table"- | "pre"- | "blockquote"- | "ul"- | "ol"- | "li"- | "hr"- | "h1"- | "h2"- | "h3"- | "h4"- | "h5"- | "h6"- )- {- table.has_complex_content = true;+ if let Some(table) = self.tables.last_mut() {+ table.visit_element(tag);}match tag {"head" | "script" | "style" | "noscript" | "template" => return,@@ -221,7 +202,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {return;}"hr" => {- self.writer().block("---", 2, 2);+ self.writer().block("---".into(), 2, 2);return;}"strong" | "b" => self.style(Style::Strong),@@ -290,7 +271,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {if let Some(mut table) =Table::from_dom(self.dom, node, depth, self.options.max_depth){- let captions = std::mem::take(&mut table.captions);+ let captions = table.take_captions();self.tables.push(table);self.tasks.push(Task::TableCell);for caption in captions.into_iter().rev() {@@ -312,8 +293,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {fn table_cell(&mut self) {let table = self.tables.last_mut().expect("table conversion is active");- if let Some(cell) = table.cells.next() {- table.in_cell = true;+ if let Some(cell) = table.next_cell() {self.capture();self.writer().table_cell();self.tasks.push(Task::TableCell);@@ -321,7 +301,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {self.children(cell.node, cell.depth + 1);} else {let table = self.tables.pop().expect("table conversion is active");- self.writer().block_output(table.finish(), 2, 2);+ self.writer().block(table.finish(), 2, 2);}}@@ -444,7 +424,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {block.push('\n');}block.push_str(&fence);- self.writer().block(&block, 2, 2);+ self.writer().block(block.into(), 2, 2);}}diff --git a/moli-html2md/src/output.rs b/moli-html2md/src/output.rsindex 4419a4b09..fbc93e471 100644--- a/moli-html2md/src/output.rs+++ b/moli-html2md/src/output.rs@@ -1,3 +1,5 @@+use std::borrow::Cow;+/// Finished Markdown fragments. Joining outputs moves their roots, not their/// text or descendant vectors. Both flattening and destruction are iterative.#[derive(Default)]@@ -9,8 +11,7 @@ pub(crate) struct Output {}enum Part {- Text(String),- Literal(&'static str),+ Text(Cow<'static, str>),Group(Output),}@@ -37,7 +38,7 @@ impl Output {};}- pub(crate) fn push_text(&mut self, text: String) {+ pub(crate) fn push_text(&mut self, text: Cow<'static, str>) {if let Some(last) = text.chars().next_back() {self.extend_tail(text.len(),@@ -48,17 +49,6 @@ impl Output {}}- pub(crate) fn push_literal(&mut self, text: &'static str) {- if let Some(last) = text.chars().next_back() {- self.extend_tail(- text.len(),- last,- text.len() - text.trim_end_matches('\n').len(),- );- self.parts.push(Part::Literal(text));- }- }-pub(crate) fn append(&mut self, output: Self) {if let Some(last) = output.last {self.extend_tail(output.len, last, output.trailing_newlines);@@ -71,7 +61,7 @@ impl Output {// allocation instead of adding a copy to the common, non-table path.if self.parts.len() == 1 {match self.parts.pop().expect("one output fragment") {- Part::Text(text) => return text,+ Part::Text(Cow::Owned(text)) => return text,part => self.parts.push(part),}}@@ -85,11 +75,6 @@ impl Output {record_copy(fragment.len());text.push_str(&fragment);}- Part::Literal(fragment) => {- #[cfg(test)]- record_copy(fragment.len());- text.push_str(fragment);- }Part::Group(mut output) => pending.extend(output.parts.drain(..).rev()),}}@@ -100,7 +85,15 @@ impl Output {impl From<String> for Output {fn from(text: String) -> Self {let mut output = Self::default();- output.push_text(text);+ output.push_text(text.into());+ output+ }+}++impl From<&'static str> for Output {+ fn from(text: &'static str) -> Self {+ let mut output = Self::default();+ output.push_text(text.into());output}}@@ -124,7 +117,7 @@ thread_local! {}#[cfg(test)]-pub(crate) fn record_copy(bytes: usize) {+fn record_copy(bytes: usize) {COPIED_BYTES.with(|count| count.set(count.get() + bytes));}diff --git a/moli-html2md/src/table.rs b/moli-html2md/src/table.rsindex 7a682d17a..99cf4456f 100644--- a/moli-html2md/src/table.rs+++ b/moli-html2md/src/table.rs@@ -6,11 +6,11 @@ pub(crate) struct Cell<Id> {}pub(crate) struct Table<Id> {- pub(crate) captions: Vec<Cell<Id>>,- pub(crate) cells: std::vec::IntoIter<Cell<Id>>,- pub(crate) content: Vec<Output>,- pub(crate) in_cell: bool,- pub(crate) has_complex_content: bool,+ captions: Vec<Cell<Id>>,+ cells: std::vec::IntoIter<Cell<Id>>,+ content: Vec<Output>,+ in_cell: bool,+ has_complex_content: bool,rows: Vec<usize>,separators: Vec<&'static str>,has_header: bool,@@ -140,12 +140,55 @@ impl<Id: Copy> Table<Id> {Some(())}+ pub(crate) fn take_captions(&mut self) -> Vec<Cell<Id>> {+ std::mem::take(&mut self.captions)+ }++ pub(crate) fn next_cell(&mut self) -> Option<Cell<Id>> {+ debug_assert!(+ !self.in_cell,+ "finish the current cell before starting another"+ );+ let cell = self.cells.next()?;+ self.in_cell = true;+ Some(cell)+ }++ pub(crate) fn finish_cell(&mut self, content: Output) {+ debug_assert!(self.in_cell, "cell conversion must be active");+ self.content.push(content);+ self.in_cell = false;+ }++ pub(crate) fn visit_element(&mut self, tag: &str) {+ if self.in_cell+ && matches!(+ tag,+ "table"+ | "pre"+ | "blockquote"+ | "ul"+ | "ol"+ | "li"+ | "hr"+ | "h1"+ | "h2"+ | "h3"+ | "h4"+ | "h5"+ | "h6"+ )+ {+ self.has_complex_content = true;+ }+ }+pub(crate) fn finish(self) -> Output {if self.has_complex_content {let mut output = Output::default();for cell in self.content.into_iter().filter(|cell| !cell.is_empty()) {if !output.is_empty() {- output.push_literal("\n\n");+ output.push_text("\n\n".into());}output.append(cell);}diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rsindex c32bb6acb..e2866b906 100644--- a/moli-html2md/src/writer.rs+++ b/moli-html2md/src/writer.rs@@ -244,21 +244,7 @@ impl<'a> Writer<'a> {self.line_digits = Some(0);}- pub(crate) fn block(&mut self, text: &str, before: usize, after: usize) {- if text.is_empty() {- self.boundary(before.max(after));- return;- }- self.boundary(before);- self.flush_breaks();- #[cfg(test)]- crate::output::record_copy(text.len());- self.output.push_str(text);- self.line_digits = None;- self.boundary(after);- }-- pub(crate) fn block_output(&mut self, output: Output, before: usize, after: usize) {+ pub(crate) fn block(&mut self, output: Output, before: usize, after: usize) {if output.is_empty() {self.boundary(before.max(after));return;@@ -267,7 +253,8 @@ impl<'a> Writer<'a> {// offsets local to the current String, never inside a finished block.self.boundary(before);self.flush_breaks();- self.prefix.push_text(std::mem::take(&mut self.output));+ self.prefix+ .push_text(std::mem::take(&mut self.output).into());self.prefix.append(output);self.line_digits = None;self.boundary(after);@@ -279,7 +266,7 @@ impl<'a> Writer<'a> {self.flush_spaces(false);let end = self.output.trim_end_matches(is_space).len();self.output.truncate(end);- self.prefix.push_text(self.output);+ self.prefix.push_text(self.output.into());self.prefix}