Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

refactor(markdown): unify output ownership and table state

ldm0a14ec9788cf2
diff --git a/moli-html2md/README.md b/moli-html2md/README.md
index 67584234b..cd6414ec5 100644
--- a/moli-html2md/README.md
+++ b/moli-html2md/README.md
@@ -38,11 +38,11 @@ their children as ordinary content. Comments and doctypes produce no output.
tasks. Raw code text uses the same iterative traversal with formatting disabled.
- Tables inspect their direct sections, rows and cells before conversion. Nested
content uses the task stack; eligibility checks never rescan cell descendants.
-- Finished table cells form an owned fragment tree. Fallback joins and writes to
- parent buffers move fragment roots without copying descendant text; the final
- output is flattened in one iterative pass. Fragment destruction is iterative
- too. Quotes, headings and list items still materialize their captured text
- when rewriting its lines.
+- Finished blocks and table cells form an owned fragment tree. Fallback joins
+ and writes to parent buffers move fragment roots without copying descendant
+ text; the final output is flattened in one iterative pass. Fragment destruction
+ is iterative too. Quotes, headings and list items still materialize their
+ captured text when rewriting its lines.
- List spacing follows block boundaries recorded during conversion. Nested
lists keep their own spacing; list items need no preliminary DOM scan.
- Ordered lists use `start` followed by sequential numbers, like Turndown core.
@@ -102,8 +102,9 @@ Targeted regressions also cover empty elements, links, attributes in headings an
tables, code whitespace and language names, list numbering, and loose lists.
An operation-count regression checks nested table fallback at 64, 128 and 256
levels with 256 text bytes per level, both with and without headers. It bounds
-bytes copied during fragment materialization and writes to parent buffers by
-output size, so a repeated flatten/copy regression fails without timing thresholds.
+bytes copied during fragment materialization by output size; completed blocks
+move into parent buffers without copying their text. A repeated flatten/copy
+regression fails without timing thresholds.
Dropping 20,000 nested fragments is also tested on a 64 KiB thread stack.
Emphasis tests cover adjacent links that can use Markdown delimiters and
intraword link boundaries that still require inline HTML.
diff --git a/moli-html2md/src/copy_tests.rs b/moli-html2md/src/copy_tests.rs
index e7fed121b..9439eecea 100644
--- a/moli-html2md/src/copy_tests.rs
+++ b/moli-html2md/src/copy_tests.rs
@@ -30,8 +30,9 @@ fn nested_table_fallback_copies_text_linearly() {
support::rendered_html(&actual).matches("<table>").count(),
1
);
- // Count actual bytes copied when materializing fragments or writing
- // strings back to a parent, not elapsed time or recursion depth.
+ // Count actual bytes copied when materializing fragments; completed
+ // blocks move into parents without copying their text. This measures
+ // work rather than elapsed time or recursion depth.
assert!(
copied <= 2 * actual.len(),
"depth={depth}, output={}, copied={copied}",
diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs
index 903f042b5..d6afabd4c 100644
--- a/moli-html2md/src/machine.rs
+++ b/moli-html2md/src/machine.rs
@@ -82,14 +82,14 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
quote.push_str(line);
}
}
- self.writer().block(&quote, 2, 2);
+ self.writer().block(quote.into(), 2, 2);
}
Task::EndHeading(level) => {
let content = self.take_writer().into_string();
if !content.is_empty() {
let heading =
format!("{} {}", "#".repeat(level), content.replace('\n', " "));
- self.writer().block(&heading, 2, 2);
+ self.writer().block(heading.into(), 2, 2);
} else {
self.writer().boundary(2);
}
@@ -98,7 +98,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
self.lists.pop();
let content = self.take_writer();
let has_blocks = self.writer().has_blocks;
- self.writer().block_output(content, before, 2);
+ self.writer().block(content, before, 2);
if before == 1 {
// A nested list alone does not make its parent item loose.
self.writer().has_blocks = has_blocks;
@@ -127,7 +127,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
let list = self.lists.last_mut().expect("list item belongs to a list");
let separation = if loose || list.loose { 2 } else { 1 };
list.loose |= loose;
- self.writer().block(&item, separation, separation);
+ self.writer().block(item.into(), separation, separation);
}
Task::RawChildren(Some(node), depth, inline) => self.raw_node(node, depth, inline),
Task::EndCode => {
@@ -140,8 +140,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
Task::EndTableCell => {
let content = self.take_writer();
let table = self.tables.last_mut().expect("cell belongs to a table");
- table.content.push(content);
- table.in_cell = false;
+ table.finish_cell(content);
}
}
}
@@ -193,26 +192,8 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
NodeKind::Other => return,
NodeKind::Element(tag) => tag,
};
- if let Some(table) = self.tables.last_mut()
- && table.in_cell
- && matches!(
- tag,
- "table"
- | "pre"
- | "blockquote"
- | "ul"
- | "ol"
- | "li"
- | "hr"
- | "h1"
- | "h2"
- | "h3"
- | "h4"
- | "h5"
- | "h6"
- )
- {
- table.has_complex_content = true;
+ if let Some(table) = self.tables.last_mut() {
+ table.visit_element(tag);
}
match tag {
"head" | "script" | "style" | "noscript" | "template" => return,
@@ -221,7 +202,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
return;
}
"hr" => {
- self.writer().block("---", 2, 2);
+ self.writer().block("---".into(), 2, 2);
return;
}
"strong" | "b" => self.style(Style::Strong),
@@ -290,7 +271,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
if let Some(mut table) =
Table::from_dom(self.dom, node, depth, self.options.max_depth)
{
- let captions = std::mem::take(&mut table.captions);
+ let captions = table.take_captions();
self.tables.push(table);
self.tasks.push(Task::TableCell);
for caption in captions.into_iter().rev() {
@@ -312,8 +293,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
fn table_cell(&mut self) {
let table = self.tables.last_mut().expect("table conversion is active");
- if let Some(cell) = table.cells.next() {
- table.in_cell = true;
+ if let Some(cell) = table.next_cell() {
self.capture();
self.writer().table_cell();
self.tasks.push(Task::TableCell);
@@ -321,7 +301,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
self.children(cell.node, cell.depth + 1);
} else {
let table = self.tables.pop().expect("table conversion is active");
- self.writer().block_output(table.finish(), 2, 2);
+ self.writer().block(table.finish(), 2, 2);
}
}
@@ -444,7 +424,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
block.push('\n');
}
block.push_str(&fence);
- self.writer().block(&block, 2, 2);
+ self.writer().block(block.into(), 2, 2);
}
}
diff --git a/moli-html2md/src/output.rs b/moli-html2md/src/output.rs
index 4419a4b09..fbc93e471 100644
--- a/moli-html2md/src/output.rs
+++ b/moli-html2md/src/output.rs
@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+
/// Finished Markdown fragments. Joining outputs moves their roots, not their
/// text or descendant vectors. Both flattening and destruction are iterative.
#[derive(Default)]
@@ -9,8 +11,7 @@ pub(crate) struct Output {
}
enum Part {
- Text(String),
- Literal(&'static str),
+ Text(Cow<'static, str>),
Group(Output),
}
@@ -37,7 +38,7 @@ impl Output {
};
}
- pub(crate) fn push_text(&mut self, text: String) {
+ pub(crate) fn push_text(&mut self, text: Cow<'static, str>) {
if let Some(last) = text.chars().next_back() {
self.extend_tail(
text.len(),
@@ -48,17 +49,6 @@ impl Output {
}
}
- pub(crate) fn push_literal(&mut self, text: &'static str) {
- if let Some(last) = text.chars().next_back() {
- self.extend_tail(
- text.len(),
- last,
- text.len() - text.trim_end_matches('\n').len(),
- );
- self.parts.push(Part::Literal(text));
- }
- }
-
pub(crate) fn append(&mut self, output: Self) {
if let Some(last) = output.last {
self.extend_tail(output.len, last, output.trailing_newlines);
@@ -71,7 +61,7 @@ impl Output {
// allocation instead of adding a copy to the common, non-table path.
if self.parts.len() == 1 {
match self.parts.pop().expect("one output fragment") {
- Part::Text(text) => return text,
+ Part::Text(Cow::Owned(text)) => return text,
part => self.parts.push(part),
}
}
@@ -85,11 +75,6 @@ impl Output {
record_copy(fragment.len());
text.push_str(&fragment);
}
- Part::Literal(fragment) => {
- #[cfg(test)]
- record_copy(fragment.len());
- text.push_str(fragment);
- }
Part::Group(mut output) => pending.extend(output.parts.drain(..).rev()),
}
}
@@ -100,7 +85,15 @@ impl Output {
impl From<String> for Output {
fn from(text: String) -> Self {
let mut output = Self::default();
- output.push_text(text);
+ output.push_text(text.into());
+ output
+ }
+}
+
+impl From<&'static str> for Output {
+ fn from(text: &'static str) -> Self {
+ let mut output = Self::default();
+ output.push_text(text.into());
output
}
}
@@ -124,7 +117,7 @@ thread_local! {
}
#[cfg(test)]
-pub(crate) fn record_copy(bytes: usize) {
+fn record_copy(bytes: usize) {
COPIED_BYTES.with(|count| count.set(count.get() + bytes));
}
diff --git a/moli-html2md/src/table.rs b/moli-html2md/src/table.rs
index 7a682d17a..99cf4456f 100644
--- a/moli-html2md/src/table.rs
+++ b/moli-html2md/src/table.rs
@@ -6,11 +6,11 @@ pub(crate) struct Cell<Id> {
}
pub(crate) struct Table<Id> {
- pub(crate) captions: Vec<Cell<Id>>,
- pub(crate) cells: std::vec::IntoIter<Cell<Id>>,
- pub(crate) content: Vec<Output>,
- pub(crate) in_cell: bool,
- pub(crate) has_complex_content: bool,
+ captions: Vec<Cell<Id>>,
+ cells: std::vec::IntoIter<Cell<Id>>,
+ content: Vec<Output>,
+ in_cell: bool,
+ has_complex_content: bool,
rows: Vec<usize>,
separators: Vec<&'static str>,
has_header: bool,
@@ -140,12 +140,55 @@ impl<Id: Copy> Table<Id> {
Some(())
}
+ pub(crate) fn take_captions(&mut self) -> Vec<Cell<Id>> {
+ std::mem::take(&mut self.captions)
+ }
+
+ pub(crate) fn next_cell(&mut self) -> Option<Cell<Id>> {
+ debug_assert!(
+ !self.in_cell,
+ "finish the current cell before starting another"
+ );
+ let cell = self.cells.next()?;
+ self.in_cell = true;
+ Some(cell)
+ }
+
+ pub(crate) fn finish_cell(&mut self, content: Output) {
+ debug_assert!(self.in_cell, "cell conversion must be active");
+ self.content.push(content);
+ self.in_cell = false;
+ }
+
+ pub(crate) fn visit_element(&mut self, tag: &str) {
+ if self.in_cell
+ && matches!(
+ tag,
+ "table"
+ | "pre"
+ | "blockquote"
+ | "ul"
+ | "ol"
+ | "li"
+ | "hr"
+ | "h1"
+ | "h2"
+ | "h3"
+ | "h4"
+ | "h5"
+ | "h6"
+ )
+ {
+ self.has_complex_content = true;
+ }
+ }
+
pub(crate) fn finish(self) -> Output {
if self.has_complex_content {
let mut output = Output::default();
for cell in self.content.into_iter().filter(|cell| !cell.is_empty()) {
if !output.is_empty() {
- output.push_literal("\n\n");
+ output.push_text("\n\n".into());
}
output.append(cell);
}
diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rs
index c32bb6acb..e2866b906 100644
--- a/moli-html2md/src/writer.rs
+++ b/moli-html2md/src/writer.rs
@@ -244,21 +244,7 @@ impl<'a> Writer<'a> {
self.line_digits = Some(0);
}
- pub(crate) fn block(&mut self, text: &str, before: usize, after: usize) {
- if text.is_empty() {
- self.boundary(before.max(after));
- return;
- }
- self.boundary(before);
- self.flush_breaks();
- #[cfg(test)]
- crate::output::record_copy(text.len());
- self.output.push_str(text);
- self.line_digits = None;
- self.boundary(after);
- }
-
- pub(crate) fn block_output(&mut self, output: Output, before: usize, after: usize) {
+ pub(crate) fn block(&mut self, output: Output, before: usize, after: usize) {
if output.is_empty() {
self.boundary(before.max(after));
return;
@@ -267,7 +253,8 @@ impl<'a> Writer<'a> {
// offsets local to the current String, never inside a finished block.
self.boundary(before);
self.flush_breaks();
- self.prefix.push_text(std::mem::take(&mut self.output));
+ self.prefix
+ .push_text(std::mem::take(&mut self.output).into());
self.prefix.append(output);
self.line_digits = None;
self.boundary(after);
@@ -279,7 +266,7 @@ impl<'a> Writer<'a> {
self.flush_spaces(false);
let end = self.output.trim_end_matches(is_space).len();
self.output.truncate(end);
- self.prefix.push_text(self.output);
+ self.prefix.push_text(self.output.into());
self.prefix
}