Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

feat(markdown): introduce moli-html2md

ldm061dbf7ea3e8d
diff --git a/Cargo.lock b/Cargo.lock
index 99de8b7d9..cdf3d51f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2308,6 +2308,18 @@ dependencies = [
"web_atoms",
]
+[[package]]
+name = "markup5ever_rcdom"
+version = "0.39.0+unofficial"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ac010f19d6c4af81eeb4018a39d7a115de9d285af45c126a4ac02e6fc5716b7"
+dependencies = [
+ "html5ever",
+ "markup5ever",
+ "tendril",
+ "xml5ever",
+]
+
[[package]]
name = "matchers"
version = "0.2.0"
@@ -2606,6 +2618,7 @@ dependencies = [
"moli-css-parse",
"moli-html-input-temporal",
"moli-html-input-type",
+ "moli-html2md",
"percent-encoding",
"selectors",
"serde_json",
@@ -2701,6 +2714,16 @@ dependencies = [
"strum",
]
+[[package]]
+name = "moli-html2md"
+version = "0.1.0"
+dependencies = [
+ "html5ever",
+ "markup5ever_rcdom",
+ "pulldown-cmark",
+ "serde_json",
+]
+
[[package]]
name = "moli-http-cache"
version = "0.1.0"
@@ -3038,6 +3061,7 @@ dependencies = [
"moli-fetch",
"moli-file-api",
"moli-geometry",
+ "moli-html2md",
"moli-http-cache",
"moli-image",
"moli-import-map",
@@ -4132,6 +4156,24 @@ version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
+[[package]]
+name = "pulldown-cmark"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
+dependencies = [
+ "bitflags",
+ "memchr",
+ "pulldown-cmark-escape",
+ "unicase",
+]
+
+[[package]]
+name = "pulldown-cmark-escape"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
+
[[package]]
name = "pxfm"
version = "0.1.30"
@@ -5607,6 +5649,12 @@ dependencies = [
"arrayvec",
]
+[[package]]
+name = "unicase"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
+
[[package]]
name = "unicode-bidi"
version = "0.3.18"
diff --git a/Cargo.toml b/Cargo.toml
index 8934278c7..ec8aa593a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -79,6 +79,7 @@ members = [
"moli-geometry",
"moli-header-field",
"moli-html-input-type",
+ "moli-html2md",
"moli-html-input-temporal",
"moli-window-features",
"moli-time",
diff --git a/moli-dom/Cargo.toml b/moli-dom/Cargo.toml
index 878b52ccd..5530db1f1 100644
--- a/moli-dom/Cargo.toml
+++ b/moli-dom/Cargo.toml
@@ -9,6 +9,7 @@ html5ever = "0.39.0"
indexmap = "2.6"
moli-css-parse = { path = "../moli-css-parse" }
moli-html-input-type = { path = "../moli-html-input-type" }
+moli-html2md = { path = "../moli-html2md" }
moli-html-input-temporal = { path = "../moli-html-input-temporal" }
percent-encoding = "2.3"
selectors = "0.40"
diff --git a/moli-dom/src/native/markdown.rs b/moli-dom/src/native/markdown.rs
new file mode 100644
index 000000000..ba6f395d4
--- /dev/null
+++ b/moli-dom/src/native/markdown.rs
@@ -0,0 +1,29 @@
+use super::{NativeDom, NativeNodeId, NodeData};
+
+impl moli_html2md::Dom for NativeDom {
+ type NodeId = NativeNodeId;
+
+ fn node_kind(&self, node: NativeNodeId) -> moli_html2md::NodeKind<'_> {
+ use moli_html2md::NodeKind;
+
+ match self.node(node).map(|node| node.data()) {
+ Some(NodeData::Document(_) | NodeData::DocumentFragment(_)) => NodeKind::Document,
+ Some(NodeData::Element(element)) => NodeKind::Element(element.local_name()),
+ Some(NodeData::Text(text)) => NodeKind::Text(text.data()),
+ Some(NodeData::CDataSection(text)) => NodeKind::Text(text.data()),
+ _ => NodeKind::Other,
+ }
+ }
+
+ fn first_child(&self, node: NativeNodeId) -> Option<NativeNodeId> {
+ NativeDom::first_child(self, node)
+ }
+
+ fn next_sibling(&self, node: NativeNodeId) -> Option<NativeNodeId> {
+ NativeDom::next_sibling(self, node)
+ }
+
+ fn attribute(&self, node: NativeNodeId, name: &str) -> Option<&str> {
+ self.node(node)?.as_element()?.attribute(name)
+ }
+}
diff --git a/moli-dom/src/native/mod.rs b/moli-dom/src/native/mod.rs
index bd30e0fd9..d7322a3c3 100644
--- a/moli-dom/src/native/mod.rs
+++ b/moli-dom/src/native/mod.rs
@@ -3,6 +3,7 @@ mod document;
mod element;
mod host;
mod html_serialization;
+mod markdown;
mod node;
mod queries;
mod scripts;
diff --git a/moli-html2md/Cargo.toml b/moli-html2md/Cargo.toml
new file mode 100644
index 000000000..1698234b1
--- /dev/null
+++ b/moli-html2md/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "moli-html2md"
+version = "0.1.0"
+edition = "2024"
+license.workspace = true
+description = "Read-only DOM to Markdown conversion with an iterative state machine"
+
+[lints]
+workspace = true
+
+[dev-dependencies]
+html5ever = "0.39"
+markup5ever_rcdom = "0.39"
+pulldown-cmark = { version = "0.13", default-features = false, features = ["html"] }
+serde_json = "1.0.145"
diff --git a/moli-html2md/README.md b/moli-html2md/README.md
new file mode 100644
index 000000000..9767df8d8
--- /dev/null
+++ b/moli-html2md/README.md
@@ -0,0 +1,86 @@
+# moli-html2md
+
+Convert an existing DOM to Markdown through a small read-only interface. This
+is an independent implementation with no runtime dependencies, HTML parser, or
+intermediate DOM. `moli-dom::native::NativeDom` implements the interface directly.
+
+```rust
+let root = dom.body_node_id().unwrap_or(dom.document_node_id());
+let markdown = moli_html2md::Converter::new(moli_html2md::Options {
+ max_depth: 1000,
+ default_code_language: Some("text".to_owned()),
+ ..Default::default()
+})
+.convert(&dom, root);
+```
+
+## DOM interface
+
+Implement `Dom` with a copyable node ID and four borrowed queries:
+
+- `node_kind`: document/fragment, lowercase element tag, decoded text, or ignored node.
+- `first_child` and `next_sibling`: ordered traversal of a stable, acyclic tree.
+- `attribute`: decoded attribute value, borrowed from the DOM.
+
+The converter does not need parent pointers, mutable access, serialization, or
+an allocation for each node. The caller chooses the root; its siblings are not
+visited. `Other` nodes and their subtrees are ignored. Unknown elements expose
+their children as ordinary content. Comments and doctypes produce no output.
+
+## State machine
+
+- A heap task stack tracks node entry, sibling continuation, and element exit.
+ Traversal stores one continuation per level instead of collecting all children.
+- An inline writer tracks desired and emitted styles, pending whitespace, and
+ block boundaries. It collapses HTML whitespace across node boundaries, keeps
+ Unicode spaces, and coalesces adjacent identical emphasis without changing nodes.
+- Lists, quotes, and headings have output buffers finalized by exit
+ tasks. Raw code text uses the same iterative traversal with formatting disabled.
+- List spacing follows block boundaries recorded during conversion. Nested
+ lists keep their own spacing; list items need no preliminary DOM scan.
+- Ordered lists use `start` followed by sequential numbers, like Turndown core.
+ `reversed` and individual `li[value]` attributes are not represented.
+- Code fences are chosen after collecting the code text. Language names retain
+ punctuation and Unicode; a name containing backticks uses a tilde fence.
+ Fallback language hints end at the first line ending.
+
+Tables, sections, rows, and cells follow Turndown core's ordinary block behavior:
+their content is emitted in DOM order, separated by blank lines. Nested tables
+use the same rule. The converter does not classify layout tables or generate
+GFM table syntax; headers, roles, borders, and spans do not select another path.
+
+Supported output includes headings, paragraphs, emphasis, strikethrough, links,
+images, lists, blockquotes, hard breaks, and fenced code. Inline HTML is used when
+Markdown delimiters cannot express an emphasis boundary.
+Script/style/head/noscript/template subtrees are omitted.
+
+`max_depth` counts the supplied root as depth zero; nodes at or beyond the limit
+are omitted. All traversal paths, including raw code, use this limit.
+
+`preformatted_code` preserves whitespace inside inline code when enabled. Its
+default is `false`, which collapses ordinary HTML whitespace and moves boundary
+spaces outside code delimiters. Fenced blocks always preserve nonempty code's
+spaces and blank lines. Empty blocks still separate surrounding text.
+
+This is a structural content dump. It does not evaluate CSS, visibility, layout,
+or JavaScript, and cannot preserve all HTML presentation (for example, arbitrary
+ordered-list numbering or table spanning). DOM snapshots used by Moli's optional
+strip/base/frame transformations remain the responsibility of the caller.
+
+## Verification
+
+`cargo test -p moli-html2md` covers an independent arena DOM, immutability,
+whitespace, escaping, code, tables, depth limits, and 20,000 nested nodes on a
+64 KiB thread stack. The [Turndown differential suite](tests/turndown/README.md)
+pins 147 upstream fixtures and 50 additional cases. It compares rendered meaning,
+checks DOM immutability, and explicitly asserts 12 documented differences that
+preserve Moli's handling of literal text, code, emphasis, and list boundaries.
+Targeted regressions also cover empty elements, links, attributes in headings and
+tables, code whitespace and language names, list numbering, and loose lists.
+Emphasis tests cover adjacent links that can use Markdown delimiters and
+intraword link boundaries that still require inline HTML.
+
+The HTML parser, reference JSON reader, and `pulldown-cmark` renderer are all
+**test-only** dependencies. Tests use checked-in reference outputs and require
+neither Node nor network access. NativeDom integration and real HTML fixtures
+are tested in `moli-renderer-v8`.
diff --git a/moli-html2md/src/converter.rs b/moli-html2md/src/converter.rs
new file mode 100644
index 000000000..51e9bac86
--- /dev/null
+++ b/moli-html2md/src/converter.rs
@@ -0,0 +1,29 @@
+//! Convert an existing DOM to Markdown without parsing HTML or modifying nodes.
+//!
+//! Implement [`Dom`] for your tree, then pass a borrowed tree and a root node to
+//! [`convert`]. Traversal and formatting use explicit stacks, including code,
+//! lists, quotes, and tables; DOM depth never becomes Rust call-stack depth.
+
+use crate::{Dom, Options, machine};
+
+/// A reusable converter. All per-document state lives inside [`Self::convert`].
+#[derive(Clone, Debug, Default)]
+pub struct Converter {
+ options: Options,
+}
+
+impl Converter {
+ pub fn new(options: Options) -> Self {
+ Self { options }
+ }
+
+ /// Convert a subtree, borrowing the DOM for this call only.
+ pub fn convert<D: Dom + ?Sized>(&self, dom: &D, root: D::NodeId) -> String {
+ machine::convert(dom, root, &self.options)
+ }
+}
+
+/// Convert a subtree using the default options.
+pub fn convert<D: Dom + ?Sized>(dom: &D, root: D::NodeId) -> String {
+ Converter::default().convert(dom, root)
+}
diff --git a/moli-html2md/src/dom.rs b/moli-html2md/src/dom.rs
new file mode 100644
index 000000000..583e68467
--- /dev/null
+++ b/moli-html2md/src/dom.rs
@@ -0,0 +1,29 @@
+/// The information needed to render a node. Text and tag names are borrowed.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum NodeKind<'a> {
+ /// A document or fragment whose children should be visited.
+ Document,
+ /// An element's normalized, lowercase HTML local name.
+ Element(&'a str),
+ /// Already decoded text, including CDATA when supported by the DOM.
+ Text(&'a str),
+ /// Comments, doctypes, processing instructions, and invalid node IDs.
+ Other,
+}
+
+/// A read-only view of an existing DOM.
+///
+/// IDs must describe a stable, acyclic tree for the duration of conversion.
+/// Sibling chains must terminate. Attributes and text must already be decoded;
+/// this crate neither parses HTML nor decodes entities a second time. No parent
+/// lookup, allocation, interior mutation, or particular node representation is
+/// required. Template contents and shadow roots are visited only if the adapter
+/// exposes them as children.
+pub trait Dom {
+ type NodeId: Copy;
+
+ fn node_kind(&self, node: Self::NodeId) -> NodeKind<'_>;
+ fn first_child(&self, node: Self::NodeId) -> Option<Self::NodeId>;
+ fn next_sibling(&self, node: Self::NodeId) -> Option<Self::NodeId>;
+ fn attribute(&self, node: Self::NodeId, name: &str) -> Option<&str>;
+}
diff --git a/moli-html2md/src/lib.rs b/moli-html2md/src/lib.rs
new file mode 100644
index 000000000..dadb292b1
--- /dev/null
+++ b/moli-html2md/src/lib.rs
@@ -0,0 +1,9 @@
+mod converter;
+mod dom;
+mod machine;
+mod options;
+mod writer;
+
+pub use converter::{Converter, convert};
+pub use dom::{Dom, NodeKind};
+pub use options::Options;
diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs
new file mode 100644
index 000000000..d6839c404
--- /dev/null
+++ b/moli-html2md/src/machine.rs
@@ -0,0 +1,426 @@
+use crate::writer::{Style, Writer, longest_run};
+use crate::{Dom, NodeKind, Options};
+
+enum Task<'a, Id> {
+ Visit(Id, usize),
+ Children(Option<Id>, usize),
+ Boundary,
+ PopStyle,
+ EndLink(usize),
+ EndQuote,
+ EndHeading(usize),
+ EndList(usize),
+ EndItem(String),
+ RawChildren(Option<Id>, usize),
+ EndCode,
+ EndPre(Option<&'a str>),
+}
+
+struct List {
+ output_depth: usize,
+ ordered: bool,
+ next: i64,
+ loose: bool,
+}
+
+struct Machine<'a, D: Dom + ?Sized> {
+ dom: &'a D,
+ options: &'a Options,
+ tasks: Vec<Task<'a, D::NodeId>>,
+ writers: Vec<Writer<'a>>,
+ lists: Vec<List>,
+ raw: String,
+ serial: usize,
+}
+
+pub(crate) fn convert<D: Dom + ?Sized>(dom: &D, root: D::NodeId, options: &Options) -> String {
+ let mut machine = Machine {
+ dom,
+ options,
+ tasks: vec![Task::Visit(root, 0)],
+ writers: vec![Writer::default()],
+ lists: Vec::new(),
+ raw: String::new(),
+ serial: 0,
+ };
+ machine.run();
+ machine.take_writer()
+}
+
+impl<'a, D: Dom + ?Sized> Machine<'a, D> {
+ fn run(&mut self) {
+ while let Some(task) = self.tasks.pop() {
+ match task {
+ Task::Visit(node, depth) => self.visit(node, depth),
+ Task::Children(Some(node), depth) => {
+ // Keep one sibling continuation, not a vector of every
+ // child. Traversal storage is bounded by depth, not width.
+ self.tasks
+ .push(Task::Children(self.dom.next_sibling(node), depth));
+ self.tasks.push(Task::Visit(node, depth));
+ }
+ Task::Children(None, _) | Task::RawChildren(None, _) => {}
+ Task::Boundary => self.writer().boundary(2),
+ Task::PopStyle => self.writer().pop_style(),
+ Task::EndLink(serial) => self.writer().end_link(serial),
+ Task::EndQuote => {
+ let content = self.take_writer();
+ let mut quote = String::new();
+ for line in content.lines() {
+ if !quote.is_empty() {
+ quote.push('\n');
+ }
+ quote.push('>');
+ if !line.is_empty() {
+ quote.push(' ');
+ quote.push_str(line);
+ }
+ }
+ self.writer().block(&quote, 2, 2);
+ }
+ Task::EndHeading(level) => {
+ let content = self.take_writer();
+ if !content.is_empty() {
+ let heading =
+ format!("{} {}", "#".repeat(level), content.replace('\n', " "));
+ self.writer().block(&heading, 2, 2);
+ } else {
+ self.writer().boundary(2);
+ }
+ }
+ Task::EndList(before) => {
+ self.lists.pop();
+ let content = self.take_writer();
+ let has_blocks = self.writer().has_blocks;
+ self.writer().block(&content, before, 2);
+ if before == 1 {
+ // A nested list alone does not make its parent item loose.
+ self.writer().has_blocks = has_blocks;
+ }
+ }
+ Task::EndItem(marker) => {
+ let loose = self.writer().has_blocks;
+ let content = self.take_writer();
+ if content.is_empty() {
+ continue;
+ }
+ let mut item = String::new();
+ for (index, line) in content.lines().enumerate() {
+ if index == 0 {
+ item.push_str(&marker);
+ } else {
+ item.push('\n');
+ if !line.is_empty() {
+ for _ in 0..marker.len() {
+ item.push(' ');
+ }
+ }
+ }
+ item.push_str(line);
+ }
+ let list = self.lists.last_mut().expect("list item belongs to a list");
+ let separation = if loose || list.loose { 2 } else { 1 };
+ list.loose |= loose;
+ self.writer().block(&item, separation, separation);
+ }
+ Task::RawChildren(Some(node), depth) => self.raw_node(node, depth),
+ Task::EndCode => {
+ let text = std::mem::take(&mut self.raw);
+ let preformatted = self.options.preformatted_code;
+ self.writer().code_with_edges(&text, preformatted);
+ }
+ Task::EndPre(language) => self.end_pre(language),
+ }
+ }
+ }
+
+ fn writer(&mut self) -> &mut Writer<'a> {
+ self.writers.last_mut().expect("conversion has an output")
+ }
+
+ fn capture(&mut self) {
+ let writer = self.writer().child();
+ self.writers.push(writer);
+ }
+
+ fn take_writer(&mut self) -> String {
+ let writer = self.writers.pop().expect("capture has an output");
+ if let Some(parent) = self.writers.last_mut() {
+ parent.last_link = parent.last_link.max(writer.last_link);
+ }
+ writer.finish()
+ }
+
+ fn children(&mut self, node: D::NodeId, depth: usize) {
+ if depth < self.options.max_depth {
+ self.tasks
+ .push(Task::Children(self.dom.first_child(node), depth));
+ }
+ }
+
+ fn style(&mut self, style: Style<'a>) {
+ if self.writer().push_style(style) {
+ self.tasks.push(Task::PopStyle);
+ }
+ }
+
+ fn visit(&mut self, node: D::NodeId, depth: usize) {
+ if depth >= self.options.max_depth {
+ return;
+ }
+ let tag = match self.dom.node_kind(node) {
+ NodeKind::Document => {
+ self.children(node, depth + 1);
+ return;
+ }
+ NodeKind::Text(text) => {
+ self.writer().text(text);
+ return;
+ }
+ NodeKind::Other => return,
+ NodeKind::Element(tag) => tag,
+ };
+ match tag {
+ "head" | "script" | "style" | "noscript" | "template" => return,
+ "br" => {
+ self.writer().hard_break();
+ return;
+ }
+ "hr" => {
+ self.writer().block("---", 2, 2);
+ return;
+ }
+ "strong" | "b" => self.style(Style::Strong),
+ "em" | "i" => self.style(Style::Emphasis),
+ "del" | "s" | "strike" => self.style(Style::Strike),
+ "a" => {
+ if let Some(href) = self
+ .dom
+ .attribute(node, "href")
+ .filter(|href| !href.is_empty())
+ {
+ self.serial += 1;
+ let serial = self.serial;
+ let style = Style::Link {
+ serial,
+ href,
+ title: self.dom.attribute(node, "title"),
+ };
+ if self.writer().push_style(style) {
+ self.tasks.push(Task::EndLink(serial));
+ }
+ }
+ }
+ "img" => {
+ let alt = self.dom.attribute(node, "alt").unwrap_or_default();
+ let title = self.dom.attribute(node, "title");
+ if let Some(src) = self
+ .dom
+ .attribute(node, "src")
+ .filter(|src| !src.is_empty())
+ {
+ self.writer().image(alt, src, title);
+ }
+ return;
+ }
+ "code" | "pre" => {
+ self.raw.clear();
+ let end = if tag == "pre" {
+ Task::EndPre(self.language(node))
+ } else {
+ Task::EndCode
+ };
+ self.tasks.push(end);
+ if depth + 1 < self.options.max_depth {
+ self.tasks
+ .push(Task::RawChildren(self.dom.first_child(node), depth + 1));
+ }
+ return;
+ }
+ "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
+ self.capture();
+ self.writer().heading();
+ self.tasks
+ .push(Task::EndHeading((tag.as_bytes()[1] - b'0') as usize));
+ }
+ "blockquote" => {
+ self.capture();
+ self.tasks.push(Task::EndQuote);
+ }
+ "ul" | "ol" => self.start_list(node, tag == "ol"),
+ "li" => self.start_item(),
+ _ if is_block(tag) => {
+ self.writer().boundary(2);
+ self.tasks.push(Task::Boundary);
+ }
+ _ => {}
+ }
+ self.children(node, depth + 1);
+ }
+
+ fn start_list(&mut self, node: D::NodeId, ordered: bool) {
+ let before = if self
+ .lists
+ .last()
+ .is_some_and(|list| self.writers.len() == list.output_depth + 1)
+ {
+ 1
+ } else {
+ 2
+ };
+ let next = self
+ .dom
+ .attribute(node, "start")
+ .and_then(|s| s.trim().parse().ok())
+ .unwrap_or(1);
+ self.capture();
+ self.lists.push(List {
+ output_depth: self.writers.len(),
+ ordered,
+ next,
+ loose: false,
+ });
+ self.tasks.push(Task::EndList(before));
+ }
+
+ fn start_item(&mut self) {
+ let Some(list) = self
+ .lists
+ .last_mut()
+ .filter(|list| list.output_depth == self.writers.len())
+ else {
+ self.writer().boundary(2);
+ self.tasks.push(Task::Boundary);
+ return;
+ };
+ let marker = if list.ordered {
+ let marker = format!("{}. ", list.next);
+ list.next = list.next.saturating_add(1);
+ marker
+ } else {
+ "- ".to_owned()
+ };
+ self.capture();
+ self.tasks.push(Task::EndItem(marker));
+ }
+
+ fn language(&self, node: D::NodeId) -> Option<&'a str> {
+ let mut child = self.dom.first_child(node);
+ while let Some(id) = child {
+ if self.dom.node_kind(id) == NodeKind::Element("code")
+ && let Some(language) = class_language(self.dom.attribute(id, "class"))
+ {
+ return Some(language);
+ }
+ child = self.dom.next_sibling(id);
+ }
+ class_language(self.dom.attribute(node, "class"))
+ .or(self.options.default_code_language.as_deref())
+ }
+
+ fn raw_node(&mut self, node: D::NodeId, depth: usize) {
+ self.tasks
+ .push(Task::RawChildren(self.dom.next_sibling(node), depth));
+ if depth >= self.options.max_depth {
+ return;
+ }
+ match self.dom.node_kind(node) {
+ NodeKind::Text(text) => self.raw.push_str(text),
+ NodeKind::Document | NodeKind::Element(_) if depth + 1 < self.options.max_depth => {
+ self.tasks
+ .push(Task::RawChildren(self.dom.first_child(node), depth + 1));
+ }
+ _ => {}
+ }
+ }
+
+ fn end_pre(&mut self, language: Option<&str>) {
+ let text = std::mem::take(&mut self.raw);
+ if text.trim().is_empty() {
+ self.writer().boundary(2);
+ return;
+ }
+ // CommonMark code blocks normalize CRLF/CR and require a closing fence
+ // on its own line. Preserve all other spaces and blank lines verbatim.
+ let text = text.replace("\r\n", "\n").replace('\r', "\n");
+ // The info string occupies one line. Tilde fences also allow language
+ // names containing backticks without changing the name itself.
+ let language = language
+ .unwrap_or_default()
+ .split(['\r', '\n'])
+ .next()
+ .unwrap();
+ let marker = if language.contains('`') { '~' } else { '`' };
+ let fence = marker
+ .to_string()
+ .repeat(3.max(longest_run(&text, marker) + 1));
+ let mut block = fence.clone();
+ for ch in language.chars() {
+ match ch {
+ '\\' => block.push_str("\\\\"),
+ '&' => block.push_str("&amp;"),
+ _ => block.push(ch),
+ }
+ }
+ block.push('\n');
+ block.push_str(&text);
+ if !text.is_empty() && !text.ends_with('\n') {
+ block.push('\n');
+ }
+ block.push_str(&fence);
+ self.writer().block(&block, 2, 2);
+ }
+}
+
+fn class_language(class: Option<&str>) -> Option<&str> {
+ class?
+ .split_ascii_whitespace()
+ .find_map(|part| part.strip_prefix("language-").filter(|s| !s.is_empty()))
+}
+
+fn is_block(tag: &str) -> bool {
+ matches!(
+ tag,
+ "address"
+ | "audio"
+ | "canvas"
+ | "frameset"
+ | "isindex"
+ | "noframes"
+ | "output"
+ | "article"
+ | "aside"
+ | "body"
+ | "caption"
+ | "center"
+ | "dd"
+ | "details"
+ | "dialog"
+ | "dir"
+ | "div"
+ | "dl"
+ | "dt"
+ | "fieldset"
+ | "figcaption"
+ | "figure"
+ | "footer"
+ | "form"
+ | "header"
+ | "hgroup"
+ | "html"
+ | "legend"
+ | "main"
+ | "menu"
+ | "nav"
+ | "p"
+ | "section"
+ | "summary"
+ | "table"
+ | "tbody"
+ | "td"
+ | "tfoot"
+ | "th"
+ | "thead"
+ | "tr"
+ )
+}
diff --git a/moli-html2md/src/options.rs b/moli-html2md/src/options.rs
new file mode 100644
index 000000000..066249363
--- /dev/null
+++ b/moli-html2md/src/options.rs
@@ -0,0 +1,23 @@
+/// Conversion limits and the fallback language for fenced code blocks.
+#[derive(Clone, Debug)]
+pub struct Options {
+ /// Visit nodes at depths `0..max_depth`, with the supplied root at depth 0.
+ /// Deeper subtrees are omitted. Set to `usize::MAX` to disable this limit.
+ pub max_depth: usize,
+ /// Used when neither `<pre>` nor its `<code>` child declares a language.
+ /// The hint ends at the first line ending, as Markdown info strings use one line.
+ pub default_code_language: Option<String>,
+ /// Preserve whitespace inside inline code, replacing line endings with spaces.
+ /// When false, inline code follows ordinary HTML whitespace collapsing.
+ pub preformatted_code: bool,
+}
+
+impl Default for Options {
+ fn default() -> Self {
+ Self {
+ max_depth: 1000,
+ default_code_language: None,
+ preformatted_code: false,
+ }
+ }
+}
diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rs
new file mode 100644
index 000000000..3ee7c0f92
--- /dev/null
+++ b/moli-html2md/src/writer.rs
@@ -0,0 +1,464 @@
+// Formatting is delayed until visible content arrives. This keeps whitespace
+// outside newly opened/closed marks and coalesces adjacent identical styles.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum Style<'a> {
+ Strong,
+ Emphasis,
+ Strike,
+ Link {
+ // Distinguish adjacent links even when their destinations match.
+ serial: usize,
+ href: &'a str,
+ title: Option<&'a str>,
+ },
+}
+
+struct OpenStyle<'a> {
+ style: Style<'a>,
+ start: usize,
+ html: bool,
+}
+
+#[derive(Default)]
+pub(crate) struct Writer<'a> {
+ output: String,
+ desired: Vec<Style<'a>>,
+ emitted: Vec<OpenStyle<'a>>,
+ space: bool,
+ preserved_spaces: String,
+ breaks: usize,
+ code: Option<String>,
+ line_digits: Option<usize>,
+ heading: bool,
+ single_line_attributes: bool,
+ pub(crate) last_link: usize,
+ // Block events in this output buffer, used when finalizing a list item.
+ pub(crate) has_blocks: bool,
+}
+
+pub(crate) fn is_space(ch: char) -> bool {
+ matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{000c}')
+}
+
+impl<'a> Writer<'a> {
+ pub(crate) fn child(&self) -> Self {
+ Self {
+ desired: self.desired.clone(),
+ single_line_attributes: self.single_line_attributes,
+ ..Self::default()
+ }
+ }
+
+ pub(crate) fn heading(&mut self) {
+ self.heading = true;
+ self.single_line_attributes = true;
+ }
+
+ pub(crate) fn push_style(&mut self, style: Style<'a>) -> bool {
+ // Repeated emphasis has the same visible meaning. Nested links cannot
+ // be represented in Markdown, so retain the outer destination.
+ if self.desired.contains(&style)
+ || matches!(style, Style::Link { .. })
+ && self.desired.iter().any(|s| matches!(s, Style::Link { .. }))
+ {
+ return false;
+ }
+ self.flush_code();
+ self.desired.push(style);
+ true
+ }
+
+ pub(crate) fn pop_style(&mut self) {
+ self.flush_code();
+ self.desired.pop();
+ }
+
+ pub(crate) fn end_link(&mut self, serial: usize) {
+ self.flush_code();
+ if self.last_link < serial {
+ // Empty anchors still carry a destination. Materialize the link
+ // after its leading whitespace, even without a text event.
+ self.prepare_inline('[');
+ }
+ self.pop_style();
+ }
+
+ pub(crate) fn text(&mut self, text: &str) {
+ self.flush_code();
+ for ch in text.chars() {
+ if is_space(ch) {
+ self.space = true;
+ continue;
+ }
+ if ch.is_whitespace() {
+ // NBSP and other Unicode spaces remain visible characters,
+ // but Markdown emphasis delimiters cannot touch whitespace.
+ if self.space {
+ self.preserved_spaces.push(' ');
+ self.space = false;
+ }
+ self.preserved_spaces.push(ch);
+ continue;
+ }
+ self.prepare_inline(ch);
+ let line_start = self.output.is_empty() || self.output.ends_with('\n');
+ if line_start {
+ self.line_digits = Some(0);
+ }
+ match ch {
+ '\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '~' => {
+ self.output.push('\\');
+ self.output.push(ch);
+ }
+ '#' if line_start || self.heading => {
+ self.output.push('\\');
+ self.output.push(ch);
+ }
+ '+' | '-' | '=' if line_start => {
+ self.output.push('\\');
+ self.output.push(ch);
+ }
+ '.' | ')' if self.line_digits.is_some_and(|digits| digits > 0) => {
+ self.output.push('\\');
+ self.output.push(ch);
+ }
+ '&' => self.output.push_str("&amp;"),
+ _ => self.output.push(ch),
+ }
+ self.line_digits = if ch.is_ascii_digit() {
+ self.line_digits.map(|n| n + 1)
+ } else {
+ None
+ };
+ }
+ }
+
+ pub(crate) fn code(&mut self, text: &str) {
+ if text.starts_with(is_space) {
+ self.flush_code();
+ self.space = true;
+ }
+ let text = text.trim_matches(is_space);
+ if !text.is_empty() {
+ if self.code.is_none() {
+ self.prepare_inline('`');
+ self.code = Some(String::new());
+ }
+ let code = self.code.as_mut().expect("code buffer was initialized");
+ let mut space = false;
+ for ch in text.chars() {
+ if is_space(ch) {
+ space = true;
+ } else {
+ if space {
+ code.push(' ');
+ space = false;
+ }
+ code.push(ch);
+ }
+ }
+ }
+ }
+
+ pub(crate) fn code_with_edges(&mut self, text: &str, preformatted: bool) {
+ if preformatted {
+ if !text.is_empty() {
+ if self.code.is_none() {
+ self.prepare_inline('`');
+ self.code = Some(String::new());
+ }
+ self.code
+ .as_mut()
+ .expect("initialized code buffer")
+ .push_str(&text.replace("\r\n", " ").replace(['\n', '\r'], " "));
+ }
+ return;
+ }
+ let content = text.trim_matches(char::is_whitespace);
+ if content.is_empty() {
+ self.text(text);
+ return;
+ }
+ let leading = text.len() - text.trim_start_matches(char::is_whitespace).len();
+ let trailing = text.trim_end_matches(char::is_whitespace).len();
+ if leading > 0 {
+ self.text(&text[..leading]);
+ }
+ self.code(content);
+ if trailing < text.len() {
+ self.flush_code();
+ self.text(&text[trailing..]);
+ }
+ }
+
+ pub(crate) fn image(&mut self, alt: &str, src: &str, title: Option<&str>) {
+ self.flush_code();
+ self.prepare_inline('!');
+ self.output.push_str("![");
+ // Image labels are literal text, not a second Markdown input.
+ for ch in clean_attribute(alt) {
+ match ch {
+ '\\' | '[' | ']' | '*' | '_' | '`' | '<' | '>' => {
+ self.output.push('\\');
+ self.output.push(ch);
+ }
+ '&' => self.output.push_str("&amp;"),
+ // Markdown image labels render soft breaks as spaces. Keep
+ // that meaning when a heading flattens its content.
+ '\n' if self.single_line_attributes => self.output.push(' '),
+ _ => self.output.push(ch),
+ }
+ }
+ self.output.push_str("](");
+ destination(&mut self.output, src, title, self.single_line_attributes);
+ self.output.push(')');
+ self.line_digits = None;
+ }
+
+ pub(crate) fn boundary(&mut self, lines: usize) {
+ self.has_blocks |= lines > 1;
+ self.flush_code();
+ self.close_to(0, None);
+ self.flush_spaces(false);
+ self.space = false;
+ self.breaks = self.breaks.max(lines);
+ }
+
+ pub(crate) fn hard_break(&mut self) {
+ self.flush_code();
+ self.close_to(0, None);
+ self.flush_spaces(false);
+ self.flush_breaks();
+ self.output.push_str(" \n");
+ self.space = false;
+ self.line_digits = Some(0);
+ }
+
+ pub(crate) fn block(&mut self, text: &str, before: usize, after: usize) {
+ if text.is_empty() {
+ self.boundary(before.max(after));
+ return;
+ }
+ self.boundary(before);
+ self.flush_breaks();
+ self.output.push_str(text);
+ self.line_digits = None;
+ self.boundary(after);
+ }
+
+ pub(crate) fn finish(mut self) -> String {
+ self.flush_code();
+ self.close_to(0, None);
+ self.flush_spaces(false);
+ let end = self.output.trim_end_matches(is_space).len();
+ self.output.truncate(end);
+ self.output
+ }
+
+ fn prepare_inline(&mut self, next: char) {
+ self.flush_breaks();
+ let common = self
+ .emitted
+ .iter()
+ .zip(&self.desired)
+ .take_while(|(a, b)| a.style == **b)
+ .count();
+ // A new link emits '[' before its label. Include that character when
+ // checking the boundaries of surrounding emphasis, even though style
+ // emission is delayed until the first character of the label arrives.
+ let link = self.desired[common..]
+ .iter()
+ .position(|style| matches!(style, Style::Link { .. }))
+ .map(|index| common + index);
+ let next_after_close = if self.space || !self.preserved_spaces.is_empty() {
+ ' '
+ } else if link.is_some() {
+ '['
+ } else {
+ next
+ };
+ self.close_to(common, Some(next_after_close));
+ self.flush_spaces(true);
+ for (index, &style) in self.desired.iter().enumerate().skip(common) {
+ let start = self.output.len();
+ // An opening emphasis delimiter between a word and punctuation
+ // cannot open a CommonMark span. Inline HTML preserves its meaning.
+ let first = if link.is_some_and(|link| index < link) {
+ '['
+ } else {
+ next
+ };
+ let html = is_punctuation(first)
+ && self
+ .output
+ .chars()
+ .next_back()
+ .is_some_and(char::is_alphanumeric);
+ match style {
+ Style::Strong => self.output.push_str(if html { "<strong>" } else { "**" }),
+ Style::Emphasis => self.output.push_str(if html { "<em>" } else { "*" }),
+ Style::Strike => self.output.push_str(if html { "<del>" } else { "~~" }),
+ Style::Link { serial, .. } => {
+ self.last_link = self.last_link.max(serial);
+ if self.output.ends_with('!') {
+ self.output.insert(self.output.len() - 1, '\\');
+ }
+ self.output.push('[');
+ }
+ }
+ self.emitted.push(OpenStyle { style, start, html });
+ }
+ }
+
+ fn flush_spaces(&mut self, trailing_ascii: bool) {
+ if !self.preserved_spaces.is_empty() {
+ // Materialize any block boundary before preserving Unicode space.
+ self.flush_breaks();
+ let text = if self.output.is_empty() || self.output.ends_with('\n') {
+ self.preserved_spaces.trim_start_matches(' ')
+ } else {
+ &self.preserved_spaces
+ };
+ self.output.push_str(text);
+ self.preserved_spaces.clear();
+ self.line_digits = None;
+ }
+ if trailing_ascii && self.space && !self.output.is_empty() && !self.output.ends_with('\n') {
+ self.output.push(' ');
+ self.line_digits = None;
+ }
+ self.space = false;
+ }
+
+ fn close_to(&mut self, count: usize, next: Option<char>) {
+ while self.emitted.len() > count {
+ let opened = self.emitted.pop().expect("nonempty style stack");
+ let closing_needs_html = next.is_some_and(char::is_alphanumeric)
+ && self.output.chars().next_back().is_some_and(is_punctuation);
+ let delimiters = match opened.style {
+ Style::Strong => Some(("**", "<strong>", "</strong>")),
+ Style::Emphasis => Some(("*", "<em>", "</em>")),
+ Style::Strike => Some(("~~", "<del>", "</del>")),
+ Style::Link { href, title, .. } => {
+ self.output.push_str("](");
+ destination(&mut self.output, href, title, self.single_line_attributes);
+ self.output.push(')');
+ None
+ }
+ };
+ if let Some((marker, open, close)) = delimiters {
+ if opened.html {
+ self.output.push_str(close);
+ } else if closing_needs_html {
+ // Only the converter's output changes. Remaining open
+ // ancestors precede this offset, so their offsets stay valid.
+ self.output
+ .replace_range(opened.start..opened.start + marker.len(), open);
+ self.output.push_str(close);
+ } else {
+ self.output.push_str(marker);
+ }
+ }
+ }
+ }
+
+ fn flush_breaks(&mut self) {
+ if self.breaks > 0 {
+ if !self.output.is_empty() {
+ let existing = self.output.len() - self.output.trim_end_matches('\n').len();
+ for _ in existing..self.breaks {
+ self.output.push('\n');
+ }
+ }
+ self.line_digits = Some(0);
+ self.breaks = 0;
+ }
+ }
+
+ fn flush_code(&mut self) {
+ if let Some(code) = self.code.take() {
+ let fence = "`".repeat(longest_run(&code, '`') + 1);
+ let padding = code.starts_with('`')
+ || code.ends_with('`')
+ || code.starts_with(' ') && code.ends_with(' ') && code.chars().any(|ch| ch != ' ');
+ self.output.push_str(&fence);
+ if padding {
+ self.output.push(' ');
+ }
+ self.output.push_str(&code);
+ if padding {
+ self.output.push(' ');
+ }
+ self.output.push_str(&fence);
+ self.line_digits = None;
+ }
+ }
+}
+
+fn is_punctuation(ch: char) -> bool {
+ // Conservatively include Unicode symbols and combining marks; choosing
+ // inline HTML for these edges is preferable to emitting literal delimiters.
+ !ch.is_alphanumeric() && !ch.is_whitespace()
+}
+
+pub(crate) fn longest_run(text: &str, marker: char) -> usize {
+ let mut longest = 0;
+ let mut current = 0;
+ for ch in text.chars() {
+ if ch == marker {
+ current += 1;
+ longest = longest.max(current);
+ } else {
+ current = 0;
+ }
+ }
+ longest
+}
+
+fn destination(output: &mut String, href: &str, title: Option<&str>, single_line: bool) {
+ for ch in href.chars() {
+ match ch {
+ '\\' | '(' | ')' => {
+ output.push('\\');
+ output.push(ch);
+ }
+ ' ' => output.push_str("%20"),
+ '\t' => output.push_str("%09"),
+ '\r' => output.push_str("%0D"),
+ '\n' => output.push_str("%0A"),
+ '<' => output.push_str("%3C"),
+ '>' => output.push_str("%3E"),
+ '&' => output.push_str("&amp;"),
+ _ => output.push(ch),
+ }
+ }
+ if let Some(title) = title.filter(|title| !title.is_empty()) {
+ output.push_str(" \"");
+ for ch in clean_attribute(title) {
+ match ch {
+ '\\' | '"' => {
+ output.push('\\');
+ output.push(ch);
+ }
+ '&' => output.push_str("&amp;"),
+ // Protect title newlines from heading line flattening.
+ '\n' if single_line => output.push_str("&#10;"),
+ _ => output.push(ch),
+ }
+ }
+ output.push('"');
+ }
+}
+
+// Attribute newlines are significant in labels and titles; indentation after a
+// newline is not. Collapse each newline/whitespace run without flattening it.
+fn clean_attribute(value: &str) -> impl Iterator<Item = char> + '_ {
+ let mut after_newline = false;
+ value.chars().filter(move |&ch| {
+ if after_newline && ch.is_whitespace() {
+ return false;
+ }
+ after_newline = ch == '\n';
+ true
+ })
+}
diff --git a/moli-html2md/tests/conversion.rs b/moli-html2md/tests/conversion.rs
new file mode 100644
index 000000000..ed76b79b0
--- /dev/null
+++ b/moli-html2md/tests/conversion.rs
@@ -0,0 +1,424 @@
+use moli_html2md::{Converter, Dom, NodeKind, Options, convert};
+
+// A second DOM implementation, independent of NativeDom or an HTML parser.
+// Its stable arena IDs and borrowed strings are enough for the converter.
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct Node {
+ kind: NodeKind<'static>,
+ attrs: Vec<(&'static str, &'static str)>,
+ first: Option<usize>,
+ last: Option<usize>,
+ next: Option<usize>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct Tree(Vec<Node>);
+
+impl Tree {
+ fn new() -> Self {
+ Self(vec![Node {
+ kind: NodeKind::Document,
+ attrs: Vec::new(),
+ first: None,
+ last: None,
+ next: None,
+ }])
+ }
+
+ fn add(&mut self, parent: usize, kind: NodeKind<'static>) -> usize {
+ let id = self.0.len();
+ self.0.push(Node {
+ kind,
+ attrs: Vec::new(),
+ first: None,
+ last: None,
+ next: None,
+ });
+ if let Some(previous) = self.0[parent].last {
+ self.0[previous].next = Some(id);
+ } else {
+ self.0[parent].first = Some(id);
+ }
+ self.0[parent].last = Some(id);
+ id
+ }
+
+ fn element(&mut self, parent: usize, tag: &'static str) -> usize {
+ self.add(parent, NodeKind::Element(tag))
+ }
+
+ fn text(&mut self, parent: usize, text: &'static str) {
+ self.add(parent, NodeKind::Text(text));
+ }
+
+ fn attr(&mut self, node: usize, name: &'static str, value: &'static str) {
+ self.0[node].attrs.push((name, value));
+ }
+
+ fn leaf(&mut self, parent: usize, tag: &'static str, text: &'static str) -> usize {
+ let element = self.element(parent, tag);
+ self.text(element, text);
+ element
+ }
+}
+
+impl Dom for Tree {
+ type NodeId = usize;
+
+ fn node_kind(&self, node: usize) -> NodeKind<'_> {
+ self.0[node].kind
+ }
+
+ fn first_child(&self, node: usize) -> Option<usize> {
+ self.0[node].first
+ }
+
+ fn next_sibling(&self, node: usize) -> Option<usize> {
+ self.0[node].next
+ }
+
+ fn attribute(&self, node: usize, name: &str) -> Option<&str> {
+ self.0[node]
+ .attrs
+ .iter()
+ .find_map(|(key, value)| (*key == name).then_some(*value))
+ }
+}
+
+#[test]
+fn coalesces_styles_without_changing_the_tree() {
+ let mut dom = Tree::new();
+ dom.leaf(0, "em", "foo");
+ dom.leaf(0, "i", "bar");
+ dom.text(0, " ");
+ let bold = dom.element(0, "strong");
+ dom.leaf(bold, "b", "nested");
+ dom.leaf(0, "strong", "!");
+ let before = dom.clone();
+ let converter = Converter::default();
+ for _ in 0..2 {
+ assert_eq!(converter.convert(&dom, 0), "*foobar* **nested!**");
+ assert_eq!(dom, before);
+ }
+}
+
+#[test]
+fn moves_whitespace_outside_formatting_but_keeps_nonbreaking_spaces() {
+ let mut dom = Tree::new();
+ dom.text(0, "by");
+ dom.leaf(0, "strong", " \t");
+ dom.leaf(0, "small", "Albert Einstein");
+ dom.text(0, "\u{a0}");
+ let em = dom.element(0, "em");
+ dom.leaf(em, "strong", " next ");
+ dom.text(0, "word");
+ assert_eq!(convert(&dom, 0), "by Albert Einstein\u{a0} ***next*** word");
+}
+
+#[test]
+fn keeps_separate_links_and_escapes_destinations_and_titles() {
+ let mut dom = Tree::new();
+ for label in ["one", "two"] {
+ let link = dom.leaf(0, "a", label);
+ dom.attr(link, "href", "/a(b) c");
+ dom.attr(link, "title", "a \"title\"");
+ }
+ assert_eq!(
+ convert(&dom, 0),
+ "[one](/a\\(b\\)%20c \"a \\\"title\\\"\")[two](/a\\(b\\)%20c \"a \\\"title\\\"\")"
+ );
+}
+
+#[test]
+fn escapes_literal_markdown_and_decoded_entities_across_text_nodes() {
+ let mut dom = Tree::new();
+ let p = dom.element(0, "p");
+ dom.text(p, "12");
+ dom.text(p, ". [x] *y* _z_ `a` <tag> &copy;");
+ dom.leaf(0, "p", "# heading");
+ dom.leaf(0, "p", "- item");
+ dom.text(0, "!");
+ let link = dom.leaf(0, "a", "link");
+ dom.attr(link, "href", "/");
+ assert_eq!(
+ convert(&dom, 0),
+ "12\\. \\[x\\] \\*y\\* \\_z\\_ \\`a\\` \\<tag\\> &amp;copy;\n\n\\# heading\n\n\\- item\n\n\\![link](/)"
+ );
+}
+
+#[test]
+fn combines_adjacent_code_and_chooses_safe_delimiters() {
+ let mut dom = Tree::new();
+ dom.leaf(0, "code", "a`");
+ dom.leaf(0, "code", "`b");
+ dom.text(0, " ");
+ dom.leaf(0, "code", " ` ");
+ dom.text(0, "after");
+ assert_eq!(convert(&dom, 0), "```a``b``` `` ` `` after");
+}
+
+#[test]
+fn preserves_preformatted_text_including_blank_lines_and_nested_elements() {
+ let mut dom = Tree::new();
+ let pre = dom.element(0, "pre");
+ let code = dom.element(pre, "code");
+ dom.attr(code, "class", "highlight language-rust");
+ dom.text(code, " first\r\n\r\n");
+ dom.leaf(code, "span", "```\n");
+ dom.text(code, "\n");
+ assert_eq!(convert(&dom, 0), "````rust\n first\n\n```\n\n````");
+}
+
+#[test]
+fn prefixes_multiline_list_items_and_quotes() {
+ let mut dom = Tree::new();
+ let quote = dom.element(0, "blockquote");
+ let list = dom.element(quote, "ol");
+ dom.attr(list, "start", "9");
+ let item = dom.element(list, "li");
+ dom.leaf(item, "p", "first");
+ dom.leaf(item, "p", "second");
+ dom.leaf(list, "li", "last");
+ assert_eq!(
+ convert(&dom, 0),
+ "> 9. first\n>\n> second\n>\n> 10. last"
+ );
+}
+
+#[test]
+fn supports_headings_images_and_non_content_elements() {
+ let mut dom = Tree::new();
+ dom.leaf(0, "h2", "Heading");
+ let image = dom.element(0, "img");
+ dom.attr(image, "src", "/image.png");
+ dom.attr(image, "alt", "[image]");
+ dom.element(0, "hr");
+ for tag in ["head", "script", "style", "noscript", "template"] {
+ dom.leaf(0, tag, "invisible");
+ }
+ dom.add(0, NodeKind::Other);
+ dom.leaf(0, "p", "visible");
+ assert_eq!(
+ convert(&dom, 0),
+ "## Heading\n\n![\\[image\\]](/image.png)\n\n---\n\nvisible"
+ );
+}
+
+#[test]
+fn expands_headerless_table_cells_as_blocks() {
+ let mut dom = Tree::new();
+ let table = dom.element(0, "table");
+ let row = dom.element(table, "tr");
+ dom.leaf(row, "td", "one");
+ dom.leaf(row, "td", "two");
+ assert_eq!(convert(&dom, 0), "one\n\ntwo");
+}
+
+#[test]
+fn preserves_table_cell_boundaries_hard_breaks_and_literal_pipes() {
+ let mut dom = Tree::new();
+ let table = dom.element(0, "table");
+ let header = dom.element(table, "tr");
+ let cell = dom.leaf(header, "th", "Key");
+ dom.attr(cell, "align", "right");
+ let row = dom.element(table, "tr");
+ let cell = dom.element(row, "td");
+ dom.text(cell, "a|b");
+ dom.element(cell, "br");
+ dom.text(cell, "c");
+ assert_eq!(convert(&dom, 0), "Key\n\na|b \nc");
+}
+
+#[test]
+fn table_spans_do_not_duplicate_cell_text() {
+ let mut dom = Tree::new();
+ let table = dom.element(0, "table");
+ let row = dom.element(table, "tr");
+ let cell = dom.leaf(row, "td", "one");
+ dom.attr(cell, "rowspan", "2");
+ dom.leaf(row, "td", "two");
+ let row = dom.element(table, "tr");
+ let cell = dom.leaf(row, "td", "end");
+ dom.attr(cell, "colspan", "999999999999");
+ assert_eq!(convert(&dom, 0), "one\n\ntwo\n\nend");
+}
+
+#[test]
+fn bounds_every_traversal_including_raw_code() {
+ for tag in [
+ "div",
+ "span",
+ "strong",
+ "blockquote",
+ "table",
+ "pre",
+ "code",
+ ] {
+ let mut dom = Tree::new();
+ dom.text(0, "visible");
+ let mut node = 0;
+ for _ in 0..30 {
+ node = dom.element(node, tag);
+ }
+ dom.text(node, "too deep");
+ let converter = Converter::new(Options {
+ max_depth: 10,
+ ..Options::default()
+ });
+ let result = converter.convert(&dom, 0);
+ assert!(!result.contains("too deep"), "{tag}: {result}");
+ assert!(result.starts_with("visible"), "{tag}: {result}");
+ }
+}
+
+#[test]
+fn converts_deep_trees_on_a_small_thread_stack() {
+ std::thread::Builder::new()
+ .stack_size(64 * 1024)
+ .spawn(|| {
+ for tag in ["span", "strong", "pre"] {
+ let mut dom = Tree::new();
+ let mut node = 0;
+ for _ in 0..20_000 {
+ node = dom.element(node, tag);
+ }
+ dom.text(node, "deep");
+ let converter = Converter::new(Options {
+ max_depth: usize::MAX,
+ ..Options::default()
+ });
+ let expected = match tag {
+ "strong" => "**deep**",
+ "pre" => "```\ndeep\n```",
+ _ => "deep",
+ };
+ assert_eq!(converter.convert(&dom, 0), expected);
+ }
+ })
+ .expect("spawn small-stack test")
+ .join()
+ .expect("iterative conversion should complete");
+}
+
+#[test]
+fn converts_deep_tables_on_a_small_thread_stack() {
+ std::thread::Builder::new()
+ .stack_size(64 * 1024)
+ .spawn(|| {
+ let mut dom = Tree::new();
+ let mut node = 0;
+ for _ in 0..10_000 {
+ node = dom.element(node, "table");
+ node = dom.element(node, "tr");
+ node = dom.element(node, "td");
+ }
+ dom.text(node, "deep");
+ let converter = Converter::new(Options {
+ max_depth: usize::MAX,
+ ..Options::default()
+ });
+ assert_eq!(converter.convert(&dom, 0), "deep");
+ })
+ .expect("spawn small-stack test")
+ .join()
+ .expect("nested tables should not recurse");
+}
+
+#[test]
+fn conversion_is_limited_to_the_supplied_subtree() {
+ let mut dom = Tree::new();
+ let first = dom.leaf(0, "p", "first");
+ dom.leaf(0, "p", "second");
+ assert_eq!(convert(&dom, first), "first");
+ let converter = Converter::new(Options {
+ max_depth: 0,
+ ..Options::default()
+ });
+ assert_eq!(converter.convert(&dom, first), "");
+}
+
+fn rendered_html(markdown: &str) -> String {
+ let parser = pulldown_cmark::Parser::new_ext(
+ markdown,
+ pulldown_cmark::Options::ENABLE_TABLES | pulldown_cmark::Options::ENABLE_STRIKETHROUGH,
+ );
+ let mut html = String::new();
+ pulldown_cmark::html::push_html(&mut html, parser);
+ html
+}
+
+#[test]
+fn markdown_parser_recovers_adjacent_and_nested_formatting() {
+ let mut dom = Tree::new();
+ dom.leaf(0, "em", "foo");
+ dom.leaf(0, "em", "bar");
+ dom.text(0, " ");
+ let strong = dom.element(0, "strong");
+ dom.leaf(strong, "em", "both");
+ dom.text(0, " ");
+ dom.leaf(0, "del", "deleted");
+ assert_eq!(
+ rendered_html(&convert(&dom, 0)),
+ "<p><em>foobar</em> <em><strong>both</strong></em> <del>deleted</del></p>\n"
+ );
+}
+
+#[test]
+fn markdown_parser_recovers_unicode_spaces_at_emphasis_boundaries() {
+ let mut dom = Tree::new();
+ dom.leaf(0, "em", "\u{a0} a \u{2003}");
+ dom.text(0, "b");
+ assert_eq!(
+ rendered_html(&convert(&dom, 0)),
+ "<p>\u{a0} <em>a</em> \u{2003}b</p>\n"
+ );
+ let mut dom = Tree::new();
+ dom.leaf(0, "p", "\u{a0} a");
+ assert_eq!(convert(&dom, 0), "\u{a0} a");
+}
+
+#[test]
+fn markdown_parser_recovers_intraword_emphasis_next_to_punctuation() {
+ for (tag, html_tag) in [("strong", "strong"), ("em", "em"), ("del", "del")] {
+ for text in ["(word)", "word!", "!word", "日本語。"] {
+ let mut dom = Tree::new();
+ dom.text(0, "before");
+ dom.leaf(0, tag, text);
+ dom.text(0, "after");
+ let markdown = convert(&dom, 0);
+ assert_eq!(
+ rendered_html(&markdown),
+ format!("<p>before<{html_tag}>{text}</{html_tag}>after</p>\n"),
+ "{markdown}"
+ );
+ }
+ }
+}
+
+#[test]
+fn markdown_parser_recovers_literal_hashes_at_the_end_of_headings() {
+ let mut dom = Tree::new();
+ dom.leaf(0, "h1", "title #");
+ assert_eq!(rendered_html(&convert(&dom, 0)), "<h1>title #</h1>\n");
+}
+
+#[test]
+fn markdown_parser_recovers_pipes_and_preformatted_code_inside_tables() {
+ let mut dom = Tree::new();
+ let table = dom.element(0, "table");
+ let header = dom.element(table, "tr");
+ dom.leaf(header, "th", "Code");
+ let row = dom.element(table, "tr");
+ let cell = dom.element(row, "td");
+ dom.leaf(cell, "code", "a|b");
+ let row = dom.element(table, "tr");
+ let cell = dom.element(row, "td");
+ dom.leaf(cell, "pre", " <a>|b\nnext\n");
+ let markdown = convert(&dom, 0);
+ let html = rendered_html(&markdown);
+ assert_eq!(
+ html,
+ "<p>Code</p>\n<p><code>a|b</code></p>\n<pre><code> &lt;a&gt;|b\nnext\n</code></pre>\n",
+ "{markdown}"
+ );
+}
diff --git a/moli-html2md/tests/fixtures/hacker-news-layout.html b/moli-html2md/tests/fixtures/hacker-news-layout.html
new file mode 100644
index 000000000..6c2c54efc
--- /dev/null
+++ b/moli-html2md/tests/fixtures/hacker-news-layout.html
@@ -0,0 +1,22 @@
+<!doctype html>
+<!-- Reduced from https://news.ycombinator.com/news: layout tables, spacer
+ rows, and colspan metadata rows. Story text and destinations are synthetic. -->
+<html><body>
+<table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">
+ <tr><td><table border="0" width="100%"><tr>
+ <td><b><a href="news">News</a></b></td>
+ <td><a href="newest">new</a> | <a href="front">past</a></td>
+ <td><a href="login?goto=news">login</a></td>
+ </tr></table></td></tr>
+ <tr style="height:10px"></tr>
+ <tr><td><table border="0" cellpadding="0" cellspacing="0">
+ <tr class="athing"><td align="right">1.</td><td><center><a href="vote?id=1&amp;how=up"><div class="votearrow"></div></a></center></td><td><a href="https://example.test/first">First story</a> (<a href="from?site=example.test">example.test</a>)</td></tr>
+ <tr><td colspan="2"></td><td>42 points by <a href="user?id=alice">alice</a> | <a href="item?id=1">8 comments</a></td></tr>
+ <tr class="spacer" style="height:5px"></tr>
+ <tr class="athing"><td align="right">2.</td><td><center><a href="vote?id=2&amp;how=up"><div class="votearrow"></div></a></center></td><td><a href="https://example.test/second">Second story</a></td></tr>
+ <tr><td colspan="2"></td><td>7 points by <a href="user?id=bob">bob</a> | <a href="item?id=2">discuss</a></td></tr>
+ <tr class="spacer" style="height:5px"></tr>
+ <tr><td colspan="2"></td><td><a href="news?p=2">More</a></td></tr>
+ </table></td></tr>
+</table>
+</body></html>
diff --git a/moli-html2md/tests/fixtures/hacker-news-layout.md b/moli-html2md/tests/fixtures/hacker-news-layout.md
new file mode 100644
index 000000000..7abae5463
--- /dev/null
+++ b/moli-html2md/tests/fixtures/hacker-news-layout.md
@@ -0,0 +1,23 @@
+**[News](news)**
+
+[new](newest) | [past](front)
+
+[login](login?goto=news)
+
+1\.
+
+[](vote?id=1&amp;how=up)
+
+[First story](https://example.test/first) ([example.test](from?site=example.test))
+
+42 points by [alice](user?id=alice) | [8 comments](item?id=1)
+
+2\.
+
+[](vote?id=2&amp;how=up)
+
+[Second story](https://example.test/second)
+
+7 points by [bob](user?id=bob) | [discuss](item?id=2)
+
+[More](news?p=2)
diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs
new file mode 100644
index 000000000..b36ea76f1
--- /dev/null
+++ b/moli-html2md/tests/regressions.rs
@@ -0,0 +1,261 @@
+mod support;
+
+use moli_html2md::{Converter, Options};
+use support::{Tree, rendered_html};
+
+fn markdown(html: &str, preformatted_code: bool) -> String {
+ let dom = Tree::parse(html);
+ Converter::new(Options {
+ preformatted_code,
+ ..Options::default()
+ })
+ .convert(&dom, dom.root)
+}
+
+#[test]
+fn empty_blocks_separate_text_even_when_they_contain_only_whitespace() {
+ for tag in [
+ "p",
+ "div",
+ "blockquote",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "ul",
+ "ol",
+ "pre",
+ "table",
+ ] {
+ for contents in ["", " \n\t "] {
+ let html = format!("before<{tag}>{contents}</{tag}>after");
+ assert_eq!(markdown(&html, false), "before\n\nafter", "{html}");
+ }
+ }
+}
+
+#[test]
+fn empty_links_keep_destinations_across_adjacent_and_nested_elements() {
+ for (html, expected) in [
+ ("<a href='/a'></a><a href='/b'></a>", "[](/a)[](/b)"),
+ (
+ "before<a href='/a'><em> </em></a>after",
+ "before [](/a)after",
+ ),
+ ("<strong><a href='/a'></a></strong>", "**[](/a)**"),
+ ("<a href='/a'><h2>heading</h2></a>", "## [heading](/a)"),
+ (
+ "<a href='/a'><blockquote>quote</blockquote></a>",
+ "> [quote](/a)",
+ ),
+ ("<a href='/a'><ul><li>item</li></ul></a>", "- [item](/a)"),
+ ("<a href='/a'><img src='/image'></a>", "[![](/image)](/a)"),
+ ] {
+ assert_eq!(markdown(html, false), expected, "{html}");
+ }
+}
+
+#[test]
+fn href_without_a_destination_does_not_create_an_empty_markdown_link() {
+ for html in [
+ "<a>label</a>",
+ "<a id='target'>label</a>",
+ "<a href=''>label</a>",
+ ] {
+ assert_eq!(markdown(html, false), "label");
+ }
+}
+
+#[test]
+fn reusable_converter_does_not_leak_link_state_between_documents() {
+ let converter = Converter::default();
+ for _ in 0..3 {
+ for (html, expected) in [
+ ("<a href='/one'>one</a>", "[one](/one)"),
+ ("<a href='/two'></a>", "[](/two)"),
+ ] {
+ let dom = Tree::parse(html);
+ assert_eq!(converter.convert(&dom, dom.root), expected);
+ }
+ }
+}
+
+#[test]
+fn preformatted_code_keeps_leading_trailing_and_internal_spaces() {
+ for (html, expected, expected_html) in [
+ (
+ "<code> first</code>",
+ "` first`",
+ "<p><code> first</code></p>\n",
+ ),
+ (
+ "<code>last </code>",
+ "`last `",
+ "<p><code>last </code></p>\n",
+ ),
+ (
+ "<code> a b </code>",
+ "` a b `",
+ "<p><code> a b </code></p>\n",
+ ),
+ ("<code> </code>", "` `", "<p><code> </code></p>\n"),
+ (
+ "<code>\tword\t</code>",
+ "`\tword\t`",
+ "<p><code>\tword\t</code></p>\n",
+ ),
+ (
+ "<code>` a `</code>",
+ "`` ` a ` ``",
+ "<p><code>` a `</code></p>\n",
+ ),
+ ] {
+ let result = markdown(html, true);
+ assert_eq!(result, expected, "{html}");
+ assert_eq!(rendered_html(&result), expected_html, "{html}");
+ }
+}
+
+#[test]
+fn preformatted_code_normalizes_line_endings_without_collapsing_spaces() {
+ let html = "<code> a\r\n b\r c\n d </code>";
+ let actual = markdown(html, true);
+ assert_eq!(
+ rendered_html(&actual),
+ "<p><code> a b c d </code></p>\n"
+ );
+ assert_eq!(markdown(html, false), "`a b c d`");
+}
+
+#[test]
+fn normal_code_moves_unicode_boundary_spaces_outside_delimiters() {
+ for space in ['\u{00a0}', '\u{2003}', '\u{2009}', '\u{202f}'] {
+ let html = format!("before<code>{space}word{space}</code>after");
+ assert_eq!(
+ markdown(&html, false),
+ format!("before{space}`word`{space}after")
+ );
+ assert_eq!(
+ rendered_html(&markdown(&html, true)),
+ format!("<p>before<code>{space}word{space}</code>after</p>\n")
+ );
+ }
+}
+
+#[test]
+fn attribute_newlines_remove_indentation_without_joining_words() {
+ for separator in ["\n ", "\r\n\t", "\n \n \t "] {
+ let html = format!("<a href='/a' title='first{separator}second'>link</a>");
+ assert_eq!(markdown(&html, false), "[link](/a \"first\nsecond\")");
+ let html =
+ format!("<img src='/i' alt='first{separator}second' title='first{separator}second'>");
+ assert_eq!(
+ markdown(&html, false),
+ "![first\nsecond](/i \"first\nsecond\")"
+ );
+ }
+}
+
+#[test]
+fn image_and_link_attributes_escape_markdown_and_preserve_literal_entities() {
+ let html = "<a href='/a\\*?x=&amp;copy;' title='a &quot;quote&quot; and \\ slash'>[label]</a>";
+ let result = markdown(html, false);
+ assert_eq!(
+ rendered_html(&result),
+ "<p><a href=\"/a%5C*?x=&amp;copy;\" title=\"a &quot;quote&quot; and \\ slash\">[label]</a></p>\n"
+ );
+ let html = "<img src='/a(b)' alt='[a] *b* `c` &amp;copy;'>";
+ assert_eq!(
+ rendered_html(&markdown(html, false)),
+ "<p><img src=\"/a(b)\" alt=\"[a] *b* `c` &amp;copy;\" /></p>\n"
+ );
+}
+
+#[test]
+fn table_attributes_keep_the_same_meaning_as_attributes_outside_tables() {
+ for content in [
+ "<a href='/a' title='first\n second'>link</a>",
+ "<img src='/i' alt='first\n second' title='first\n second'>",
+ ] {
+ let inline = rendered_html(&markdown(content, false));
+ let html = format!("<table><tr><th>{content}</th></tr></table>");
+ assert_eq!(rendered_html(&markdown(&html, false)), inline, "{html}",);
+ }
+}
+
+#[test]
+fn heading_attributes_keep_the_same_meaning_as_attributes_outside_headings() {
+ for content in [
+ "<a href='/a' title='first\n second'>link</a>",
+ "<img src='/i' alt='first\n second' title='first\n second'>",
+ ] {
+ let inline = rendered_html(&markdown(content, false));
+ let inline = inline
+ .strip_prefix("<p>")
+ .unwrap()
+ .strip_suffix("</p>\n")
+ .unwrap();
+ let html = format!("<h2>{content}</h2>");
+ assert_eq!(
+ rendered_html(&markdown(&html, false)),
+ format!("<h2>{inline}</h2>\n")
+ );
+ }
+}
+
+#[test]
+fn images_without_sources_do_not_emit_placeholder_markdown() {
+ for html in [
+ "a<img>b",
+ "a<img alt='label'>b",
+ "a<img src='' alt='label'>b",
+ ] {
+ assert_eq!(markdown(html, false), "ab");
+ }
+}
+
+#[test]
+fn list_paragraphs_and_code_blocks_require_blank_lines_between_items() {
+ for (html, expected) in [
+ ("<ul><li>one</li><li>two</li></ul>", "- one\n- two"),
+ ("<ul><li><p>one</p></li><li>two</li></ul>", "- one\n\n- two"),
+ ("<ul><li>one</li><li><p>two</p></li></ul>", "- one\n\n- two"),
+ ("<ol><li></li><li>two</li></ol>", "2. two"),
+ (
+ "<ul><li><pre><code>x</code></pre></li><li>tail</li></ul>",
+ "- ```\n x\n ```\n\n- tail",
+ ),
+ ] {
+ assert_eq!(markdown(html, false), expected, "{html}");
+ }
+ assert_eq!(
+ rendered_html(&markdown("<ul><li><p>one</p></li><li>two</li></ul>", false)),
+ "<ul>\n<li>\n<p>one</p>\n</li>\n<li>\n<p>two</p>\n</li>\n</ul>\n"
+ );
+}
+
+#[test]
+fn empty_pre_blocks_disappear_and_nonempty_pre_keeps_blank_lines() {
+ for contents in ["", " ", "\n", "\n \n"] {
+ assert_eq!(
+ markdown(&format!("<pre><code>{contents}</code></pre>"), false),
+ ""
+ );
+ }
+ assert_eq!(
+ markdown("<pre><code>\nfirst\n\nlast\n\n</code></pre>", false),
+ "```\n\nfirst\n\nlast\n\n```"
+ );
+}
+
+#[test]
+fn fallback_block_elements_separate_neighboring_words() {
+ for tag in ["audio", "canvas", "output"] {
+ assert_eq!(
+ markdown(&format!("before<{tag}>content</{tag}>after"), false),
+ "before\n\ncontent\n\nafter"
+ );
+ }
+}
diff --git a/moli-html2md/tests/rules.rs b/moli-html2md/tests/rules.rs
new file mode 100644
index 000000000..5f3a1e543
--- /dev/null
+++ b/moli-html2md/tests/rules.rs
@@ -0,0 +1,230 @@
+mod support;
+
+use moli_html2md::{Converter, Options};
+use support::{Tree, rendered_html};
+
+fn markdown_with(html: &str, options: Options) -> String {
+ let dom = Tree::parse(html);
+ let before = dom.clone();
+ let result = Converter::new(options).convert(&dom, dom.root);
+ assert_eq!(dom, before);
+ result
+}
+
+fn markdown(html: &str) -> String {
+ markdown_with(html, Options::default())
+}
+
+#[test]
+fn ordered_lists_use_start_and_sequential_numbers() {
+ for (attributes, start) in [
+ ("", 1),
+ ("reversed", 1),
+ ("start='4'", 4),
+ ("reversed start='4'", 4),
+ ] {
+ let html = format!(
+ "<ol {attributes}><li value='20'>first</li><li value='-3'>second</li><li>third</li></ol>"
+ );
+ let result = markdown(&html);
+ assert_eq!(
+ result,
+ format!(
+ "{start}. first\n{}. second\n{}. third",
+ start + 1,
+ start + 2
+ )
+ );
+ let start_attribute = if start == 1 {
+ String::new()
+ } else {
+ format!(" start=\"{start}\"")
+ };
+ assert_eq!(
+ rendered_html(&result),
+ format!(
+ "<ol{start_attribute}>\n<li>first</li>\n<li>second</li>\n<li>third</li>\n</ol>\n"
+ )
+ );
+ }
+}
+
+#[test]
+fn nested_lists_keep_independent_start_numbers() {
+ assert_eq!(
+ markdown(
+ "<ol start='4'><li>outer<ol reversed start='9'><li value='100'>inner</li><li>next</li></ol></li><li value='70'>tail</li></ol>"
+ ),
+ "4. outer\n 9. inner\n 10. next\n5. tail"
+ );
+}
+
+#[test]
+fn code_languages_preserve_punctuation_and_unicode() {
+ for language in ["text/x-rust", "中文", "C#", "objective-c++", "foo:bar"] {
+ let html = format!("<pre><code class='language-{language}'>x</code></pre>");
+ let result = markdown(&html);
+ assert_eq!(result, format!("```{language}\nx\n```"));
+ assert_eq!(
+ rendered_html(&result),
+ format!("<pre><code class=\"language-{language}\">x\n</code></pre>\n")
+ );
+ }
+}
+
+#[test]
+fn code_languages_preserve_literal_backslashes_and_entities() {
+ for (language, expected) in [
+ ("f\\*oo", "f\\*oo"),
+ ("literal&amp;copy;", "literal&amp;copy;"),
+ ] {
+ let result = markdown(&format!(
+ "<pre><code class='language-{language}'>x</code></pre>"
+ ));
+ assert_eq!(
+ rendered_html(&result),
+ format!("<pre><code class=\"language-{expected}\">x\n</code></pre>\n")
+ );
+ }
+}
+
+#[test]
+fn language_backticks_use_a_tilde_fence_that_contains_the_entire_code() {
+ let result = markdown("<pre><code class='language-a`b'>first\n~~~\nlast\n</code></pre>");
+ assert_eq!(result, "~~~~a`b\nfirst\n~~~\nlast\n~~~~");
+ assert_eq!(
+ rendered_html(&result),
+ "<pre><code class=\"language-a`b\">first\n~~~\nlast\n</code></pre>\n"
+ );
+}
+
+#[test]
+fn fallback_language_stays_on_the_fence_line() {
+ for ending in ["\n", "\r\n", "\r"] {
+ let result = markdown_with(
+ "<pre>x</pre>",
+ Options {
+ default_code_language: Some(format!("text/x-rust{ending}```{ending}outside")),
+ ..Options::default()
+ },
+ );
+ assert_eq!(result, "```text/x-rust\nx\n```");
+ }
+}
+
+#[test]
+fn list_spacing_uses_converted_blocks_including_wrapped_blocks() {
+ for (body, expected) in [
+ ("<h2>Heading</h2>", "- ## Heading\n\n- tail"),
+ ("intro<hr>", "- intro\n\n ---\n\n- tail"),
+ ("<section>one</section>", "- one\n\n- tail"),
+ ("<span><p>one</p></span>", "- one\n\n- tail"),
+ ] {
+ let result = markdown(&format!("<ul><li>{body}</li><li>tail</li></ul>"));
+ assert_eq!(result, expected, "{body}");
+ assert!(rendered_html(&result).contains("<p>tail</p>"), "{body}");
+ }
+}
+
+#[test]
+fn nested_list_blocks_do_not_make_outer_items_loose() {
+ for (inner, expected) in [
+ (
+ "<li>one</li><li>two</li>",
+ "- outer\n - one\n - two\n- tail",
+ ),
+ (
+ "<li><p>one</p></li><li><p>two</p></li>",
+ "- outer\n - one\n\n - two\n- tail",
+ ),
+ ] {
+ let result = markdown(&format!(
+ "<ul><li>outer<ul>{inner}</ul></li><li>tail</li></ul>"
+ ));
+ assert_eq!(result, expected);
+ assert!(rendered_html(&result).contains("<li>tail</li>"));
+ }
+ assert_eq!(
+ markdown("<ul><li><p>outer</p><ul><li>inner</li></ul></li><li>tail</li></ul>"),
+ "- outer\n\n - inner\n\n- tail"
+ );
+}
+
+#[test]
+fn omitted_subtrees_do_not_affect_list_spacing() {
+ let html = "<ul><li>one<span><h2>too deep</h2></span></li><li>two</li></ul>";
+ assert_eq!(
+ markdown_with(
+ html,
+ Options {
+ max_depth: 4,
+ ..Options::default()
+ }
+ ),
+ "- one\n- two"
+ );
+ assert_eq!(
+ markdown("<ul><li>one<script>ignored</script></li><li>two</li></ul>"),
+ "- one\n- two"
+ );
+}
+
+#[test]
+fn emphasis_uses_markdown_between_adjacent_links() {
+ for (tag, marker, html_tag) in [
+ ("b", "**", "strong"),
+ ("em", "*", "em"),
+ ("del", "~~", "del"),
+ ] {
+ for label in ["Hacker News", "News!"] {
+ let result = markdown(&format!(
+ "<{tag}><a href='news'>{label}</a></{tag}><a href='newest'>new</a>"
+ ));
+ assert_eq!(
+ result,
+ format!("{marker}[{label}](news){marker}[new](newest)")
+ );
+ assert_eq!(
+ rendered_html(&result),
+ format!(
+ "<p><{html_tag}><a href=\"news\">{label}</a></{html_tag}><a href=\"newest\">new</a></p>\n"
+ )
+ );
+ }
+ }
+ assert_eq!(
+ markdown("<strong>News!</strong><a href='newest'>new</a>"),
+ "**News!**[new](newest)"
+ );
+}
+
+#[test]
+fn emphasized_links_keep_html_at_intraword_boundaries() {
+ for tag in ["strong", "em"] {
+ for suffix in [" after", "after", "<a href='/next'>next</a>"] {
+ let result = markdown(&format!(
+ "before<{tag}><a href='/'>text</a></{tag}>{suffix}"
+ ));
+ let suffix_markdown = if suffix.starts_with('<') {
+ "[next](/next)"
+ } else {
+ suffix
+ };
+ let suffix_html = if suffix.starts_with('<') {
+ "<a href=\"/next\">next</a>"
+ } else {
+ suffix
+ };
+ assert_eq!(
+ result,
+ format!("before<{tag}>[text](/)</{tag}>{suffix_markdown}")
+ );
+ assert_eq!(
+ rendered_html(&result),
+ format!("<p>before<{tag}><a href=\"/\">text</a></{tag}>{suffix_html}</p>\n")
+ );
+ }
+ let result = markdown(&format!("<{tag}><a href='/'>text</a></{tag}>after"));
+ assert_eq!(result, format!("<{tag}>[text](/)</{tag}>after"));
+ }
+}
diff --git a/moli-html2md/tests/support/mod.rs b/moli-html2md/tests/support/mod.rs
new file mode 100644
index 000000000..379a15b50
--- /dev/null
+++ b/moli-html2md/tests/support/mod.rs
@@ -0,0 +1,121 @@
+use html5ever::tendril::TendrilSink;
+use markup5ever_rcdom::{NodeData, RcDom};
+use moli_html2md::{Dom, NodeKind};
+
+/// An independent test adapter. Parsing and copying happen only in tests;
+/// the production converter sees the same four borrowed queries as NativeDom.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Tree {
+ pub nodes: Vec<Node>,
+ pub root: usize,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Node {
+ pub tag: Option<String>,
+ pub text: Option<String>,
+ pub attrs: Vec<(String, String)>,
+ pub first: Option<usize>,
+ pub next: Option<usize>,
+}
+
+impl Tree {
+ pub fn parse(html: &str) -> Self {
+ let parsed = html5ever::parse_document(RcDom::default(), Default::default()).one(format!(
+ "<x-turndown id='turndown-root'>{html}</x-turndown>"
+ ));
+ let mut nodes: Vec<Node> = Vec::new();
+ let mut root = 0;
+ // RcDom's destructor clears descendants even when handles survive.
+ // Keep the parsed document alive until the complete arena is copied.
+ let mut stack = vec![(parsed.document.clone(), None::<usize>)];
+ let mut last_children: Vec<Option<usize>> = Vec::new();
+ while let Some((node, parent)) = stack.pop() {
+ let id = nodes.len();
+ let (tag, text, attrs) = match &node.data {
+ NodeData::Element { name, attrs, .. } => (
+ Some(name.local.to_string()),
+ None,
+ attrs
+ .borrow()
+ .iter()
+ .map(|a| (a.name.local.to_string(), a.value.to_string()))
+ .collect(),
+ ),
+ NodeData::Text { contents } => {
+ (None, Some(contents.borrow().to_string()), Vec::new())
+ }
+ _ => (None, None, Vec::new()),
+ };
+ if attrs
+ .iter()
+ .any(|(key, value)| key == "id" && value == "turndown-root")
+ {
+ root = id;
+ }
+ nodes.push(Node {
+ tag,
+ text,
+ attrs,
+ first: None,
+ next: None,
+ });
+ last_children.push(None);
+ if let Some(parent) = parent {
+ if let Some(previous) = last_children[parent] {
+ nodes[previous].next = Some(id);
+ } else {
+ nodes[parent].first = Some(id);
+ }
+ last_children[parent] = Some(id);
+ }
+ stack.extend(
+ node.children
+ .borrow()
+ .iter()
+ .rev()
+ .map(|child| (child.clone(), Some(id))),
+ );
+ }
+ assert_ne!(root, 0, "fixture wrapper must be present");
+ Self { nodes, root }
+ }
+}
+
+impl Dom for Tree {
+ type NodeId = usize;
+
+ fn node_kind(&self, node: usize) -> NodeKind<'_> {
+ let node = &self.nodes[node];
+ if let Some(tag) = &node.tag {
+ NodeKind::Element(tag)
+ } else if let Some(text) = &node.text {
+ NodeKind::Text(text)
+ } else {
+ NodeKind::Other
+ }
+ }
+
+ fn first_child(&self, node: usize) -> Option<usize> {
+ self.nodes[node].first
+ }
+ fn next_sibling(&self, node: usize) -> Option<usize> {
+ self.nodes[node].next
+ }
+ fn attribute(&self, node: usize, name: &str) -> Option<&str> {
+ self.nodes[node]
+ .attrs
+ .iter()
+ .find_map(|(key, value)| (key == name).then_some(value.as_str()))
+ }
+}
+
+pub fn rendered_html(markdown: &str) -> String {
+ let parser = pulldown_cmark::Parser::new_ext(
+ markdown,
+ pulldown_cmark::Options::ENABLE_TABLES | pulldown_cmark::Options::ENABLE_STRIKETHROUGH,
+ );
+ let mut html = String::new();
+ pulldown_cmark::html::push_html(&mut html, parser);
+ html
+}
diff --git a/moli-html2md/tests/tables.rs b/moli-html2md/tests/tables.rs
new file mode 100644
index 000000000..d32386370
--- /dev/null
+++ b/moli-html2md/tests/tables.rs
@@ -0,0 +1,127 @@
+mod support;
+
+use moli_html2md::{Converter, Options, convert};
+use support::{Tree, rendered_html};
+
+fn markdown(html: &str) -> String {
+ let dom = Tree::parse(html);
+ let before = dom.clone();
+ let result = convert(&dom, dom.root);
+ assert_eq!(dom, before);
+ result
+}
+
+#[test]
+fn hacker_news_layout_tables_preserve_story_and_metadata_order() {
+ let result = markdown(include_str!("fixtures/hacker-news-layout.html"));
+ assert_eq!(
+ result,
+ include_str!("fixtures/hacker-news-layout.md").trim_end()
+ );
+ let html = rendered_html(&result);
+ assert!(!html.contains("<table>"));
+ assert!(!html.contains("<br>"));
+ assert_eq!(html.matches("<a ").count(), 14);
+}
+
+#[test]
+fn nested_tables_expand_cells_in_dom_order() {
+ let html = "<table><tr><td>before</td></tr><tr><td><table><tr><th>Key</th><th>Value</th></tr><tr><td>a|b</td><td><code>c|d</code></td></tr></table></td></tr><tr><td>after</td></tr></table>";
+ let actual = markdown(html);
+ assert_eq!(actual, "before\n\nKey\n\nValue\n\na|b\n\n`c|d`\n\nafter");
+ assert_eq!(
+ rendered_html(&actual),
+ "<p>before</p>\n<p>Key</p>\n<p>Value</p>\n<p>a|b</p>\n<p><code>c|d</code></p>\n<p>after</p>\n"
+ );
+}
+
+#[test]
+fn wrapping_a_table_preserves_its_content() {
+ let data =
+ "<table><tr><td>one</td><td>two</td></tr><tr><td>three</td><td>four</td></tr></table>";
+ assert_eq!(
+ markdown(&format!("<table><tr><td>{data}</td></tr></table>")),
+ markdown(data)
+ );
+ assert_eq!(markdown(data), "one\n\ntwo\n\nthree\n\nfour");
+}
+
+#[test]
+fn table_roles_do_not_change_conversion() {
+ for role in [
+ "",
+ "presentation",
+ "none",
+ " PRESENTATION ",
+ "table",
+ "grid",
+ ] {
+ let html = format!(
+ "<table role='{role}'><tr><th>Name</th><th>Value</th></tr><tr><td>one</td><td>two</td></tr></table>"
+ );
+ assert_eq!(markdown(&html), "Name\n\nValue\n\none\n\ntwo");
+ }
+}
+
+#[test]
+fn table_cells_preserve_block_syntax_and_code_whitespace() {
+ let html = "<table><tr><td>intro</td><td><h2>Heading</h2><pre><code> a\n b\n</code></pre><blockquote>quote</blockquote><ul><li>item</li></ul></td><td>end</td></tr></table>";
+ assert_eq!(
+ markdown(html),
+ "intro\n\n## Heading\n\n```\n a\n b\n```\n\n> quote\n\n- item\n\nend"
+ );
+}
+
+#[test]
+fn captions_sections_headers_and_spacer_rows_preserve_content_order() {
+ for (html, expected) in [
+ (
+ "<table role='table'><tr><td>a</td><td>b</td></tr><tr></tr><tr><td>c</td><td>d</td></tr></table>",
+ "a\n\nb\n\nc\n\nd",
+ ),
+ (
+ "<table><caption>Data</caption><tr><td>a</td><td>b</td></tr><tr></tr><tr><td>c</td><td>d</td></tr></table>",
+ "Data\n\na\n\nb\n\nc\n\nd",
+ ),
+ (
+ "<table><thead><tr><td>a</td><td>b</td></tr></thead><tbody><tr></tr><tr><td>c</td><td>d</td></tr></tbody><tfoot><tr><td>end</td></tr></tfoot></table>",
+ "a\n\nb\n\nc\n\nd\n\nend",
+ ),
+ (
+ "<table><tr><th>a</th><th>b</th></tr><tr></tr><tr><td>c</td><td>d</td></tr></table>",
+ "a\n\nb\n\nc\n\nd",
+ ),
+ ] {
+ assert_eq!(markdown(html), expected, "{html}");
+ }
+}
+
+#[test]
+fn table_presentation_attributes_do_not_change_conversion() {
+ for attributes in [
+ "",
+ "border='0' width='100%'",
+ "border='1'",
+ "style='border:0'",
+ ] {
+ let html = format!(
+ "<table {attributes}><tr><td><a href='/'>Home</a></td><td><a href='/help'>Help</a></td></tr></table>"
+ );
+ assert_eq!(markdown(&html), "[Home](/)\n\n[Help](/help)");
+ }
+}
+
+#[test]
+fn nested_tables_obey_the_conversion_depth_limit() {
+ let dom = Tree::parse(
+ "<table><tr><td>visible<div><table><tr><td>too deep</td></tr></table></div></td></tr></table>",
+ );
+ for max_depth in [6, 7] {
+ let converter = Converter::new(Options {
+ max_depth,
+ ..Options::default()
+ });
+ assert_eq!(converter.convert(&dom, dom.root), "visible");
+ }
+ assert_eq!(convert(&dom, dom.root), "visible\n\ntoo deep");
+}
diff --git a/moli-html2md/tests/turndown.rs b/moli-html2md/tests/turndown.rs
new file mode 100644
index 000000000..e1d8fc14a
--- /dev/null
+++ b/moli-html2md/tests/turndown.rs
@@ -0,0 +1,91 @@
+mod support;
+
+use std::sync::OnceLock;
+
+use moli_html2md::{Converter, Options};
+use serde_json::Value;
+use support::{Tree, rendered_html};
+
+fn corpus() -> &'static Value {
+ static CORPUS: OnceLock<Value> = OnceLock::new();
+ CORPUS.get_or_init(|| {
+ serde_json::from_str(include_str!("turndown/cases.json")).expect("valid reference corpus")
+ })
+}
+
+fn expectations() -> &'static Value {
+ static EXPECTATIONS: OnceLock<Value> = OnceLock::new();
+ EXPECTATIONS.get_or_init(|| {
+ serde_json::from_str(include_str!("turndown/expectations.json"))
+ .expect("valid intentional differences")
+ })
+}
+
+fn check_case(index: usize) {
+ let case = &corpus()["cases"][index];
+ let input = case["html"].as_str().expect("HTML fixture");
+ let expected = case["markdown"].as_str().expect("reference Markdown");
+ let dom = Tree::parse(input);
+ let before = dom.clone();
+ let converter = Converter::new(Options {
+ preformatted_code: case["preformatted_code"].as_bool().unwrap_or(false),
+ ..Options::default()
+ });
+ let actual = converter.convert(&dom, dom.root);
+ assert_eq!(dom, before, "conversion modified DOM");
+ let difference = &expectations()[case["id"].as_str().expect("fixture ID")];
+ if let Some(markdown) = difference["markdown"].as_str() {
+ assert!(
+ !difference["reason"]
+ .as_str()
+ .expect("documented difference")
+ .is_empty()
+ );
+ assert_ne!(
+ rendered_html(markdown),
+ rendered_html(expected),
+ "review an obsolete difference"
+ );
+ assert_eq!(actual, markdown, "{}: {}", case["id"], difference["reason"]);
+ return;
+ }
+ let expected_html = rendered_html(expected);
+ let actual_html = rendered_html(&actual);
+ if actual_html != expected_html {
+ println!(
+ "DIFF {}",
+ serde_json::json!({
+ "id": case["id"], "name": case["name"], "input": input,
+ "expected": expected, "actual": actual,
+ "expected_html": expected_html, "actual_html": actual_html,
+ })
+ );
+ }
+ assert_eq!(
+ actual_html, expected_html,
+ "case {}: {}",
+ case["id"], case["name"]
+ );
+}
+
+macro_rules! case {
+ ($name:ident, $index:expr) => {
+ #[test]
+ fn $name() {
+ check_case($index);
+ }
+ };
+}
+
+include!("turndown/cases.rs");
+
+#[test]
+fn every_intentional_difference_names_a_unique_existing_case() {
+ let mut ids = std::collections::HashSet::new();
+ for case in corpus()["cases"].as_array().expect("cases") {
+ assert!(ids.insert(case["id"].as_str().expect("fixture ID")));
+ }
+ for id in expectations().as_object().expect("difference map").keys() {
+ assert!(ids.contains(id.as_str()), "unused difference: {id}");
+ }
+}
diff --git a/moli-html2md/tests/turndown/LICENSE b/moli-html2md/tests/turndown/LICENSE
new file mode 100644
index 000000000..02b58935f
--- /dev/null
+++ b/moli-html2md/tests/turndown/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Dom Christie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/moli-html2md/tests/turndown/README.md b/moli-html2md/tests/turndown/README.md
new file mode 100644
index 000000000..a9292b338
--- /dev/null
+++ b/moli-html2md/tests/turndown/README.md
@@ -0,0 +1,51 @@
+# Turndown differential tests
+
+Reference: [mixmark-io/turndown](https://github.com/mixmark-io/turndown/tree/aa84dfa3e2361edea8c43acbfc2b7a9363494bfd),
+revision `aa84dfa3e2361edea8c43acbfc2b7a9363494bfd`, package version 7.2.4,
+using `@mixmark-io/domino` 2.2.0.
+
+`cases.json` contains all 147 HTML fixtures from upstream `test/index.html`,
+plus the 50 independently written cases in `additional.json`. The upstream
+fixtures retain their MIT license in `LICENSE`. Production Rust code does not
+include the upstream JavaScript implementation.
+
+## What is compared
+
+Turndown is configured to match Moli's ATX headings, dash bullets, star emphasis,
+fenced code, inline links, and horizontal rules. Per-case style-only options
+(such as referenced links or tilde fences) are normalized to this configuration.
+The semantic `preformattedCode` option is preserved and tested against Moli's
+`preformatted_code` option. This corpus tests conversion behavior, not Turndown's
+JavaScript plugin API or every output-style option.
+
+Each case parses the source HTML through an independent test DOM implementing
+`Dom`, converts it, and checks that the DOM is unchanged. Both Markdown outputs
+are rendered with `pulldown-cmark`, then their HTML is compared exactly. This
+accepts equivalent delimiter choices and list indentation while detecting lost
+text, incorrect links, changed paragraph/list structure, and visible delimiters.
+There are also exact Markdown assertions in the original and regression suites.
+
+`expectations.json` documents 12 deliberate differences with exact Moli outputs.
+These preserve bare pre blocks and avoid upstream losses of literal characters,
+emphasis, whitespace, and list boundaries. They remain active assertions; no
+case is skipped. A reference change that makes a difference obsolete also fails
+the test. Separate converter tests cover table expansion using Turndown core's
+ordinary block behavior and strikethrough as a Moli extension.
+
+## Regeneration
+
+Normal Rust tests use the checked-in data and require neither Node nor network
+access. To update the reference intentionally:
+
+```sh
+git clone https://github.com/mixmark-io/turndown.git /tmp/moli-turndown/upstream
+git -C /tmp/moli-turndown/upstream checkout aa84dfa3e2361edea8c43acbfc2b7a9363494bfd
+npm install --prefix /tmp/moli-turndown --ignore-scripts --no-audit --no-fund @mixmark-io/[email protected]
+node moli-html2md/tests/turndown/generate.mjs /tmp/moli-turndown/upstream
+cargo test -p moli-html2md
+```
+
+The generator executes the pinned upstream source. It adapts import extensions
+for Node's ESM loader and provides CommonJS `require` for Domino, without changing
+conversion rules. The corpus records both the upstream revision and normalized
+options. `cases.rs` provides a separate named Rust test for every fixture.
diff --git a/moli-html2md/tests/turndown/additional.json b/moli-html2md/tests/turndown/additional.json
new file mode 100644
index 000000000..841c64ef2
--- /dev/null
+++ b/moli-html2md/tests/turndown/additional.json
@@ -0,0 +1,52 @@
+[
+ {"id":"extra_empty_href","name":"empty href keeps link text","html":"before<a href=''>label</a>after"},
+ {"id":"extra_empty_link","name":"empty link label","html":"before<a href='/target'></a>after"},
+ {"id":"extra_blank_link","name":"whitespace-only link label","html":"before<a href='/target'> </a>after"},
+ {"id":"extra_missing_image_src","name":"image alt without source","html":"before<img alt='missing'>after"},
+ {"id":"extra_img_alt_whitespace","name":"image alt with multiple lines and indentation","html":"<img src='/x' alt='one\n \n two'>"},
+ {"id":"extra_img_alt_code","name":"image label containing code delimiters","html":"<img src='/x' alt='a`b'>"},
+ {"id":"extra_adjacent_lists","name":"adjacent unordered lists stay separate","html":"<ul><li>one</li></ul><ul><li>two</li></ul>"},
+ {"id":"extra_adjacent_ordered_lists","name":"adjacent ordered lists stay separate","html":"<ol><li>one</li></ol><ol start='4'><li>four</li></ol>"},
+ {"id":"extra_empty_list","name":"empty list","html":"before<ul></ul>after"},
+ {"id":"extra_empty_list_item","name":"empty list item","html":"<ul><li></li><li>text</li></ul>"},
+ {"id":"extra_loose_list","name":"paragraph list items stay loose","html":"<ul><li><p>one</p></li><li><p>two</p></li></ul>"},
+ {"id":"extra_mixed_loose_list","name":"paragraph followed by a plain list item","html":"<ul><li><p>one</p></li><li>two</li></ul>"},
+ {"id":"extra_list_nested_only","name":"list item containing only a nested list","html":"<ul><li><ul><li>inner</li></ul></li></ul>"},
+ {"id":"extra_list_nested_tail","name":"text after a nested list","html":"<ul><li>start<ul><li>inner</li></ul>end</li></ul>"},
+ {"id":"extra_list_nested_tail_p","name":"paragraph after a nested list","html":"<ul><li><p>start</p><ul><li>inner</li></ul><p>end</p></li></ul>"},
+ {"id":"extra_list_code","name":"fenced code inside list item","html":"<ol start='9'><li><pre><code> first\nsecond\n</code></pre></li><li>end</li></ol>"},
+ {"id":"extra_list_quote","name":"list item containing only a quote","html":"<ul><li><blockquote><p>quote</p></blockquote></li></ul>"},
+ {"id":"extra_empty_quote","name":"empty quote still separates blocks","html":"before<blockquote></blockquote>after"},
+ {"id":"extra_quote_blank_lines","name":"quote with paragraphs and hard breaks","html":"<blockquote><p>one<br>two</p><p>three</p></blockquote>"},
+ {"id":"extra_empty_heading","name":"empty heading boundary","html":"before<h2></h2>after"},
+ {"id":"extra_empty_pre","name":"empty pre","html":"before<pre></pre>after"},
+ {"id":"extra_empty_pre_code","name":"empty pre code","html":"before<pre><code></code></pre>after"},
+ {"id":"extra_blank_pre_code","name":"whitespace-only pre code","html":"<pre><code> \n</code></pre>"},
+ {"id":"extra_adjacent_code","name":"adjacent code spans","html":"<code>a</code><code>b</code>"},
+ {"id":"extra_code_gap","name":"separated code spans","html":"<code>a</code> <code>b</code>"},
+ {"id":"extra_code_nested_format","name":"markup inside inline code","html":"<code><strong>a</strong><em>b</em></code>"},
+ {"id":"extra_code_many_runs","name":"code span chooses an unused delimiter length","html":"<code>a```b`````c</code>"},
+ {"id":"extra_code_only_ticks","name":"code containing only backticks","html":"<code>```</code>"},
+ {"id":"extra_code_nbsp","name":"code with nonbreaking boundary spaces","html":"before<code>&nbsp;a&nbsp;</code>after"},
+ {"id":"extra_preformatted_internal","name":"preformatted inline code preserves multiple spaces","html":"<code> a b </code>","original_options":{"preformattedCode":true}},
+ {"id":"extra_preformatted_blank","name":"preformatted whitespace-only code","html":"before<code> </code>after","original_options":{"preformattedCode":true}},
+ {"id":"extra_literal_entities","name":"decoded entity text remains literal","html":"<p>&amp;copy; &amp;lt; &amp;#32;</p>"},
+ {"id":"extra_literal_html","name":"literal HTML tags remain visible text","html":"<p>&lt;b&gt;literal&lt;/b&gt;</p>"},
+ {"id":"extra_split_list_marker","name":"ordered marker split across elements","html":"<p>12<span>.</span> text</p>"},
+ {"id":"extra_split_emphasis_literal","name":"asterisks split across elements","html":"<p>*<span>word</span>*</p>"},
+ {"id":"extra_link_backslash","name":"link destination with backslashes","html":"<a href='a\\b'>link</a>"},
+ {"id":"extra_link_entities","name":"link destination with literal entity syntax","html":"<a href='/?q=&amp;copy;'>link</a>"},
+ {"id":"extra_link_leading_space","name":"leading whitespace inside link label","html":"before<a href='/'> label</a>after"},
+ {"id":"extra_nested_link_emphasis","name":"emphasis nested in link","html":"<a href='/'><strong><em>text</em></strong></a>"},
+ {"id":"extra_emphasis_nested_link","name":"link nested in emphasis","html":"<em><a href='/'>text</a></em>"},
+ {"id":"extra_emphasis_intraword_link","name":"link in intraword emphasis","html":"before<em><a href='/'>text</a></em>after"},
+ {"id":"extra_emphasis_intraword_punctuation","name":"intraword emphasis around punctuation","html":"before<em>(text)</em>after"},
+ {"id":"extra_empty_emphasis","name":"empty formatting preserves surrounding spaces","html":"a<strong> </strong>b<em> </em>c"},
+ {"id":"extra_nbsp_comment","name":"nonbreaking whitespace around ignored comment","html":"a&nbsp;<!--x--> b"},
+ {"id":"extra_leading_br","name":"leading hard break","html":"<p><br>text</p>"},
+ {"id":"extra_consecutive_br","name":"consecutive hard breaks","html":"<p>one<br><br>two</p>"},
+ {"id":"extra_blank_br_container","name":"otherwise empty span containing hard break","html":"a<span><br></span>b"},
+ {"id":"extra_empty_block_inside_inline","name":"empty block inside emphasis","html":"<strong>a<div></div>b</strong>"},
+ {"id":"extra_audio_boundary","name":"audio fallback content is a block","html":"before<audio>fallback</audio>after"},
+ {"id":"extra_output_boundary","name":"output element block boundary","html":"before<output>result</output>after"}
+]
diff --git a/moli-html2md/tests/turndown/cases.json b/moli-html2md/tests/turndown/cases.json
new file mode 100644
index 000000000..3fa328753
--- /dev/null
+++ b/moli-html2md/tests/turndown/cases.json
@@ -0,0 +1,1604 @@
+{
+ "upstream": {
+ "repository": "https://github.com/mixmark-io/turndown",
+ "revision": "aa84dfa3e2361edea8c43acbfc2b7a9363494bfd",
+ "version": "7.2.4",
+ "domino": "2.2.0"
+ },
+ "options": {
+ "headingStyle": "atx",
+ "hr": "---",
+ "bulletListMarker": "-",
+ "codeBlockStyle": "fenced",
+ "fence": "```",
+ "emDelimiter": "*",
+ "strongDelimiter": "**",
+ "linkStyle": "inlined",
+ "br": " "
+ },
+ "cases": [
+ {
+ "id": "upstream_001",
+ "name": "p",
+ "html": "<p>Lorem ipsum</p>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Lorem ipsum"
+ },
+ {
+ "id": "upstream_002",
+ "name": "multiple ps",
+ "html": "\n <p>Lorem</p>\n <p>ipsum</p>\n <p>sit</p>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Lorem\n\nipsum\n\nsit"
+ },
+ {
+ "id": "upstream_003",
+ "name": "em",
+ "html": "<em>em element</em>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "*em element*"
+ },
+ {
+ "id": "upstream_004",
+ "name": "i",
+ "html": "<i>i element</i>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "*i element*"
+ },
+ {
+ "id": "upstream_005",
+ "name": "strong",
+ "html": "<strong>strong element</strong>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "**strong element**"
+ },
+ {
+ "id": "upstream_006",
+ "name": "b",
+ "html": "<b>b element</b>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "**b element**"
+ },
+ {
+ "id": "upstream_007",
+ "name": "code",
+ "html": "<code>code element</code>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "`code element`"
+ },
+ {
+ "id": "upstream_008",
+ "name": "code containing a backtick",
+ "html": "<code>There is a literal backtick (`) here</code>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "``There is a literal backtick (`) here``"
+ },
+ {
+ "id": "upstream_009",
+ "name": "code containing three or more backticks",
+ "html": "<code>here are three ``` here are four ```` that's it</code>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "`here are three ``` here are four ```` that's it`"
+ },
+ {
+ "id": "upstream_010",
+ "name": "code containing one or more backticks",
+ "html": "<code>here are three ``` here are four ```` here is one ` that's it</code>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "``here are three ``` here are four ```` here is one ` that's it``"
+ },
+ {
+ "id": "upstream_011",
+ "name": "code starting with a backtick",
+ "html": "<code>`starting with a backtick</code>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "`` `starting with a backtick ``"
+ },
+ {
+ "id": "upstream_012",
+ "name": "code containing markdown syntax",
+ "html": "<code>_emphasis_</code>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "`_emphasis_`"
+ },
+ {
+ "id": "upstream_013",
+ "name": "code containing markdown syntax in a span",
+ "html": "<code><span>_emphasis_</span></code>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "`_emphasis_`"
+ },
+ {
+ "id": "upstream_014",
+ "name": "h1",
+ "html": "<h1>Level One Heading</h1>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "# Level One Heading"
+ },
+ {
+ "id": "upstream_015",
+ "name": "escape = when used as heading",
+ "html": "===",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\==="
+ },
+ {
+ "id": "upstream_016",
+ "name": "not escaping = outside of a heading",
+ "html": "A sentence containing =",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "A sentence containing ="
+ },
+ {
+ "id": "upstream_017",
+ "name": "h1 as atx",
+ "html": "<h1>Level One Heading with ATX</h1>",
+ "original_options": {
+ "headingStyle": "atx"
+ },
+ "preformatted_code": false,
+ "markdown": "# Level One Heading with ATX"
+ },
+ {
+ "id": "upstream_018",
+ "name": "h2",
+ "html": "<h2>Level Two Heading</h2>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "## Level Two Heading"
+ },
+ {
+ "id": "upstream_019",
+ "name": "h2 as atx",
+ "html": "<h2>Level Two Heading with ATX</h2>",
+ "original_options": {
+ "headingStyle": "atx"
+ },
+ "preformatted_code": false,
+ "markdown": "## Level Two Heading with ATX"
+ },
+ {
+ "id": "upstream_020",
+ "name": "h3",
+ "html": "<h3>Level Three Heading</h3>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "### Level Three Heading"
+ },
+ {
+ "id": "upstream_021",
+ "name": "heading with child",
+ "html": "<h4>Level Four Heading with <code>child</code></h4>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "#### Level Four Heading with `child`"
+ },
+ {
+ "id": "upstream_022",
+ "name": "invalid heading",
+ "html": "<h7>Level Seven Heading?</h7>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Level Seven Heading?"
+ },
+ {
+ "id": "upstream_023",
+ "name": "hr",
+ "html": "<hr>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "---"
+ },
+ {
+ "id": "upstream_024",
+ "name": "hr with closing tag",
+ "html": "<hr>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "---"
+ },
+ {
+ "id": "upstream_025",
+ "name": "hr with option",
+ "html": "<hr>",
+ "original_options": {
+ "hr": "- - -"
+ },
+ "preformatted_code": false,
+ "markdown": "---"
+ },
+ {
+ "id": "upstream_026",
+ "name": "br",
+ "html": "More<br>after the break",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "More \nafter the break"
+ },
+ {
+ "id": "upstream_027",
+ "name": "br with visible line-ending",
+ "html": "More<br>after the break",
+ "original_options": {
+ "br": "\\"
+ },
+ "preformatted_code": false,
+ "markdown": "More \nafter the break"
+ },
+ {
+ "id": "upstream_028",
+ "name": "img with no alt",
+ "html": "<img src=\"http://example.com/logo.png\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![](http://example.com/logo.png)"
+ },
+ {
+ "id": "upstream_029",
+ "name": "img with relative src",
+ "html": "<img src=\"logo.png\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![](logo.png)"
+ },
+ {
+ "id": "upstream_030",
+ "name": "img with alt",
+ "html": "<img src=\"logo.png\" alt=\"img with alt\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![img with alt](logo.png)"
+ },
+ {
+ "id": "upstream_031",
+ "name": "img with escaped content in alt",
+ "html": "<img src=\"logo.png\" alt=\"_img_ *with* [alt]\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![\\_img\\_ \\*with\\* \\[alt\\]](logo.png)"
+ },
+ {
+ "id": "upstream_032",
+ "name": "img with no src",
+ "html": "<img>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": ""
+ },
+ {
+ "id": "upstream_033",
+ "name": "img with parenthesis in src",
+ "html": "<img src=\"logo.png?(query)\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![](logo.png?\\(query\\))"
+ },
+ {
+ "id": "upstream_034",
+ "name": "img with a new line in alt",
+ "html": "<img src=\"logo.png\" alt=\"img with\n alt\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![img with\nalt](logo.png)"
+ },
+ {
+ "id": "upstream_035",
+ "name": "img with more than one new line in alt",
+ "html": "<img src=\"logo.png\" alt=\"img with\n \n alt\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![img with\nalt](logo.png)"
+ },
+ {
+ "id": "upstream_036",
+ "name": "img with new lines in title",
+ "html": "<img src=\"logo.png\" title=\"the\n \n title\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![](logo.png \"the\ntitle\")"
+ },
+ {
+ "id": "upstream_037",
+ "name": "img with quotes in title",
+ "html": "<img src=\"logo.png\" title=\"&quot;hello&quot;\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![](logo.png \"\\\"hello\\\"\")"
+ },
+ {
+ "id": "upstream_038",
+ "name": "img with space in URL",
+ "html": "<img src=\"logo 1.png\" alt=\"An image\">",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![An image](<logo 1.png>)"
+ },
+ {
+ "id": "upstream_039",
+ "name": "a",
+ "html": "<a href=\"http://example.com\">An anchor</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[An anchor](http://example.com)"
+ },
+ {
+ "id": "upstream_040",
+ "name": "a with space in URL",
+ "html": "<a href=\"http://example.com/foo bar\">An anchor</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[An anchor](<http://example.com/foo bar>)"
+ },
+ {
+ "id": "upstream_041",
+ "name": "a with title",
+ "html": "<a href=\"http://example.com\" title=\"Title for link\">An anchor</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[An anchor](http://example.com \"Title for link\")"
+ },
+ {
+ "id": "upstream_042",
+ "name": "a with multiline title",
+ "html": "<a href=\"http://example.com\" title=\"Title for\n \n link\">An anchor</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[An anchor](http://example.com \"Title for\nlink\")"
+ },
+ {
+ "id": "upstream_043",
+ "name": "a with quotes in title",
+ "html": "<a href=\"http://example.com\" title=\"&quot;hello&quot;\">An anchor</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[An anchor](http://example.com \"\\\"hello\\\"\")"
+ },
+ {
+ "id": "upstream_044",
+ "name": "a with parenthesis in query",
+ "html": "<a href=\"http://example.com?(query)\">An anchor</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[An anchor](http://example.com?\\(query\\))"
+ },
+ {
+ "id": "upstream_045",
+ "name": "a without a src",
+ "html": "<a id=\"about-anchor\">Anchor without a title</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Anchor without a title"
+ },
+ {
+ "id": "upstream_046",
+ "name": "a with a child",
+ "html": "<a href=\"http://example.com/code\">Some <code>code</code></a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[Some `code`](http://example.com/code)"
+ },
+ {
+ "id": "upstream_047",
+ "name": "a with a brackets in text",
+ "html": "<a href=\"http://example.com/code\">Some [text]</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[Some \\[text\\]](http://example.com/code)"
+ },
+ {
+ "id": "upstream_048",
+ "name": "a reference",
+ "html": "<a href=\"http://example.com\">Reference link</a>",
+ "original_options": {
+ "linkStyle": "referenced"
+ },
+ "preformatted_code": false,
+ "markdown": "[Reference link](http://example.com)"
+ },
+ {
+ "id": "upstream_049",
+ "name": "a reference with space in URL",
+ "html": "<a href=\"http://example.com/foo bar\">Reference link</a>",
+ "original_options": {
+ "linkStyle": "referenced"
+ },
+ "preformatted_code": false,
+ "markdown": "[Reference link](<http://example.com/foo bar>)"
+ },
+ {
+ "id": "upstream_050",
+ "name": "a reference with collapsed style",
+ "html": "<a href=\"http://example.com\">Reference link with collapsed style</a>",
+ "original_options": {
+ "linkStyle": "referenced",
+ "linkReferenceStyle": "collapsed"
+ },
+ "preformatted_code": false,
+ "markdown": "[Reference link with collapsed style](http://example.com)"
+ },
+ {
+ "id": "upstream_051",
+ "name": "a reference with shortcut style",
+ "html": "<a href=\"http://example.com\">Reference link with shortcut style</a>",
+ "original_options": {
+ "linkStyle": "referenced",
+ "linkReferenceStyle": "shortcut"
+ },
+ "preformatted_code": false,
+ "markdown": "[Reference link with shortcut style](http://example.com)"
+ },
+ {
+ "id": "upstream_052",
+ "name": "a reference with title",
+ "html": "<a href=\"http://example.com\" title=\"Hello &quot; World\">Reference link</a>",
+ "original_options": {
+ "linkStyle": "referenced"
+ },
+ "preformatted_code": false,
+ "markdown": "[Reference link](http://example.com \"Hello \\\" World\")"
+ },
+ {
+ "id": "upstream_053",
+ "name": "pre/code block",
+ "html": "<pre><code>def code_block\n # 42 &lt; 9001\n \"Hello world!\"\nend</code></pre>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "```\ndef code_block\n # 42 < 9001\n \"Hello world!\"\nend\n```"
+ },
+ {
+ "id": "upstream_054",
+ "name": "multiple pre/code blocks",
+ "html": "<pre><code>def first_code_block\n # 42 &lt; 9001\n \"Hello world!\"\nend</code></pre>\n\n<p>next:</p>\n\n<pre><code>def second_code_block\n # 42 &lt; 9001\n \"Hello world!\"\nend</code></pre>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "```\ndef first_code_block\n # 42 < 9001\n \"Hello world!\"\nend\n```\n\nnext:\n\n```\ndef second_code_block\n # 42 < 9001\n \"Hello world!\"\nend\n```"
+ },
+ {
+ "id": "upstream_055",
+ "name": "pre/code block with multiple new lines",
+ "html": "<div><pre><code>Multiple new lines\n\n\nshould not be\n\n\nremoved</code></pre></div>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "```\nMultiple new lines\n\n\nshould not be\n\n\nremoved\n```"
+ },
+ {
+ "id": "upstream_056",
+ "name": "fenced pre/code block",
+ "html": "\n <pre><code>def a_fenced_code block; end</code></pre>\n ",
+ "original_options": {
+ "codeBlockStyle": "fenced"
+ },
+ "preformatted_code": false,
+ "markdown": "```\ndef a_fenced_code block; end\n```"
+ },
+ {
+ "id": "upstream_057",
+ "name": "pre/code block fenced with ~",
+ "html": "\n <pre><code>def a_fenced_code block; end</code></pre>\n ",
+ "original_options": {
+ "codeBlockStyle": "fenced",
+ "fence": "~~~"
+ },
+ "preformatted_code": false,
+ "markdown": "```\ndef a_fenced_code block; end\n```"
+ },
+ {
+ "id": "upstream_058",
+ "name": "escaping ~~~",
+ "html": "\n <pre>~~~ foo</pre>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\~~~ foo"
+ },
+ {
+ "id": "upstream_059",
+ "name": "not escaping ~~~",
+ "html": "A sentence containing ~~~",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "A sentence containing ~~~"
+ },
+ {
+ "id": "upstream_060",
+ "name": "fenced pre/code block with language",
+ "html": "\n <pre><code class=\"language-ruby\">def a_fenced_code block; end</code></pre>\n ",
+ "original_options": {
+ "codeBlockStyle": "fenced"
+ },
+ "preformatted_code": false,
+ "markdown": "```ruby\ndef a_fenced_code block; end\n```"
+ },
+ {
+ "id": "upstream_061",
+ "name": "empty pre does not throw error",
+ "html": "\n <pre></pre>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": ""
+ },
+ {
+ "id": "upstream_062",
+ "name": "ol",
+ "html": "\n <ol>\n <li>Ordered list item 1</li>\n <li>Ordered list item 2</li>\n <li>Ordered list item 3</li>\n </ol>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "1. Ordered list item 1\n2. Ordered list item 2\n3. Ordered list item 3"
+ },
+ {
+ "id": "upstream_063",
+ "name": "ol with start",
+ "html": "\n <ol start=\"42\">\n <li>Ordered list item 42</li>\n <li>Ordered list item 43</li>\n <li>Ordered list item 44</li>\n </ol>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "42. Ordered list item 42\n43. Ordered list item 43\n44. Ordered list item 44"
+ },
+ {
+ "id": "upstream_064",
+ "name": "ol with content",
+ "html": "\n <ol start=\"42\">\n <li>\n <p>Ordered list item 42</p>\n <p>Ordered list's additional content</p>\n </li>\n </ol>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "42. Ordered list item 42\n \n Ordered list's additional content"
+ },
+ {
+ "id": "upstream_065",
+ "name": "list spacing",
+ "html": "\n <p>A paragraph.</p>\n <ol>\n <li>Ordered list item 1</li>\n <li>Ordered list item 2</li>\n <li>Ordered list item 3</li>\n </ol>\n <p>Another paragraph.</p>\n <ul>\n <li>Unordered list item 1</li>\n <li>Unordered list item 2</li>\n <li>Unordered list item 3</li>\n </ul>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "A paragraph.\n\n1. Ordered list item 1\n2. Ordered list item 2\n3. Ordered list item 3\n\nAnother paragraph.\n\n- Unordered list item 1\n- Unordered list item 2\n- Unordered list item 3"
+ },
+ {
+ "id": "upstream_066",
+ "name": "ul",
+ "html": "\n <ul>\n <li>Unordered list item 1</li>\n <li>Unordered list item 2</li>\n <li>Unordered list item 3</li>\n </ul>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "- Unordered list item 1\n- Unordered list item 2\n- Unordered list item 3"
+ },
+ {
+ "id": "upstream_067",
+ "name": "ul with custom bullet",
+ "html": "\n <ul>\n <li>Unordered list item 1</li>\n <li>Unordered list item 2</li>\n <li>Unordered list item 3</li>\n </ul>\n ",
+ "original_options": {
+ "bulletListMarker": "-"
+ },
+ "preformatted_code": false,
+ "markdown": "- Unordered list item 1\n- Unordered list item 2\n- Unordered list item 3"
+ },
+ {
+ "id": "upstream_068",
+ "name": "ul with paragraph",
+ "html": "\n <ul>\n <li><p>List item with paragraph</p></li>\n <li>List item without paragraph</li>\n </ul>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "- List item with paragraph\n \n- List item without paragraph"
+ },
+ {
+ "id": "upstream_069",
+ "name": "ol with paragraphs",
+ "html": "\n <ol>\n <li>\n <p>This is a paragraph in a list item.</p>\n <p>This is a paragraph in the same list item as above.</p>\n </li>\n <li>\n <p>A paragraph in a second list item.</p>\n </li>\n </ol>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "1. This is a paragraph in a list item.\n \n This is a paragraph in the same list item as above.\n \n2. A paragraph in a second list item."
+ },
+ {
+ "id": "upstream_070",
+ "name": "nested uls",
+ "html": "\n <ul>\n <li>This is a list item at root level</li>\n <li>This is another item at root level</li>\n <li>\n <ul>\n <li>This is a nested list item</li>\n <li>This is another nested list item</li>\n <li>\n <ul>\n <li>This is a deeply nested list item</li>\n <li>This is another deeply nested list item</li>\n <li>This is a third deeply nested list item</li>\n </ul>\n </li>\n </ul>\n </li>\n <li>This is a third item at root level</li>\n </ul>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "- This is a list item at root level\n- This is another item at root level\n- - This is a nested list item\n - This is another nested list item\n - - This is a deeply nested list item\n - This is another deeply nested list item\n - This is a third deeply nested list item\n- This is a third item at root level"
+ },
+ {
+ "id": "upstream_071",
+ "name": "nested ols and uls",
+ "html": "\n <ul>\n <li>This is a list item at root level</li>\n <li>This is another item at root level</li>\n <li>\n <ol>\n <li>This is a nested list item</li>\n <li>This is another nested list item</li>\n <li>\n <ul>\n <li>This is a deeply nested list item</li>\n <li>This is another deeply nested list item</li>\n <li>This is a third deeply nested list item</li>\n </ul>\n </li>\n </ol>\n </li>\n <li>This is a third item at root level</li>\n </ul>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "- This is a list item at root level\n- This is another item at root level\n- 1. This is a nested list item\n 2. This is another nested list item\n 3. - This is a deeply nested list item\n - This is another deeply nested list item\n - This is a third deeply nested list item\n- This is a third item at root level"
+ },
+ {
+ "id": "upstream_072",
+ "name": "ul with blockquote",
+ "html": "\n <ul>\n <li>\n <p>A list item with a blockquote:</p>\n <blockquote>\n <p>This is a blockquote inside a list item.</p>\n </blockquote>\n </li>\n </ul>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "- A list item with a blockquote:\n \n > This is a blockquote inside a list item."
+ },
+ {
+ "id": "upstream_073",
+ "name": "blockquote",
+ "html": "\n <blockquote>\n <p>This is a paragraph within a blockquote.</p>\n <p>This is another paragraph within a blockquote.</p>\n </blockquote>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "> This is a paragraph within a blockquote.\n> \n> This is another paragraph within a blockquote."
+ },
+ {
+ "id": "upstream_074",
+ "name": "nested blockquotes",
+ "html": "\n <blockquote>\n <p>This is the first level of quoting.</p>\n <blockquote>\n <p>This is a paragraph in a nested blockquote.</p>\n </blockquote>\n <p>Back to the first level.</p>\n </blockquote>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "> This is the first level of quoting.\n> \n> > This is a paragraph in a nested blockquote.\n> \n> Back to the first level."
+ },
+ {
+ "id": "upstream_075",
+ "name": "html in blockquote",
+ "html": "\n <blockquote>\n <h2>This is a header.</h2>\n <ol>\n <li>This is the first list item.</li>\n <li>This is the second list item.</li>\n </ol>\n <p>A code block:</p>\n <pre><code>return 1 &lt; 2 ? shell_exec('echo $input | $markdown_script') : 0;</code></pre>\n </blockquote>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "> ## This is a header.\n> \n> 1. This is the first list item.\n> 2. This is the second list item.\n> \n> A code block:\n> \n> ```\n> return 1 < 2 ? shell_exec('echo $input | $markdown_script') : 0;\n> ```"
+ },
+ {
+ "id": "upstream_076",
+ "name": "multiple divs",
+ "html": "\n <div>A div</div>\n <div>Another div</div>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "A div\n\nAnother div"
+ },
+ {
+ "id": "upstream_077",
+ "name": "multiple divs",
+ "html": "\n <div>A div</div>\n <div>Another div</div>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "A div\n\nAnother div"
+ },
+ {
+ "id": "upstream_078",
+ "name": "comment",
+ "html": "<!-- comment -->",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": ""
+ },
+ {
+ "id": "upstream_079",
+ "name": "pre/code with comment",
+ "html": "\n <pre><code>Hello<!-- comment --> world</code></pre>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "```\nHello world\n```"
+ },
+ {
+ "id": "upstream_080",
+ "name": "leading whitespace in heading",
+ "html": "<h3>\n h3 with leading whitespace</h3>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "### h3 with leading whitespace"
+ },
+ {
+ "id": "upstream_081",
+ "name": "trailing whitespace in li",
+ "html": "\n <ol>\n <li>Chapter One\n <ol>\n <li>Section One</li>\n <li>Section Two with trailing whitespace </li>\n <li>Section Three with trailing whitespace </li>\n </ol>\n </li>\n <li>Chapter Two</li>\n <li>Chapter Three with trailing whitespace </li>\n </ol>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "1. Chapter One\n 1. Section One\n 2. Section Two with trailing whitespace\n 3. Section Three with trailing whitespace\n2. Chapter Two\n3. Chapter Three with trailing whitespace"
+ },
+ {
+ "id": "upstream_082",
+ "name": "multilined and bizarre formatting",
+ "html": "\n <ul>\n <li>\n Indented li with leading/trailing newlines\n </li>\n <li>\n <strong>Strong with trailing space inside li with leading/trailing whitespace </strong> </li>\n <li>li without whitespace</li>\n <li> Leading space, text, lots of whitespace …\n text\n </li>\n \n </ul>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "- Indented li with leading/trailing newlines\n- **Strong with trailing space inside li with leading/trailing whitespace**\n- li without whitespace\n- Leading space, text, lots of whitespace … text"
+ },
+ {
+ "id": "upstream_083",
+ "name": "whitespace between inline elements",
+ "html": "\n <p>I <a href=\"http://example.com/need\">need</a> <a href=\"http://www.example.com/more\">more</a> spaces!</p>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "I [need](http://example.com/need) [more](http://www.example.com/more) spaces!"
+ },
+ {
+ "id": "upstream_084",
+ "name": "whitespace in inline elements",
+ "html": "Text with no space after the period.<em> Text in em with leading/trailing spaces </em><strong>text in strong with trailing space </strong>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Text with no space after the period. *Text in em with leading/trailing spaces* **text in strong with trailing space**"
+ },
+ {
+ "id": "upstream_085",
+ "name": "whitespace in nested inline elements",
+ "html": "Text at root <strong><a href=\"http://www.example.com\">link text with trailing space in strong </a></strong>more text at root",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Text at root **[link text with trailing space in strong](http://www.example.com)** more text at root"
+ },
+ {
+ "id": "upstream_086",
+ "name": "blank inline elements",
+ "html": "\n Text before blank em … <em></em> text after blank em\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Text before blank em … text after blank em"
+ },
+ {
+ "id": "upstream_087",
+ "name": "blank block elements",
+ "html": "\n Text before blank div … <div></div> text after blank div\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Text before blank div …\n\ntext after blank div"
+ },
+ {
+ "id": "upstream_088",
+ "name": "blank inline element with br",
+ "html": "<strong><br></strong>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": ""
+ },
+ {
+ "id": "upstream_089",
+ "name": "whitespace between blocks",
+ "html": "<div><div>Content in a nested div</div></div>\n<div>Content in another div</div>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Content in a nested div\n\nContent in another div"
+ },
+ {
+ "id": "upstream_090",
+ "name": "escaping backslashes",
+ "html": "backslash \\",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "backslash \\\\"
+ },
+ {
+ "id": "upstream_091",
+ "name": "escaping headings with #",
+ "html": "### This is not a heading",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\### This is not a heading"
+ },
+ {
+ "id": "upstream_092",
+ "name": "not escaping # outside of a heading",
+ "html": "#This is not # a heading",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "#This is not # a heading"
+ },
+ {
+ "id": "upstream_093",
+ "name": "escaping em markdown with *",
+ "html": "To add emphasis, surround text with *. For example: *this is emphasis*",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "To add emphasis, surround text with \\*. For example: \\*this is emphasis\\*"
+ },
+ {
+ "id": "upstream_094",
+ "name": "escaping em markdown with _",
+ "html": "To add emphasis, surround text with _. For example: _this is emphasis_",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "To add emphasis, surround text with \\_. For example: \\_this is emphasis\\_"
+ },
+ {
+ "id": "upstream_095",
+ "name": "not escaping within code",
+ "html": "<pre><code>def this_is_a_method; end;</code></pre>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "```\ndef this_is_a_method; end;\n```"
+ },
+ {
+ "id": "upstream_096",
+ "name": "escaping strong markdown with *",
+ "html": "To add strong emphasis, surround text with **. For example: **this is strong**",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "To add strong emphasis, surround text with \\*\\*. For example: \\*\\*this is strong\\*\\*"
+ },
+ {
+ "id": "upstream_097",
+ "name": "escaping strong markdown with _",
+ "html": "To add strong emphasis, surround text with __. For example: __this is strong__",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "To add strong emphasis, surround text with \\_\\_. For example: \\_\\_this is strong\\_\\_"
+ },
+ {
+ "id": "upstream_098",
+ "name": "escaping hr markdown with *",
+ "html": "* * *",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\* \\* \\*"
+ },
+ {
+ "id": "upstream_099",
+ "name": "escaping hr markdown with -",
+ "html": "- - -",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\- - -"
+ },
+ {
+ "id": "upstream_100",
+ "name": "escaping hr markdown with _",
+ "html": "_ _ _",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\_ \\_ \\_"
+ },
+ {
+ "id": "upstream_101",
+ "name": "escaping hr markdown without spaces",
+ "html": "***",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\*\\*\\*"
+ },
+ {
+ "id": "upstream_102",
+ "name": "escaping hr markdown with more than 3 characters",
+ "html": "* * * * *",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\* \\* \\* \\* \\*"
+ },
+ {
+ "id": "upstream_103",
+ "name": "escaping ol markdown",
+ "html": "1984. by George Orwell",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "1984\\. by George Orwell"
+ },
+ {
+ "id": "upstream_104",
+ "name": "not escaping . outside of an ol",
+ "html": "1984.George Orwell wrote 1984.",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "1984.George Orwell wrote 1984."
+ },
+ {
+ "id": "upstream_105",
+ "name": "escaping ul markdown *",
+ "html": "* An unordered list item",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\* An unordered list item"
+ },
+ {
+ "id": "upstream_106",
+ "name": "escaping ul markdown -",
+ "html": "- An unordered list item",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\- An unordered list item"
+ },
+ {
+ "id": "upstream_107",
+ "name": "escaping ul markdown +",
+ "html": "+ An unordered list item",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\+ An unordered list item"
+ },
+ {
+ "id": "upstream_108",
+ "name": "not escaping - outside of a ul",
+ "html": "Hello-world, 45 - 3 is 42",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Hello-world, 45 - 3 is 42"
+ },
+ {
+ "id": "upstream_109",
+ "name": "not escaping + outside of a ul",
+ "html": "+1 and another +",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "+1 and another +"
+ },
+ {
+ "id": "upstream_110",
+ "name": "escaping *",
+ "html": "You can use * for multiplication",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "You can use \\* for multiplication"
+ },
+ {
+ "id": "upstream_111",
+ "name": "escaping ** inside strong tags",
+ "html": "<strong>**test</strong>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "**\\*\\*test**"
+ },
+ {
+ "id": "upstream_112",
+ "name": "escaping _ inside em tags",
+ "html": "<em>test_italics</em>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "*test\\_italics*"
+ },
+ {
+ "id": "upstream_113",
+ "name": "escaping > as blockquote",
+ "html": "&gt; Blockquote in markdown",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\> Blockquote in markdown"
+ },
+ {
+ "id": "upstream_114",
+ "name": "escaping > as blockquote without space",
+ "html": "&gt;Blockquote in markdown",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\>Blockquote in markdown"
+ },
+ {
+ "id": "upstream_115",
+ "name": "not escaping > outside of a blockquote",
+ "html": "42 &gt; 1",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "42 > 1"
+ },
+ {
+ "id": "upstream_116",
+ "name": "escaping code",
+ "html": "`not code`",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\`not code\\`"
+ },
+ {
+ "id": "upstream_117",
+ "name": "escaping []",
+ "html": "[This] is a sentence with brackets",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\[This\\] is a sentence with brackets"
+ },
+ {
+ "id": "upstream_118",
+ "name": "escaping [",
+ "html": "<a href=\"http://www.example.com\">c[iao</a>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "[c\\[iao](http://www.example.com)"
+ },
+ {
+ "id": "upstream_119",
+ "name": "escaping * performance",
+ "html": "fasdf *883 asdf wer qweasd fsd asdf asdfaqwe rqwefrsdf",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "fasdf \\*883 asdf wer qweasd fsd asdf asdfaqwe rqwefrsdf"
+ },
+ {
+ "id": "upstream_120",
+ "name": "escaping multiple asterisks",
+ "html": "<p>* * ** It aims to be*</p>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\* \\* \\*\\* It aims to be\\*"
+ },
+ {
+ "id": "upstream_121",
+ "name": "escaping delimiters around short words and numbers",
+ "html": "<p>_Really_? Is that what it _is_? A **2000** year-old computer?</p>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "\\_Really\\_? Is that what it \\_is\\_? A \\*\\*2000\\*\\* year-old computer?"
+ },
+ {
+ "id": "upstream_122",
+ "name": "non-markdown block elements",
+ "html": "\n Foo\n <div>Bar</div>\n Baz\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Foo\n\nBar\n\nBaz"
+ },
+ {
+ "id": "upstream_123",
+ "name": "non-markdown inline elements",
+ "html": "\n Foo <span>Bar</span>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Foo Bar"
+ },
+ {
+ "id": "upstream_124",
+ "name": "blank inline elements",
+ "html": "\n Hello <em></em>world\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Hello world"
+ },
+ {
+ "id": "upstream_125",
+ "name": "elements with a single void element",
+ "html": "\n <p><img src=\"http://example.com/logo.png\"></p>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![](http://example.com/logo.png)"
+ },
+ {
+ "id": "upstream_126",
+ "name": "elements with a nested void element",
+ "html": "\n <p><span><img src=\"http://example.com/logo.png\"></span></p>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "![](http://example.com/logo.png)"
+ },
+ {
+ "id": "upstream_127",
+ "name": "text separated by a space in an element",
+ "html": "\n <p>Foo<span> </span>Bar</p>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Foo Bar"
+ },
+ {
+ "id": "upstream_128",
+ "name": "text separated by a non-breaking space in an element",
+ "html": "\n <p>Foo<span>&nbsp;</span>Bar</p>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Foo Bar"
+ },
+ {
+ "id": "upstream_129",
+ "name": "triple tildes inside code",
+ "html": "\n<pre><code>~~~\nCode\n~~~\n</code></pre>\n ",
+ "original_options": {
+ "codeBlockStyle": "fenced",
+ "fence": "~~~"
+ },
+ "preformatted_code": false,
+ "markdown": "```\n~~~\nCode\n~~~\n```"
+ },
+ {
+ "id": "upstream_130",
+ "name": "triple ticks inside code",
+ "html": "\n<pre><code>```\nCode\n```\n</code></pre>\n ",
+ "original_options": {
+ "codeBlockStyle": "fenced",
+ "fence": "```"
+ },
+ "preformatted_code": false,
+ "markdown": "````\n```\nCode\n```\n````"
+ },
+ {
+ "id": "upstream_131",
+ "name": "four ticks inside code",
+ "html": "\n<pre><code>````\nCode\n````\n</code></pre>\n ",
+ "original_options": {
+ "codeBlockStyle": "fenced",
+ "fence": "```"
+ },
+ "preformatted_code": false,
+ "markdown": "`````\n````\nCode\n````\n`````"
+ },
+ {
+ "id": "upstream_132",
+ "name": "empty line in start/end of code block",
+ "html": "\n<pre><code>\nCode\n\n</code></pre>\n ",
+ "original_options": {
+ "codeBlockStyle": "fenced",
+ "fence": "```"
+ },
+ "preformatted_code": false,
+ "markdown": "```\n\nCode\n\n```"
+ },
+ {
+ "id": "upstream_133",
+ "name": "text separated by ASCII and nonASCII space in an element",
+ "html": "\n <p>Foo<span> &nbsp; </span>Bar</p>\n ",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "Foo   Bar"
+ },
+ {
+ "id": "upstream_134",
+ "name": "list-like text with non-breaking spaces",
+ "html": "&nbsp;1. First<br>&nbsp;2. Second",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": " 1. First \n 2. Second"
+ },
+ {
+ "id": "upstream_135",
+ "name": "element with trailing nonASCII WS followed by nonWS",
+ "html": "<i>foo&nbsp;</i>bar",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "*foo* bar"
+ },
+ {
+ "id": "upstream_136",
+ "name": "element with trailing nonASCII WS followed by nonASCII WS",
+ "html": "<i>foo&nbsp;</i>&nbsp;bar",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "*foo*  bar"
+ },
+ {
+ "id": "upstream_137",
+ "name": "element with trailing ASCII WS followed by nonASCII WS",
+ "html": "<i>foo </i>&nbsp;bar",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "*foo*  bar"
+ },
+ {
+ "id": "upstream_138",
+ "name": "element with trailing nonASCII WS followed by ASCII WS",
+ "html": "<i>foo&nbsp;</i> bar",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "*foo*  bar"
+ },
+ {
+ "id": "upstream_139",
+ "name": "nonWS followed by element with leading nonASCII WS",
+ "html": "foo<i>&nbsp;bar</i>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "foo *bar*"
+ },
+ {
+ "id": "upstream_140",
+ "name": "nonASCII WS followed by element with leading nonASCII WS",
+ "html": "foo&nbsp;<i>&nbsp;bar</i>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "foo  *bar*"
+ },
+ {
+ "id": "upstream_141",
+ "name": "nonASCII WS followed by element with leading ASCII WS",
+ "html": "foo&nbsp;<i> bar</i>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "foo  *bar*"
+ },
+ {
+ "id": "upstream_142",
+ "name": "ASCII WS followed by element with leading nonASCII WS",
+ "html": "foo <i>&nbsp;bar</i>",
+ "original_options": {},
+ "preformatted_code": false,
+ "markdown": "foo  *bar*"
+ },
+ {
+ "id": "upstream_143",
+ "name": "preformatted code with leading whitespace",
+ "html": "Four spaces <code> make an indented code block in Markdown</code>",
+ "original_options": {
+ "preformattedCode": true
+ },
+ "preformatted_code": true,
+ "markdown": "Four spaces ` make an indented code block in Markdown`"
+ },
+ {
+ "id": "upstream_144",
+ "name": "preformatted code with trailing whitespace",
+ "html": "<code>A line break </code> <b> note the spaces</b>",
+ "original_options": {
+ "preformattedCode": true
+ },
+ "preformatted_code": true,
+ "markdown": "`A line break ` **note the spaces**"
+ },
+ {
+ "id": "upstream_145",
+ "name": "preformatted code tightly surrounded",
+ "html": "<b>tight</b><code>code</code><b>wrap</b>",
+ "original_options": {
+ "preformattedCode": true
+ },
+ "preformatted_code": true,
+ "markdown": "**tight**`code`**wrap**"
+ },
+ {
+ "id": "upstream_146",
+ "name": "preformatted code loosely surrounded",
+ "html": "<b>not so tight </b><code>code</code><b> wrap</b>",
+ "original_options": {
+ "preformattedCode": true
+ },
+ "preformatted_code": true,
+ "markdown": "**not so tight** `code` **wrap**"
+ },
+ {
+ "id": "upstream_147",
+ "name": "preformatted code with newlines",
+ "html": "\n<code>\n\n nasty\ncode\n\n</code>\n ",
+ "original_options": {
+ "preformattedCode": true
+ },
+ "preformatted_code": true,
+ "markdown": "` nasty code `"
+ },
+ {
+ "id": "extra_empty_href",
+ "name": "empty href keeps link text",
+ "html": "before<a href=''>label</a>after",
+ "preformatted_code": false,
+ "markdown": "beforelabelafter"
+ },
+ {
+ "id": "extra_empty_link",
+ "name": "empty link label",
+ "html": "before<a href='/target'></a>after",
+ "preformatted_code": false,
+ "markdown": "before[](/target)after"
+ },
+ {
+ "id": "extra_blank_link",
+ "name": "whitespace-only link label",
+ "html": "before<a href='/target'> </a>after",
+ "preformatted_code": false,
+ "markdown": "before [](/target)after"
+ },
+ {
+ "id": "extra_missing_image_src",
+ "name": "image alt without source",
+ "html": "before<img alt='missing'>after",
+ "preformatted_code": false,
+ "markdown": "beforeafter"
+ },
+ {
+ "id": "extra_img_alt_whitespace",
+ "name": "image alt with multiple lines and indentation",
+ "html": "<img src='/x' alt='one\n \n two'>",
+ "preformatted_code": false,
+ "markdown": "![one\ntwo](/x)"
+ },
+ {
+ "id": "extra_img_alt_code",
+ "name": "image label containing code delimiters",
+ "html": "<img src='/x' alt='a`b'>",
+ "preformatted_code": false,
+ "markdown": "![a\\`b](/x)"
+ },
+ {
+ "id": "extra_adjacent_lists",
+ "name": "adjacent unordered lists stay separate",
+ "html": "<ul><li>one</li></ul><ul><li>two</li></ul>",
+ "preformatted_code": false,
+ "markdown": "- one\n\n- two"
+ },
+ {
+ "id": "extra_adjacent_ordered_lists",
+ "name": "adjacent ordered lists stay separate",
+ "html": "<ol><li>one</li></ol><ol start='4'><li>four</li></ol>",
+ "preformatted_code": false,
+ "markdown": "1. one\n\n4. four"
+ },
+ {
+ "id": "extra_empty_list",
+ "name": "empty list",
+ "html": "before<ul></ul>after",
+ "preformatted_code": false,
+ "markdown": "before\n\nafter"
+ },
+ {
+ "id": "extra_empty_list_item",
+ "name": "empty list item",
+ "html": "<ul><li></li><li>text</li></ul>",
+ "preformatted_code": false,
+ "markdown": "- text"
+ },
+ {
+ "id": "extra_loose_list",
+ "name": "paragraph list items stay loose",
+ "html": "<ul><li><p>one</p></li><li><p>two</p></li></ul>",
+ "preformatted_code": false,
+ "markdown": "- one\n \n- two"
+ },
+ {
+ "id": "extra_mixed_loose_list",
+ "name": "paragraph followed by a plain list item",
+ "html": "<ul><li><p>one</p></li><li>two</li></ul>",
+ "preformatted_code": false,
+ "markdown": "- one\n \n- two"
+ },
+ {
+ "id": "extra_list_nested_only",
+ "name": "list item containing only a nested list",
+ "html": "<ul><li><ul><li>inner</li></ul></li></ul>",
+ "preformatted_code": false,
+ "markdown": "- - inner"
+ },
+ {
+ "id": "extra_list_nested_tail",
+ "name": "text after a nested list",
+ "html": "<ul><li>start<ul><li>inner</li></ul>end</li></ul>",
+ "preformatted_code": false,
+ "markdown": "- start\n - innerend"
+ },
+ {
+ "id": "extra_list_nested_tail_p",
+ "name": "paragraph after a nested list",
+ "html": "<ul><li><p>start</p><ul><li>inner</li></ul><p>end</p></li></ul>",
+ "preformatted_code": false,
+ "markdown": "- start\n \n - inner\n \n end"
+ },
+ {
+ "id": "extra_list_code",
+ "name": "fenced code inside list item",
+ "html": "<ol start='9'><li><pre><code> first\nsecond\n</code></pre></li><li>end</li></ol>",
+ "preformatted_code": false,
+ "markdown": "9. ```\n first\n second\n ```\n \n10. end"
+ },
+ {
+ "id": "extra_list_quote",
+ "name": "list item containing only a quote",
+ "html": "<ul><li><blockquote><p>quote</p></blockquote></li></ul>",
+ "preformatted_code": false,
+ "markdown": "- > quote"
+ },
+ {
+ "id": "extra_empty_quote",
+ "name": "empty quote still separates blocks",
+ "html": "before<blockquote></blockquote>after",
+ "preformatted_code": false,
+ "markdown": "before\n\nafter"
+ },
+ {
+ "id": "extra_quote_blank_lines",
+ "name": "quote with paragraphs and hard breaks",
+ "html": "<blockquote><p>one<br>two</p><p>three</p></blockquote>",
+ "preformatted_code": false,
+ "markdown": "> one \n> two\n> \n> three"
+ },
+ {
+ "id": "extra_empty_heading",
+ "name": "empty heading boundary",
+ "html": "before<h2></h2>after",
+ "preformatted_code": false,
+ "markdown": "before\n\nafter"
+ },
+ {
+ "id": "extra_empty_pre",
+ "name": "empty pre",
+ "html": "before<pre></pre>after",
+ "preformatted_code": false,
+ "markdown": "before\n\nafter"
+ },
+ {
+ "id": "extra_empty_pre_code",
+ "name": "empty pre code",
+ "html": "before<pre><code></code></pre>after",
+ "preformatted_code": false,
+ "markdown": "before\n\nafter"
+ },
+ {
+ "id": "extra_blank_pre_code",
+ "name": "whitespace-only pre code",
+ "html": "<pre><code> \n</code></pre>",
+ "preformatted_code": false,
+ "markdown": ""
+ },
+ {
+ "id": "extra_adjacent_code",
+ "name": "adjacent code spans",
+ "html": "<code>a</code><code>b</code>",
+ "preformatted_code": false,
+ "markdown": "`a``b`"
+ },
+ {
+ "id": "extra_code_gap",
+ "name": "separated code spans",
+ "html": "<code>a</code> <code>b</code>",
+ "preformatted_code": false,
+ "markdown": "`a` `b`"
+ },
+ {
+ "id": "extra_code_nested_format",
+ "name": "markup inside inline code",
+ "html": "<code><strong>a</strong><em>b</em></code>",
+ "preformatted_code": false,
+ "markdown": "`**a***b*`"
+ },
+ {
+ "id": "extra_code_many_runs",
+ "name": "code span chooses an unused delimiter length",
+ "html": "<code>a```b`````c</code>",
+ "preformatted_code": false,
+ "markdown": "`a```b`````c`"
+ },
+ {
+ "id": "extra_code_only_ticks",
+ "name": "code containing only backticks",
+ "html": "<code>```</code>",
+ "preformatted_code": false,
+ "markdown": "` ``` `"
+ },
+ {
+ "id": "extra_code_nbsp",
+ "name": "code with nonbreaking boundary spaces",
+ "html": "before<code>&nbsp;a&nbsp;</code>after",
+ "preformatted_code": false,
+ "markdown": "before `a` after"
+ },
+ {
+ "id": "extra_preformatted_internal",
+ "name": "preformatted inline code preserves multiple spaces",
+ "html": "<code> a b </code>",
+ "original_options": {
+ "preformattedCode": true
+ },
+ "preformatted_code": true,
+ "markdown": "` a b `"
+ },
+ {
+ "id": "extra_preformatted_blank",
+ "name": "preformatted whitespace-only code",
+ "html": "before<code> </code>after",
+ "original_options": {
+ "preformattedCode": true
+ },
+ "preformatted_code": true,
+ "markdown": "beforeafter"
+ },
+ {
+ "id": "extra_literal_entities",
+ "name": "decoded entity text remains literal",
+ "html": "<p>&amp;copy; &amp;lt; &amp;#32;</p>",
+ "preformatted_code": false,
+ "markdown": "&copy; &lt; &#32;"
+ },
+ {
+ "id": "extra_literal_html",
+ "name": "literal HTML tags remain visible text",
+ "html": "<p>&lt;b&gt;literal&lt;/b&gt;</p>",
+ "preformatted_code": false,
+ "markdown": "<b>literal</b>"
+ },
+ {
+ "id": "extra_split_list_marker",
+ "name": "ordered marker split across elements",
+ "html": "<p>12<span>.</span> text</p>",
+ "preformatted_code": false,
+ "markdown": "12. text"
+ },
+ {
+ "id": "extra_split_emphasis_literal",
+ "name": "asterisks split across elements",
+ "html": "<p>*<span>word</span>*</p>",
+ "preformatted_code": false,
+ "markdown": "\\*word\\*"
+ },
+ {
+ "id": "extra_link_backslash",
+ "name": "link destination with backslashes",
+ "html": "<a href='a\\b'>link</a>",
+ "preformatted_code": false,
+ "markdown": "[link](a\\b)"
+ },
+ {
+ "id": "extra_link_entities",
+ "name": "link destination with literal entity syntax",
+ "html": "<a href='/?q=&amp;copy;'>link</a>",
+ "preformatted_code": false,
+ "markdown": "[link](/?q=&copy;)"
+ },
+ {
+ "id": "extra_link_leading_space",
+ "name": "leading whitespace inside link label",
+ "html": "before<a href='/'> label</a>after",
+ "preformatted_code": false,
+ "markdown": "before [label](/)after"
+ },
+ {
+ "id": "extra_nested_link_emphasis",
+ "name": "emphasis nested in link",
+ "html": "<a href='/'><strong><em>text</em></strong></a>",
+ "preformatted_code": false,
+ "markdown": "[***text***](/)"
+ },
+ {
+ "id": "extra_emphasis_nested_link",
+ "name": "link nested in emphasis",
+ "html": "<em><a href='/'>text</a></em>",
+ "preformatted_code": false,
+ "markdown": "*[text](/)*"
+ },
+ {
+ "id": "extra_emphasis_intraword_link",
+ "name": "link in intraword emphasis",
+ "html": "before<em><a href='/'>text</a></em>after",
+ "preformatted_code": false,
+ "markdown": "before*[text](/)*after"
+ },
+ {
+ "id": "extra_emphasis_intraword_punctuation",
+ "name": "intraword emphasis around punctuation",
+ "html": "before<em>(text)</em>after",
+ "preformatted_code": false,
+ "markdown": "before*(text)*after"
+ },
+ {
+ "id": "extra_empty_emphasis",
+ "name": "empty formatting preserves surrounding spaces",
+ "html": "a<strong> </strong>b<em> </em>c",
+ "preformatted_code": false,
+ "markdown": "a b c"
+ },
+ {
+ "id": "extra_nbsp_comment",
+ "name": "nonbreaking whitespace around ignored comment",
+ "html": "a&nbsp;<!--x--> b",
+ "preformatted_code": false,
+ "markdown": "a  b"
+ },
+ {
+ "id": "extra_leading_br",
+ "name": "leading hard break",
+ "html": "<p><br>text</p>",
+ "preformatted_code": false,
+ "markdown": " \ntext"
+ },
+ {
+ "id": "extra_consecutive_br",
+ "name": "consecutive hard breaks",
+ "html": "<p>one<br><br>two</p>",
+ "preformatted_code": false,
+ "markdown": "one \n \ntwo"
+ },
+ {
+ "id": "extra_blank_br_container",
+ "name": "otherwise empty span containing hard break",
+ "html": "a<span><br></span>b",
+ "preformatted_code": false,
+ "markdown": "a \nb"
+ },
+ {
+ "id": "extra_empty_block_inside_inline",
+ "name": "empty block inside emphasis",
+ "html": "<strong>a<div></div>b</strong>",
+ "preformatted_code": false,
+ "markdown": "**a\n\nb**"
+ },
+ {
+ "id": "extra_audio_boundary",
+ "name": "audio fallback content is a block",
+ "html": "before<audio>fallback</audio>after",
+ "preformatted_code": false,
+ "markdown": "before\n\nfallback\n\nafter"
+ },
+ {
+ "id": "extra_output_boundary",
+ "name": "output element block boundary",
+ "html": "before<output>result</output>after",
+ "preformatted_code": false,
+ "markdown": "before\n\nresult\n\nafter"
+ }
+ ]
+}
diff --git a/moli-html2md/tests/turndown/cases.rs b/moli-html2md/tests/turndown/cases.rs
new file mode 100644
index 000000000..0d5e0e2a9
--- /dev/null
+++ b/moli-html2md/tests/turndown/cases.rs
@@ -0,0 +1,198 @@
+// Generated by generate.mjs. See README.md and LICENSE.
+case!(upstream_001, 0);
+case!(upstream_002, 1);
+case!(upstream_003, 2);
+case!(upstream_004, 3);
+case!(upstream_005, 4);
+case!(upstream_006, 5);
+case!(upstream_007, 6);
+case!(upstream_008, 7);
+case!(upstream_009, 8);
+case!(upstream_010, 9);
+case!(upstream_011, 10);
+case!(upstream_012, 11);
+case!(upstream_013, 12);
+case!(upstream_014, 13);
+case!(upstream_015, 14);
+case!(upstream_016, 15);
+case!(upstream_017, 16);
+case!(upstream_018, 17);
+case!(upstream_019, 18);
+case!(upstream_020, 19);
+case!(upstream_021, 20);
+case!(upstream_022, 21);
+case!(upstream_023, 22);
+case!(upstream_024, 23);
+case!(upstream_025, 24);
+case!(upstream_026, 25);
+case!(upstream_027, 26);
+case!(upstream_028, 27);
+case!(upstream_029, 28);
+case!(upstream_030, 29);
+case!(upstream_031, 30);
+case!(upstream_032, 31);
+case!(upstream_033, 32);
+case!(upstream_034, 33);
+case!(upstream_035, 34);
+case!(upstream_036, 35);
+case!(upstream_037, 36);
+case!(upstream_038, 37);
+case!(upstream_039, 38);
+case!(upstream_040, 39);
+case!(upstream_041, 40);
+case!(upstream_042, 41);
+case!(upstream_043, 42);
+case!(upstream_044, 43);
+case!(upstream_045, 44);
+case!(upstream_046, 45);
+case!(upstream_047, 46);
+case!(upstream_048, 47);
+case!(upstream_049, 48);
+case!(upstream_050, 49);
+case!(upstream_051, 50);
+case!(upstream_052, 51);
+case!(upstream_053, 52);
+case!(upstream_054, 53);
+case!(upstream_055, 54);
+case!(upstream_056, 55);
+case!(upstream_057, 56);
+case!(upstream_058, 57);
+case!(upstream_059, 58);
+case!(upstream_060, 59);
+case!(upstream_061, 60);
+case!(upstream_062, 61);
+case!(upstream_063, 62);
+case!(upstream_064, 63);
+case!(upstream_065, 64);
+case!(upstream_066, 65);
+case!(upstream_067, 66);
+case!(upstream_068, 67);
+case!(upstream_069, 68);
+case!(upstream_070, 69);
+case!(upstream_071, 70);
+case!(upstream_072, 71);
+case!(upstream_073, 72);
+case!(upstream_074, 73);
+case!(upstream_075, 74);
+case!(upstream_076, 75);
+case!(upstream_077, 76);
+case!(upstream_078, 77);
+case!(upstream_079, 78);
+case!(upstream_080, 79);
+case!(upstream_081, 80);
+case!(upstream_082, 81);
+case!(upstream_083, 82);
+case!(upstream_084, 83);
+case!(upstream_085, 84);
+case!(upstream_086, 85);
+case!(upstream_087, 86);
+case!(upstream_088, 87);
+case!(upstream_089, 88);
+case!(upstream_090, 89);
+case!(upstream_091, 90);
+case!(upstream_092, 91);
+case!(upstream_093, 92);
+case!(upstream_094, 93);
+case!(upstream_095, 94);
+case!(upstream_096, 95);
+case!(upstream_097, 96);
+case!(upstream_098, 97);
+case!(upstream_099, 98);
+case!(upstream_100, 99);
+case!(upstream_101, 100);
+case!(upstream_102, 101);
+case!(upstream_103, 102);
+case!(upstream_104, 103);
+case!(upstream_105, 104);
+case!(upstream_106, 105);
+case!(upstream_107, 106);
+case!(upstream_108, 107);
+case!(upstream_109, 108);
+case!(upstream_110, 109);
+case!(upstream_111, 110);
+case!(upstream_112, 111);
+case!(upstream_113, 112);
+case!(upstream_114, 113);
+case!(upstream_115, 114);
+case!(upstream_116, 115);
+case!(upstream_117, 116);
+case!(upstream_118, 117);
+case!(upstream_119, 118);
+case!(upstream_120, 119);
+case!(upstream_121, 120);
+case!(upstream_122, 121);
+case!(upstream_123, 122);
+case!(upstream_124, 123);
+case!(upstream_125, 124);
+case!(upstream_126, 125);
+case!(upstream_127, 126);
+case!(upstream_128, 127);
+case!(upstream_129, 128);
+case!(upstream_130, 129);
+case!(upstream_131, 130);
+case!(upstream_132, 131);
+case!(upstream_133, 132);
+case!(upstream_134, 133);
+case!(upstream_135, 134);
+case!(upstream_136, 135);
+case!(upstream_137, 136);
+case!(upstream_138, 137);
+case!(upstream_139, 138);
+case!(upstream_140, 139);
+case!(upstream_141, 140);
+case!(upstream_142, 141);
+case!(upstream_143, 142);
+case!(upstream_144, 143);
+case!(upstream_145, 144);
+case!(upstream_146, 145);
+case!(upstream_147, 146);
+case!(extra_empty_href, 147);
+case!(extra_empty_link, 148);
+case!(extra_blank_link, 149);
+case!(extra_missing_image_src, 150);
+case!(extra_img_alt_whitespace, 151);
+case!(extra_img_alt_code, 152);
+case!(extra_adjacent_lists, 153);
+case!(extra_adjacent_ordered_lists, 154);
+case!(extra_empty_list, 155);
+case!(extra_empty_list_item, 156);
+case!(extra_loose_list, 157);
+case!(extra_mixed_loose_list, 158);
+case!(extra_list_nested_only, 159);
+case!(extra_list_nested_tail, 160);
+case!(extra_list_nested_tail_p, 161);
+case!(extra_list_code, 162);
+case!(extra_list_quote, 163);
+case!(extra_empty_quote, 164);
+case!(extra_quote_blank_lines, 165);
+case!(extra_empty_heading, 166);
+case!(extra_empty_pre, 167);
+case!(extra_empty_pre_code, 168);
+case!(extra_blank_pre_code, 169);
+case!(extra_adjacent_code, 170);
+case!(extra_code_gap, 171);
+case!(extra_code_nested_format, 172);
+case!(extra_code_many_runs, 173);
+case!(extra_code_only_ticks, 174);
+case!(extra_code_nbsp, 175);
+case!(extra_preformatted_internal, 176);
+case!(extra_preformatted_blank, 177);
+case!(extra_literal_entities, 178);
+case!(extra_literal_html, 179);
+case!(extra_split_list_marker, 180);
+case!(extra_split_emphasis_literal, 181);
+case!(extra_link_backslash, 182);
+case!(extra_link_entities, 183);
+case!(extra_link_leading_space, 184);
+case!(extra_nested_link_emphasis, 185);
+case!(extra_emphasis_nested_link, 186);
+case!(extra_emphasis_intraword_link, 187);
+case!(extra_emphasis_intraword_punctuation, 188);
+case!(extra_empty_emphasis, 189);
+case!(extra_nbsp_comment, 190);
+case!(extra_leading_br, 191);
+case!(extra_consecutive_br, 192);
+case!(extra_blank_br_container, 193);
+case!(extra_empty_block_inside_inline, 194);
+case!(extra_audio_boundary, 195);
+case!(extra_output_boundary, 196);
diff --git a/moli-html2md/tests/turndown/expectations.json b/moli-html2md/tests/turndown/expectations.json
new file mode 100644
index 000000000..f9b20051c
--- /dev/null
+++ b/moli-html2md/tests/turndown/expectations.json
@@ -0,0 +1,50 @@
+{
+ "upstream_058": {
+ "markdown": "```\n~~~ foo\n```",
+ "reason": "Moli fences bare pre elements as well as pre/code, preserving preformatted text. Turndown only recognizes a pre/code pair."
+ },
+ "extra_adjacent_code": {
+ "markdown": "`ab`",
+ "reason": "Adjacent code elements contain ab. Turndown joins their delimiters into one code span whose text contains two extra backticks."
+ },
+ "extra_code_nested_format": {
+ "markdown": "`ab`",
+ "reason": "Code captures raw descendant text. Turndown includes the Markdown markers of nested strong/em elements in the code text."
+ },
+ "extra_emphasis_intraword_link": {
+ "markdown": "before<em>[text](/)</em>after",
+ "reason": "Turndown's star delimiters do not open emphasis at this punctuation boundary. Inline HTML preserves the original emphasized link."
+ },
+ "extra_emphasis_intraword_punctuation": {
+ "markdown": "before<em>(text)</em>after",
+ "reason": "CommonMark does not recognize Turndown's intraword delimiters next to punctuation. Inline HTML preserves emphasis without literal stars."
+ },
+ "extra_empty_block_inside_inline": {
+ "markdown": "**a**\n\n**b**",
+ "reason": "Emphasis must close and reopen across paragraph boundaries; delimiters spanning the blank line become literal text."
+ },
+ "extra_list_nested_tail": {
+ "markdown": "- start\n - inner\n\n end",
+ "reason": "The trailing text belongs to the outer item. Turndown appends it to the nested item, losing the block boundary and joining words."
+ },
+ "extra_literal_entities": {
+ "markdown": "&amp;copy; &amp;lt; &amp;#32;",
+ "reason": "DOM text is already decoded. Escape literal entity syntax so a Markdown reader does not decode it a second time."
+ },
+ "extra_literal_html": {
+ "markdown": "\\<b\\>literal\\</b\\>",
+ "reason": "Escaped HTML in the source is visible text, not markup. Turndown turns the decoded text into executable Markdown HTML tags."
+ },
+ "extra_link_entities": {
+ "markdown": "[link](/?q=&amp;copy;)",
+ "reason": "The literal entity syntax in the decoded href must not be decoded again into a different URL."
+ },
+ "extra_preformatted_blank": {
+ "markdown": "before` `after",
+ "reason": "Explicit preformatted-code mode preserves the spaces inside code. Turndown removes a whitespace-only code element entirely."
+ },
+ "extra_split_list_marker": {
+ "markdown": "12\\. text",
+ "reason": "Escape list markers across text-node boundaries. Turndown escapes nodes individually and accidentally creates an ordered list."
+ }
+}
diff --git a/moli-html2md/tests/turndown/generate.mjs b/moli-html2md/tests/turndown/generate.mjs
new file mode 100644
index 000000000..34072bdf9
--- /dev/null
+++ b/moli-html2md/tests/turndown/generate.mjs
@@ -0,0 +1,58 @@
+// Regenerate the offline reference corpus from the pinned Turndown checkout.
+// See README.md in this directory for dependencies and invocation.
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { readFileSync, writeFileSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const revision = 'aa84dfa3e2361edea8c43acbfc2b7a9363494bfd';
+const upstream = resolve(process.argv[2]);
+assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: upstream, encoding: 'utf8' }).trim(), revision);
+const require = createRequire(join(upstream, 'package.json'));
+const domino = require('@mixmark-io/domino');
+assert.equal(require('@mixmark-io/domino/package.json').version, '2.2.0');
+const destination = dirname(fileURLToPath(import.meta.url));
+const temporary = mkdtempSync(join(tmpdir(), 'moli-turndown-esm-'));
+try {
+ // Node's ESM loader needs file extensions; Turndown's build normally supplies
+ // these through Rollup. Only import paths and CommonJS require setup change.
+ for (const name of ['collapse-whitespace', 'commonmark-rules', 'html-parser', 'node', 'root-node', 'rules', 'turndown', 'utilities']) {
+ const original = readFileSync(join(upstream, 'src', name + '.js'), 'utf8');
+ const source = original.replace(/from '(\.\/[^']+)'/g, "from '$1.mjs'");
+ const header = `import { createRequire } from 'node:module';\nconst require = createRequire(${JSON.stringify(join(upstream, 'package.json'))});\n`;
+ writeFileSync(join(temporary, name + '.mjs'), header + source);
+ }
+ const { default: Turndown } = await import(pathToFileURL(join(temporary, 'turndown.mjs')));
+ const options = {
+ headingStyle: 'atx', hr: '---', bulletListMarker: '-',
+ codeBlockStyle: 'fenced', fence: '```', emDelimiter: '*',
+ strongDelimiter: '**', linkStyle: 'inlined', br: ' ',
+ };
+ const document = domino.createDocument(readFileSync(join(upstream, 'test', 'index.html'), 'utf8'));
+ const upstreamCases = Array.from(document.querySelectorAll('.case')).map((node, index) => ({
+ id: `upstream_${String(index + 1).padStart(3, '0')}`,
+ name: node.getAttribute('data-name'),
+ html: node.querySelector('.input').innerHTML,
+ original_options: JSON.parse(node.getAttribute('data-options') || '{}'),
+ }));
+ const extra = JSON.parse(readFileSync(join(destination, 'additional.json'), 'utf8'));
+ const cases = [...upstreamCases, ...extra].map(test => {
+ const preformattedCode = test.original_options?.preformattedCode || false;
+ const reference = new Turndown({ ...options, preformattedCode });
+ return { ...test, preformatted_code: preformattedCode, markdown: reference.turndown(test.html) };
+ });
+ const corpus = {
+ upstream: { repository: 'https://github.com/mixmark-io/turndown', revision, version: '7.2.4', domino: '2.2.0' },
+ options,
+ cases,
+ };
+ writeFileSync(join(destination, 'cases.json'), JSON.stringify(corpus, null, 2) + '\n');
+ writeFileSync(join(destination, 'LICENSE'), readFileSync(join(upstream, 'LICENSE')));
+ writeFileSync(join(destination, 'cases.rs'), '// Generated by generate.mjs. See README.md and LICENSE.\n' + cases.map((test, index) => `case!(${test.id}, ${index});`).join('\n') + '\n');
+ console.log(`Generated ${upstreamCases.length} upstream cases and ${extra.length} additional cases.`);
+} finally {
+ rmSync(temporary, { recursive: true, force: true });
+}
diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.toml
index b9f38b59d..d5ad20d59 100644
--- a/moli-renderer-v8/Cargo.toml
+++ b/moli-renderer-v8/Cargo.toml
@@ -53,6 +53,7 @@ moli-fetch = { path = "../moli-fetch" }
moli-file-api = { path = "../moli-file-api" }
moli-geometry = { path = "../moli-geometry" }
moli-http-cache = { path = "../moli-http-cache" }
+moli-html2md = { path = "../moli-html2md" }
moli-image = { path = "../moli-image" }
moli-import-map = { path = "../moli-import-map" }
moli-indexeddb = { path = "../moli-indexeddb" }
diff --git a/moli-renderer-v8/src/runtime/page_dump.rs b/moli-renderer-v8/src/runtime/page_dump.rs
index 07599a27c..fac217816 100644
--- a/moli-renderer-v8/src/runtime/page_dump.rs
+++ b/moli-renderer-v8/src/runtime/page_dump.rs
@@ -1,4 +1,4 @@
-use crate::dom::native::{NativeDom, NativeNodeId, NodeData};
+use crate::dom::native::{NativeDom, NativeNodeId};
use crate::runtime::page_surface::{
RendererPageDumpFormat, RendererPageDumpOptions, RendererPageDumpStripOptions,
};
@@ -8,6 +8,13 @@ use super::page_vm::PageVm;
impl PageVm {
pub(crate) fn render_page_dump(&mut self, options: RendererPageDumpOptions) -> String {
+ if options.format == RendererPageDumpFormat::Markdown
+ && !options.with_base
+ && !options.with_frames
+ && options.strip == RendererPageDumpStripOptions::default()
+ {
+ return render_markdown_document(self.vm().document_runtime.dom_host().dom());
+ }
let mut dom = self.vm().document_runtime.dom_host().dom().clone();
if options.with_base {
@@ -178,408 +185,12 @@ fn collect_node_ids(dom: &NativeDom, node_id: NativeNodeId, out: &mut Vec<Native
fn render_markdown_document(dom: &NativeDom) -> String {
let root = dom.body_node_id().unwrap_or(dom.document_node_id());
- let mut out = String::new();
- render_markdown_node(dom, root, &mut out, 0, MAX_DOM_OUTPUT_TREE_DEPTH);
- normalize_markdown(&out)
-}
-
-enum MarkdownFrame {
- Enter {
- node_id: NativeNodeId,
- list_depth: usize,
- remaining_tree_depth: usize,
- target: usize,
- },
- Append {
- target: usize,
- text: &'static str,
- },
- FinishInline {
- source: usize,
- target: usize,
- kind: MarkdownInlineKind,
- },
-}
-
-enum MarkdownInlineKind {
- Paragraph,
- Anchor { href: String },
- ListItem { list_depth: usize },
- Strong,
- Emphasis,
- Code,
-}
-
-fn render_markdown_node(
- dom: &NativeDom,
- node_id: NativeNodeId,
- out: &mut String,
- list_depth: usize,
- remaining_tree_depth: usize,
-) {
- render_markdown_nodes(dom, [node_id], out, list_depth, remaining_tree_depth);
-}
-
-fn render_markdown_nodes(
- dom: &NativeDom,
- roots: impl IntoIterator<Item = NativeNodeId>,
- out: &mut String,
- list_depth: usize,
- remaining_tree_depth: usize,
-) {
- let mut buffers = vec![String::new()];
- let mut stack = roots
- .into_iter()
- .map(|node_id| MarkdownFrame::Enter {
- node_id,
- list_depth,
- remaining_tree_depth,
- target: 0,
- })
- .collect::<Vec<_>>();
- stack.reverse();
-
- while let Some(frame) = stack.pop() {
- match frame {
- MarkdownFrame::Enter {
- node_id,
- list_depth,
- remaining_tree_depth,
- target,
- } => render_markdown_enter_node(
- dom,
- node_id,
- &mut buffers,
- &mut stack,
- list_depth,
- remaining_tree_depth,
- target,
- ),
- MarkdownFrame::Append { target, text } => buffers[target].push_str(text),
- MarkdownFrame::FinishInline {
- source,
- target,
- kind,
- } => {
- let text = std::mem::take(&mut buffers[source]);
- finish_inline_markdown(kind, &text, &mut buffers[target]);
- }
- }
- }
-
- out.push_str(&buffers[0]);
-}
-
-fn render_markdown_enter_node(
- dom: &NativeDom,
- node_id: NativeNodeId,
- buffers: &mut Vec<String>,
- stack: &mut Vec<MarkdownFrame>,
- list_depth: usize,
- remaining_tree_depth: usize,
- target: usize,
-) {
- let Some(next_tree_depth) = remaining_tree_depth.checked_sub(1) else {
- return;
- };
- let Some(node) = dom.node(node_id) else {
- return;
- };
-
- match node.kind() {
- NodeData::Document(_) | NodeData::DocumentFragment(_) => {
- push_child_markdown_frames(stack, dom, node_id, list_depth, next_tree_depth, target);
- }
- NodeData::Text(text) => {
- push_markdown_text(&mut buffers[target], text.data());
- }
- NodeData::CDataSection(cdata) => {
- push_markdown_text(&mut buffers[target], cdata.data());
- }
- NodeData::Comment(_) | NodeData::DocumentType(_) | NodeData::ProcessingInstruction(_) => {}
- NodeData::Element(element) => match element.local_name() {
- "head" | "script" | "style" | "noscript" => {}
- "br" => buffers[target].push('\n'),
- "hr" => buffers[target].push_str("\n---\n"),
- "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
- let level = element.local_name()[1..].parse::<usize>().unwrap_or(1);
- let text = collapse_whitespace(&dom.text_content(node_id).unwrap_or_default());
- if !text.is_empty() {
- buffers[target].push('\n');
- buffers[target].push_str(&"#".repeat(level));
- buffers[target].push(' ');
- buffers[target].push_str(&text);
- buffers[target].push_str("\n\n");
- }
- }
- "p" => {
- push_inline_children_markdown_frames(
- buffers,
- stack,
- dom,
- node_id,
- next_tree_depth,
- target,
- MarkdownInlineKind::Paragraph,
- );
- }
- "pre" => {
- let text = dom.text_content(node_id).unwrap_or_default();
- if !text.trim().is_empty() {
- buffers[target].push_str("\n```text\n");
- buffers[target].push_str(text.trim_end());
- buffers[target].push_str("\n```\n\n");
- }
- }
- "code" => {
- let text = dom.text_content(node_id).unwrap_or_default();
- finish_inline_markdown(MarkdownInlineKind::Code, &text, &mut buffers[target]);
- }
- "a" => {
- let href = dom.get_attribute(node_id, "href").unwrap_or_default();
- push_inline_children_markdown_frames(
- buffers,
- stack,
- dom,
- node_id,
- next_tree_depth,
- target,
- MarkdownInlineKind::Anchor { href },
- );
- }
- "img" => {
- let alt = dom.get_attribute(node_id, "alt").unwrap_or_default();
- let src = dom.get_attribute(node_id, "src").unwrap_or_default();
- buffers[target].push_str("![");
- buffers[target].push_str(&alt);
- buffers[target].push_str("](");
- buffers[target].push_str(&src);
- buffers[target].push(')');
- }
- "ul" => {
- stack.push(MarkdownFrame::Append { target, text: "\n" });
- push_child_markdown_frames(
- stack,
- dom,
- node_id,
- list_depth + 1,
- next_tree_depth,
- target,
- );
- stack.push(MarkdownFrame::Append { target, text: "\n" });
- }
- "ol" => {
- stack.push(MarkdownFrame::Append { target, text: "\n" });
- push_child_markdown_frames(
- stack,
- dom,
- node_id,
- list_depth + 1,
- next_tree_depth,
- target,
- );
- stack.push(MarkdownFrame::Append { target, text: "\n" });
- }
- "li" => {
- push_inline_children_markdown_frames(
- buffers,
- stack,
- dom,
- node_id,
- next_tree_depth,
- target,
- MarkdownInlineKind::ListItem { list_depth },
- );
- }
- "strong" | "b" => {
- push_inline_children_markdown_frames(
- buffers,
- stack,
- dom,
- node_id,
- next_tree_depth,
- target,
- MarkdownInlineKind::Strong,
- );
- }
- "em" | "i" => {
- push_inline_children_markdown_frames(
- buffers,
- stack,
- dom,
- node_id,
- next_tree_depth,
- target,
- MarkdownInlineKind::Emphasis,
- );
- }
- _ => {
- if matches!(
- element.local_name(),
- "div" | "section" | "article" | "main" | "header" | "footer" | "aside" | "nav"
- ) {
- buffers[target].push('\n');
- stack.push(MarkdownFrame::Append { target, text: "\n" });
- }
- push_child_markdown_frames(
- stack,
- dom,
- node_id,
- list_depth,
- next_tree_depth,
- target,
- );
- }
- },
- }
-}
-
-fn push_inline_children_markdown_frames(
- buffers: &mut Vec<String>,
- stack: &mut Vec<MarkdownFrame>,
- dom: &NativeDom,
- node_id: NativeNodeId,
- remaining_tree_depth: usize,
- target: usize,
- kind: MarkdownInlineKind,
-) {
- let source = buffers.len();
- buffers.push(String::new());
- stack.push(MarkdownFrame::FinishInline {
- source,
- target,
- kind,
- });
- push_child_markdown_frames(stack, dom, node_id, 0, remaining_tree_depth, source);
-}
-
-fn push_child_markdown_frames(
- stack: &mut Vec<MarkdownFrame>,
- dom: &NativeDom,
- node_id: NativeNodeId,
- list_depth: usize,
- remaining_tree_depth: usize,
- target: usize,
-) {
- let child_ids = dom.child_ids(node_id).collect::<Vec<_>>();
- for child_id in child_ids.into_iter().rev() {
- stack.push(MarkdownFrame::Enter {
- node_id: child_id,
- list_depth,
- remaining_tree_depth,
- target,
- });
- }
-}
-
-fn finish_inline_markdown(kind: MarkdownInlineKind, text: &str, out: &mut String) {
- // Spaces at inline element edges belong outside Markdown delimiters, but
- // still separate this element's text from its siblings. Block edges trim.
- let preserve_spacing = matches!(
- kind,
- MarkdownInlineKind::Anchor { .. }
- | MarkdownInlineKind::Strong
- | MarkdownInlineKind::Emphasis
- | MarkdownInlineKind::Code
- );
- let trailing_space = preserve_spacing && text.ends_with(char::is_whitespace);
- if preserve_spacing && text.starts_with(char::is_whitespace) {
- push_markdown_text(out, " ");
- }
- let text = collapse_whitespace(text);
- let text = text.as_str();
- match kind {
- MarkdownInlineKind::Paragraph => {
- if !text.is_empty() {
- out.push('\n');
- out.push_str(text);
- out.push_str("\n\n");
- }
- }
- MarkdownInlineKind::Anchor { href } => {
- if href.is_empty() {
- out.push_str(text);
- } else {
- let label = if text.is_empty() { href.as_str() } else { text };
- out.push('[');
- out.push_str(label);
- out.push_str("](");
- out.push_str(&href);
- out.push(')');
- }
- }
- MarkdownInlineKind::ListItem { list_depth } => {
- if !text.is_empty() {
- out.push_str(&" ".repeat(list_depth.saturating_sub(1)));
- out.push_str("- ");
- out.push_str(text);
- out.push('\n');
- }
- }
- MarkdownInlineKind::Strong => {
- if !text.is_empty() {
- out.push_str("**");
- out.push_str(text);
- out.push_str("**");
- }
- }
- MarkdownInlineKind::Emphasis => {
- if !text.is_empty() {
- out.push('*');
- out.push_str(text);
- out.push('*');
- }
- }
- MarkdownInlineKind::Code => {
- if !text.is_empty() {
- out.push('`');
- out.push_str(text);
- out.push('`');
- }
- }
- }
- if trailing_space {
- push_markdown_text(out, " ");
- }
-}
-
-fn push_markdown_text(out: &mut String, text: &str) {
- // Collapse whitespace across text nodes without discarding leading,
- // trailing, or whitespace-only separators in the source inline flow.
- for character in text.chars() {
- if character.is_whitespace() {
- if !out.ends_with(char::is_whitespace) {
- out.push(' ');
- }
- } else {
- out.push(character);
- }
- }
-}
-
-fn normalize_markdown(input: &str) -> String {
- let mut out = String::new();
- let mut previous_blank = false;
- for line in input.lines() {
- let trimmed = line.trim_end();
- if trimmed.is_empty() {
- if !previous_blank && !out.is_empty() {
- out.push('\n');
- }
- previous_blank = true;
- continue;
- }
- if !out.is_empty() {
- out.push('\n');
- }
- out.push_str(trimmed);
- previous_blank = false;
- }
- out.trim().to_owned()
-}
-
-fn collapse_whitespace(input: &str) -> String {
- input.split_whitespace().collect::<Vec<_>>().join(" ")
+ moli_html2md::Converter::new(moli_html2md::Options {
+ max_depth: MAX_DOM_OUTPUT_TREE_DEPTH,
+ default_code_language: Some("text".to_owned()),
+ ..Default::default()
+ })
+ .convert(dom, root)
}
#[cfg(test)]
@@ -653,7 +264,7 @@ mod tests {
"<p>a<span> </span>b<strong> </strong>c<em> </em>d<a> </a>e</p>",
"a b c d e",
),
- ("<p>one&nbsp;<span>two</span></p>", "one two"),
+ ("<p>one&nbsp;<span>two</span></p>", "one\u{00a0}two"),
(
"<p><span>one</span><!-- comment --> <span>two</span></p>",
"one two",
@@ -674,6 +285,10 @@ mod tests {
"<p>before<a href='/docs'> docs </a>after</p>",
"before [docs](/docs) after",
),
+ (
+ "<p>before<a href='/docs'><em> docs </em></a>after</p>",
+ "before [*docs*](/docs) after",
+ ),
(
"<p>before<strong> bold </strong>after</p>",
"before **bold** after",
@@ -716,12 +331,15 @@ mod tests {
for (html, expected) in [
(
"<span>Main menu</span><div>Main menu</div>",
- "Main menu\nMain menu",
+ "Main menu\n\nMain menu",
+ ),
+ (
+ "<div>outer<div>inner</div>tail</div>",
+ "outer\n\ninner\n\ntail",
),
- ("<div>outer<div>inner</div>tail</div>", "outer\ninner\ntail"),
(
"<span>by <small>Albert Einstein</small></span><div>Tags:\n <a>change</a>\n <a>deep-thoughts</a> <a>thinking</a> <a>world</a></div>",
- "by Albert Einstein\nTags: change deep-thoughts thinking world",
+ "by Albert Einstein\n\nTags: change deep-thoughts thinking world",
),
] {
assert_eq!(markdown_from_html(html), expected, "HTML: {html}");
@@ -755,19 +373,196 @@ mod tests {
}
#[test]
- fn markdown_renderer_truncates_deep_tree_with_heap_stack_walk() {
+ fn markdown_renderer_expands_table_cells() {
+ assert_eq!(
+ markdown_from_html(
+ "<table><tr><th>Name</th><th>Count</th></tr><tr><td>moli</td><td>2</td></tr></table>"
+ ),
+ "Name\n\nCount\n\nmoli\n\n2"
+ );
+ }
+
+ #[test]
+ fn markdown_renderer_flattens_hacker_news_layout_tables() {
+ let dom = HtmlParser::SCRIPTING_DISABLED.parse(
+ test_url(),
+ include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.html").to_owned(),
+ );
+ let before = dom.serialize_document();
+ assert_eq!(
+ render_markdown_document(&dom),
+ include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md").trim_end()
+ );
+ assert_eq!(dom.serialize_document(), before);
+ }
+
+ #[test]
+ fn markdown_renderer_preserves_ordered_and_nested_lists() {
+ assert_eq!(
+ markdown_from_html(
+ "<ol start='3'><li>three<ul><li>nested</li></ul></li><li>four</li></ol>"
+ ),
+ "3. three\n - nested\n4. four"
+ );
+ }
+
+ #[test]
+ fn markdown_renderer_preserves_blockquotes_and_hard_breaks() {
+ assert_eq!(
+ markdown_from_html("<blockquote><p>first<br>second</p></blockquote>"),
+ "> first \n> second"
+ );
+ }
+
+ #[test]
+ fn markdown_renderer_preserves_code_language_fences_and_blank_lines() {
+ assert_eq!(
+ markdown_from_html(
+ "<pre><code class='language-rust'>let s = \"```\";\n\n\nend\n</code></pre>"
+ ),
+ "````rust\nlet s = \"```\";\n\n\nend\n````"
+ );
+ assert_eq!(
+ markdown_from_html("<p>before<code> a`b </code>after</p>"),
+ "before ``a`b`` after"
+ );
+ }
+
+ #[test]
+ fn markdown_renderer_skips_non_content_tags() {
+ assert_eq!(
+ markdown_from_html(
+ "<head><title>title</title></head><body><script>script</script><style>style</style><noscript>noscript</noscript><p>Visible</p></body>"
+ ),
+ "Visible"
+ );
+ }
+
+ #[test]
+ fn markdown_renderer_keeps_boundaries_around_empty_blocks() {
+ for tag in ["p", "div", "h2", "blockquote", "ul", "ol", "pre", "table"] {
+ for content in ["", " \n\t "] {
+ let html = format!("before<{tag}>{content}</{tag}>after");
+ assert_eq!(markdown_from_html(&html), "before\n\nafter", "{html}");
+ }
+ }
+ }
+
+ #[test]
+ fn markdown_renderer_keeps_empty_links_and_links_around_blocks() {
+ for (html, expected) in [
+ ("<a href='/a'></a><a href='/b'></a>", "[](/a)[](/b)"),
+ ("<a href=''>label</a>", "label"),
+ ("<a id='target'>label</a>", "label"),
+ ("<a href='/a'><h2>heading</h2></a>", "## [heading](/a)"),
+ (
+ "<a href='/a'><blockquote>quote</blockquote></a>",
+ "> [quote](/a)",
+ ),
+ ("<a href='/a'><ul><li>item</li></ul></a>", "- [item](/a)"),
+ ("<a href='/a'><img src='/i'></a>", "[![](/i)](/a)"),
+ ] {
+ assert_eq!(markdown_from_html(html), expected, "{html}");
+ }
+ }
+
+ #[test]
+ fn markdown_renderer_keeps_paragraphs_in_list_items() {
+ for (html, expected) in [
+ ("<ul><li><p>one</p></li><li>two</li></ul>", "- one\n\n- two"),
+ ("<ul><li>one</li><li><p>two</p></li></ul>", "- one\n\n- two"),
+ ("<ol start='3'><li></li><li>four</li></ol>", "4. four"),
+ (
+ "<ul><li>parent<ul><li>child</li></ul>tail</li></ul>",
+ "- parent\n - child\n\n tail",
+ ),
+ ] {
+ assert_eq!(markdown_from_html(html), expected, "{html}");
+ }
+ }
+
+ #[test]
+ fn markdown_renderer_normalizes_image_and_link_attribute_newlines() {
+ for (html, expected) in [
+ (
+ "<a href='/a' title='one\n two'>link</a>",
+ "[link](/a \"one\ntwo\")",
+ ),
+ (
+ "<img src='/i' alt='one\n two' title='one\n two'>",
+ "![one\ntwo](/i \"one\ntwo\")",
+ ),
+ (
+ "<h2><a href='/a' title='one\n two'>link</a></h2>",
+ "## [link](/a \"one&#10;two\")",
+ ),
+ ("a<img alt='label'>b<img src='' alt='label'>c", "abc"),
+ ] {
+ assert_eq!(markdown_from_html(html), expected, "{html}");
+ }
+ }
+
+ #[test]
+ fn markdown_renderer_preserves_literal_entities_and_split_list_markers() {
+ assert_eq!(
+ markdown_from_html("<p>&amp;copy; &lt;b&gt;literal&lt;/b&gt;</p>"),
+ "&amp;copy; \\<b\\>literal\\</b\\>"
+ );
let mut dom = NativeDom::new_html(test_url());
let body = dom.create_element("body");
assert!(dom.append_child(dom.document_node_id(), body));
-
- let mut parent = body;
- for _ in 0..(MAX_DOM_OUTPUT_TREE_DEPTH + 32) {
- let child = dom.create_element("div");
- assert!(dom.append_child(parent, child));
- parent = child;
+ for chunk in ["12", ".", " ", "item"] {
+ append_text(&mut dom, body, chunk);
}
- append_text(&mut dom, parent, "too deep");
+ assert_eq!(render_markdown_document(&dom), "12\\. item");
+ }
+
+ #[test]
+ fn markdown_converter_supports_preformatted_inline_code_on_native_dom() {
+ let dom = HtmlParser::SCRIPTING_DISABLED.parse(
+ test_url(),
+ "<body>before<code> a b </code>after</body>".to_owned(),
+ );
+ let before = dom.serialize_document();
+ let root = dom.body_node_id().expect("body");
+ let converter = moli_html2md::Converter::new(moli_html2md::Options {
+ preformatted_code: true,
+ ..Default::default()
+ });
+ assert_eq!(converter.convert(&dom, root), "before` a b `after");
+ assert_eq!(render_markdown_document(&dom), "before `a b` after");
+ assert_eq!(dom.serialize_document(), before);
+ }
+
+ #[test]
+ fn markdown_renderer_reads_dom_without_mutating_nodes() {
+ let dom = HtmlParser::SCRIPTING_DISABLED.parse(
+ test_url(),
+ "<body><p><em>foo</em><i>bar</i></p><pre> a b\n</pre></body>".to_owned(),
+ );
+ let before = dom.serialize_document();
+ assert_eq!(
+ render_markdown_document(&dom),
+ "*foobar*\n\n```text\n a b\n```"
+ );
+ assert_eq!(dom.serialize_document(), before);
+ }
- assert_eq!(render_markdown_document(&dom), "");
+ #[test]
+ fn markdown_renderer_truncates_deep_tree() {
+ for tag in ["div", "span", "strong", "blockquote"] {
+ let mut dom = NativeDom::new_html(test_url());
+ let body = dom.create_element("body");
+ assert!(dom.append_child(dom.document_node_id(), body));
+ append_text(&mut dom, body, "Visible");
+ let mut parent = body;
+ for _ in 0..(MAX_DOM_OUTPUT_TREE_DEPTH + 32) {
+ let child = dom.create_element(tag);
+ assert!(dom.append_child(parent, child));
+ parent = child;
+ }
+ append_text(&mut dom, parent, "too deep");
+ assert_eq!(render_markdown_document(&dom), "Visible", "tag: {tag}");
+ }
}
}
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index 104ded56a..f35d96720 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -49,6 +49,8 @@ mod disable_css_boundaries;
mod disable_js;
#[path = "fetch_cli/eval.rs"]
mod eval;
+#[path = "fetch_cli/markdown.rs"]
+mod markdown;
#[path = "fetch_cli/readiness.rs"]
mod readiness;
#[path = "fetch_cli/redirect_chain.rs"]
diff --git a/moli/tests/fetch_cli/markdown.rs b/moli/tests/fetch_cli/markdown.rs
new file mode 100644
index 000000000..3eadba80a
--- /dev/null
+++ b/moli/tests/fetch_cli/markdown.rs
@@ -0,0 +1,40 @@
+use anyhow::Result;
+use base64::{Engine as _, engine::general_purpose::STANDARD};
+use std::process::Command;
+
+const HTML: &str = include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.html");
+const MARKDOWN: &str = include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md");
+
+fn assert_layout_table_dump(args: &[&str]) -> Result<()> {
+ let url = format!("data:text/html;base64,{}", STANDARD.encode(HTML));
+ let output = Command::new(env!("CARGO_BIN_EXE_moli"))
+ .args([
+ "fetch",
+ "--dump",
+ "markdown",
+ "--wait-until",
+ "load",
+ "--timeout",
+ "10000",
+ ])
+ .args(args)
+ .arg(url)
+ .output()?;
+ assert!(
+ output.status.success(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ assert_eq!(String::from_utf8(output.stdout)?, MARKDOWN.trim_end());
+ Ok(())
+}
+
+#[test]
+fn hacker_news_layout_tables_dump_markdown() -> Result<()> {
+ assert_layout_table_dump(&[])
+}
+
+#[test]
+fn hacker_news_layout_tables_dump_markdown_with_layout() -> Result<()> {
+ assert_layout_table_dump(&["--layout"])
+}