Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat(markdown): introduce moli-html2md
ldm061dbf7ea3e8d
diff --git a/Cargo.lock b/Cargo.lockindex 99de8b7d9..cdf3d51f0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2308,6 +2308,18 @@ dependencies = ["web_atoms",]+[[package]]+name = "markup5ever_rcdom"+version = "0.39.0+unofficial"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3ac010f19d6c4af81eeb4018a39d7a115de9d285af45c126a4ac02e6fc5716b7"+dependencies = [+ "html5ever",+ "markup5ever",+ "tendril",+ "xml5ever",+]+[[package]]name = "matchers"version = "0.2.0"@@ -2606,6 +2618,7 @@ dependencies = ["moli-css-parse","moli-html-input-temporal","moli-html-input-type",+ "moli-html2md","percent-encoding","selectors","serde_json",@@ -2701,6 +2714,16 @@ dependencies = ["strum",]+[[package]]+name = "moli-html2md"+version = "0.1.0"+dependencies = [+ "html5ever",+ "markup5ever_rcdom",+ "pulldown-cmark",+ "serde_json",+]+[[package]]name = "moli-http-cache"version = "0.1.0"@@ -3038,6 +3061,7 @@ dependencies = ["moli-fetch","moli-file-api","moli-geometry",+ "moli-html2md","moli-http-cache","moli-image","moli-import-map",@@ -4132,6 +4156,24 @@ version = "2.0.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"+[[package]]+name = "pulldown-cmark"+version = "0.13.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"+dependencies = [+ "bitflags",+ "memchr",+ "pulldown-cmark-escape",+ "unicase",+]++[[package]]+name = "pulldown-cmark-escape"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"+[[package]]name = "pxfm"version = "0.1.30"@@ -5607,6 +5649,12 @@ dependencies = ["arrayvec",]+[[package]]+name = "unicase"+version = "2.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"+[[package]]name = "unicode-bidi"version = "0.3.18"diff --git a/Cargo.toml b/Cargo.tomlindex 8934278c7..ec8aa593a 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -79,6 +79,7 @@ members = ["moli-geometry","moli-header-field","moli-html-input-type",+ "moli-html2md","moli-html-input-temporal","moli-window-features","moli-time",diff --git a/moli-dom/Cargo.toml b/moli-dom/Cargo.tomlindex 878b52ccd..5530db1f1 100644--- a/moli-dom/Cargo.toml+++ b/moli-dom/Cargo.toml@@ -9,6 +9,7 @@ html5ever = "0.39.0"indexmap = "2.6"moli-css-parse = { path = "../moli-css-parse" }moli-html-input-type = { path = "../moli-html-input-type" }+moli-html2md = { path = "../moli-html2md" }moli-html-input-temporal = { path = "../moli-html-input-temporal" }percent-encoding = "2.3"selectors = "0.40"diff --git a/moli-dom/src/native/markdown.rs b/moli-dom/src/native/markdown.rsnew file mode 100644index 000000000..ba6f395d4--- /dev/null+++ b/moli-dom/src/native/markdown.rs@@ -0,0 +1,29 @@+use super::{NativeDom, NativeNodeId, NodeData};++impl moli_html2md::Dom for NativeDom {+ type NodeId = NativeNodeId;++ fn node_kind(&self, node: NativeNodeId) -> moli_html2md::NodeKind<'_> {+ use moli_html2md::NodeKind;++ match self.node(node).map(|node| node.data()) {+ Some(NodeData::Document(_) | NodeData::DocumentFragment(_)) => NodeKind::Document,+ Some(NodeData::Element(element)) => NodeKind::Element(element.local_name()),+ Some(NodeData::Text(text)) => NodeKind::Text(text.data()),+ Some(NodeData::CDataSection(text)) => NodeKind::Text(text.data()),+ _ => NodeKind::Other,+ }+ }++ fn first_child(&self, node: NativeNodeId) -> Option<NativeNodeId> {+ NativeDom::first_child(self, node)+ }++ fn next_sibling(&self, node: NativeNodeId) -> Option<NativeNodeId> {+ NativeDom::next_sibling(self, node)+ }++ fn attribute(&self, node: NativeNodeId, name: &str) -> Option<&str> {+ self.node(node)?.as_element()?.attribute(name)+ }+}diff --git a/moli-dom/src/native/mod.rs b/moli-dom/src/native/mod.rsindex bd30e0fd9..d7322a3c3 100644--- a/moli-dom/src/native/mod.rs+++ b/moli-dom/src/native/mod.rs@@ -3,6 +3,7 @@ mod document;mod element;mod host;mod html_serialization;+mod markdown;mod node;mod queries;mod scripts;diff --git a/moli-html2md/Cargo.toml b/moli-html2md/Cargo.tomlnew file mode 100644index 000000000..1698234b1--- /dev/null+++ b/moli-html2md/Cargo.toml@@ -0,0 +1,15 @@+[package]+name = "moli-html2md"+version = "0.1.0"+edition = "2024"+license.workspace = true+description = "Read-only DOM to Markdown conversion with an iterative state machine"++[lints]+workspace = true++[dev-dependencies]+html5ever = "0.39"+markup5ever_rcdom = "0.39"+pulldown-cmark = { version = "0.13", default-features = false, features = ["html"] }+serde_json = "1.0.145"diff --git a/moli-html2md/README.md b/moli-html2md/README.mdnew file mode 100644index 000000000..9767df8d8--- /dev/null+++ b/moli-html2md/README.md@@ -0,0 +1,86 @@+# moli-html2md++Convert an existing DOM to Markdown through a small read-only interface. This+is an independent implementation with no runtime dependencies, HTML parser, or+intermediate DOM. `moli-dom::native::NativeDom` implements the interface directly.++```rust+let root = dom.body_node_id().unwrap_or(dom.document_node_id());+let markdown = moli_html2md::Converter::new(moli_html2md::Options {+ max_depth: 1000,+ default_code_language: Some("text".to_owned()),+ ..Default::default()+})+.convert(&dom, root);+```++## DOM interface++Implement `Dom` with a copyable node ID and four borrowed queries:++- `node_kind`: document/fragment, lowercase element tag, decoded text, or ignored node.+- `first_child` and `next_sibling`: ordered traversal of a stable, acyclic tree.+- `attribute`: decoded attribute value, borrowed from the DOM.++The converter does not need parent pointers, mutable access, serialization, or+an allocation for each node. The caller chooses the root; its siblings are not+visited. `Other` nodes and their subtrees are ignored. Unknown elements expose+their children as ordinary content. Comments and doctypes produce no output.++## State machine++- A heap task stack tracks node entry, sibling continuation, and element exit.+ Traversal stores one continuation per level instead of collecting all children.+- An inline writer tracks desired and emitted styles, pending whitespace, and+ block boundaries. It collapses HTML whitespace across node boundaries, keeps+ Unicode spaces, and coalesces adjacent identical emphasis without changing nodes.+- Lists, quotes, and headings have output buffers finalized by exit+ tasks. Raw code text uses the same iterative traversal with formatting disabled.+- List spacing follows block boundaries recorded during conversion. Nested+ lists keep their own spacing; list items need no preliminary DOM scan.+- Ordered lists use `start` followed by sequential numbers, like Turndown core.+ `reversed` and individual `li[value]` attributes are not represented.+- Code fences are chosen after collecting the code text. Language names retain+ punctuation and Unicode; a name containing backticks uses a tilde fence.+ Fallback language hints end at the first line ending.++Tables, sections, rows, and cells follow Turndown core's ordinary block behavior:+their content is emitted in DOM order, separated by blank lines. Nested tables+use the same rule. The converter does not classify layout tables or generate+GFM table syntax; headers, roles, borders, and spans do not select another path.++Supported output includes headings, paragraphs, emphasis, strikethrough, links,+images, lists, blockquotes, hard breaks, and fenced code. Inline HTML is used when+Markdown delimiters cannot express an emphasis boundary.+Script/style/head/noscript/template subtrees are omitted.++`max_depth` counts the supplied root as depth zero; nodes at or beyond the limit+are omitted. All traversal paths, including raw code, use this limit.++`preformatted_code` preserves whitespace inside inline code when enabled. Its+default is `false`, which collapses ordinary HTML whitespace and moves boundary+spaces outside code delimiters. Fenced blocks always preserve nonempty code's+spaces and blank lines. Empty blocks still separate surrounding text.++This is a structural content dump. It does not evaluate CSS, visibility, layout,+or JavaScript, and cannot preserve all HTML presentation (for example, arbitrary+ordered-list numbering or table spanning). DOM snapshots used by Moli's optional+strip/base/frame transformations remain the responsibility of the caller.++## Verification++`cargo test -p moli-html2md` covers an independent arena DOM, immutability,+whitespace, escaping, code, tables, depth limits, and 20,000 nested nodes on a+64 KiB thread stack. The [Turndown differential suite](tests/turndown/README.md)+pins 147 upstream fixtures and 50 additional cases. It compares rendered meaning,+checks DOM immutability, and explicitly asserts 12 documented differences that+preserve Moli's handling of literal text, code, emphasis, and list boundaries.+Targeted regressions also cover empty elements, links, attributes in headings and+tables, code whitespace and language names, list numbering, and loose lists.+Emphasis tests cover adjacent links that can use Markdown delimiters and+intraword link boundaries that still require inline HTML.++The HTML parser, reference JSON reader, and `pulldown-cmark` renderer are all+**test-only** dependencies. Tests use checked-in reference outputs and require+neither Node nor network access. NativeDom integration and real HTML fixtures+are tested in `moli-renderer-v8`.diff --git a/moli-html2md/src/converter.rs b/moli-html2md/src/converter.rsnew file mode 100644index 000000000..51e9bac86--- /dev/null+++ b/moli-html2md/src/converter.rs@@ -0,0 +1,29 @@+//! Convert an existing DOM to Markdown without parsing HTML or modifying nodes.+//!+//! Implement [`Dom`] for your tree, then pass a borrowed tree and a root node to+//! [`convert`]. Traversal and formatting use explicit stacks, including code,+//! lists, quotes, and tables; DOM depth never becomes Rust call-stack depth.++use crate::{Dom, Options, machine};++/// A reusable converter. All per-document state lives inside [`Self::convert`].+#[derive(Clone, Debug, Default)]+pub struct Converter {+ options: Options,+}++impl Converter {+ pub fn new(options: Options) -> Self {+ Self { options }+ }++ /// Convert a subtree, borrowing the DOM for this call only.+ pub fn convert<D: Dom + ?Sized>(&self, dom: &D, root: D::NodeId) -> String {+ machine::convert(dom, root, &self.options)+ }+}++/// Convert a subtree using the default options.+pub fn convert<D: Dom + ?Sized>(dom: &D, root: D::NodeId) -> String {+ Converter::default().convert(dom, root)+}diff --git a/moli-html2md/src/dom.rs b/moli-html2md/src/dom.rsnew file mode 100644index 000000000..583e68467--- /dev/null+++ b/moli-html2md/src/dom.rs@@ -0,0 +1,29 @@+/// The information needed to render a node. Text and tag names are borrowed.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum NodeKind<'a> {+ /// A document or fragment whose children should be visited.+ Document,+ /// An element's normalized, lowercase HTML local name.+ Element(&'a str),+ /// Already decoded text, including CDATA when supported by the DOM.+ Text(&'a str),+ /// Comments, doctypes, processing instructions, and invalid node IDs.+ Other,+}++/// A read-only view of an existing DOM.+///+/// IDs must describe a stable, acyclic tree for the duration of conversion.+/// Sibling chains must terminate. Attributes and text must already be decoded;+/// this crate neither parses HTML nor decodes entities a second time. No parent+/// lookup, allocation, interior mutation, or particular node representation is+/// required. Template contents and shadow roots are visited only if the adapter+/// exposes them as children.+pub trait Dom {+ type NodeId: Copy;++ fn node_kind(&self, node: Self::NodeId) -> NodeKind<'_>;+ fn first_child(&self, node: Self::NodeId) -> Option<Self::NodeId>;+ fn next_sibling(&self, node: Self::NodeId) -> Option<Self::NodeId>;+ fn attribute(&self, node: Self::NodeId, name: &str) -> Option<&str>;+}diff --git a/moli-html2md/src/lib.rs b/moli-html2md/src/lib.rsnew file mode 100644index 000000000..dadb292b1--- /dev/null+++ b/moli-html2md/src/lib.rs@@ -0,0 +1,9 @@+mod converter;+mod dom;+mod machine;+mod options;+mod writer;++pub use converter::{Converter, convert};+pub use dom::{Dom, NodeKind};+pub use options::Options;diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rsnew file mode 100644index 000000000..d6839c404--- /dev/null+++ b/moli-html2md/src/machine.rs@@ -0,0 +1,426 @@+use crate::writer::{Style, Writer, longest_run};+use crate::{Dom, NodeKind, Options};++enum Task<'a, Id> {+ Visit(Id, usize),+ Children(Option<Id>, usize),+ Boundary,+ PopStyle,+ EndLink(usize),+ EndQuote,+ EndHeading(usize),+ EndList(usize),+ EndItem(String),+ RawChildren(Option<Id>, usize),+ EndCode,+ EndPre(Option<&'a str>),+}++struct List {+ output_depth: usize,+ ordered: bool,+ next: i64,+ loose: bool,+}++struct Machine<'a, D: Dom + ?Sized> {+ dom: &'a D,+ options: &'a Options,+ tasks: Vec<Task<'a, D::NodeId>>,+ writers: Vec<Writer<'a>>,+ lists: Vec<List>,+ raw: String,+ serial: usize,+}++pub(crate) fn convert<D: Dom + ?Sized>(dom: &D, root: D::NodeId, options: &Options) -> String {+ let mut machine = Machine {+ dom,+ options,+ tasks: vec![Task::Visit(root, 0)],+ writers: vec![Writer::default()],+ lists: Vec::new(),+ raw: String::new(),+ serial: 0,+ };+ machine.run();+ machine.take_writer()+}++impl<'a, D: Dom + ?Sized> Machine<'a, D> {+ fn run(&mut self) {+ while let Some(task) = self.tasks.pop() {+ match task {+ Task::Visit(node, depth) => self.visit(node, depth),+ Task::Children(Some(node), depth) => {+ // Keep one sibling continuation, not a vector of every+ // child. Traversal storage is bounded by depth, not width.+ self.tasks+ .push(Task::Children(self.dom.next_sibling(node), depth));+ self.tasks.push(Task::Visit(node, depth));+ }+ Task::Children(None, _) | Task::RawChildren(None, _) => {}+ Task::Boundary => self.writer().boundary(2),+ Task::PopStyle => self.writer().pop_style(),+ Task::EndLink(serial) => self.writer().end_link(serial),+ Task::EndQuote => {+ let content = self.take_writer();+ let mut quote = String::new();+ for line in content.lines() {+ if !quote.is_empty() {+ quote.push('\n');+ }+ quote.push('>');+ if !line.is_empty() {+ quote.push(' ');+ quote.push_str(line);+ }+ }+ self.writer().block("e, 2, 2);+ }+ Task::EndHeading(level) => {+ let content = self.take_writer();+ if !content.is_empty() {+ let heading =+ format!("{} {}", "#".repeat(level), content.replace('\n', " "));+ self.writer().block(&heading, 2, 2);+ } else {+ self.writer().boundary(2);+ }+ }+ Task::EndList(before) => {+ self.lists.pop();+ let content = self.take_writer();+ let has_blocks = self.writer().has_blocks;+ self.writer().block(&content, before, 2);+ if before == 1 {+ // A nested list alone does not make its parent item loose.+ self.writer().has_blocks = has_blocks;+ }+ }+ Task::EndItem(marker) => {+ let loose = self.writer().has_blocks;+ let content = self.take_writer();+ if content.is_empty() {+ continue;+ }+ let mut item = String::new();+ for (index, line) in content.lines().enumerate() {+ if index == 0 {+ item.push_str(&marker);+ } else {+ item.push('\n');+ if !line.is_empty() {+ for _ in 0..marker.len() {+ item.push(' ');+ }+ }+ }+ item.push_str(line);+ }+ let list = self.lists.last_mut().expect("list item belongs to a list");+ let separation = if loose || list.loose { 2 } else { 1 };+ list.loose |= loose;+ self.writer().block(&item, separation, separation);+ }+ Task::RawChildren(Some(node), depth) => self.raw_node(node, depth),+ Task::EndCode => {+ let text = std::mem::take(&mut self.raw);+ let preformatted = self.options.preformatted_code;+ self.writer().code_with_edges(&text, preformatted);+ }+ Task::EndPre(language) => self.end_pre(language),+ }+ }+ }++ fn writer(&mut self) -> &mut Writer<'a> {+ self.writers.last_mut().expect("conversion has an output")+ }++ fn capture(&mut self) {+ let writer = self.writer().child();+ self.writers.push(writer);+ }++ fn take_writer(&mut self) -> String {+ let writer = self.writers.pop().expect("capture has an output");+ if let Some(parent) = self.writers.last_mut() {+ parent.last_link = parent.last_link.max(writer.last_link);+ }+ writer.finish()+ }++ fn children(&mut self, node: D::NodeId, depth: usize) {+ if depth < self.options.max_depth {+ self.tasks+ .push(Task::Children(self.dom.first_child(node), depth));+ }+ }++ fn style(&mut self, style: Style<'a>) {+ if self.writer().push_style(style) {+ self.tasks.push(Task::PopStyle);+ }+ }++ fn visit(&mut self, node: D::NodeId, depth: usize) {+ if depth >= self.options.max_depth {+ return;+ }+ let tag = match self.dom.node_kind(node) {+ NodeKind::Document => {+ self.children(node, depth + 1);+ return;+ }+ NodeKind::Text(text) => {+ self.writer().text(text);+ return;+ }+ NodeKind::Other => return,+ NodeKind::Element(tag) => tag,+ };+ match tag {+ "head" | "script" | "style" | "noscript" | "template" => return,+ "br" => {+ self.writer().hard_break();+ return;+ }+ "hr" => {+ self.writer().block("---", 2, 2);+ return;+ }+ "strong" | "b" => self.style(Style::Strong),+ "em" | "i" => self.style(Style::Emphasis),+ "del" | "s" | "strike" => self.style(Style::Strike),+ "a" => {+ if let Some(href) = self+ .dom+ .attribute(node, "href")+ .filter(|href| !href.is_empty())+ {+ self.serial += 1;+ let serial = self.serial;+ let style = Style::Link {+ serial,+ href,+ title: self.dom.attribute(node, "title"),+ };+ if self.writer().push_style(style) {+ self.tasks.push(Task::EndLink(serial));+ }+ }+ }+ "img" => {+ let alt = self.dom.attribute(node, "alt").unwrap_or_default();+ let title = self.dom.attribute(node, "title");+ if let Some(src) = self+ .dom+ .attribute(node, "src")+ .filter(|src| !src.is_empty())+ {+ self.writer().image(alt, src, title);+ }+ return;+ }+ "code" | "pre" => {+ self.raw.clear();+ let end = if tag == "pre" {+ Task::EndPre(self.language(node))+ } else {+ Task::EndCode+ };+ self.tasks.push(end);+ if depth + 1 < self.options.max_depth {+ self.tasks+ .push(Task::RawChildren(self.dom.first_child(node), depth + 1));+ }+ return;+ }+ "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {+ self.capture();+ self.writer().heading();+ self.tasks+ .push(Task::EndHeading((tag.as_bytes()[1] - b'0') as usize));+ }+ "blockquote" => {+ self.capture();+ self.tasks.push(Task::EndQuote);+ }+ "ul" | "ol" => self.start_list(node, tag == "ol"),+ "li" => self.start_item(),+ _ if is_block(tag) => {+ self.writer().boundary(2);+ self.tasks.push(Task::Boundary);+ }+ _ => {}+ }+ self.children(node, depth + 1);+ }++ fn start_list(&mut self, node: D::NodeId, ordered: bool) {+ let before = if self+ .lists+ .last()+ .is_some_and(|list| self.writers.len() == list.output_depth + 1)+ {+ 1+ } else {+ 2+ };+ let next = self+ .dom+ .attribute(node, "start")+ .and_then(|s| s.trim().parse().ok())+ .unwrap_or(1);+ self.capture();+ self.lists.push(List {+ output_depth: self.writers.len(),+ ordered,+ next,+ loose: false,+ });+ self.tasks.push(Task::EndList(before));+ }++ fn start_item(&mut self) {+ let Some(list) = self+ .lists+ .last_mut()+ .filter(|list| list.output_depth == self.writers.len())+ else {+ self.writer().boundary(2);+ self.tasks.push(Task::Boundary);+ return;+ };+ let marker = if list.ordered {+ let marker = format!("{}. ", list.next);+ list.next = list.next.saturating_add(1);+ marker+ } else {+ "- ".to_owned()+ };+ self.capture();+ self.tasks.push(Task::EndItem(marker));+ }++ fn language(&self, node: D::NodeId) -> Option<&'a str> {+ let mut child = self.dom.first_child(node);+ while let Some(id) = child {+ if self.dom.node_kind(id) == NodeKind::Element("code")+ && let Some(language) = class_language(self.dom.attribute(id, "class"))+ {+ return Some(language);+ }+ child = self.dom.next_sibling(id);+ }+ class_language(self.dom.attribute(node, "class"))+ .or(self.options.default_code_language.as_deref())+ }++ fn raw_node(&mut self, node: D::NodeId, depth: usize) {+ self.tasks+ .push(Task::RawChildren(self.dom.next_sibling(node), depth));+ if depth >= self.options.max_depth {+ return;+ }+ match self.dom.node_kind(node) {+ NodeKind::Text(text) => self.raw.push_str(text),+ NodeKind::Document | NodeKind::Element(_) if depth + 1 < self.options.max_depth => {+ self.tasks+ .push(Task::RawChildren(self.dom.first_child(node), depth + 1));+ }+ _ => {}+ }+ }++ fn end_pre(&mut self, language: Option<&str>) {+ let text = std::mem::take(&mut self.raw);+ if text.trim().is_empty() {+ self.writer().boundary(2);+ return;+ }+ // CommonMark code blocks normalize CRLF/CR and require a closing fence+ // on its own line. Preserve all other spaces and blank lines verbatim.+ let text = text.replace("\r\n", "\n").replace('\r', "\n");+ // The info string occupies one line. Tilde fences also allow language+ // names containing backticks without changing the name itself.+ let language = language+ .unwrap_or_default()+ .split(['\r', '\n'])+ .next()+ .unwrap();+ let marker = if language.contains('`') { '~' } else { '`' };+ let fence = marker+ .to_string()+ .repeat(3.max(longest_run(&text, marker) + 1));+ let mut block = fence.clone();+ for ch in language.chars() {+ match ch {+ '\\' => block.push_str("\\\\"),+ '&' => block.push_str("&"),+ _ => block.push(ch),+ }+ }+ block.push('\n');+ block.push_str(&text);+ if !text.is_empty() && !text.ends_with('\n') {+ block.push('\n');+ }+ block.push_str(&fence);+ self.writer().block(&block, 2, 2);+ }+}++fn class_language(class: Option<&str>) -> Option<&str> {+ class?+ .split_ascii_whitespace()+ .find_map(|part| part.strip_prefix("language-").filter(|s| !s.is_empty()))+}++fn is_block(tag: &str) -> bool {+ matches!(+ tag,+ "address"+ | "audio"+ | "canvas"+ | "frameset"+ | "isindex"+ | "noframes"+ | "output"+ | "article"+ | "aside"+ | "body"+ | "caption"+ | "center"+ | "dd"+ | "details"+ | "dialog"+ | "dir"+ | "div"+ | "dl"+ | "dt"+ | "fieldset"+ | "figcaption"+ | "figure"+ | "footer"+ | "form"+ | "header"+ | "hgroup"+ | "html"+ | "legend"+ | "main"+ | "menu"+ | "nav"+ | "p"+ | "section"+ | "summary"+ | "table"+ | "tbody"+ | "td"+ | "tfoot"+ | "th"+ | "thead"+ | "tr"+ )+}diff --git a/moli-html2md/src/options.rs b/moli-html2md/src/options.rsnew file mode 100644index 000000000..066249363--- /dev/null+++ b/moli-html2md/src/options.rs@@ -0,0 +1,23 @@+/// Conversion limits and the fallback language for fenced code blocks.+#[derive(Clone, Debug)]+pub struct Options {+ /// Visit nodes at depths `0..max_depth`, with the supplied root at depth 0.+ /// Deeper subtrees are omitted. Set to `usize::MAX` to disable this limit.+ pub max_depth: usize,+ /// Used when neither `<pre>` nor its `<code>` child declares a language.+ /// The hint ends at the first line ending, as Markdown info strings use one line.+ pub default_code_language: Option<String>,+ /// Preserve whitespace inside inline code, replacing line endings with spaces.+ /// When false, inline code follows ordinary HTML whitespace collapsing.+ pub preformatted_code: bool,+}++impl Default for Options {+ fn default() -> Self {+ Self {+ max_depth: 1000,+ default_code_language: None,+ preformatted_code: false,+ }+ }+}diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rsnew file mode 100644index 000000000..3ee7c0f92--- /dev/null+++ b/moli-html2md/src/writer.rs@@ -0,0 +1,464 @@+// Formatting is delayed until visible content arrives. This keeps whitespace+// outside newly opened/closed marks and coalesces adjacent identical styles.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(crate) enum Style<'a> {+ Strong,+ Emphasis,+ Strike,+ Link {+ // Distinguish adjacent links even when their destinations match.+ serial: usize,+ href: &'a str,+ title: Option<&'a str>,+ },+}++struct OpenStyle<'a> {+ style: Style<'a>,+ start: usize,+ html: bool,+}++#[derive(Default)]+pub(crate) struct Writer<'a> {+ output: String,+ desired: Vec<Style<'a>>,+ emitted: Vec<OpenStyle<'a>>,+ space: bool,+ preserved_spaces: String,+ breaks: usize,+ code: Option<String>,+ line_digits: Option<usize>,+ heading: bool,+ single_line_attributes: bool,+ pub(crate) last_link: usize,+ // Block events in this output buffer, used when finalizing a list item.+ pub(crate) has_blocks: bool,+}++pub(crate) fn is_space(ch: char) -> bool {+ matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{000c}')+}++impl<'a> Writer<'a> {+ pub(crate) fn child(&self) -> Self {+ Self {+ desired: self.desired.clone(),+ single_line_attributes: self.single_line_attributes,+ ..Self::default()+ }+ }++ pub(crate) fn heading(&mut self) {+ self.heading = true;+ self.single_line_attributes = true;+ }++ pub(crate) fn push_style(&mut self, style: Style<'a>) -> bool {+ // Repeated emphasis has the same visible meaning. Nested links cannot+ // be represented in Markdown, so retain the outer destination.+ if self.desired.contains(&style)+ || matches!(style, Style::Link { .. })+ && self.desired.iter().any(|s| matches!(s, Style::Link { .. }))+ {+ return false;+ }+ self.flush_code();+ self.desired.push(style);+ true+ }++ pub(crate) fn pop_style(&mut self) {+ self.flush_code();+ self.desired.pop();+ }++ pub(crate) fn end_link(&mut self, serial: usize) {+ self.flush_code();+ if self.last_link < serial {+ // Empty anchors still carry a destination. Materialize the link+ // after its leading whitespace, even without a text event.+ self.prepare_inline('[');+ }+ self.pop_style();+ }++ pub(crate) fn text(&mut self, text: &str) {+ self.flush_code();+ for ch in text.chars() {+ if is_space(ch) {+ self.space = true;+ continue;+ }+ if ch.is_whitespace() {+ // NBSP and other Unicode spaces remain visible characters,+ // but Markdown emphasis delimiters cannot touch whitespace.+ if self.space {+ self.preserved_spaces.push(' ');+ self.space = false;+ }+ self.preserved_spaces.push(ch);+ continue;+ }+ self.prepare_inline(ch);+ let line_start = self.output.is_empty() || self.output.ends_with('\n');+ if line_start {+ self.line_digits = Some(0);+ }+ match ch {+ '\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '~' => {+ self.output.push('\\');+ self.output.push(ch);+ }+ '#' if line_start || self.heading => {+ self.output.push('\\');+ self.output.push(ch);+ }+ '+' | '-' | '=' if line_start => {+ self.output.push('\\');+ self.output.push(ch);+ }+ '.' | ')' if self.line_digits.is_some_and(|digits| digits > 0) => {+ self.output.push('\\');+ self.output.push(ch);+ }+ '&' => self.output.push_str("&"),+ _ => self.output.push(ch),+ }+ self.line_digits = if ch.is_ascii_digit() {+ self.line_digits.map(|n| n + 1)+ } else {+ None+ };+ }+ }++ pub(crate) fn code(&mut self, text: &str) {+ if text.starts_with(is_space) {+ self.flush_code();+ self.space = true;+ }+ let text = text.trim_matches(is_space);+ if !text.is_empty() {+ if self.code.is_none() {+ self.prepare_inline('`');+ self.code = Some(String::new());+ }+ let code = self.code.as_mut().expect("code buffer was initialized");+ let mut space = false;+ for ch in text.chars() {+ if is_space(ch) {+ space = true;+ } else {+ if space {+ code.push(' ');+ space = false;+ }+ code.push(ch);+ }+ }+ }+ }++ pub(crate) fn code_with_edges(&mut self, text: &str, preformatted: bool) {+ if preformatted {+ if !text.is_empty() {+ if self.code.is_none() {+ self.prepare_inline('`');+ self.code = Some(String::new());+ }+ self.code+ .as_mut()+ .expect("initialized code buffer")+ .push_str(&text.replace("\r\n", " ").replace(['\n', '\r'], " "));+ }+ return;+ }+ let content = text.trim_matches(char::is_whitespace);+ if content.is_empty() {+ self.text(text);+ return;+ }+ let leading = text.len() - text.trim_start_matches(char::is_whitespace).len();+ let trailing = text.trim_end_matches(char::is_whitespace).len();+ if leading > 0 {+ self.text(&text[..leading]);+ }+ self.code(content);+ if trailing < text.len() {+ self.flush_code();+ self.text(&text[trailing..]);+ }+ }++ pub(crate) fn image(&mut self, alt: &str, src: &str, title: Option<&str>) {+ self.flush_code();+ self.prepare_inline('!');+ self.output.push_str("![");+ // Image labels are literal text, not a second Markdown input.+ for ch in clean_attribute(alt) {+ match ch {+ '\\' | '[' | ']' | '*' | '_' | '`' | '<' | '>' => {+ self.output.push('\\');+ self.output.push(ch);+ }+ '&' => self.output.push_str("&"),+ // Markdown image labels render soft breaks as spaces. Keep+ // that meaning when a heading flattens its content.+ '\n' if self.single_line_attributes => self.output.push(' '),+ _ => self.output.push(ch),+ }+ }+ self.output.push_str("](");+ destination(&mut self.output, src, title, self.single_line_attributes);+ self.output.push(')');+ self.line_digits = None;+ }++ pub(crate) fn boundary(&mut self, lines: usize) {+ self.has_blocks |= lines > 1;+ self.flush_code();+ self.close_to(0, None);+ self.flush_spaces(false);+ self.space = false;+ self.breaks = self.breaks.max(lines);+ }++ pub(crate) fn hard_break(&mut self) {+ self.flush_code();+ self.close_to(0, None);+ self.flush_spaces(false);+ self.flush_breaks();+ self.output.push_str(" \n");+ self.space = false;+ self.line_digits = Some(0);+ }++ pub(crate) fn block(&mut self, text: &str, before: usize, after: usize) {+ if text.is_empty() {+ self.boundary(before.max(after));+ return;+ }+ self.boundary(before);+ self.flush_breaks();+ self.output.push_str(text);+ self.line_digits = None;+ self.boundary(after);+ }++ pub(crate) fn finish(mut self) -> String {+ self.flush_code();+ self.close_to(0, None);+ self.flush_spaces(false);+ let end = self.output.trim_end_matches(is_space).len();+ self.output.truncate(end);+ self.output+ }++ fn prepare_inline(&mut self, next: char) {+ self.flush_breaks();+ let common = self+ .emitted+ .iter()+ .zip(&self.desired)+ .take_while(|(a, b)| a.style == **b)+ .count();+ // A new link emits '[' before its label. Include that character when+ // checking the boundaries of surrounding emphasis, even though style+ // emission is delayed until the first character of the label arrives.+ let link = self.desired[common..]+ .iter()+ .position(|style| matches!(style, Style::Link { .. }))+ .map(|index| common + index);+ let next_after_close = if self.space || !self.preserved_spaces.is_empty() {+ ' '+ } else if link.is_some() {+ '['+ } else {+ next+ };+ self.close_to(common, Some(next_after_close));+ self.flush_spaces(true);+ for (index, &style) in self.desired.iter().enumerate().skip(common) {+ let start = self.output.len();+ // An opening emphasis delimiter between a word and punctuation+ // cannot open a CommonMark span. Inline HTML preserves its meaning.+ let first = if link.is_some_and(|link| index < link) {+ '['+ } else {+ next+ };+ let html = is_punctuation(first)+ && self+ .output+ .chars()+ .next_back()+ .is_some_and(char::is_alphanumeric);+ match style {+ Style::Strong => self.output.push_str(if html { "<strong>" } else { "**" }),+ Style::Emphasis => self.output.push_str(if html { "<em>" } else { "*" }),+ Style::Strike => self.output.push_str(if html { "<del>" } else { "~~" }),+ Style::Link { serial, .. } => {+ self.last_link = self.last_link.max(serial);+ if self.output.ends_with('!') {+ self.output.insert(self.output.len() - 1, '\\');+ }+ self.output.push('[');+ }+ }+ self.emitted.push(OpenStyle { style, start, html });+ }+ }++ fn flush_spaces(&mut self, trailing_ascii: bool) {+ if !self.preserved_spaces.is_empty() {+ // Materialize any block boundary before preserving Unicode space.+ self.flush_breaks();+ let text = if self.output.is_empty() || self.output.ends_with('\n') {+ self.preserved_spaces.trim_start_matches(' ')+ } else {+ &self.preserved_spaces+ };+ self.output.push_str(text);+ self.preserved_spaces.clear();+ self.line_digits = None;+ }+ if trailing_ascii && self.space && !self.output.is_empty() && !self.output.ends_with('\n') {+ self.output.push(' ');+ self.line_digits = None;+ }+ self.space = false;+ }++ fn close_to(&mut self, count: usize, next: Option<char>) {+ while self.emitted.len() > count {+ let opened = self.emitted.pop().expect("nonempty style stack");+ let closing_needs_html = next.is_some_and(char::is_alphanumeric)+ && self.output.chars().next_back().is_some_and(is_punctuation);+ let delimiters = match opened.style {+ Style::Strong => Some(("**", "<strong>", "</strong>")),+ Style::Emphasis => Some(("*", "<em>", "</em>")),+ Style::Strike => Some(("~~", "<del>", "</del>")),+ Style::Link { href, title, .. } => {+ self.output.push_str("](");+ destination(&mut self.output, href, title, self.single_line_attributes);+ self.output.push(')');+ None+ }+ };+ if let Some((marker, open, close)) = delimiters {+ if opened.html {+ self.output.push_str(close);+ } else if closing_needs_html {+ // Only the converter's output changes. Remaining open+ // ancestors precede this offset, so their offsets stay valid.+ self.output+ .replace_range(opened.start..opened.start + marker.len(), open);+ self.output.push_str(close);+ } else {+ self.output.push_str(marker);+ }+ }+ }+ }++ fn flush_breaks(&mut self) {+ if self.breaks > 0 {+ if !self.output.is_empty() {+ let existing = self.output.len() - self.output.trim_end_matches('\n').len();+ for _ in existing..self.breaks {+ self.output.push('\n');+ }+ }+ self.line_digits = Some(0);+ self.breaks = 0;+ }+ }++ fn flush_code(&mut self) {+ if let Some(code) = self.code.take() {+ let fence = "`".repeat(longest_run(&code, '`') + 1);+ let padding = code.starts_with('`')+ || code.ends_with('`')+ || code.starts_with(' ') && code.ends_with(' ') && code.chars().any(|ch| ch != ' ');+ self.output.push_str(&fence);+ if padding {+ self.output.push(' ');+ }+ self.output.push_str(&code);+ if padding {+ self.output.push(' ');+ }+ self.output.push_str(&fence);+ self.line_digits = None;+ }+ }+}++fn is_punctuation(ch: char) -> bool {+ // Conservatively include Unicode symbols and combining marks; choosing+ // inline HTML for these edges is preferable to emitting literal delimiters.+ !ch.is_alphanumeric() && !ch.is_whitespace()+}++pub(crate) fn longest_run(text: &str, marker: char) -> usize {+ let mut longest = 0;+ let mut current = 0;+ for ch in text.chars() {+ if ch == marker {+ current += 1;+ longest = longest.max(current);+ } else {+ current = 0;+ }+ }+ longest+}++fn destination(output: &mut String, href: &str, title: Option<&str>, single_line: bool) {+ for ch in href.chars() {+ match ch {+ '\\' | '(' | ')' => {+ output.push('\\');+ output.push(ch);+ }+ ' ' => output.push_str("%20"),+ '\t' => output.push_str("%09"),+ '\r' => output.push_str("%0D"),+ '\n' => output.push_str("%0A"),+ '<' => output.push_str("%3C"),+ '>' => output.push_str("%3E"),+ '&' => output.push_str("&"),+ _ => output.push(ch),+ }+ }+ if let Some(title) = title.filter(|title| !title.is_empty()) {+ output.push_str(" \"");+ for ch in clean_attribute(title) {+ match ch {+ '\\' | '"' => {+ output.push('\\');+ output.push(ch);+ }+ '&' => output.push_str("&"),+ // Protect title newlines from heading line flattening.+ '\n' if single_line => output.push_str(" "),+ _ => output.push(ch),+ }+ }+ output.push('"');+ }+}++// Attribute newlines are significant in labels and titles; indentation after a+// newline is not. Collapse each newline/whitespace run without flattening it.+fn clean_attribute(value: &str) -> impl Iterator<Item = char> + '_ {+ let mut after_newline = false;+ value.chars().filter(move |&ch| {+ if after_newline && ch.is_whitespace() {+ return false;+ }+ after_newline = ch == '\n';+ true+ })+}diff --git a/moli-html2md/tests/conversion.rs b/moli-html2md/tests/conversion.rsnew file mode 100644index 000000000..ed76b79b0--- /dev/null+++ b/moli-html2md/tests/conversion.rs@@ -0,0 +1,424 @@+use moli_html2md::{Converter, Dom, NodeKind, Options, convert};++// A second DOM implementation, independent of NativeDom or an HTML parser.+// Its stable arena IDs and borrowed strings are enough for the converter.+#[derive(Clone, Debug, PartialEq, Eq)]+struct Node {+ kind: NodeKind<'static>,+ attrs: Vec<(&'static str, &'static str)>,+ first: Option<usize>,+ last: Option<usize>,+ next: Option<usize>,+}++#[derive(Clone, Debug, PartialEq, Eq)]+struct Tree(Vec<Node>);++impl Tree {+ fn new() -> Self {+ Self(vec![Node {+ kind: NodeKind::Document,+ attrs: Vec::new(),+ first: None,+ last: None,+ next: None,+ }])+ }++ fn add(&mut self, parent: usize, kind: NodeKind<'static>) -> usize {+ let id = self.0.len();+ self.0.push(Node {+ kind,+ attrs: Vec::new(),+ first: None,+ last: None,+ next: None,+ });+ if let Some(previous) = self.0[parent].last {+ self.0[previous].next = Some(id);+ } else {+ self.0[parent].first = Some(id);+ }+ self.0[parent].last = Some(id);+ id+ }++ fn element(&mut self, parent: usize, tag: &'static str) -> usize {+ self.add(parent, NodeKind::Element(tag))+ }++ fn text(&mut self, parent: usize, text: &'static str) {+ self.add(parent, NodeKind::Text(text));+ }++ fn attr(&mut self, node: usize, name: &'static str, value: &'static str) {+ self.0[node].attrs.push((name, value));+ }++ fn leaf(&mut self, parent: usize, tag: &'static str, text: &'static str) -> usize {+ let element = self.element(parent, tag);+ self.text(element, text);+ element+ }+}++impl Dom for Tree {+ type NodeId = usize;++ fn node_kind(&self, node: usize) -> NodeKind<'_> {+ self.0[node].kind+ }++ fn first_child(&self, node: usize) -> Option<usize> {+ self.0[node].first+ }++ fn next_sibling(&self, node: usize) -> Option<usize> {+ self.0[node].next+ }++ fn attribute(&self, node: usize, name: &str) -> Option<&str> {+ self.0[node]+ .attrs+ .iter()+ .find_map(|(key, value)| (*key == name).then_some(*value))+ }+}++#[test]+fn coalesces_styles_without_changing_the_tree() {+ let mut dom = Tree::new();+ dom.leaf(0, "em", "foo");+ dom.leaf(0, "i", "bar");+ dom.text(0, " ");+ let bold = dom.element(0, "strong");+ dom.leaf(bold, "b", "nested");+ dom.leaf(0, "strong", "!");+ let before = dom.clone();+ let converter = Converter::default();+ for _ in 0..2 {+ assert_eq!(converter.convert(&dom, 0), "*foobar* **nested!**");+ assert_eq!(dom, before);+ }+}++#[test]+fn moves_whitespace_outside_formatting_but_keeps_nonbreaking_spaces() {+ let mut dom = Tree::new();+ dom.text(0, "by");+ dom.leaf(0, "strong", " \t");+ dom.leaf(0, "small", "Albert Einstein");+ dom.text(0, "\u{a0}");+ let em = dom.element(0, "em");+ dom.leaf(em, "strong", " next ");+ dom.text(0, "word");+ assert_eq!(convert(&dom, 0), "by Albert Einstein\u{a0} ***next*** word");+}++#[test]+fn keeps_separate_links_and_escapes_destinations_and_titles() {+ let mut dom = Tree::new();+ for label in ["one", "two"] {+ let link = dom.leaf(0, "a", label);+ dom.attr(link, "href", "/a(b) c");+ dom.attr(link, "title", "a \"title\"");+ }+ assert_eq!(+ convert(&dom, 0),+ "[one](/a\\(b\\)%20c \"a \\\"title\\\"\")[two](/a\\(b\\)%20c \"a \\\"title\\\"\")"+ );+}++#[test]+fn escapes_literal_markdown_and_decoded_entities_across_text_nodes() {+ let mut dom = Tree::new();+ let p = dom.element(0, "p");+ dom.text(p, "12");+ dom.text(p, ". [x] *y* _z_ `a` <tag> ©");+ dom.leaf(0, "p", "# heading");+ dom.leaf(0, "p", "- item");+ dom.text(0, "!");+ let link = dom.leaf(0, "a", "link");+ dom.attr(link, "href", "/");+ assert_eq!(+ convert(&dom, 0),+ "12\\. \\[x\\] \\*y\\* \\_z\\_ \\`a\\` \\<tag\\> &copy;\n\n\\# heading\n\n\\- item\n\n\\"+ );+}++#[test]+fn combines_adjacent_code_and_chooses_safe_delimiters() {+ let mut dom = Tree::new();+ dom.leaf(0, "code", "a`");+ dom.leaf(0, "code", "`b");+ dom.text(0, " ");+ dom.leaf(0, "code", " ` ");+ dom.text(0, "after");+ assert_eq!(convert(&dom, 0), "```a``b``` `` ` `` after");+}++#[test]+fn preserves_preformatted_text_including_blank_lines_and_nested_elements() {+ let mut dom = Tree::new();+ let pre = dom.element(0, "pre");+ let code = dom.element(pre, "code");+ dom.attr(code, "class", "highlight language-rust");+ dom.text(code, " first\r\n\r\n");+ dom.leaf(code, "span", "```\n");+ dom.text(code, "\n");+ assert_eq!(convert(&dom, 0), "````rust\n first\n\n```\n\n````");+}++#[test]+fn prefixes_multiline_list_items_and_quotes() {+ let mut dom = Tree::new();+ let quote = dom.element(0, "blockquote");+ let list = dom.element(quote, "ol");+ dom.attr(list, "start", "9");+ let item = dom.element(list, "li");+ dom.leaf(item, "p", "first");+ dom.leaf(item, "p", "second");+ dom.leaf(list, "li", "last");+ assert_eq!(+ convert(&dom, 0),+ "> 9. first\n>\n> second\n>\n> 10. last"+ );+}++#[test]+fn supports_headings_images_and_non_content_elements() {+ let mut dom = Tree::new();+ dom.leaf(0, "h2", "Heading");+ let image = dom.element(0, "img");+ dom.attr(image, "src", "/image.png");+ dom.attr(image, "alt", "[image]");+ dom.element(0, "hr");+ for tag in ["head", "script", "style", "noscript", "template"] {+ dom.leaf(0, tag, "invisible");+ }+ dom.add(0, NodeKind::Other);+ dom.leaf(0, "p", "visible");+ assert_eq!(+ convert(&dom, 0),+ "## Heading\n\n![\\[image\\]](/image.png)\n\n---\n\nvisible"+ );+}++#[test]+fn expands_headerless_table_cells_as_blocks() {+ let mut dom = Tree::new();+ let table = dom.element(0, "table");+ let row = dom.element(table, "tr");+ dom.leaf(row, "td", "one");+ dom.leaf(row, "td", "two");+ assert_eq!(convert(&dom, 0), "one\n\ntwo");+}++#[test]+fn preserves_table_cell_boundaries_hard_breaks_and_literal_pipes() {+ let mut dom = Tree::new();+ let table = dom.element(0, "table");+ let header = dom.element(table, "tr");+ let cell = dom.leaf(header, "th", "Key");+ dom.attr(cell, "align", "right");+ let row = dom.element(table, "tr");+ let cell = dom.element(row, "td");+ dom.text(cell, "a|b");+ dom.element(cell, "br");+ dom.text(cell, "c");+ assert_eq!(convert(&dom, 0), "Key\n\na|b \nc");+}++#[test]+fn table_spans_do_not_duplicate_cell_text() {+ let mut dom = Tree::new();+ let table = dom.element(0, "table");+ let row = dom.element(table, "tr");+ let cell = dom.leaf(row, "td", "one");+ dom.attr(cell, "rowspan", "2");+ dom.leaf(row, "td", "two");+ let row = dom.element(table, "tr");+ let cell = dom.leaf(row, "td", "end");+ dom.attr(cell, "colspan", "999999999999");+ assert_eq!(convert(&dom, 0), "one\n\ntwo\n\nend");+}++#[test]+fn bounds_every_traversal_including_raw_code() {+ for tag in [+ "div",+ "span",+ "strong",+ "blockquote",+ "table",+ "pre",+ "code",+ ] {+ let mut dom = Tree::new();+ dom.text(0, "visible");+ let mut node = 0;+ for _ in 0..30 {+ node = dom.element(node, tag);+ }+ dom.text(node, "too deep");+ let converter = Converter::new(Options {+ max_depth: 10,+ ..Options::default()+ });+ let result = converter.convert(&dom, 0);+ assert!(!result.contains("too deep"), "{tag}: {result}");+ assert!(result.starts_with("visible"), "{tag}: {result}");+ }+}++#[test]+fn converts_deep_trees_on_a_small_thread_stack() {+ std::thread::Builder::new()+ .stack_size(64 * 1024)+ .spawn(|| {+ for tag in ["span", "strong", "pre"] {+ let mut dom = Tree::new();+ let mut node = 0;+ for _ in 0..20_000 {+ node = dom.element(node, tag);+ }+ dom.text(node, "deep");+ let converter = Converter::new(Options {+ max_depth: usize::MAX,+ ..Options::default()+ });+ let expected = match tag {+ "strong" => "**deep**",+ "pre" => "```\ndeep\n```",+ _ => "deep",+ };+ assert_eq!(converter.convert(&dom, 0), expected);+ }+ })+ .expect("spawn small-stack test")+ .join()+ .expect("iterative conversion should complete");+}++#[test]+fn converts_deep_tables_on_a_small_thread_stack() {+ std::thread::Builder::new()+ .stack_size(64 * 1024)+ .spawn(|| {+ let mut dom = Tree::new();+ let mut node = 0;+ for _ in 0..10_000 {+ node = dom.element(node, "table");+ node = dom.element(node, "tr");+ node = dom.element(node, "td");+ }+ dom.text(node, "deep");+ let converter = Converter::new(Options {+ max_depth: usize::MAX,+ ..Options::default()+ });+ assert_eq!(converter.convert(&dom, 0), "deep");+ })+ .expect("spawn small-stack test")+ .join()+ .expect("nested tables should not recurse");+}++#[test]+fn conversion_is_limited_to_the_supplied_subtree() {+ let mut dom = Tree::new();+ let first = dom.leaf(0, "p", "first");+ dom.leaf(0, "p", "second");+ assert_eq!(convert(&dom, first), "first");+ let converter = Converter::new(Options {+ max_depth: 0,+ ..Options::default()+ });+ assert_eq!(converter.convert(&dom, first), "");+}++fn rendered_html(markdown: &str) -> String {+ let parser = pulldown_cmark::Parser::new_ext(+ markdown,+ pulldown_cmark::Options::ENABLE_TABLES | pulldown_cmark::Options::ENABLE_STRIKETHROUGH,+ );+ let mut html = String::new();+ pulldown_cmark::html::push_html(&mut html, parser);+ html+}++#[test]+fn markdown_parser_recovers_adjacent_and_nested_formatting() {+ let mut dom = Tree::new();+ dom.leaf(0, "em", "foo");+ dom.leaf(0, "em", "bar");+ dom.text(0, " ");+ let strong = dom.element(0, "strong");+ dom.leaf(strong, "em", "both");+ dom.text(0, " ");+ dom.leaf(0, "del", "deleted");+ assert_eq!(+ rendered_html(&convert(&dom, 0)),+ "<p><em>foobar</em> <em><strong>both</strong></em> <del>deleted</del></p>\n"+ );+}++#[test]+fn markdown_parser_recovers_unicode_spaces_at_emphasis_boundaries() {+ let mut dom = Tree::new();+ dom.leaf(0, "em", "\u{a0} a \u{2003}");+ dom.text(0, "b");+ assert_eq!(+ rendered_html(&convert(&dom, 0)),+ "<p>\u{a0} <em>a</em> \u{2003}b</p>\n"+ );+ let mut dom = Tree::new();+ dom.leaf(0, "p", "\u{a0} a");+ assert_eq!(convert(&dom, 0), "\u{a0} a");+}++#[test]+fn markdown_parser_recovers_intraword_emphasis_next_to_punctuation() {+ for (tag, html_tag) in [("strong", "strong"), ("em", "em"), ("del", "del")] {+ for text in ["(word)", "word!", "!word", "日本語。"] {+ let mut dom = Tree::new();+ dom.text(0, "before");+ dom.leaf(0, tag, text);+ dom.text(0, "after");+ let markdown = convert(&dom, 0);+ assert_eq!(+ rendered_html(&markdown),+ format!("<p>before<{html_tag}>{text}</{html_tag}>after</p>\n"),+ "{markdown}"+ );+ }+ }+}++#[test]+fn markdown_parser_recovers_literal_hashes_at_the_end_of_headings() {+ let mut dom = Tree::new();+ dom.leaf(0, "h1", "title #");+ assert_eq!(rendered_html(&convert(&dom, 0)), "<h1>title #</h1>\n");+}++#[test]+fn markdown_parser_recovers_pipes_and_preformatted_code_inside_tables() {+ let mut dom = Tree::new();+ let table = dom.element(0, "table");+ let header = dom.element(table, "tr");+ dom.leaf(header, "th", "Code");+ let row = dom.element(table, "tr");+ let cell = dom.element(row, "td");+ dom.leaf(cell, "code", "a|b");+ let row = dom.element(table, "tr");+ let cell = dom.element(row, "td");+ dom.leaf(cell, "pre", " <a>|b\nnext\n");+ let markdown = convert(&dom, 0);+ let html = rendered_html(&markdown);+ assert_eq!(+ html,+ "<p>Code</p>\n<p><code>a|b</code></p>\n<pre><code> <a>|b\nnext\n</code></pre>\n",+ "{markdown}"+ );+}diff --git a/moli-html2md/tests/fixtures/hacker-news-layout.html b/moli-html2md/tests/fixtures/hacker-news-layout.htmlnew file mode 100644index 000000000..6c2c54efc--- /dev/null+++ b/moli-html2md/tests/fixtures/hacker-news-layout.html@@ -0,0 +1,22 @@+<!doctype html>+<!-- Reduced from https://news.ycombinator.com/news: layout tables, spacer+ rows, and colspan metadata rows. Story text and destinations are synthetic. -->+<html><body>+<table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">+ <tr><td><table border="0" width="100%"><tr>+ <td><b><a href="news">News</a></b></td>+ <td><a href="newest">new</a> | <a href="front">past</a></td>+ <td><a href="login?goto=news">login</a></td>+ </tr></table></td></tr>+ <tr style="height:10px"></tr>+ <tr><td><table border="0" cellpadding="0" cellspacing="0">+ <tr class="athing"><td align="right">1.</td><td><center><a href="vote?id=1&how=up"><div class="votearrow"></div></a></center></td><td><a href="https://example.test/first">First story</a> (<a href="from?site=example.test">example.test</a>)</td></tr>+ <tr><td colspan="2"></td><td>42 points by <a href="user?id=alice">alice</a> | <a href="item?id=1">8 comments</a></td></tr>+ <tr class="spacer" style="height:5px"></tr>+ <tr class="athing"><td align="right">2.</td><td><center><a href="vote?id=2&how=up"><div class="votearrow"></div></a></center></td><td><a href="https://example.test/second">Second story</a></td></tr>+ <tr><td colspan="2"></td><td>7 points by <a href="user?id=bob">bob</a> | <a href="item?id=2">discuss</a></td></tr>+ <tr class="spacer" style="height:5px"></tr>+ <tr><td colspan="2"></td><td><a href="news?p=2">More</a></td></tr>+ </table></td></tr>+</table>+</body></html>diff --git a/moli-html2md/tests/fixtures/hacker-news-layout.md b/moli-html2md/tests/fixtures/hacker-news-layout.mdnew file mode 100644index 000000000..7abae5463--- /dev/null+++ b/moli-html2md/tests/fixtures/hacker-news-layout.md@@ -0,0 +1,23 @@+**[News](news)**++[new](newest) | [past](front)++[login](login?goto=news)++1\.++[](vote?id=1&how=up)++[First story](https://example.test/first) ([example.test](from?site=example.test))++42 points by [alice](user?id=alice) | [8 comments](item?id=1)++2\.++[](vote?id=2&how=up)++[Second story](https://example.test/second)++7 points by [bob](user?id=bob) | [discuss](item?id=2)++[More](news?p=2)diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rsnew file mode 100644index 000000000..b36ea76f1--- /dev/null+++ b/moli-html2md/tests/regressions.rs@@ -0,0 +1,261 @@+mod support;++use moli_html2md::{Converter, Options};+use support::{Tree, rendered_html};++fn markdown(html: &str, preformatted_code: bool) -> String {+ let dom = Tree::parse(html);+ Converter::new(Options {+ preformatted_code,+ ..Options::default()+ })+ .convert(&dom, dom.root)+}++#[test]+fn empty_blocks_separate_text_even_when_they_contain_only_whitespace() {+ for tag in [+ "p",+ "div",+ "blockquote",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+ "ul",+ "ol",+ "pre",+ "table",+ ] {+ for contents in ["", " \n\t "] {+ let html = format!("before<{tag}>{contents}</{tag}>after");+ assert_eq!(markdown(&html, false), "before\n\nafter", "{html}");+ }+ }+}++#[test]+fn empty_links_keep_destinations_across_adjacent_and_nested_elements() {+ for (html, expected) in [+ ("<a href='/a'></a><a href='/b'></a>", "[](/a)[](/b)"),+ (+ "before<a href='/a'><em> </em></a>after",+ "before [](/a)after",+ ),+ ("<strong><a href='/a'></a></strong>", "**[](/a)**"),+ ("<a href='/a'><h2>heading</h2></a>", "## [heading](/a)"),+ (+ "<a href='/a'><blockquote>quote</blockquote></a>",+ "> [quote](/a)",+ ),+ ("<a href='/a'><ul><li>item</li></ul></a>", "- [item](/a)"),+ ("<a href='/a'><img src='/image'></a>", "[](/a)"),+ ] {+ assert_eq!(markdown(html, false), expected, "{html}");+ }+}++#[test]+fn href_without_a_destination_does_not_create_an_empty_markdown_link() {+ for html in [+ "<a>label</a>",+ "<a id='target'>label</a>",+ "<a href=''>label</a>",+ ] {+ assert_eq!(markdown(html, false), "label");+ }+}++#[test]+fn reusable_converter_does_not_leak_link_state_between_documents() {+ let converter = Converter::default();+ for _ in 0..3 {+ for (html, expected) in [+ ("<a href='/one'>one</a>", "[one](/one)"),+ ("<a href='/two'></a>", "[](/two)"),+ ] {+ let dom = Tree::parse(html);+ assert_eq!(converter.convert(&dom, dom.root), expected);+ }+ }+}++#[test]+fn preformatted_code_keeps_leading_trailing_and_internal_spaces() {+ for (html, expected, expected_html) in [+ (+ "<code> first</code>",+ "` first`",+ "<p><code> first</code></p>\n",+ ),+ (+ "<code>last </code>",+ "`last `",+ "<p><code>last </code></p>\n",+ ),+ (+ "<code> a b </code>",+ "` a b `",+ "<p><code> a b </code></p>\n",+ ),+ ("<code> </code>", "` `", "<p><code> </code></p>\n"),+ (+ "<code>\tword\t</code>",+ "`\tword\t`",+ "<p><code>\tword\t</code></p>\n",+ ),+ (+ "<code>` a `</code>",+ "`` ` a ` ``",+ "<p><code>` a `</code></p>\n",+ ),+ ] {+ let result = markdown(html, true);+ assert_eq!(result, expected, "{html}");+ assert_eq!(rendered_html(&result), expected_html, "{html}");+ }+}++#[test]+fn preformatted_code_normalizes_line_endings_without_collapsing_spaces() {+ let html = "<code> a\r\n b\r c\n d </code>";+ let actual = markdown(html, true);+ assert_eq!(+ rendered_html(&actual),+ "<p><code> a b c d </code></p>\n"+ );+ assert_eq!(markdown(html, false), "`a b c d`");+}++#[test]+fn normal_code_moves_unicode_boundary_spaces_outside_delimiters() {+ for space in ['\u{00a0}', '\u{2003}', '\u{2009}', '\u{202f}'] {+ let html = format!("before<code>{space}word{space}</code>after");+ assert_eq!(+ markdown(&html, false),+ format!("before{space}`word`{space}after")+ );+ assert_eq!(+ rendered_html(&markdown(&html, true)),+ format!("<p>before<code>{space}word{space}</code>after</p>\n")+ );+ }+}++#[test]+fn attribute_newlines_remove_indentation_without_joining_words() {+ for separator in ["\n ", "\r\n\t", "\n \n \t "] {+ let html = format!("<a href='/a' title='first{separator}second'>link</a>");+ assert_eq!(markdown(&html, false), "[link](/a \"first\nsecond\")");+ let html =+ format!("<img src='/i' alt='first{separator}second' title='first{separator}second'>");+ assert_eq!(+ markdown(&html, false),+ ""+ );+ }+}++#[test]+fn image_and_link_attributes_escape_markdown_and_preserve_literal_entities() {+ let html = "<a href='/a\\*?x=&copy;' title='a "quote" and \\ slash'>[label]</a>";+ let result = markdown(html, false);+ assert_eq!(+ rendered_html(&result),+ "<p><a href=\"/a%5C*?x=&copy;\" title=\"a "quote" and \\ slash\">[label]</a></p>\n"+ );+ let html = "<img src='/a(b)' alt='[a] *b* `c` &copy;'>";+ assert_eq!(+ rendered_html(&markdown(html, false)),+ "<p><img src=\"/a(b)\" alt=\"[a] *b* `c` &copy;\" /></p>\n"+ );+}++#[test]+fn table_attributes_keep_the_same_meaning_as_attributes_outside_tables() {+ for content in [+ "<a href='/a' title='first\n second'>link</a>",+ "<img src='/i' alt='first\n second' title='first\n second'>",+ ] {+ let inline = rendered_html(&markdown(content, false));+ let html = format!("<table><tr><th>{content}</th></tr></table>");+ assert_eq!(rendered_html(&markdown(&html, false)), inline, "{html}",);+ }+}++#[test]+fn heading_attributes_keep_the_same_meaning_as_attributes_outside_headings() {+ for content in [+ "<a href='/a' title='first\n second'>link</a>",+ "<img src='/i' alt='first\n second' title='first\n second'>",+ ] {+ let inline = rendered_html(&markdown(content, false));+ let inline = inline+ .strip_prefix("<p>")+ .unwrap()+ .strip_suffix("</p>\n")+ .unwrap();+ let html = format!("<h2>{content}</h2>");+ assert_eq!(+ rendered_html(&markdown(&html, false)),+ format!("<h2>{inline}</h2>\n")+ );+ }+}++#[test]+fn images_without_sources_do_not_emit_placeholder_markdown() {+ for html in [+ "a<img>b",+ "a<img alt='label'>b",+ "a<img src='' alt='label'>b",+ ] {+ assert_eq!(markdown(html, false), "ab");+ }+}++#[test]+fn list_paragraphs_and_code_blocks_require_blank_lines_between_items() {+ for (html, expected) in [+ ("<ul><li>one</li><li>two</li></ul>", "- one\n- two"),+ ("<ul><li><p>one</p></li><li>two</li></ul>", "- one\n\n- two"),+ ("<ul><li>one</li><li><p>two</p></li></ul>", "- one\n\n- two"),+ ("<ol><li></li><li>two</li></ol>", "2. two"),+ (+ "<ul><li><pre><code>x</code></pre></li><li>tail</li></ul>",+ "- ```\n x\n ```\n\n- tail",+ ),+ ] {+ assert_eq!(markdown(html, false), expected, "{html}");+ }+ assert_eq!(+ rendered_html(&markdown("<ul><li><p>one</p></li><li>two</li></ul>", false)),+ "<ul>\n<li>\n<p>one</p>\n</li>\n<li>\n<p>two</p>\n</li>\n</ul>\n"+ );+}++#[test]+fn empty_pre_blocks_disappear_and_nonempty_pre_keeps_blank_lines() {+ for contents in ["", " ", "\n", "\n \n"] {+ assert_eq!(+ markdown(&format!("<pre><code>{contents}</code></pre>"), false),+ ""+ );+ }+ assert_eq!(+ markdown("<pre><code>\nfirst\n\nlast\n\n</code></pre>", false),+ "```\n\nfirst\n\nlast\n\n```"+ );+}++#[test]+fn fallback_block_elements_separate_neighboring_words() {+ for tag in ["audio", "canvas", "output"] {+ assert_eq!(+ markdown(&format!("before<{tag}>content</{tag}>after"), false),+ "before\n\ncontent\n\nafter"+ );+ }+}diff --git a/moli-html2md/tests/rules.rs b/moli-html2md/tests/rules.rsnew file mode 100644index 000000000..5f3a1e543--- /dev/null+++ b/moli-html2md/tests/rules.rs@@ -0,0 +1,230 @@+mod support;++use moli_html2md::{Converter, Options};+use support::{Tree, rendered_html};++fn markdown_with(html: &str, options: Options) -> String {+ let dom = Tree::parse(html);+ let before = dom.clone();+ let result = Converter::new(options).convert(&dom, dom.root);+ assert_eq!(dom, before);+ result+}++fn markdown(html: &str) -> String {+ markdown_with(html, Options::default())+}++#[test]+fn ordered_lists_use_start_and_sequential_numbers() {+ for (attributes, start) in [+ ("", 1),+ ("reversed", 1),+ ("start='4'", 4),+ ("reversed start='4'", 4),+ ] {+ let html = format!(+ "<ol {attributes}><li value='20'>first</li><li value='-3'>second</li><li>third</li></ol>"+ );+ let result = markdown(&html);+ assert_eq!(+ result,+ format!(+ "{start}. first\n{}. second\n{}. third",+ start + 1,+ start + 2+ )+ );+ let start_attribute = if start == 1 {+ String::new()+ } else {+ format!(" start=\"{start}\"")+ };+ assert_eq!(+ rendered_html(&result),+ format!(+ "<ol{start_attribute}>\n<li>first</li>\n<li>second</li>\n<li>third</li>\n</ol>\n"+ )+ );+ }+}++#[test]+fn nested_lists_keep_independent_start_numbers() {+ assert_eq!(+ markdown(+ "<ol start='4'><li>outer<ol reversed start='9'><li value='100'>inner</li><li>next</li></ol></li><li value='70'>tail</li></ol>"+ ),+ "4. outer\n 9. inner\n 10. next\n5. tail"+ );+}++#[test]+fn code_languages_preserve_punctuation_and_unicode() {+ for language in ["text/x-rust", "中文", "C#", "objective-c++", "foo:bar"] {+ let html = format!("<pre><code class='language-{language}'>x</code></pre>");+ let result = markdown(&html);+ assert_eq!(result, format!("```{language}\nx\n```"));+ assert_eq!(+ rendered_html(&result),+ format!("<pre><code class=\"language-{language}\">x\n</code></pre>\n")+ );+ }+}++#[test]+fn code_languages_preserve_literal_backslashes_and_entities() {+ for (language, expected) in [+ ("f\\*oo", "f\\*oo"),+ ("literal&copy;", "literal&copy;"),+ ] {+ let result = markdown(&format!(+ "<pre><code class='language-{language}'>x</code></pre>"+ ));+ assert_eq!(+ rendered_html(&result),+ format!("<pre><code class=\"language-{expected}\">x\n</code></pre>\n")+ );+ }+}++#[test]+fn language_backticks_use_a_tilde_fence_that_contains_the_entire_code() {+ let result = markdown("<pre><code class='language-a`b'>first\n~~~\nlast\n</code></pre>");+ assert_eq!(result, "~~~~a`b\nfirst\n~~~\nlast\n~~~~");+ assert_eq!(+ rendered_html(&result),+ "<pre><code class=\"language-a`b\">first\n~~~\nlast\n</code></pre>\n"+ );+}++#[test]+fn fallback_language_stays_on_the_fence_line() {+ for ending in ["\n", "\r\n", "\r"] {+ let result = markdown_with(+ "<pre>x</pre>",+ Options {+ default_code_language: Some(format!("text/x-rust{ending}```{ending}outside")),+ ..Options::default()+ },+ );+ assert_eq!(result, "```text/x-rust\nx\n```");+ }+}++#[test]+fn list_spacing_uses_converted_blocks_including_wrapped_blocks() {+ for (body, expected) in [+ ("<h2>Heading</h2>", "- ## Heading\n\n- tail"),+ ("intro<hr>", "- intro\n\n ---\n\n- tail"),+ ("<section>one</section>", "- one\n\n- tail"),+ ("<span><p>one</p></span>", "- one\n\n- tail"),+ ] {+ let result = markdown(&format!("<ul><li>{body}</li><li>tail</li></ul>"));+ assert_eq!(result, expected, "{body}");+ assert!(rendered_html(&result).contains("<p>tail</p>"), "{body}");+ }+}++#[test]+fn nested_list_blocks_do_not_make_outer_items_loose() {+ for (inner, expected) in [+ (+ "<li>one</li><li>two</li>",+ "- outer\n - one\n - two\n- tail",+ ),+ (+ "<li><p>one</p></li><li><p>two</p></li>",+ "- outer\n - one\n\n - two\n- tail",+ ),+ ] {+ let result = markdown(&format!(+ "<ul><li>outer<ul>{inner}</ul></li><li>tail</li></ul>"+ ));+ assert_eq!(result, expected);+ assert!(rendered_html(&result).contains("<li>tail</li>"));+ }+ assert_eq!(+ markdown("<ul><li><p>outer</p><ul><li>inner</li></ul></li><li>tail</li></ul>"),+ "- outer\n\n - inner\n\n- tail"+ );+}++#[test]+fn omitted_subtrees_do_not_affect_list_spacing() {+ let html = "<ul><li>one<span><h2>too deep</h2></span></li><li>two</li></ul>";+ assert_eq!(+ markdown_with(+ html,+ Options {+ max_depth: 4,+ ..Options::default()+ }+ ),+ "- one\n- two"+ );+ assert_eq!(+ markdown("<ul><li>one<script>ignored</script></li><li>two</li></ul>"),+ "- one\n- two"+ );+}++#[test]+fn emphasis_uses_markdown_between_adjacent_links() {+ for (tag, marker, html_tag) in [+ ("b", "**", "strong"),+ ("em", "*", "em"),+ ("del", "~~", "del"),+ ] {+ for label in ["Hacker News", "News!"] {+ let result = markdown(&format!(+ "<{tag}><a href='news'>{label}</a></{tag}><a href='newest'>new</a>"+ ));+ assert_eq!(+ result,+ format!("{marker}[{label}](news){marker}[new](newest)")+ );+ assert_eq!(+ rendered_html(&result),+ format!(+ "<p><{html_tag}><a href=\"news\">{label}</a></{html_tag}><a href=\"newest\">new</a></p>\n"+ )+ );+ }+ }+ assert_eq!(+ markdown("<strong>News!</strong><a href='newest'>new</a>"),+ "**News!**[new](newest)"+ );+}++#[test]+fn emphasized_links_keep_html_at_intraword_boundaries() {+ for tag in ["strong", "em"] {+ for suffix in [" after", "after", "<a href='/next'>next</a>"] {+ let result = markdown(&format!(+ "before<{tag}><a href='/'>text</a></{tag}>{suffix}"+ ));+ let suffix_markdown = if suffix.starts_with('<') {+ "[next](/next)"+ } else {+ suffix+ };+ let suffix_html = if suffix.starts_with('<') {+ "<a href=\"/next\">next</a>"+ } else {+ suffix+ };+ assert_eq!(+ result,+ format!("before<{tag}>[text](/)</{tag}>{suffix_markdown}")+ );+ assert_eq!(+ rendered_html(&result),+ format!("<p>before<{tag}><a href=\"/\">text</a></{tag}>{suffix_html}</p>\n")+ );+ }+ let result = markdown(&format!("<{tag}><a href='/'>text</a></{tag}>after"));+ assert_eq!(result, format!("<{tag}>[text](/)</{tag}>after"));+ }+}diff --git a/moli-html2md/tests/support/mod.rs b/moli-html2md/tests/support/mod.rsnew file mode 100644index 000000000..379a15b50--- /dev/null+++ b/moli-html2md/tests/support/mod.rs@@ -0,0 +1,121 @@+use html5ever::tendril::TendrilSink;+use markup5ever_rcdom::{NodeData, RcDom};+use moli_html2md::{Dom, NodeKind};++/// An independent test adapter. Parsing and copying happen only in tests;+/// the production converter sees the same four borrowed queries as NativeDom.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct Tree {+ pub nodes: Vec<Node>,+ pub root: usize,+}++#[derive(Clone, Debug, PartialEq, Eq)]+pub struct Node {+ pub tag: Option<String>,+ pub text: Option<String>,+ pub attrs: Vec<(String, String)>,+ pub first: Option<usize>,+ pub next: Option<usize>,+}++impl Tree {+ pub fn parse(html: &str) -> Self {+ let parsed = html5ever::parse_document(RcDom::default(), Default::default()).one(format!(+ "<x-turndown id='turndown-root'>{html}</x-turndown>"+ ));+ let mut nodes: Vec<Node> = Vec::new();+ let mut root = 0;+ // RcDom's destructor clears descendants even when handles survive.+ // Keep the parsed document alive until the complete arena is copied.+ let mut stack = vec![(parsed.document.clone(), None::<usize>)];+ let mut last_children: Vec<Option<usize>> = Vec::new();+ while let Some((node, parent)) = stack.pop() {+ let id = nodes.len();+ let (tag, text, attrs) = match &node.data {+ NodeData::Element { name, attrs, .. } => (+ Some(name.local.to_string()),+ None,+ attrs+ .borrow()+ .iter()+ .map(|a| (a.name.local.to_string(), a.value.to_string()))+ .collect(),+ ),+ NodeData::Text { contents } => {+ (None, Some(contents.borrow().to_string()), Vec::new())+ }+ _ => (None, None, Vec::new()),+ };+ if attrs+ .iter()+ .any(|(key, value)| key == "id" && value == "turndown-root")+ {+ root = id;+ }+ nodes.push(Node {+ tag,+ text,+ attrs,+ first: None,+ next: None,+ });+ last_children.push(None);+ if let Some(parent) = parent {+ if let Some(previous) = last_children[parent] {+ nodes[previous].next = Some(id);+ } else {+ nodes[parent].first = Some(id);+ }+ last_children[parent] = Some(id);+ }+ stack.extend(+ node.children+ .borrow()+ .iter()+ .rev()+ .map(|child| (child.clone(), Some(id))),+ );+ }+ assert_ne!(root, 0, "fixture wrapper must be present");+ Self { nodes, root }+ }+}++impl Dom for Tree {+ type NodeId = usize;++ fn node_kind(&self, node: usize) -> NodeKind<'_> {+ let node = &self.nodes[node];+ if let Some(tag) = &node.tag {+ NodeKind::Element(tag)+ } else if let Some(text) = &node.text {+ NodeKind::Text(text)+ } else {+ NodeKind::Other+ }+ }++ fn first_child(&self, node: usize) -> Option<usize> {+ self.nodes[node].first+ }+ fn next_sibling(&self, node: usize) -> Option<usize> {+ self.nodes[node].next+ }+ fn attribute(&self, node: usize, name: &str) -> Option<&str> {+ self.nodes[node]+ .attrs+ .iter()+ .find_map(|(key, value)| (key == name).then_some(value.as_str()))+ }+}++pub fn rendered_html(markdown: &str) -> String {+ let parser = pulldown_cmark::Parser::new_ext(+ markdown,+ pulldown_cmark::Options::ENABLE_TABLES | pulldown_cmark::Options::ENABLE_STRIKETHROUGH,+ );+ let mut html = String::new();+ pulldown_cmark::html::push_html(&mut html, parser);+ html+}diff --git a/moli-html2md/tests/tables.rs b/moli-html2md/tests/tables.rsnew file mode 100644index 000000000..d32386370--- /dev/null+++ b/moli-html2md/tests/tables.rs@@ -0,0 +1,127 @@+mod support;++use moli_html2md::{Converter, Options, convert};+use support::{Tree, rendered_html};++fn markdown(html: &str) -> String {+ let dom = Tree::parse(html);+ let before = dom.clone();+ let result = convert(&dom, dom.root);+ assert_eq!(dom, before);+ result+}++#[test]+fn hacker_news_layout_tables_preserve_story_and_metadata_order() {+ let result = markdown(include_str!("fixtures/hacker-news-layout.html"));+ assert_eq!(+ result,+ include_str!("fixtures/hacker-news-layout.md").trim_end()+ );+ let html = rendered_html(&result);+ assert!(!html.contains("<table>"));+ assert!(!html.contains("<br>"));+ assert_eq!(html.matches("<a ").count(), 14);+}++#[test]+fn nested_tables_expand_cells_in_dom_order() {+ let html = "<table><tr><td>before</td></tr><tr><td><table><tr><th>Key</th><th>Value</th></tr><tr><td>a|b</td><td><code>c|d</code></td></tr></table></td></tr><tr><td>after</td></tr></table>";+ let actual = markdown(html);+ assert_eq!(actual, "before\n\nKey\n\nValue\n\na|b\n\n`c|d`\n\nafter");+ assert_eq!(+ rendered_html(&actual),+ "<p>before</p>\n<p>Key</p>\n<p>Value</p>\n<p>a|b</p>\n<p><code>c|d</code></p>\n<p>after</p>\n"+ );+}++#[test]+fn wrapping_a_table_preserves_its_content() {+ let data =+ "<table><tr><td>one</td><td>two</td></tr><tr><td>three</td><td>four</td></tr></table>";+ assert_eq!(+ markdown(&format!("<table><tr><td>{data}</td></tr></table>")),+ markdown(data)+ );+ assert_eq!(markdown(data), "one\n\ntwo\n\nthree\n\nfour");+}++#[test]+fn table_roles_do_not_change_conversion() {+ for role in [+ "",+ "presentation",+ "none",+ " PRESENTATION ",+ "table",+ "grid",+ ] {+ let html = format!(+ "<table role='{role}'><tr><th>Name</th><th>Value</th></tr><tr><td>one</td><td>two</td></tr></table>"+ );+ assert_eq!(markdown(&html), "Name\n\nValue\n\none\n\ntwo");+ }+}++#[test]+fn table_cells_preserve_block_syntax_and_code_whitespace() {+ let html = "<table><tr><td>intro</td><td><h2>Heading</h2><pre><code> a\n b\n</code></pre><blockquote>quote</blockquote><ul><li>item</li></ul></td><td>end</td></tr></table>";+ assert_eq!(+ markdown(html),+ "intro\n\n## Heading\n\n```\n a\n b\n```\n\n> quote\n\n- item\n\nend"+ );+}++#[test]+fn captions_sections_headers_and_spacer_rows_preserve_content_order() {+ for (html, expected) in [+ (+ "<table role='table'><tr><td>a</td><td>b</td></tr><tr></tr><tr><td>c</td><td>d</td></tr></table>",+ "a\n\nb\n\nc\n\nd",+ ),+ (+ "<table><caption>Data</caption><tr><td>a</td><td>b</td></tr><tr></tr><tr><td>c</td><td>d</td></tr></table>",+ "Data\n\na\n\nb\n\nc\n\nd",+ ),+ (+ "<table><thead><tr><td>a</td><td>b</td></tr></thead><tbody><tr></tr><tr><td>c</td><td>d</td></tr></tbody><tfoot><tr><td>end</td></tr></tfoot></table>",+ "a\n\nb\n\nc\n\nd\n\nend",+ ),+ (+ "<table><tr><th>a</th><th>b</th></tr><tr></tr><tr><td>c</td><td>d</td></tr></table>",+ "a\n\nb\n\nc\n\nd",+ ),+ ] {+ assert_eq!(markdown(html), expected, "{html}");+ }+}++#[test]+fn table_presentation_attributes_do_not_change_conversion() {+ for attributes in [+ "",+ "border='0' width='100%'",+ "border='1'",+ "style='border:0'",+ ] {+ let html = format!(+ "<table {attributes}><tr><td><a href='/'>Home</a></td><td><a href='/help'>Help</a></td></tr></table>"+ );+ assert_eq!(markdown(&html), "[Home](/)\n\n[Help](/help)");+ }+}++#[test]+fn nested_tables_obey_the_conversion_depth_limit() {+ let dom = Tree::parse(+ "<table><tr><td>visible<div><table><tr><td>too deep</td></tr></table></div></td></tr></table>",+ );+ for max_depth in [6, 7] {+ let converter = Converter::new(Options {+ max_depth,+ ..Options::default()+ });+ assert_eq!(converter.convert(&dom, dom.root), "visible");+ }+ assert_eq!(convert(&dom, dom.root), "visible\n\ntoo deep");+}diff --git a/moli-html2md/tests/turndown.rs b/moli-html2md/tests/turndown.rsnew file mode 100644index 000000000..e1d8fc14a--- /dev/null+++ b/moli-html2md/tests/turndown.rs@@ -0,0 +1,91 @@+mod support;++use std::sync::OnceLock;++use moli_html2md::{Converter, Options};+use serde_json::Value;+use support::{Tree, rendered_html};++fn corpus() -> &'static Value {+ static CORPUS: OnceLock<Value> = OnceLock::new();+ CORPUS.get_or_init(|| {+ serde_json::from_str(include_str!("turndown/cases.json")).expect("valid reference corpus")+ })+}++fn expectations() -> &'static Value {+ static EXPECTATIONS: OnceLock<Value> = OnceLock::new();+ EXPECTATIONS.get_or_init(|| {+ serde_json::from_str(include_str!("turndown/expectations.json"))+ .expect("valid intentional differences")+ })+}++fn check_case(index: usize) {+ let case = &corpus()["cases"][index];+ let input = case["html"].as_str().expect("HTML fixture");+ let expected = case["markdown"].as_str().expect("reference Markdown");+ let dom = Tree::parse(input);+ let before = dom.clone();+ let converter = Converter::new(Options {+ preformatted_code: case["preformatted_code"].as_bool().unwrap_or(false),+ ..Options::default()+ });+ let actual = converter.convert(&dom, dom.root);+ assert_eq!(dom, before, "conversion modified DOM");+ let difference = &expectations()[case["id"].as_str().expect("fixture ID")];+ if let Some(markdown) = difference["markdown"].as_str() {+ assert!(+ !difference["reason"]+ .as_str()+ .expect("documented difference")+ .is_empty()+ );+ assert_ne!(+ rendered_html(markdown),+ rendered_html(expected),+ "review an obsolete difference"+ );+ assert_eq!(actual, markdown, "{}: {}", case["id"], difference["reason"]);+ return;+ }+ let expected_html = rendered_html(expected);+ let actual_html = rendered_html(&actual);+ if actual_html != expected_html {+ println!(+ "DIFF {}",+ serde_json::json!({+ "id": case["id"], "name": case["name"], "input": input,+ "expected": expected, "actual": actual,+ "expected_html": expected_html, "actual_html": actual_html,+ })+ );+ }+ assert_eq!(+ actual_html, expected_html,+ "case {}: {}",+ case["id"], case["name"]+ );+}++macro_rules! case {+ ($name:ident, $index:expr) => {+ #[test]+ fn $name() {+ check_case($index);+ }+ };+}++include!("turndown/cases.rs");++#[test]+fn every_intentional_difference_names_a_unique_existing_case() {+ let mut ids = std::collections::HashSet::new();+ for case in corpus()["cases"].as_array().expect("cases") {+ assert!(ids.insert(case["id"].as_str().expect("fixture ID")));+ }+ for id in expectations().as_object().expect("difference map").keys() {+ assert!(ids.contains(id.as_str()), "unused difference: {id}");+ }+}diff --git a/moli-html2md/tests/turndown/LICENSE b/moli-html2md/tests/turndown/LICENSEnew file mode 100644index 000000000..02b58935f--- /dev/null+++ b/moli-html2md/tests/turndown/LICENSE@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2017 Dom Christie++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/moli-html2md/tests/turndown/README.md b/moli-html2md/tests/turndown/README.mdnew file mode 100644index 000000000..a9292b338--- /dev/null+++ b/moli-html2md/tests/turndown/README.md@@ -0,0 +1,51 @@+# Turndown differential tests++Reference: [mixmark-io/turndown](https://github.com/mixmark-io/turndown/tree/aa84dfa3e2361edea8c43acbfc2b7a9363494bfd),+revision `aa84dfa3e2361edea8c43acbfc2b7a9363494bfd`, package version 7.2.4,+using `@mixmark-io/domino` 2.2.0.++`cases.json` contains all 147 HTML fixtures from upstream `test/index.html`,+plus the 50 independently written cases in `additional.json`. The upstream+fixtures retain their MIT license in `LICENSE`. Production Rust code does not+include the upstream JavaScript implementation.++## What is compared++Turndown is configured to match Moli's ATX headings, dash bullets, star emphasis,+fenced code, inline links, and horizontal rules. Per-case style-only options+(such as referenced links or tilde fences) are normalized to this configuration.+The semantic `preformattedCode` option is preserved and tested against Moli's+`preformatted_code` option. This corpus tests conversion behavior, not Turndown's+JavaScript plugin API or every output-style option.++Each case parses the source HTML through an independent test DOM implementing+`Dom`, converts it, and checks that the DOM is unchanged. Both Markdown outputs+are rendered with `pulldown-cmark`, then their HTML is compared exactly. This+accepts equivalent delimiter choices and list indentation while detecting lost+text, incorrect links, changed paragraph/list structure, and visible delimiters.+There are also exact Markdown assertions in the original and regression suites.++`expectations.json` documents 12 deliberate differences with exact Moli outputs.+These preserve bare pre blocks and avoid upstream losses of literal characters,+emphasis, whitespace, and list boundaries. They remain active assertions; no+case is skipped. A reference change that makes a difference obsolete also fails+the test. Separate converter tests cover table expansion using Turndown core's+ordinary block behavior and strikethrough as a Moli extension.++## Regeneration++Normal Rust tests use the checked-in data and require neither Node nor network+access. To update the reference intentionally:++```sh+git clone https://github.com/mixmark-io/turndown.git /tmp/moli-turndown/upstream+git -C /tmp/moli-turndown/upstream checkout aa84dfa3e2361edea8c43acbfc2b7a9363494bfd+npm install --prefix /tmp/moli-turndown --ignore-scripts --no-audit --no-fund @mixmark-io/[email protected]+node moli-html2md/tests/turndown/generate.mjs /tmp/moli-turndown/upstream+cargo test -p moli-html2md+```++The generator executes the pinned upstream source. It adapts import extensions+for Node's ESM loader and provides CommonJS `require` for Domino, without changing+conversion rules. The corpus records both the upstream revision and normalized+options. `cases.rs` provides a separate named Rust test for every fixture.diff --git a/moli-html2md/tests/turndown/additional.json b/moli-html2md/tests/turndown/additional.jsonnew file mode 100644index 000000000..841c64ef2--- /dev/null+++ b/moli-html2md/tests/turndown/additional.json@@ -0,0 +1,52 @@+[+ {"id":"extra_empty_href","name":"empty href keeps link text","html":"before<a href=''>label</a>after"},+ {"id":"extra_empty_link","name":"empty link label","html":"before<a href='/target'></a>after"},+ {"id":"extra_blank_link","name":"whitespace-only link label","html":"before<a href='/target'> </a>after"},+ {"id":"extra_missing_image_src","name":"image alt without source","html":"before<img alt='missing'>after"},+ {"id":"extra_img_alt_whitespace","name":"image alt with multiple lines and indentation","html":"<img src='/x' alt='one\n \n two'>"},+ {"id":"extra_img_alt_code","name":"image label containing code delimiters","html":"<img src='/x' alt='a`b'>"},+ {"id":"extra_adjacent_lists","name":"adjacent unordered lists stay separate","html":"<ul><li>one</li></ul><ul><li>two</li></ul>"},+ {"id":"extra_adjacent_ordered_lists","name":"adjacent ordered lists stay separate","html":"<ol><li>one</li></ol><ol start='4'><li>four</li></ol>"},+ {"id":"extra_empty_list","name":"empty list","html":"before<ul></ul>after"},+ {"id":"extra_empty_list_item","name":"empty list item","html":"<ul><li></li><li>text</li></ul>"},+ {"id":"extra_loose_list","name":"paragraph list items stay loose","html":"<ul><li><p>one</p></li><li><p>two</p></li></ul>"},+ {"id":"extra_mixed_loose_list","name":"paragraph followed by a plain list item","html":"<ul><li><p>one</p></li><li>two</li></ul>"},+ {"id":"extra_list_nested_only","name":"list item containing only a nested list","html":"<ul><li><ul><li>inner</li></ul></li></ul>"},+ {"id":"extra_list_nested_tail","name":"text after a nested list","html":"<ul><li>start<ul><li>inner</li></ul>end</li></ul>"},+ {"id":"extra_list_nested_tail_p","name":"paragraph after a nested list","html":"<ul><li><p>start</p><ul><li>inner</li></ul><p>end</p></li></ul>"},+ {"id":"extra_list_code","name":"fenced code inside list item","html":"<ol start='9'><li><pre><code> first\nsecond\n</code></pre></li><li>end</li></ol>"},+ {"id":"extra_list_quote","name":"list item containing only a quote","html":"<ul><li><blockquote><p>quote</p></blockquote></li></ul>"},+ {"id":"extra_empty_quote","name":"empty quote still separates blocks","html":"before<blockquote></blockquote>after"},+ {"id":"extra_quote_blank_lines","name":"quote with paragraphs and hard breaks","html":"<blockquote><p>one<br>two</p><p>three</p></blockquote>"},+ {"id":"extra_empty_heading","name":"empty heading boundary","html":"before<h2></h2>after"},+ {"id":"extra_empty_pre","name":"empty pre","html":"before<pre></pre>after"},+ {"id":"extra_empty_pre_code","name":"empty pre code","html":"before<pre><code></code></pre>after"},+ {"id":"extra_blank_pre_code","name":"whitespace-only pre code","html":"<pre><code> \n</code></pre>"},+ {"id":"extra_adjacent_code","name":"adjacent code spans","html":"<code>a</code><code>b</code>"},+ {"id":"extra_code_gap","name":"separated code spans","html":"<code>a</code> <code>b</code>"},+ {"id":"extra_code_nested_format","name":"markup inside inline code","html":"<code><strong>a</strong><em>b</em></code>"},+ {"id":"extra_code_many_runs","name":"code span chooses an unused delimiter length","html":"<code>a```b`````c</code>"},+ {"id":"extra_code_only_ticks","name":"code containing only backticks","html":"<code>```</code>"},+ {"id":"extra_code_nbsp","name":"code with nonbreaking boundary spaces","html":"before<code> a </code>after"},+ {"id":"extra_preformatted_internal","name":"preformatted inline code preserves multiple spaces","html":"<code> a b </code>","original_options":{"preformattedCode":true}},+ {"id":"extra_preformatted_blank","name":"preformatted whitespace-only code","html":"before<code> </code>after","original_options":{"preformattedCode":true}},+ {"id":"extra_literal_entities","name":"decoded entity text remains literal","html":"<p>&copy; &lt; &#32;</p>"},+ {"id":"extra_literal_html","name":"literal HTML tags remain visible text","html":"<p><b>literal</b></p>"},+ {"id":"extra_split_list_marker","name":"ordered marker split across elements","html":"<p>12<span>.</span> text</p>"},+ {"id":"extra_split_emphasis_literal","name":"asterisks split across elements","html":"<p>*<span>word</span>*</p>"},+ {"id":"extra_link_backslash","name":"link destination with backslashes","html":"<a href='a\\b'>link</a>"},+ {"id":"extra_link_entities","name":"link destination with literal entity syntax","html":"<a href='/?q=&copy;'>link</a>"},+ {"id":"extra_link_leading_space","name":"leading whitespace inside link label","html":"before<a href='/'> label</a>after"},+ {"id":"extra_nested_link_emphasis","name":"emphasis nested in link","html":"<a href='/'><strong><em>text</em></strong></a>"},+ {"id":"extra_emphasis_nested_link","name":"link nested in emphasis","html":"<em><a href='/'>text</a></em>"},+ {"id":"extra_emphasis_intraword_link","name":"link in intraword emphasis","html":"before<em><a href='/'>text</a></em>after"},+ {"id":"extra_emphasis_intraword_punctuation","name":"intraword emphasis around punctuation","html":"before<em>(text)</em>after"},+ {"id":"extra_empty_emphasis","name":"empty formatting preserves surrounding spaces","html":"a<strong> </strong>b<em> </em>c"},+ {"id":"extra_nbsp_comment","name":"nonbreaking whitespace around ignored comment","html":"a <!--x--> b"},+ {"id":"extra_leading_br","name":"leading hard break","html":"<p><br>text</p>"},+ {"id":"extra_consecutive_br","name":"consecutive hard breaks","html":"<p>one<br><br>two</p>"},+ {"id":"extra_blank_br_container","name":"otherwise empty span containing hard break","html":"a<span><br></span>b"},+ {"id":"extra_empty_block_inside_inline","name":"empty block inside emphasis","html":"<strong>a<div></div>b</strong>"},+ {"id":"extra_audio_boundary","name":"audio fallback content is a block","html":"before<audio>fallback</audio>after"},+ {"id":"extra_output_boundary","name":"output element block boundary","html":"before<output>result</output>after"}+]diff --git a/moli-html2md/tests/turndown/cases.json b/moli-html2md/tests/turndown/cases.jsonnew file mode 100644index 000000000..3fa328753--- /dev/null+++ b/moli-html2md/tests/turndown/cases.json@@ -0,0 +1,1604 @@+{+ "upstream": {+ "repository": "https://github.com/mixmark-io/turndown",+ "revision": "aa84dfa3e2361edea8c43acbfc2b7a9363494bfd",+ "version": "7.2.4",+ "domino": "2.2.0"+ },+ "options": {+ "headingStyle": "atx",+ "hr": "---",+ "bulletListMarker": "-",+ "codeBlockStyle": "fenced",+ "fence": "```",+ "emDelimiter": "*",+ "strongDelimiter": "**",+ "linkStyle": "inlined",+ "br": " "+ },+ "cases": [+ {+ "id": "upstream_001",+ "name": "p",+ "html": "<p>Lorem ipsum</p>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Lorem ipsum"+ },+ {+ "id": "upstream_002",+ "name": "multiple ps",+ "html": "\n <p>Lorem</p>\n <p>ipsum</p>\n <p>sit</p>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Lorem\n\nipsum\n\nsit"+ },+ {+ "id": "upstream_003",+ "name": "em",+ "html": "<em>em element</em>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "*em element*"+ },+ {+ "id": "upstream_004",+ "name": "i",+ "html": "<i>i element</i>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "*i element*"+ },+ {+ "id": "upstream_005",+ "name": "strong",+ "html": "<strong>strong element</strong>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "**strong element**"+ },+ {+ "id": "upstream_006",+ "name": "b",+ "html": "<b>b element</b>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "**b element**"+ },+ {+ "id": "upstream_007",+ "name": "code",+ "html": "<code>code element</code>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "`code element`"+ },+ {+ "id": "upstream_008",+ "name": "code containing a backtick",+ "html": "<code>There is a literal backtick (`) here</code>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "``There is a literal backtick (`) here``"+ },+ {+ "id": "upstream_009",+ "name": "code containing three or more backticks",+ "html": "<code>here are three ``` here are four ```` that's it</code>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "`here are three ``` here are four ```` that's it`"+ },+ {+ "id": "upstream_010",+ "name": "code containing one or more backticks",+ "html": "<code>here are three ``` here are four ```` here is one ` that's it</code>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "``here are three ``` here are four ```` here is one ` that's it``"+ },+ {+ "id": "upstream_011",+ "name": "code starting with a backtick",+ "html": "<code>`starting with a backtick</code>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "`` `starting with a backtick ``"+ },+ {+ "id": "upstream_012",+ "name": "code containing markdown syntax",+ "html": "<code>_emphasis_</code>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "`_emphasis_`"+ },+ {+ "id": "upstream_013",+ "name": "code containing markdown syntax in a span",+ "html": "<code><span>_emphasis_</span></code>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "`_emphasis_`"+ },+ {+ "id": "upstream_014",+ "name": "h1",+ "html": "<h1>Level One Heading</h1>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "# Level One Heading"+ },+ {+ "id": "upstream_015",+ "name": "escape = when used as heading",+ "html": "===",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\==="+ },+ {+ "id": "upstream_016",+ "name": "not escaping = outside of a heading",+ "html": "A sentence containing =",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "A sentence containing ="+ },+ {+ "id": "upstream_017",+ "name": "h1 as atx",+ "html": "<h1>Level One Heading with ATX</h1>",+ "original_options": {+ "headingStyle": "atx"+ },+ "preformatted_code": false,+ "markdown": "# Level One Heading with ATX"+ },+ {+ "id": "upstream_018",+ "name": "h2",+ "html": "<h2>Level Two Heading</h2>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "## Level Two Heading"+ },+ {+ "id": "upstream_019",+ "name": "h2 as atx",+ "html": "<h2>Level Two Heading with ATX</h2>",+ "original_options": {+ "headingStyle": "atx"+ },+ "preformatted_code": false,+ "markdown": "## Level Two Heading with ATX"+ },+ {+ "id": "upstream_020",+ "name": "h3",+ "html": "<h3>Level Three Heading</h3>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "### Level Three Heading"+ },+ {+ "id": "upstream_021",+ "name": "heading with child",+ "html": "<h4>Level Four Heading with <code>child</code></h4>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "#### Level Four Heading with `child`"+ },+ {+ "id": "upstream_022",+ "name": "invalid heading",+ "html": "<h7>Level Seven Heading?</h7>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Level Seven Heading?"+ },+ {+ "id": "upstream_023",+ "name": "hr",+ "html": "<hr>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "---"+ },+ {+ "id": "upstream_024",+ "name": "hr with closing tag",+ "html": "<hr>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "---"+ },+ {+ "id": "upstream_025",+ "name": "hr with option",+ "html": "<hr>",+ "original_options": {+ "hr": "- - -"+ },+ "preformatted_code": false,+ "markdown": "---"+ },+ {+ "id": "upstream_026",+ "name": "br",+ "html": "More<br>after the break",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "More \nafter the break"+ },+ {+ "id": "upstream_027",+ "name": "br with visible line-ending",+ "html": "More<br>after the break",+ "original_options": {+ "br": "\\"+ },+ "preformatted_code": false,+ "markdown": "More \nafter the break"+ },+ {+ "id": "upstream_028",+ "name": "img with no alt",+ "html": "<img src=\"http://example.com/logo.png\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_029",+ "name": "img with relative src",+ "html": "<img src=\"logo.png\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_030",+ "name": "img with alt",+ "html": "<img src=\"logo.png\" alt=\"img with alt\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_031",+ "name": "img with escaped content in alt",+ "html": "<img src=\"logo.png\" alt=\"_img_ *with* [alt]\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "![\\_img\\_ \\*with\\* \\[alt\\]](logo.png)"+ },+ {+ "id": "upstream_032",+ "name": "img with no src",+ "html": "<img>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_033",+ "name": "img with parenthesis in src",+ "html": "<img src=\"logo.png?(query)\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ")"+ },+ {+ "id": "upstream_034",+ "name": "img with a new line in alt",+ "html": "<img src=\"logo.png\" alt=\"img with\n alt\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_035",+ "name": "img with more than one new line in alt",+ "html": "<img src=\"logo.png\" alt=\"img with\n \n alt\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_036",+ "name": "img with new lines in title",+ "html": "<img src=\"logo.png\" title=\"the\n \n title\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_037",+ "name": "img with quotes in title",+ "html": "<img src=\"logo.png\" title=\""hello"\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_038",+ "name": "img with space in URL",+ "html": "<img src=\"logo 1.png\" alt=\"An image\">",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_039",+ "name": "a",+ "html": "<a href=\"http://example.com\">An anchor</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[An anchor](http://example.com)"+ },+ {+ "id": "upstream_040",+ "name": "a with space in URL",+ "html": "<a href=\"http://example.com/foo bar\">An anchor</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[An anchor](<http://example.com/foo bar>)"+ },+ {+ "id": "upstream_041",+ "name": "a with title",+ "html": "<a href=\"http://example.com\" title=\"Title for link\">An anchor</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[An anchor](http://example.com \"Title for link\")"+ },+ {+ "id": "upstream_042",+ "name": "a with multiline title",+ "html": "<a href=\"http://example.com\" title=\"Title for\n \n link\">An anchor</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[An anchor](http://example.com \"Title for\nlink\")"+ },+ {+ "id": "upstream_043",+ "name": "a with quotes in title",+ "html": "<a href=\"http://example.com\" title=\""hello"\">An anchor</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[An anchor](http://example.com \"\\\"hello\\\"\")"+ },+ {+ "id": "upstream_044",+ "name": "a with parenthesis in query",+ "html": "<a href=\"http://example.com?(query)\">An anchor</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[An anchor](http://example.com?\\(query\\))"+ },+ {+ "id": "upstream_045",+ "name": "a without a src",+ "html": "<a id=\"about-anchor\">Anchor without a title</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Anchor without a title"+ },+ {+ "id": "upstream_046",+ "name": "a with a child",+ "html": "<a href=\"http://example.com/code\">Some <code>code</code></a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[Some `code`](http://example.com/code)"+ },+ {+ "id": "upstream_047",+ "name": "a with a brackets in text",+ "html": "<a href=\"http://example.com/code\">Some [text]</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[Some \\[text\\]](http://example.com/code)"+ },+ {+ "id": "upstream_048",+ "name": "a reference",+ "html": "<a href=\"http://example.com\">Reference link</a>",+ "original_options": {+ "linkStyle": "referenced"+ },+ "preformatted_code": false,+ "markdown": "[Reference link](http://example.com)"+ },+ {+ "id": "upstream_049",+ "name": "a reference with space in URL",+ "html": "<a href=\"http://example.com/foo bar\">Reference link</a>",+ "original_options": {+ "linkStyle": "referenced"+ },+ "preformatted_code": false,+ "markdown": "[Reference link](<http://example.com/foo bar>)"+ },+ {+ "id": "upstream_050",+ "name": "a reference with collapsed style",+ "html": "<a href=\"http://example.com\">Reference link with collapsed style</a>",+ "original_options": {+ "linkStyle": "referenced",+ "linkReferenceStyle": "collapsed"+ },+ "preformatted_code": false,+ "markdown": "[Reference link with collapsed style](http://example.com)"+ },+ {+ "id": "upstream_051",+ "name": "a reference with shortcut style",+ "html": "<a href=\"http://example.com\">Reference link with shortcut style</a>",+ "original_options": {+ "linkStyle": "referenced",+ "linkReferenceStyle": "shortcut"+ },+ "preformatted_code": false,+ "markdown": "[Reference link with shortcut style](http://example.com)"+ },+ {+ "id": "upstream_052",+ "name": "a reference with title",+ "html": "<a href=\"http://example.com\" title=\"Hello " World\">Reference link</a>",+ "original_options": {+ "linkStyle": "referenced"+ },+ "preformatted_code": false,+ "markdown": "[Reference link](http://example.com \"Hello \\\" World\")"+ },+ {+ "id": "upstream_053",+ "name": "pre/code block",+ "html": "<pre><code>def code_block\n # 42 < 9001\n \"Hello world!\"\nend</code></pre>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "```\ndef code_block\n # 42 < 9001\n \"Hello world!\"\nend\n```"+ },+ {+ "id": "upstream_054",+ "name": "multiple pre/code blocks",+ "html": "<pre><code>def first_code_block\n # 42 < 9001\n \"Hello world!\"\nend</code></pre>\n\n<p>next:</p>\n\n<pre><code>def second_code_block\n # 42 < 9001\n \"Hello world!\"\nend</code></pre>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "```\ndef first_code_block\n # 42 < 9001\n \"Hello world!\"\nend\n```\n\nnext:\n\n```\ndef second_code_block\n # 42 < 9001\n \"Hello world!\"\nend\n```"+ },+ {+ "id": "upstream_055",+ "name": "pre/code block with multiple new lines",+ "html": "<div><pre><code>Multiple new lines\n\n\nshould not be\n\n\nremoved</code></pre></div>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "```\nMultiple new lines\n\n\nshould not be\n\n\nremoved\n```"+ },+ {+ "id": "upstream_056",+ "name": "fenced pre/code block",+ "html": "\n <pre><code>def a_fenced_code block; end</code></pre>\n ",+ "original_options": {+ "codeBlockStyle": "fenced"+ },+ "preformatted_code": false,+ "markdown": "```\ndef a_fenced_code block; end\n```"+ },+ {+ "id": "upstream_057",+ "name": "pre/code block fenced with ~",+ "html": "\n <pre><code>def a_fenced_code block; end</code></pre>\n ",+ "original_options": {+ "codeBlockStyle": "fenced",+ "fence": "~~~"+ },+ "preformatted_code": false,+ "markdown": "```\ndef a_fenced_code block; end\n```"+ },+ {+ "id": "upstream_058",+ "name": "escaping ~~~",+ "html": "\n <pre>~~~ foo</pre>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\~~~ foo"+ },+ {+ "id": "upstream_059",+ "name": "not escaping ~~~",+ "html": "A sentence containing ~~~",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "A sentence containing ~~~"+ },+ {+ "id": "upstream_060",+ "name": "fenced pre/code block with language",+ "html": "\n <pre><code class=\"language-ruby\">def a_fenced_code block; end</code></pre>\n ",+ "original_options": {+ "codeBlockStyle": "fenced"+ },+ "preformatted_code": false,+ "markdown": "```ruby\ndef a_fenced_code block; end\n```"+ },+ {+ "id": "upstream_061",+ "name": "empty pre does not throw error",+ "html": "\n <pre></pre>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_062",+ "name": "ol",+ "html": "\n <ol>\n <li>Ordered list item 1</li>\n <li>Ordered list item 2</li>\n <li>Ordered list item 3</li>\n </ol>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "1. Ordered list item 1\n2. Ordered list item 2\n3. Ordered list item 3"+ },+ {+ "id": "upstream_063",+ "name": "ol with start",+ "html": "\n <ol start=\"42\">\n <li>Ordered list item 42</li>\n <li>Ordered list item 43</li>\n <li>Ordered list item 44</li>\n </ol>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "42. Ordered list item 42\n43. Ordered list item 43\n44. Ordered list item 44"+ },+ {+ "id": "upstream_064",+ "name": "ol with content",+ "html": "\n <ol start=\"42\">\n <li>\n <p>Ordered list item 42</p>\n <p>Ordered list's additional content</p>\n </li>\n </ol>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "42. Ordered list item 42\n \n Ordered list's additional content"+ },+ {+ "id": "upstream_065",+ "name": "list spacing",+ "html": "\n <p>A paragraph.</p>\n <ol>\n <li>Ordered list item 1</li>\n <li>Ordered list item 2</li>\n <li>Ordered list item 3</li>\n </ol>\n <p>Another paragraph.</p>\n <ul>\n <li>Unordered list item 1</li>\n <li>Unordered list item 2</li>\n <li>Unordered list item 3</li>\n </ul>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "A paragraph.\n\n1. Ordered list item 1\n2. Ordered list item 2\n3. Ordered list item 3\n\nAnother paragraph.\n\n- Unordered list item 1\n- Unordered list item 2\n- Unordered list item 3"+ },+ {+ "id": "upstream_066",+ "name": "ul",+ "html": "\n <ul>\n <li>Unordered list item 1</li>\n <li>Unordered list item 2</li>\n <li>Unordered list item 3</li>\n </ul>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "- Unordered list item 1\n- Unordered list item 2\n- Unordered list item 3"+ },+ {+ "id": "upstream_067",+ "name": "ul with custom bullet",+ "html": "\n <ul>\n <li>Unordered list item 1</li>\n <li>Unordered list item 2</li>\n <li>Unordered list item 3</li>\n </ul>\n ",+ "original_options": {+ "bulletListMarker": "-"+ },+ "preformatted_code": false,+ "markdown": "- Unordered list item 1\n- Unordered list item 2\n- Unordered list item 3"+ },+ {+ "id": "upstream_068",+ "name": "ul with paragraph",+ "html": "\n <ul>\n <li><p>List item with paragraph</p></li>\n <li>List item without paragraph</li>\n </ul>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "- List item with paragraph\n \n- List item without paragraph"+ },+ {+ "id": "upstream_069",+ "name": "ol with paragraphs",+ "html": "\n <ol>\n <li>\n <p>This is a paragraph in a list item.</p>\n <p>This is a paragraph in the same list item as above.</p>\n </li>\n <li>\n <p>A paragraph in a second list item.</p>\n </li>\n </ol>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "1. This is a paragraph in a list item.\n \n This is a paragraph in the same list item as above.\n \n2. A paragraph in a second list item."+ },+ {+ "id": "upstream_070",+ "name": "nested uls",+ "html": "\n <ul>\n <li>This is a list item at root level</li>\n <li>This is another item at root level</li>\n <li>\n <ul>\n <li>This is a nested list item</li>\n <li>This is another nested list item</li>\n <li>\n <ul>\n <li>This is a deeply nested list item</li>\n <li>This is another deeply nested list item</li>\n <li>This is a third deeply nested list item</li>\n </ul>\n </li>\n </ul>\n </li>\n <li>This is a third item at root level</li>\n </ul>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "- This is a list item at root level\n- This is another item at root level\n- - This is a nested list item\n - This is another nested list item\n - - This is a deeply nested list item\n - This is another deeply nested list item\n - This is a third deeply nested list item\n- This is a third item at root level"+ },+ {+ "id": "upstream_071",+ "name": "nested ols and uls",+ "html": "\n <ul>\n <li>This is a list item at root level</li>\n <li>This is another item at root level</li>\n <li>\n <ol>\n <li>This is a nested list item</li>\n <li>This is another nested list item</li>\n <li>\n <ul>\n <li>This is a deeply nested list item</li>\n <li>This is another deeply nested list item</li>\n <li>This is a third deeply nested list item</li>\n </ul>\n </li>\n </ol>\n </li>\n <li>This is a third item at root level</li>\n </ul>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "- This is a list item at root level\n- This is another item at root level\n- 1. This is a nested list item\n 2. This is another nested list item\n 3. - This is a deeply nested list item\n - This is another deeply nested list item\n - This is a third deeply nested list item\n- This is a third item at root level"+ },+ {+ "id": "upstream_072",+ "name": "ul with blockquote",+ "html": "\n <ul>\n <li>\n <p>A list item with a blockquote:</p>\n <blockquote>\n <p>This is a blockquote inside a list item.</p>\n </blockquote>\n </li>\n </ul>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "- A list item with a blockquote:\n \n > This is a blockquote inside a list item."+ },+ {+ "id": "upstream_073",+ "name": "blockquote",+ "html": "\n <blockquote>\n <p>This is a paragraph within a blockquote.</p>\n <p>This is another paragraph within a blockquote.</p>\n </blockquote>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "> This is a paragraph within a blockquote.\n> \n> This is another paragraph within a blockquote."+ },+ {+ "id": "upstream_074",+ "name": "nested blockquotes",+ "html": "\n <blockquote>\n <p>This is the first level of quoting.</p>\n <blockquote>\n <p>This is a paragraph in a nested blockquote.</p>\n </blockquote>\n <p>Back to the first level.</p>\n </blockquote>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "> This is the first level of quoting.\n> \n> > This is a paragraph in a nested blockquote.\n> \n> Back to the first level."+ },+ {+ "id": "upstream_075",+ "name": "html in blockquote",+ "html": "\n <blockquote>\n <h2>This is a header.</h2>\n <ol>\n <li>This is the first list item.</li>\n <li>This is the second list item.</li>\n </ol>\n <p>A code block:</p>\n <pre><code>return 1 < 2 ? shell_exec('echo $input | $markdown_script') : 0;</code></pre>\n </blockquote>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "> ## This is a header.\n> \n> 1. This is the first list item.\n> 2. This is the second list item.\n> \n> A code block:\n> \n> ```\n> return 1 < 2 ? shell_exec('echo $input | $markdown_script') : 0;\n> ```"+ },+ {+ "id": "upstream_076",+ "name": "multiple divs",+ "html": "\n <div>A div</div>\n <div>Another div</div>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "A div\n\nAnother div"+ },+ {+ "id": "upstream_077",+ "name": "multiple divs",+ "html": "\n <div>A div</div>\n <div>Another div</div>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "A div\n\nAnother div"+ },+ {+ "id": "upstream_078",+ "name": "comment",+ "html": "<!-- comment -->",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_079",+ "name": "pre/code with comment",+ "html": "\n <pre><code>Hello<!-- comment --> world</code></pre>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "```\nHello world\n```"+ },+ {+ "id": "upstream_080",+ "name": "leading whitespace in heading",+ "html": "<h3>\n h3 with leading whitespace</h3>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "### h3 with leading whitespace"+ },+ {+ "id": "upstream_081",+ "name": "trailing whitespace in li",+ "html": "\n <ol>\n <li>Chapter One\n <ol>\n <li>Section One</li>\n <li>Section Two with trailing whitespace </li>\n <li>Section Three with trailing whitespace </li>\n </ol>\n </li>\n <li>Chapter Two</li>\n <li>Chapter Three with trailing whitespace </li>\n </ol>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "1. Chapter One\n 1. Section One\n 2. Section Two with trailing whitespace\n 3. Section Three with trailing whitespace\n2. Chapter Two\n3. Chapter Three with trailing whitespace"+ },+ {+ "id": "upstream_082",+ "name": "multilined and bizarre formatting",+ "html": "\n <ul>\n <li>\n Indented li with leading/trailing newlines\n </li>\n <li>\n <strong>Strong with trailing space inside li with leading/trailing whitespace </strong> </li>\n <li>li without whitespace</li>\n <li> Leading space, text, lots of whitespace …\n text\n </li>\n \n </ul>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "- Indented li with leading/trailing newlines\n- **Strong with trailing space inside li with leading/trailing whitespace**\n- li without whitespace\n- Leading space, text, lots of whitespace … text"+ },+ {+ "id": "upstream_083",+ "name": "whitespace between inline elements",+ "html": "\n <p>I <a href=\"http://example.com/need\">need</a> <a href=\"http://www.example.com/more\">more</a> spaces!</p>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "I [need](http://example.com/need) [more](http://www.example.com/more) spaces!"+ },+ {+ "id": "upstream_084",+ "name": "whitespace in inline elements",+ "html": "Text with no space after the period.<em> Text in em with leading/trailing spaces </em><strong>text in strong with trailing space </strong>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Text with no space after the period. *Text in em with leading/trailing spaces* **text in strong with trailing space**"+ },+ {+ "id": "upstream_085",+ "name": "whitespace in nested inline elements",+ "html": "Text at root <strong><a href=\"http://www.example.com\">link text with trailing space in strong </a></strong>more text at root",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Text at root **[link text with trailing space in strong](http://www.example.com)** more text at root"+ },+ {+ "id": "upstream_086",+ "name": "blank inline elements",+ "html": "\n Text before blank em … <em></em> text after blank em\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Text before blank em … text after blank em"+ },+ {+ "id": "upstream_087",+ "name": "blank block elements",+ "html": "\n Text before blank div … <div></div> text after blank div\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Text before blank div …\n\ntext after blank div"+ },+ {+ "id": "upstream_088",+ "name": "blank inline element with br",+ "html": "<strong><br></strong>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_089",+ "name": "whitespace between blocks",+ "html": "<div><div>Content in a nested div</div></div>\n<div>Content in another div</div>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Content in a nested div\n\nContent in another div"+ },+ {+ "id": "upstream_090",+ "name": "escaping backslashes",+ "html": "backslash \\",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "backslash \\\\"+ },+ {+ "id": "upstream_091",+ "name": "escaping headings with #",+ "html": "### This is not a heading",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\### This is not a heading"+ },+ {+ "id": "upstream_092",+ "name": "not escaping # outside of a heading",+ "html": "#This is not # a heading",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "#This is not # a heading"+ },+ {+ "id": "upstream_093",+ "name": "escaping em markdown with *",+ "html": "To add emphasis, surround text with *. For example: *this is emphasis*",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "To add emphasis, surround text with \\*. For example: \\*this is emphasis\\*"+ },+ {+ "id": "upstream_094",+ "name": "escaping em markdown with _",+ "html": "To add emphasis, surround text with _. For example: _this is emphasis_",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "To add emphasis, surround text with \\_. For example: \\_this is emphasis\\_"+ },+ {+ "id": "upstream_095",+ "name": "not escaping within code",+ "html": "<pre><code>def this_is_a_method; end;</code></pre>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "```\ndef this_is_a_method; end;\n```"+ },+ {+ "id": "upstream_096",+ "name": "escaping strong markdown with *",+ "html": "To add strong emphasis, surround text with **. For example: **this is strong**",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "To add strong emphasis, surround text with \\*\\*. For example: \\*\\*this is strong\\*\\*"+ },+ {+ "id": "upstream_097",+ "name": "escaping strong markdown with _",+ "html": "To add strong emphasis, surround text with __. For example: __this is strong__",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "To add strong emphasis, surround text with \\_\\_. For example: \\_\\_this is strong\\_\\_"+ },+ {+ "id": "upstream_098",+ "name": "escaping hr markdown with *",+ "html": "* * *",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\* \\* \\*"+ },+ {+ "id": "upstream_099",+ "name": "escaping hr markdown with -",+ "html": "- - -",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\- - -"+ },+ {+ "id": "upstream_100",+ "name": "escaping hr markdown with _",+ "html": "_ _ _",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\_ \\_ \\_"+ },+ {+ "id": "upstream_101",+ "name": "escaping hr markdown without spaces",+ "html": "***",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\*\\*\\*"+ },+ {+ "id": "upstream_102",+ "name": "escaping hr markdown with more than 3 characters",+ "html": "* * * * *",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\* \\* \\* \\* \\*"+ },+ {+ "id": "upstream_103",+ "name": "escaping ol markdown",+ "html": "1984. by George Orwell",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "1984\\. by George Orwell"+ },+ {+ "id": "upstream_104",+ "name": "not escaping . outside of an ol",+ "html": "1984.George Orwell wrote 1984.",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "1984.George Orwell wrote 1984."+ },+ {+ "id": "upstream_105",+ "name": "escaping ul markdown *",+ "html": "* An unordered list item",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\* An unordered list item"+ },+ {+ "id": "upstream_106",+ "name": "escaping ul markdown -",+ "html": "- An unordered list item",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\- An unordered list item"+ },+ {+ "id": "upstream_107",+ "name": "escaping ul markdown +",+ "html": "+ An unordered list item",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\+ An unordered list item"+ },+ {+ "id": "upstream_108",+ "name": "not escaping - outside of a ul",+ "html": "Hello-world, 45 - 3 is 42",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Hello-world, 45 - 3 is 42"+ },+ {+ "id": "upstream_109",+ "name": "not escaping + outside of a ul",+ "html": "+1 and another +",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "+1 and another +"+ },+ {+ "id": "upstream_110",+ "name": "escaping *",+ "html": "You can use * for multiplication",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "You can use \\* for multiplication"+ },+ {+ "id": "upstream_111",+ "name": "escaping ** inside strong tags",+ "html": "<strong>**test</strong>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "**\\*\\*test**"+ },+ {+ "id": "upstream_112",+ "name": "escaping _ inside em tags",+ "html": "<em>test_italics</em>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "*test\\_italics*"+ },+ {+ "id": "upstream_113",+ "name": "escaping > as blockquote",+ "html": "> Blockquote in markdown",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\> Blockquote in markdown"+ },+ {+ "id": "upstream_114",+ "name": "escaping > as blockquote without space",+ "html": ">Blockquote in markdown",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\>Blockquote in markdown"+ },+ {+ "id": "upstream_115",+ "name": "not escaping > outside of a blockquote",+ "html": "42 > 1",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "42 > 1"+ },+ {+ "id": "upstream_116",+ "name": "escaping code",+ "html": "`not code`",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\`not code\\`"+ },+ {+ "id": "upstream_117",+ "name": "escaping []",+ "html": "[This] is a sentence with brackets",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\[This\\] is a sentence with brackets"+ },+ {+ "id": "upstream_118",+ "name": "escaping [",+ "html": "<a href=\"http://www.example.com\">c[iao</a>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "[c\\[iao](http://www.example.com)"+ },+ {+ "id": "upstream_119",+ "name": "escaping * performance",+ "html": "fasdf *883 asdf wer qweasd fsd asdf asdfaqwe rqwefrsdf",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "fasdf \\*883 asdf wer qweasd fsd asdf asdfaqwe rqwefrsdf"+ },+ {+ "id": "upstream_120",+ "name": "escaping multiple asterisks",+ "html": "<p>* * ** It aims to be*</p>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\* \\* \\*\\* It aims to be\\*"+ },+ {+ "id": "upstream_121",+ "name": "escaping delimiters around short words and numbers",+ "html": "<p>_Really_? Is that what it _is_? A **2000** year-old computer?</p>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "\\_Really\\_? Is that what it \\_is\\_? A \\*\\*2000\\*\\* year-old computer?"+ },+ {+ "id": "upstream_122",+ "name": "non-markdown block elements",+ "html": "\n Foo\n <div>Bar</div>\n Baz\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Foo\n\nBar\n\nBaz"+ },+ {+ "id": "upstream_123",+ "name": "non-markdown inline elements",+ "html": "\n Foo <span>Bar</span>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Foo Bar"+ },+ {+ "id": "upstream_124",+ "name": "blank inline elements",+ "html": "\n Hello <em></em>world\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Hello world"+ },+ {+ "id": "upstream_125",+ "name": "elements with a single void element",+ "html": "\n <p><img src=\"http://example.com/logo.png\"></p>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_126",+ "name": "elements with a nested void element",+ "html": "\n <p><span><img src=\"http://example.com/logo.png\"></span></p>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "upstream_127",+ "name": "text separated by a space in an element",+ "html": "\n <p>Foo<span> </span>Bar</p>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Foo Bar"+ },+ {+ "id": "upstream_128",+ "name": "text separated by a non-breaking space in an element",+ "html": "\n <p>Foo<span> </span>Bar</p>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Foo Bar"+ },+ {+ "id": "upstream_129",+ "name": "triple tildes inside code",+ "html": "\n<pre><code>~~~\nCode\n~~~\n</code></pre>\n ",+ "original_options": {+ "codeBlockStyle": "fenced",+ "fence": "~~~"+ },+ "preformatted_code": false,+ "markdown": "```\n~~~\nCode\n~~~\n```"+ },+ {+ "id": "upstream_130",+ "name": "triple ticks inside code",+ "html": "\n<pre><code>```\nCode\n```\n</code></pre>\n ",+ "original_options": {+ "codeBlockStyle": "fenced",+ "fence": "```"+ },+ "preformatted_code": false,+ "markdown": "````\n```\nCode\n```\n````"+ },+ {+ "id": "upstream_131",+ "name": "four ticks inside code",+ "html": "\n<pre><code>````\nCode\n````\n</code></pre>\n ",+ "original_options": {+ "codeBlockStyle": "fenced",+ "fence": "```"+ },+ "preformatted_code": false,+ "markdown": "`````\n````\nCode\n````\n`````"+ },+ {+ "id": "upstream_132",+ "name": "empty line in start/end of code block",+ "html": "\n<pre><code>\nCode\n\n</code></pre>\n ",+ "original_options": {+ "codeBlockStyle": "fenced",+ "fence": "```"+ },+ "preformatted_code": false,+ "markdown": "```\n\nCode\n\n```"+ },+ {+ "id": "upstream_133",+ "name": "text separated by ASCII and nonASCII space in an element",+ "html": "\n <p>Foo<span> </span>Bar</p>\n ",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "Foo Bar"+ },+ {+ "id": "upstream_134",+ "name": "list-like text with non-breaking spaces",+ "html": " 1. First<br> 2. Second",+ "original_options": {},+ "preformatted_code": false,+ "markdown": " 1. First \n 2. Second"+ },+ {+ "id": "upstream_135",+ "name": "element with trailing nonASCII WS followed by nonWS",+ "html": "<i>foo </i>bar",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "*foo* bar"+ },+ {+ "id": "upstream_136",+ "name": "element with trailing nonASCII WS followed by nonASCII WS",+ "html": "<i>foo </i> bar",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "*foo* bar"+ },+ {+ "id": "upstream_137",+ "name": "element with trailing ASCII WS followed by nonASCII WS",+ "html": "<i>foo </i> bar",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "*foo* bar"+ },+ {+ "id": "upstream_138",+ "name": "element with trailing nonASCII WS followed by ASCII WS",+ "html": "<i>foo </i> bar",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "*foo* bar"+ },+ {+ "id": "upstream_139",+ "name": "nonWS followed by element with leading nonASCII WS",+ "html": "foo<i> bar</i>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "foo *bar*"+ },+ {+ "id": "upstream_140",+ "name": "nonASCII WS followed by element with leading nonASCII WS",+ "html": "foo <i> bar</i>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "foo *bar*"+ },+ {+ "id": "upstream_141",+ "name": "nonASCII WS followed by element with leading ASCII WS",+ "html": "foo <i> bar</i>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "foo *bar*"+ },+ {+ "id": "upstream_142",+ "name": "ASCII WS followed by element with leading nonASCII WS",+ "html": "foo <i> bar</i>",+ "original_options": {},+ "preformatted_code": false,+ "markdown": "foo *bar*"+ },+ {+ "id": "upstream_143",+ "name": "preformatted code with leading whitespace",+ "html": "Four spaces <code> make an indented code block in Markdown</code>",+ "original_options": {+ "preformattedCode": true+ },+ "preformatted_code": true,+ "markdown": "Four spaces ` make an indented code block in Markdown`"+ },+ {+ "id": "upstream_144",+ "name": "preformatted code with trailing whitespace",+ "html": "<code>A line break </code> <b> note the spaces</b>",+ "original_options": {+ "preformattedCode": true+ },+ "preformatted_code": true,+ "markdown": "`A line break ` **note the spaces**"+ },+ {+ "id": "upstream_145",+ "name": "preformatted code tightly surrounded",+ "html": "<b>tight</b><code>code</code><b>wrap</b>",+ "original_options": {+ "preformattedCode": true+ },+ "preformatted_code": true,+ "markdown": "**tight**`code`**wrap**"+ },+ {+ "id": "upstream_146",+ "name": "preformatted code loosely surrounded",+ "html": "<b>not so tight </b><code>code</code><b> wrap</b>",+ "original_options": {+ "preformattedCode": true+ },+ "preformatted_code": true,+ "markdown": "**not so tight** `code` **wrap**"+ },+ {+ "id": "upstream_147",+ "name": "preformatted code with newlines",+ "html": "\n<code>\n\n nasty\ncode\n\n</code>\n ",+ "original_options": {+ "preformattedCode": true+ },+ "preformatted_code": true,+ "markdown": "` nasty code `"+ },+ {+ "id": "extra_empty_href",+ "name": "empty href keeps link text",+ "html": "before<a href=''>label</a>after",+ "preformatted_code": false,+ "markdown": "beforelabelafter"+ },+ {+ "id": "extra_empty_link",+ "name": "empty link label",+ "html": "before<a href='/target'></a>after",+ "preformatted_code": false,+ "markdown": "before[](/target)after"+ },+ {+ "id": "extra_blank_link",+ "name": "whitespace-only link label",+ "html": "before<a href='/target'> </a>after",+ "preformatted_code": false,+ "markdown": "before [](/target)after"+ },+ {+ "id": "extra_missing_image_src",+ "name": "image alt without source",+ "html": "before<img alt='missing'>after",+ "preformatted_code": false,+ "markdown": "beforeafter"+ },+ {+ "id": "extra_img_alt_whitespace",+ "name": "image alt with multiple lines and indentation",+ "html": "<img src='/x' alt='one\n \n two'>",+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "extra_img_alt_code",+ "name": "image label containing code delimiters",+ "html": "<img src='/x' alt='a`b'>",+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "extra_adjacent_lists",+ "name": "adjacent unordered lists stay separate",+ "html": "<ul><li>one</li></ul><ul><li>two</li></ul>",+ "preformatted_code": false,+ "markdown": "- one\n\n- two"+ },+ {+ "id": "extra_adjacent_ordered_lists",+ "name": "adjacent ordered lists stay separate",+ "html": "<ol><li>one</li></ol><ol start='4'><li>four</li></ol>",+ "preformatted_code": false,+ "markdown": "1. one\n\n4. four"+ },+ {+ "id": "extra_empty_list",+ "name": "empty list",+ "html": "before<ul></ul>after",+ "preformatted_code": false,+ "markdown": "before\n\nafter"+ },+ {+ "id": "extra_empty_list_item",+ "name": "empty list item",+ "html": "<ul><li></li><li>text</li></ul>",+ "preformatted_code": false,+ "markdown": "- text"+ },+ {+ "id": "extra_loose_list",+ "name": "paragraph list items stay loose",+ "html": "<ul><li><p>one</p></li><li><p>two</p></li></ul>",+ "preformatted_code": false,+ "markdown": "- one\n \n- two"+ },+ {+ "id": "extra_mixed_loose_list",+ "name": "paragraph followed by a plain list item",+ "html": "<ul><li><p>one</p></li><li>two</li></ul>",+ "preformatted_code": false,+ "markdown": "- one\n \n- two"+ },+ {+ "id": "extra_list_nested_only",+ "name": "list item containing only a nested list",+ "html": "<ul><li><ul><li>inner</li></ul></li></ul>",+ "preformatted_code": false,+ "markdown": "- - inner"+ },+ {+ "id": "extra_list_nested_tail",+ "name": "text after a nested list",+ "html": "<ul><li>start<ul><li>inner</li></ul>end</li></ul>",+ "preformatted_code": false,+ "markdown": "- start\n - innerend"+ },+ {+ "id": "extra_list_nested_tail_p",+ "name": "paragraph after a nested list",+ "html": "<ul><li><p>start</p><ul><li>inner</li></ul><p>end</p></li></ul>",+ "preformatted_code": false,+ "markdown": "- start\n \n - inner\n \n end"+ },+ {+ "id": "extra_list_code",+ "name": "fenced code inside list item",+ "html": "<ol start='9'><li><pre><code> first\nsecond\n</code></pre></li><li>end</li></ol>",+ "preformatted_code": false,+ "markdown": "9. ```\n first\n second\n ```\n \n10. end"+ },+ {+ "id": "extra_list_quote",+ "name": "list item containing only a quote",+ "html": "<ul><li><blockquote><p>quote</p></blockquote></li></ul>",+ "preformatted_code": false,+ "markdown": "- > quote"+ },+ {+ "id": "extra_empty_quote",+ "name": "empty quote still separates blocks",+ "html": "before<blockquote></blockquote>after",+ "preformatted_code": false,+ "markdown": "before\n\nafter"+ },+ {+ "id": "extra_quote_blank_lines",+ "name": "quote with paragraphs and hard breaks",+ "html": "<blockquote><p>one<br>two</p><p>three</p></blockquote>",+ "preformatted_code": false,+ "markdown": "> one \n> two\n> \n> three"+ },+ {+ "id": "extra_empty_heading",+ "name": "empty heading boundary",+ "html": "before<h2></h2>after",+ "preformatted_code": false,+ "markdown": "before\n\nafter"+ },+ {+ "id": "extra_empty_pre",+ "name": "empty pre",+ "html": "before<pre></pre>after",+ "preformatted_code": false,+ "markdown": "before\n\nafter"+ },+ {+ "id": "extra_empty_pre_code",+ "name": "empty pre code",+ "html": "before<pre><code></code></pre>after",+ "preformatted_code": false,+ "markdown": "before\n\nafter"+ },+ {+ "id": "extra_blank_pre_code",+ "name": "whitespace-only pre code",+ "html": "<pre><code> \n</code></pre>",+ "preformatted_code": false,+ "markdown": ""+ },+ {+ "id": "extra_adjacent_code",+ "name": "adjacent code spans",+ "html": "<code>a</code><code>b</code>",+ "preformatted_code": false,+ "markdown": "`a``b`"+ },+ {+ "id": "extra_code_gap",+ "name": "separated code spans",+ "html": "<code>a</code> <code>b</code>",+ "preformatted_code": false,+ "markdown": "`a` `b`"+ },+ {+ "id": "extra_code_nested_format",+ "name": "markup inside inline code",+ "html": "<code><strong>a</strong><em>b</em></code>",+ "preformatted_code": false,+ "markdown": "`**a***b*`"+ },+ {+ "id": "extra_code_many_runs",+ "name": "code span chooses an unused delimiter length",+ "html": "<code>a```b`````c</code>",+ "preformatted_code": false,+ "markdown": "`a```b`````c`"+ },+ {+ "id": "extra_code_only_ticks",+ "name": "code containing only backticks",+ "html": "<code>```</code>",+ "preformatted_code": false,+ "markdown": "` ``` `"+ },+ {+ "id": "extra_code_nbsp",+ "name": "code with nonbreaking boundary spaces",+ "html": "before<code> a </code>after",+ "preformatted_code": false,+ "markdown": "before `a` after"+ },+ {+ "id": "extra_preformatted_internal",+ "name": "preformatted inline code preserves multiple spaces",+ "html": "<code> a b </code>",+ "original_options": {+ "preformattedCode": true+ },+ "preformatted_code": true,+ "markdown": "` a b `"+ },+ {+ "id": "extra_preformatted_blank",+ "name": "preformatted whitespace-only code",+ "html": "before<code> </code>after",+ "original_options": {+ "preformattedCode": true+ },+ "preformatted_code": true,+ "markdown": "beforeafter"+ },+ {+ "id": "extra_literal_entities",+ "name": "decoded entity text remains literal",+ "html": "<p>&copy; &lt; &#32;</p>",+ "preformatted_code": false,+ "markdown": "© <  "+ },+ {+ "id": "extra_literal_html",+ "name": "literal HTML tags remain visible text",+ "html": "<p><b>literal</b></p>",+ "preformatted_code": false,+ "markdown": "<b>literal</b>"+ },+ {+ "id": "extra_split_list_marker",+ "name": "ordered marker split across elements",+ "html": "<p>12<span>.</span> text</p>",+ "preformatted_code": false,+ "markdown": "12. text"+ },+ {+ "id": "extra_split_emphasis_literal",+ "name": "asterisks split across elements",+ "html": "<p>*<span>word</span>*</p>",+ "preformatted_code": false,+ "markdown": "\\*word\\*"+ },+ {+ "id": "extra_link_backslash",+ "name": "link destination with backslashes",+ "html": "<a href='a\\b'>link</a>",+ "preformatted_code": false,+ "markdown": "[link](a\\b)"+ },+ {+ "id": "extra_link_entities",+ "name": "link destination with literal entity syntax",+ "html": "<a href='/?q=&copy;'>link</a>",+ "preformatted_code": false,+ "markdown": "[link](/?q=©)"+ },+ {+ "id": "extra_link_leading_space",+ "name": "leading whitespace inside link label",+ "html": "before<a href='/'> label</a>after",+ "preformatted_code": false,+ "markdown": "before [label](/)after"+ },+ {+ "id": "extra_nested_link_emphasis",+ "name": "emphasis nested in link",+ "html": "<a href='/'><strong><em>text</em></strong></a>",+ "preformatted_code": false,+ "markdown": "[***text***](/)"+ },+ {+ "id": "extra_emphasis_nested_link",+ "name": "link nested in emphasis",+ "html": "<em><a href='/'>text</a></em>",+ "preformatted_code": false,+ "markdown": "*[text](/)*"+ },+ {+ "id": "extra_emphasis_intraword_link",+ "name": "link in intraword emphasis",+ "html": "before<em><a href='/'>text</a></em>after",+ "preformatted_code": false,+ "markdown": "before*[text](/)*after"+ },+ {+ "id": "extra_emphasis_intraword_punctuation",+ "name": "intraword emphasis around punctuation",+ "html": "before<em>(text)</em>after",+ "preformatted_code": false,+ "markdown": "before*(text)*after"+ },+ {+ "id": "extra_empty_emphasis",+ "name": "empty formatting preserves surrounding spaces",+ "html": "a<strong> </strong>b<em> </em>c",+ "preformatted_code": false,+ "markdown": "a b c"+ },+ {+ "id": "extra_nbsp_comment",+ "name": "nonbreaking whitespace around ignored comment",+ "html": "a <!--x--> b",+ "preformatted_code": false,+ "markdown": "a b"+ },+ {+ "id": "extra_leading_br",+ "name": "leading hard break",+ "html": "<p><br>text</p>",+ "preformatted_code": false,+ "markdown": " \ntext"+ },+ {+ "id": "extra_consecutive_br",+ "name": "consecutive hard breaks",+ "html": "<p>one<br><br>two</p>",+ "preformatted_code": false,+ "markdown": "one \n \ntwo"+ },+ {+ "id": "extra_blank_br_container",+ "name": "otherwise empty span containing hard break",+ "html": "a<span><br></span>b",+ "preformatted_code": false,+ "markdown": "a \nb"+ },+ {+ "id": "extra_empty_block_inside_inline",+ "name": "empty block inside emphasis",+ "html": "<strong>a<div></div>b</strong>",+ "preformatted_code": false,+ "markdown": "**a\n\nb**"+ },+ {+ "id": "extra_audio_boundary",+ "name": "audio fallback content is a block",+ "html": "before<audio>fallback</audio>after",+ "preformatted_code": false,+ "markdown": "before\n\nfallback\n\nafter"+ },+ {+ "id": "extra_output_boundary",+ "name": "output element block boundary",+ "html": "before<output>result</output>after",+ "preformatted_code": false,+ "markdown": "before\n\nresult\n\nafter"+ }+ ]+}diff --git a/moli-html2md/tests/turndown/cases.rs b/moli-html2md/tests/turndown/cases.rsnew file mode 100644index 000000000..0d5e0e2a9--- /dev/null+++ b/moli-html2md/tests/turndown/cases.rs@@ -0,0 +1,198 @@+// Generated by generate.mjs. See README.md and LICENSE.+case!(upstream_001, 0);+case!(upstream_002, 1);+case!(upstream_003, 2);+case!(upstream_004, 3);+case!(upstream_005, 4);+case!(upstream_006, 5);+case!(upstream_007, 6);+case!(upstream_008, 7);+case!(upstream_009, 8);+case!(upstream_010, 9);+case!(upstream_011, 10);+case!(upstream_012, 11);+case!(upstream_013, 12);+case!(upstream_014, 13);+case!(upstream_015, 14);+case!(upstream_016, 15);+case!(upstream_017, 16);+case!(upstream_018, 17);+case!(upstream_019, 18);+case!(upstream_020, 19);+case!(upstream_021, 20);+case!(upstream_022, 21);+case!(upstream_023, 22);+case!(upstream_024, 23);+case!(upstream_025, 24);+case!(upstream_026, 25);+case!(upstream_027, 26);+case!(upstream_028, 27);+case!(upstream_029, 28);+case!(upstream_030, 29);+case!(upstream_031, 30);+case!(upstream_032, 31);+case!(upstream_033, 32);+case!(upstream_034, 33);+case!(upstream_035, 34);+case!(upstream_036, 35);+case!(upstream_037, 36);+case!(upstream_038, 37);+case!(upstream_039, 38);+case!(upstream_040, 39);+case!(upstream_041, 40);+case!(upstream_042, 41);+case!(upstream_043, 42);+case!(upstream_044, 43);+case!(upstream_045, 44);+case!(upstream_046, 45);+case!(upstream_047, 46);+case!(upstream_048, 47);+case!(upstream_049, 48);+case!(upstream_050, 49);+case!(upstream_051, 50);+case!(upstream_052, 51);+case!(upstream_053, 52);+case!(upstream_054, 53);+case!(upstream_055, 54);+case!(upstream_056, 55);+case!(upstream_057, 56);+case!(upstream_058, 57);+case!(upstream_059, 58);+case!(upstream_060, 59);+case!(upstream_061, 60);+case!(upstream_062, 61);+case!(upstream_063, 62);+case!(upstream_064, 63);+case!(upstream_065, 64);+case!(upstream_066, 65);+case!(upstream_067, 66);+case!(upstream_068, 67);+case!(upstream_069, 68);+case!(upstream_070, 69);+case!(upstream_071, 70);+case!(upstream_072, 71);+case!(upstream_073, 72);+case!(upstream_074, 73);+case!(upstream_075, 74);+case!(upstream_076, 75);+case!(upstream_077, 76);+case!(upstream_078, 77);+case!(upstream_079, 78);+case!(upstream_080, 79);+case!(upstream_081, 80);+case!(upstream_082, 81);+case!(upstream_083, 82);+case!(upstream_084, 83);+case!(upstream_085, 84);+case!(upstream_086, 85);+case!(upstream_087, 86);+case!(upstream_088, 87);+case!(upstream_089, 88);+case!(upstream_090, 89);+case!(upstream_091, 90);+case!(upstream_092, 91);+case!(upstream_093, 92);+case!(upstream_094, 93);+case!(upstream_095, 94);+case!(upstream_096, 95);+case!(upstream_097, 96);+case!(upstream_098, 97);+case!(upstream_099, 98);+case!(upstream_100, 99);+case!(upstream_101, 100);+case!(upstream_102, 101);+case!(upstream_103, 102);+case!(upstream_104, 103);+case!(upstream_105, 104);+case!(upstream_106, 105);+case!(upstream_107, 106);+case!(upstream_108, 107);+case!(upstream_109, 108);+case!(upstream_110, 109);+case!(upstream_111, 110);+case!(upstream_112, 111);+case!(upstream_113, 112);+case!(upstream_114, 113);+case!(upstream_115, 114);+case!(upstream_116, 115);+case!(upstream_117, 116);+case!(upstream_118, 117);+case!(upstream_119, 118);+case!(upstream_120, 119);+case!(upstream_121, 120);+case!(upstream_122, 121);+case!(upstream_123, 122);+case!(upstream_124, 123);+case!(upstream_125, 124);+case!(upstream_126, 125);+case!(upstream_127, 126);+case!(upstream_128, 127);+case!(upstream_129, 128);+case!(upstream_130, 129);+case!(upstream_131, 130);+case!(upstream_132, 131);+case!(upstream_133, 132);+case!(upstream_134, 133);+case!(upstream_135, 134);+case!(upstream_136, 135);+case!(upstream_137, 136);+case!(upstream_138, 137);+case!(upstream_139, 138);+case!(upstream_140, 139);+case!(upstream_141, 140);+case!(upstream_142, 141);+case!(upstream_143, 142);+case!(upstream_144, 143);+case!(upstream_145, 144);+case!(upstream_146, 145);+case!(upstream_147, 146);+case!(extra_empty_href, 147);+case!(extra_empty_link, 148);+case!(extra_blank_link, 149);+case!(extra_missing_image_src, 150);+case!(extra_img_alt_whitespace, 151);+case!(extra_img_alt_code, 152);+case!(extra_adjacent_lists, 153);+case!(extra_adjacent_ordered_lists, 154);+case!(extra_empty_list, 155);+case!(extra_empty_list_item, 156);+case!(extra_loose_list, 157);+case!(extra_mixed_loose_list, 158);+case!(extra_list_nested_only, 159);+case!(extra_list_nested_tail, 160);+case!(extra_list_nested_tail_p, 161);+case!(extra_list_code, 162);+case!(extra_list_quote, 163);+case!(extra_empty_quote, 164);+case!(extra_quote_blank_lines, 165);+case!(extra_empty_heading, 166);+case!(extra_empty_pre, 167);+case!(extra_empty_pre_code, 168);+case!(extra_blank_pre_code, 169);+case!(extra_adjacent_code, 170);+case!(extra_code_gap, 171);+case!(extra_code_nested_format, 172);+case!(extra_code_many_runs, 173);+case!(extra_code_only_ticks, 174);+case!(extra_code_nbsp, 175);+case!(extra_preformatted_internal, 176);+case!(extra_preformatted_blank, 177);+case!(extra_literal_entities, 178);+case!(extra_literal_html, 179);+case!(extra_split_list_marker, 180);+case!(extra_split_emphasis_literal, 181);+case!(extra_link_backslash, 182);+case!(extra_link_entities, 183);+case!(extra_link_leading_space, 184);+case!(extra_nested_link_emphasis, 185);+case!(extra_emphasis_nested_link, 186);+case!(extra_emphasis_intraword_link, 187);+case!(extra_emphasis_intraword_punctuation, 188);+case!(extra_empty_emphasis, 189);+case!(extra_nbsp_comment, 190);+case!(extra_leading_br, 191);+case!(extra_consecutive_br, 192);+case!(extra_blank_br_container, 193);+case!(extra_empty_block_inside_inline, 194);+case!(extra_audio_boundary, 195);+case!(extra_output_boundary, 196);diff --git a/moli-html2md/tests/turndown/expectations.json b/moli-html2md/tests/turndown/expectations.jsonnew file mode 100644index 000000000..f9b20051c--- /dev/null+++ b/moli-html2md/tests/turndown/expectations.json@@ -0,0 +1,50 @@+{+ "upstream_058": {+ "markdown": "```\n~~~ foo\n```",+ "reason": "Moli fences bare pre elements as well as pre/code, preserving preformatted text. Turndown only recognizes a pre/code pair."+ },+ "extra_adjacent_code": {+ "markdown": "`ab`",+ "reason": "Adjacent code elements contain ab. Turndown joins their delimiters into one code span whose text contains two extra backticks."+ },+ "extra_code_nested_format": {+ "markdown": "`ab`",+ "reason": "Code captures raw descendant text. Turndown includes the Markdown markers of nested strong/em elements in the code text."+ },+ "extra_emphasis_intraword_link": {+ "markdown": "before<em>[text](/)</em>after",+ "reason": "Turndown's star delimiters do not open emphasis at this punctuation boundary. Inline HTML preserves the original emphasized link."+ },+ "extra_emphasis_intraword_punctuation": {+ "markdown": "before<em>(text)</em>after",+ "reason": "CommonMark does not recognize Turndown's intraword delimiters next to punctuation. Inline HTML preserves emphasis without literal stars."+ },+ "extra_empty_block_inside_inline": {+ "markdown": "**a**\n\n**b**",+ "reason": "Emphasis must close and reopen across paragraph boundaries; delimiters spanning the blank line become literal text."+ },+ "extra_list_nested_tail": {+ "markdown": "- start\n - inner\n\n end",+ "reason": "The trailing text belongs to the outer item. Turndown appends it to the nested item, losing the block boundary and joining words."+ },+ "extra_literal_entities": {+ "markdown": "&copy; &lt; &#32;",+ "reason": "DOM text is already decoded. Escape literal entity syntax so a Markdown reader does not decode it a second time."+ },+ "extra_literal_html": {+ "markdown": "\\<b\\>literal\\</b\\>",+ "reason": "Escaped HTML in the source is visible text, not markup. Turndown turns the decoded text into executable Markdown HTML tags."+ },+ "extra_link_entities": {+ "markdown": "[link](/?q=&copy;)",+ "reason": "The literal entity syntax in the decoded href must not be decoded again into a different URL."+ },+ "extra_preformatted_blank": {+ "markdown": "before` `after",+ "reason": "Explicit preformatted-code mode preserves the spaces inside code. Turndown removes a whitespace-only code element entirely."+ },+ "extra_split_list_marker": {+ "markdown": "12\\. text",+ "reason": "Escape list markers across text-node boundaries. Turndown escapes nodes individually and accidentally creates an ordered list."+ }+}diff --git a/moli-html2md/tests/turndown/generate.mjs b/moli-html2md/tests/turndown/generate.mjsnew file mode 100644index 000000000..34072bdf9--- /dev/null+++ b/moli-html2md/tests/turndown/generate.mjs@@ -0,0 +1,58 @@+// Regenerate the offline reference corpus from the pinned Turndown checkout.+// See README.md in this directory for dependencies and invocation.+import assert from 'node:assert/strict';+import { execFileSync } from 'node:child_process';+import { createRequire } from 'node:module';+import { readFileSync, writeFileSync, mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, resolve } from 'node:path';+import { fileURLToPath, pathToFileURL } from 'node:url';++const revision = 'aa84dfa3e2361edea8c43acbfc2b7a9363494bfd';+const upstream = resolve(process.argv[2]);+assert.equal(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: upstream, encoding: 'utf8' }).trim(), revision);+const require = createRequire(join(upstream, 'package.json'));+const domino = require('@mixmark-io/domino');+assert.equal(require('@mixmark-io/domino/package.json').version, '2.2.0');+const destination = dirname(fileURLToPath(import.meta.url));+const temporary = mkdtempSync(join(tmpdir(), 'moli-turndown-esm-'));+try {+ // Node's ESM loader needs file extensions; Turndown's build normally supplies+ // these through Rollup. Only import paths and CommonJS require setup change.+ for (const name of ['collapse-whitespace', 'commonmark-rules', 'html-parser', 'node', 'root-node', 'rules', 'turndown', 'utilities']) {+ const original = readFileSync(join(upstream, 'src', name + '.js'), 'utf8');+ const source = original.replace(/from '(\.\/[^']+)'/g, "from '$1.mjs'");+ const header = `import { createRequire } from 'node:module';\nconst require = createRequire(${JSON.stringify(join(upstream, 'package.json'))});\n`;+ writeFileSync(join(temporary, name + '.mjs'), header + source);+ }+ const { default: Turndown } = await import(pathToFileURL(join(temporary, 'turndown.mjs')));+ const options = {+ headingStyle: 'atx', hr: '---', bulletListMarker: '-',+ codeBlockStyle: 'fenced', fence: '```', emDelimiter: '*',+ strongDelimiter: '**', linkStyle: 'inlined', br: ' ',+ };+ const document = domino.createDocument(readFileSync(join(upstream, 'test', 'index.html'), 'utf8'));+ const upstreamCases = Array.from(document.querySelectorAll('.case')).map((node, index) => ({+ id: `upstream_${String(index + 1).padStart(3, '0')}`,+ name: node.getAttribute('data-name'),+ html: node.querySelector('.input').innerHTML,+ original_options: JSON.parse(node.getAttribute('data-options') || '{}'),+ }));+ const extra = JSON.parse(readFileSync(join(destination, 'additional.json'), 'utf8'));+ const cases = [...upstreamCases, ...extra].map(test => {+ const preformattedCode = test.original_options?.preformattedCode || false;+ const reference = new Turndown({ ...options, preformattedCode });+ return { ...test, preformatted_code: preformattedCode, markdown: reference.turndown(test.html) };+ });+ const corpus = {+ upstream: { repository: 'https://github.com/mixmark-io/turndown', revision, version: '7.2.4', domino: '2.2.0' },+ options,+ cases,+ };+ writeFileSync(join(destination, 'cases.json'), JSON.stringify(corpus, null, 2) + '\n');+ writeFileSync(join(destination, 'LICENSE'), readFileSync(join(upstream, 'LICENSE')));+ writeFileSync(join(destination, 'cases.rs'), '// Generated by generate.mjs. See README.md and LICENSE.\n' + cases.map((test, index) => `case!(${test.id}, ${index});`).join('\n') + '\n');+ console.log(`Generated ${upstreamCases.length} upstream cases and ${extra.length} additional cases.`);+} finally {+ rmSync(temporary, { recursive: true, force: true });+}diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.tomlindex b9f38b59d..d5ad20d59 100644--- a/moli-renderer-v8/Cargo.toml+++ b/moli-renderer-v8/Cargo.toml@@ -53,6 +53,7 @@ moli-fetch = { path = "../moli-fetch" }moli-file-api = { path = "../moli-file-api" }moli-geometry = { path = "../moli-geometry" }moli-http-cache = { path = "../moli-http-cache" }+moli-html2md = { path = "../moli-html2md" }moli-image = { path = "../moli-image" }moli-import-map = { path = "../moli-import-map" }moli-indexeddb = { path = "../moli-indexeddb" }diff --git a/moli-renderer-v8/src/runtime/page_dump.rs b/moli-renderer-v8/src/runtime/page_dump.rsindex 07599a27c..fac217816 100644--- a/moli-renderer-v8/src/runtime/page_dump.rs+++ b/moli-renderer-v8/src/runtime/page_dump.rs@@ -1,4 +1,4 @@-use crate::dom::native::{NativeDom, NativeNodeId, NodeData};+use crate::dom::native::{NativeDom, NativeNodeId};use crate::runtime::page_surface::{RendererPageDumpFormat, RendererPageDumpOptions, RendererPageDumpStripOptions,};@@ -8,6 +8,13 @@ use super::page_vm::PageVm;impl PageVm {pub(crate) fn render_page_dump(&mut self, options: RendererPageDumpOptions) -> String {+ if options.format == RendererPageDumpFormat::Markdown+ && !options.with_base+ && !options.with_frames+ && options.strip == RendererPageDumpStripOptions::default()+ {+ return render_markdown_document(self.vm().document_runtime.dom_host().dom());+ }let mut dom = self.vm().document_runtime.dom_host().dom().clone();if options.with_base {@@ -178,408 +185,12 @@ fn collect_node_ids(dom: &NativeDom, node_id: NativeNodeId, out: &mut Vec<Nativefn render_markdown_document(dom: &NativeDom) -> String {let root = dom.body_node_id().unwrap_or(dom.document_node_id());- let mut out = String::new();- render_markdown_node(dom, root, &mut out, 0, MAX_DOM_OUTPUT_TREE_DEPTH);- normalize_markdown(&out)-}--enum MarkdownFrame {- Enter {- node_id: NativeNodeId,- list_depth: usize,- remaining_tree_depth: usize,- target: usize,- },- Append {- target: usize,- text: &'static str,- },- FinishInline {- source: usize,- target: usize,- kind: MarkdownInlineKind,- },-}--enum MarkdownInlineKind {- Paragraph,- Anchor { href: String },- ListItem { list_depth: usize },- Strong,- Emphasis,- Code,-}--fn render_markdown_node(- dom: &NativeDom,- node_id: NativeNodeId,- out: &mut String,- list_depth: usize,- remaining_tree_depth: usize,-) {- render_markdown_nodes(dom, [node_id], out, list_depth, remaining_tree_depth);-}--fn render_markdown_nodes(- dom: &NativeDom,- roots: impl IntoIterator<Item = NativeNodeId>,- out: &mut String,- list_depth: usize,- remaining_tree_depth: usize,-) {- let mut buffers = vec![String::new()];- let mut stack = roots- .into_iter()- .map(|node_id| MarkdownFrame::Enter {- node_id,- list_depth,- remaining_tree_depth,- target: 0,- })- .collect::<Vec<_>>();- stack.reverse();-- while let Some(frame) = stack.pop() {- match frame {- MarkdownFrame::Enter {- node_id,- list_depth,- remaining_tree_depth,- target,- } => render_markdown_enter_node(- dom,- node_id,- &mut buffers,- &mut stack,- list_depth,- remaining_tree_depth,- target,- ),- MarkdownFrame::Append { target, text } => buffers[target].push_str(text),- MarkdownFrame::FinishInline {- source,- target,- kind,- } => {- let text = std::mem::take(&mut buffers[source]);- finish_inline_markdown(kind, &text, &mut buffers[target]);- }- }- }-- out.push_str(&buffers[0]);-}--fn render_markdown_enter_node(- dom: &NativeDom,- node_id: NativeNodeId,- buffers: &mut Vec<String>,- stack: &mut Vec<MarkdownFrame>,- list_depth: usize,- remaining_tree_depth: usize,- target: usize,-) {- let Some(next_tree_depth) = remaining_tree_depth.checked_sub(1) else {- return;- };- let Some(node) = dom.node(node_id) else {- return;- };-- match node.kind() {- NodeData::Document(_) | NodeData::DocumentFragment(_) => {- push_child_markdown_frames(stack, dom, node_id, list_depth, next_tree_depth, target);- }- NodeData::Text(text) => {- push_markdown_text(&mut buffers[target], text.data());- }- NodeData::CDataSection(cdata) => {- push_markdown_text(&mut buffers[target], cdata.data());- }- NodeData::Comment(_) | NodeData::DocumentType(_) | NodeData::ProcessingInstruction(_) => {}- NodeData::Element(element) => match element.local_name() {- "head" | "script" | "style" | "noscript" => {}- "br" => buffers[target].push('\n'),- "hr" => buffers[target].push_str("\n---\n"),- "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {- let level = element.local_name()[1..].parse::<usize>().unwrap_or(1);- let text = collapse_whitespace(&dom.text_content(node_id).unwrap_or_default());- if !text.is_empty() {- buffers[target].push('\n');- buffers[target].push_str(&"#".repeat(level));- buffers[target].push(' ');- buffers[target].push_str(&text);- buffers[target].push_str("\n\n");- }- }- "p" => {- push_inline_children_markdown_frames(- buffers,- stack,- dom,- node_id,- next_tree_depth,- target,- MarkdownInlineKind::Paragraph,- );- }- "pre" => {- let text = dom.text_content(node_id).unwrap_or_default();- if !text.trim().is_empty() {- buffers[target].push_str("\n```text\n");- buffers[target].push_str(text.trim_end());- buffers[target].push_str("\n```\n\n");- }- }- "code" => {- let text = dom.text_content(node_id).unwrap_or_default();- finish_inline_markdown(MarkdownInlineKind::Code, &text, &mut buffers[target]);- }- "a" => {- let href = dom.get_attribute(node_id, "href").unwrap_or_default();- push_inline_children_markdown_frames(- buffers,- stack,- dom,- node_id,- next_tree_depth,- target,- MarkdownInlineKind::Anchor { href },- );- }- "img" => {- let alt = dom.get_attribute(node_id, "alt").unwrap_or_default();- let src = dom.get_attribute(node_id, "src").unwrap_or_default();- buffers[target].push_str("![");- buffers[target].push_str(&alt);- buffers[target].push_str("](");- buffers[target].push_str(&src);- buffers[target].push(')');- }- "ul" => {- stack.push(MarkdownFrame::Append { target, text: "\n" });- push_child_markdown_frames(- stack,- dom,- node_id,- list_depth + 1,- next_tree_depth,- target,- );- stack.push(MarkdownFrame::Append { target, text: "\n" });- }- "ol" => {- stack.push(MarkdownFrame::Append { target, text: "\n" });- push_child_markdown_frames(- stack,- dom,- node_id,- list_depth + 1,- next_tree_depth,- target,- );- stack.push(MarkdownFrame::Append { target, text: "\n" });- }- "li" => {- push_inline_children_markdown_frames(- buffers,- stack,- dom,- node_id,- next_tree_depth,- target,- MarkdownInlineKind::ListItem { list_depth },- );- }- "strong" | "b" => {- push_inline_children_markdown_frames(- buffers,- stack,- dom,- node_id,- next_tree_depth,- target,- MarkdownInlineKind::Strong,- );- }- "em" | "i" => {- push_inline_children_markdown_frames(- buffers,- stack,- dom,- node_id,- next_tree_depth,- target,- MarkdownInlineKind::Emphasis,- );- }- _ => {- if matches!(- element.local_name(),- "div" | "section" | "article" | "main" | "header" | "footer" | "aside" | "nav"- ) {- buffers[target].push('\n');- stack.push(MarkdownFrame::Append { target, text: "\n" });- }- push_child_markdown_frames(- stack,- dom,- node_id,- list_depth,- next_tree_depth,- target,- );- }- },- }-}--fn push_inline_children_markdown_frames(- buffers: &mut Vec<String>,- stack: &mut Vec<MarkdownFrame>,- dom: &NativeDom,- node_id: NativeNodeId,- remaining_tree_depth: usize,- target: usize,- kind: MarkdownInlineKind,-) {- let source = buffers.len();- buffers.push(String::new());- stack.push(MarkdownFrame::FinishInline {- source,- target,- kind,- });- push_child_markdown_frames(stack, dom, node_id, 0, remaining_tree_depth, source);-}--fn push_child_markdown_frames(- stack: &mut Vec<MarkdownFrame>,- dom: &NativeDom,- node_id: NativeNodeId,- list_depth: usize,- remaining_tree_depth: usize,- target: usize,-) {- let child_ids = dom.child_ids(node_id).collect::<Vec<_>>();- for child_id in child_ids.into_iter().rev() {- stack.push(MarkdownFrame::Enter {- node_id: child_id,- list_depth,- remaining_tree_depth,- target,- });- }-}--fn finish_inline_markdown(kind: MarkdownInlineKind, text: &str, out: &mut String) {- // Spaces at inline element edges belong outside Markdown delimiters, but- // still separate this element's text from its siblings. Block edges trim.- let preserve_spacing = matches!(- kind,- MarkdownInlineKind::Anchor { .. }- | MarkdownInlineKind::Strong- | MarkdownInlineKind::Emphasis- | MarkdownInlineKind::Code- );- let trailing_space = preserve_spacing && text.ends_with(char::is_whitespace);- if preserve_spacing && text.starts_with(char::is_whitespace) {- push_markdown_text(out, " ");- }- let text = collapse_whitespace(text);- let text = text.as_str();- match kind {- MarkdownInlineKind::Paragraph => {- if !text.is_empty() {- out.push('\n');- out.push_str(text);- out.push_str("\n\n");- }- }- MarkdownInlineKind::Anchor { href } => {- if href.is_empty() {- out.push_str(text);- } else {- let label = if text.is_empty() { href.as_str() } else { text };- out.push('[');- out.push_str(label);- out.push_str("](");- out.push_str(&href);- out.push(')');- }- }- MarkdownInlineKind::ListItem { list_depth } => {- if !text.is_empty() {- out.push_str(&" ".repeat(list_depth.saturating_sub(1)));- out.push_str("- ");- out.push_str(text);- out.push('\n');- }- }- MarkdownInlineKind::Strong => {- if !text.is_empty() {- out.push_str("**");- out.push_str(text);- out.push_str("**");- }- }- MarkdownInlineKind::Emphasis => {- if !text.is_empty() {- out.push('*');- out.push_str(text);- out.push('*');- }- }- MarkdownInlineKind::Code => {- if !text.is_empty() {- out.push('`');- out.push_str(text);- out.push('`');- }- }- }- if trailing_space {- push_markdown_text(out, " ");- }-}--fn push_markdown_text(out: &mut String, text: &str) {- // Collapse whitespace across text nodes without discarding leading,- // trailing, or whitespace-only separators in the source inline flow.- for character in text.chars() {- if character.is_whitespace() {- if !out.ends_with(char::is_whitespace) {- out.push(' ');- }- } else {- out.push(character);- }- }-}--fn normalize_markdown(input: &str) -> String {- let mut out = String::new();- let mut previous_blank = false;- for line in input.lines() {- let trimmed = line.trim_end();- if trimmed.is_empty() {- if !previous_blank && !out.is_empty() {- out.push('\n');- }- previous_blank = true;- continue;- }- if !out.is_empty() {- out.push('\n');- }- out.push_str(trimmed);- previous_blank = false;- }- out.trim().to_owned()-}--fn collapse_whitespace(input: &str) -> String {- input.split_whitespace().collect::<Vec<_>>().join(" ")+ moli_html2md::Converter::new(moli_html2md::Options {+ max_depth: MAX_DOM_OUTPUT_TREE_DEPTH,+ default_code_language: Some("text".to_owned()),+ ..Default::default()+ })+ .convert(dom, root)}#[cfg(test)]@@ -653,7 +264,7 @@ mod tests {"<p>a<span> </span>b<strong> </strong>c<em> </em>d<a> </a>e</p>","a b c d e",),- ("<p>one <span>two</span></p>", "one two"),+ ("<p>one <span>two</span></p>", "one\u{00a0}two"),("<p><span>one</span><!-- comment --> <span>two</span></p>","one two",@@ -674,6 +285,10 @@ mod tests {"<p>before<a href='/docs'> docs </a>after</p>","before [docs](/docs) after",),+ (+ "<p>before<a href='/docs'><em> docs </em></a>after</p>",+ "before [*docs*](/docs) after",+ ),("<p>before<strong> bold </strong>after</p>","before **bold** after",@@ -716,12 +331,15 @@ mod tests {for (html, expected) in [("<span>Main menu</span><div>Main menu</div>",- "Main menu\nMain menu",+ "Main menu\n\nMain menu",+ ),+ (+ "<div>outer<div>inner</div>tail</div>",+ "outer\n\ninner\n\ntail",),- ("<div>outer<div>inner</div>tail</div>", "outer\ninner\ntail"),("<span>by <small>Albert Einstein</small></span><div>Tags:\n <a>change</a>\n <a>deep-thoughts</a> <a>thinking</a> <a>world</a></div>",- "by Albert Einstein\nTags: change deep-thoughts thinking world",+ "by Albert Einstein\n\nTags: change deep-thoughts thinking world",),] {assert_eq!(markdown_from_html(html), expected, "HTML: {html}");@@ -755,19 +373,196 @@ mod tests {}#[test]- fn markdown_renderer_truncates_deep_tree_with_heap_stack_walk() {+ fn markdown_renderer_expands_table_cells() {+ assert_eq!(+ markdown_from_html(+ "<table><tr><th>Name</th><th>Count</th></tr><tr><td>moli</td><td>2</td></tr></table>"+ ),+ "Name\n\nCount\n\nmoli\n\n2"+ );+ }++ #[test]+ fn markdown_renderer_flattens_hacker_news_layout_tables() {+ let dom = HtmlParser::SCRIPTING_DISABLED.parse(+ test_url(),+ include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.html").to_owned(),+ );+ let before = dom.serialize_document();+ assert_eq!(+ render_markdown_document(&dom),+ include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md").trim_end()+ );+ assert_eq!(dom.serialize_document(), before);+ }++ #[test]+ fn markdown_renderer_preserves_ordered_and_nested_lists() {+ assert_eq!(+ markdown_from_html(+ "<ol start='3'><li>three<ul><li>nested</li></ul></li><li>four</li></ol>"+ ),+ "3. three\n - nested\n4. four"+ );+ }++ #[test]+ fn markdown_renderer_preserves_blockquotes_and_hard_breaks() {+ assert_eq!(+ markdown_from_html("<blockquote><p>first<br>second</p></blockquote>"),+ "> first \n> second"+ );+ }++ #[test]+ fn markdown_renderer_preserves_code_language_fences_and_blank_lines() {+ assert_eq!(+ markdown_from_html(+ "<pre><code class='language-rust'>let s = \"```\";\n\n\nend\n</code></pre>"+ ),+ "````rust\nlet s = \"```\";\n\n\nend\n````"+ );+ assert_eq!(+ markdown_from_html("<p>before<code> a`b </code>after</p>"),+ "before ``a`b`` after"+ );+ }++ #[test]+ fn markdown_renderer_skips_non_content_tags() {+ assert_eq!(+ markdown_from_html(+ "<head><title>title</title></head><body><script>script</script><style>style</style><noscript>noscript</noscript><p>Visible</p></body>"+ ),+ "Visible"+ );+ }++ #[test]+ fn markdown_renderer_keeps_boundaries_around_empty_blocks() {+ for tag in ["p", "div", "h2", "blockquote", "ul", "ol", "pre", "table"] {+ for content in ["", " \n\t "] {+ let html = format!("before<{tag}>{content}</{tag}>after");+ assert_eq!(markdown_from_html(&html), "before\n\nafter", "{html}");+ }+ }+ }++ #[test]+ fn markdown_renderer_keeps_empty_links_and_links_around_blocks() {+ for (html, expected) in [+ ("<a href='/a'></a><a href='/b'></a>", "[](/a)[](/b)"),+ ("<a href=''>label</a>", "label"),+ ("<a id='target'>label</a>", "label"),+ ("<a href='/a'><h2>heading</h2></a>", "## [heading](/a)"),+ (+ "<a href='/a'><blockquote>quote</blockquote></a>",+ "> [quote](/a)",+ ),+ ("<a href='/a'><ul><li>item</li></ul></a>", "- [item](/a)"),+ ("<a href='/a'><img src='/i'></a>", "[](/a)"),+ ] {+ assert_eq!(markdown_from_html(html), expected, "{html}");+ }+ }++ #[test]+ fn markdown_renderer_keeps_paragraphs_in_list_items() {+ for (html, expected) in [+ ("<ul><li><p>one</p></li><li>two</li></ul>", "- one\n\n- two"),+ ("<ul><li>one</li><li><p>two</p></li></ul>", "- one\n\n- two"),+ ("<ol start='3'><li></li><li>four</li></ol>", "4. four"),+ (+ "<ul><li>parent<ul><li>child</li></ul>tail</li></ul>",+ "- parent\n - child\n\n tail",+ ),+ ] {+ assert_eq!(markdown_from_html(html), expected, "{html}");+ }+ }++ #[test]+ fn markdown_renderer_normalizes_image_and_link_attribute_newlines() {+ for (html, expected) in [+ (+ "<a href='/a' title='one\n two'>link</a>",+ "[link](/a \"one\ntwo\")",+ ),+ (+ "<img src='/i' alt='one\n two' title='one\n two'>",+ "",+ ),+ (+ "<h2><a href='/a' title='one\n two'>link</a></h2>",+ "## [link](/a \"one two\")",+ ),+ ("a<img alt='label'>b<img src='' alt='label'>c", "abc"),+ ] {+ assert_eq!(markdown_from_html(html), expected, "{html}");+ }+ }++ #[test]+ fn markdown_renderer_preserves_literal_entities_and_split_list_markers() {+ assert_eq!(+ markdown_from_html("<p>&copy; <b>literal</b></p>"),+ "&copy; \\<b\\>literal\\</b\\>"+ );let mut dom = NativeDom::new_html(test_url());let body = dom.create_element("body");assert!(dom.append_child(dom.document_node_id(), body));-- let mut parent = body;- for _ in 0..(MAX_DOM_OUTPUT_TREE_DEPTH + 32) {- let child = dom.create_element("div");- assert!(dom.append_child(parent, child));- parent = child;+ for chunk in ["12", ".", " ", "item"] {+ append_text(&mut dom, body, chunk);}- append_text(&mut dom, parent, "too deep");+ assert_eq!(render_markdown_document(&dom), "12\\. item");+ }++ #[test]+ fn markdown_converter_supports_preformatted_inline_code_on_native_dom() {+ let dom = HtmlParser::SCRIPTING_DISABLED.parse(+ test_url(),+ "<body>before<code> a b </code>after</body>".to_owned(),+ );+ let before = dom.serialize_document();+ let root = dom.body_node_id().expect("body");+ let converter = moli_html2md::Converter::new(moli_html2md::Options {+ preformatted_code: true,+ ..Default::default()+ });+ assert_eq!(converter.convert(&dom, root), "before` a b `after");+ assert_eq!(render_markdown_document(&dom), "before `a b` after");+ assert_eq!(dom.serialize_document(), before);+ }++ #[test]+ fn markdown_renderer_reads_dom_without_mutating_nodes() {+ let dom = HtmlParser::SCRIPTING_DISABLED.parse(+ test_url(),+ "<body><p><em>foo</em><i>bar</i></p><pre> a b\n</pre></body>".to_owned(),+ );+ let before = dom.serialize_document();+ assert_eq!(+ render_markdown_document(&dom),+ "*foobar*\n\n```text\n a b\n```"+ );+ assert_eq!(dom.serialize_document(), before);+ }- assert_eq!(render_markdown_document(&dom), "");+ #[test]+ fn markdown_renderer_truncates_deep_tree() {+ for tag in ["div", "span", "strong", "blockquote"] {+ let mut dom = NativeDom::new_html(test_url());+ let body = dom.create_element("body");+ assert!(dom.append_child(dom.document_node_id(), body));+ append_text(&mut dom, body, "Visible");+ let mut parent = body;+ for _ in 0..(MAX_DOM_OUTPUT_TREE_DEPTH + 32) {+ let child = dom.create_element(tag);+ assert!(dom.append_child(parent, child));+ parent = child;+ }+ append_text(&mut dom, parent, "too deep");+ assert_eq!(render_markdown_document(&dom), "Visible", "tag: {tag}");+ }}}diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex 104ded56a..f35d96720 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -49,6 +49,8 @@ mod disable_css_boundaries;mod disable_js;#[path = "fetch_cli/eval.rs"]mod eval;+#[path = "fetch_cli/markdown.rs"]+mod markdown;#[path = "fetch_cli/readiness.rs"]mod readiness;#[path = "fetch_cli/redirect_chain.rs"]diff --git a/moli/tests/fetch_cli/markdown.rs b/moli/tests/fetch_cli/markdown.rsnew file mode 100644index 000000000..3eadba80a--- /dev/null+++ b/moli/tests/fetch_cli/markdown.rs@@ -0,0 +1,40 @@+use anyhow::Result;+use base64::{Engine as _, engine::general_purpose::STANDARD};+use std::process::Command;++const HTML: &str = include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.html");+const MARKDOWN: &str = include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md");++fn assert_layout_table_dump(args: &[&str]) -> Result<()> {+ let url = format!("data:text/html;base64,{}", STANDARD.encode(HTML));+ let output = Command::new(env!("CARGO_BIN_EXE_moli"))+ .args([+ "fetch",+ "--dump",+ "markdown",+ "--wait-until",+ "load",+ "--timeout",+ "10000",+ ])+ .args(args)+ .arg(url)+ .output()?;+ assert!(+ output.status.success(),+ "{}",+ String::from_utf8_lossy(&output.stderr)+ );+ assert_eq!(String::from_utf8(output.stdout)?, MARKDOWN.trim_end());+ Ok(())+}++#[test]+fn hacker_news_layout_tables_dump_markdown() -> Result<()> {+ assert_layout_table_dump(&[])+}++#[test]+fn hacker_news_layout_tables_dump_markdown_with_layout() -> Result<()> {+ assert_layout_table_dump(&["--layout"])+}