Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
Preserve content semantics in HTML to Markdown conversion
kevinc496e37cb448
diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rsindex dbbcf43d7..d34b687e4 100644--- a/moli-html2md/src/machine.rs+++ b/moli-html2md/src/machine.rs@@ -234,9 +234,9 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {.attribute(node, "src").filter(|src| !src.is_empty()){- // Lazy-image placeholders carry no visible image. Keep+ // Embedded placeholders carry no useful image. Keep// accessible alternate text without emitting a data URI.- if is_empty_svg_placeholder(src) {+ if is_noncontent_data_image(src) {if !alt.is_empty()&& !self.dom@@ -441,38 +441,52 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {}}-fn is_empty_svg_placeholder(src: &str) -> bool {+fn is_noncontent_data_image(src: &str) -> bool {use base64::Engine as _;let Some((media_type, payload)) = src- .strip_prefix("data:")+ .get(5..)+ .filter(|_| {+ src.get(..5)+ .is_some_and(|scheme| scheme.eq_ignore_ascii_case("data:"))+ }).and_then(|uri| uri.split_once(','))else {return false;};let mut parts = media_type.split(';');- if !parts- .next()- .is_some_and(|kind| kind.eq_ignore_ascii_case("image/svg+xml"))+ let Some(kind) = parts.next() else {+ return false;+ };+ if !["image/svg+xml", "image/gif", "image/png"]+ .iter()+ .any(|supported| kind.eq_ignore_ascii_case(supported))+ || payload.len() > 1_000_000{return false;}let encoded = parts.any(|part| part.eq_ignore_ascii_case("base64"));- let svg = if encoded {+ let bytes = if encoded {let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(payload) else {return false;};- let Ok(text) = String::from_utf8(bytes) else {- return false;- };- text+ bytes} else {- let Ok(text) = percent_encoding::percent_decode_str(payload).decode_utf8() else {- return false;- };- text.into_owned()+ percent_encoding::percent_decode_str(payload).collect::<Vec<_>>()+ };+ if kind.eq_ignore_ascii_case("image/gif") {+ return (bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a"))+ && bytes.get(6..10) == Some(&[1, 0, 1, 0][..]);+ }+ if kind.eq_ignore_ascii_case("image/png") {+ return bytes.starts_with(b"\x89PNG\r\n\x1a\n")+ && bytes.get(12..16) == Some(&b"IHDR"[..])+ && bytes.get(16..24) == Some(&[0, 0, 0, 1, 0, 0, 0, 1][..]);+ }+ let Ok(svg) = std::str::from_utf8(&bytes) else {+ return false;};- let Ok(document) = roxmltree::Document::parse(&svg) else {+ let Ok(document) = roxmltree::Document::parse(svg) else {return false;};let root = document.root_element();diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rsindex e2866b906..01b87e78a 100644--- a/moli-html2md/src/writer.rs+++ b/moli-html2md/src/writer.rs@@ -171,6 +171,14 @@ impl<'a> Writer<'a> {}pub(crate) fn code_with_edges(&mut self, text: &str, preformatted: bool) {+ // Each call represents a separate HTML code element. Keep adjacent+ // elements distinct even when the source has no whitespace between them.+ if self.code.is_some() {+ self.flush_code();+ if !text.is_empty() {+ self.space = true;+ }+ }if preformatted {if !text.is_empty() {if self.code.is_none() {diff --git a/moli-html2md/tests/conversion.rs b/moli-html2md/tests/conversion.rsindex 7dc4cfcd9..46b855872 100644--- a/moli-html2md/tests/conversion.rs+++ b/moli-html2md/tests/conversion.rs@@ -147,14 +147,14 @@ fn escapes_literal_markdown_and_decoded_entities_across_text_nodes() {}#[test]-fn combines_adjacent_code_and_chooses_safe_delimiters() {+fn keeps_adjacent_code_elements_distinct_and_chooses_safe_delimiters() {let mut dom = Tree::new();dom.leaf(0, "code", "a`");dom.leaf(0, "code", "`b");dom.text(0, " ");dom.leaf(0, "code", " ` ");dom.text(0, "after");- assert_eq!(convert(&dom, 0), "```a``b``` `` ` `` after");+ assert_eq!(convert(&dom, 0), "`` a` `` `` `b `` `` ` `` after");}#[test]diff --git a/moli-html2md/tests/inline_boundaries.rs b/moli-html2md/tests/inline_boundaries.rsindex fd5b475ff..a1a3f11ed 100644--- a/moli-html2md/tests/inline_boundaries.rs+++ b/moli-html2md/tests/inline_boundaries.rs@@ -76,7 +76,7 @@ fn inline_code_breaks_keep_separation_at_code_span_edges() {..Options::default()},);- assert_eq!(result, "before`git status`after");+ assert_eq!(result, "before`git ` `status`after");}#[test]diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rsindex 1e9d5e1c7..7b547f627 100644--- a/moli-html2md/tests/regressions.rs+++ b/moli-html2md/tests/regressions.rs@@ -144,6 +144,20 @@ fn normal_code_moves_unicode_boundary_spaces_outside_delimiters() {}}+#[test]+fn adjacent_code_elements_keep_distinct_values() {+ let html = "<div><code>name</code><code>string</code></div>";+ assert_eq!(markdown(html, false), "`name` `string`");+ assert_eq!(+ rendered_html(&markdown(html, false)),+ "<p><code>name</code> <code>string</code></p>\n"+ );+ assert_eq!(+ markdown("<code>first</code>suffix<code>second</code>", false),+ "`first`suffix`second`"+ );+}+#[test]fn attribute_newlines_remove_indentation_without_joining_words() {for separator in ["\n ", "\r\n\t", "\n \n \t "] {@@ -252,6 +266,33 @@ fn empty_svg_lazy_image_placeholders_do_not_emit_data_uris() {assert!(markdown(&html, false).contains(text_svg));}+#[test]+fn single_pixel_embedded_raster_placeholders_do_not_emit_data_uris() {+ let gifs = [+ "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==",+ "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==",+ ];+ for src in gifs {+ let html = format!("before<img alt='Product photo' src=\"{src}\">after");+ assert_eq!(markdown(&html, false), "beforeProduct photoafter");+ }+ let png = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z8DwHwAFgAI/ScL/nwAAAABJRU5ErkJggg==";+ let html = format!("before<img aria-hidden='true' src=\"{png}\">after");+ assert_eq!(markdown(&html, false), "beforeafter");++ use base64::Engine as _;+ let mut wide = base64::engine::general_purpose::STANDARD+ .decode(gifs[0].split_once(',').unwrap().1)+ .unwrap();+ wide[6] = 2;+ let wide = format!(+ "data:image/gif;base64,{}",+ base64::engine::general_purpose::STANDARD.encode(wide)+ );+ let html = format!("<img alt='Wide image' src=\"{wide}\">");+ assert!(markdown(&html, false).contains(&wide));+}+#[test]fn list_paragraphs_and_code_blocks_require_blank_lines_between_items() {for (html, expected) in [diff --git a/moli-html2md/tests/turndown/expectations.json b/moli-html2md/tests/turndown/expectations.jsonindex f9b20051c..15a45d349 100644--- a/moli-html2md/tests/turndown/expectations.json+++ b/moli-html2md/tests/turndown/expectations.json@@ -4,8 +4,8 @@"reason": "Moli fences bare pre elements as well as pre/code, preserving preformatted text. Turndown only recognizes a pre/code pair."},"extra_adjacent_code": {- "markdown": "`ab`",- "reason": "Adjacent code elements contain ab. Turndown joins their delimiters into one code span whose text contains two extra backticks."+ "markdown": "`a` `b`",+ "reason": "Separate HTML code elements remain separate Markdown code spans, preserving their semantic boundary. Turndown joins their delimiters into one code span whose text contains two extra backticks."},"extra_code_nested_format": {"markdown": "`ab`",