Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

Preserve content semantics in HTML to Markdown conversion

kevinc496e37cb448
diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs
index dbbcf43d7..d34b687e4 100644
--- a/moli-html2md/src/machine.rs
+++ b/moli-html2md/src/machine.rs
@@ -234,9 +234,9 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
.attribute(node, "src")
.filter(|src| !src.is_empty())
{
- // Lazy-image placeholders carry no visible image. Keep
+ // Embedded placeholders carry no useful image. Keep
// accessible alternate text without emitting a data URI.
- if is_empty_svg_placeholder(src) {
+ if is_noncontent_data_image(src) {
if !alt.is_empty()
&& !self
.dom
@@ -441,38 +441,52 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
}
}
-fn is_empty_svg_placeholder(src: &str) -> bool {
+fn is_noncontent_data_image(src: &str) -> bool {
use base64::Engine as _;
let Some((media_type, payload)) = src
- .strip_prefix("data:")
+ .get(5..)
+ .filter(|_| {
+ src.get(..5)
+ .is_some_and(|scheme| scheme.eq_ignore_ascii_case("data:"))
+ })
.and_then(|uri| uri.split_once(','))
else {
return false;
};
let mut parts = media_type.split(';');
- if !parts
- .next()
- .is_some_and(|kind| kind.eq_ignore_ascii_case("image/svg+xml"))
+ let Some(kind) = parts.next() else {
+ return false;
+ };
+ if !["image/svg+xml", "image/gif", "image/png"]
+ .iter()
+ .any(|supported| kind.eq_ignore_ascii_case(supported))
+ || payload.len() > 1_000_000
{
return false;
}
let encoded = parts.any(|part| part.eq_ignore_ascii_case("base64"));
- let svg = if encoded {
+ let bytes = if encoded {
let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(payload) else {
return false;
};
- let Ok(text) = String::from_utf8(bytes) else {
- return false;
- };
- text
+ bytes
} else {
- let Ok(text) = percent_encoding::percent_decode_str(payload).decode_utf8() else {
- return false;
- };
- text.into_owned()
+ percent_encoding::percent_decode_str(payload).collect::<Vec<_>>()
+ };
+ if kind.eq_ignore_ascii_case("image/gif") {
+ return (bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a"))
+ && bytes.get(6..10) == Some(&[1, 0, 1, 0][..]);
+ }
+ if kind.eq_ignore_ascii_case("image/png") {
+ return bytes.starts_with(b"\x89PNG\r\n\x1a\n")
+ && bytes.get(12..16) == Some(&b"IHDR"[..])
+ && bytes.get(16..24) == Some(&[0, 0, 0, 1, 0, 0, 0, 1][..]);
+ }
+ let Ok(svg) = std::str::from_utf8(&bytes) else {
+ return false;
};
- let Ok(document) = roxmltree::Document::parse(&svg) else {
+ let Ok(document) = roxmltree::Document::parse(svg) else {
return false;
};
let root = document.root_element();
diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rs
index e2866b906..01b87e78a 100644
--- a/moli-html2md/src/writer.rs
+++ b/moli-html2md/src/writer.rs
@@ -171,6 +171,14 @@ impl<'a> Writer<'a> {
}
pub(crate) fn code_with_edges(&mut self, text: &str, preformatted: bool) {
+ // Each call represents a separate HTML code element. Keep adjacent
+ // elements distinct even when the source has no whitespace between them.
+ if self.code.is_some() {
+ self.flush_code();
+ if !text.is_empty() {
+ self.space = true;
+ }
+ }
if preformatted {
if !text.is_empty() {
if self.code.is_none() {
diff --git a/moli-html2md/tests/conversion.rs b/moli-html2md/tests/conversion.rs
index 7dc4cfcd9..46b855872 100644
--- a/moli-html2md/tests/conversion.rs
+++ b/moli-html2md/tests/conversion.rs
@@ -147,14 +147,14 @@ fn escapes_literal_markdown_and_decoded_entities_across_text_nodes() {
}
#[test]
-fn combines_adjacent_code_and_chooses_safe_delimiters() {
+fn keeps_adjacent_code_elements_distinct_and_chooses_safe_delimiters() {
let mut dom = Tree::new();
dom.leaf(0, "code", "a`");
dom.leaf(0, "code", "`b");
dom.text(0, " ");
dom.leaf(0, "code", " ` ");
dom.text(0, "after");
- assert_eq!(convert(&dom, 0), "```a``b``` `` ` `` after");
+ assert_eq!(convert(&dom, 0), "`` a` `` `` `b `` `` ` `` after");
}
#[test]
diff --git a/moli-html2md/tests/inline_boundaries.rs b/moli-html2md/tests/inline_boundaries.rs
index fd5b475ff..a1a3f11ed 100644
--- a/moli-html2md/tests/inline_boundaries.rs
+++ b/moli-html2md/tests/inline_boundaries.rs
@@ -76,7 +76,7 @@ fn inline_code_breaks_keep_separation_at_code_span_edges() {
..Options::default()
},
);
- assert_eq!(result, "before`git status`after");
+ assert_eq!(result, "before`git ` `status`after");
}
#[test]
diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs
index 1e9d5e1c7..7b547f627 100644
--- a/moli-html2md/tests/regressions.rs
+++ b/moli-html2md/tests/regressions.rs
@@ -144,6 +144,20 @@ fn normal_code_moves_unicode_boundary_spaces_outside_delimiters() {
}
}
+#[test]
+fn adjacent_code_elements_keep_distinct_values() {
+ let html = "<div><code>name</code><code>string</code></div>";
+ assert_eq!(markdown(html, false), "`name` `string`");
+ assert_eq!(
+ rendered_html(&markdown(html, false)),
+ "<p><code>name</code> <code>string</code></p>\n"
+ );
+ assert_eq!(
+ markdown("<code>first</code>suffix<code>second</code>", false),
+ "`first`suffix`second`"
+ );
+}
+
#[test]
fn attribute_newlines_remove_indentation_without_joining_words() {
for separator in ["\n ", "\r\n\t", "\n \n \t "] {
@@ -252,6 +266,33 @@ fn empty_svg_lazy_image_placeholders_do_not_emit_data_uris() {
assert!(markdown(&html, false).contains(text_svg));
}
+#[test]
+fn single_pixel_embedded_raster_placeholders_do_not_emit_data_uris() {
+ let gifs = [
+ "data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==",
+ "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==",
+ ];
+ for src in gifs {
+ let html = format!("before<img alt='Product photo' src=\"{src}\">after");
+ assert_eq!(markdown(&html, false), "beforeProduct photoafter");
+ }
+ let png = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z8DwHwAFgAI/ScL/nwAAAABJRU5ErkJggg==";
+ let html = format!("before<img aria-hidden='true' src=\"{png}\">after");
+ assert_eq!(markdown(&html, false), "beforeafter");
+
+ use base64::Engine as _;
+ let mut wide = base64::engine::general_purpose::STANDARD
+ .decode(gifs[0].split_once(',').unwrap().1)
+ .unwrap();
+ wide[6] = 2;
+ let wide = format!(
+ "data:image/gif;base64,{}",
+ base64::engine::general_purpose::STANDARD.encode(wide)
+ );
+ let html = format!("<img alt='Wide image' src=\"{wide}\">");
+ assert!(markdown(&html, false).contains(&wide));
+}
+
#[test]
fn list_paragraphs_and_code_blocks_require_blank_lines_between_items() {
for (html, expected) in [
diff --git a/moli-html2md/tests/turndown/expectations.json b/moli-html2md/tests/turndown/expectations.json
index f9b20051c..15a45d349 100644
--- a/moli-html2md/tests/turndown/expectations.json
+++ b/moli-html2md/tests/turndown/expectations.json
@@ -4,8 +4,8 @@
"reason": "Moli fences bare pre elements as well as pre/code, preserving preformatted text. Turndown only recognizes a pre/code pair."
},
"extra_adjacent_code": {
- "markdown": "`ab`",
- "reason": "Adjacent code elements contain ab. Turndown joins their delimiters into one code span whose text contains two extra backticks."
+ "markdown": "`a` `b`",
+ "reason": "Separate HTML code elements remain separate Markdown code spans, preserving their semantic boundary. Turndown joins their delimiters into one code span whose text contains two extra backticks."
},
"extra_code_nested_format": {
"markdown": "`ab`",