Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

Use CommonMark punctuation class and retain compatibility scenarios

kevin3560c85f4ba3
diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rs
index 6684a58f5..325da4bf4 100644
--- a/moli-html2md/src/writer.rs
+++ b/moli-html2md/src/writer.rs
@@ -465,7 +465,10 @@ impl<'a> Writer<'a> {
'&' => self.output.push_str("&amp;"),
'<' => self.output.push_str("&lt;"),
'>' => self.output.push_str("&gt;"),
- '\\' | '`' | '*' | '_' | '[' | ']' | '|' | '~' => {
+ // CommonMark permits escaping every ASCII punctuation
+ // character. Inline HTML content still parses Markdown,
+ // so protect the whole syntax class rather than a subset.
+ ch if ch.is_ascii_punctuation() => {
self.output.push('\\');
self.output.push(ch);
}
diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs
index 9b6ce3e8d..8fbf1a057 100644
--- a/moli-html2md/tests/regressions.rs
+++ b/moli-html2md/tests/regressions.rs
@@ -212,6 +212,48 @@ fn preformatted_block_children_keep_text_boundaries() {
);
}
+#[test]
+fn adjacent_code_preserves_documentation_examples_as_literal_text() {
+ let code_values = |html: &str| {
+ let tree = Tree::parse(html);
+ tree.nodes
+ .iter()
+ .filter(|node| node.tag.as_deref() == Some("code"))
+ .map(|node| {
+ let mut text = String::new();
+ let mut child = node.first;
+ while let Some(id) = child {
+ let node = &tree.nodes[id];
+ assert!(node.tag.is_none(), "code gained formatting: {html}");
+ text.push_str(node.text.as_deref().unwrap_or_default());
+ child = node.next;
+ }
+ text
+ })
+ .collect::<Vec<_>>()
+ };
+ for value in [
+ r"curl https://example.com/a?x=1&amp;y=2",
+ r"![icon](image.png) **bold** _name_ ~~old~~",
+ r"&lt;T&gt; &amp;amp; &#96;value&#96; C:\work\file",
+ r"!&quot;#$%&amp;'()*+,-./:;&lt;=&gt;?@[\]^_&#96;{|}~",
+ "变量—café 🙂",
+ ] {
+ // The HTML source is the oracle: Markdown conversion must preserve
+ // each code value, including syntax, entities and Unicode characters.
+ let source = format!("<code>{value}</code><code>next</code>");
+ let expected = code_values(&source);
+ for preformatted in [false, true] {
+ let output = markdown(&source, preformatted);
+ assert_eq!(
+ code_values(&rendered_html(&output)),
+ expected,
+ "{source}: {output}"
+ );
+ }
+ }
+}
+
#[test]
fn attribute_newlines_remove_indentation_without_joining_words() {
for separator in ["\n ", "\r\n\t", "\n \n \t "] {
diff --git a/moli-protocol/src/domains/dom/tests/geometry.rs b/moli-protocol/src/domains/dom/tests/geometry.rs
index 732b19517..072322f83 100644
--- a/moli-protocol/src/domains/dom/tests/geometry.rs
+++ b/moli-protocol/src/domains/dom/tests/geometry.rs
@@ -1388,9 +1388,10 @@ async fn scroll_into_view_if_needed_accepts_element_node() {
let mut ctx = TestContext::new();
load_bc(&mut ctx, "BID-A");
- let spacer = "<div style='height:200vh'></div>";
+ // Keep the text-row scenario while making its height independent of fonts.
+ let rows = "<div style='height:4vh'>row</div>".repeat(60);
let url = format!(
- "data:text/html,<!doctype html><html><body>{spacer}<div id='box'></div></body></html>"
+ "data:text/html,<!doctype html><html><body>{rows}<div id='box'></div></body></html>"
);
navigate_to_url_and_wait_for_load_async(&mut ctx, 1, url).await;
@@ -1542,9 +1543,10 @@ async fn scroll_into_view_if_needed_supports_object_id() {
let mut ctx = TestContext::new();
load_bc(&mut ctx, "BID-A");
- let spacer = "<div style='height:200vh'></div>";
+ // Keep the text-row scenario while making its height independent of fonts.
+ let rows = "<div style='height:4vh'>row</div>".repeat(60);
let url = format!(
- "data:text/html,<!doctype html><html><body>{spacer}<div id='box'></div></body></html>"
+ "data:text/html,<!doctype html><html><body>{rows}<div id='box'></div></body></html>"
);
navigate_to_url_and_wait_for_load_async(&mut ctx, 1, url).await;
@@ -1591,9 +1593,10 @@ async fn scroll_into_view_if_needed_supports_object_id() {
async fn scroll_into_view_if_needed_uses_first_rendered_child_of_display_contents() {
let mut ctx = TestContext::new();
load_bc(&mut ctx, "BID-A");
- let spacer = "<div style='height:200vh'></div>";
+ // Keep the text-row scenario while making its height independent of fonts.
+ let rows = "<div style='height:4vh'>row</div>".repeat(60);
let url = format!(
- "data:text/html,<!doctype html><body>{spacer}<button id=contents style='display:contents'>target</button></body>"
+ "data:text/html,<!doctype html><body>{rows}<button id=contents style='display:contents'>target</button></body>"
);
navigate_to_url_and_wait_for_load_async(&mut ctx, 30, url).await;
diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rs
index 84e3229a2..a1599c989 100644
--- a/moli/src/fetch_dump/tests.rs
+++ b/moli/src/fetch_dump/tests.rs
@@ -132,7 +132,7 @@ async fn render_page_dump_default_html_uses_renderer_live_serialize() -> Result<
#[tokio::test]
async fn strip_ui_keeps_content_inside_details() -> Result<()> {
let (_browser, mut page, http_server) = load_page(
- "<!doctype html><html><body><nav>Menu</nav><main><details><summary>Product details</summary><p>Critical product description</p></details></main></body></html>",
+ "<!doctype html><html><body><nav><details><summary>Menu</summary><p>Navigation entry</p></details></nav><main><details><summary>Product details</summary><p>Critical product description</p><button>Buy now</button></details><details open><summary>Specifications</summary><p>Material: cotton</p></details></main></body></html>",
)
.await?;
let rendered = render_page_dump_with_options_async(
@@ -150,8 +150,12 @@ async fn strip_ui_keeps_content_inside_details() -> Result<()> {
)
.await?;
assert!(!rendered.contains("Menu"));
+ assert!(!rendered.contains("Navigation entry"));
+ assert!(!rendered.contains("Buy now"));
assert!(rendered.contains("Product details"));
assert!(rendered.contains("Critical product description"));
+ assert!(rendered.contains("Specifications"));
+ assert!(rendered.contains("Material: cotton"));
http_server.abort();
Ok(())
}