Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
Use CommonMark punctuation class and retain compatibility scenarios
kevin3560c85f4ba3
diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rsindex 6684a58f5..325da4bf4 100644--- a/moli-html2md/src/writer.rs+++ b/moli-html2md/src/writer.rs@@ -465,7 +465,10 @@ impl<'a> Writer<'a> {'&' => self.output.push_str("&"),'<' => self.output.push_str("<"),'>' => self.output.push_str(">"),- '\\' | '`' | '*' | '_' | '[' | ']' | '|' | '~' => {+ // CommonMark permits escaping every ASCII punctuation+ // character. Inline HTML content still parses Markdown,+ // so protect the whole syntax class rather than a subset.+ ch if ch.is_ascii_punctuation() => {self.output.push('\\');self.output.push(ch);}diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rsindex 9b6ce3e8d..8fbf1a057 100644--- a/moli-html2md/tests/regressions.rs+++ b/moli-html2md/tests/regressions.rs@@ -212,6 +212,48 @@ fn preformatted_block_children_keep_text_boundaries() {);}+#[test]+fn adjacent_code_preserves_documentation_examples_as_literal_text() {+ let code_values = |html: &str| {+ let tree = Tree::parse(html);+ tree.nodes+ .iter()+ .filter(|node| node.tag.as_deref() == Some("code"))+ .map(|node| {+ let mut text = String::new();+ let mut child = node.first;+ while let Some(id) = child {+ let node = &tree.nodes[id];+ assert!(node.tag.is_none(), "code gained formatting: {html}");+ text.push_str(node.text.as_deref().unwrap_or_default());+ child = node.next;+ }+ text+ })+ .collect::<Vec<_>>()+ };+ for value in [+ r"curl https://example.com/a?x=1&y=2",+ r" **bold** _name_ ~~old~~",+ r"<T> &amp; `value` C:\work\file",+ r"!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~",+ "变量—café 🙂",+ ] {+ // The HTML source is the oracle: Markdown conversion must preserve+ // each code value, including syntax, entities and Unicode characters.+ let source = format!("<code>{value}</code><code>next</code>");+ let expected = code_values(&source);+ for preformatted in [false, true] {+ let output = markdown(&source, preformatted);+ assert_eq!(+ code_values(&rendered_html(&output)),+ expected,+ "{source}: {output}"+ );+ }+ }+}+#[test]fn attribute_newlines_remove_indentation_without_joining_words() {for separator in ["\n ", "\r\n\t", "\n \n \t "] {diff --git a/moli-protocol/src/domains/dom/tests/geometry.rs b/moli-protocol/src/domains/dom/tests/geometry.rsindex 732b19517..072322f83 100644--- a/moli-protocol/src/domains/dom/tests/geometry.rs+++ b/moli-protocol/src/domains/dom/tests/geometry.rs@@ -1388,9 +1388,10 @@ async fn scroll_into_view_if_needed_accepts_element_node() {let mut ctx = TestContext::new();load_bc(&mut ctx, "BID-A");- let spacer = "<div style='height:200vh'></div>";+ // Keep the text-row scenario while making its height independent of fonts.+ let rows = "<div style='height:4vh'>row</div>".repeat(60);let url = format!(- "data:text/html,<!doctype html><html><body>{spacer}<div id='box'></div></body></html>"+ "data:text/html,<!doctype html><html><body>{rows}<div id='box'></div></body></html>");navigate_to_url_and_wait_for_load_async(&mut ctx, 1, url).await;@@ -1542,9 +1543,10 @@ async fn scroll_into_view_if_needed_supports_object_id() {let mut ctx = TestContext::new();load_bc(&mut ctx, "BID-A");- let spacer = "<div style='height:200vh'></div>";+ // Keep the text-row scenario while making its height independent of fonts.+ let rows = "<div style='height:4vh'>row</div>".repeat(60);let url = format!(- "data:text/html,<!doctype html><html><body>{spacer}<div id='box'></div></body></html>"+ "data:text/html,<!doctype html><html><body>{rows}<div id='box'></div></body></html>");navigate_to_url_and_wait_for_load_async(&mut ctx, 1, url).await;@@ -1591,9 +1593,10 @@ async fn scroll_into_view_if_needed_supports_object_id() {async fn scroll_into_view_if_needed_uses_first_rendered_child_of_display_contents() {let mut ctx = TestContext::new();load_bc(&mut ctx, "BID-A");- let spacer = "<div style='height:200vh'></div>";+ // Keep the text-row scenario while making its height independent of fonts.+ let rows = "<div style='height:4vh'>row</div>".repeat(60);let url = format!(- "data:text/html,<!doctype html><body>{spacer}<button id=contents style='display:contents'>target</button></body>"+ "data:text/html,<!doctype html><body>{rows}<button id=contents style='display:contents'>target</button></body>");navigate_to_url_and_wait_for_load_async(&mut ctx, 30, url).await;diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rsindex 84e3229a2..a1599c989 100644--- a/moli/src/fetch_dump/tests.rs+++ b/moli/src/fetch_dump/tests.rs@@ -132,7 +132,7 @@ async fn render_page_dump_default_html_uses_renderer_live_serialize() -> Result<#[tokio::test]async fn strip_ui_keeps_content_inside_details() -> Result<()> {let (_browser, mut page, http_server) = load_page(- "<!doctype html><html><body><nav>Menu</nav><main><details><summary>Product details</summary><p>Critical product description</p></details></main></body></html>",+ "<!doctype html><html><body><nav><details><summary>Menu</summary><p>Navigation entry</p></details></nav><main><details><summary>Product details</summary><p>Critical product description</p><button>Buy now</button></details><details open><summary>Specifications</summary><p>Material: cotton</p></details></main></body></html>",).await?;let rendered = render_page_dump_with_options_async(@@ -150,8 +150,12 @@ async fn strip_ui_keeps_content_inside_details() -> Result<()> {).await?;assert!(!rendered.contains("Menu"));+ assert!(!rendered.contains("Navigation entry"));+ assert!(!rendered.contains("Buy now"));assert!(rendered.contains("Product details"));assert!(rendered.contains("Critical product description"));+ assert!(rendered.contains("Specifications"));+ assert!(rendered.contains("Material: cotton"));http_server.abort();Ok(())}