Craft Anything: understand how any software works, and craft your own.
Rust
Read HTML pages' scripts and preloads correctly in more cases (#645)
BFlat0eea78d7aa50
diff --git a/crates/js/src/html.rs b/crates/js/src/html.rsindex 46b75396..4d7a27ca 100644--- a/crates/js/src/html.rs+++ b/crates/js/src/html.rs@@ -30,7 +30,7 @@ pub struct HtmlFacts {fn attributes(src: &str, lower: &str, mut i: usize) -> (Vec<(String, String)>, usize) {let b = lower.as_bytes();let mut out = Vec::new();- while i < b.len() && out.len() < 64 {+ while i < b.len() {while b.get(i).is_some_and(|c| c.is_ascii_whitespace() || *c == b'/') {i += 1;}@@ -75,7 +75,7 @@ fn attributes(src: &str, lower: &str, mut i: usize) -> (Vec<(String, String)>, u}if name.is_empty() {i += 1;- } else {+ } else if out.len() < 64 {out.push((name, decode_entities(&value)));}}@@ -83,13 +83,59 @@ fn attributes(src: &str, lower: &str, mut i: usize) -> (Vec<(String, String)>, u}fn decode_entities(s: &str) -> String {- s.replace("&", "&").replace(""", "\"").replace("'", "'").replace("<", "<").replace(">", ">")+ let mut out = String::with_capacity(s.len());+ let mut rest = s;+ while let Some(at) = rest.find('&') {+ out.push_str(rest.get(..at).unwrap_or_default());+ rest = rest.get(at + 1..).unwrap_or_default();+ let decoded = rest+ .strip_prefix('#')+ .and_then(|number| {+ let (digits, radix) = number.strip_prefix(['x', 'X']).map_or((number, 10), |n| (n, 16));+ let count = digits.bytes().take_while(|c| if radix == 16 { c.is_ascii_hexdigit() } else { c.is_ascii_digit() }).count();+ if count == 0 {+ return None;+ }+ let value = digits.get(..count).and_then(|s| u32::from_str_radix(s, radix).ok()).unwrap_or(u32::MAX);+ let consumed = rest.len() - digits.len() + count;+ let consumed = consumed + usize::from(rest.as_bytes().get(consumed) == Some(&b';'));+ // HTML maps the numeric C1 references through the Windows-1252 table.+ const C1: [u32; 32] = [+ 0x20ac, 0x81, 0x201a, 0x192, 0x201e, 0x2026, 0x2020, 0x2021, 0x2c6, 0x2030, 0x160, 0x2039, 0x152, 0x8d, 0x17d, 0x8f, 0x90, 0x2018, 0x2019,+ 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x2dc, 0x2122, 0x161, 0x203a, 0x153, 0x9d, 0x17e, 0x178,+ ];+ let value = value.checked_sub(0x80).and_then(|i| C1.get(i as usize)).copied().unwrap_or(value);+ Some((char::from_u32(value).filter(|c| *c != '\0').unwrap_or('\u{fffd}'), consumed))+ })+ .or_else(|| {+ [("amp;", '&'), ("quot;", '"'), ("apos;", '\''), ("lt;", '<'), ("gt;", '>')]+ .into_iter()+ .find_map(|(name, value)| rest.starts_with(name).then_some((value, name.len())))+ });+ if let Some((c, consumed)) = decoded {+ out.push(c);+ rest = rest.get(consumed..).unwrap_or_default();+ } else {+ out.push('&');+ }+ }+ out.push_str(rest);+ out}fn get<'v>(attrs: &'v [(String, String)], k: &str) -> Option<&'v str> {attrs.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str())}+/// Find an actual closing tag, not a longer name such as `</scripture>`.+fn closing_tag(lower: &str, from: usize, name: &str) -> Option<usize> {+ let needle = format!("</{name}");+ lower.get(from..)?.match_indices(&needle).find_map(|(offset, _)| {+ let at = from + offset;+ lower.as_bytes().get(at + needle.len()).filter(|c| c.is_ascii_whitespace() || matches!(c, b'/' | b'>')).map(|_| at)+ })+}+/// Scan an HTML document.pub fn scan_html(src: &str) -> HtmlFacts {let lower = src.to_ascii_lowercase();@@ -104,13 +150,16 @@ pub fn scan_html(src: &str) -> HtmlFacts {}let at = pos + rel;let rest = lower.get(at + 1..).unwrap_or("");- let name_len = rest.bytes().take_while(|c| c.is_ascii_alphanumeric() || *c == b'-').count();- let name = rest.get(..name_len).unwrap_or("");- let after = at + 1 + name_len;+ // Inspect only the names we handle: malformed runs of `<` must not rescan the remaining document.+ let name = ["script", "link", "meta", "title", "style", "textarea", "xmp", "iframe", "noembed", "noframes", "plaintext"]+ .into_iter()+ .find(|name| rest.starts_with(*name) && rest.as_bytes().get(name.len()).is_none_or(|c| c.is_ascii_whitespace() || matches!(*c, b'/' | b'>')))+ .unwrap_or_default();+ let after = at + 1 + name.len();match name {"script" => {let (attrs, end) = attributes(src, &lower, after);- let close = lower.get(end..).and_then(|r| r.find("</script")).map_or(lower.len(), |c| end + c);+ let close = closing_tag(&lower, end, "script").unwrap_or(lower.len());let module = get(&attrs, "type").is_some_and(|t| t.eq_ignore_ascii_case("module"));let line = lines.line(u32::try_from(at).unwrap_or(u32::MAX));let src_attr = get(&attrs, "src").map(String::from);@@ -122,10 +171,16 @@ pub fn scan_html(src: &str) -> HtmlFacts {}pos = close.max(end).max(at + 1);}+ "style" | "textarea" | "xmp" | "iframe" | "noembed" | "noframes" => {+ let (_, end) = attributes(src, &lower, after);+ pos = closing_tag(&lower, end, name).unwrap_or(lower.len()).max(at + 1);+ }+ "plaintext" => break,"link" => {let (attrs, end) = attributes(src, &lower, after);let rel_attr = get(&attrs, "rel").unwrap_or("").to_ascii_lowercase();- if (rel_attr == "modulepreload" || (rel_attr == "preload" && get(&attrs, "as") == Some("script")))+ let script_destination = get(&attrs, "as").is_some_and(|value| value.eq_ignore_ascii_case("script"));+ if rel_attr.split_ascii_whitespace().any(|token| token == "modulepreload" || (token == "preload" && script_destination))&& let Some(h) = get(&attrs, "href")&& out.preloads.len() < MAX_TAGS{@@ -142,13 +197,19 @@ pub fn scan_html(src: &str) -> HtmlFacts {}"title" => {let (_, end) = attributes(src, &lower, after);- let close = lower.get(end..).and_then(|r| r.find("</title")).map_or(end, |c| end + c);+ let close = closing_tag(&lower, end, "title").unwrap_or(lower.len());out.title = src.get(end..close).map(|t| t.trim().chars().take(200).collect());pos = close.max(at + 1);}_ if rest.starts_with("!--") => {pos = lower.get(at..).and_then(|r| r.find("-->")).map_or(lower.len(), |c| at + c + 3);}+ _ if rest.as_bytes().first().is_some_and(u8::is_ascii_alphabetic)+ || rest.strip_prefix('/').and_then(|s| s.as_bytes().first()).is_some_and(u8::is_ascii_alphabetic) =>+ {+ let (_, end) = attributes(src, &lower, after);+ pos = end.max(at + 1);+ }_ => pos = at + 1,}}diff --git a/crates/js/tests/regression_html_end_tags.rs b/crates/js/tests/regression_html_end_tags.rsnew file mode 100644index 00000000..0747bea2--- /dev/null+++ b/crates/js/tests/regression_html_end_tags.rs@@ -0,0 +1,18 @@+use craft_anything_js::html::scan_html;++#[test]+fn regression_closing_tag_lookalikes_do_not_end_scripts() {+ for lookalike in ["</scripture>", "</script_widget>", "</script:widget>"] {+ let code = format!("const label = '{lookalike}'; window.after = true;");+ let page = format!("<script>{code}</ScRiPt ><script src='next.js'></script>");+ let f = scan_html(&page);+ let (start, end) = f.scripts[0].inline.unwrap();+ assert_eq!(&page[start as usize..end as usize], code);+ assert_eq!(f.scripts.len(), 2);+ assert_eq!(f.scripts[1].src.as_deref(), Some("next.js"));+ }+ assert_eq!(scan_html("<title>a</titleish>b</title>").title.as_deref(), Some("a</titleish>b"));+ let unclosed = scan_html("<title>caption</titleish><script src='ghost.js'></script>");+ assert!(unclosed.scripts.is_empty());+ assert!(unclosed.title.as_deref().unwrap().starts_with("caption</titleish>"));+}diff --git a/crates/js/tests/regression_html_entities.rs b/crates/js/tests/regression_html_entities.rsnew file mode 100644index 00000000..3b79120f--- /dev/null+++ b/crates/js/tests/regression_html_entities.rs@@ -0,0 +1,15 @@+use craft_anything_js::html::scan_html;+fn decode_entities(value: &str) -> String {+ scan_html(&format!("<script src=\"{value}\"></script>")).scripts[0].src.clone().unwrap()+}++#[test]+fn regression_asset_references_decode_once() {+ let f = scan_html(r#"<script src="/app.js?a=1&b=2/3&quot;"></script><link rel="modulepreload" href="/modules/main.js">"#);+ assert_eq!(f.scripts[0].src.as_deref(), Some("/app.js?a=1&b=2/3""));+ assert_eq!(f.preloads, ["/modules/main.js"]);+ assert_eq!(decode_entities("&unknown; &#x; &#not; &amp; 🙂"), "&unknown; &#x; &#not; & 🙂");+ assert_eq!(decode_entities("� � � €"), "� � � €");+ assert_eq!(decode_entities("/app.js /modules.js"), "/app.js /modules.js");+ assert_eq!(decode_entities(&format!("&#{}47;app.js", "0".repeat(80))), "/app.js");+}diff --git a/crates/js/tests/regression_html_preloads.rs b/crates/js/tests/regression_html_preloads.rsnew file mode 100644index 00000000..230d875f--- /dev/null+++ b/crates/js/tests/regression_html_preloads.rs@@ -0,0 +1,9 @@+use craft_anything_js::html::scan_html;++#[test]+fn regression_preload_relationships_are_tokens() {+ let f = scan_html(+ r#"<link rel="alternate PRELOAD" as="SCRIPT" href="one.js"><link rel="stylesheet modulepreload" href="two.js"><link rel="notpreload" as="script" href="no.js"><link rel="preload" as="style" href="no.css"><link rel="preloadish" as="script" href="other.js">"#,+ );+ assert_eq!(f.preloads, ["one.js", "two.js"]);+}diff --git a/crates/js/tests/regression_html_quoted_tags.rs b/crates/js/tests/regression_html_quoted_tags.rsnew file mode 100644index 00000000..dbea234e--- /dev/null+++ b/crates/js/tests/regression_html_quoted_tags.rs@@ -0,0 +1,15 @@+use craft_anything_js::html::scan_html;++#[test]+fn regression_markup_in_attributes_is_not_scanned_as_tags() {+ for extra in [String::new(), (0..70).map(|i| format!(" a{i}='value'")).collect()] {+ let page = format!("<div{extra} data-example=\"<script src='ghost.js'></script>\"><script src='real.js'></script></div>");+ let f = scan_html(&page);+ assert_eq!(f.scripts.len(), 1, "{:?}", f.scripts);+ assert_eq!(f.scripts[0].src.as_deref(), Some("real.js"));+ }+ for text in ["1 < 2", "1 <2", "x < y"] {+ let page = format!("<p>{text} <script src='real.js'></script></p>");+ assert_eq!(scan_html(&page).scripts[0].src.as_deref(), Some("real.js"));+ }+}diff --git a/crates/js/tests/regression_html_raw_text.rs b/crates/js/tests/regression_html_raw_text.rsnew file mode 100644index 00000000..349d8ca9--- /dev/null+++ b/crates/js/tests/regression_html_raw_text.rs@@ -0,0 +1,12 @@+use craft_anything_js::html::scan_html;++#[test]+fn regression_text_elements_do_not_invent_script_dependencies() {+ for tag in ["style", "textarea", "xmp", "iframe", "noembed", "noframes"] {+ let page = format!("<{tag}><script src='ghost.js'></script></{tag}><script src='real.js'></script>");+ let f = scan_html(&page);+ assert_eq!(f.scripts.len(), 1, "{tag}: {:?}", f.scripts);+ assert_eq!(f.scripts[0].src.as_deref(), Some("real.js"));+ }+ assert!(scan_html("<plaintext><script src='ghost.js'></script>").scripts.is_empty());+}diff --git a/crates/js/tests/regression_html_tag_names.rs b/crates/js/tests/regression_html_tag_names.rsnew file mode 100644index 00000000..33f5346f--- /dev/null+++ b/crates/js/tests/regression_html_tag_names.rs@@ -0,0 +1,21 @@+use craft_anything_js::html::scan_html;++#[test]+fn regression_html_many_invalid_tag_starts_still_finds_scripts() {+ let html = "<".repeat(80_000) + "<script src='/after-invalid-tags.js'></script>";+ let facts = scan_html(&html);+ assert_eq!(facts.scripts.len(), 1);+ assert_eq!(facts.scripts[0].src.as_deref(), Some("/after-invalid-tags.js"));+}++#[test]+fn regression_custom_tag_names_are_not_builtin_tags() {+ let f = scan_html(+ r#"<script:widget src="ghost.js"></script:widget><script_widget src="other.js"></script_widget><link:widget rel="modulepreload" href="ghost.js"><meta:widget http-equiv="content-security-policy" content="bad"><title:widget>wrong</title:widget><script src="real.js"></script>"#,+ );+ assert_eq!(f.scripts.iter().filter_map(|s| s.src.as_deref()).collect::<Vec<_>>(), ["real.js"]);+ assert!(f.preloads.is_empty());+ assert!(f.csp.is_none());+ assert!(f.title.is_none());+ assert_eq!(scan_html("<SCRIPT src='upper.js'></SCRIPT>").scripts[0].src.as_deref(), Some("upper.js"));+}diff --git a/docs/ROADMAP.md b/docs/ROADMAP.mdindex c1cae368..c424f4fe 100644--- a/docs/ROADMAP.md+++ b/docs/ROADMAP.md@@ -518,6 +518,13 @@ Owner: open · Status: in progress- [ ] `native_publisher_show` hashes the file minus the signature fields, not section by section;a file with unusual gaps between sections can read as different+### R3.13 Web inspection correctness+Owner: open · Status: in progress+- [x] The HTML scanner finds script preloads in `rel` token lists, decodes asset URLs once (numeric+ and named references), keeps script and title text through lookalike closing tags, and skips+ raw-text elements, markup inside attribute values and custom tag names such as+ `<script:widget>` (#591–#596, by Afloat16)+---## M4 Release