Skip to content

Craft Anything: understand how any software works, and craft your own.

Rust

Read HTML pages' scripts and preloads correctly in more cases (#645)

BFlat0eea78d7aa50
diff --git a/crates/js/src/html.rs b/crates/js/src/html.rs
index 46b75396..4d7a27ca 100644
--- a/crates/js/src/html.rs
+++ b/crates/js/src/html.rs
@@ -30,7 +30,7 @@ pub struct HtmlFacts {
fn attributes(src: &str, lower: &str, mut i: usize) -> (Vec<(String, String)>, usize) {
let b = lower.as_bytes();
let mut out = Vec::new();
- while i < b.len() && out.len() < 64 {
+ while i < b.len() {
while b.get(i).is_some_and(|c| c.is_ascii_whitespace() || *c == b'/') {
i += 1;
}
@@ -75,7 +75,7 @@ fn attributes(src: &str, lower: &str, mut i: usize) -> (Vec<(String, String)>, u
}
if name.is_empty() {
i += 1;
- } else {
+ } else if out.len() < 64 {
out.push((name, decode_entities(&value)));
}
}
@@ -83,13 +83,59 @@ fn attributes(src: &str, lower: &str, mut i: usize) -> (Vec<(String, String)>, u
}
fn decode_entities(s: &str) -> String {
- s.replace("&amp;", "&").replace("&quot;", "\"").replace("&#39;", "'").replace("&lt;", "<").replace("&gt;", ">")
+ let mut out = String::with_capacity(s.len());
+ let mut rest = s;
+ while let Some(at) = rest.find('&') {
+ out.push_str(rest.get(..at).unwrap_or_default());
+ rest = rest.get(at + 1..).unwrap_or_default();
+ let decoded = rest
+ .strip_prefix('#')
+ .and_then(|number| {
+ let (digits, radix) = number.strip_prefix(['x', 'X']).map_or((number, 10), |n| (n, 16));
+ let count = digits.bytes().take_while(|c| if radix == 16 { c.is_ascii_hexdigit() } else { c.is_ascii_digit() }).count();
+ if count == 0 {
+ return None;
+ }
+ let value = digits.get(..count).and_then(|s| u32::from_str_radix(s, radix).ok()).unwrap_or(u32::MAX);
+ let consumed = rest.len() - digits.len() + count;
+ let consumed = consumed + usize::from(rest.as_bytes().get(consumed) == Some(&b';'));
+ // HTML maps the numeric C1 references through the Windows-1252 table.
+ const C1: [u32; 32] = [
+ 0x20ac, 0x81, 0x201a, 0x192, 0x201e, 0x2026, 0x2020, 0x2021, 0x2c6, 0x2030, 0x160, 0x2039, 0x152, 0x8d, 0x17d, 0x8f, 0x90, 0x2018, 0x2019,
+ 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x2dc, 0x2122, 0x161, 0x203a, 0x153, 0x9d, 0x17e, 0x178,
+ ];
+ let value = value.checked_sub(0x80).and_then(|i| C1.get(i as usize)).copied().unwrap_or(value);
+ Some((char::from_u32(value).filter(|c| *c != '\0').unwrap_or('\u{fffd}'), consumed))
+ })
+ .or_else(|| {
+ [("amp;", '&'), ("quot;", '"'), ("apos;", '\''), ("lt;", '<'), ("gt;", '>')]
+ .into_iter()
+ .find_map(|(name, value)| rest.starts_with(name).then_some((value, name.len())))
+ });
+ if let Some((c, consumed)) = decoded {
+ out.push(c);
+ rest = rest.get(consumed..).unwrap_or_default();
+ } else {
+ out.push('&');
+ }
+ }
+ out.push_str(rest);
+ out
}
fn get<'v>(attrs: &'v [(String, String)], k: &str) -> Option<&'v str> {
attrs.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str())
}
+/// Find an actual closing tag, not a longer name such as `</scripture>`.
+fn closing_tag(lower: &str, from: usize, name: &str) -> Option<usize> {
+ let needle = format!("</{name}");
+ lower.get(from..)?.match_indices(&needle).find_map(|(offset, _)| {
+ let at = from + offset;
+ lower.as_bytes().get(at + needle.len()).filter(|c| c.is_ascii_whitespace() || matches!(c, b'/' | b'>')).map(|_| at)
+ })
+}
+
/// Scan an HTML document.
pub fn scan_html(src: &str) -> HtmlFacts {
let lower = src.to_ascii_lowercase();
@@ -104,13 +150,16 @@ pub fn scan_html(src: &str) -> HtmlFacts {
}
let at = pos + rel;
let rest = lower.get(at + 1..).unwrap_or("");
- let name_len = rest.bytes().take_while(|c| c.is_ascii_alphanumeric() || *c == b'-').count();
- let name = rest.get(..name_len).unwrap_or("");
- let after = at + 1 + name_len;
+ // Inspect only the names we handle: malformed runs of `<` must not rescan the remaining document.
+ let name = ["script", "link", "meta", "title", "style", "textarea", "xmp", "iframe", "noembed", "noframes", "plaintext"]
+ .into_iter()
+ .find(|name| rest.starts_with(*name) && rest.as_bytes().get(name.len()).is_none_or(|c| c.is_ascii_whitespace() || matches!(*c, b'/' | b'>')))
+ .unwrap_or_default();
+ let after = at + 1 + name.len();
match name {
"script" => {
let (attrs, end) = attributes(src, &lower, after);
- let close = lower.get(end..).and_then(|r| r.find("</script")).map_or(lower.len(), |c| end + c);
+ let close = closing_tag(&lower, end, "script").unwrap_or(lower.len());
let module = get(&attrs, "type").is_some_and(|t| t.eq_ignore_ascii_case("module"));
let line = lines.line(u32::try_from(at).unwrap_or(u32::MAX));
let src_attr = get(&attrs, "src").map(String::from);
@@ -122,10 +171,16 @@ pub fn scan_html(src: &str) -> HtmlFacts {
}
pos = close.max(end).max(at + 1);
}
+ "style" | "textarea" | "xmp" | "iframe" | "noembed" | "noframes" => {
+ let (_, end) = attributes(src, &lower, after);
+ pos = closing_tag(&lower, end, name).unwrap_or(lower.len()).max(at + 1);
+ }
+ "plaintext" => break,
"link" => {
let (attrs, end) = attributes(src, &lower, after);
let rel_attr = get(&attrs, "rel").unwrap_or("").to_ascii_lowercase();
- if (rel_attr == "modulepreload" || (rel_attr == "preload" && get(&attrs, "as") == Some("script")))
+ let script_destination = get(&attrs, "as").is_some_and(|value| value.eq_ignore_ascii_case("script"));
+ if rel_attr.split_ascii_whitespace().any(|token| token == "modulepreload" || (token == "preload" && script_destination))
&& let Some(h) = get(&attrs, "href")
&& out.preloads.len() < MAX_TAGS
{
@@ -142,13 +197,19 @@ pub fn scan_html(src: &str) -> HtmlFacts {
}
"title" => {
let (_, end) = attributes(src, &lower, after);
- let close = lower.get(end..).and_then(|r| r.find("</title")).map_or(end, |c| end + c);
+ let close = closing_tag(&lower, end, "title").unwrap_or(lower.len());
out.title = src.get(end..close).map(|t| t.trim().chars().take(200).collect());
pos = close.max(at + 1);
}
_ if rest.starts_with("!--") => {
pos = lower.get(at..).and_then(|r| r.find("-->")).map_or(lower.len(), |c| at + c + 3);
}
+ _ if rest.as_bytes().first().is_some_and(u8::is_ascii_alphabetic)
+ || rest.strip_prefix('/').and_then(|s| s.as_bytes().first()).is_some_and(u8::is_ascii_alphabetic) =>
+ {
+ let (_, end) = attributes(src, &lower, after);
+ pos = end.max(at + 1);
+ }
_ => pos = at + 1,
}
}
diff --git a/crates/js/tests/regression_html_end_tags.rs b/crates/js/tests/regression_html_end_tags.rs
new file mode 100644
index 00000000..0747bea2
--- /dev/null
+++ b/crates/js/tests/regression_html_end_tags.rs
@@ -0,0 +1,18 @@
+use craft_anything_js::html::scan_html;
+
+#[test]
+fn regression_closing_tag_lookalikes_do_not_end_scripts() {
+ for lookalike in ["</scripture>", "</script_widget>", "</script:widget>"] {
+ let code = format!("const label = '{lookalike}'; window.after = true;");
+ let page = format!("<script>{code}</ScRiPt ><script src='next.js'></script>");
+ let f = scan_html(&page);
+ let (start, end) = f.scripts[0].inline.unwrap();
+ assert_eq!(&page[start as usize..end as usize], code);
+ assert_eq!(f.scripts.len(), 2);
+ assert_eq!(f.scripts[1].src.as_deref(), Some("next.js"));
+ }
+ assert_eq!(scan_html("<title>a</titleish>b</title>").title.as_deref(), Some("a</titleish>b"));
+ let unclosed = scan_html("<title>caption</titleish><script src='ghost.js'></script>");
+ assert!(unclosed.scripts.is_empty());
+ assert!(unclosed.title.as_deref().unwrap().starts_with("caption</titleish>"));
+}
diff --git a/crates/js/tests/regression_html_entities.rs b/crates/js/tests/regression_html_entities.rs
new file mode 100644
index 00000000..3b79120f
--- /dev/null
+++ b/crates/js/tests/regression_html_entities.rs
@@ -0,0 +1,15 @@
+use craft_anything_js::html::scan_html;
+fn decode_entities(value: &str) -> String {
+ scan_html(&format!("<script src=\"{value}\"></script>")).scripts[0].src.clone().unwrap()
+}
+
+#[test]
+fn regression_asset_references_decode_once() {
+ let f = scan_html(r#"<script src="/app.js?a=1&amp;b=&#x32;&#47;&#51;&amp;quot;"></script><link rel="modulepreload" href="&#x2f;modules&#47;main.js">"#);
+ assert_eq!(f.scripts[0].src.as_deref(), Some("/app.js?a=1&b=2/3&quot;"));
+ assert_eq!(f.preloads, ["/modules/main.js"]);
+ assert_eq!(decode_entities("&unknown; &#x; &#not; &amp;amp; &#x1f642;"), "&unknown; &#x; &#not; &amp; 🙂");
+ assert_eq!(decode_entities("&#0; &#xD800; &#x110000; &#x80;"), "� � � €");
+ assert_eq!(decode_entities("&#47app.js &#x2Fmodules.js"), "/app.js /modules.js");
+ assert_eq!(decode_entities(&format!("&#{}47;app.js", "0".repeat(80))), "/app.js");
+}
diff --git a/crates/js/tests/regression_html_preloads.rs b/crates/js/tests/regression_html_preloads.rs
new file mode 100644
index 00000000..230d875f
--- /dev/null
+++ b/crates/js/tests/regression_html_preloads.rs
@@ -0,0 +1,9 @@
+use craft_anything_js::html::scan_html;
+
+#[test]
+fn regression_preload_relationships_are_tokens() {
+ let f = scan_html(
+ r#"<link rel="alternate PRELOAD" as="SCRIPT" href="one.js"><link rel="stylesheet modulepreload" href="two.js"><link rel="notpreload" as="script" href="no.js"><link rel="preload" as="style" href="no.css"><link rel="preloadish" as="script" href="other.js">"#,
+ );
+ assert_eq!(f.preloads, ["one.js", "two.js"]);
+}
diff --git a/crates/js/tests/regression_html_quoted_tags.rs b/crates/js/tests/regression_html_quoted_tags.rs
new file mode 100644
index 00000000..dbea234e
--- /dev/null
+++ b/crates/js/tests/regression_html_quoted_tags.rs
@@ -0,0 +1,15 @@
+use craft_anything_js::html::scan_html;
+
+#[test]
+fn regression_markup_in_attributes_is_not_scanned_as_tags() {
+ for extra in [String::new(), (0..70).map(|i| format!(" a{i}='value'")).collect()] {
+ let page = format!("<div{extra} data-example=\"<script src='ghost.js'></script>\"><script src='real.js'></script></div>");
+ let f = scan_html(&page);
+ assert_eq!(f.scripts.len(), 1, "{:?}", f.scripts);
+ assert_eq!(f.scripts[0].src.as_deref(), Some("real.js"));
+ }
+ for text in ["1 < 2", "1 <2", "x < y"] {
+ let page = format!("<p>{text} <script src='real.js'></script></p>");
+ assert_eq!(scan_html(&page).scripts[0].src.as_deref(), Some("real.js"));
+ }
+}
diff --git a/crates/js/tests/regression_html_raw_text.rs b/crates/js/tests/regression_html_raw_text.rs
new file mode 100644
index 00000000..349d8ca9
--- /dev/null
+++ b/crates/js/tests/regression_html_raw_text.rs
@@ -0,0 +1,12 @@
+use craft_anything_js::html::scan_html;
+
+#[test]
+fn regression_text_elements_do_not_invent_script_dependencies() {
+ for tag in ["style", "textarea", "xmp", "iframe", "noembed", "noframes"] {
+ let page = format!("<{tag}><script src='ghost.js'></script></{tag}><script src='real.js'></script>");
+ let f = scan_html(&page);
+ assert_eq!(f.scripts.len(), 1, "{tag}: {:?}", f.scripts);
+ assert_eq!(f.scripts[0].src.as_deref(), Some("real.js"));
+ }
+ assert!(scan_html("<plaintext><script src='ghost.js'></script>").scripts.is_empty());
+}
diff --git a/crates/js/tests/regression_html_tag_names.rs b/crates/js/tests/regression_html_tag_names.rs
new file mode 100644
index 00000000..33f5346f
--- /dev/null
+++ b/crates/js/tests/regression_html_tag_names.rs
@@ -0,0 +1,21 @@
+use craft_anything_js::html::scan_html;
+
+#[test]
+fn regression_html_many_invalid_tag_starts_still_finds_scripts() {
+ let html = "<".repeat(80_000) + "<script src='/after-invalid-tags.js'></script>";
+ let facts = scan_html(&html);
+ assert_eq!(facts.scripts.len(), 1);
+ assert_eq!(facts.scripts[0].src.as_deref(), Some("/after-invalid-tags.js"));
+}
+
+#[test]
+fn regression_custom_tag_names_are_not_builtin_tags() {
+ let f = scan_html(
+ r#"<script:widget src="ghost.js"></script:widget><script_widget src="other.js"></script_widget><link:widget rel="modulepreload" href="ghost.js"><meta:widget http-equiv="content-security-policy" content="bad"><title:widget>wrong</title:widget><script src="real.js"></script>"#,
+ );
+ assert_eq!(f.scripts.iter().filter_map(|s| s.src.as_deref()).collect::<Vec<_>>(), ["real.js"]);
+ assert!(f.preloads.is_empty());
+ assert!(f.csp.is_none());
+ assert!(f.title.is_none());
+ assert_eq!(scan_html("<SCRIPT src='upper.js'></SCRIPT>").scripts[0].src.as_deref(), Some("upper.js"));
+}
diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md
index c1cae368..c424f4fe 100644
--- a/docs/ROADMAP.md
+++ b/docs/ROADMAP.md
@@ -518,6 +518,13 @@ Owner: open · Status: in progress
- [ ] `native_publisher_show` hashes the file minus the signature fields, not section by section;
a file with unusual gaps between sections can read as different
+### R3.13 Web inspection correctness
+Owner: open · Status: in progress
+- [x] The HTML scanner finds script preloads in `rel` token lists, decodes asset URLs once (numeric
+ and named references), keeps script and title text through lookalike closing tags, and skips
+ raw-text elements, markup inside attribute values and custom tag names such as
+ `<script:widget>` (#591–#596, by Afloat16)
+
---
## M4 Release