Skip to content

Craft Anything: understand how any software works, and craft your own.

Rust
12.4 KB
Raw Blame
//! HTML pages: script references, inline scripts, module preloads and the Content Security Policy.
//! A tolerant tag scanner, not a full HTML parser.

use serde::{Deserialize, Serialize};

const MAX_TAGS: usize = 2_000;

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HtmlScript {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub src: Option<String>,
    pub module: bool,
    /// Byte range of inline script text.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub inline: Option<(u32, u32)>,
    pub line: u32,
}

#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct HtmlFacts {
    pub scripts: Vec<HtmlScript>,
    pub preloads: Vec<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub csp: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub title: Option<String>,
}

/// Attributes of the tag starting at `start` (just after the tag name). Returns (attrs, end of tag).
fn attributes(src: &str, lower: &str, mut i: usize) -> (Vec<(String, String)>, usize) {
    let b = lower.as_bytes();
    let mut out = Vec::new();
    while i < b.len() {
        while b.get(i).is_some_and(|c| c.is_ascii_whitespace() || *c == b'/') {
            i += 1;
        }
        match b.get(i) {
            None => break,
            Some(b'>') => return (out, i + 1),
            _ => {}
        }
        let ns = i;
        while b.get(i).is_some_and(|c| !c.is_ascii_whitespace() && *c != b'=' && *c != b'>') {
            i += 1;
        }
        let name = lower.get(ns..i).unwrap_or("").to_string();
        while b.get(i).is_some_and(u8::is_ascii_whitespace) {
            i += 1;
        }
        let mut value = String::new();
        if b.get(i) == Some(&b'=') {
            i += 1;
            while b.get(i).is_some_and(u8::is_ascii_whitespace) {
                i += 1;
            }
            match b.get(i) {
                Some(q @ (b'"' | b'\'')) => {
                    let q = *q;
                    let vs = i + 1;
                    i = vs;
                    while b.get(i).is_some_and(|c| *c != q) {
                        i += 1;
                    }
                    value = src.get(vs..i).unwrap_or("").to_string();
                    i += 1;
                }
                _ => {
                    let vs = i;
                    while b.get(i).is_some_and(|c| !c.is_ascii_whitespace() && *c != b'>') {
                        i += 1;
                    }
                    value = src.get(vs..i).unwrap_or("").to_string();
                }
            }
        }
        if name.is_empty() {
            i += 1;
        } else if out.len() < 64 {
            out.push((name, decode_entities(&value)));
        }
    }
    (out, i.min(b.len()))
}

fn decode_entities(s: &str) -> String {
    let mut out = String::with_capacity(s.len());
    let mut rest = s;
    while let Some(at) = rest.find('&') {
        out.push_str(rest.get(..at).unwrap_or_default());
        rest = rest.get(at + 1..).unwrap_or_default();
        let decoded = rest
            .strip_prefix('#')
            .and_then(|number| {
                let (digits, radix) = number.strip_prefix(['x', 'X']).map_or((number, 10), |n| (n, 16));
                let count = digits.bytes().take_while(|c| if radix == 16 { c.is_ascii_hexdigit() } else { c.is_ascii_digit() }).count();
                if count == 0 {
                    return None;
                }
                let value = digits.get(..count).and_then(|s| u32::from_str_radix(s, radix).ok()).unwrap_or(u32::MAX);
                let consumed = rest.len() - digits.len() + count;
                let consumed = consumed + usize::from(rest.as_bytes().get(consumed) == Some(&b';'));
                // HTML maps the numeric C1 references through the Windows-1252 table.
                const C1: [u32; 32] = [
                    0x20ac, 0x81, 0x201a, 0x192, 0x201e, 0x2026, 0x2020, 0x2021, 0x2c6, 0x2030, 0x160, 0x2039, 0x152, 0x8d, 0x17d, 0x8f, 0x90, 0x2018, 0x2019,
                    0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x2dc, 0x2122, 0x161, 0x203a, 0x153, 0x9d, 0x17e, 0x178,
                ];
                let value = value.checked_sub(0x80).and_then(|i| C1.get(i as usize)).copied().unwrap_or(value);
                Some((char::from_u32(value).filter(|c| *c != '\0').unwrap_or('\u{fffd}'), consumed))
            })
            .or_else(|| {
                [("amp;", '&'), ("quot;", '"'), ("apos;", '\''), ("lt;", '<'), ("gt;", '>')]
                    .into_iter()
                    .find_map(|(name, value)| rest.starts_with(name).then_some((value, name.len())))
            });
        if let Some((c, consumed)) = decoded {
            out.push(c);
            rest = rest.get(consumed..).unwrap_or_default();
        } else {
            out.push('&');
        }
    }
    out.push_str(rest);
    out
}

fn get<'v>(attrs: &'v [(String, String)], k: &str) -> Option<&'v str> {
    attrs.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str())
}

/// Find an actual closing tag, not a longer name such as `</scripture>`.
fn closing_tag(lower: &str, from: usize, name: &str) -> Option<usize> {
    let needle = format!("</{name}");
    lower.get(from..)?.match_indices(&needle).find_map(|(offset, _)| {
        let at = from + offset;
        lower.as_bytes().get(at + needle.len()).filter(|c| c.is_ascii_whitespace() || matches!(c, b'/' | b'>')).map(|_| at)
    })
}

/// Scan an HTML document.
pub fn scan_html(src: &str) -> HtmlFacts {
    let lower = src.to_ascii_lowercase();
    let lines = crate::util::LineIndex::new(src);
    let mut out = HtmlFacts::default();
    let mut pos = 0usize;
    let mut tags = 0usize;
    while let Some(rel) = lower.get(pos..).and_then(|r| r.find('<')) {
        tags += 1;
        if tags > MAX_TAGS * 50 || out.scripts.len() >= MAX_TAGS {
            break;
        }
        let at = pos + rel;
        let rest = lower.get(at + 1..).unwrap_or("");
        // Inspect only the names we handle: malformed runs of `<` must not rescan the remaining document.
        let name = ["script", "link", "meta", "title", "style", "textarea", "xmp", "iframe", "noembed", "noframes", "plaintext"]
            .into_iter()
            .find(|name| rest.starts_with(*name) && rest.as_bytes().get(name.len()).is_none_or(|c| c.is_ascii_whitespace() || matches!(*c, b'/' | b'>')))
            .unwrap_or_default();
        let after = at + 1 + name.len();
        match name {
            "script" => {
                let (attrs, end) = attributes(src, &lower, after);
                let close = closing_tag(&lower, end, "script").unwrap_or(lower.len());
                let module = get(&attrs, "type").is_some_and(|t| t.eq_ignore_ascii_case("module"));
                let line = lines.line(u32::try_from(at).unwrap_or(u32::MAX));
                let src_attr = get(&attrs, "src").map(String::from);
                let js = javascript_type(get(&attrs, "type"));
                let inline = (src_attr.is_none() && close > end && js && src.get(end..close).is_some_and(|t| !t.trim().is_empty()))
                    .then(|| (u32::try_from(end).unwrap_or(u32::MAX), u32::try_from(close).unwrap_or(u32::MAX)));
                if js && (src_attr.is_some() || inline.is_some()) {
                    out.scripts.push(HtmlScript { src: src_attr, module, inline, line });
                }
                pos = close.max(end).max(at + 1);
            }
            "style" | "textarea" | "xmp" | "iframe" | "noembed" | "noframes" => {
                let (_, end) = attributes(src, &lower, after);
                pos = closing_tag(&lower, end, name).unwrap_or(lower.len()).max(at + 1);
            }
            "plaintext" => break,
            "link" => {
                let (attrs, end) = attributes(src, &lower, after);
                let rel_attr = get(&attrs, "rel").unwrap_or("").to_ascii_lowercase();
                let script_destination = get(&attrs, "as").is_some_and(|value| value.eq_ignore_ascii_case("script"));
                if rel_attr.split_ascii_whitespace().any(|token| token == "modulepreload" || (token == "preload" && script_destination))
                    && let Some(h) = get(&attrs, "href")
                    && out.preloads.len() < MAX_TAGS
                {
                    out.preloads.push(h.to_string());
                }
                pos = end.max(at + 1);
            }
            "meta" => {
                let (attrs, end) = attributes(src, &lower, after);
                if get(&attrs, "http-equiv").is_some_and(|h| h.eq_ignore_ascii_case("content-security-policy")) {
                    out.csp = get(&attrs, "content").map(String::from);
                }
                pos = end.max(at + 1);
            }
            "title" => {
                let (_, end) = attributes(src, &lower, after);
                let close = closing_tag(&lower, end, "title").unwrap_or(lower.len());
                out.title = src.get(end..close).map(|t| t.trim().chars().take(200).collect());
                pos = close.max(at + 1);
            }
            _ if rest.starts_with("!--") => {
                pos = lower.get(at..).and_then(|r| r.find("-->")).map_or(lower.len(), |c| at + c + 3);
            }
            _ if rest.as_bytes().first().is_some_and(u8::is_ascii_alphabetic)
                || rest.strip_prefix('/').and_then(|s| s.as_bytes().first()).is_some_and(u8::is_ascii_alphabetic) =>
            {
                let (_, end) = attributes(src, &lower, after);
                pos = end.max(at + 1);
            }
            _ => pos = at + 1,
        }
    }
    out
}

/// Whether a `<script type>` is code the page runs: no type, a JavaScript MIME type or `module`
/// (and JSX for in-browser Babel, which is code too). Import maps, speculation rules and data
/// blocks such as JSON or HTML templates aren't JavaScript (#285).
fn javascript_type(t: Option<&str>) -> bool {
    let Some(t) = t else { return true };
    let essence = t.split(';').next().unwrap_or_default().trim().to_ascii_lowercase();
    essence.is_empty()
        || essence == "module"
        || matches!(essence.as_str(), "text/babel" | "text/jsx" | "text/jscript" | "text/livescript")
        || [
            "text/javascript",
            "application/javascript",
            "application/ecmascript",
            "text/ecmascript",
            "application/x-javascript",
            "application/x-ecmascript",
            "text/x-javascript",
            "text/x-ecmascript",
        ]
        .iter()
        .any(|m| essence == *m || essence.strip_prefix(m).is_some_and(|v| v.starts_with('1')))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn finds_scripts_and_csp() {
        let h = r#"<!doctype html><html><head><meta http-equiv="Content-Security-Policy" content="default-src 'self'">
<title>Notes</title><link rel="modulepreload" href="/assets/vendor-abc.js">
<script type="module" src="./app.js"></script><!-- <script src="no.js"></script> -->
<script>window.x = 1;</script><script type="application/json">{}</script></head></html>"#;
        let f = scan_html(h);
        assert_eq!(f.scripts.len(), 2);
        assert_eq!(f.scripts[0].src.as_deref(), Some("./app.js"));
        assert!(f.scripts[0].module);
        let (s, e) = f.scripts[1].inline.unwrap();
        assert_eq!(&h[s as usize..e as usize], "window.x = 1;");
        assert_eq!(f.preloads, vec!["/assets/vendor-abc.js"]);
        assert_eq!(f.csp.as_deref(), Some("default-src 'self'"));
        assert_eq!(f.title.as_deref(), Some("Notes"));
    }

    /// #285: import maps, speculation rules and HTML templates were read as JavaScript.
    #[test]
    fn only_javascript_types_are_scripts() {
        for t in ["importmap", "speculationrules", "text/html", "application/json", "text/x-template"] {
            let f = scan_html(&format!(r#"<script type="{t}">{{"imports": {{}}}}</script><script type="{t}" src="data.json"></script>"#));
            assert!(f.scripts.is_empty(), "{t}: {:?}", f.scripts);
        }
        for t in ["", "module", "text/javascript", "Application/JavaScript", "text/javascript; charset=utf-8", "text/babel"] {
            assert_eq!(scan_html(&format!(r#"<script type="{t}">fetch("/real-api");</script>"#)).scripts.len(), 1, "{t}");
        }
    }

    #[test]
    fn junk_does_not_panic() {
        for s in ["<", "<script", "<script src=", "<script src='x", "<<<<>>>", "<script>é", "<meta http-equiv=content-security-policy content=\"é", "<!--"] {
            let _ = scan_html(s);
        }
    }
}