Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

12.3 KB
Raw Blame
use std::borrow::Cow;

use moli_web_mime::request_header_content_type_essence;

// Header strings represent HTTP bytes one-for-one, including obs-text. UTF-8
// decoding would either replace bytes or combine a valid multibyte sequence.
pub(crate) fn decode_http_header_bytes(data: &[u8]) -> Cow<'_, str> {
    moli_header_field::decode_header_value(data)
}

pub(crate) fn parse_http_response_header_line(data: &[u8]) -> Option<(String, Vec<u8>)> {
    let end = data
        .iter()
        .rposition(|byte| !matches!(byte, b'\r' | b'\n'))
        .map_or(0, |index| index + 1);
    let line = &data[..end];
    let separator = line.iter().position(|byte| *byte == b':')?;
    let name = trim_http_ows(&line[..separator]);
    if name.is_empty() {
        return None;
    }
    let value = trim_http_ows(&line[separator + 1..]);
    Some((decode_http_header_bytes(name).into_owned(), value.to_vec()))
}

fn trim_http_ows(bytes: &[u8]) -> &[u8] {
    let start = bytes
        .iter()
        .position(|byte| !matches!(byte, b' ' | b'\t'))
        .unwrap_or(bytes.len());
    let bytes = &bytes[start..];
    let end = bytes
        .iter()
        .rposition(|byte| !matches!(byte, b' ' | b'\t'))
        .map_or(0, |index| index + 1);
    &bytes[..end]
}

pub(crate) fn parse_http_header_line(line: &str) -> Option<(String, String)> {
    let line = line.trim_end_matches(['\r', '\n']);
    let (name, value) = line.split_once(':')?;
    let name = name.trim_matches([' ', '\t']);
    if name.is_empty() {
        return None;
    }
    // Only HTTP OWS is framing. NBSP and NEL are ordinary header bytes.
    Some((name.to_owned(), value.trim_matches([' ', '\t']).to_owned()))
}

pub fn is_forbidden_request_header_name(name: &str) -> bool {
    let name = name.to_ascii_lowercase();
    matches!(
        name.as_str(),
        "accept-charset"
            | "accept-encoding"
            | "access-control-request-headers"
            | "access-control-request-method"
            | "connection"
            | "content-length"
            | "cookie"
            | "cookie2"
            | "date"
            | "dnt"
            | "expect"
            | "host"
            | "keep-alive"
            | "origin"
            | "referer"
            | "set-cookie"
            | "te"
            | "trailer"
            | "transfer-encoding"
            | "upgrade"
            | "via"
    ) || name.starts_with("proxy-")
        || name.starts_with("sec-")
}

pub fn is_forbidden_response_header_name(name: &str) -> bool {
    let name = name.to_ascii_lowercase();
    matches!(name.as_str(), "set-cookie" | "set-cookie2")
}

pub fn is_forbidden_request_header_override_value(name: &str, value: &str) -> bool {
    let name = name.to_ascii_lowercase();
    if !matches!(
        name.as_str(),
        "x-http-method-override" | "x-http-method" | "x-method-override"
    ) {
        return false;
    }
    split_http_header_list(value).any(|method| {
        matches!(
            method.to_ascii_uppercase().as_str(),
            "CONNECT" | "TRACE" | "TRACK"
        )
    })
}

/// Fetch's "get, decode, and split" list splitting, after combining and
/// decoding the header fields. Quotes and escapes are preserved, so commas
/// and tokens inside a quoted string cannot become separate list members.
pub fn split_http_header_list(value: &str) -> impl Iterator<Item = &str> {
    let mut quoted = false;
    let mut escaped = false;
    value
        .split(move |character| {
            if escaped {
                escaped = false;
                false
            } else if quoted && character == '\\' {
                escaped = true;
                false
            } else if character == '"' {
                quoted = !quoted;
                false
            } else {
                character == ',' && !quoted
            }
        })
        .map(|member| member.trim_matches(is_http_whitespace))
}

pub fn is_no_cors_safelisted_request_header(name: &str, value: &str) -> bool {
    matches!(
        name.to_ascii_lowercase().as_str(),
        "accept" | "accept-language" | "content-language" | "content-type"
    ) && is_cors_safelisted_request_header(name, value)
}

pub fn is_cors_safelisted_method(method: &str) -> bool {
    matches!(
        method.to_ascii_uppercase().as_str(),
        "GET" | "HEAD" | "POST"
    )
}

pub fn cors_unsafe_request_header_names(request_headers: &[(String, String)]) -> Vec<String> {
    let mut unsafe_names = Vec::<String>::new();
    let mut potentially_unsafe_names = Vec::<String>::new();
    let mut safelist_value_size = 0usize;

    for (name, value) in request_headers {
        let lower = name.to_ascii_lowercase();
        if is_cors_safelisted_request_header(&lower, value) {
            safelist_value_size += value.len();
            if !potentially_unsafe_names
                .iter()
                .any(|existing| existing == &lower)
            {
                potentially_unsafe_names.push(lower);
            }
        } else if !unsafe_names.iter().any(|existing| existing == &lower) {
            unsafe_names.push(lower);
        }
    }

    if safelist_value_size > 1024 {
        for name in potentially_unsafe_names {
            if !unsafe_names.iter().any(|existing| existing == &name) {
                unsafe_names.push(name);
            }
        }
    }

    unsafe_names.sort();
    unsafe_names
}

pub fn is_cors_safelisted_request_header(name: &str, value: &str) -> bool {
    if value.len() > 128 {
        return false;
    }
    match name.to_ascii_lowercase().as_str() {
        "accept" => !value.bytes().any(is_cors_unsafe_request_header_byte),
        "accept-language" | "content-language" => value.bytes().all(|byte| {
            byte.is_ascii_alphanumeric()
                || matches!(byte, b' ' | b'*' | b',' | b'-' | b'.' | b';' | b'=')
        }),
        "content-type" => is_cors_safelisted_request_content_type(value),
        "range" => is_cors_safelisted_request_range(value),
        _ => false,
    }
}

pub fn is_cors_safelisted_request_content_type(value: &str) -> bool {
    if value.bytes().any(is_cors_unsafe_request_header_byte) {
        return false;
    }
    let Some(essence) = request_header_content_type_essence(value) else {
        return false;
    };
    matches!(
        essence.as_str(),
        "application/x-www-form-urlencoded" | "multipart/form-data" | "text/plain"
    )
}

pub fn is_cors_safelisted_request_range(value: &str) -> bool {
    let Some(range) = value.trim().strip_prefix("bytes=") else {
        return false;
    };
    let mut parts = range.split('-');
    let Some(start) = parts.next() else {
        return false;
    };
    let Some(end) = parts.next() else {
        return false;
    };
    if parts.next().is_some() {
        return false;
    }
    let Ok(start) = start.parse::<u64>() else {
        return false;
    };
    end.is_empty() || end.parse::<u64>().is_ok_and(|end| start <= end)
}

pub fn is_cors_unsafe_request_header_byte(byte: u8) -> bool {
    matches!(
        byte,
        0x00..=0x08
            | 0x0a..=0x1f
            | 0x7f
            | b'"'
            | b'('
            | b')'
            | b':'
            | b'<'
            | b'>'
            | b'?'
            | b'@'
            | b'['
            | b'\\'
            | b']'
            | b'{'
            | b'}'
    )
}

fn is_http_whitespace(ch: char) -> bool {
    matches!(ch, '\t' | '\n' | '\r' | ' ')
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn filters_forbidden_request_header_names_and_method_overrides() {
        assert!(is_forbidden_request_header_name("Cookie"));
        assert!(is_forbidden_request_header_name("Proxy-Authorization"));
        assert!(is_forbidden_request_header_name("Sec-Fetch-Site"));
        assert!(!is_forbidden_request_header_name("X-Test"));

        assert!(is_forbidden_request_header_override_value(
            "x-http-method-override",
            "GET, track "
        ));
        assert!(is_forbidden_request_header_override_value(
            "x-method-override",
            "\tTRACE"
        ));
        assert!(!is_forbidden_request_header_override_value(
            "x-http-method",
            "GETTRACE"
        ));
        assert!(!is_forbidden_request_header_override_value(
            "x-http-method",
            "\",TRACE\","
        ));
    }

    #[test]
    fn method_override_filter_respects_quoted_list_members() {
        for name in [
            "X-HTTP-Method",
            "X-HTTP-Method-Override",
            "X-Method-Override",
        ] {
            for value in [
                r#""GET,TRACE,POST""#,
                r#""GET\",TRACK,POST""#,
                r#"prefix"one,CONNECT,two"suffix"#,
                r#""unterminated,TRACE"#,
                r#""TRACE""#,
            ] {
                assert!(
                    !is_forbidden_request_header_override_value(name, value),
                    "{name}: {value}"
                );
            }
            for value in [
                r#""GET,TRACE", TRACK"#,
                r#""GET\",TRACE", CONNECT"#,
                r#""GET\\", CONNECT"#,
                r#"prefix"CONNECT", TRACE"#,
            ] {
                assert!(
                    is_forbidden_request_header_override_value(name, value),
                    "{name}: {value}"
                );
            }
        }
    }

    #[test]
    fn no_cors_safelist_keeps_fetch_header_subset() {
        assert!(is_no_cors_safelisted_request_header("accept", "text/html"));
        assert!(!is_no_cors_safelisted_request_header("accept", "\""));
        assert!(is_no_cors_safelisted_request_header(
            "accept-language",
            "en-US, zh"
        ));
        assert!(!is_no_cors_safelisted_request_header(
            "accept-language",
            "@"
        ));
        assert!(is_no_cors_safelisted_request_header(
            "content-type",
            "text/plain;charset=UTF-8"
        ));
        assert!(is_no_cors_safelisted_request_header(
            "content-type",
            "Multipart/Form-Data; boundary=abc"
        ));
        assert!(is_no_cors_safelisted_request_header(
            "content-type",
            &format!("text/plain;{}", "s".repeat(116))
        ));
        assert!(!is_no_cors_safelisted_request_header(
            "content-type",
            "text/html"
        ));
        assert!(is_no_cors_safelisted_request_header(
            "content-type",
            "text/plain;charset=UTF-8, text/plain"
        ));
        for value in [
            "text/plain, text/plain",
            "application/json, text/plain",
            "text/plain, application/json",
            "text/plain;charset=\"utf8\", extra",
        ] {
            assert!(!is_no_cors_safelisted_request_header("content-type", value));
        }
        assert!(!is_no_cors_safelisted_request_header("range", "bytes=0-1"));
    }

    #[test]
    fn cors_safelist_accepts_single_range_values() {
        assert!(is_cors_safelisted_request_header("range", "bytes=100-200"));
        assert!(is_cors_safelisted_request_header("range", "bytes=200-"));
        assert!(!is_cors_safelisted_request_header("range", "bytes=200-100"));
        assert!(!is_cors_safelisted_request_header("range", "bytes=abc-def"));
        assert!(!is_cors_safelisted_request_header("range", ""));
    }

    #[test]
    fn cors_unsafe_request_header_names_sorts_dedupes_and_applies_safelist_size_limit() {
        let headers = vec![
            ("X-Test".to_owned(), "yes".to_owned()),
            ("x-test".to_owned(), "again".to_owned()),
            ("Range".to_owned(), "bytes=0-1".to_owned()),
            ("Accept-Language".to_owned(), "a".repeat(1025)),
            ("Content-Type".to_owned(), "text/plain".to_owned()),
        ];

        assert_eq!(
            cors_unsafe_request_header_names(&headers),
            vec!["accept-language".to_owned(), "x-test".to_owned()]
        );

        let large_safelisted = (0..9)
            .map(|_| ("Accept".to_owned(), "a".repeat(128)))
            .collect::<Vec<_>>();

        assert_eq!(
            cors_unsafe_request_header_names(&large_safelisted),
            vec!["accept".to_owned()]
        );
    }

    #[test]
    fn response_forbidden_headers_match_fetch_filtering() {
        assert!(is_forbidden_response_header_name("Set-Cookie"));
        assert!(is_forbidden_response_header_name("set-cookie2"));
        assert!(!is_forbidden_response_header_name("content-type"));
    }
}