Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use std::borrow::Cow;
use moli_web_mime::request_header_content_type_essence;
// Header strings represent HTTP bytes one-for-one, including obs-text. UTF-8
// decoding would either replace bytes or combine a valid multibyte sequence.
pub(crate) fn decode_http_header_bytes(data: &[u8]) -> Cow<'_, str> {
moli_header_field::decode_header_value(data)
}
pub(crate) fn parse_http_response_header_line(data: &[u8]) -> Option<(String, Vec<u8>)> {
let end = data
.iter()
.rposition(|byte| !matches!(byte, b'\r' | b'\n'))
.map_or(0, |index| index + 1);
let line = &data[..end];
let separator = line.iter().position(|byte| *byte == b':')?;
let name = trim_http_ows(&line[..separator]);
if name.is_empty() {
return None;
}
let value = trim_http_ows(&line[separator + 1..]);
Some((decode_http_header_bytes(name).into_owned(), value.to_vec()))
}
fn trim_http_ows(bytes: &[u8]) -> &[u8] {
let start = bytes
.iter()
.position(|byte| !matches!(byte, b' ' | b'\t'))
.unwrap_or(bytes.len());
let bytes = &bytes[start..];
let end = bytes
.iter()
.rposition(|byte| !matches!(byte, b' ' | b'\t'))
.map_or(0, |index| index + 1);
&bytes[..end]
}
pub(crate) fn parse_http_header_line(line: &str) -> Option<(String, String)> {
let line = line.trim_end_matches(['\r', '\n']);
let (name, value) = line.split_once(':')?;
let name = name.trim_matches([' ', '\t']);
if name.is_empty() {
return None;
}
// Only HTTP OWS is framing. NBSP and NEL are ordinary header bytes.
Some((name.to_owned(), value.trim_matches([' ', '\t']).to_owned()))
}
pub fn is_forbidden_request_header_name(name: &str) -> bool {
let name = name.to_ascii_lowercase();
matches!(
name.as_str(),
"accept-charset"
| "accept-encoding"
| "access-control-request-headers"
| "access-control-request-method"
| "connection"
| "content-length"
| "cookie"
| "cookie2"
| "date"
| "dnt"
| "expect"
| "host"
| "keep-alive"
| "origin"
| "referer"
| "set-cookie"
| "te"
| "trailer"
| "transfer-encoding"
| "upgrade"
| "via"
) || name.starts_with("proxy-")
|| name.starts_with("sec-")
}
pub fn is_forbidden_response_header_name(name: &str) -> bool {
let name = name.to_ascii_lowercase();
matches!(name.as_str(), "set-cookie" | "set-cookie2")
}
pub fn is_forbidden_request_header_override_value(name: &str, value: &str) -> bool {
let name = name.to_ascii_lowercase();
if !matches!(
name.as_str(),
"x-http-method-override" | "x-http-method" | "x-method-override"
) {
return false;
}
split_http_header_list(value).any(|method| {
matches!(
method.to_ascii_uppercase().as_str(),
"CONNECT" | "TRACE" | "TRACK"
)
})
}
/// Fetch's "get, decode, and split" list splitting, after combining and
/// decoding the header fields. Quotes and escapes are preserved, so commas
/// and tokens inside a quoted string cannot become separate list members.
pub fn split_http_header_list(value: &str) -> impl Iterator<Item = &str> {
let mut quoted = false;
let mut escaped = false;
value
.split(move |character| {
if escaped {
escaped = false;
false
} else if quoted && character == '\\' {
escaped = true;
false
} else if character == '"' {
quoted = !quoted;
false
} else {
character == ',' && !quoted
}
})
.map(|member| member.trim_matches(is_http_whitespace))
}
pub fn is_no_cors_safelisted_request_header(name: &str, value: &str) -> bool {
matches!(
name.to_ascii_lowercase().as_str(),
"accept" | "accept-language" | "content-language" | "content-type"
) && is_cors_safelisted_request_header(name, value)
}
pub fn is_cors_safelisted_method(method: &str) -> bool {
matches!(
method.to_ascii_uppercase().as_str(),
"GET" | "HEAD" | "POST"
)
}
pub fn cors_unsafe_request_header_names(request_headers: &[(String, String)]) -> Vec<String> {
let mut unsafe_names = Vec::<String>::new();
let mut potentially_unsafe_names = Vec::<String>::new();
let mut safelist_value_size = 0usize;
for (name, value) in request_headers {
let lower = name.to_ascii_lowercase();
if is_cors_safelisted_request_header(&lower, value) {
safelist_value_size += value.len();
if !potentially_unsafe_names
.iter()
.any(|existing| existing == &lower)
{
potentially_unsafe_names.push(lower);
}
} else if !unsafe_names.iter().any(|existing| existing == &lower) {
unsafe_names.push(lower);
}
}
if safelist_value_size > 1024 {
for name in potentially_unsafe_names {
if !unsafe_names.iter().any(|existing| existing == &name) {
unsafe_names.push(name);
}
}
}
unsafe_names.sort();
unsafe_names
}
pub fn is_cors_safelisted_request_header(name: &str, value: &str) -> bool {
if value.len() > 128 {
return false;
}
match name.to_ascii_lowercase().as_str() {
"accept" => !value.bytes().any(is_cors_unsafe_request_header_byte),
"accept-language" | "content-language" => value.bytes().all(|byte| {
byte.is_ascii_alphanumeric()
|| matches!(byte, b' ' | b'*' | b',' | b'-' | b'.' | b';' | b'=')
}),
"content-type" => is_cors_safelisted_request_content_type(value),
"range" => is_cors_safelisted_request_range(value),
_ => false,
}
}
pub fn is_cors_safelisted_request_content_type(value: &str) -> bool {
if value.bytes().any(is_cors_unsafe_request_header_byte) {
return false;
}
let Some(essence) = request_header_content_type_essence(value) else {
return false;
};
matches!(
essence.as_str(),
"application/x-www-form-urlencoded" | "multipart/form-data" | "text/plain"
)
}
pub fn is_cors_safelisted_request_range(value: &str) -> bool {
let Some(range) = value.trim().strip_prefix("bytes=") else {
return false;
};
let mut parts = range.split('-');
let Some(start) = parts.next() else {
return false;
};
let Some(end) = parts.next() else {
return false;
};
if parts.next().is_some() {
return false;
}
let Ok(start) = start.parse::<u64>() else {
return false;
};
end.is_empty() || end.parse::<u64>().is_ok_and(|end| start <= end)
}
pub fn is_cors_unsafe_request_header_byte(byte: u8) -> bool {
matches!(
byte,
0x00..=0x08
| 0x0a..=0x1f
| 0x7f
| b'"'
| b'('
| b')'
| b':'
| b'<'
| b'>'
| b'?'
| b'@'
| b'['
| b'\\'
| b']'
| b'{'
| b'}'
)
}
fn is_http_whitespace(ch: char) -> bool {
matches!(ch, '\t' | '\n' | '\r' | ' ')
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn filters_forbidden_request_header_names_and_method_overrides() {
assert!(is_forbidden_request_header_name("Cookie"));
assert!(is_forbidden_request_header_name("Proxy-Authorization"));
assert!(is_forbidden_request_header_name("Sec-Fetch-Site"));
assert!(!is_forbidden_request_header_name("X-Test"));
assert!(is_forbidden_request_header_override_value(
"x-http-method-override",
"GET, track "
));
assert!(is_forbidden_request_header_override_value(
"x-method-override",
"\tTRACE"
));
assert!(!is_forbidden_request_header_override_value(
"x-http-method",
"GETTRACE"
));
assert!(!is_forbidden_request_header_override_value(
"x-http-method",
"\",TRACE\","
));
}
#[test]
fn method_override_filter_respects_quoted_list_members() {
for name in [
"X-HTTP-Method",
"X-HTTP-Method-Override",
"X-Method-Override",
] {
for value in [
r#""GET,TRACE,POST""#,
r#""GET\",TRACK,POST""#,
r#"prefix"one,CONNECT,two"suffix"#,
r#""unterminated,TRACE"#,
r#""TRACE""#,
] {
assert!(
!is_forbidden_request_header_override_value(name, value),
"{name}: {value}"
);
}
for value in [
r#""GET,TRACE", TRACK"#,
r#""GET\",TRACE", CONNECT"#,
r#""GET\\", CONNECT"#,
r#"prefix"CONNECT", TRACE"#,
] {
assert!(
is_forbidden_request_header_override_value(name, value),
"{name}: {value}"
);
}
}
}
#[test]
fn no_cors_safelist_keeps_fetch_header_subset() {
assert!(is_no_cors_safelisted_request_header("accept", "text/html"));
assert!(!is_no_cors_safelisted_request_header("accept", "\""));
assert!(is_no_cors_safelisted_request_header(
"accept-language",
"en-US, zh"
));
assert!(!is_no_cors_safelisted_request_header(
"accept-language",
"@"
));
assert!(is_no_cors_safelisted_request_header(
"content-type",
"text/plain;charset=UTF-8"
));
assert!(is_no_cors_safelisted_request_header(
"content-type",
"Multipart/Form-Data; boundary=abc"
));
assert!(is_no_cors_safelisted_request_header(
"content-type",
&format!("text/plain;{}", "s".repeat(116))
));
assert!(!is_no_cors_safelisted_request_header(
"content-type",
"text/html"
));
assert!(is_no_cors_safelisted_request_header(
"content-type",
"text/plain;charset=UTF-8, text/plain"
));
for value in [
"text/plain, text/plain",
"application/json, text/plain",
"text/plain, application/json",
"text/plain;charset=\"utf8\", extra",
] {
assert!(!is_no_cors_safelisted_request_header("content-type", value));
}
assert!(!is_no_cors_safelisted_request_header("range", "bytes=0-1"));
}
#[test]
fn cors_safelist_accepts_single_range_values() {
assert!(is_cors_safelisted_request_header("range", "bytes=100-200"));
assert!(is_cors_safelisted_request_header("range", "bytes=200-"));
assert!(!is_cors_safelisted_request_header("range", "bytes=200-100"));
assert!(!is_cors_safelisted_request_header("range", "bytes=abc-def"));
assert!(!is_cors_safelisted_request_header("range", ""));
}
#[test]
fn cors_unsafe_request_header_names_sorts_dedupes_and_applies_safelist_size_limit() {
let headers = vec![
("X-Test".to_owned(), "yes".to_owned()),
("x-test".to_owned(), "again".to_owned()),
("Range".to_owned(), "bytes=0-1".to_owned()),
("Accept-Language".to_owned(), "a".repeat(1025)),
("Content-Type".to_owned(), "text/plain".to_owned()),
];
assert_eq!(
cors_unsafe_request_header_names(&headers),
vec!["accept-language".to_owned(), "x-test".to_owned()]
);
let large_safelisted = (0..9)
.map(|_| ("Accept".to_owned(), "a".repeat(128)))
.collect::<Vec<_>>();
assert_eq!(
cors_unsafe_request_header_names(&large_safelisted),
vec!["accept".to_owned()]
);
}
#[test]
fn response_forbidden_headers_match_fetch_filtering() {
assert!(is_forbidden_response_header_name("Set-Cookie"));
assert!(is_forbidden_response_header_name("set-cookie2"));
assert!(!is_forbidden_response_header_name("content-type"));
}
}