Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): distinguish request header bytes from UTF-8 strings
ldm04ed30be874ce
diff --git a/moli-fetch/src/blocking/cache/key.rs b/moli-fetch/src/blocking/cache/key.rsindex 00c847732..9c80480a8 100644--- a/moli-fetch/src/blocking/cache/key.rs+++ b/moli-fetch/src/blocking/cache/key.rs@@ -23,7 +23,7 @@ pub(super) fn cache_store_and_key_for_request(|| !config.default_request_headers().is_empty()|| !subresource_validation_allows_http_cache(request)|| cookie_header.is_some()- || !request_headers_allow_http_cache(&request.request_headers)+ || !request_headers_allow_http_cache(&request.request_headers.to_byte_strings())|| !matches!(request_url.scheme(), "http" | "https")|| !request_url.username().is_empty()|| request_url.password().is_some()diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex d0279d39e..741fe9af8 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -160,26 +160,38 @@ pub fn outgoing_request_headers(outgoing_request_headers_for_url(config, request, &request.url, cookie_header)}+// String views used by observation and cache metadata are isomorphic byte+// projections. The transport itself keeps the encoded representation below.pub(crate) fn outgoing_request_headers_for_url(config: &FetchConfig,request: &Request,request_url: &Url,cookie_header: Option<&str>,) -> Vec<(String, String)> {- let mut outgoing = Vec::new();+ outgoing_request_header_bytes_for_url(config, request, request_url, cookie_header)+ .to_byte_strings()+}++fn outgoing_request_header_bytes_for_url(+ config: &FetchConfig,+ request: &Request,+ request_url: &Url,+ cookie_header: Option<&str>,+) -> crate::RequestHeaders {+ let mut outgoing = crate::RequestHeaders::default();if let Some(cookie_header) = cookie_header {- outgoing.push(("Cookie".to_owned(), cookie_header.to_owned()));+ outgoing.push(("Cookie".to_owned(), cookie_header.as_bytes().to_vec()));}for (name, value) in config.default_request_headers() {if cookie_header.is_some() && name.eq_ignore_ascii_case("cookie") {continue;}- outgoing.push((name.clone(), value.clone()));+ outgoing.push((name.clone(), value.as_bytes().to_vec()));}- for (name, value) in &request.request_headers {+ for (name, value) in request.request_headers.iter() {if cookie_header.is_some() && name.eq_ignore_ascii_case("cookie") {continue;}@@ -187,7 +199,7 @@ pub(crate) fn outgoing_request_headers_for_url(}if request.has_browser_identity_override() {- append_header_if_missing(+ append_encoded_header_if_missing(&mut outgoing,"User-Agent",request.browser_identity(config).user_agent().to_owned(),@@ -199,13 +211,13 @@ pub(crate) fn outgoing_request_headers_for_url(append_browser_storage_access_header(&mut outgoing, request, request_url);if let Some(origin) = request_origin_header_value(request, request_url) {- append_header_if_missing(&mut outgoing, "Origin", origin);+ append_encoded_header_if_missing(&mut outgoing, "Origin", origin);}if !header_present(&outgoing, "referer")&& let Some(referer) = referrer_header_value_for_request(request, request_url){- outgoing.push(("Referer".to_owned(), referer));+ outgoing.push(("Referer".to_owned(), referer.into_bytes()));}if !header_present(&outgoing, "authorization")@@ -217,7 +229,7 @@ pub(crate) fn outgoing_request_headers_for_url(// 401 retry body while Digest/NTLM/Negotiate stay on the buffered path.outgoing.push(("Authorization".to_owned(),- format!("Basic {}", encode_basic_auth(username, password)),+ format!("Basic {}", encode_basic_auth(username, password)).into_bytes(),));}@@ -231,7 +243,8 @@ pub(crate) fn outgoing_request_headers_for_url(format!("Basic {}",encode_basic_auth(&auth.username, &auth.password)- ),+ )+ .into_bytes(),));}@@ -251,20 +264,34 @@ pub(crate) fn network_request_extra_info_from_headers(}}-fn header_present(headers: &[(String, String)], name: &str) -> bool {+fn header_present<V>(headers: &[(String, V)], name: &str) -> bool {headers.iter().any(|(header_name, _)| header_name.eq_ignore_ascii_case(name))}+fn header_utf8_byte_string(value: &str) -> String {+ crate::headers::decode_http_header_bytes(value.as_bytes()).into_owned()+}+fn append_header_if_missing(headers: &mut Vec<(String, String)>, name: &str, value: String) {if !header_present(headers, name) {- headers.push((name.to_owned(), value));+ headers.push((name.to_owned(), header_utf8_byte_string(&value)));+ }+}++fn append_encoded_header_if_missing(+ headers: &mut crate::RequestHeaders,+ name: &str,+ value: String,+) {+ if !header_present(headers, name) {+ headers.push((name.to_owned(), value.into_bytes()));}}fn append_browser_navigation_headers(- outgoing: &mut Vec<(String, String)>,+ outgoing: &mut crate::RequestHeaders,config: &FetchConfig,request: &Request,request_url: &Url,@@ -273,12 +300,12 @@ fn append_browser_navigation_headers(return;}- append_header_if_missing(+ append_encoded_header_if_missing(outgoing,"Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7".to_owned(),);- append_header_if_missing(+ append_encoded_header_if_missing(outgoing,"Accept-Language",request@@ -286,9 +313,9 @@ fn append_browser_navigation_headers(.accept_language().to_owned(),);- append_header_if_missing(outgoing, "Upgrade-Insecure-Requests", "1".to_owned());- append_header_if_missing(outgoing, "Sec-Fetch-Mode", "navigate".to_owned());- append_header_if_missing(+ append_encoded_header_if_missing(outgoing, "Upgrade-Insecure-Requests", "1".to_owned());+ append_encoded_header_if_missing(outgoing, "Sec-Fetch-Mode", "navigate".to_owned());+ append_encoded_header_if_missing(outgoing,"Sec-Fetch-Dest",if request.is_subframe_navigation_request() {@@ -298,25 +325,25 @@ fn append_browser_navigation_headers(}.to_owned(),);- append_header_if_missing(+ append_encoded_header_if_missing(outgoing,"Sec-Fetch-Site",request_sec_fetch_site(request, request_url),);if request.is_top_level_navigation_request() && request.cookie_context.initiator_url.is_none() {- append_header_if_missing(outgoing, "Sec-Fetch-User", "?1".to_owned());+ append_encoded_header_if_missing(outgoing, "Sec-Fetch-User", "?1".to_owned());}if request.browser_navigation_kind() == crate::BrowserNavigationRequestKind::Reload {- append_header_if_missing(outgoing, "Cache-Control", "max-age=0".to_owned());+ append_encoded_header_if_missing(outgoing, "Cache-Control", "max-age=0".to_owned());}append_browser_client_hints(outgoing, request.browser_identity(config));}fn append_browser_subresource_headers(- outgoing: &mut Vec<(String, String)>,+ outgoing: &mut crate::RequestHeaders,config: &FetchConfig,request: &Request,request_url: &Url,@@ -354,8 +381,8 @@ fn append_browser_subresource_headers(BrowserRequestMetadata::TextTrack => "text/vtt,*/*;q=0.1",_ => "*/*",};- append_header_if_missing(outgoing, "Accept", accept.to_owned());- append_header_if_missing(+ append_encoded_header_if_missing(outgoing, "Accept", accept.to_owned());+ append_encoded_header_if_missing(outgoing,"Accept-Language",request@@ -363,12 +390,12 @@ fn append_browser_subresource_headers(.accept_language().to_owned(),);- append_header_if_missing(+ append_encoded_header_if_missing(outgoing,"Sec-Fetch-Site",request_sec_fetch_site(request, request_url),);- append_header_if_missing(+ append_encoded_header_if_missing(outgoing,"Sec-Fetch-Mode",request.request_mode.as_ref().to_owned(),@@ -390,7 +417,7 @@ fn append_browser_subresource_headers(| BrowserRequestMetadata::Ping| BrowserRequestMetadata::Xhr => "empty",};- append_header_if_missing(outgoing, "Sec-Fetch-Dest", destination.to_owned());+ append_encoded_header_if_missing(outgoing, "Sec-Fetch-Dest", destination.to_owned());append_browser_client_hints(outgoing, request.browser_identity(config));}}@@ -413,19 +440,19 @@ fn request_origin_header_value(request: &Request, request_url: &Url) -> Option<S}fn append_browser_client_hints(- outgoing: &mut Vec<(String, String)>,+ outgoing: &mut crate::RequestHeaders,identity: &moli_browser_profile::BrowserIdentityProfile,) {let Some(sec_ch_ua) = identity.sec_ch_ua_value() else {return;};- append_header_if_missing(outgoing, "Sec-CH-UA", sec_ch_ua);- append_header_if_missing(+ append_encoded_header_if_missing(outgoing, "Sec-CH-UA", sec_ch_ua);+ append_encoded_header_if_missing(outgoing,"Sec-CH-UA-Mobile",if identity.mobile() { "?1" } else { "?0" }.to_owned(),);- append_header_if_missing(+ append_encoded_header_if_missing(outgoing,"Sec-CH-UA-Platform",format!("\"{}\"", identity.platform()),@@ -433,7 +460,7 @@ fn append_browser_client_hints(}fn append_browser_storage_access_header(- outgoing: &mut Vec<(String, String)>,+ outgoing: &mut crate::RequestHeaders,request: &Request,request_url: &Url,) {@@ -457,7 +484,7 @@ fn append_browser_storage_access_header(// Moli does not currently expose a third-party-cookie blocking mode,// so its network cookie policy has the same effective state. If such a// blocker is added, this value must be derived from that policy instead.- append_header_if_missing(outgoing, "Sec-Fetch-Storage-Access", "active".to_owned());+ append_encoded_header_if_missing(outgoing, "Sec-Fetch-Storage-Access", "active".to_owned());}fn request_sec_fetch_site(request: &Request, request_url: &Url) -> String {@@ -653,7 +680,7 @@ pub(crate) fn configure_easy(let mut headers = RequestHeaderList::default();let mut outgoing_headers =- outgoing_request_headers_for_url(config, request, request_url, cookie_header);+ outgoing_request_header_bytes_for_url(config, request, request_url, cookie_header);// A 407 can come from a transparent proxy even when no explicit proxy// route is configured. In that case the connected endpoint receives the// challenge response as a normal request header. Explicit proxies use the@@ -662,31 +689,37 @@ pub(crate) fn configure_easy(&& let Some(authorization) = proxy_header_authorization(request)&& !header_present(&outgoing_headers, "proxy-authorization"){- outgoing_headers.push(("Proxy-Authorization".to_owned(), authorization));+ outgoing_headers.push(("Proxy-Authorization".to_owned(), authorization.into_bytes()));}if let Some(web_bot_auth) = config.web_bot_auth() {+ let mut signed_headers = outgoing_headers.to_byte_strings();web_bot_auth- .append_request_headers(&mut outgoing_headers, &request.method, request_url)+ .append_request_headers(&mut signed_headers, &request.method, request_url).with_context(|| anyhow!("failed to sign web bot auth request for {request_url}"))?;+ outgoing_headers = crate::RequestHeaders::from_byte_strings(&signed_headers)?;}let mut has_content_type_header = false;- for (name, value) in &outgoing_headers {+ for (name, value) in outgoing_headers.iter() {has_content_type_header |= name.eq_ignore_ascii_case("content-type");- let header_line = if value.is_empty() {- format!("{name}:")- } else {- format!("{name}: {value}")- };+ let mut header_line = format!("{name}:").into_bytes();+ if !value.is_empty() {+ header_line.push(b' ');+ header_line.extend_from_slice(value);+ }headers.append(&header_line).context("failed to build request header")?;}if let Some(validation_headers) = validation_headers {- for (name, value) in validation_headers {+ // Validators were read from response headers and are ByteStrings.+ let validation_headers = crate::RequestHeaders::from_byte_strings(&validation_headers)?;+ for (name, value) in validation_headers.iter() {has_content_type_header |= name.eq_ignore_ascii_case("content-type");+ let mut line = format!("{name}: ").into_bytes();+ line.extend_from_slice(value);headers- .append(&format!("{name}: {value}"))+ .append(&line).context("failed to build cache validation request header")?;}}@@ -698,7 +731,7 @@ pub(crate) fn configure_easy(// for POST bodies and bodyless PUT requests. Browser requests only send Content-Type when// BodyInit or caller headers produce one, so suppress curl's transport default.headers- .append("Content-Type:")+ .append(b"Content-Type:").context("failed to suppress curl default content-type")?;}@@ -742,7 +775,7 @@ pub(crate) fn configure_easy(}crate::runtime::FetchTransferHandler::set_request_headers(easy, headers)?;- Ok(outgoing_headers)+ Ok(outgoing_headers.to_byte_strings())}fn configure_proxy_headers<H: Handler>(diff --git a/moli-fetch/src/blocking/request_headers.rs b/moli-fetch/src/blocking/request_headers.rsindex 0c4ca295a..cbed29086 100644--- a/moli-fetch/src/blocking/request_headers.rs+++ b/moli-fetch/src/blocking/request_headers.rs@@ -3,7 +3,7 @@ use std::ffi::CString;use anyhow::{Context, Result};// curl::easy::List only accepts UTF-8 strings. Own a libcurl list here so the-// transport can send ByteString header values without re-encoding them.+// transport can send already encoded header values without re-encoding them.#[derive(Default)]pub(crate) struct RequestHeaderList {raw: *mut curl_sys::curl_slist,@@ -14,18 +14,8 @@ pub(crate) struct RequestHeaderList {unsafe impl Send for RequestHeaderList {}impl RequestHeaderList {- pub(crate) fn append(&mut self, line: &str) -> Result<()> {- let line = if line.is_ascii() {- CString::new(line.as_bytes())- } else {- let bytes = line- .chars()- .map(|ch| u8::try_from(u32::from(ch)))- .collect::<std::result::Result<Vec<_>, _>>()- .context("HTTP request header contains a non-byte character")?;- CString::new(bytes)- }- .context("HTTP request header contains NUL")?;+ pub(crate) fn append(&mut self, line: &[u8]) -> Result<()> {+ let line = CString::new(line).context("HTTP request header contains NUL")?;// SAFETY: self.raw is null or a live list owned by self. libcurl copies// the NUL-terminated string; failure leaves the existing list intact.let raw = unsafe { curl_sys::curl_slist_append(self.raw, line.as_ptr()) };@@ -54,11 +44,10 @@ mod tests {use super::*;#[test]- fn rejects_non_byte_characters_and_nul() {+ fn rejects_nul_without_discarding_the_list() {let mut headers = RequestHeaderList::default();- headers.append("X-Valid: \u{ff}").unwrap();- assert!(headers.append("X-Invalid: \u{100}").is_err());- assert!(headers.append("X-Invalid: before\0after").is_err());- headers.append("X-Valid: after").unwrap();+ headers.append(b"X-Valid: \xff").unwrap();+ assert!(headers.append(b"X-Invalid: before\0after").is_err());+ headers.append(b"X-Valid: after").unwrap();}}diff --git a/moli-fetch/src/client_hints.rs b/moli-fetch/src/client_hints.rsindex dbe16b752..7ef173957 100644--- a/moli-fetch/src/client_hints.rs+++ b/moli-fetch/src/client_hints.rs@@ -222,7 +222,7 @@ pub(crate) fn prepare_client_hint_request(};effective_request.request_headers- .push((hint.header_name().to_owned(), value));+ .push((hint.header_name().to_owned(), value.into_bytes()));sent_hints.insert(hint);}}@@ -255,8 +255,9 @@ fn configured_client_hint_names(config: &FetchConfig, request: &Request) -> BTreconfig.default_request_headers().iter()- .chain(&request.request_headers)- .filter_map(|(name, _)| ClientHint::from_header_name(name))+ .map(|(name, _)| name)+ .chain(request.request_headers.iter().map(|(name, _)| name))+ .filter_map(|name| ClientHint::from_header_name(name)).collect()}@@ -335,10 +336,13 @@ mod tests {let second =prepare_client_hint_request(&preferences, &restarts, &config, &request, &request.url);assert!(second.request.request_headers.iter().any(|(name, value)| {- name.eq_ignore_ascii_case("sec-ch-ua-arch") && value == "\"x86\""+ name.eq_ignore_ascii_case("sec-ch-ua-arch") && value == b"\"x86\""}));assert!(second.request.request_headers.iter().any(|(name, value)| {- name.eq_ignore_ascii_case("sec-ch-ua-full-version-list") && value.contains("145.0.0.0")+ name.eq_ignore_ascii_case("sec-ch-ua-full-version-list")+ && value+ .windows(b"145.0.0.0".len())+ .any(|part| part == b"145.0.0.0")}));assert_eq!(seconddiff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rsindex b66e97609..a5a3f03b1 100644--- a/moli-fetch/src/lib.rs+++ b/moli-fetch/src/lib.rs@@ -19,6 +19,7 @@ mod proxy;mod proxy_connect;mod referrer_policy;mod request;+mod request_headers;mod request_policy;mod response;mod runtime;@@ -67,6 +68,7 @@ pub use request::{RequestPriorityHints, RequestRedirectMode, RequestResourceType, ResourceLoadPriority,ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, SubresourceRequestMetadata,};+pub use request_headers::RequestHeaders;pub use request_policy::{is_bad_port, should_request_be_blocked_due_to_bad_port};pub use response::{NegotiatedHttpVersion, NetworkRequestExtraInfo, NetworkResponseExtraInfo, RawResponse,diff --git a/moli-fetch/src/request.rs b/moli-fetch/src/request.rsindex 7ae14e4bb..0cb46d374 100644--- a/moli-fetch/src/request.rs+++ b/moli-fetch/src/request.rs@@ -29,7 +29,8 @@ pub struct Request {pub url: Url,pub method: String,pub body: Option<Vec<u8>>,- pub request_headers: Vec<(String, String)>,+ /// Wire bytes, with the source encoding resolved before header merging.+ pub request_headers: crate::RequestHeaders,cache_mode: RequestCacheMode,pub resource_type: RequestResourceType,subresource_request_metadata: Option<SubresourceRequestMetadata>,@@ -385,7 +386,7 @@ impl Request {url,method: "GET".to_owned(),body: None,- request_headers: vec![],+ request_headers: crate::RequestHeaders::default(),cache_mode: RequestCacheMode::Default,resource_type: RequestResourceType::Raw,subresource_request_metadata: None,@@ -416,7 +417,7 @@ impl Request {url,method: "GET".to_owned(),body: None,- request_headers: vec![],+ request_headers: crate::RequestHeaders::default(),cache_mode: RequestCacheMode::Default,resource_type: RequestResourceType::Raw,subresource_request_metadata: None,@@ -441,11 +442,13 @@ impl Request {}}+ /// String header values use UTF-8. To supply HTTP bytes or WebIDL+ /// ByteStrings, pass an explicitly constructed `RequestHeaders` instead.pub fn new(method: &str,raw_url: &str,body: Option<String>,- request_headers: Vec<(String, String)>,+ request_headers: impl Into<crate::RequestHeaders>,) -> Result<Self> {Self::new_bytes(method,@@ -461,7 +464,7 @@ impl Request {method: &str,raw_url: &str,body: Option<Vec<u8>>,- request_headers: Vec<(String, String)>,+ request_headers: impl Into<crate::RequestHeaders>,) -> Result<Self> {let url = Url::parse(raw_url).with_context(|| anyhow!("failed to parse request url `{raw_url}`"))?;@@ -472,13 +475,13 @@ impl Request {method: &str,url: Url,body: Option<Vec<u8>>,- request_headers: Vec<(String, String)>,+ request_headers: impl Into<crate::RequestHeaders>,) -> Self {Self {url,method: method.to_owned(),body,- request_headers,+ request_headers: request_headers.into(),cache_mode: RequestCacheMode::Default,resource_type: RequestResourceType::Raw,subresource_request_metadata: None,@@ -687,7 +690,7 @@ impl Request {method: &str,raw_url: &str,body: Option<Vec<u8>>,- headers: Vec<(String, String)>,+ headers: impl Into<crate::RequestHeaders>,origin: WebOrigin,) -> Result<Self> {Ok(Self::new_bytes(method, raw_url, body, headers)?.with_request_origin(origin))@@ -697,7 +700,7 @@ impl Request {method: &str,url: Url,body: Option<Vec<u8>>,- headers: Vec<(String, String)>,+ headers: impl Into<crate::RequestHeaders>,origin: WebOrigin,) -> Self {Self::new_http_with_url(method, url, body, headers).with_request_origin(origin)diff --git a/moli-fetch/src/request_headers.rs b/moli-fetch/src/request_headers.rsnew file mode 100644index 000000000..ebc16615b--- /dev/null+++ b/moli-fetch/src/request_headers.rs@@ -0,0 +1,82 @@+use anyhow::{Context, Result};++/// Encoded request header values. Ordinary Rust strings use UTF-8; WebIDL+/// ByteStrings must enter through `from_byte_strings` before headers are merged.+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]+pub struct RequestHeaders(Vec<(String, Vec<u8>)>);++impl RequestHeaders {+ pub fn from_bytes(headers: Vec<(String, Vec<u8>)>) -> Self {+ Self(headers)+ }++ pub fn from_byte_strings(headers: &[(String, String)]) -> Result<Self> {+ headers+ .iter()+ .map(|(name, value)| {+ let bytes = value+ .chars()+ .map(|ch| u8::try_from(u32::from(ch)))+ .collect::<std::result::Result<Vec<_>, _>>()+ .context("HTTP header ByteString contains a non-byte character")?;+ Ok((name.clone(), bytes))+ })+ .collect::<Result<Vec<_>>>()+ .map(Self)+ }++ /// A lossless string projection for Fetch and network observation surfaces.+ pub fn to_byte_strings(&self) -> Vec<(String, String)> {+ self.0+ .iter()+ .map(|(name, value)| {+ (+ name.clone(),+ crate::headers::decode_http_header_bytes(value).into_owned(),+ )+ })+ .collect()+ }+}++impl From<Vec<(String, String)>> for RequestHeaders {+ fn from(headers: Vec<(String, String)>) -> Self {+ Self(+ headers+ .into_iter()+ .map(|(name, value)| (name, value.into_bytes()))+ .collect(),+ )+ }+}++impl std::ops::Deref for RequestHeaders {+ type Target = Vec<(String, Vec<u8>)>;++ fn deref(&self) -> &Self::Target {+ &self.0+ }+}++impl std::ops::DerefMut for RequestHeaders {+ fn deref_mut(&mut self) -> &mut Self::Target {+ &mut self.0+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn string_encodings_are_explicit() {+ let headers = vec![("X-Test".into(), "é".into())];+ assert_eq!(RequestHeaders::from(headers.clone())[0].1, [0xc3, 0xa9]);+ assert_eq!(+ RequestHeaders::from_byte_strings(&headers).unwrap()[0].1,+ [0xe9]+ );+ assert!(RequestHeaders::from_byte_strings(&[("X-Test".into(), "中".into())]).is_err());+ assert!(RequestHeaders::from_byte_strings(&[("X-Test".into(), "\u{100}".into())]).is_err());+ }+}diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex 2eff43475..e9989e518 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -3709,7 +3709,7 @@ fn network_fetch_failure_for_request(current_url.clone(),request.method.clone(),request.body.clone(),- request.request_headers.clone(),+ request.request_headers.to_byte_strings(),redirect_chain.to_vec(),),)@@ -3718,7 +3718,7 @@ fn network_fetch_failure_for_request(#[cfg(test)]fn request_panics_for_testing(request: &Request) -> bool {request.request_headers.iter().any(|(name, value)| {- name.eq_ignore_ascii_case("x-moli-test-panic") && value == "runtime-worker"+ name.eq_ignore_ascii_case("x-moli-test-panic") && value == b"runtime-worker"})}diff --git a/moli-fetch/src/tests/cookie_store.rs b/moli-fetch/src/tests/cookie_store.rsindex e4c708116..7ee78e7e3 100644--- a/moli-fetch/src/tests/cookie_store.rs+++ b/moli-fetch/src/tests/cookie_store.rs@@ -1008,7 +1008,8 @@ fn explicit_navigation_headers_override_browser_defaults() {("Sec-Fetch-Site".to_owned(), "cross-site".to_owned()),("Sec-Fetch-User".to_owned(), "?0".to_owned()),("Cache-Control".to_owned(), "no-cache".to_owned()),- ];+ ]+ .into();let headers = outgoing_request_headers(&config, &request, None);diff --git a/moli-fetch/src/tests/header_bytes.rs b/moli-fetch/src/tests/header_bytes.rsindex 35b53ec91..c07f00fca 100644--- a/moli-fetch/src/tests/header_bytes.rs+++ b/moli-fetch/src/tests/header_bytes.rs@@ -69,11 +69,18 @@ async fn http_header_bytes_survive_all_transports_and_observation() -> Result<()let listener = TcpListener::bind("127.0.0.1:0").await?;let url = format!("http://{}/bytes", listener.local_addr()?);let server = tokio::spawn(serve_responses(listener, vec![response(&headers); 3]));- let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ let mut config = FetchConfig::default();+ config.set_default_request_headers(vec![("X-Config".into(), "é中".into())]);+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());for mode in ["buffered", "html", "raw"] {let recorder = NetworkObservationRecorder::default();- let request = Request::new("GET", &url, None, vec![("X-Bytes".into(), value.clone())])?- .with_network_observation_recorder(recorder.clone());+ let request = Request::new(+ "GET",+ &url,+ None,+ crate::RequestHeaders::from_bytes(vec![("X-Bytes".into(), bytes.clone())]),+ )?+ .with_network_observation_recorder(recorder.clone());let (head, body) = fetch_in_mode(&client, request, mode).await?;// The HTML stream intentionally decodes the body as text, independently// of header ByteStrings. The raw and buffered paths preserve its bytes.@@ -119,6 +126,11 @@ async fn http_header_bytes_survive_all_transports_and_observation() -> Result<()}let expected_line = [b"X-Bytes: ".as_slice(), bytes.as_slice(), b"\r\n"].concat();for request in server.await?? {+ assert!(+ request+ .windows("X-Config: é中\r\n".len())+ .any(|part| part == "X-Config: é中\r\n".as_bytes())+ );assert!(request.windows(expected_line.len())@@ -129,6 +141,27 @@ async fn http_header_bytes_survive_all_transports_and_observation() -> Result<()Ok(())}+#[tokio::test]+async fn ordinary_request_header_strings_keep_utf8_encoding() -> Result<()> {+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let url = format!("http://{}/utf8", listener.local_addr()?);+ let server = tokio::spawn(serve_responses(listener, vec![response(b""); 3]));+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ for mode in ["buffered", "html", "raw"] {+ let request = Request::new("GET", &url, None, vec![("X-Utf8".into(), "é中".into())])?;+ fetch_in_mode(&client, request, mode).await?;+ }+ for request in server.await?? {+ assert!(+ request+ .windows("X-Utf8: é中\r\n".len())+ .any(|part| part == "X-Utf8: é中\r\n".as_bytes())+ );+ }+ assert!(client.shutdown().is_clean());+ Ok(())+}+#[tokio::test]async fn http_header_bytes_survive_redirect_handle_reuse() -> Result<()> {let listener = TcpListener::bind("127.0.0.1:0").await?;@@ -139,7 +172,9 @@ async fn http_header_bytes_survive_redirect_handle_reuse() -> Result<()> {vec![redirect, response(b"X-Bytes: \xff\r\n")],));let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());- let request = Request::new("GET", &url, None, vec![("X-Bytes".into(), "\u{ff}".into())])?;+ let mut headers = crate::RequestHeaders::from(vec![("X-Utf8".into(), "é中".into())]);+ headers.push(("X-Bytes".into(), vec![0xff]));+ let request = Request::new("GET", &url, None, headers)?;let (head, body) = fetch_in_mode(&client, request, "raw").await?;assert!(head.redirected);assert_eq!(head.final_url.path(), "/final");@@ -148,6 +183,11 @@ async fn http_header_bytes_survive_redirect_handle_reuse() -> Result<()> {let requests = server.await??;assert_eq!(requests.len(), 2);for request in requests {+ assert!(+ request+ .windows("X-Utf8: é中\r\n".len())+ .any(|part| part == "X-Utf8: é中\r\n".as_bytes())+ );assert!(request.windows(b"X-Bytes: \xff\r\n".len())diff --git a/moli-protocol/src/conn/downloads.rs b/moli-protocol/src/conn/downloads.rsindex 8dc9d6e80..bd4484c0f 100644--- a/moli-protocol/src/conn/downloads.rs+++ b/moli-protocol/src/conn/downloads.rs@@ -411,7 +411,7 @@ impl CdpConnection {let mut request = Request::get(&activation.url).map_err(|error| format!("invalid download url: {error}"))?;- request.request_headers = owner.request_headers;+ request.request_headers = owner.request_headers.into();request = request.with_top_level_navigation_cookie_context().with_page_network_policy()diff --git a/moli-protocol/src/domains/fetch/commands.rs b/moli-protocol/src/domains/fetch/commands.rsindex d7dd9d86f..66cbc1998 100644--- a/moli-protocol/src/domains/fetch/commands.rs+++ b/moli-protocol/src/domains/fetch/commands.rs@@ -225,6 +225,18 @@ fn start_devtools_continue_intercepted_request_command(&request_id,) {let mut pending = pending;+ // Resolve protocol Unicode strings before mixing them with the saved+ // Fetch/XHR ByteStrings, including across multiple interception stages.+ let command_headers = command.headers.clone().map(|headers| {+ if matches!(+ pending.resource_type,+ SubresourceResourceType::Fetch | SubresourceResourceType::Xhr+ ) {+ moli_fetch::RequestHeaders::from(headers).to_byte_strings()+ } else {+ headers+ }+ });if pending.request_stage_pause_state().is_some_and(|chain| !chain.remaining_sessions.is_empty())@@ -233,7 +245,7 @@ fn start_devtools_continue_intercepted_request_command(parsed_url,command.method.clone(),command.post_data.clone(),- command.headers.clone(),+ command_headers,);let Some(event) =super::subresource::next_chained_subresource_request_pause_event(conn, pending)@@ -257,7 +269,7 @@ fn start_devtools_continue_intercepted_request_command(} else {chain_body};- let headers = command.headers.clone().or(chain_headers);+ let headers = command_headers.or(chain_headers);pending.request_stage_chain = None;let configured_response_stage = conn.target_fetch_subresource_interception_snapshot_for_owner(owner)diff --git a/moli-protocol/src/domains/network/tests/subresource.rs b/moli-protocol/src/domains/network/tests/subresource.rsindex acd860eae..8141c3883 100644--- a/moli-protocol/src/domains/network/tests/subresource.rs+++ b/moli-protocol/src/domains/network/tests/subresource.rs@@ -3597,3 +3597,164 @@ xhr.send();server.abort();}++#[tokio::test(flavor = "multi_thread")]+async fn cdp_utf8_headers_coexist_with_fetch_and_xhr_byte_strings() {+ const PROBE: &str = r#"+ const headers = { 'x-web': '\u00e9\u00ff', 'x-override': '\u00e9' };+ function xhrProbe(async) {+ return new Promise((resolve, reject) => {+ const xhr = new XMLHttpRequest();+ xhr.open('GET', '/echo', async);+ for (const [name, value] of Object.entries(headers)) xhr.setRequestHeader(name, value);+ xhr.onload = () => resolve(JSON.parse(xhr.responseText));+ xhr.onerror = () => reject(new Error('XHR failed'));+ xhr.send();+ });+ }+ const results = await Promise.all([+ fetch('/redirect', { headers }).then(response => response.json()),+ xhrProbe(true),+ xhrProbe(false)+ ]);+ "#;+ async fn echo(headers: axum::http::HeaderMap) -> impl IntoResponse {+ axum::Json(json!({+ "cdp": headers.get("x-cdp").map(|value| value.as_bytes()),+ "web": headers.get("x-web").map(|value| value.as_bytes()),+ "override": headers.get("x-override").map(|value| value.as_bytes()),+ }))+ }+ let worker_script = format!("(async () => {{ {PROBE} postMessage(results); }})()");+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let server = tokio::spawn(async move {+ axum::serve(+ listener,+ Router::new()+ .route("/page", get(plain_page))+ .route("/echo", get(echo))+ .route(+ "/redirect",+ get(|| async { axum::response::Redirect::temporary("/echo") }),+ )+ .route(+ "/worker.js",+ get(move || async move {+ ([(CONTENT_TYPE.as_str(), "text/javascript")], worker_script)+ }),+ ),+ )+ .await+ .unwrap();+ });+ let mut ctx = TestContext::new();+ let mut bc = BrowserContext::new("BID-1".into());+ bc.set_active_target_id("TID-1");+ bc.attach_active_session("SID-1");+ ctx.conn.install_browser_context_fixture_for_test(bc);+ ctx.process_async(json!({+ "id": 0, "method": "Network.enable", "sessionId": "SID-1"+ }))+ .await;+ ctx.expect_result(0, json!({}), Some("SID-1"));+ ctx.process_async(json!({+ "id": 1, "method": "Network.setExtraHTTPHeaders", "sessionId": "SID-1",+ "params": { "headers": { "x-cdp": "é中", "X-Override": "中" } }+ }))+ .await;+ ctx.expect_result(1, json!({}), Some("SID-1"));+ ctx.process_async(json!({+ "id": 2, "method": "Page.navigate", "sessionId": "SID-1",+ "params": { "url": format!("http://{addr}/page") }+ }))+ .await;+ ctx.process_async(json!({+ "id": 3, "method": "Runtime.evaluate", "sessionId": "SID-1",+ "params": {+ "expression": format!(r#"(async () => {{+ {PROBE}+ const workerResults = await new Promise((resolve, reject) => {{+ const worker = new Worker('/worker.js');+ worker.onmessage = event => {{ worker.terminate(); resolve(event.data); }};+ worker.onerror = () => reject(new Error('Worker failed'));+ }});+ return JSON.stringify(results.concat(workerResults));+ }})()"#),+ "awaitPromise": true, "returnByValue": true+ }+ }))+ .await;+ wait_until_messages(+ &mut ctx,+ Some("SID-1"),+ "mixed header encoding probe",+ |messages| messages.iter().any(|message| message["id"] == json!(3)),+ )+ .await;+ let expected =+ json!({ "cdp": [0xc3, 0xa9, 0xe4, 0xb8, 0xad], "web": [0xe9, 0xff], "override": [0xe9] });+ ctx.expect_result(3, json!({ "result": { "type": "string", "value": serde_json::to_string(&vec![expected; 6]).unwrap() } }), Some("SID-1"));+ ctx.process_async(json!({+ "id": 4, "method": "Fetch.enable", "sessionId": "SID-1",+ "params": { "patterns": [{ "urlPattern": "*/echo", "requestStage": "Request" }] }+ }))+ .await;+ ctx.expect_result(4, json!({}), Some("SID-1"));+ for replace_headers in [false, true] {+ ctx.sent.clear();+ ctx.process_async(json!({+ "id": 5, "method": "Runtime.evaluate", "sessionId": "SID-1",+ "params": {+ "expression": r#"fetch('/echo', {headers: {'x-web': '\u00e9\u00ff', 'x-override': '\u00e9'}}).then(response => response.text())"#,+ "awaitPromise": true, "returnByValue": true+ }+ })).await;+ wait_until_messages(+ &mut ctx,+ Some("SID-1"),+ "byte header request pause",+ |messages| {+ messages+ .iter()+ .any(|message| message["method"] == "Fetch.requestPaused")+ },+ )+ .await;+ let messages = ctx.take_all();+ let paused = messages+ .iter()+ .find(|message| message["method"] == "Fetch.requestPaused")+ .unwrap();+ let mut params = json!({"requestId": paused["params"]["requestId"]});+ if replace_headers {+ params["headers"] = json!([+ {"name": "x-cdp", "value": "é中"},+ {"name": "x-web", "value": "é中"},+ {"name": "x-override", "value": "é"}+ ]);+ }+ ctx.process_async(json!({+ "id": 6, "method": "Fetch.continueRequest", "sessionId": "SID-1", "params": params+ }))+ .await;+ wait_until_messages(+ &mut ctx,+ Some("SID-1"),+ "continued byte header request",+ |messages| messages.iter().any(|message| message["id"] == json!(5)),+ )+ .await;+ let expected = if replace_headers {+ json!({"cdp": [0xc3, 0xa9, 0xe4, 0xb8, 0xad], "web": [0xc3, 0xa9, 0xe4, 0xb8, 0xad], "override": [0xc3, 0xa9]})+ } else {+ json!({"cdp": [0xc3, 0xa9, 0xe4, 0xb8, 0xad], "web": [0xe9, 0xff], "override": [0xe9]})+ };+ ctx.expect_result(+ 5,+ json!({"result": {"type": "string", "value": expected.to_string()}}),+ Some("SID-1"),+ );+ }+ server.abort();+}diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rsindex 5dd1589de..bd9ce2a8a 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs@@ -208,7 +208,7 @@ impl JsContextHost {.with_network_partition_key(network_partition_key);let request_url = request.url.as_str().to_owned();let request_method = request.method.clone();- let request_headers = request.request_headers.clone();+ let request_headers = request.request_headers.to_byte_strings();if let Some(client_id) = service_worker_client_id&& let Some(response) = browser_context_runtime.fetch_service_worker_child_main_resource_for_reserved_client(diff --git a/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs b/moli-renderer-v8/src/native_bridge/context_host/service_workers.rsindex 59243ea4e..6ea4e8be6 100644--- a/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs@@ -1693,7 +1693,7 @@ impl JsContextHost {client_id: ServiceWorkerClientId,url: Url,method: String,- headers: Vec<(String, String)>,+ headers: impl Into<moli_fetch::RequestHeaders>,body: Option<Vec<u8>>,destination: ServiceWorkerRequestDestination,request_mode: moli_fetch::RequestMode,@@ -1707,7 +1707,7 @@ impl JsContextHost {resulting_client_id: None,url,method,- headers,+ headers: headers.into().to_byte_strings(),body,destination,request_mode,diff --git a/moli-renderer-v8/src/network/backend/memory_cache.rs b/moli-renderer-v8/src/network/backend/memory_cache.rsindex ec4bfaff9..684637283 100644--- a/moli-renderer-v8/src/network/backend/memory_cache.rs+++ b/moli-renderer-v8/src/network/backend/memory_cache.rs@@ -616,7 +616,7 @@ pub(in crate::network) fn script_text_cache_key(request: &Request) -> ScriptTextintegrity: request.subresource_request_metadata().and_then(|metadata| metadata.integrity.clone()),- request_headers: request.request_headers.clone(),+ request_headers: request.request_headers.to_byte_strings(),}}diff --git a/moli-renderer-v8/src/network/context/tests.rs b/moli-renderer-v8/src/network/context/tests.rsindex 6cc2e6862..dea733c0e 100644--- a/moli-renderer-v8/src/network/context/tests.rs+++ b/moli-renderer-v8/src/network/context/tests.rs@@ -245,12 +245,12 @@ async fn registry_replaces_future_transport_without_rebinding_existing_loads() {).expect("replacement request policy");assert_eq!(- existing_request.request_headers,+ existing_request.request_headers.to_byte_strings(),vec![("x-request-policy".to_owned(), "original".to_owned())],"an in-flight request keeps its prepared configuration");assert_eq!(- replacement_request.request_headers,+ replacement_request.request_headers.to_byte_strings(),vec![("x-request-policy".to_owned(), "replacement".to_owned())],"new requests use the replacement configuration");diff --git a/moli-renderer-v8/src/network/policy.rs b/moli-renderer-v8/src/network/policy.rsindex 6237e2dc7..ac93404fb 100644--- a/moli-renderer-v8/src/network/policy.rs+++ b/moli-renderer-v8/src/network/policy.rs@@ -479,20 +479,20 @@ impl PageNetworkPolicySnapshot {fn merge_page_network_policy_headers(context_headers: &[(Box<str>, Box<str>)],- request_headers: &[(String, String)],-) -> Vec<(String, String)> {- let mut merged = IndexMap::<String, (String, String)>::new();+ request_headers: &moli_fetch::RequestHeaders,+) -> moli_fetch::RequestHeaders {+ let mut merged = IndexMap::<String, (String, Vec<u8>)>::new();for (name, value) in context_headers {merged.entry(header_name_key(name))- .or_insert_with(|| (name.to_string(), value.to_string()));+ .or_insert_with(|| (name.to_string(), value.as_bytes().to_vec()));}- for (name, value) in request_headers {+ for (name, value) in request_headers.iter() {let key = header_name_key(name);merged.shift_remove(&key);merged.insert(key, (name.clone(), value.clone()));}- merged.into_values().collect()+ moli_fetch::RequestHeaders::from_bytes(merged.into_values().collect())}fn header_name_key(name: &str) -> String {@@ -532,7 +532,7 @@ mod tests {).unwrap();assert_eq!(- original.request_headers,+ original.request_headers.to_byte_strings(),vec![("x-owner".to_owned(), "first".to_owned())]);}@@ -589,7 +589,7 @@ mod tests {).unwrap();assert_eq!(- request.request_headers,+ request.request_headers.to_byte_strings(),vec![("x-policy-revision".to_owned(), "one".to_owned())]);assert!(policy.snapshot().network_offline());@@ -632,7 +632,7 @@ mod tests {).unwrap();assert_eq!(- request.request_headers,+ request.request_headers.to_byte_strings(),vec![("x-policy-revision".to_owned(), "one".to_owned())],"request configuration must remain the one captured at registration");diff --git a/moli-renderer-v8/src/network/request_client/tests.rs b/moli-renderer-v8/src/network/request_client/tests.rsindex 1791cf44b..d044fc321 100644--- a/moli-renderer-v8/src/network/request_client/tests.rs+++ b/moli-renderer-v8/src/network/request_client/tests.rs@@ -370,7 +370,8 @@ fn merge_loader_network_policy_headers_uses_header_name_keys_and_request_order()request.request_headers = vec![("x-test".to_owned(), "request".to_owned()),("X-New".to_owned(), "new".to_owned()),- ];+ ]+ .into();let merged = policy.snapshot().apply_to_request(request)@@ -378,7 +379,7 @@ fn merge_loader_network_policy_headers_uses_header_name_keys_and_request_order().request_headers;assert_eq!(- merged,+ merged.to_byte_strings(),vec![("Accept".to_owned(), "text/html".to_owned()),("x-test".to_owned(), "request".to_owned()),diff --git a/moli-renderer-v8/src/network_host.rs b/moli-renderer-v8/src/network_host.rsindex 40ea8bf66..721aa70f0 100644--- a/moli-renderer-v8/src/network_host.rs+++ b/moli-renderer-v8/src/network_host.rs@@ -167,6 +167,7 @@ pub(crate) use self::stylesheet_subresource::{pub(crate) use self::text_track::{TextTrackResourceFetchStart, start_text_track_resource_fetch, text_track_response_result,};+pub(crate) use self::url_helpers::merge_byte_string_request_headers;pub(in crate::network_host) use self::url_helpers::merge_subresource_request_headers;pub(crate) use self::url_helpers::resolve_context_url;#[cfg(test)]diff --git a/moli-renderer-v8/src/network_host/async_fetch.rs b/moli-renderer-v8/src/network_host/async_fetch.rsindex f2b193365..2c28c2aa4 100644--- a/moli-renderer-v8/src/network_host/async_fetch.rs+++ b/moli-renderer-v8/src/network_host/async_fetch.rs@@ -23,7 +23,7 @@ pub(crate) async fn fetch_browser_subresource_with_preflight_and_network_metadatrequest: Request,cancel_handle: Option<FetchCancelHandle>,) -> Result<NetworkFetchResult<Response>, String> {- let preflight_request_headers = request.request_headers.clone();+ let preflight_request_headers = request.request_headers.to_byte_strings();fetch_browser_subresource_with_preflight_headers_and_observer(loader,request,@@ -218,7 +218,7 @@ impl ManualCorsRedirectState {});self.request.apply_redirect_status(redirect_status);self.request.url = next_url;- self.preflight_request_headers = self.request.request_headers.clone();+ self.preflight_request_headers = self.request.request_headers.to_byte_strings();Ok(ManualCorsRedirectTransition::FollowedRedirect)}diff --git a/moli-renderer-v8/src/network_host/csp_reports.rs b/moli-renderer-v8/src/network_host/csp_reports.rsindex 08a1feb3c..2114ac219 100644--- a/moli-renderer-v8/src/network_host/csp_reports.rs+++ b/moli-renderer-v8/src/network_host/csp_reports.rs@@ -340,7 +340,7 @@ fn dispatch_service_worker_content_security_policy_report(internal_id,request_url: request.url,request_method: request.method,- request_headers: request.request_headers,+ request_headers: request.request_headers.to_byte_strings(),request_body: request_body_text,response_status_text: None,skip_fetch_security_validation: true,@@ -388,7 +388,7 @@ fn spawn_content_security_policy_report_fetch(},request.url,request.method,- request.request_headers,+ request.request_headers.to_byte_strings(),request_body_text,);}@@ -407,7 +407,7 @@ fn report_subresource_fetch_info(url: request.url.clone(),websocket_socket_id: None,method: request.method.clone(),- request_headers: request.request_headers.clone(),+ request_headers: request.request_headers.to_byte_strings(),request_body: report_request_body_text(request),request_body_bytes: request.body.clone(),resource_type: SubresourceResourceType::CspReport,diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs b/moli-renderer-v8/src/network_host/fetch/bindings/paths.rsindex 6d0184a94..d13002a71 100644--- a/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs+++ b/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs@@ -204,7 +204,8 @@ pub(super) fn spawn_network_fetch(&prepared.method,prepared.resolved_url.clone(),prepared.body.clone(),- prepared.request_headers.clone(),+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)+ .expect("prepared Fetch/XHR headers are ByteStrings"),prepared.request_origin.clone(),).with_initiator_url(&prepared.document_url)diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/request.rs b/moli-renderer-v8/src/network_host/fetch/bindings/request.rsindex 5f40e78bc..60a60c2a0 100644--- a/moli-renderer-v8/src/network_host/fetch/bindings/request.rs+++ b/moli-renderer-v8/src/network_host/fetch/bindings/request.rs@@ -76,7 +76,7 @@ pub(super) fn prepare_window_fetch_request<'s>(request_headers.push(("Content-Type".to_owned(), String::new()));}let request_headers =- merge_subresource_request_headers(host.extra_http_headers(), &request_headers);+ merge_byte_string_request_headers(host.extra_http_headers(), &request_headers);let resolved_url = resolve_context_url(&base_url, &parsed.url, None)?;Ok(PreparedWindowFetchRequest {diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs b/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rsindex a1ca1ff50..a662529b1 100644--- a/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs+++ b/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs@@ -85,7 +85,8 @@ pub(super) fn dispatch_service_worker_fetch(client_id,prepared.resolved_url.clone(),prepared.method.clone(),- prepared.request_headers.clone(),+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)+ .expect("prepared Fetch/XHR headers are ByteStrings"),prepared.body.clone(),ServiceWorkerRequestDestination::Empty,prepared.request_mode,diff --git a/moli-renderer-v8/src/network_host/url_helpers.rs b/moli-renderer-v8/src/network_host/url_helpers.rsindex e68f564dc..3c7efa40e 100644--- a/moli-renderer-v8/src/network_host/url_helpers.rs+++ b/moli-renderer-v8/src/network_host/url_helpers.rs@@ -36,6 +36,16 @@ pub(in crate::network_host) fn merge_subresource_request_headers(merged.into_values().collect()}+/// Renderer Fetch/XHR state carries isomorphically decoded HTTP bytes. Convert+/// the page's ordinary UTF-8 strings before merging them with WebIDL values.+pub(crate) fn merge_byte_string_request_headers(+ context_headers: &[(String, String)],+ request_headers: &[(String, String)],+) -> Vec<(String, String)> {+ let context = moli_fetch::RequestHeaders::from(context_headers.to_vec()).to_byte_strings();+ merge_subresource_request_headers(&context, request_headers)+}+fn header_name_key(name: &str) -> String {HeaderName::from_bytes(name.as_bytes()).map(|name| name.as_str().to_owned())diff --git a/moli-renderer-v8/src/network_host/xhr/send.rs b/moli-renderer-v8/src/network_host/xhr/send.rsindex 1189b64b9..c69a63d3a 100644--- a/moli-renderer-v8/src/network_host/xhr/send.rs+++ b/moli-renderer-v8/src/network_host/xhr/send.rs@@ -287,7 +287,8 @@ fn send_synchronous_network_xhr(&prepared.method,prepared.resolved_url.as_str(),prepared.send_body.clone(),- prepared.request_headers.clone(),+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)+ .expect("prepared Fetch/XHR headers are ByteStrings"),).expect("xhr request url was already resolved").with_initiator_url(&prepared.document_url)diff --git a/moli-renderer-v8/src/network_host/xhr/send/paths.rs b/moli-renderer-v8/src/network_host/xhr/send/paths.rsindex 56a0698d3..be8dce53b 100644--- a/moli-renderer-v8/src/network_host/xhr/send/paths.rs+++ b/moli-renderer-v8/src/network_host/xhr/send/paths.rs@@ -105,7 +105,8 @@ pub(super) fn dispatch_service_worker_xhr(client_id,prepared.resolved_url.clone(),prepared.method.clone(),- prepared.request_headers.clone(),+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)+ .expect("prepared Fetch/XHR headers are ByteStrings"),prepared.send_body.clone(),ServiceWorkerRequestDestination::Empty,moli_fetch::RequestMode::Cors,@@ -301,7 +302,8 @@ pub(super) fn spawn_network_xhr_fetch(&prepared.method,prepared.resolved_url.as_str(),prepared.send_body.clone(),- prepared.request_headers.clone(),+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)+ .expect("prepared Fetch/XHR headers are ByteStrings"),).expect("xhr request url was already resolved").with_initiator_url(&prepared.document_url)diff --git a/moli-renderer-v8/src/network_host/xhr/send/request.rs b/moli-renderer-v8/src/network_host/xhr/send/request.rsindex bc426f130..129f55767 100644--- a/moli-renderer-v8/src/network_host/xhr/send/request.rs+++ b/moli-renderer-v8/src/network_host/xhr/send/request.rs@@ -208,7 +208,7 @@ fn xhr_request_headers(default_content_type,suppress_default_content_type,);- let merged = merge_subresource_request_headers(host.extra_http_headers(), &author_headers);+ let merged = merge_byte_string_request_headers(host.extra_http_headers(), &author_headers);(merged, author_headers)}diff --git a/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs b/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rsindex ef8a53b47..800f7a6d9 100644--- a/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs+++ b/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs@@ -204,7 +204,7 @@ impl RendererBrowserContextRuntime {resulting_client_id: Some(client_id),url: request.url.clone(),method: request.method.clone(),- headers: request.request_headers.clone(),+ headers: request.request_headers.to_byte_strings(),body: request.body.clone(),destination,request_mode: request.request_mode,@@ -275,7 +275,7 @@ impl RendererBrowserContextRuntime {resulting_client_id: None,url: request.url.clone(),method: request.method.clone(),- headers: request.request_headers.clone(),+ headers: request.request_headers.to_byte_strings(),body: request.body.clone(),destination,request_mode: request.request_mode,diff --git a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rsindex 74ad7a85a..ecae78a67 100644--- a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs+++ b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs@@ -1361,7 +1361,7 @@ mod tests {assert_eq!(request.method, "POST");assert_eq!(request.body, Some(body));assert_eq!(- request.request_headers,+ request.request_headers.to_byte_strings(),vec![("Content-Type".to_owned(),"application/x-www-form-urlencoded".to_owned(),diff --git a/moli-renderer-v8/src/script_vm/subresource_fetch.rs b/moli-renderer-v8/src/script_vm/subresource_fetch.rsindex ee93862e5..8a8ac06e3 100644--- a/moli-renderer-v8/src/script_vm/subresource_fetch.rs+++ b/moli-renderer-v8/src/script_vm/subresource_fetch.rs@@ -1409,11 +1409,19 @@ impl ScriptVm {// up the ambient Page loader here would silently rebind policy/backend// to a newer Document identity.let loader = pending.load.request_client();+ let encoded_headers = if matches!(+ pending.info.resource_type,+ SubresourceResourceType::Fetch | SubresourceResourceType::Xhr+ ) {+ moli_fetch::RequestHeaders::from_byte_strings(&request_headers)?+ } else {+ moli_fetch::RequestHeaders::from(request_headers.clone())+ };let mut request = moli_fetch::Request::new(&request_method,request_url.as_str(),request_body.clone(),- request_headers.clone(),+ encoded_headers,)?.with_initiator_url(&pending.info.document_url).with_request_origin(pending.request_origin.clone())@@ -1679,11 +1687,19 @@ impl ScriptVm {));}let loader = pending_fetch.load.request_client();+ let encoded_headers = if matches!(+ pending_fetch.info.resource_type,+ SubresourceResourceType::Fetch | SubresourceResourceType::Xhr+ ) {+ moli_fetch::RequestHeaders::from_byte_strings(&original_request_headers)?+ } else {+ moli_fetch::RequestHeaders::from(original_request_headers.clone())+ };let mut request = moli_fetch::Request::new(&request_method,request_url.as_str(),request_body.clone(),- original_request_headers.clone(),+ encoded_headers,)?.with_initiator_url(&pending_fetch.info.document_url).with_request_origin(pending_fetch.request_origin.clone())diff --git a/moli-renderer-v8/src/service_worker_runtime/events.rs b/moli-renderer-v8/src/service_worker_runtime/events.rsindex 30480cb1f..042a08819 100644--- a/moli-renderer-v8/src/service_worker_runtime/events.rs+++ b/moli-renderer-v8/src/service_worker_runtime/events.rs@@ -470,6 +470,7 @@ pub(crate) struct ServiceWorkerFetchRequest {pub(crate) resulting_client_id: Option<ServiceWorkerClientId>,pub(crate) url: Url,pub(crate) method: String,+ /// Isomorphically decoded wire bytes, ready for the worker's Fetch Headers.pub(crate) headers: Vec<(String, String)>,pub(crate) body: Option<Vec<u8>>,pub(crate) destination: ServiceWorkerRequestDestination,diff --git a/moli-renderer-v8/src/service_worker_runtime/service/event_completion.rs b/moli-renderer-v8/src/service_worker_runtime/service/event_completion.rsindex 900ea27a8..8bae2c0ef 100644--- a/moli-renderer-v8/src/service_worker_runtime/service/event_completion.rs+++ b/moli-renderer-v8/src/service_worker_runtime/service/event_completion.rs@@ -529,7 +529,7 @@ pub(super) fn service_worker_fetch_diagnostic_from_job_result(document_url: job.network_context.document_url.as_str().to_owned(),request_url: job.request.url.as_str().to_owned(),method: job.request.method.clone(),- request_headers: job.request.request_headers.clone(),+ request_headers: job.request.request_headers.to_byte_strings(),request_body: request_body_text(&job.request.body),destination: service_worker_fetch_diagnostic_destination(job.network_context.resource_type).to_owned(),diff --git a/moli-renderer-v8/src/service_worker_runtime/service/event_dispatch.rs b/moli-renderer-v8/src/service_worker_runtime/service/event_dispatch.rsindex 719c49292..578bec6cb 100644--- a/moli-renderer-v8/src/service_worker_runtime/service/event_dispatch.rs+++ b/moli-renderer-v8/src/service_worker_runtime/service/event_dispatch.rs@@ -40,10 +40,12 @@ fn navigation_preload_request_for_job() -> Result<moli_fetch::Request, String> {let mut headers = job.request.request_headers.clone();headers.retain(|(name, _)| !name.eq_ignore_ascii_case("service-worker-navigation-preload"));- headers.push((+ let preload_header = moli_fetch::RequestHeaders::from_byte_strings(&[("Service-Worker-Navigation-Preload".to_owned(),header_value.to_owned(),- ));+ )])+ .map_err(|error| error.to_string())?;+ headers.extend_from_slice(&preload_header);let mut request = super::fetch_settlement::service_worker_network_fallback_request_for_job(job);request.request_headers = headers;Ok(request@@ -247,7 +249,8 @@ impl ServiceWorkerRuntimeService {&request.method,request.url.clone(),request.body.clone(),- request.headers.clone(),+ moli_fetch::RequestHeaders::from_byte_strings(&request.headers)+ .expect("service worker request headers are ByteStrings"),origin,).with_initiator_url(&dispatch.network_context.document_url)diff --git a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rsindex 3827aaea6..7afa42c73 100644--- a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs+++ b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs@@ -55,7 +55,7 @@ fn fetch_request_for_job(job: &ServiceWorkerFetchJob) -> ServiceWorkerFetchRequeresulting_client_id: job.resulting_client_id,url: job.request.url.clone(),method: job.request.method.clone(),- headers: job.request.request_headers.clone(),+ headers: job.request.request_headers.to_byte_strings(),body: job.request.body.clone(),destination: job.destination,request_mode: job.request.request_mode,@@ -167,7 +167,7 @@ fn apply_service_worker_synthetic_redirect(negotiated_http_version: None,}).map_err(|error| error.to_string())?;- job.cors_preflight_request_headers = job.request.request_headers.clone();+ job.cors_preflight_request_headers = job.request.request_headers.to_byte_strings();Ok(())}@@ -201,7 +201,7 @@ impl ServiceWorkerRuntimeService {resulting_client_id: Some(client_id),url: request.url.clone(),method: request.method.clone(),- headers: request.request_headers.clone(),+ headers: request.request_headers.to_byte_strings(),body: request.body.clone(),destination,request_mode: request.request_mode,@@ -267,7 +267,7 @@ impl ServiceWorkerRuntimeService {job.network_context,job.request.url,job.request.method,- job.request.request_headers,+ job.request.request_headers.to_byte_strings(),request_body_text(&job.request.body),);}@@ -385,7 +385,7 @@ impl ServiceWorkerRuntimeService {internal_id: job.internal_id,request_url: job.request.url.clone(),request_method: job.request.method.clone(),- request_headers: job.request.request_headers.clone(),+ request_headers: job.request.request_headers.to_byte_strings(),request_body: request_body_text(&job.request.body),body_source_id: started.body_source_id,network_request_headers: None,@@ -638,7 +638,7 @@ impl ServiceWorkerRuntimeService {internal_id: job.internal_id,request_url: job.request.url,request_method: job.request.method,- request_headers: job.request.request_headers,+ request_headers: job.request.request_headers.to_byte_strings(),request_body: request_body_text(&job.request.body),response_status_text: Some(response.status_text),skip_fetch_security_validation: true,@@ -691,7 +691,7 @@ impl ServiceWorkerRuntimeService {internal_id: job.internal_id,request_url: job.request.url,request_method: job.request.method,- request_headers: job.request.request_headers,+ request_headers: job.request.request_headers.to_byte_strings(),request_body: request_body_text(&job.request.body),response_status_text: None,skip_fetch_security_validation: false,@@ -1960,12 +1960,12 @@ mod tests {assert_eq!(request_body_text(&job.request.body), None);assert_eq!(job.request.body, None);assert_eq!(- job.request.request_headers,+ job.request.request_headers.to_byte_strings(),vec![("x-keep".to_owned(), "yes".to_owned())]);assert_eq!(job.cors_preflight_request_headers,- job.request.request_headers+ job.request.request_headers.to_byte_strings());assert_eq!(job.request.redirect_count(), 1);assert_eq!(job.request.redirect_chain().len(), 1);diff --git a/moli-renderer-v8/src/worker/global_scope/content_security_policy.rs b/moli-renderer-v8/src/worker/global_scope/content_security_policy.rsindex b704ea406..55568a024 100644--- a/moli-renderer-v8/src/worker/global_scope/content_security_policy.rs+++ b/moli-renderer-v8/src/worker/global_scope/content_security_policy.rs@@ -288,7 +288,7 @@ fn pause_worker_content_security_policy_report_for_fetch_interception(url: request.url.clone(),websocket_socket_id: None,method: request.method.clone(),- request_headers: request.request_headers.clone(),+ request_headers: request.request_headers.to_byte_strings(),request_body: request_body.clone(),request_body_bytes,resource_type: SubresourceResourceType::CspReport,@@ -373,7 +373,7 @@ fn dispatch_worker_content_security_policy_report_to_service_worker(resulting_client_id: None,url: request.url.clone(),method: request.method.clone(),- headers: request.request_headers.clone(),+ headers: request.request_headers.to_byte_strings(),body: request.body.clone(),destination: ServiceWorkerRequestDestination::Report,request_mode: request.request_mode,@@ -687,7 +687,7 @@ fn record_worker_content_security_policy_report_failure(document_url,request.url,request.method,- request.request_headers,+ request.request_headers.to_byte_strings(),request_body,SubresourceResourceType::CspReport,message,@@ -713,7 +713,7 @@ fn send_worker_content_security_policy_report_success(document_url,request.url,request.method,- request.request_headers,+ request.request_headers.to_byte_strings(),request_body,SubresourceResourceType::CspReport,head.request_cookie_report.clone(),@@ -751,7 +751,7 @@ fn send_worker_content_security_policy_report_failure(document_url,request.url,request.method,- request.request_headers,+ request.request_headers.to_byte_strings(),request_body,SubresourceResourceType::CspReport,message,diff --git a/moli-renderer-v8/src/worker/global_scope/fetch.rs b/moli-renderer-v8/src/worker/global_scope/fetch.rsindex 26e34d42a..6bd52934b 100644--- a/moli-renderer-v8/src/worker/global_scope/fetch.rs+++ b/moli-renderer-v8/src/worker/global_scope/fetch.rs@@ -184,7 +184,8 @@ pub(in crate::worker) fn spawn_worker_fetch_network(&method,resolved_url.as_str(),body,- headers,+ moli_fetch::RequestHeaders::from_byte_strings(&headers)+ .expect("prepared worker Fetch headers are ByteStrings"),moli_url::WebOrigin::from_url(&document_url),) {Ok(request) => {@@ -519,7 +520,8 @@ pub(in crate::worker) fn spawn_worker_xhr_network(&method,resolved_url.as_str(),body,- headers,+ moli_fetch::RequestHeaders::from_byte_strings(&headers)+ .expect("prepared worker XHR headers are ByteStrings"),moli_url::WebOrigin::from_url(&document_url),).map(|request| {diff --git a/moli-renderer-v8/src/worker/global_scope/mod.rs b/moli-renderer-v8/src/worker/global_scope/mod.rsindex 89c8399e3..986ae5d54 100644--- a/moli-renderer-v8/src/worker/global_scope/mod.rs+++ b/moli-renderer-v8/src/worker/global_scope/mod.rs@@ -6555,7 +6555,8 @@ fn merge_worker_request_headers(context_headers: &[(String, String)],request_headers: &[(String, String)],) -> Vec<(String, String)> {- let mut merged = context_headers.to_vec();+ // Worker Fetch/XHR headers are ByteStrings; page extra headers are UTF-8.+ let mut merged = moli_fetch::RequestHeaders::from(context_headers.to_vec()).to_byte_strings();for (name, value) in request_headers {let lower = name.to_ascii_lowercase();merged.retain(|(existing_name, _)| existing_name.to_ascii_lowercase() != lower);diff --git a/moli-renderer-v8/src/worker/global_scope/xhr.rs b/moli-renderer-v8/src/worker/global_scope/xhr.rsindex 09bfad621..3e859973b 100644--- a/moli-renderer-v8/src/worker/global_scope/xhr.rs+++ b/moli-renderer-v8/src/worker/global_scope/xhr.rs@@ -566,7 +566,8 @@ fn send_synchronous_worker_xhr(&prepared.method,prepared.resolved_url.as_str(),prepared.send_body.clone(),- prepared.request_headers.clone(),+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)+ .expect("prepared Fetch/XHR headers are ByteStrings"),) {Ok(request) => {let mut request = requestdiff --git a/moli/src/app.rs b/moli/src/app.rsindex 60c91604b..a3f4a5133 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -215,7 +215,7 @@ fn read_script_file_arg(option: &str, path: &Path) -> Result<String> {fn build_fetch_request(url: &str, config: &AppConfig) -> Result<Request> {let mut request = Request::get(url)?;// Keep CLI-provided headers scoped to the initial document navigation.- request.request_headers = config.fetch.request_headers.clone();+ request.request_headers = config.fetch.request_headers.clone().into();Ok(request)}diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex f35d96720..19dabe4c8 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -1872,6 +1872,40 @@ fn cli_fetch_repeats_custom_request_headers() -> Result<()> {Ok(())}+#[test]+fn cli_fetch_keeps_unicode_header_values_as_utf8_bytes() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let listener = runtime.block_on(TcpListener::bind("127.0.0.1:0"))?;+ let url = format!("http://{}/", listener.local_addr()?);+ let server = runtime.spawn(async move {+ let (mut stream, _) = listener.accept().await?;+ let mut request = Vec::new();+ while !request.windows(4).any(|part| part == b"\r\n\r\n") {+ let mut chunk = [0; 1024];+ let count = stream.read(&mut chunk).await?;+ anyhow::ensure!(count != 0 && request.len() < 64 * 1024, "incomplete request");+ request.extend_from_slice(&chunk[..count]);+ }+ stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await?;+ Ok::<_, anyhow::Error>(request)+ });+ let output = run_fetch_cli_with_args(&url, &["-H", "X-Utf8: é中"])?;+ assert!(+ output.status.success(),+ "{}",+ String::from_utf8_lossy(&output.stderr)+ );+ let request = runtime.block_on(async {+ tokio::time::timeout(std::time::Duration::from_secs(5), server).await+ })???;+ assert!(+ request+ .windows("X-Utf8: é中\r\n".len())+ .any(|part| part == "X-Utf8: é中\r\n".as_bytes())+ );+ Ok(())+}+#[test]fn cli_fetch_preserves_header_values_with_embedded_colons_and_empty_values() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;