Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): distinguish request header bytes from UTF-8 strings

ldm04ed30be874ce
diff --git a/moli-fetch/src/blocking/cache/key.rs b/moli-fetch/src/blocking/cache/key.rs
index 00c847732..9c80480a8 100644
--- a/moli-fetch/src/blocking/cache/key.rs
+++ b/moli-fetch/src/blocking/cache/key.rs
@@ -23,7 +23,7 @@ pub(super) fn cache_store_and_key_for_request(
|| !config.default_request_headers().is_empty()
|| !subresource_validation_allows_http_cache(request)
|| cookie_header.is_some()
- || !request_headers_allow_http_cache(&request.request_headers)
+ || !request_headers_allow_http_cache(&request.request_headers.to_byte_strings())
|| !matches!(request_url.scheme(), "http" | "https")
|| !request_url.username().is_empty()
|| request_url.password().is_some()
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs
index d0279d39e..741fe9af8 100644
--- a/moli-fetch/src/blocking/mod.rs
+++ b/moli-fetch/src/blocking/mod.rs
@@ -160,26 +160,38 @@ pub fn outgoing_request_headers(
outgoing_request_headers_for_url(config, request, &request.url, cookie_header)
}
+// String views used by observation and cache metadata are isomorphic byte
+// projections. The transport itself keeps the encoded representation below.
pub(crate) fn outgoing_request_headers_for_url(
config: &FetchConfig,
request: &Request,
request_url: &Url,
cookie_header: Option<&str>,
) -> Vec<(String, String)> {
- let mut outgoing = Vec::new();
+ outgoing_request_header_bytes_for_url(config, request, request_url, cookie_header)
+ .to_byte_strings()
+}
+
+fn outgoing_request_header_bytes_for_url(
+ config: &FetchConfig,
+ request: &Request,
+ request_url: &Url,
+ cookie_header: Option<&str>,
+) -> crate::RequestHeaders {
+ let mut outgoing = crate::RequestHeaders::default();
if let Some(cookie_header) = cookie_header {
- outgoing.push(("Cookie".to_owned(), cookie_header.to_owned()));
+ outgoing.push(("Cookie".to_owned(), cookie_header.as_bytes().to_vec()));
}
for (name, value) in config.default_request_headers() {
if cookie_header.is_some() && name.eq_ignore_ascii_case("cookie") {
continue;
}
- outgoing.push((name.clone(), value.clone()));
+ outgoing.push((name.clone(), value.as_bytes().to_vec()));
}
- for (name, value) in &request.request_headers {
+ for (name, value) in request.request_headers.iter() {
if cookie_header.is_some() && name.eq_ignore_ascii_case("cookie") {
continue;
}
@@ -187,7 +199,7 @@ pub(crate) fn outgoing_request_headers_for_url(
}
if request.has_browser_identity_override() {
- append_header_if_missing(
+ append_encoded_header_if_missing(
&mut outgoing,
"User-Agent",
request.browser_identity(config).user_agent().to_owned(),
@@ -199,13 +211,13 @@ pub(crate) fn outgoing_request_headers_for_url(
append_browser_storage_access_header(&mut outgoing, request, request_url);
if let Some(origin) = request_origin_header_value(request, request_url) {
- append_header_if_missing(&mut outgoing, "Origin", origin);
+ append_encoded_header_if_missing(&mut outgoing, "Origin", origin);
}
if !header_present(&outgoing, "referer")
&& let Some(referer) = referrer_header_value_for_request(request, request_url)
{
- outgoing.push(("Referer".to_owned(), referer));
+ outgoing.push(("Referer".to_owned(), referer.into_bytes()));
}
if !header_present(&outgoing, "authorization")
@@ -217,7 +229,7 @@ pub(crate) fn outgoing_request_headers_for_url(
// 401 retry body while Digest/NTLM/Negotiate stay on the buffered path.
outgoing.push((
"Authorization".to_owned(),
- format!("Basic {}", encode_basic_auth(username, password)),
+ format!("Basic {}", encode_basic_auth(username, password)).into_bytes(),
));
}
@@ -231,7 +243,8 @@ pub(crate) fn outgoing_request_headers_for_url(
format!(
"Basic {}",
encode_basic_auth(&auth.username, &auth.password)
- ),
+ )
+ .into_bytes(),
));
}
@@ -251,20 +264,34 @@ pub(crate) fn network_request_extra_info_from_headers(
}
}
-fn header_present(headers: &[(String, String)], name: &str) -> bool {
+fn header_present<V>(headers: &[(String, V)], name: &str) -> bool {
headers
.iter()
.any(|(header_name, _)| header_name.eq_ignore_ascii_case(name))
}
+fn header_utf8_byte_string(value: &str) -> String {
+ crate::headers::decode_http_header_bytes(value.as_bytes()).into_owned()
+}
+
fn append_header_if_missing(headers: &mut Vec<(String, String)>, name: &str, value: String) {
if !header_present(headers, name) {
- headers.push((name.to_owned(), value));
+ headers.push((name.to_owned(), header_utf8_byte_string(&value)));
+ }
+}
+
+fn append_encoded_header_if_missing(
+ headers: &mut crate::RequestHeaders,
+ name: &str,
+ value: String,
+) {
+ if !header_present(headers, name) {
+ headers.push((name.to_owned(), value.into_bytes()));
}
}
fn append_browser_navigation_headers(
- outgoing: &mut Vec<(String, String)>,
+ outgoing: &mut crate::RequestHeaders,
config: &FetchConfig,
request: &Request,
request_url: &Url,
@@ -273,12 +300,12 @@ fn append_browser_navigation_headers(
return;
}
- append_header_if_missing(
+ append_encoded_header_if_missing(
outgoing,
"Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7".to_owned(),
);
- append_header_if_missing(
+ append_encoded_header_if_missing(
outgoing,
"Accept-Language",
request
@@ -286,9 +313,9 @@ fn append_browser_navigation_headers(
.accept_language()
.to_owned(),
);
- append_header_if_missing(outgoing, "Upgrade-Insecure-Requests", "1".to_owned());
- append_header_if_missing(outgoing, "Sec-Fetch-Mode", "navigate".to_owned());
- append_header_if_missing(
+ append_encoded_header_if_missing(outgoing, "Upgrade-Insecure-Requests", "1".to_owned());
+ append_encoded_header_if_missing(outgoing, "Sec-Fetch-Mode", "navigate".to_owned());
+ append_encoded_header_if_missing(
outgoing,
"Sec-Fetch-Dest",
if request.is_subframe_navigation_request() {
@@ -298,25 +325,25 @@ fn append_browser_navigation_headers(
}
.to_owned(),
);
- append_header_if_missing(
+ append_encoded_header_if_missing(
outgoing,
"Sec-Fetch-Site",
request_sec_fetch_site(request, request_url),
);
if request.is_top_level_navigation_request() && request.cookie_context.initiator_url.is_none() {
- append_header_if_missing(outgoing, "Sec-Fetch-User", "?1".to_owned());
+ append_encoded_header_if_missing(outgoing, "Sec-Fetch-User", "?1".to_owned());
}
if request.browser_navigation_kind() == crate::BrowserNavigationRequestKind::Reload {
- append_header_if_missing(outgoing, "Cache-Control", "max-age=0".to_owned());
+ append_encoded_header_if_missing(outgoing, "Cache-Control", "max-age=0".to_owned());
}
append_browser_client_hints(outgoing, request.browser_identity(config));
}
fn append_browser_subresource_headers(
- outgoing: &mut Vec<(String, String)>,
+ outgoing: &mut crate::RequestHeaders,
config: &FetchConfig,
request: &Request,
request_url: &Url,
@@ -354,8 +381,8 @@ fn append_browser_subresource_headers(
BrowserRequestMetadata::TextTrack => "text/vtt,*/*;q=0.1",
_ => "*/*",
};
- append_header_if_missing(outgoing, "Accept", accept.to_owned());
- append_header_if_missing(
+ append_encoded_header_if_missing(outgoing, "Accept", accept.to_owned());
+ append_encoded_header_if_missing(
outgoing,
"Accept-Language",
request
@@ -363,12 +390,12 @@ fn append_browser_subresource_headers(
.accept_language()
.to_owned(),
);
- append_header_if_missing(
+ append_encoded_header_if_missing(
outgoing,
"Sec-Fetch-Site",
request_sec_fetch_site(request, request_url),
);
- append_header_if_missing(
+ append_encoded_header_if_missing(
outgoing,
"Sec-Fetch-Mode",
request.request_mode.as_ref().to_owned(),
@@ -390,7 +417,7 @@ fn append_browser_subresource_headers(
| BrowserRequestMetadata::Ping
| BrowserRequestMetadata::Xhr => "empty",
};
- append_header_if_missing(outgoing, "Sec-Fetch-Dest", destination.to_owned());
+ append_encoded_header_if_missing(outgoing, "Sec-Fetch-Dest", destination.to_owned());
append_browser_client_hints(outgoing, request.browser_identity(config));
}
}
@@ -413,19 +440,19 @@ fn request_origin_header_value(request: &Request, request_url: &Url) -> Option<S
}
fn append_browser_client_hints(
- outgoing: &mut Vec<(String, String)>,
+ outgoing: &mut crate::RequestHeaders,
identity: &moli_browser_profile::BrowserIdentityProfile,
) {
let Some(sec_ch_ua) = identity.sec_ch_ua_value() else {
return;
};
- append_header_if_missing(outgoing, "Sec-CH-UA", sec_ch_ua);
- append_header_if_missing(
+ append_encoded_header_if_missing(outgoing, "Sec-CH-UA", sec_ch_ua);
+ append_encoded_header_if_missing(
outgoing,
"Sec-CH-UA-Mobile",
if identity.mobile() { "?1" } else { "?0" }.to_owned(),
);
- append_header_if_missing(
+ append_encoded_header_if_missing(
outgoing,
"Sec-CH-UA-Platform",
format!("\"{}\"", identity.platform()),
@@ -433,7 +460,7 @@ fn append_browser_client_hints(
}
fn append_browser_storage_access_header(
- outgoing: &mut Vec<(String, String)>,
+ outgoing: &mut crate::RequestHeaders,
request: &Request,
request_url: &Url,
) {
@@ -457,7 +484,7 @@ fn append_browser_storage_access_header(
// Moli does not currently expose a third-party-cookie blocking mode,
// so its network cookie policy has the same effective state. If such a
// blocker is added, this value must be derived from that policy instead.
- append_header_if_missing(outgoing, "Sec-Fetch-Storage-Access", "active".to_owned());
+ append_encoded_header_if_missing(outgoing, "Sec-Fetch-Storage-Access", "active".to_owned());
}
fn request_sec_fetch_site(request: &Request, request_url: &Url) -> String {
@@ -653,7 +680,7 @@ pub(crate) fn configure_easy(
let mut headers = RequestHeaderList::default();
let mut outgoing_headers =
- outgoing_request_headers_for_url(config, request, request_url, cookie_header);
+ outgoing_request_header_bytes_for_url(config, request, request_url, cookie_header);
// A 407 can come from a transparent proxy even when no explicit proxy
// route is configured. In that case the connected endpoint receives the
// challenge response as a normal request header. Explicit proxies use the
@@ -662,31 +689,37 @@ pub(crate) fn configure_easy(
&& let Some(authorization) = proxy_header_authorization(request)
&& !header_present(&outgoing_headers, "proxy-authorization")
{
- outgoing_headers.push(("Proxy-Authorization".to_owned(), authorization));
+ outgoing_headers.push(("Proxy-Authorization".to_owned(), authorization.into_bytes()));
}
if let Some(web_bot_auth) = config.web_bot_auth() {
+ let mut signed_headers = outgoing_headers.to_byte_strings();
web_bot_auth
- .append_request_headers(&mut outgoing_headers, &request.method, request_url)
+ .append_request_headers(&mut signed_headers, &request.method, request_url)
.with_context(|| anyhow!("failed to sign web bot auth request for {request_url}"))?;
+ outgoing_headers = crate::RequestHeaders::from_byte_strings(&signed_headers)?;
}
let mut has_content_type_header = false;
- for (name, value) in &outgoing_headers {
+ for (name, value) in outgoing_headers.iter() {
has_content_type_header |= name.eq_ignore_ascii_case("content-type");
- let header_line = if value.is_empty() {
- format!("{name}:")
- } else {
- format!("{name}: {value}")
- };
+ let mut header_line = format!("{name}:").into_bytes();
+ if !value.is_empty() {
+ header_line.push(b' ');
+ header_line.extend_from_slice(value);
+ }
headers
.append(&header_line)
.context("failed to build request header")?;
}
if let Some(validation_headers) = validation_headers {
- for (name, value) in validation_headers {
+ // Validators were read from response headers and are ByteStrings.
+ let validation_headers = crate::RequestHeaders::from_byte_strings(&validation_headers)?;
+ for (name, value) in validation_headers.iter() {
has_content_type_header |= name.eq_ignore_ascii_case("content-type");
+ let mut line = format!("{name}: ").into_bytes();
+ line.extend_from_slice(value);
headers
- .append(&format!("{name}: {value}"))
+ .append(&line)
.context("failed to build cache validation request header")?;
}
}
@@ -698,7 +731,7 @@ pub(crate) fn configure_easy(
// for POST bodies and bodyless PUT requests. Browser requests only send Content-Type when
// BodyInit or caller headers produce one, so suppress curl's transport default.
headers
- .append("Content-Type:")
+ .append(b"Content-Type:")
.context("failed to suppress curl default content-type")?;
}
@@ -742,7 +775,7 @@ pub(crate) fn configure_easy(
}
crate::runtime::FetchTransferHandler::set_request_headers(easy, headers)?;
- Ok(outgoing_headers)
+ Ok(outgoing_headers.to_byte_strings())
}
fn configure_proxy_headers<H: Handler>(
diff --git a/moli-fetch/src/blocking/request_headers.rs b/moli-fetch/src/blocking/request_headers.rs
index 0c4ca295a..cbed29086 100644
--- a/moli-fetch/src/blocking/request_headers.rs
+++ b/moli-fetch/src/blocking/request_headers.rs
@@ -3,7 +3,7 @@ use std::ffi::CString;
use anyhow::{Context, Result};
// curl::easy::List only accepts UTF-8 strings. Own a libcurl list here so the
-// transport can send ByteString header values without re-encoding them.
+// transport can send already encoded header values without re-encoding them.
#[derive(Default)]
pub(crate) struct RequestHeaderList {
raw: *mut curl_sys::curl_slist,
@@ -14,18 +14,8 @@ pub(crate) struct RequestHeaderList {
unsafe impl Send for RequestHeaderList {}
impl RequestHeaderList {
- pub(crate) fn append(&mut self, line: &str) -> Result<()> {
- let line = if line.is_ascii() {
- CString::new(line.as_bytes())
- } else {
- let bytes = line
- .chars()
- .map(|ch| u8::try_from(u32::from(ch)))
- .collect::<std::result::Result<Vec<_>, _>>()
- .context("HTTP request header contains a non-byte character")?;
- CString::new(bytes)
- }
- .context("HTTP request header contains NUL")?;
+ pub(crate) fn append(&mut self, line: &[u8]) -> Result<()> {
+ let line = CString::new(line).context("HTTP request header contains NUL")?;
// SAFETY: self.raw is null or a live list owned by self. libcurl copies
// the NUL-terminated string; failure leaves the existing list intact.
let raw = unsafe { curl_sys::curl_slist_append(self.raw, line.as_ptr()) };
@@ -54,11 +44,10 @@ mod tests {
use super::*;
#[test]
- fn rejects_non_byte_characters_and_nul() {
+ fn rejects_nul_without_discarding_the_list() {
let mut headers = RequestHeaderList::default();
- headers.append("X-Valid: \u{ff}").unwrap();
- assert!(headers.append("X-Invalid: \u{100}").is_err());
- assert!(headers.append("X-Invalid: before\0after").is_err());
- headers.append("X-Valid: after").unwrap();
+ headers.append(b"X-Valid: \xff").unwrap();
+ assert!(headers.append(b"X-Invalid: before\0after").is_err());
+ headers.append(b"X-Valid: after").unwrap();
}
}
diff --git a/moli-fetch/src/client_hints.rs b/moli-fetch/src/client_hints.rs
index dbe16b752..7ef173957 100644
--- a/moli-fetch/src/client_hints.rs
+++ b/moli-fetch/src/client_hints.rs
@@ -222,7 +222,7 @@ pub(crate) fn prepare_client_hint_request(
};
effective_request
.request_headers
- .push((hint.header_name().to_owned(), value));
+ .push((hint.header_name().to_owned(), value.into_bytes()));
sent_hints.insert(hint);
}
}
@@ -255,8 +255,9 @@ fn configured_client_hint_names(config: &FetchConfig, request: &Request) -> BTre
config
.default_request_headers()
.iter()
- .chain(&request.request_headers)
- .filter_map(|(name, _)| ClientHint::from_header_name(name))
+ .map(|(name, _)| name)
+ .chain(request.request_headers.iter().map(|(name, _)| name))
+ .filter_map(|name| ClientHint::from_header_name(name))
.collect()
}
@@ -335,10 +336,13 @@ mod tests {
let second =
prepare_client_hint_request(&preferences, &restarts, &config, &request, &request.url);
assert!(second.request.request_headers.iter().any(|(name, value)| {
- name.eq_ignore_ascii_case("sec-ch-ua-arch") && value == "\"x86\""
+ name.eq_ignore_ascii_case("sec-ch-ua-arch") && value == b"\"x86\""
}));
assert!(second.request.request_headers.iter().any(|(name, value)| {
- name.eq_ignore_ascii_case("sec-ch-ua-full-version-list") && value.contains("145.0.0.0")
+ name.eq_ignore_ascii_case("sec-ch-ua-full-version-list")
+ && value
+ .windows(b"145.0.0.0".len())
+ .any(|part| part == b"145.0.0.0")
}));
assert_eq!(
second
diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs
index b66e97609..a5a3f03b1 100644
--- a/moli-fetch/src/lib.rs
+++ b/moli-fetch/src/lib.rs
@@ -19,6 +19,7 @@ mod proxy;
mod proxy_connect;
mod referrer_policy;
mod request;
+mod request_headers;
mod request_policy;
mod response;
mod runtime;
@@ -67,6 +68,7 @@ pub use request::{
RequestPriorityHints, RequestRedirectMode, RequestResourceType, ResourceLoadPriority,
ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, SubresourceRequestMetadata,
};
+pub use request_headers::RequestHeaders;
pub use request_policy::{is_bad_port, should_request_be_blocked_due_to_bad_port};
pub use response::{
NegotiatedHttpVersion, NetworkRequestExtraInfo, NetworkResponseExtraInfo, RawResponse,
diff --git a/moli-fetch/src/request.rs b/moli-fetch/src/request.rs
index 7ae14e4bb..0cb46d374 100644
--- a/moli-fetch/src/request.rs
+++ b/moli-fetch/src/request.rs
@@ -29,7 +29,8 @@ pub struct Request {
pub url: Url,
pub method: String,
pub body: Option<Vec<u8>>,
- pub request_headers: Vec<(String, String)>,
+ /// Wire bytes, with the source encoding resolved before header merging.
+ pub request_headers: crate::RequestHeaders,
cache_mode: RequestCacheMode,
pub resource_type: RequestResourceType,
subresource_request_metadata: Option<SubresourceRequestMetadata>,
@@ -385,7 +386,7 @@ impl Request {
url,
method: "GET".to_owned(),
body: None,
- request_headers: vec![],
+ request_headers: crate::RequestHeaders::default(),
cache_mode: RequestCacheMode::Default,
resource_type: RequestResourceType::Raw,
subresource_request_metadata: None,
@@ -416,7 +417,7 @@ impl Request {
url,
method: "GET".to_owned(),
body: None,
- request_headers: vec![],
+ request_headers: crate::RequestHeaders::default(),
cache_mode: RequestCacheMode::Default,
resource_type: RequestResourceType::Raw,
subresource_request_metadata: None,
@@ -441,11 +442,13 @@ impl Request {
}
}
+ /// String header values use UTF-8. To supply HTTP bytes or WebIDL
+ /// ByteStrings, pass an explicitly constructed `RequestHeaders` instead.
pub fn new(
method: &str,
raw_url: &str,
body: Option<String>,
- request_headers: Vec<(String, String)>,
+ request_headers: impl Into<crate::RequestHeaders>,
) -> Result<Self> {
Self::new_bytes(
method,
@@ -461,7 +464,7 @@ impl Request {
method: &str,
raw_url: &str,
body: Option<Vec<u8>>,
- request_headers: Vec<(String, String)>,
+ request_headers: impl Into<crate::RequestHeaders>,
) -> Result<Self> {
let url = Url::parse(raw_url)
.with_context(|| anyhow!("failed to parse request url `{raw_url}`"))?;
@@ -472,13 +475,13 @@ impl Request {
method: &str,
url: Url,
body: Option<Vec<u8>>,
- request_headers: Vec<(String, String)>,
+ request_headers: impl Into<crate::RequestHeaders>,
) -> Self {
Self {
url,
method: method.to_owned(),
body,
- request_headers,
+ request_headers: request_headers.into(),
cache_mode: RequestCacheMode::Default,
resource_type: RequestResourceType::Raw,
subresource_request_metadata: None,
@@ -687,7 +690,7 @@ impl Request {
method: &str,
raw_url: &str,
body: Option<Vec<u8>>,
- headers: Vec<(String, String)>,
+ headers: impl Into<crate::RequestHeaders>,
origin: WebOrigin,
) -> Result<Self> {
Ok(Self::new_bytes(method, raw_url, body, headers)?.with_request_origin(origin))
@@ -697,7 +700,7 @@ impl Request {
method: &str,
url: Url,
body: Option<Vec<u8>>,
- headers: Vec<(String, String)>,
+ headers: impl Into<crate::RequestHeaders>,
origin: WebOrigin,
) -> Self {
Self::new_http_with_url(method, url, body, headers).with_request_origin(origin)
diff --git a/moli-fetch/src/request_headers.rs b/moli-fetch/src/request_headers.rs
new file mode 100644
index 000000000..ebc16615b
--- /dev/null
+++ b/moli-fetch/src/request_headers.rs
@@ -0,0 +1,82 @@
+use anyhow::{Context, Result};
+
+/// Encoded request header values. Ordinary Rust strings use UTF-8; WebIDL
+/// ByteStrings must enter through `from_byte_strings` before headers are merged.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+pub struct RequestHeaders(Vec<(String, Vec<u8>)>);
+
+impl RequestHeaders {
+ pub fn from_bytes(headers: Vec<(String, Vec<u8>)>) -> Self {
+ Self(headers)
+ }
+
+ pub fn from_byte_strings(headers: &[(String, String)]) -> Result<Self> {
+ headers
+ .iter()
+ .map(|(name, value)| {
+ let bytes = value
+ .chars()
+ .map(|ch| u8::try_from(u32::from(ch)))
+ .collect::<std::result::Result<Vec<_>, _>>()
+ .context("HTTP header ByteString contains a non-byte character")?;
+ Ok((name.clone(), bytes))
+ })
+ .collect::<Result<Vec<_>>>()
+ .map(Self)
+ }
+
+ /// A lossless string projection for Fetch and network observation surfaces.
+ pub fn to_byte_strings(&self) -> Vec<(String, String)> {
+ self.0
+ .iter()
+ .map(|(name, value)| {
+ (
+ name.clone(),
+ crate::headers::decode_http_header_bytes(value).into_owned(),
+ )
+ })
+ .collect()
+ }
+}
+
+impl From<Vec<(String, String)>> for RequestHeaders {
+ fn from(headers: Vec<(String, String)>) -> Self {
+ Self(
+ headers
+ .into_iter()
+ .map(|(name, value)| (name, value.into_bytes()))
+ .collect(),
+ )
+ }
+}
+
+impl std::ops::Deref for RequestHeaders {
+ type Target = Vec<(String, Vec<u8>)>;
+
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
+impl std::ops::DerefMut for RequestHeaders {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.0
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn string_encodings_are_explicit() {
+ let headers = vec![("X-Test".into(), "é".into())];
+ assert_eq!(RequestHeaders::from(headers.clone())[0].1, [0xc3, 0xa9]);
+ assert_eq!(
+ RequestHeaders::from_byte_strings(&headers).unwrap()[0].1,
+ [0xe9]
+ );
+ assert!(RequestHeaders::from_byte_strings(&[("X-Test".into(), "中".into())]).is_err());
+ assert!(RequestHeaders::from_byte_strings(&[("X-Test".into(), "\u{100}".into())]).is_err());
+ }
+}
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index 2eff43475..e9989e518 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -3709,7 +3709,7 @@ fn network_fetch_failure_for_request(
current_url.clone(),
request.method.clone(),
request.body.clone(),
- request.request_headers.clone(),
+ request.request_headers.to_byte_strings(),
redirect_chain.to_vec(),
),
)
@@ -3718,7 +3718,7 @@ fn network_fetch_failure_for_request(
#[cfg(test)]
fn request_panics_for_testing(request: &Request) -> bool {
request.request_headers.iter().any(|(name, value)| {
- name.eq_ignore_ascii_case("x-moli-test-panic") && value == "runtime-worker"
+ name.eq_ignore_ascii_case("x-moli-test-panic") && value == b"runtime-worker"
})
}
diff --git a/moli-fetch/src/tests/cookie_store.rs b/moli-fetch/src/tests/cookie_store.rs
index e4c708116..7ee78e7e3 100644
--- a/moli-fetch/src/tests/cookie_store.rs
+++ b/moli-fetch/src/tests/cookie_store.rs
@@ -1008,7 +1008,8 @@ fn explicit_navigation_headers_override_browser_defaults() {
("Sec-Fetch-Site".to_owned(), "cross-site".to_owned()),
("Sec-Fetch-User".to_owned(), "?0".to_owned()),
("Cache-Control".to_owned(), "no-cache".to_owned()),
- ];
+ ]
+ .into();
let headers = outgoing_request_headers(&config, &request, None);
diff --git a/moli-fetch/src/tests/header_bytes.rs b/moli-fetch/src/tests/header_bytes.rs
index 35b53ec91..c07f00fca 100644
--- a/moli-fetch/src/tests/header_bytes.rs
+++ b/moli-fetch/src/tests/header_bytes.rs
@@ -69,11 +69,18 @@ async fn http_header_bytes_survive_all_transports_and_observation() -> Result<()
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/bytes", listener.local_addr()?);
let server = tokio::spawn(serve_responses(listener, vec![response(&headers); 3]));
- let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ let mut config = FetchConfig::default();
+ config.set_default_request_headers(vec![("X-Config".into(), "é中".into())]);
+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());
for mode in ["buffered", "html", "raw"] {
let recorder = NetworkObservationRecorder::default();
- let request = Request::new("GET", &url, None, vec![("X-Bytes".into(), value.clone())])?
- .with_network_observation_recorder(recorder.clone());
+ let request = Request::new(
+ "GET",
+ &url,
+ None,
+ crate::RequestHeaders::from_bytes(vec![("X-Bytes".into(), bytes.clone())]),
+ )?
+ .with_network_observation_recorder(recorder.clone());
let (head, body) = fetch_in_mode(&client, request, mode).await?;
// The HTML stream intentionally decodes the body as text, independently
// of header ByteStrings. The raw and buffered paths preserve its bytes.
@@ -119,6 +126,11 @@ async fn http_header_bytes_survive_all_transports_and_observation() -> Result<()
}
let expected_line = [b"X-Bytes: ".as_slice(), bytes.as_slice(), b"\r\n"].concat();
for request in server.await?? {
+ assert!(
+ request
+ .windows("X-Config: é中\r\n".len())
+ .any(|part| part == "X-Config: é中\r\n".as_bytes())
+ );
assert!(
request
.windows(expected_line.len())
@@ -129,6 +141,27 @@ async fn http_header_bytes_survive_all_transports_and_observation() -> Result<()
Ok(())
}
+#[tokio::test]
+async fn ordinary_request_header_strings_keep_utf8_encoding() -> Result<()> {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/utf8", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(listener, vec![response(b""); 3]));
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ for mode in ["buffered", "html", "raw"] {
+ let request = Request::new("GET", &url, None, vec![("X-Utf8".into(), "é中".into())])?;
+ fetch_in_mode(&client, request, mode).await?;
+ }
+ for request in server.await?? {
+ assert!(
+ request
+ .windows("X-Utf8: é中\r\n".len())
+ .any(|part| part == "X-Utf8: é中\r\n".as_bytes())
+ );
+ }
+ assert!(client.shutdown().is_clean());
+ Ok(())
+}
+
#[tokio::test]
async fn http_header_bytes_survive_redirect_handle_reuse() -> Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
@@ -139,7 +172,9 @@ async fn http_header_bytes_survive_redirect_handle_reuse() -> Result<()> {
vec![redirect, response(b"X-Bytes: \xff\r\n")],
));
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
- let request = Request::new("GET", &url, None, vec![("X-Bytes".into(), "\u{ff}".into())])?;
+ let mut headers = crate::RequestHeaders::from(vec![("X-Utf8".into(), "é中".into())]);
+ headers.push(("X-Bytes".into(), vec![0xff]));
+ let request = Request::new("GET", &url, None, headers)?;
let (head, body) = fetch_in_mode(&client, request, "raw").await?;
assert!(head.redirected);
assert_eq!(head.final_url.path(), "/final");
@@ -148,6 +183,11 @@ async fn http_header_bytes_survive_redirect_handle_reuse() -> Result<()> {
let requests = server.await??;
assert_eq!(requests.len(), 2);
for request in requests {
+ assert!(
+ request
+ .windows("X-Utf8: é中\r\n".len())
+ .any(|part| part == "X-Utf8: é中\r\n".as_bytes())
+ );
assert!(
request
.windows(b"X-Bytes: \xff\r\n".len())
diff --git a/moli-protocol/src/conn/downloads.rs b/moli-protocol/src/conn/downloads.rs
index 8dc9d6e80..bd4484c0f 100644
--- a/moli-protocol/src/conn/downloads.rs
+++ b/moli-protocol/src/conn/downloads.rs
@@ -411,7 +411,7 @@ impl CdpConnection {
let mut request = Request::get(&activation.url)
.map_err(|error| format!("invalid download url: {error}"))?;
- request.request_headers = owner.request_headers;
+ request.request_headers = owner.request_headers.into();
request = request
.with_top_level_navigation_cookie_context()
.with_page_network_policy()
diff --git a/moli-protocol/src/domains/fetch/commands.rs b/moli-protocol/src/domains/fetch/commands.rs
index d7dd9d86f..66cbc1998 100644
--- a/moli-protocol/src/domains/fetch/commands.rs
+++ b/moli-protocol/src/domains/fetch/commands.rs
@@ -225,6 +225,18 @@ fn start_devtools_continue_intercepted_request_command(
&request_id,
) {
let mut pending = pending;
+ // Resolve protocol Unicode strings before mixing them with the saved
+ // Fetch/XHR ByteStrings, including across multiple interception stages.
+ let command_headers = command.headers.clone().map(|headers| {
+ if matches!(
+ pending.resource_type,
+ SubresourceResourceType::Fetch | SubresourceResourceType::Xhr
+ ) {
+ moli_fetch::RequestHeaders::from(headers).to_byte_strings()
+ } else {
+ headers
+ }
+ });
if pending
.request_stage_pause_state()
.is_some_and(|chain| !chain.remaining_sessions.is_empty())
@@ -233,7 +245,7 @@ fn start_devtools_continue_intercepted_request_command(
parsed_url,
command.method.clone(),
command.post_data.clone(),
- command.headers.clone(),
+ command_headers,
);
let Some(event) =
super::subresource::next_chained_subresource_request_pause_event(conn, pending)
@@ -257,7 +269,7 @@ fn start_devtools_continue_intercepted_request_command(
} else {
chain_body
};
- let headers = command.headers.clone().or(chain_headers);
+ let headers = command_headers.or(chain_headers);
pending.request_stage_chain = None;
let configured_response_stage = conn
.target_fetch_subresource_interception_snapshot_for_owner(owner)
diff --git a/moli-protocol/src/domains/network/tests/subresource.rs b/moli-protocol/src/domains/network/tests/subresource.rs
index acd860eae..8141c3883 100644
--- a/moli-protocol/src/domains/network/tests/subresource.rs
+++ b/moli-protocol/src/domains/network/tests/subresource.rs
@@ -3597,3 +3597,164 @@ xhr.send();
server.abort();
}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn cdp_utf8_headers_coexist_with_fetch_and_xhr_byte_strings() {
+ const PROBE: &str = r#"
+ const headers = { 'x-web': '\u00e9\u00ff', 'x-override': '\u00e9' };
+ function xhrProbe(async) {
+ return new Promise((resolve, reject) => {
+ const xhr = new XMLHttpRequest();
+ xhr.open('GET', '/echo', async);
+ for (const [name, value] of Object.entries(headers)) xhr.setRequestHeader(name, value);
+ xhr.onload = () => resolve(JSON.parse(xhr.responseText));
+ xhr.onerror = () => reject(new Error('XHR failed'));
+ xhr.send();
+ });
+ }
+ const results = await Promise.all([
+ fetch('/redirect', { headers }).then(response => response.json()),
+ xhrProbe(true),
+ xhrProbe(false)
+ ]);
+ "#;
+ async fn echo(headers: axum::http::HeaderMap) -> impl IntoResponse {
+ axum::Json(json!({
+ "cdp": headers.get("x-cdp").map(|value| value.as_bytes()),
+ "web": headers.get("x-web").map(|value| value.as_bytes()),
+ "override": headers.get("x-override").map(|value| value.as_bytes()),
+ }))
+ }
+ let worker_script = format!("(async () => {{ {PROBE} postMessage(results); }})()");
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ Router::new()
+ .route("/page", get(plain_page))
+ .route("/echo", get(echo))
+ .route(
+ "/redirect",
+ get(|| async { axum::response::Redirect::temporary("/echo") }),
+ )
+ .route(
+ "/worker.js",
+ get(move || async move {
+ ([(CONTENT_TYPE.as_str(), "text/javascript")], worker_script)
+ }),
+ ),
+ )
+ .await
+ .unwrap();
+ });
+ let mut ctx = TestContext::new();
+ let mut bc = BrowserContext::new("BID-1".into());
+ bc.set_active_target_id("TID-1");
+ bc.attach_active_session("SID-1");
+ ctx.conn.install_browser_context_fixture_for_test(bc);
+ ctx.process_async(json!({
+ "id": 0, "method": "Network.enable", "sessionId": "SID-1"
+ }))
+ .await;
+ ctx.expect_result(0, json!({}), Some("SID-1"));
+ ctx.process_async(json!({
+ "id": 1, "method": "Network.setExtraHTTPHeaders", "sessionId": "SID-1",
+ "params": { "headers": { "x-cdp": "é中", "X-Override": "中" } }
+ }))
+ .await;
+ ctx.expect_result(1, json!({}), Some("SID-1"));
+ ctx.process_async(json!({
+ "id": 2, "method": "Page.navigate", "sessionId": "SID-1",
+ "params": { "url": format!("http://{addr}/page") }
+ }))
+ .await;
+ ctx.process_async(json!({
+ "id": 3, "method": "Runtime.evaluate", "sessionId": "SID-1",
+ "params": {
+ "expression": format!(r#"(async () => {{
+ {PROBE}
+ const workerResults = await new Promise((resolve, reject) => {{
+ const worker = new Worker('/worker.js');
+ worker.onmessage = event => {{ worker.terminate(); resolve(event.data); }};
+ worker.onerror = () => reject(new Error('Worker failed'));
+ }});
+ return JSON.stringify(results.concat(workerResults));
+ }})()"#),
+ "awaitPromise": true, "returnByValue": true
+ }
+ }))
+ .await;
+ wait_until_messages(
+ &mut ctx,
+ Some("SID-1"),
+ "mixed header encoding probe",
+ |messages| messages.iter().any(|message| message["id"] == json!(3)),
+ )
+ .await;
+ let expected =
+ json!({ "cdp": [0xc3, 0xa9, 0xe4, 0xb8, 0xad], "web": [0xe9, 0xff], "override": [0xe9] });
+ ctx.expect_result(3, json!({ "result": { "type": "string", "value": serde_json::to_string(&vec![expected; 6]).unwrap() } }), Some("SID-1"));
+ ctx.process_async(json!({
+ "id": 4, "method": "Fetch.enable", "sessionId": "SID-1",
+ "params": { "patterns": [{ "urlPattern": "*/echo", "requestStage": "Request" }] }
+ }))
+ .await;
+ ctx.expect_result(4, json!({}), Some("SID-1"));
+ for replace_headers in [false, true] {
+ ctx.sent.clear();
+ ctx.process_async(json!({
+ "id": 5, "method": "Runtime.evaluate", "sessionId": "SID-1",
+ "params": {
+ "expression": r#"fetch('/echo', {headers: {'x-web': '\u00e9\u00ff', 'x-override': '\u00e9'}}).then(response => response.text())"#,
+ "awaitPromise": true, "returnByValue": true
+ }
+ })).await;
+ wait_until_messages(
+ &mut ctx,
+ Some("SID-1"),
+ "byte header request pause",
+ |messages| {
+ messages
+ .iter()
+ .any(|message| message["method"] == "Fetch.requestPaused")
+ },
+ )
+ .await;
+ let messages = ctx.take_all();
+ let paused = messages
+ .iter()
+ .find(|message| message["method"] == "Fetch.requestPaused")
+ .unwrap();
+ let mut params = json!({"requestId": paused["params"]["requestId"]});
+ if replace_headers {
+ params["headers"] = json!([
+ {"name": "x-cdp", "value": "é中"},
+ {"name": "x-web", "value": "é中"},
+ {"name": "x-override", "value": "é"}
+ ]);
+ }
+ ctx.process_async(json!({
+ "id": 6, "method": "Fetch.continueRequest", "sessionId": "SID-1", "params": params
+ }))
+ .await;
+ wait_until_messages(
+ &mut ctx,
+ Some("SID-1"),
+ "continued byte header request",
+ |messages| messages.iter().any(|message| message["id"] == json!(5)),
+ )
+ .await;
+ let expected = if replace_headers {
+ json!({"cdp": [0xc3, 0xa9, 0xe4, 0xb8, 0xad], "web": [0xc3, 0xa9, 0xe4, 0xb8, 0xad], "override": [0xc3, 0xa9]})
+ } else {
+ json!({"cdp": [0xc3, 0xa9, 0xe4, 0xb8, 0xad], "web": [0xe9, 0xff], "override": [0xe9]})
+ };
+ ctx.expect_result(
+ 5,
+ json!({"result": {"type": "string", "value": expected.to_string()}}),
+ Some("SID-1"),
+ );
+ }
+ server.abort();
+}
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
index 5dd1589de..bd9ce2a8a 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
@@ -208,7 +208,7 @@ impl JsContextHost {
.with_network_partition_key(network_partition_key);
let request_url = request.url.as_str().to_owned();
let request_method = request.method.clone();
- let request_headers = request.request_headers.clone();
+ let request_headers = request.request_headers.to_byte_strings();
if let Some(client_id) = service_worker_client_id
&& let Some(response) = browser_context_runtime
.fetch_service_worker_child_main_resource_for_reserved_client(
diff --git a/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs b/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs
index 59243ea4e..6ea4e8be6 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/service_workers.rs
@@ -1693,7 +1693,7 @@ impl JsContextHost {
client_id: ServiceWorkerClientId,
url: Url,
method: String,
- headers: Vec<(String, String)>,
+ headers: impl Into<moli_fetch::RequestHeaders>,
body: Option<Vec<u8>>,
destination: ServiceWorkerRequestDestination,
request_mode: moli_fetch::RequestMode,
@@ -1707,7 +1707,7 @@ impl JsContextHost {
resulting_client_id: None,
url,
method,
- headers,
+ headers: headers.into().to_byte_strings(),
body,
destination,
request_mode,
diff --git a/moli-renderer-v8/src/network/backend/memory_cache.rs b/moli-renderer-v8/src/network/backend/memory_cache.rs
index ec4bfaff9..684637283 100644
--- a/moli-renderer-v8/src/network/backend/memory_cache.rs
+++ b/moli-renderer-v8/src/network/backend/memory_cache.rs
@@ -616,7 +616,7 @@ pub(in crate::network) fn script_text_cache_key(request: &Request) -> ScriptText
integrity: request
.subresource_request_metadata()
.and_then(|metadata| metadata.integrity.clone()),
- request_headers: request.request_headers.clone(),
+ request_headers: request.request_headers.to_byte_strings(),
}
}
diff --git a/moli-renderer-v8/src/network/context/tests.rs b/moli-renderer-v8/src/network/context/tests.rs
index 6cc2e6862..dea733c0e 100644
--- a/moli-renderer-v8/src/network/context/tests.rs
+++ b/moli-renderer-v8/src/network/context/tests.rs
@@ -245,12 +245,12 @@ async fn registry_replaces_future_transport_without_rebinding_existing_loads() {
)
.expect("replacement request policy");
assert_eq!(
- existing_request.request_headers,
+ existing_request.request_headers.to_byte_strings(),
vec![("x-request-policy".to_owned(), "original".to_owned())],
"an in-flight request keeps its prepared configuration"
);
assert_eq!(
- replacement_request.request_headers,
+ replacement_request.request_headers.to_byte_strings(),
vec![("x-request-policy".to_owned(), "replacement".to_owned())],
"new requests use the replacement configuration"
);
diff --git a/moli-renderer-v8/src/network/policy.rs b/moli-renderer-v8/src/network/policy.rs
index 6237e2dc7..ac93404fb 100644
--- a/moli-renderer-v8/src/network/policy.rs
+++ b/moli-renderer-v8/src/network/policy.rs
@@ -479,20 +479,20 @@ impl PageNetworkPolicySnapshot {
fn merge_page_network_policy_headers(
context_headers: &[(Box<str>, Box<str>)],
- request_headers: &[(String, String)],
-) -> Vec<(String, String)> {
- let mut merged = IndexMap::<String, (String, String)>::new();
+ request_headers: &moli_fetch::RequestHeaders,
+) -> moli_fetch::RequestHeaders {
+ let mut merged = IndexMap::<String, (String, Vec<u8>)>::new();
for (name, value) in context_headers {
merged
.entry(header_name_key(name))
- .or_insert_with(|| (name.to_string(), value.to_string()));
+ .or_insert_with(|| (name.to_string(), value.as_bytes().to_vec()));
}
- for (name, value) in request_headers {
+ for (name, value) in request_headers.iter() {
let key = header_name_key(name);
merged.shift_remove(&key);
merged.insert(key, (name.clone(), value.clone()));
}
- merged.into_values().collect()
+ moli_fetch::RequestHeaders::from_bytes(merged.into_values().collect())
}
fn header_name_key(name: &str) -> String {
@@ -532,7 +532,7 @@ mod tests {
)
.unwrap();
assert_eq!(
- original.request_headers,
+ original.request_headers.to_byte_strings(),
vec![("x-owner".to_owned(), "first".to_owned())]
);
}
@@ -589,7 +589,7 @@ mod tests {
)
.unwrap();
assert_eq!(
- request.request_headers,
+ request.request_headers.to_byte_strings(),
vec![("x-policy-revision".to_owned(), "one".to_owned())]
);
assert!(policy.snapshot().network_offline());
@@ -632,7 +632,7 @@ mod tests {
)
.unwrap();
assert_eq!(
- request.request_headers,
+ request.request_headers.to_byte_strings(),
vec![("x-policy-revision".to_owned(), "one".to_owned())],
"request configuration must remain the one captured at registration"
);
diff --git a/moli-renderer-v8/src/network/request_client/tests.rs b/moli-renderer-v8/src/network/request_client/tests.rs
index 1791cf44b..d044fc321 100644
--- a/moli-renderer-v8/src/network/request_client/tests.rs
+++ b/moli-renderer-v8/src/network/request_client/tests.rs
@@ -370,7 +370,8 @@ fn merge_loader_network_policy_headers_uses_header_name_keys_and_request_order()
request.request_headers = vec![
("x-test".to_owned(), "request".to_owned()),
("X-New".to_owned(), "new".to_owned()),
- ];
+ ]
+ .into();
let merged = policy
.snapshot()
.apply_to_request(request)
@@ -378,7 +379,7 @@ fn merge_loader_network_policy_headers_uses_header_name_keys_and_request_order()
.request_headers;
assert_eq!(
- merged,
+ merged.to_byte_strings(),
vec![
("Accept".to_owned(), "text/html".to_owned()),
("x-test".to_owned(), "request".to_owned()),
diff --git a/moli-renderer-v8/src/network_host.rs b/moli-renderer-v8/src/network_host.rs
index 40ea8bf66..721aa70f0 100644
--- a/moli-renderer-v8/src/network_host.rs
+++ b/moli-renderer-v8/src/network_host.rs
@@ -167,6 +167,7 @@ pub(crate) use self::stylesheet_subresource::{
pub(crate) use self::text_track::{
TextTrackResourceFetchStart, start_text_track_resource_fetch, text_track_response_result,
};
+pub(crate) use self::url_helpers::merge_byte_string_request_headers;
pub(in crate::network_host) use self::url_helpers::merge_subresource_request_headers;
pub(crate) use self::url_helpers::resolve_context_url;
#[cfg(test)]
diff --git a/moli-renderer-v8/src/network_host/async_fetch.rs b/moli-renderer-v8/src/network_host/async_fetch.rs
index f2b193365..2c28c2aa4 100644
--- a/moli-renderer-v8/src/network_host/async_fetch.rs
+++ b/moli-renderer-v8/src/network_host/async_fetch.rs
@@ -23,7 +23,7 @@ pub(crate) async fn fetch_browser_subresource_with_preflight_and_network_metadat
request: Request,
cancel_handle: Option<FetchCancelHandle>,
) -> Result<NetworkFetchResult<Response>, String> {
- let preflight_request_headers = request.request_headers.clone();
+ let preflight_request_headers = request.request_headers.to_byte_strings();
fetch_browser_subresource_with_preflight_headers_and_observer(
loader,
request,
@@ -218,7 +218,7 @@ impl ManualCorsRedirectState {
});
self.request.apply_redirect_status(redirect_status);
self.request.url = next_url;
- self.preflight_request_headers = self.request.request_headers.clone();
+ self.preflight_request_headers = self.request.request_headers.to_byte_strings();
Ok(ManualCorsRedirectTransition::FollowedRedirect)
}
diff --git a/moli-renderer-v8/src/network_host/csp_reports.rs b/moli-renderer-v8/src/network_host/csp_reports.rs
index 08a1feb3c..2114ac219 100644
--- a/moli-renderer-v8/src/network_host/csp_reports.rs
+++ b/moli-renderer-v8/src/network_host/csp_reports.rs
@@ -340,7 +340,7 @@ fn dispatch_service_worker_content_security_policy_report(
internal_id,
request_url: request.url,
request_method: request.method,
- request_headers: request.request_headers,
+ request_headers: request.request_headers.to_byte_strings(),
request_body: request_body_text,
response_status_text: None,
skip_fetch_security_validation: true,
@@ -388,7 +388,7 @@ fn spawn_content_security_policy_report_fetch(
},
request.url,
request.method,
- request.request_headers,
+ request.request_headers.to_byte_strings(),
request_body_text,
);
}
@@ -407,7 +407,7 @@ fn report_subresource_fetch_info(
url: request.url.clone(),
websocket_socket_id: None,
method: request.method.clone(),
- request_headers: request.request_headers.clone(),
+ request_headers: request.request_headers.to_byte_strings(),
request_body: report_request_body_text(request),
request_body_bytes: request.body.clone(),
resource_type: SubresourceResourceType::CspReport,
diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs b/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs
index 6d0184a94..d13002a71 100644
--- a/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs
+++ b/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs
@@ -204,7 +204,8 @@ pub(super) fn spawn_network_fetch(
&prepared.method,
prepared.resolved_url.clone(),
prepared.body.clone(),
- prepared.request_headers.clone(),
+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)
+ .expect("prepared Fetch/XHR headers are ByteStrings"),
prepared.request_origin.clone(),
)
.with_initiator_url(&prepared.document_url)
diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/request.rs b/moli-renderer-v8/src/network_host/fetch/bindings/request.rs
index 5f40e78bc..60a60c2a0 100644
--- a/moli-renderer-v8/src/network_host/fetch/bindings/request.rs
+++ b/moli-renderer-v8/src/network_host/fetch/bindings/request.rs
@@ -76,7 +76,7 @@ pub(super) fn prepare_window_fetch_request<'s>(
request_headers.push(("Content-Type".to_owned(), String::new()));
}
let request_headers =
- merge_subresource_request_headers(host.extra_http_headers(), &request_headers);
+ merge_byte_string_request_headers(host.extra_http_headers(), &request_headers);
let resolved_url = resolve_context_url(&base_url, &parsed.url, None)?;
Ok(PreparedWindowFetchRequest {
diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs b/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs
index a1ca1ff50..a662529b1 100644
--- a/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs
+++ b/moli-renderer-v8/src/network_host/fetch/bindings/service_worker.rs
@@ -85,7 +85,8 @@ pub(super) fn dispatch_service_worker_fetch(
client_id,
prepared.resolved_url.clone(),
prepared.method.clone(),
- prepared.request_headers.clone(),
+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)
+ .expect("prepared Fetch/XHR headers are ByteStrings"),
prepared.body.clone(),
ServiceWorkerRequestDestination::Empty,
prepared.request_mode,
diff --git a/moli-renderer-v8/src/network_host/url_helpers.rs b/moli-renderer-v8/src/network_host/url_helpers.rs
index e68f564dc..3c7efa40e 100644
--- a/moli-renderer-v8/src/network_host/url_helpers.rs
+++ b/moli-renderer-v8/src/network_host/url_helpers.rs
@@ -36,6 +36,16 @@ pub(in crate::network_host) fn merge_subresource_request_headers(
merged.into_values().collect()
}
+/// Renderer Fetch/XHR state carries isomorphically decoded HTTP bytes. Convert
+/// the page's ordinary UTF-8 strings before merging them with WebIDL values.
+pub(crate) fn merge_byte_string_request_headers(
+ context_headers: &[(String, String)],
+ request_headers: &[(String, String)],
+) -> Vec<(String, String)> {
+ let context = moli_fetch::RequestHeaders::from(context_headers.to_vec()).to_byte_strings();
+ merge_subresource_request_headers(&context, request_headers)
+}
+
fn header_name_key(name: &str) -> String {
HeaderName::from_bytes(name.as_bytes())
.map(|name| name.as_str().to_owned())
diff --git a/moli-renderer-v8/src/network_host/xhr/send.rs b/moli-renderer-v8/src/network_host/xhr/send.rs
index 1189b64b9..c69a63d3a 100644
--- a/moli-renderer-v8/src/network_host/xhr/send.rs
+++ b/moli-renderer-v8/src/network_host/xhr/send.rs
@@ -287,7 +287,8 @@ fn send_synchronous_network_xhr(
&prepared.method,
prepared.resolved_url.as_str(),
prepared.send_body.clone(),
- prepared.request_headers.clone(),
+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)
+ .expect("prepared Fetch/XHR headers are ByteStrings"),
)
.expect("xhr request url was already resolved")
.with_initiator_url(&prepared.document_url)
diff --git a/moli-renderer-v8/src/network_host/xhr/send/paths.rs b/moli-renderer-v8/src/network_host/xhr/send/paths.rs
index 56a0698d3..be8dce53b 100644
--- a/moli-renderer-v8/src/network_host/xhr/send/paths.rs
+++ b/moli-renderer-v8/src/network_host/xhr/send/paths.rs
@@ -105,7 +105,8 @@ pub(super) fn dispatch_service_worker_xhr(
client_id,
prepared.resolved_url.clone(),
prepared.method.clone(),
- prepared.request_headers.clone(),
+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)
+ .expect("prepared Fetch/XHR headers are ByteStrings"),
prepared.send_body.clone(),
ServiceWorkerRequestDestination::Empty,
moli_fetch::RequestMode::Cors,
@@ -301,7 +302,8 @@ pub(super) fn spawn_network_xhr_fetch(
&prepared.method,
prepared.resolved_url.as_str(),
prepared.send_body.clone(),
- prepared.request_headers.clone(),
+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)
+ .expect("prepared Fetch/XHR headers are ByteStrings"),
)
.expect("xhr request url was already resolved")
.with_initiator_url(&prepared.document_url)
diff --git a/moli-renderer-v8/src/network_host/xhr/send/request.rs b/moli-renderer-v8/src/network_host/xhr/send/request.rs
index bc426f130..129f55767 100644
--- a/moli-renderer-v8/src/network_host/xhr/send/request.rs
+++ b/moli-renderer-v8/src/network_host/xhr/send/request.rs
@@ -208,7 +208,7 @@ fn xhr_request_headers(
default_content_type,
suppress_default_content_type,
);
- let merged = merge_subresource_request_headers(host.extra_http_headers(), &author_headers);
+ let merged = merge_byte_string_request_headers(host.extra_http_headers(), &author_headers);
(merged, author_headers)
}
diff --git a/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs b/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs
index ef8a53b47..800f7a6d9 100644
--- a/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs
+++ b/moli-renderer-v8/src/runtime/browser_context_runtime/service_workers.rs
@@ -204,7 +204,7 @@ impl RendererBrowserContextRuntime {
resulting_client_id: Some(client_id),
url: request.url.clone(),
method: request.method.clone(),
- headers: request.request_headers.clone(),
+ headers: request.request_headers.to_byte_strings(),
body: request.body.clone(),
destination,
request_mode: request.request_mode,
@@ -275,7 +275,7 @@ impl RendererBrowserContextRuntime {
resulting_client_id: None,
url: request.url.clone(),
method: request.method.clone(),
- headers: request.request_headers.clone(),
+ headers: request.request_headers.to_byte_strings(),
body: request.body.clone(),
destination,
request_mode: request.request_mode,
diff --git a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
index 74ad7a85a..ecae78a67 100644
--- a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
+++ b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
@@ -1361,7 +1361,7 @@ mod tests {
assert_eq!(request.method, "POST");
assert_eq!(request.body, Some(body));
assert_eq!(
- request.request_headers,
+ request.request_headers.to_byte_strings(),
vec![(
"Content-Type".to_owned(),
"application/x-www-form-urlencoded".to_owned(),
diff --git a/moli-renderer-v8/src/script_vm/subresource_fetch.rs b/moli-renderer-v8/src/script_vm/subresource_fetch.rs
index ee93862e5..8a8ac06e3 100644
--- a/moli-renderer-v8/src/script_vm/subresource_fetch.rs
+++ b/moli-renderer-v8/src/script_vm/subresource_fetch.rs
@@ -1409,11 +1409,19 @@ impl ScriptVm {
// up the ambient Page loader here would silently rebind policy/backend
// to a newer Document identity.
let loader = pending.load.request_client();
+ let encoded_headers = if matches!(
+ pending.info.resource_type,
+ SubresourceResourceType::Fetch | SubresourceResourceType::Xhr
+ ) {
+ moli_fetch::RequestHeaders::from_byte_strings(&request_headers)?
+ } else {
+ moli_fetch::RequestHeaders::from(request_headers.clone())
+ };
let mut request = moli_fetch::Request::new(
&request_method,
request_url.as_str(),
request_body.clone(),
- request_headers.clone(),
+ encoded_headers,
)?
.with_initiator_url(&pending.info.document_url)
.with_request_origin(pending.request_origin.clone())
@@ -1679,11 +1687,19 @@ impl ScriptVm {
));
}
let loader = pending_fetch.load.request_client();
+ let encoded_headers = if matches!(
+ pending_fetch.info.resource_type,
+ SubresourceResourceType::Fetch | SubresourceResourceType::Xhr
+ ) {
+ moli_fetch::RequestHeaders::from_byte_strings(&original_request_headers)?
+ } else {
+ moli_fetch::RequestHeaders::from(original_request_headers.clone())
+ };
let mut request = moli_fetch::Request::new(
&request_method,
request_url.as_str(),
request_body.clone(),
- original_request_headers.clone(),
+ encoded_headers,
)?
.with_initiator_url(&pending_fetch.info.document_url)
.with_request_origin(pending_fetch.request_origin.clone())
diff --git a/moli-renderer-v8/src/service_worker_runtime/events.rs b/moli-renderer-v8/src/service_worker_runtime/events.rs
index 30480cb1f..042a08819 100644
--- a/moli-renderer-v8/src/service_worker_runtime/events.rs
+++ b/moli-renderer-v8/src/service_worker_runtime/events.rs
@@ -470,6 +470,7 @@ pub(crate) struct ServiceWorkerFetchRequest {
pub(crate) resulting_client_id: Option<ServiceWorkerClientId>,
pub(crate) url: Url,
pub(crate) method: String,
+ /// Isomorphically decoded wire bytes, ready for the worker's Fetch Headers.
pub(crate) headers: Vec<(String, String)>,
pub(crate) body: Option<Vec<u8>>,
pub(crate) destination: ServiceWorkerRequestDestination,
diff --git a/moli-renderer-v8/src/service_worker_runtime/service/event_completion.rs b/moli-renderer-v8/src/service_worker_runtime/service/event_completion.rs
index 900ea27a8..8bae2c0ef 100644
--- a/moli-renderer-v8/src/service_worker_runtime/service/event_completion.rs
+++ b/moli-renderer-v8/src/service_worker_runtime/service/event_completion.rs
@@ -529,7 +529,7 @@ pub(super) fn service_worker_fetch_diagnostic_from_job_result(
document_url: job.network_context.document_url.as_str().to_owned(),
request_url: job.request.url.as_str().to_owned(),
method: job.request.method.clone(),
- request_headers: job.request.request_headers.clone(),
+ request_headers: job.request.request_headers.to_byte_strings(),
request_body: request_body_text(&job.request.body),
destination: service_worker_fetch_diagnostic_destination(job.network_context.resource_type)
.to_owned(),
diff --git a/moli-renderer-v8/src/service_worker_runtime/service/event_dispatch.rs b/moli-renderer-v8/src/service_worker_runtime/service/event_dispatch.rs
index 719c49292..578bec6cb 100644
--- a/moli-renderer-v8/src/service_worker_runtime/service/event_dispatch.rs
+++ b/moli-renderer-v8/src/service_worker_runtime/service/event_dispatch.rs
@@ -40,10 +40,12 @@ fn navigation_preload_request_for_job(
) -> Result<moli_fetch::Request, String> {
let mut headers = job.request.request_headers.clone();
headers.retain(|(name, _)| !name.eq_ignore_ascii_case("service-worker-navigation-preload"));
- headers.push((
+ let preload_header = moli_fetch::RequestHeaders::from_byte_strings(&[(
"Service-Worker-Navigation-Preload".to_owned(),
header_value.to_owned(),
- ));
+ )])
+ .map_err(|error| error.to_string())?;
+ headers.extend_from_slice(&preload_header);
let mut request = super::fetch_settlement::service_worker_network_fallback_request_for_job(job);
request.request_headers = headers;
Ok(request
@@ -247,7 +249,8 @@ impl ServiceWorkerRuntimeService {
&request.method,
request.url.clone(),
request.body.clone(),
- request.headers.clone(),
+ moli_fetch::RequestHeaders::from_byte_strings(&request.headers)
+ .expect("service worker request headers are ByteStrings"),
origin,
)
.with_initiator_url(&dispatch.network_context.document_url)
diff --git a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs
index 3827aaea6..7afa42c73 100644
--- a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs
+++ b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs
@@ -55,7 +55,7 @@ fn fetch_request_for_job(job: &ServiceWorkerFetchJob) -> ServiceWorkerFetchReque
resulting_client_id: job.resulting_client_id,
url: job.request.url.clone(),
method: job.request.method.clone(),
- headers: job.request.request_headers.clone(),
+ headers: job.request.request_headers.to_byte_strings(),
body: job.request.body.clone(),
destination: job.destination,
request_mode: job.request.request_mode,
@@ -167,7 +167,7 @@ fn apply_service_worker_synthetic_redirect(
negotiated_http_version: None,
})
.map_err(|error| error.to_string())?;
- job.cors_preflight_request_headers = job.request.request_headers.clone();
+ job.cors_preflight_request_headers = job.request.request_headers.to_byte_strings();
Ok(())
}
@@ -201,7 +201,7 @@ impl ServiceWorkerRuntimeService {
resulting_client_id: Some(client_id),
url: request.url.clone(),
method: request.method.clone(),
- headers: request.request_headers.clone(),
+ headers: request.request_headers.to_byte_strings(),
body: request.body.clone(),
destination,
request_mode: request.request_mode,
@@ -267,7 +267,7 @@ impl ServiceWorkerRuntimeService {
job.network_context,
job.request.url,
job.request.method,
- job.request.request_headers,
+ job.request.request_headers.to_byte_strings(),
request_body_text(&job.request.body),
);
}
@@ -385,7 +385,7 @@ impl ServiceWorkerRuntimeService {
internal_id: job.internal_id,
request_url: job.request.url.clone(),
request_method: job.request.method.clone(),
- request_headers: job.request.request_headers.clone(),
+ request_headers: job.request.request_headers.to_byte_strings(),
request_body: request_body_text(&job.request.body),
body_source_id: started.body_source_id,
network_request_headers: None,
@@ -638,7 +638,7 @@ impl ServiceWorkerRuntimeService {
internal_id: job.internal_id,
request_url: job.request.url,
request_method: job.request.method,
- request_headers: job.request.request_headers,
+ request_headers: job.request.request_headers.to_byte_strings(),
request_body: request_body_text(&job.request.body),
response_status_text: Some(response.status_text),
skip_fetch_security_validation: true,
@@ -691,7 +691,7 @@ impl ServiceWorkerRuntimeService {
internal_id: job.internal_id,
request_url: job.request.url,
request_method: job.request.method,
- request_headers: job.request.request_headers,
+ request_headers: job.request.request_headers.to_byte_strings(),
request_body: request_body_text(&job.request.body),
response_status_text: None,
skip_fetch_security_validation: false,
@@ -1960,12 +1960,12 @@ mod tests {
assert_eq!(request_body_text(&job.request.body), None);
assert_eq!(job.request.body, None);
assert_eq!(
- job.request.request_headers,
+ job.request.request_headers.to_byte_strings(),
vec![("x-keep".to_owned(), "yes".to_owned())]
);
assert_eq!(
job.cors_preflight_request_headers,
- job.request.request_headers
+ job.request.request_headers.to_byte_strings()
);
assert_eq!(job.request.redirect_count(), 1);
assert_eq!(job.request.redirect_chain().len(), 1);
diff --git a/moli-renderer-v8/src/worker/global_scope/content_security_policy.rs b/moli-renderer-v8/src/worker/global_scope/content_security_policy.rs
index b704ea406..55568a024 100644
--- a/moli-renderer-v8/src/worker/global_scope/content_security_policy.rs
+++ b/moli-renderer-v8/src/worker/global_scope/content_security_policy.rs
@@ -288,7 +288,7 @@ fn pause_worker_content_security_policy_report_for_fetch_interception(
url: request.url.clone(),
websocket_socket_id: None,
method: request.method.clone(),
- request_headers: request.request_headers.clone(),
+ request_headers: request.request_headers.to_byte_strings(),
request_body: request_body.clone(),
request_body_bytes,
resource_type: SubresourceResourceType::CspReport,
@@ -373,7 +373,7 @@ fn dispatch_worker_content_security_policy_report_to_service_worker(
resulting_client_id: None,
url: request.url.clone(),
method: request.method.clone(),
- headers: request.request_headers.clone(),
+ headers: request.request_headers.to_byte_strings(),
body: request.body.clone(),
destination: ServiceWorkerRequestDestination::Report,
request_mode: request.request_mode,
@@ -687,7 +687,7 @@ fn record_worker_content_security_policy_report_failure(
document_url,
request.url,
request.method,
- request.request_headers,
+ request.request_headers.to_byte_strings(),
request_body,
SubresourceResourceType::CspReport,
message,
@@ -713,7 +713,7 @@ fn send_worker_content_security_policy_report_success(
document_url,
request.url,
request.method,
- request.request_headers,
+ request.request_headers.to_byte_strings(),
request_body,
SubresourceResourceType::CspReport,
head.request_cookie_report.clone(),
@@ -751,7 +751,7 @@ fn send_worker_content_security_policy_report_failure(
document_url,
request.url,
request.method,
- request.request_headers,
+ request.request_headers.to_byte_strings(),
request_body,
SubresourceResourceType::CspReport,
message,
diff --git a/moli-renderer-v8/src/worker/global_scope/fetch.rs b/moli-renderer-v8/src/worker/global_scope/fetch.rs
index 26e34d42a..6bd52934b 100644
--- a/moli-renderer-v8/src/worker/global_scope/fetch.rs
+++ b/moli-renderer-v8/src/worker/global_scope/fetch.rs
@@ -184,7 +184,8 @@ pub(in crate::worker) fn spawn_worker_fetch_network(
&method,
resolved_url.as_str(),
body,
- headers,
+ moli_fetch::RequestHeaders::from_byte_strings(&headers)
+ .expect("prepared worker Fetch headers are ByteStrings"),
moli_url::WebOrigin::from_url(&document_url),
) {
Ok(request) => {
@@ -519,7 +520,8 @@ pub(in crate::worker) fn spawn_worker_xhr_network(
&method,
resolved_url.as_str(),
body,
- headers,
+ moli_fetch::RequestHeaders::from_byte_strings(&headers)
+ .expect("prepared worker XHR headers are ByteStrings"),
moli_url::WebOrigin::from_url(&document_url),
)
.map(|request| {
diff --git a/moli-renderer-v8/src/worker/global_scope/mod.rs b/moli-renderer-v8/src/worker/global_scope/mod.rs
index 89c8399e3..986ae5d54 100644
--- a/moli-renderer-v8/src/worker/global_scope/mod.rs
+++ b/moli-renderer-v8/src/worker/global_scope/mod.rs
@@ -6555,7 +6555,8 @@ fn merge_worker_request_headers(
context_headers: &[(String, String)],
request_headers: &[(String, String)],
) -> Vec<(String, String)> {
- let mut merged = context_headers.to_vec();
+ // Worker Fetch/XHR headers are ByteStrings; page extra headers are UTF-8.
+ let mut merged = moli_fetch::RequestHeaders::from(context_headers.to_vec()).to_byte_strings();
for (name, value) in request_headers {
let lower = name.to_ascii_lowercase();
merged.retain(|(existing_name, _)| existing_name.to_ascii_lowercase() != lower);
diff --git a/moli-renderer-v8/src/worker/global_scope/xhr.rs b/moli-renderer-v8/src/worker/global_scope/xhr.rs
index 09bfad621..3e859973b 100644
--- a/moli-renderer-v8/src/worker/global_scope/xhr.rs
+++ b/moli-renderer-v8/src/worker/global_scope/xhr.rs
@@ -566,7 +566,8 @@ fn send_synchronous_worker_xhr(
&prepared.method,
prepared.resolved_url.as_str(),
prepared.send_body.clone(),
- prepared.request_headers.clone(),
+ moli_fetch::RequestHeaders::from_byte_strings(&prepared.request_headers)
+ .expect("prepared Fetch/XHR headers are ByteStrings"),
) {
Ok(request) => {
let mut request = request
diff --git a/moli/src/app.rs b/moli/src/app.rs
index 60c91604b..a3f4a5133 100644
--- a/moli/src/app.rs
+++ b/moli/src/app.rs
@@ -215,7 +215,7 @@ fn read_script_file_arg(option: &str, path: &Path) -> Result<String> {
fn build_fetch_request(url: &str, config: &AppConfig) -> Result<Request> {
let mut request = Request::get(url)?;
// Keep CLI-provided headers scoped to the initial document navigation.
- request.request_headers = config.fetch.request_headers.clone();
+ request.request_headers = config.fetch.request_headers.clone().into();
Ok(request)
}
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index f35d96720..19dabe4c8 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -1872,6 +1872,40 @@ fn cli_fetch_repeats_custom_request_headers() -> Result<()> {
Ok(())
}
+#[test]
+fn cli_fetch_keeps_unicode_header_values_as_utf8_bytes() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let listener = runtime.block_on(TcpListener::bind("127.0.0.1:0"))?;
+ let url = format!("http://{}/", listener.local_addr()?);
+ let server = runtime.spawn(async move {
+ let (mut stream, _) = listener.accept().await?;
+ let mut request = Vec::new();
+ while !request.windows(4).any(|part| part == b"\r\n\r\n") {
+ let mut chunk = [0; 1024];
+ let count = stream.read(&mut chunk).await?;
+ anyhow::ensure!(count != 0 && request.len() < 64 * 1024, "incomplete request");
+ request.extend_from_slice(&chunk[..count]);
+ }
+ stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await?;
+ Ok::<_, anyhow::Error>(request)
+ });
+ let output = run_fetch_cli_with_args(&url, &["-H", "X-Utf8: é中"])?;
+ assert!(
+ output.status.success(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let request = runtime.block_on(async {
+ tokio::time::timeout(std::time::Duration::from_secs(5), server).await
+ })???;
+ assert!(
+ request
+ .windows("X-Utf8: é中\r\n".len())
+ .any(|part| part == "X-Utf8: é中\r\n".as_bytes())
+ );
+ Ok(())
+}
+
#[test]
fn cli_fetch_preserves_header_values_with_embedded_colons_and_empty_values() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;