Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): preserve explicit empty Content-Type headers
ldm014f6d073be8f
diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex 5c78f0aad..078f6bf9e 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -698,16 +698,10 @@ pub(crate) fn configure_easy(let mut has_content_type_header = false;for (name, value) in outgoing_headers.iter() {- let is_content_type = name.eq_ignore_ascii_case("content-type");- has_content_type_header |= is_content_type;+ has_content_type_header |= name.eq_ignore_ascii_case("content-type");let mut header_line = name.as_bytes().to_vec();// curl sends an empty field for `Name;`, but removes it for `Name:`.- // Empty Content-Type remains the internal upload-default suppression marker.- header_line.push(if value.is_empty() && !is_content_type {- b';'- } else {- b':'- });+ header_line.push(if value.is_empty() { b';' } else { b':' });if !value.is_empty() {header_line.push(b' ');header_line.extend_from_slice(value);@@ -728,13 +722,10 @@ pub(crate) fn configure_easy(.context("failed to build cache validation request header")?;}}- if (request.method.eq_ignore_ascii_case("POST")- || (request.method == "PUT" && request.body.is_none()))- && !has_content_type_header- {- // libcurl otherwise synthesizes `Content-Type: application/x-www-form-urlencoded`- // for POST bodies and bodyless PUT requests. Browser requests only send Content-Type when- // BodyInit or caller headers produce one, so suppress curl's transport default.+ if !has_content_type_header {+ // BodyInit and caller headers determine whether Content-Type is present.+ // Suppress libcurl's upload default only in the curl header list, keeping+ // this transport instruction distinct from an explicit empty header.headers.append("Content-Type:").context("failed to suppress curl default content-type")?;diff --git a/moli-fetch/src/tests/header_bytes.rs b/moli-fetch/src/tests/header_bytes.rsindex c84094de0..0dd2b364b 100644--- a/moli-fetch/src/tests/header_bytes.rs+++ b/moli-fetch/src/tests/header_bytes.rs@@ -53,6 +53,64 @@ async fn fetch_in_mode(Ok((head, body.into_materialized_bytes().await?))}+#[tokio::test]+async fn request_content_type_distinguishes_absent_empty_and_nonempty_in_all_transports()+-> Result<()> {+ let cases = [+ ("GET", None),+ ("POST", None),+ ("POST", Some("")),+ ("POST", Some("body")),+ ("PUT", None),+ ("PUT", Some("body")),+ ("PATCH", Some("body")),+ ("DELETE", Some("body")),+ ];+ for mode in ["buffered", "html", "raw"] {+ for content_type in [Some(""), None, Some("application/custom")] {+ let server = ScriptedHttpServer::spawn(+ cases.iter().map(|_| ScriptedResponse::ok("ok")).collect(),+ );+ let client =+ FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ for (method, body) in cases {+ let mut headers = vec![("X-Test".to_owned(), String::new())];+ if let Some(value) = content_type {+ headers.push(("cOnTeNt-TyPe".to_owned(), value.to_owned()));+ }+ let recorder = NetworkObservationRecorder::default();+ let request =+ Request::new(method, &server.url(), body.map(str::to_owned), headers)?+ .with_network_observation_recorder(recorder.clone());+ fetch_in_mode(&client, request, mode).await?;+ let journal = recorder.snapshot();+ let observed = journal.final_request_observation().unwrap().headers();+ assert_eq!(+ observed+ .iter()+ .find(|(name, _)| name.eq_ignore_ascii_case("content-type"))+ .map(|(_, value)| value.as_str()),+ content_type,+ "{mode}, {method}, body={body:?}: {observed:?}"+ );+ }+ let requests = server.requests();+ server.shutdown();+ assert!(client.shutdown().is_clean());+ assert_eq!(requests.len(), cases.len());+ for (request, (method, body)) in requests.iter().zip(cases) {+ assert_eq!(+ request_head_header_value(request, "content-type"),+ content_type,+ "{mode}, {method}, body={body:?}: {request}"+ );+ assert_eq!(request_head_header_value(request, "x-test"), Some(""));+ }+ }+ }+ Ok(())+}+#[tokio::test]async fn http_header_bytes_survive_all_transports_and_observation() -> Result<()> {let bytes: Vec<u8> = (0x80..=0xff).collect();diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/request.rs b/moli-renderer-v8/src/network_host/fetch/bindings/request.rsindex b9862a850..f7df440a6 100644--- a/moli-renderer-v8/src/network_host/fetch/bindings/request.rs+++ b/moli-renderer-v8/src/network_host/fetch/bindings/request.rs@@ -40,7 +40,7 @@ pub(super) fn prepare_window_fetch_request<'s>(fetch_context: crate::native_bridge::WindowFetchContext,host: &JsContextHost,) -> Result<PreparedWindowFetchRequest, String> {- let mut request_headers = parsed.headers;+ let request_headers = parsed.headers;// Receiver capture and WebIDL conversion are complete before this pure// preparation stage. Never inspect `args.this()` here: doing so could bind// the operation to a replacement LocalWindow after an author getter@@ -72,9 +72,6 @@ pub(super) fn prepare_window_fetch_request<'s>(let policy_context = effective_subresource_policy_context(scope, host, request_scope);let network_partition_key = active_subresource_network_partition_key(host, request_scope);let cors_preflight_request_headers = request_headers.clone();- if parsed.suppress_default_content_type {- request_headers.push(("Content-Type".to_owned(), String::new()));- }let request_headers =merge_byte_string_request_headers(host.extra_http_headers(), &request_headers);let resolved_url = resolve_context_url(&base_url, &parsed.url, None)?;diff --git a/moli-renderer-v8/src/network_host/fetch/input.rs b/moli-renderer-v8/src/network_host/fetch/input.rsindex 115ee19bf..f5cdb04f2 100644--- a/moli-renderer-v8/src/network_host/fetch/input.rs+++ b/moli-renderer-v8/src/network_host/fetch/input.rs@@ -7,7 +7,6 @@ pub(super) struct ParsedWindowFetchInput {pub(super) method: String,pub(super) body: Option<Vec<u8>>,pub(super) headers: Vec<(String, String)>,- pub(super) suppress_default_content_type: bool,pub(super) request_mode: moli_fetch::RequestMode,pub(super) credentials_mode: moli_fetch::RequestCredentialsMode,pub(super) redirect_mode: moli_fetch::RequestRedirectMode,@@ -53,17 +52,9 @@ pub(super) fn parse_window_fetch_input<'s>(} else {inherited.headers.clone()};- let suppress_default_content_type = if init.body_present {- if !init.headers_present {- append_default_body_content_type(&mut headers, init.body_content_type.as_deref());- }- init.suppress_default_content_type- || (body.is_some()- && init.body_content_type.is_none()- && !has_header(&headers, "content-type"))- } else {- body.is_some() && !has_header(&headers, "content-type")- };+ if init.body_present && !init.headers_present {+ append_default_body_content_type(&mut headers, init.body_content_type.as_deref());+ }let inherited_credentials = request_object_credentials_mode(scope, req_obj)?;let request_mode = init.request_mode@@ -90,7 +81,6 @@ pub(super) fn parse_window_fetch_input<'s>(method,body,headers,- suppress_default_content_type,request_mode,credentials_mode,redirect_mode,@@ -123,7 +113,6 @@ pub(super) fn parse_window_fetch_input<'s>(method: init.method,body: init.body,headers,- suppress_default_content_type: init.suppress_default_content_type,request_mode,credentials_mode,redirect_mode,diff --git a/moli-renderer-v8/src/network_host/request/init.rs b/moli-renderer-v8/src/network_host/request/init.rsindex 31156c30b..0e42573ba 100644--- a/moli-renderer-v8/src/network_host/request/init.rs+++ b/moli-renderer-v8/src/network_host/request/init.rs@@ -13,7 +13,6 @@ pub(crate) struct ParsedFetchInit {pub(crate) body_content_type: Option<String>,pub(crate) headers: Vec<(String, String)>,pub(crate) headers_present: bool,- pub(crate) suppress_default_content_type: bool,pub(crate) request_mode: Option<RequestMode>,pub(crate) credentials_mode: Option<RequestCredentialsMode>,pub(crate) redirect_mode: Option<RequestRedirectMode>,@@ -35,7 +34,6 @@ impl Default for ParsedFetchInit {body_content_type: None,headers: Vec::new(),headers_present: false,- suppress_default_content_type: false,request_mode: None,credentials_mode: None,redirect_mode: None,@@ -156,14 +154,7 @@ pub(crate) fn parse_fetch_init<'s>(.has(scope, v8str(scope, "headers").into()).unwrap_or(false);let mut extra_headers = init.headers.unwrap_or_default();- let mut suppress_default_content_type = false;- if let Some(prepared_body) = prepared_body.as_ref() {- if let Some(content_type) = prepared_body.content_type.as_deref() {- append_default_body_content_type(&mut extra_headers, Some(content_type));- } else if !has_header(&extra_headers, "content-type") {- suppress_default_content_type = true;- }- }+ append_default_body_content_type(&mut extra_headers, body_content_type.as_deref());let request_mode = init.mode.map(|value| value.0);let credentials_mode = init.credentials_mode.map(|value| value.0);@@ -180,7 +171,6 @@ pub(crate) fn parse_fetch_init<'s>(body_content_type,headers: extra_headers,headers_present,- suppress_default_content_type,request_mode,credentials_mode,redirect_mode,diff --git a/moli-renderer-v8/src/network_host/xhr/send/request.rs b/moli-renderer-v8/src/network_host/xhr/send/request.rsindex d0bb1705a..dc95803d1 100644--- a/moli-renderer-v8/src/network_host/xhr/send/request.rs+++ b/moli-renderer-v8/src/network_host/xhr/send/request.rs@@ -80,13 +80,8 @@ pub(super) fn prepare_xhr_send_request<'s>(let network_partition_key = active_subresource_network_partition_key(host, owner);let resolved_url =resolve_context_url(&base_url, &url_str, None).map_err(XhrSendPrepareError::Url)?;- let (request_headers, cors_preflight_request_headers) = xhr_request_headers(- scope,- host,- xhr,- prepared_body.default_content_type,- prepared_body.suppress_default_content_type,- );+ let (request_headers, cors_preflight_request_headers) =+ xhr_request_headers(scope, host, xhr, prepared_body.default_content_type);let credentials_mode =if xhr_state_bool_property(scope, xhr, XHR_WITH_CREDENTIALS_SLOT).unwrap_or(false) {moli_fetch::RequestCredentialsMode::Include@@ -115,7 +110,6 @@ pub(super) fn prepare_xhr_send_request<'s>(pub(crate) struct PreparedXhrSendBody {pub(crate) body: Option<Vec<u8>>,pub(crate) default_content_type: Option<String>,- pub(crate) suppress_default_content_type: bool,}impl PreparedXhrSendBody {@@ -123,16 +117,13 @@ impl PreparedXhrSendBody {Self {body: None,default_content_type: None,- suppress_default_content_type: false,}}fn new(body: Vec<u8>, default_content_type: Option<String>) -> Self {- let suppress_default_content_type = default_content_type.is_none();Self {body: Some(body),default_content_type,- suppress_default_content_type,}}}@@ -200,14 +191,8 @@ fn xhr_request_headers(host: &JsContextHost,xhr: v8::Local<'_, v8::Object>,default_content_type: Option<String>,- suppress_default_content_type: bool,) -> (moli_fetch::RequestHeaders, Vec<(String, String)>) {- let author_headers = xhr_author_request_headers(- scope,- xhr,- default_content_type,- suppress_default_content_type,- );+ let author_headers = xhr_author_request_headers(scope, xhr, default_content_type);let merged = merge_byte_string_request_headers(host.extra_http_headers(), &author_headers);(merged, author_headers)}@@ -216,7 +201,6 @@ pub(crate) fn xhr_author_request_headers(scope: &mut v8::PinScope<'_, '_>,xhr: v8::Local<'_, v8::Object>,default_content_type: Option<String>,- suppress_default_content_type: bool,) -> Vec<(String, String)> {let headers_json = xhr_state_string_property(scope, xhr, XHR_REQUEST_HEADERS_SLOT).unwrap_or_else(|| "[]".to_owned());@@ -229,10 +213,6 @@ pub(crate) fn xhr_author_request_headers(&& !has_header(&author_headers, CONTENT_TYPE_HEADER){author_headers.push((CONTENT_TYPE_HEADER.to_owned(), default_content_type));- } else if suppress_default_content_type && !has_header(&author_headers, CONTENT_TYPE_HEADER) {- // An empty header value is intentional: the fetch transport serializes this- // as `Content-Type:` so the HTTP stack does not synthesize its own upload default.- author_headers.push((CONTENT_TYPE_HEADER.to_owned(), String::new()));}author_headers}diff --git a/moli-renderer-v8/src/worker/global_scope/fetch.rs b/moli-renderer-v8/src/worker/global_scope/fetch.rsindex 8728dece9..cbbb1e4af 100644--- a/moli-renderer-v8/src/worker/global_scope/fetch.rs+++ b/moli-renderer-v8/src/worker/global_scope/fetch.rs@@ -149,14 +149,13 @@ pub(in crate::worker) fn spawn_worker_fetch_network(resolved_url: Url,method: String,body: Option<Vec<u8>>,- mut headers: moli_fetch::RequestHeaders,+ headers: moli_fetch::RequestHeaders,request_mode: RequestMode,credentials_mode: RequestCredentialsMode,redirect_mode: RequestRedirectMode,priority: Option<moli_fetch::FetchPriorityHint>,request_metadata: ServiceWorkerFetchRequestMetadata,auth: Option<crate::protocol_types::SubresourceAuthCredentials>,- suppress_default_content_type: bool,allow_headers_first: bool,redirect_headers: Option<moli_fetch::RequestHeaders>,redirect_chain: Vec<moli_fetch::RedirectInfo>,@@ -177,11 +176,6 @@ pub(in crate::worker) fn spawn_worker_fetch_network()} else {let cors_preflight_request_headers = headers.to_byte_strings();- if suppress_default_content_type {- // An empty header value is intentional: the fetch transport serializes this- // as `Content-Type:` so the HTTP stack does not synthesize its own upload default.- headers.push(("Content-Type".to_owned(), Vec::new()));- }match Request::new_browser_bytes(&method,resolved_url.as_str(),@@ -403,7 +397,6 @@ fn spawn_worker_fetch_service_worker(redirect_mode: RequestRedirectMode,priority: Option<moli_fetch::FetchPriorityHint>,request_metadata: ServiceWorkerFetchRequestMetadata,- suppress_default_content_type: bool,) {let (direct_completion_tx, direct_completion_rx) = tokio::sync::oneshot::channel();let dispatch = ServiceWorkerFetchDispatch {@@ -471,7 +464,6 @@ fn spawn_worker_fetch_service_worker(priority,request_metadata,None,- suppress_default_content_type,true,None,Vec::new(),@@ -739,7 +731,6 @@ pub(in crate::worker) fn continue_pending_worker_fetch(priority,request_metadata,auth,- false,allow_headers_first,redirect_headers,redirect_chain,@@ -1829,7 +1820,6 @@ pub(in crate::worker) struct ResolvedWorkerFetchInput<'s> {method: String,body: Option<Vec<u8>>,headers: Vec<(String, String)>,- suppress_default_content_type: bool,request_mode: moli_fetch::RequestMode,credentials_mode: RequestCredentialsMode,redirect_mode: RequestRedirectMode,@@ -1858,7 +1848,6 @@ pub(in crate::worker) fn resolve_worker_fetch_input<'s>(method,body,headers,- suppress_default_content_type,request_mode,credentials_mode,redirect_mode,@@ -1885,17 +1874,9 @@ pub(in crate::worker) fn resolve_worker_fetch_input<'s>(} else {inherited.headers.clone()};- let suppress_default_content_type = if init.body_present {- if !init.headers_present {- append_default_body_content_type(&mut headers, init.body_content_type.as_deref());- }- init.suppress_default_content_type- || (body.is_some()- && init.body_content_type.is_none()- && !has_header(&headers, "content-type"))- } else {- body.is_some() && !has_header(&headers, "content-type")- };+ if init.body_present && !init.headers_present {+ append_default_body_content_type(&mut headers, init.body_content_type.as_deref());+ }let inherited_credentials = request_object_credentials_mode(scope, req_obj)?;let request_mode = init.request_mode@@ -1931,7 +1912,6 @@ pub(in crate::worker) fn resolve_worker_fetch_input<'s>(method,body,headers,- suppress_default_content_type,request_mode,credentials_mode,redirect_mode,@@ -1970,7 +1950,6 @@ pub(in crate::worker) fn resolve_worker_fetch_input<'s>(init.method,init.body,headers,- init.suppress_default_content_type,request_mode,credentials_mode,redirect_mode,@@ -1988,7 +1967,6 @@ pub(in crate::worker) fn resolve_worker_fetch_input<'s>(method,body,headers,- suppress_default_content_type,request_mode,credentials_mode,redirect_mode,@@ -2052,7 +2030,6 @@ pub(in crate::worker) fn worker_fetch_callback<'s>(method,body,headers: request_headers,- suppress_default_content_type,request_mode,credentials_mode,redirect_mode,@@ -2355,7 +2332,6 @@ pub(in crate::worker) fn worker_fetch_callback<'s>(redirect_mode,priority,request_metadata,- suppress_default_content_type,);} else {spawn_worker_fetch_network(@@ -2376,7 +2352,6 @@ pub(in crate::worker) fn worker_fetch_callback<'s>(priority,request_metadata,None,- suppress_default_content_type,true,None,Vec::new(),diff --git a/moli-renderer-v8/src/worker/global_scope/mod.rs b/moli-renderer-v8/src/worker/global_scope/mod.rsindex e2dcf7d44..8ea6ed283 100644--- a/moli-renderer-v8/src/worker/global_scope/mod.rs+++ b/moli-renderer-v8/src/worker/global_scope/mod.rs@@ -82,11 +82,11 @@ use crate::network_host::{error_pending_network_body_stream_with_reason, extract_subresource_auth_challenge,fetch_browser_subresource_raw_stream_with_preflight_headers_and_network_metadata,fetch_browser_subresource_with_preflight_headers_and_network_metadata,- filter_cors_exposed_response_headers, filter_headers_for_guard, has_header,- is_cors_policy_failure_message, local_url_response_result, parse_fetch_init,- prepare_xhr_send_body_from_args, request_input_snapshot, request_object_credentials_mode,- reset_xhr_response_for_request_error, resolve_context_url, set_xhr_state_bool,- set_xhr_state_number, throw_synchronous_xhr_failure, validate_fetch_response_security_policy,+ filter_cors_exposed_response_headers, filter_headers_for_guard, is_cors_policy_failure_message,+ local_url_response_result, parse_fetch_init, prepare_xhr_send_body_from_args,+ request_input_snapshot, request_object_credentials_mode, reset_xhr_response_for_request_error,+ resolve_context_url, set_xhr_state_bool, set_xhr_state_number, throw_synchronous_xhr_failure,+ validate_fetch_response_security_policy,validate_fetch_response_security_policy_with_body_classified, xhr_author_request_headers,xhr_dispatch_progress_event, xhr_ensure_send_allowed, xhr_state_bool_property,xhr_state_number_property, xhr_state_string_property,diff --git a/moli-renderer-v8/src/worker/global_scope/xhr.rs b/moli-renderer-v8/src/worker/global_scope/xhr.rsindex 8735918d3..2c46b8651 100644--- a/moli-renderer-v8/src/worker/global_scope/xhr.rs+++ b/moli-renderer-v8/src/worker/global_scope/xhr.rs@@ -1176,12 +1176,8 @@ pub(in crate::worker) fn prepare_worker_xhr_send_request<'s>(})?;let resolved_url = resolve_context_url(&document_url, &url_str, None).map_err(WorkerXhrSendPrepareError::Request)?;- let request_headers = xhr_author_request_headers(- scope,- xhr,- prepared_body.default_content_type,- prepared_body.suppress_default_content_type,- );+ let request_headers =+ xhr_author_request_headers(scope, xhr, prepared_body.default_content_type);let credentials_mode =if xhr_state_bool_property(scope, xhr, XHR_WITH_CREDENTIALS_SLOT).unwrap_or(false) {RequestCredentialsMode::Includediff --git a/moli-wpt-compat/fixtures/wpt/ported/fetch/fetch-body-content-type-basic.html b/moli-wpt-compat/fixtures/wpt/ported/fetch/fetch-body-content-type-basic.htmlindex c294b9e02..7b06fc9ae 100644--- a/moli-wpt-compat/fixtures/wpt/ported/fetch/fetch-body-content-type-basic.html+++ b/moli-wpt-compat/fixtures/wpt/ported/fetch/fetch-body-content-type-basic.html@@ -236,4 +236,62 @@ promise_test(async function () {"fetch(Request) should preserve inherited explicit BufferSource Content-Type",);}, "fetch() inherits Request body headers without adding transport defaults");++async function contentTypeVariants(url) {+ const results = [];+ const bodies = [+ ["no body", undefined, null],+ ["text", "hello", "text/plain;charset=UTF-8"],+ ["bytes", new Uint8Array([1, 2, 3]), null],+ ["untyped Blob", new Blob(["hello"]), null],+ ["typed Blob", new Blob(["hello"], { type: "text/custom" }), "text/custom"],+ ];+ for (const [label, body, defaultType] of bodies) {+ for (const contentType of [null, "", "application/custom"]) {+ const headers = contentType === null ? {} : { "cOnTeNt-TyPe": contentType };+ const init = { method: "POST", headers, body };+ const expected = contentType === null ? defaultType : contentType;+ const description = label + "/" + JSON.stringify(contentType);+ const direct = await fetch(url, init);+ results.push(["fetch/" + description, (await direct.json()).content_type, expected]);+ const inherited = await fetch(new Request(url, init));+ results.push(["fetch(Request)/" + description, (await inherited.json()).content_type, expected]);+ const xhrType = await new Promise(function (resolve, reject) {+ const xhr = new XMLHttpRequest();+ xhr.open("POST", url);+ if (contentType !== null) xhr.setRequestHeader("cOnTeNt-TyPe", contentType);+ xhr.responseType = "json";+ xhr.onload = function () { resolve(xhr.response.content_type); };+ xhr.onerror = function () { reject(new Error("XHR failed for " + description)); };+ xhr.send(body);+ });+ results.push(["XHR/" + description, xhrType, expected]);+ }+ }+ return results;+}++promise_test(async function () {+ for (const [label, actual, expected] of await contentTypeVariants(ECHO_BODY_HEX_URL)) {+ assert_equals(actual, expected, label);+ }+}, "Window fetch and XHR distinguish absent, empty and nonempty Content-Type on the wire");++promise_test(async function (t) {+ const url = new URL(ECHO_BODY_HEX_URL, location.href).href;+ const source = "(" + contentTypeVariants.toString() + ")(" + JSON.stringify(url) + ")" ++ ".then(results => postMessage({results}), error => postMessage({error: String(error)}));";+ const workerUrl = URL.createObjectURL(new Blob([source], { type: "text/javascript" }));+ t.add_cleanup(function () { URL.revokeObjectURL(workerUrl); });+ const worker = new Worker(workerUrl);+ t.add_cleanup(function () { worker.terminate(); });+ const message = await new Promise(function (resolve, reject) {+ worker.onmessage = resolve;+ worker.onerror = function (event) { reject(new Error(event.message)); };+ });+ assert_equals(message.data.error, undefined, "worker requests complete");+ for (const [label, actual, expected] of message.data.results) {+ assert_equals(actual, expected, label);+ }+}, "Worker fetch and XHR distinguish absent, empty and nonempty Content-Type on the wire");</script>