Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): reject non-GET blob URL requests
ldm08bcdbb299dd2
diff --git a/moli-renderer-v8/src/network/request_client.rs b/moli-renderer-v8/src/network/request_client.rsindex 70e3824b5..365954720 100644--- a/moli-renderer-v8/src/network/request_client.rs+++ b/moli-renderer-v8/src/network/request_client.rs@@ -282,7 +282,7 @@ impl ResourceRequestClient {request: Request,) -> Result<Response> {let request = self.apply_network_policy(request)?;- if let Some(result) = local_text_response(&request.url) {+ if let Some(result) = local_text_response(&request) {return result;}let timing_enabled = moli_trace::cdp_nav_timing_enabled();@@ -446,7 +446,7 @@ impl ResourceRequestClient {F: FnOnce(Result<Response>) + Send + 'static,{let request = self.apply_network_policy(request)?;- if let Some(result) = local_text_response(&request.url) {+ if let Some(result) = local_text_response(&request) {let task_runner = resource_load.task_runner();task_runner.spawn(async move {resource_load.finish();@@ -662,7 +662,7 @@ impl ResourceRequestClient {whereF: FnOnce(Result<Response>) + Send + 'static,{- if let Some(result) = local_text_response(&request.url) {+ if let Some(result) = local_text_response(&request) {callback(result);return Ok(());}@@ -676,7 +676,7 @@ impl ResourceRequestClient {request: Request,cancel_handle: FetchCancelHandle,) -> Result<StreamingRawResponse> {- if let Some(response) = local_text_response(&request.url) {+ if let Some(response) = local_text_response(&request) {return streaming_raw_response_from_local_response(response?);}@@ -712,7 +712,7 @@ impl ResourceRequestClient {request: Request,cancel_handle: FetchCancelHandle,) -> Result<NetworkFetchResult<StreamingRawResponse>> {- if let Some(response) = local_text_response(&request.url) {+ if let Some(response) = local_text_response(&request) {return Ok(NetworkFetchResult::without_request_observation(streaming_raw_response_from_local_response(response?)?,));@@ -1121,8 +1121,8 @@ fn streaming_raw_response_from_cached_subresource(streaming_raw_response_from_head_and_body(head, response.clone_body_bytes())}-fn local_text_response(url: &url::Url) -> Option<Result<Response>> {- crate::network_host::local_url_response_result(url)+fn local_text_response(request: &Request) -> Option<Result<Response>> {+ crate::network_host::local_url_response_result(&request.url, &request.method).map(|result| result.map_err(anyhow::Error::msg))}diff --git a/moli-renderer-v8/src/network/request_client/tests.rs b/moli-renderer-v8/src/network/request_client/tests.rsindex 2f076e64e..1791cf44b 100644--- a/moli-renderer-v8/src/network/request_client/tests.rs+++ b/moli-renderer-v8/src/network/request_client/tests.rs@@ -994,6 +994,66 @@ async fn text_stream_fetch_handles_local_data_stylesheet_urls() -> Result<()> {Ok(())}+#[tokio::test(flavor = "multi_thread")]+async fn local_blob_method_errors_reach_streaming_and_callback_consumers() -> Result<()> {+ let loader = ResourceRequestClient::new(&FetchConfig::default())?;+ let origin = moli_url::WebOrigin::from_url(&Url::parse("https://method.test/page")?);+ for method in [+ "HEAD", "POST", "PUT", "DELETE", "OPTIONS", "PATCH", "CUSTOM",+ ] {+ let request = Request::new_browser_bytes(+ method,+ "blob:https://method.test/unregistered",+ None,+ Vec::new(),+ origin.clone(),+ )?;+ let expected = format!("blob URL fetch requires GET, got `{method}`");+ let error = loader.fetch_text_stream(request.clone()).await.unwrap_err();+ assert_eq!(error.to_string(), expected);+ let error = loader+ .fetch_cacheable_script_text_stream(request.clone())+ .await+ .unwrap_err();+ assert_eq!(error.to_string(), expected);+ let error = loader+ .fetch_raw_stream_with_cancel(request.clone(), FetchCancelHandle::new())+ .await+ .expect_err("raw stream must reject non-GET blob requests");+ assert_eq!(error.to_string(), expected);+ let error = loader+ .fetch_raw_stream_with_cancel_and_network_metadata(+ request.clone(),+ FetchCancelHandle::new(),+ )+ .await+ .expect_err("observed raw stream must reject non-GET blob requests");+ assert_eq!(error.to_string(), expected);+ let (tx, rx) = oneshot::channel();+ loader.fetch_text_callback(request, move |result| {+ tx.send(result).expect("callback receiver is live");+ })?;+ let error = rx.await?.unwrap_err();+ assert_eq!(error.to_string(), expected);++ let data = loader+ .fetch_text_stream(Request::new_browser_bytes(+ method,+ "data:text/plain,payload",+ None,+ Vec::new(),+ origin.clone(),+ )?)+ .await?;+ assert_eq!(+ data.status, 200,+ "data: must not inherit the blob method restriction"+ );+ assert_eq!(data.body_text(), "payload");+ }+ Ok(())+}+#[tokio::test(flavor = "multi_thread")]async fn concurrent_script_text_waiter_preserves_owner_cache_state() -> Result<()> {let listener = TcpListener::bind("127.0.0.1:0").await?;diff --git a/moli-renderer-v8/src/network_host/browser_response.rs b/moli-renderer-v8/src/network_host/browser_response.rsindex 2b4dbc345..151521872 100644--- a/moli-renderer-v8/src/network_host/browser_response.rs+++ b/moli-renderer-v8/src/network_host/browser_response.rs@@ -48,8 +48,9 @@ pub(crate) fn data_url_response(url: &url::Url) -> Option<Response> {))}+/// Reads a local resource for consumers whose requests always use GET.pub(crate) fn local_url_response(url: &url::Url) -> Option<Response> {- local_url_response_result(url).and_then(Result::ok)+ local_url_response_result(url, "GET").and_then(Result::ok)}/// Resolves renderer-owned URL schemes without falling through to the network@@ -58,8 +59,15 @@ pub(crate) fn local_url_response(url: &url::Url) -> Option<Response> {/// `None` means that the URL is not owned by this resolver. `Some(Err(..))`/// means that it is a local URL and therefore must fail locally instead of/// being handed to libcurl.-pub(crate) fn local_url_response_result(url: &url::Url) -> Option<Result<Response, String>> {+/// `method` is the request's already normalized method.+pub(crate) fn local_url_response_result(+ url: &url::Url,+ method: &str,+) -> Option<Result<Response, String>> {match url.scheme() {+ "blob" if method != "GET" => {+ Some(Err(format!("blob URL fetch requires GET, got `{method}`")))+ }"blob" => {Some(blob_url_response(url).ok_or_else(|| format!("blob URL `{url}` is unavailable")))}@@ -107,7 +115,7 @@ mod tests {fn unavailable_blob_url_is_a_local_failure() {let url = url::Url::parse("blob:https://example.test/not-registered").unwrap();- let error = local_url_response_result(&url)+ let error = local_url_response_result(&url, "GET").expect("blob URL must be owned by the local resolver").expect_err("an unregistered blob URL must fail locally");diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs b/moli-renderer-v8/src/network_host/fetch/bindings/paths.rsindex ae5350900..6d0184a94 100644--- a/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs+++ b/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs@@ -144,11 +144,15 @@ pub(super) fn resolve_local_fetch(host: &mut JsContextHost,prepared: &PreparedWindowFetchRequest,) -> Result<Option<Response>, String> {- let Some(response) = local_url_response(&prepared.resolved_url) else {- if prepared.resolved_url.scheme() != "blob" {- return Ok(None);- }- let message = FILE_NOT_FOUND_ERROR_TEXT.to_owned();+ let Some(result) = local_url_response_result(&prepared.resolved_url, &prepared.method) else {+ return Ok(None);+ };+ let response = result.map_err(|message| {+ let message = if prepared.resolved_url.scheme() == "blob" && prepared.method == "GET" {+ FILE_NOT_FOUND_ERROR_TEXT.to_owned()+ } else {+ message+ };host.record_subresource_network(SubresourceNetworkRecord::failure(prepared.frame_id.clone(),prepared.document_url.clone(),@@ -159,8 +163,8 @@ pub(super) fn resolve_local_fetch(SubresourceResourceType::Fetch,message.clone(),));- return Err(message);- };+ message+ })?;host.record_subresource_network(SubresourceNetworkRecord::success_with_body(prepared.frame_id.clone(),diff --git a/moli-renderer-v8/src/network_host/xhr/send.rs b/moli-renderer-v8/src/network_host/xhr/send.rsindex 8a09c2f80..02e0dfb46 100644--- a/moli-renderer-v8/src/network_host/xhr/send.rs+++ b/moli-renderer-v8/src/network_host/xhr/send.rs@@ -181,13 +181,22 @@ pub(super) fn xhr_send_callback<'s>(return;}- if let Some(response) = local_url_response(&prepared.resolved_url) {- if !xhr_is_async(scope, xhr) {- record_xhr_response_success(host, &prepared, &response);- apply_xhr_response(scope, xhr, response);- return;+ if let Some(result) = local_url_response_result(&prepared.resolved_url, &prepared.method) {+ match result {+ Ok(response) if async_request => {+ queue_local_xhr_response(scope, host, xhr, prepared, response);+ }+ Ok(response) => {+ record_xhr_response_success(host, &prepared, &response);+ apply_xhr_response(scope, xhr, response);+ }+ Err(message) if async_request => {+ record_url_policy_xhr_failure(scope, host, xhr, prepared, message);+ }+ Err(message) => {+ record_synchronous_xhr_failure(scope, host, xhr, prepared, message);+ }}- queue_local_xhr_response(scope, host, xhr, prepared, response);return;}diff --git a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rsindex 03f70fca4..74ad7a85a 100644--- a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs+++ b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs@@ -74,8 +74,10 @@ pub(super) async fn load_followed_location_navigation() -> Result<LoadedFollowedLocationNavigation> {debug!(%url, "starting pre-commit location navigation fetch");if let Some(response) = about_blank_navigation_response(&url)- .or_else(|| crate::network_host::local_url_response(&url))+ .map(Ok)+ .or_else(|| crate::network_host::local_url_response_result(&url, &request_method)){+ let response = response.map_err(anyhow::Error::msg)?;if matches!(response.status, 204 | 205) {return Ok(LoadedFollowedLocationNavigation::NoDocument);}@@ -1256,7 +1258,11 @@ impl PageVm {#[cfg(test)]mod tests {- use super::{about_blank_navigation_response, build_followed_location_navigation_request};+ use super::{+ LoadedFollowedLocationNavigation, about_blank_navigation_response,+ build_followed_location_navigation_request, load_followed_location_navigation,+ };+ use crate::network::ResourceRequestClient;use moli_fetch::{BrowserNavigationRequestKind, outgoing_request_headers};use url::Url;@@ -1363,6 +1369,49 @@ mod tests {);}+ #[tokio::test]+ async fn local_navigation_fetch_preserves_the_request_method() {+ let loader = ResourceRequestClient::new(&Default::default()).unwrap();+ let initiator = Url::parse("https://navigation.test/form").unwrap();+ for method in ["HEAD", "POST"] {+ let result = load_followed_location_navigation(+ &loader,+ initiator.clone(),+ Url::parse("blob:https://navigation.test/unregistered").unwrap(),+ method.to_owned(),+ None,+ Vec::new(),+ BrowserNavigationRequestKind::Navigate,+ )+ .await;+ match result {+ Err(error) => assert_eq!(+ error.to_string(),+ format!("blob URL fetch requires GET, got `{method}`")+ ),+ Ok(_) => panic!("blob navigation must not substitute GET for {method}"),+ }+ }+ let data = load_followed_location_navigation(+ &loader,+ initiator,+ Url::parse("data:text/html,<p>payload</p>").unwrap(),+ "POST".to_owned(),+ Some(b"upload".to_vec()),+ Vec::new(),+ BrowserNavigationRequestKind::Navigate,+ )+ .await+ .unwrap();+ assert!(matches!(+ data,+ LoadedFollowedLocationNavigation::ExternalDocument {+ response_status: 200,+ ..+ }+ ));+ }+#[test]fn about_blank_followed_location_navigation_uses_synthetic_response() {let url = Url::parse("about:blank#fragment").unwrap();diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/fetch_xhr.rs b/moli-renderer-v8/src/runtime/page_vm/tests/fetch_xhr.rsindex 28ae6f987..da79d1173 100644--- a/moli-renderer-v8/src/runtime/page_vm/tests/fetch_xhr.rs+++ b/moli-renderer-v8/src/runtime/page_vm/tests/fetch_xhr.rs@@ -4000,6 +4000,129 @@ async fn window_fetch_file_url_rejects_before_interception_or_transport() {.await;}+#[tokio::test]+async fn blob_fetch_and_xhr_reject_non_get_methods_in_window_and_worker() {+ run_page_vm_async_test(async move {+ for worker in [false, true] {+ let mut page_vm = test_page_vm();+ let local_executor = page_vm.local_executor.clone();+ let probe = r#"(async () => {+ const check = (value, message) => { if (!value) throw new Error(message); };+ const blob = URL.createObjectURL(new Blob(['payload'], {type: 'text/plain'}));+ try {+ for (const api of ['fetch', 'fetch-clone', 'xhr-async', 'xhr-sync']) {+ for (const scheme of ['blob', 'data']) {+ const raw = scheme === 'blob' ? blob : 'data:text/plain,payload';+ const methods = scheme === 'blob'+ ? ['GET', 'gEt', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH', 'CUSTOM']+ : ['GET', 'POST', 'CUSTOM'];+ for (const fragment of ['', '#fragment']) {+ for (const method of methods) {+ const url = raw + fragment;+ const success = scheme === 'data' || method.toUpperCase() === 'GET';+ const label = [api, scheme, fragment, method].join(':');+ if (api.startsWith('fetch')) {+ let response;+ try {+ response = await (api === 'fetch-clone'+ ? fetch(new Request(url, {method}).clone())+ : fetch(url, {method}));+ } catch (error) {+ check(!success && error instanceof TypeError, label + ': wrong rejection ' + error);+ continue;+ }+ check(success, label + ': unexpectedly fulfilled');+ check(response.status === 200 && response.url.split('#')[0] === raw, label + ': response metadata');+ check(await response.text() === 'payload', label + ': response body');+ continue;+ }+ const async = api === 'xhr-async';+ const xhr = new XMLHttpRequest();+ xhr.open(method, url, async);+ const checkResponse = () => {+ check(xhr.readyState === 4 && xhr.status === (success ? 200 : 0), label + ': state/status');+ check(xhr.responseText === (success ? 'payload' : ''), label + ': response text');+ check(xhr.responseURL.split('#')[0] === (success ? raw : ''), label + ': response URL');+ check(xhr.getResponseHeader('Content-Type') === (success ? 'text/plain' : null), label + ': headers');+ };+ if (!async) {+ let failure;+ try { xhr.send(); } catch (error) { failure = error; }+ check(success ? failure === undefined : failure instanceof DOMException && failure.name === 'NetworkError', label + ': sync exception');+ checkResponse();+ continue;+ }+ await new Promise((resolve, reject) => {+ let returned = false;+ const events = [];+ xhr.onload = () => events.push('load');+ xhr.onerror = () => events.push('error');+ xhr.onloadend = () => {+ events.push('loadend');+ try {+ check(returned, label + ': completion during send');+ check(events.join(',') === (success ? 'load,loadend' : 'error,loadend'), label + ': terminal events');+ checkResponse();+ resolve();+ } catch (error) { reject(error); }+ };+ xhr.send();+ returned = true;+ });+ }+ }+ }+ }+ } finally { URL.revokeObjectURL(blob); }+ })()"#;+ let script = if worker {+ let source = serde_json::to_string(&format!(+ "{probe}.then(() => postMessage('ok'), error => postMessage(String(error)))"+ ))+ .expect("worker source");+ format!(+ r#"globalThis.__blobMethodResult = 'pending';+ const source = URL.createObjectURL(new Blob([{source}]));+ const worker = new Worker(source);+ worker.onmessage = event => {{+ globalThis.__blobMethodResult = event.data;+ worker.terminate();+ URL.revokeObjectURL(source);+ }};+ worker.onerror = event => {{ globalThis.__blobMethodResult = event.message; }};"#+ )+ } else {+ format!(+ "globalThis.__blobMethodResult = 'pending'; {probe}.then(() => {{ globalThis.__blobMethodResult = 'ok'; }}, error => {{ globalThis.__blobMethodResult = String(error); }})"+ )+ };+ let (result, network_output) = local_executor+ .run(async move {+ page_vm.vm_mut().eval(&script)?;+ drive_websocket_until_done(+ &mut page_vm,+ "String(globalThis.__blobMethodResult !== 'pending')",+ "blob method checks should complete",+ )+ .await?;+ let result = page_vm.vm_mut().eval("globalThis.__blobMethodResult")?;+ Ok::<_, anyhow::Error>((result, page_vm.vm_mut().take_network_output()))+ })+ .await+ .expect("blob method probe should run on owner lane");+ assert_eq!(result, "ok", "worker={worker}");+ let (records, _, _) = split_network_output_items(network_output);+ let failures = records+ .iter()+ .filter(|record| matches!(record.outcome(), SubresourceNetworkOutcome::Failure { error_text }+ if error_text.contains("blob URL fetch requires GET")))+ .count();+ assert_eq!(failures, 56, "each rejected request must record a local network error; worker={worker}");+ }+ })+ .await;+}+#[tokio::test]async fn window_fetch_revoked_blob_url_records_file_not_found_failure() {run_page_vm_async_test(async move {diff --git a/moli-renderer-v8/src/worker/global_scope/fetch.rs b/moli-renderer-v8/src/worker/global_scope/fetch.rsindex 998d460e3..38d06b2d1 100644--- a/moli-renderer-v8/src/worker/global_scope/fetch.rs+++ b/moli-renderer-v8/src/worker/global_scope/fetch.rs@@ -166,11 +166,11 @@ pub(in crate::worker) fn spawn_worker_fetch_network(.validate_request_mode(request_mode, &moli_url::WebOrigin::from_url(&document_url)){(Err(message), None)- } else if matches!(resolved_url.scheme(), "blob" | "data") {+ } else if let Some(result) = local_url_response_result(&resolved_url, &method) {(- local_url_response(&resolved_url)+ result.map(|response| WorkerFetchResponse::Materialized(Box::new(response)))- .ok_or_else(|| format!("fetch: local url `{resolved_url}` is unavailable")),+ .map_err(|error| format!("fetch: {error}")),None,)} else {diff --git a/moli-renderer-v8/src/worker/global_scope/mod.rs b/moli-renderer-v8/src/worker/global_scope/mod.rsindex 350d6dbdf..89c8399e3 100644--- a/moli-renderer-v8/src/worker/global_scope/mod.rs+++ b/moli-renderer-v8/src/worker/global_scope/mod.rs@@ -83,7 +83,7 @@ use crate::network_host::{fetch_browser_subresource_raw_stream_with_preflight_headers_and_network_metadata,fetch_browser_subresource_with_preflight_headers_and_network_metadata,filter_cors_exposed_response_headers, filter_headers_for_guard, has_header,- is_cors_policy_failure_message, local_url_response, parse_fetch_init,+ is_cors_policy_failure_message, local_url_response_result, parse_fetch_init,prepare_xhr_send_body_from_args, request_input_snapshot, request_object_credentials_mode,reset_xhr_response_for_request_error, resolve_context_url, set_xhr_state_bool,set_xhr_state_number, throw_synchronous_xhr_failure, validate_fetch_response_security_policy,diff --git a/moli-renderer-v8/src/worker/global_scope/xhr.rs b/moli-renderer-v8/src/worker/global_scope/xhr.rsindex 78eb73246..09bfad621 100644--- a/moli-renderer-v8/src/worker/global_scope/xhr.rs+++ b/moli-renderer-v8/src/worker/global_scope/xhr.rs@@ -408,15 +408,32 @@ pub(crate) fn try_worker_xhr_send_callback<'s>(return true;}- if let Some(response) = local_url_response(&prepared.resolved_url) {- apply_xhr_response(scope, xhr, response);+ let local_response = local_url_response_result(&prepared.resolved_url, &prepared.method);+ if !async_request && let Some(result) = local_response {+ match result {+ Ok(response) => apply_xhr_response(scope, xhr, response),+ Err(message) => {+ record_worker_subresource_failure(+ &state.borrow(),+ prepared.document_url,+ prepared.resolved_url,+ prepared.method,+ prepared.request_headers,+ request_body_text(&prepared.send_body),+ SubresourceResourceType::Xhr,+ message,+ );+ throw_synchronous_xhr_failure(scope, xhr, &request_url, "NetworkError");+ }+ }return true;}let loader = state.borrow().loader.clone();let cancel_handle = FetchCancelHandle::new();- let intercept_request_stage = fetch_subresource_interception_enabled+ let intercept_request_stage = local_response.is_none()+ && fetch_subresource_interception_enabled&& fetch_subresource_interception_resource_type.is_none_or(|expected| {expected.has_same_cdp_fetch_interception_type(SubresourceResourceType::Xhr)});@@ -475,6 +492,17 @@ pub(crate) fn try_worker_xhr_send_callback<'s>(set_xhr_state_number(scope, xhr, XHR_ACTIVE_INTERNAL_ID_SLOT, xhr_id as f64);schedule_worker_xhr_timeout(scope, &state, xhr, xhr_id);+ if let Some(result) = local_response {+ // Local responses and network errors still complete asynchronously, so+ // abort(), open() and timeout processing use the ordinary pending XHR.+ let _ = state.borrow().xhr_completion_tx.send(WorkerXhrCompletion {+ xhr_id,+ network_request_headers: None,+ result: result.map(|response| WorkerXhrResponse::Materialized(Box::new(response))),+ });+ return true;+ }+if intercept_request_stage {let info = PendingSubresourceFetchInfo {internal_id: 0,