Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): reject non-GET blob URL requests

ldm08bcdbb299dd2
diff --git a/moli-renderer-v8/src/network/request_client.rs b/moli-renderer-v8/src/network/request_client.rs
index 70e3824b5..365954720 100644
--- a/moli-renderer-v8/src/network/request_client.rs
+++ b/moli-renderer-v8/src/network/request_client.rs
@@ -282,7 +282,7 @@ impl ResourceRequestClient {
request: Request,
) -> Result<Response> {
let request = self.apply_network_policy(request)?;
- if let Some(result) = local_text_response(&request.url) {
+ if let Some(result) = local_text_response(&request) {
return result;
}
let timing_enabled = moli_trace::cdp_nav_timing_enabled();
@@ -446,7 +446,7 @@ impl ResourceRequestClient {
F: FnOnce(Result<Response>) + Send + 'static,
{
let request = self.apply_network_policy(request)?;
- if let Some(result) = local_text_response(&request.url) {
+ if let Some(result) = local_text_response(&request) {
let task_runner = resource_load.task_runner();
task_runner.spawn(async move {
resource_load.finish();
@@ -662,7 +662,7 @@ impl ResourceRequestClient {
where
F: FnOnce(Result<Response>) + Send + 'static,
{
- if let Some(result) = local_text_response(&request.url) {
+ if let Some(result) = local_text_response(&request) {
callback(result);
return Ok(());
}
@@ -676,7 +676,7 @@ impl ResourceRequestClient {
request: Request,
cancel_handle: FetchCancelHandle,
) -> Result<StreamingRawResponse> {
- if let Some(response) = local_text_response(&request.url) {
+ if let Some(response) = local_text_response(&request) {
return streaming_raw_response_from_local_response(response?);
}
@@ -712,7 +712,7 @@ impl ResourceRequestClient {
request: Request,
cancel_handle: FetchCancelHandle,
) -> Result<NetworkFetchResult<StreamingRawResponse>> {
- if let Some(response) = local_text_response(&request.url) {
+ if let Some(response) = local_text_response(&request) {
return Ok(NetworkFetchResult::without_request_observation(
streaming_raw_response_from_local_response(response?)?,
));
@@ -1121,8 +1121,8 @@ fn streaming_raw_response_from_cached_subresource(
streaming_raw_response_from_head_and_body(head, response.clone_body_bytes())
}
-fn local_text_response(url: &url::Url) -> Option<Result<Response>> {
- crate::network_host::local_url_response_result(url)
+fn local_text_response(request: &Request) -> Option<Result<Response>> {
+ crate::network_host::local_url_response_result(&request.url, &request.method)
.map(|result| result.map_err(anyhow::Error::msg))
}
diff --git a/moli-renderer-v8/src/network/request_client/tests.rs b/moli-renderer-v8/src/network/request_client/tests.rs
index 2f076e64e..1791cf44b 100644
--- a/moli-renderer-v8/src/network/request_client/tests.rs
+++ b/moli-renderer-v8/src/network/request_client/tests.rs
@@ -994,6 +994,66 @@ async fn text_stream_fetch_handles_local_data_stylesheet_urls() -> Result<()> {
Ok(())
}
+#[tokio::test(flavor = "multi_thread")]
+async fn local_blob_method_errors_reach_streaming_and_callback_consumers() -> Result<()> {
+ let loader = ResourceRequestClient::new(&FetchConfig::default())?;
+ let origin = moli_url::WebOrigin::from_url(&Url::parse("https://method.test/page")?);
+ for method in [
+ "HEAD", "POST", "PUT", "DELETE", "OPTIONS", "PATCH", "CUSTOM",
+ ] {
+ let request = Request::new_browser_bytes(
+ method,
+ "blob:https://method.test/unregistered",
+ None,
+ Vec::new(),
+ origin.clone(),
+ )?;
+ let expected = format!("blob URL fetch requires GET, got `{method}`");
+ let error = loader.fetch_text_stream(request.clone()).await.unwrap_err();
+ assert_eq!(error.to_string(), expected);
+ let error = loader
+ .fetch_cacheable_script_text_stream(request.clone())
+ .await
+ .unwrap_err();
+ assert_eq!(error.to_string(), expected);
+ let error = loader
+ .fetch_raw_stream_with_cancel(request.clone(), FetchCancelHandle::new())
+ .await
+ .expect_err("raw stream must reject non-GET blob requests");
+ assert_eq!(error.to_string(), expected);
+ let error = loader
+ .fetch_raw_stream_with_cancel_and_network_metadata(
+ request.clone(),
+ FetchCancelHandle::new(),
+ )
+ .await
+ .expect_err("observed raw stream must reject non-GET blob requests");
+ assert_eq!(error.to_string(), expected);
+ let (tx, rx) = oneshot::channel();
+ loader.fetch_text_callback(request, move |result| {
+ tx.send(result).expect("callback receiver is live");
+ })?;
+ let error = rx.await?.unwrap_err();
+ assert_eq!(error.to_string(), expected);
+
+ let data = loader
+ .fetch_text_stream(Request::new_browser_bytes(
+ method,
+ "data:text/plain,payload",
+ None,
+ Vec::new(),
+ origin.clone(),
+ )?)
+ .await?;
+ assert_eq!(
+ data.status, 200,
+ "data: must not inherit the blob method restriction"
+ );
+ assert_eq!(data.body_text(), "payload");
+ }
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn concurrent_script_text_waiter_preserves_owner_cache_state() -> Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
diff --git a/moli-renderer-v8/src/network_host/browser_response.rs b/moli-renderer-v8/src/network_host/browser_response.rs
index 2b4dbc345..151521872 100644
--- a/moli-renderer-v8/src/network_host/browser_response.rs
+++ b/moli-renderer-v8/src/network_host/browser_response.rs
@@ -48,8 +48,9 @@ pub(crate) fn data_url_response(url: &url::Url) -> Option<Response> {
))
}
+/// Reads a local resource for consumers whose requests always use GET.
pub(crate) fn local_url_response(url: &url::Url) -> Option<Response> {
- local_url_response_result(url).and_then(Result::ok)
+ local_url_response_result(url, "GET").and_then(Result::ok)
}
/// Resolves renderer-owned URL schemes without falling through to the network
@@ -58,8 +59,15 @@ pub(crate) fn local_url_response(url: &url::Url) -> Option<Response> {
/// `None` means that the URL is not owned by this resolver. `Some(Err(..))`
/// means that it is a local URL and therefore must fail locally instead of
/// being handed to libcurl.
-pub(crate) fn local_url_response_result(url: &url::Url) -> Option<Result<Response, String>> {
+/// `method` is the request's already normalized method.
+pub(crate) fn local_url_response_result(
+ url: &url::Url,
+ method: &str,
+) -> Option<Result<Response, String>> {
match url.scheme() {
+ "blob" if method != "GET" => {
+ Some(Err(format!("blob URL fetch requires GET, got `{method}`")))
+ }
"blob" => {
Some(blob_url_response(url).ok_or_else(|| format!("blob URL `{url}` is unavailable")))
}
@@ -107,7 +115,7 @@ mod tests {
fn unavailable_blob_url_is_a_local_failure() {
let url = url::Url::parse("blob:https://example.test/not-registered").unwrap();
- let error = local_url_response_result(&url)
+ let error = local_url_response_result(&url, "GET")
.expect("blob URL must be owned by the local resolver")
.expect_err("an unregistered blob URL must fail locally");
diff --git a/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs b/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs
index ae5350900..6d0184a94 100644
--- a/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs
+++ b/moli-renderer-v8/src/network_host/fetch/bindings/paths.rs
@@ -144,11 +144,15 @@ pub(super) fn resolve_local_fetch(
host: &mut JsContextHost,
prepared: &PreparedWindowFetchRequest,
) -> Result<Option<Response>, String> {
- let Some(response) = local_url_response(&prepared.resolved_url) else {
- if prepared.resolved_url.scheme() != "blob" {
- return Ok(None);
- }
- let message = FILE_NOT_FOUND_ERROR_TEXT.to_owned();
+ let Some(result) = local_url_response_result(&prepared.resolved_url, &prepared.method) else {
+ return Ok(None);
+ };
+ let response = result.map_err(|message| {
+ let message = if prepared.resolved_url.scheme() == "blob" && prepared.method == "GET" {
+ FILE_NOT_FOUND_ERROR_TEXT.to_owned()
+ } else {
+ message
+ };
host.record_subresource_network(SubresourceNetworkRecord::failure(
prepared.frame_id.clone(),
prepared.document_url.clone(),
@@ -159,8 +163,8 @@ pub(super) fn resolve_local_fetch(
SubresourceResourceType::Fetch,
message.clone(),
));
- return Err(message);
- };
+ message
+ })?;
host.record_subresource_network(
SubresourceNetworkRecord::success_with_body(
prepared.frame_id.clone(),
diff --git a/moli-renderer-v8/src/network_host/xhr/send.rs b/moli-renderer-v8/src/network_host/xhr/send.rs
index 8a09c2f80..02e0dfb46 100644
--- a/moli-renderer-v8/src/network_host/xhr/send.rs
+++ b/moli-renderer-v8/src/network_host/xhr/send.rs
@@ -181,13 +181,22 @@ pub(super) fn xhr_send_callback<'s>(
return;
}
- if let Some(response) = local_url_response(&prepared.resolved_url) {
- if !xhr_is_async(scope, xhr) {
- record_xhr_response_success(host, &prepared, &response);
- apply_xhr_response(scope, xhr, response);
- return;
+ if let Some(result) = local_url_response_result(&prepared.resolved_url, &prepared.method) {
+ match result {
+ Ok(response) if async_request => {
+ queue_local_xhr_response(scope, host, xhr, prepared, response);
+ }
+ Ok(response) => {
+ record_xhr_response_success(host, &prepared, &response);
+ apply_xhr_response(scope, xhr, response);
+ }
+ Err(message) if async_request => {
+ record_url_policy_xhr_failure(scope, host, xhr, prepared, message);
+ }
+ Err(message) => {
+ record_synchronous_xhr_failure(scope, host, xhr, prepared, message);
+ }
}
- queue_local_xhr_response(scope, host, xhr, prepared, response);
return;
}
diff --git a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
index 03f70fca4..74ad7a85a 100644
--- a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
+++ b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
@@ -74,8 +74,10 @@ pub(super) async fn load_followed_location_navigation(
) -> Result<LoadedFollowedLocationNavigation> {
debug!(%url, "starting pre-commit location navigation fetch");
if let Some(response) = about_blank_navigation_response(&url)
- .or_else(|| crate::network_host::local_url_response(&url))
+ .map(Ok)
+ .or_else(|| crate::network_host::local_url_response_result(&url, &request_method))
{
+ let response = response.map_err(anyhow::Error::msg)?;
if matches!(response.status, 204 | 205) {
return Ok(LoadedFollowedLocationNavigation::NoDocument);
}
@@ -1256,7 +1258,11 @@ impl PageVm {
#[cfg(test)]
mod tests {
- use super::{about_blank_navigation_response, build_followed_location_navigation_request};
+ use super::{
+ LoadedFollowedLocationNavigation, about_blank_navigation_response,
+ build_followed_location_navigation_request, load_followed_location_navigation,
+ };
+ use crate::network::ResourceRequestClient;
use moli_fetch::{BrowserNavigationRequestKind, outgoing_request_headers};
use url::Url;
@@ -1363,6 +1369,49 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn local_navigation_fetch_preserves_the_request_method() {
+ let loader = ResourceRequestClient::new(&Default::default()).unwrap();
+ let initiator = Url::parse("https://navigation.test/form").unwrap();
+ for method in ["HEAD", "POST"] {
+ let result = load_followed_location_navigation(
+ &loader,
+ initiator.clone(),
+ Url::parse("blob:https://navigation.test/unregistered").unwrap(),
+ method.to_owned(),
+ None,
+ Vec::new(),
+ BrowserNavigationRequestKind::Navigate,
+ )
+ .await;
+ match result {
+ Err(error) => assert_eq!(
+ error.to_string(),
+ format!("blob URL fetch requires GET, got `{method}`")
+ ),
+ Ok(_) => panic!("blob navigation must not substitute GET for {method}"),
+ }
+ }
+ let data = load_followed_location_navigation(
+ &loader,
+ initiator,
+ Url::parse("data:text/html,<p>payload</p>").unwrap(),
+ "POST".to_owned(),
+ Some(b"upload".to_vec()),
+ Vec::new(),
+ BrowserNavigationRequestKind::Navigate,
+ )
+ .await
+ .unwrap();
+ assert!(matches!(
+ data,
+ LoadedFollowedLocationNavigation::ExternalDocument {
+ response_status: 200,
+ ..
+ }
+ ));
+ }
+
#[test]
fn about_blank_followed_location_navigation_uses_synthetic_response() {
let url = Url::parse("about:blank#fragment").unwrap();
diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/fetch_xhr.rs b/moli-renderer-v8/src/runtime/page_vm/tests/fetch_xhr.rs
index 28ae6f987..da79d1173 100644
--- a/moli-renderer-v8/src/runtime/page_vm/tests/fetch_xhr.rs
+++ b/moli-renderer-v8/src/runtime/page_vm/tests/fetch_xhr.rs
@@ -4000,6 +4000,129 @@ async fn window_fetch_file_url_rejects_before_interception_or_transport() {
.await;
}
+#[tokio::test]
+async fn blob_fetch_and_xhr_reject_non_get_methods_in_window_and_worker() {
+ run_page_vm_async_test(async move {
+ for worker in [false, true] {
+ let mut page_vm = test_page_vm();
+ let local_executor = page_vm.local_executor.clone();
+ let probe = r#"(async () => {
+ const check = (value, message) => { if (!value) throw new Error(message); };
+ const blob = URL.createObjectURL(new Blob(['payload'], {type: 'text/plain'}));
+ try {
+ for (const api of ['fetch', 'fetch-clone', 'xhr-async', 'xhr-sync']) {
+ for (const scheme of ['blob', 'data']) {
+ const raw = scheme === 'blob' ? blob : 'data:text/plain,payload';
+ const methods = scheme === 'blob'
+ ? ['GET', 'gEt', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH', 'CUSTOM']
+ : ['GET', 'POST', 'CUSTOM'];
+ for (const fragment of ['', '#fragment']) {
+ for (const method of methods) {
+ const url = raw + fragment;
+ const success = scheme === 'data' || method.toUpperCase() === 'GET';
+ const label = [api, scheme, fragment, method].join(':');
+ if (api.startsWith('fetch')) {
+ let response;
+ try {
+ response = await (api === 'fetch-clone'
+ ? fetch(new Request(url, {method}).clone())
+ : fetch(url, {method}));
+ } catch (error) {
+ check(!success && error instanceof TypeError, label + ': wrong rejection ' + error);
+ continue;
+ }
+ check(success, label + ': unexpectedly fulfilled');
+ check(response.status === 200 && response.url.split('#')[0] === raw, label + ': response metadata');
+ check(await response.text() === 'payload', label + ': response body');
+ continue;
+ }
+ const async = api === 'xhr-async';
+ const xhr = new XMLHttpRequest();
+ xhr.open(method, url, async);
+ const checkResponse = () => {
+ check(xhr.readyState === 4 && xhr.status === (success ? 200 : 0), label + ': state/status');
+ check(xhr.responseText === (success ? 'payload' : ''), label + ': response text');
+ check(xhr.responseURL.split('#')[0] === (success ? raw : ''), label + ': response URL');
+ check(xhr.getResponseHeader('Content-Type') === (success ? 'text/plain' : null), label + ': headers');
+ };
+ if (!async) {
+ let failure;
+ try { xhr.send(); } catch (error) { failure = error; }
+ check(success ? failure === undefined : failure instanceof DOMException && failure.name === 'NetworkError', label + ': sync exception');
+ checkResponse();
+ continue;
+ }
+ await new Promise((resolve, reject) => {
+ let returned = false;
+ const events = [];
+ xhr.onload = () => events.push('load');
+ xhr.onerror = () => events.push('error');
+ xhr.onloadend = () => {
+ events.push('loadend');
+ try {
+ check(returned, label + ': completion during send');
+ check(events.join(',') === (success ? 'load,loadend' : 'error,loadend'), label + ': terminal events');
+ checkResponse();
+ resolve();
+ } catch (error) { reject(error); }
+ };
+ xhr.send();
+ returned = true;
+ });
+ }
+ }
+ }
+ }
+ } finally { URL.revokeObjectURL(blob); }
+ })()"#;
+ let script = if worker {
+ let source = serde_json::to_string(&format!(
+ "{probe}.then(() => postMessage('ok'), error => postMessage(String(error)))"
+ ))
+ .expect("worker source");
+ format!(
+ r#"globalThis.__blobMethodResult = 'pending';
+ const source = URL.createObjectURL(new Blob([{source}]));
+ const worker = new Worker(source);
+ worker.onmessage = event => {{
+ globalThis.__blobMethodResult = event.data;
+ worker.terminate();
+ URL.revokeObjectURL(source);
+ }};
+ worker.onerror = event => {{ globalThis.__blobMethodResult = event.message; }};"#
+ )
+ } else {
+ format!(
+ "globalThis.__blobMethodResult = 'pending'; {probe}.then(() => {{ globalThis.__blobMethodResult = 'ok'; }}, error => {{ globalThis.__blobMethodResult = String(error); }})"
+ )
+ };
+ let (result, network_output) = local_executor
+ .run(async move {
+ page_vm.vm_mut().eval(&script)?;
+ drive_websocket_until_done(
+ &mut page_vm,
+ "String(globalThis.__blobMethodResult !== 'pending')",
+ "blob method checks should complete",
+ )
+ .await?;
+ let result = page_vm.vm_mut().eval("globalThis.__blobMethodResult")?;
+ Ok::<_, anyhow::Error>((result, page_vm.vm_mut().take_network_output()))
+ })
+ .await
+ .expect("blob method probe should run on owner lane");
+ assert_eq!(result, "ok", "worker={worker}");
+ let (records, _, _) = split_network_output_items(network_output);
+ let failures = records
+ .iter()
+ .filter(|record| matches!(record.outcome(), SubresourceNetworkOutcome::Failure { error_text }
+ if error_text.contains("blob URL fetch requires GET")))
+ .count();
+ assert_eq!(failures, 56, "each rejected request must record a local network error; worker={worker}");
+ }
+ })
+ .await;
+}
+
#[tokio::test]
async fn window_fetch_revoked_blob_url_records_file_not_found_failure() {
run_page_vm_async_test(async move {
diff --git a/moli-renderer-v8/src/worker/global_scope/fetch.rs b/moli-renderer-v8/src/worker/global_scope/fetch.rs
index 998d460e3..38d06b2d1 100644
--- a/moli-renderer-v8/src/worker/global_scope/fetch.rs
+++ b/moli-renderer-v8/src/worker/global_scope/fetch.rs
@@ -166,11 +166,11 @@ pub(in crate::worker) fn spawn_worker_fetch_network(
.validate_request_mode(request_mode, &moli_url::WebOrigin::from_url(&document_url))
{
(Err(message), None)
- } else if matches!(resolved_url.scheme(), "blob" | "data") {
+ } else if let Some(result) = local_url_response_result(&resolved_url, &method) {
(
- local_url_response(&resolved_url)
+ result
.map(|response| WorkerFetchResponse::Materialized(Box::new(response)))
- .ok_or_else(|| format!("fetch: local url `{resolved_url}` is unavailable")),
+ .map_err(|error| format!("fetch: {error}")),
None,
)
} else {
diff --git a/moli-renderer-v8/src/worker/global_scope/mod.rs b/moli-renderer-v8/src/worker/global_scope/mod.rs
index 350d6dbdf..89c8399e3 100644
--- a/moli-renderer-v8/src/worker/global_scope/mod.rs
+++ b/moli-renderer-v8/src/worker/global_scope/mod.rs
@@ -83,7 +83,7 @@ use crate::network_host::{
fetch_browser_subresource_raw_stream_with_preflight_headers_and_network_metadata,
fetch_browser_subresource_with_preflight_headers_and_network_metadata,
filter_cors_exposed_response_headers, filter_headers_for_guard, has_header,
- is_cors_policy_failure_message, local_url_response, parse_fetch_init,
+ is_cors_policy_failure_message, local_url_response_result, parse_fetch_init,
prepare_xhr_send_body_from_args, request_input_snapshot, request_object_credentials_mode,
reset_xhr_response_for_request_error, resolve_context_url, set_xhr_state_bool,
set_xhr_state_number, throw_synchronous_xhr_failure, validate_fetch_response_security_policy,
diff --git a/moli-renderer-v8/src/worker/global_scope/xhr.rs b/moli-renderer-v8/src/worker/global_scope/xhr.rs
index 78eb73246..09bfad621 100644
--- a/moli-renderer-v8/src/worker/global_scope/xhr.rs
+++ b/moli-renderer-v8/src/worker/global_scope/xhr.rs
@@ -408,15 +408,32 @@ pub(crate) fn try_worker_xhr_send_callback<'s>(
return true;
}
- if let Some(response) = local_url_response(&prepared.resolved_url) {
- apply_xhr_response(scope, xhr, response);
+ let local_response = local_url_response_result(&prepared.resolved_url, &prepared.method);
+ if !async_request && let Some(result) = local_response {
+ match result {
+ Ok(response) => apply_xhr_response(scope, xhr, response),
+ Err(message) => {
+ record_worker_subresource_failure(
+ &state.borrow(),
+ prepared.document_url,
+ prepared.resolved_url,
+ prepared.method,
+ prepared.request_headers,
+ request_body_text(&prepared.send_body),
+ SubresourceResourceType::Xhr,
+ message,
+ );
+ throw_synchronous_xhr_failure(scope, xhr, &request_url, "NetworkError");
+ }
+ }
return true;
}
let loader = state.borrow().loader.clone();
let cancel_handle = FetchCancelHandle::new();
- let intercept_request_stage = fetch_subresource_interception_enabled
+ let intercept_request_stage = local_response.is_none()
+ && fetch_subresource_interception_enabled
&& fetch_subresource_interception_resource_type.is_none_or(|expected| {
expected.has_same_cdp_fetch_interception_type(SubresourceResourceType::Xhr)
});
@@ -475,6 +492,17 @@ pub(crate) fn try_worker_xhr_send_callback<'s>(
set_xhr_state_number(scope, xhr, XHR_ACTIVE_INTERNAL_ID_SLOT, xhr_id as f64);
schedule_worker_xhr_timeout(scope, &state, xhr, xhr_id);
+ if let Some(result) = local_response {
+ // Local responses and network errors still complete asynchronously, so
+ // abort(), open() and timeout processing use the ordinary pending XHR.
+ let _ = state.borrow().xhr_completion_tx.send(WorkerXhrCompletion {
+ xhr_id,
+ network_request_headers: None,
+ result: result.map(|response| WorkerXhrResponse::Materialized(Box::new(response))),
+ });
+ return true;
+ }
+
if intercept_request_stage {
let info = PendingSubresourceFetchInfo {
internal_id: 0,