Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(cdp): allow loopback navigations under offline emulation
BibekPathakf3ef88f6dcce
diff --git a/moli-protocol-server/src/protocol_server/tests/websocket.rs b/moli-protocol-server/src/protocol_server/tests/websocket.rsindex 9d06f8aa7..0b6ce3b63 100644--- a/moli-protocol-server/src/protocol_server/tests/websocket.rs+++ b/moli-protocol-server/src/protocol_server/tests/websocket.rs@@ -2044,6 +2044,292 @@ async fn websocket_cdp_raw_client_runtime_evaluate_immediately_after_page_navigafixture_server.abort();}+#[tokio::test]+async fn websocket_cdp_offline_emulation_loopback_navigation_proceeds_and_evaluates() {+ async fn page() -> impl IntoResponse {+ (+ [(axum::http::header::CONTENT_TYPE.as_str(), "text/html")],+ "<!doctype html><html><body><div id='probe'>online</div></body></html>",+ )+ }+ async fn probe() -> &'static str {+ "pong"+ }+ let fixture_app = Router::new()+ .route("/", get(page))+ .route("/probe", get(probe));+ let fixture_listener = TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind fixture listener");+ let fixture_addr = fixture_listener.local_addr().expect("fixture addr");+ let fixture_server =+ tokio::spawn(async move { axum::serve(fixture_listener, fixture_app).await });+ let fixture_url = format!("http://{fixture_addr}/");++ let (cdp_addr, protocol_server) = spawn_test_protocol_server().await;+ let (mut socket, _) = connect_async(format!(+ "ws://{cdp_addr}/devtools/browser/{DEFAULT_BROWSER_ID}"+ ))+ .await+ .expect("connect to cdp websocket");++ socket+ .send(WsMessage::Text(+ json!({ "id": 1_u64, "method": "Target.createBrowserContext" })+ .to_string()+ .into(),+ ))+ .await+ .expect("send createBrowserContext");+ let create_browser_context = recv_until_id(&mut socket, 1).await;+ let browser_context_id = create_browser_context+ .iter()+ .find(|message| message["id"] == json!(1_u64))+ .and_then(|message| message["result"]["browserContextId"].as_str())+ .expect("browserContextId")+ .to_owned();++ socket+ .send(WsMessage::Text(+ json!({+ "id": 2_u64,+ "method": "Target.createTarget",+ "params": { "browserContextId": browser_context_id, "url": "about:blank" }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send createTarget");+ let create_target = recv_until_id(&mut socket, 2).await;+ let target_id = create_target+ .iter()+ .find(|message| message["id"] == json!(2_u64))+ .and_then(|message| message["result"]["targetId"].as_str())+ .expect("targetId")+ .to_owned();++ socket+ .send(WsMessage::Text(+ json!({+ "id": 3_u64,+ "method": "Target.attachToTarget",+ "params": { "targetId": target_id, "flatten": true }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send attachToTarget");+ let attach = recv_until_id(&mut socket, 3).await;+ let session_id = attach+ .iter()+ .find(|message| message["id"] == json!(3_u64))+ .and_then(|message| message["result"]["sessionId"].as_str())+ .expect("sessionId")+ .to_owned();++ // Reproduce the offline scenario from the bug report: enable offline+ // emulation, then navigate to a loopback fixture. Chromium does not+ // hard-block loopback navigations under offline emulation, so the+ // navigation must proceed and the committed page must stay usable.+ socket+ .send(WsMessage::Text(+ json!({+ "id": 4_u64,+ "method": "Network.emulateNetworkConditions",+ "sessionId": session_id,+ "params": {+ "offline": true,+ "latency": 0,+ "downloadThroughput": -1,+ "uploadThroughput": -1+ }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send emulateNetworkConditions");+ let emulate = recv_until_id(&mut socket, 4).await;+ let emulate_response = emulate+ .iter()+ .find(|message| message["id"] == json!(4_u64))+ .expect("emulateNetworkConditions response");+ assert!(+ emulate_response.get("error").is_none(),+ "offline emulation should be accepted; got {emulate_response}"+ );++ socket+ .send(WsMessage::Text(+ json!({+ "id": 5_u64,+ "method": "Page.navigate",+ "sessionId": session_id,+ "params": { "url": fixture_url }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send Page.navigate");+ let navigate = recv_until_id(&mut socket, 5).await;+ let navigate_response = navigate+ .iter()+ .find(|message| message["id"] == json!(5_u64))+ .expect("Page.navigate response");+ assert!(+ navigate_response.get("error").is_none(),+ "loopback navigation must proceed under offline emulation; got {navigate_response}"+ );+ assert!(+ navigate_response["result"]["frameId"].is_string(),+ "Page.navigate must resolve with a frameId; got {navigate_response}"+ );++ socket+ .send(WsMessage::Text(+ json!({+ "id": 6_u64,+ "method": "Runtime.evaluate",+ "sessionId": session_id,+ "params": { "expression": "1 + 1", "returnByValue": true }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send Runtime.evaluate");+ let evaluate = recv_until_id(&mut socket, 6).await;+ let evaluate_response = evaluate+ .iter()+ .find(|message| message["id"] == json!(6_u64))+ .expect("Runtime.evaluate response");+ assert!(+ evaluate_response.get("error").is_none(),+ "Runtime.evaluate must not error after loopback navigation; got {evaluate_response}"+ );+ assert_eq!(+ evaluate_response["result"]["result"]["value"],+ json!(2),+ "Runtime.evaluate must observe the committed loopback document"+ );++ // The loopback document keeps navigator.onLine true, matching the known+ // Chromium quirk where offline emulation leaves loopback pages online.+ socket+ .send(WsMessage::Text(+ json!({+ "id": 7_u64,+ "method": "Runtime.evaluate",+ "sessionId": session_id,+ "params": { "expression": "navigator.onLine", "returnByValue": true }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send navigator.onLine evaluate");+ let on_line = recv_until_id(&mut socket, 7).await;+ let on_line_response = on_line+ .iter()+ .find(|message| message["id"] == json!(7_u64))+ .expect("navigator.onLine response");+ assert_eq!(+ on_line_response["result"]["result"]["value"],+ json!(true),+ "navigator.onLine should stay true on a loopback page; got {on_line_response}"+ );++ // Subresource fetches to the loopback fixture also succeed while offline+ // emulation stays active, because the committed document is online.+ socket+ .send(WsMessage::Text(+ json!({+ "id": 8_u64,+ "method": "Runtime.evaluate",+ "sessionId": session_id,+ "params": {+ "expression": "fetch('/probe').then((r) => r.text()).catch(() => 'offline')",+ "awaitPromise": true,+ "returnByValue": true+ }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send loopback fetch evaluate");+ let fetched = recv_until_id(&mut socket, 8).await;+ let fetch_response = fetched+ .iter()+ .find(|message| message["id"] == json!(8_u64))+ .expect("loopback fetch response");+ assert_eq!(+ fetch_response["result"]["result"]["value"],+ json!("pong"),+ "loopback subresource fetch must succeed while offline; got {fetch_response}"+ );++ // Restoring offline:false is accepted and the target stays responsive.+ socket+ .send(WsMessage::Text(+ json!({+ "id": 9_u64,+ "method": "Network.emulateNetworkConditions",+ "sessionId": session_id,+ "params": {+ "offline": false,+ "latency": 0,+ "downloadThroughput": -1,+ "uploadThroughput": -1+ }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send offline restore");+ let restore = recv_until_id(&mut socket, 9).await;+ let restore_response = restore+ .iter()+ .find(|message| message["id"] == json!(9_u64))+ .expect("offline restore response");+ assert!(+ restore_response.get("error").is_none(),+ "offline:false must be accepted; got {restore_response}"+ );++ socket+ .send(WsMessage::Text(+ json!({+ "id": 10_u64,+ "method": "Runtime.evaluate",+ "sessionId": session_id,+ "params": { "expression": "1 + 1", "returnByValue": true }+ })+ .to_string()+ .into(),+ ))+ .await+ .expect("send post-restore evaluate");+ let restored = recv_until_id(&mut socket, 10).await;+ let restored_response = restored+ .iter()+ .find(|message| message["id"] == json!(10_u64))+ .expect("post-restore evaluate response");+ assert_eq!(+ restored_response["result"]["result"]["value"],+ json!(2),+ "target must remain responsive after restoring offline; got {restored_response}"+ );++ let _ = socket.close(None).await;+ abort_test_cdp_server(protocol_server).await;+ fixture_server.abort();+}+#[tokio::test]async fn websocket_cdp_runtime_control_command_waits_for_navigation_attachment_cutover() {let release_tail = Arc::new(tokio::sync::Notify::new());diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rsindex d3692de7b..fe02e2c1a 100644--- a/moli-protocol/src/conn/runtime_load.rs+++ b/moli-protocol/src/conn/runtime_load.rs@@ -858,9 +858,12 @@ impl BackgroundNavigationLoadJob {}ensure_url_not_blocked_for_load_inputs(&self.load_inputs, &self.raw_url)?;- if self.load_inputs.network_offline {+ if network_offline_blocks_url(self.load_inputs.network_offline, &self.raw_url) {return Err("Network emulation offline".to_owned());}+ if self.load_inputs.network_offline {+ self.load_inputs.network_offline = false;+ }let requested_url = Url::parse(&self.raw_url).map_err(|error| {format!("failed to parse request url `{}`: {error}", self.raw_url)@@ -1562,7 +1565,7 @@ impl CdpConnection {document_activity: load_inputs.document_activity,browser_resource_runtime,navigator_identity,- network_offline: load_inputs.network_offline,+ network_offline: load_inputs.network_offline && !url_is_loopback(final_url),bypass_service_worker: load_inputs.bypass_service_worker,cache_disabled: load_inputs.cache_disabled,blocked_url_patterns: load_inputs.blocked_url_patterns,@@ -2280,7 +2283,7 @@ impl CdpConnection {async fn load_navigation_request_via_runtime_with_network_events_and_load_inputs_async(&mut self,owner: &CommandOwnerScope,- load_inputs: TargetNavigationLoadInputs,+ mut load_inputs: TargetNavigationLoadInputs,method: &str,raw_url: &str,body: Option<Vec<u8>>,@@ -2351,9 +2354,12 @@ impl CdpConnection {}ensure_url_not_blocked_for_load_inputs(&load_inputs, raw_url)?;- if load_inputs.network_offline {+ if network_offline_blocks_url(load_inputs.network_offline, raw_url) {return Err("Network emulation offline".to_owned());}+ if load_inputs.network_offline {+ load_inputs.network_offline = false;+ }let requested_url = Url::parse(raw_url).map_err(|error| format!("failed to parse request url `{raw_url}`: {error}"))?;@@ -2965,7 +2971,7 @@ impl CdpConnection {) -> Result<NetworkFetchResult<NavigationResponse>, String> {let load_inputs = self.navigation_load_inputs_for_session_owner(None);ensure_url_not_blocked_for_load_inputs(&load_inputs, raw_url)?;- if load_inputs.network_offline {+ if network_offline_blocks_url(load_inputs.network_offline, raw_url) {return Err("Network emulation offline".to_owned());}let resource_storage = load_inputs.resource_storage_handles();@@ -3005,7 +3011,7 @@ impl CdpConnection {request_load_policy,);ensure_url_not_blocked_for_load_inputs(&load_inputs, raw_url)?;- if load_inputs.network_offline {+ if network_offline_blocks_url(load_inputs.network_offline, raw_url) {return Err("Network emulation offline".to_owned());}@@ -3097,7 +3103,7 @@ impl CdpConnection {auth: Option<SubresourceAuthCredentials>,) -> Result<NetworkFetchResult<StreamingRawResponse>, String> {ensure_url_not_blocked_for_load_inputs(load_inputs, raw_url)?;- if load_inputs.network_offline {+ if network_offline_blocks_url(load_inputs.network_offline, raw_url) {return Err("Network emulation offline".to_owned());}@@ -3897,6 +3903,33 @@ fn ensure_url_not_blocked_for_load_inputs(}}+/// Chromium does not hard-block loopback navigations under emulated offline+/// conditions (`Network.emulateNetworkConditions {offline:true}`); requests+/// to `localhost`/`*.localhost` and loopback IPs keep working while the page+/// is otherwise offline. Match that quirk so loopback navigations proceed.+fn url_is_loopback(url: &Url) -> bool {+ match url.host() {+ Some(url::Host::Ipv4(ip)) => ip.is_loopback(),+ Some(url::Host::Ipv6(ip)) => ip.is_loopback(),+ Some(url::Host::Domain(domain)) => {+ let domain = domain.trim_end_matches('.');+ let domain = domain.to_ascii_lowercase();+ domain == "localhost" || domain.ends_with(".localhost")+ }+ None => false,+ }+}++/// Resolves whether the emulated network-offline policy should block a+/// navigation to `raw_url`. Loopback destinations are exempt so that a+/// navigation to a local fixture proceeds like Chromium.+fn network_offline_blocks_url(network_offline: bool, raw_url: &str) -> bool {+ network_offline+ && Url::parse(raw_url)+ .map(|url| !url_is_loopback(&url))+ .unwrap_or(true)+}+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub(crate) enum NavigationLoadInputOverrideMode {FreshlyBuiltPage,@@ -3940,9 +3973,10 @@ mod tests {use super::{BackgroundNavigationEarlyResult, decode_data_url_body, decode_data_url_response,decode_text_html_data_url, decoded_data_url_navigation_response,- inline_html_navigation_source,+ inline_html_navigation_source, network_offline_blocks_url, url_is_loopback,};use serde_json::json;+ use url::Url;#[test]fn decode_text_html_data_url_uses_data_url_processor() {@@ -4065,4 +4099,70 @@ mod tests {}));}++ #[test]+ fn url_is_loopback_accepts_local_hosts_and_loopback_addresses() {+ for url in [+ "http://localhost/",+ "http://localhost:8765/page.html",+ "http://LOCALHOST/",+ "http://localhost./",+ "http://app.localhost/page",+ "http://127.0.0.1/",+ "http://127.0.0.1:8765/page.html",+ "http://127.0.0.2/",+ "http://[::1]/",+ "http://[::1]:8765/page.html",+ ] {+ let parsed = Url::parse(url).expect(url);+ assert!(+ url_is_loopback(&parsed),+ "expected `{url}` to be treated as loopback"+ );+ }+ }++ #[test]+ fn url_is_loopback_rejects_remote_hosts_and_schemeless_urls() {+ for url in [+ "http://example.test/",+ "http://example.test:8765/offline",+ "https://example.com/",+ "http://8.8.8.8/",+ "http://192.168.1.10/",+ "http://localhost.evil.example/",+ "about:blank",+ "data:text/html,hello",+ ] {+ let parsed = Url::parse(url).expect(url);+ assert!(+ !url_is_loopback(&parsed),+ "expected `{url}` not to be treated as loopback"+ );+ }+ }++ #[test]+ fn network_offline_blocks_url_exempts_loopback_destinations_only() {+ assert!(network_offline_blocks_url(true, "http://example.test/"));+ assert!(network_offline_blocks_url(+ true,+ "http://example.test/offline"+ ));+ assert!(network_offline_blocks_url(true, "http://8.8.8.8/"));+ assert!(!network_offline_blocks_url(+ true,+ "http://127.0.0.1:8765/page.html"+ ));+ assert!(!network_offline_blocks_url(+ true,+ "http://localhost/page.html"+ ));+ assert!(!network_offline_blocks_url(true, "http://[::1]/"));+ assert!(!network_offline_blocks_url(false, "http://example.test/"));+ assert!(!network_offline_blocks_url(+ false,+ "http://127.0.0.1:8765/page.html"+ ));+ }}