Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(cdp): refresh resource sources before searching
ldm070b4b1eb1544
diff --git a/moli-core/src/page/resource_search_support.rs b/moli-core/src/page/resource_search_support.rsindex 013276ace..5e4ec90de 100644--- a/moli-core/src/page/resource_search_support.rs+++ b/moli-core/src/page/resource_search_support.rs@@ -6,6 +6,11 @@ use super::{};impl Page {+ /// Capture current resource records before selecting the immutable source.+ pub fn start_resource_search_snapshot(&self) -> Result<PendingPageCommand> {+ self.start_page_command(RendererPageCommand::ResourceSearchSnapshot)+ }+pub fn start_text_search_by_lines(&self,text: String,diff --git a/moli-protocol/src/domains/page/resource_search.rs b/moli-protocol/src/domains/page/resource_search.rsindex 767fc2fae..420387221 100644--- a/moli-protocol/src/domains/page/resource_search.rs+++ b/moli-protocol/src/domains/page/resource_search.rs@@ -39,6 +39,7 @@ struct SearchInResourceParams {#[derive(Debug, Clone, Copy, PartialEq, Eq)]enum ResourceSearchSource {+ SubresourceSnapshot,SelectedText,ChildDocument,}@@ -129,11 +130,11 @@ pub(super) fn try_start_search_in_resource_command(if is_root_frame {let main_document = conn.current_main_document_resource_for_session_owner(cmd.session_id);- let selected = if main_document.as_ref().is_some_and(|resource| {+ if main_document.as_ref().is_some_and(|resource| {resource.frame_id == root_frame_id&& resource_urls_match(resource.url.as_str(), ¶ms.url)}) {- match main_document.and_then(|resource| {+ let selected = match main_document.and_then(|resource| {resource.body.map(|body| (body, resource.response_headers, resource.from_cache))@@ -152,14 +153,27 @@ pub(super) fn try_start_search_in_resource_command(}Some(_) => SelectedResource::Unavailable,None => SelectedResource::Unavailable,- }- } else {- let Some(page) = loaded_page(conn, &owner) else {- return complete_error(CONTENT_UNAVAILABLE);};- select_subresource(page, &root_frame_id, true, ¶ms.url, materialize_limit)+ return start_selected_resource_search(conn, cmd.id, owner, params, selected);+ }++ let Some(page) = loaded_page(conn, &owner) else {+ return complete_error(CONTENT_UNAVAILABLE);+ };+ // Native replies can arrive before their adapter updates the Page cache.+ // Select from a fresh renderer snapshot, even after getResourceTree.+ return match page.start_resource_search_snapshot() {+ Ok(pending) => pending_step(+ cmd.id,+ owner,+ PendingSearchInResourceCommand {+ params,+ source: ResourceSearchSource::SubresourceSnapshot,+ pending,+ },+ ),+ Err(error) => complete_error(format!("Failed to search resource: {error}")),};- return start_selected_resource_search(conn, cmd.id, owner, params, selected);}let Some(page) = loaded_page(conn, &owner) else {@@ -199,6 +213,30 @@ pub(super) fn complete_search_in_resource_command(Ok(completion) => completion,Err(message) => return complete_error(format!("Failed to search resource: {message}")),};+ if page.renderer_agent_attachment_id() != completion.renderer_agent_attachment_id() {+ return complete_error(CONTENT_UNAVAILABLE);+ }+ if completed.source == ResourceSearchSource::SubresourceSnapshot {+ if let Err(error) =+ page.finish_unit_runtime_page_command(completion, "resource search snapshot")+ {+ return complete_error(format!("Failed to search resource: {error}"));+ }+ let selected = select_subresource(+ page,+ &completed.params.frame_id,+ true,+ &completed.params.url,+ materialize_limit,+ );+ return start_selected_resource_search(+ conn,+ command_id,+ owner.clone(),+ completed.params,+ selected,+ );+ }let outcome = match page.finish_resource_search_by_lines(completion) {Ok(outcome) => outcome,Err(error) => return complete_error(format!("Failed to search resource: {error}")),@@ -210,10 +248,6 @@ pub(super) fn complete_search_in_resource_command((_, RendererResourceTextSearchOutcome::ContentUnavailable) => {complete_error(CONTENT_UNAVAILABLE)}- (- ResourceSearchSource::SelectedText,- RendererResourceTextSearchOutcome::ResourceNotFound,- ) => complete_error(RESOURCE_NOT_FOUND),(ResourceSearchSource::ChildDocument,RendererResourceTextSearchOutcome::ResourceNotFound,@@ -233,6 +267,9 @@ pub(super) fn complete_search_in_resource_command(selected,)}+ (_, RendererResourceTextSearchOutcome::ResourceNotFound) => {+ complete_error(RESOURCE_NOT_FOUND)+ }}}diff --git a/moli-protocol/src/domains/page/tests/resources.rs b/moli-protocol/src/domains/page/tests/resources.rsindex 2ab4e5eb5..2a77fbf0a 100644--- a/moli-protocol/src/domains/page/tests/resources.rs+++ b/moli-protocol/src/domains/page/tests/resources.rs@@ -602,6 +602,287 @@ async fn frame_and_resource_trees_report_main_document_response_mime() {server.abort();}+#[tokio::test(flavor = "multi_thread")]+async fn search_in_resource_reads_renderer_sources_before_native_adapter_settlement() {+ use moli_core::{RendererNativeOperation, RendererNativeProtocolResponse, RendererPageCommand};++ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let server = tokio::spawn(async move {+ axum::serve(+ listener,+ axum::Router::new().route(+ "/source.js",+ axum::routing::get(|| async {+ (+ [(+ axum::http::header::CONTENT_TYPE.as_str(),+ "application/javascript; charset=utf-8",+ )],+ "const freshSourceNeedle = 'script source';\nconsole.log('resource-search-ready');",+ )+ }),+ ),+ )+ .await+ .unwrap();+ });+ let document_url = format!("http://{addr}/document");+ let script_url = format!("http://{addr}/source.js");+ let mut ctx = TestContext::new();+ load_bc_with_target(+ &mut ctx,+ "BID-RESOURCE-SEARCH-SETTLEMENT",+ "TID-RESOURCE-SEARCH-SETTLEMENT",+ &document_url,+ );+ ctx.install_buffered_navigation_fixture_for_session_owner(+ url::Url::parse(&document_url).unwrap(),+ "<!doctype html><body>resource search</body>".to_owned(),+ None,+ )+ .await;+ for (id, method) in [(1200, "Runtime.enable"), (1201, "Page.enable")] {+ ctx.process_async(json!({"id": id, "method": method})).await;+ take_response_by_id(&mut ctx, id);+ }++ let (pending, guard) = ctx+ .conn+ .loaded_page_mut_for_protocol_access(None)+ .unwrap()+ .start_cdp_call(+ 1202,+ None,+ RendererNativeOperation::new(+ RendererPageCommand::EvaluateExpression {+ expression: format!(+ "const script = document.createElement('script'); script.src = {}; document.head.appendChild(script);",+ json!(script_url),+ ),+ await_promise: false,+ },+ |reply| {+ reply.expect("dynamic script insertion should succeed");+ RendererNativeProtocolResponse::success(json!({}))+ },+ ),+ )+ .unwrap();+ // Keep renderer completion separate from adapter cache settlement.+ let held_insertion = pending.wait().await.unwrap();+ ctx.wait_for_scheduler_message("external script execution", |message| {+ message["method"] == "Runtime.consoleAPICalled"+ && message["params"]["args"][0]["value"] == "resource-search-ready"+ })+ .await;++ let raw_tree = json!({"id": 1203, "method": "Page.getResourceTree"}).to_string();+ let pending_tree = ctx+ .conn+ .try_start_pending_command_dispatch(&raw_tree)+ .unwrap();+ assert_eq!(pending_tree.kind_name(), "Native");+ let held_tree = pending_tree.wait().await;+ let tree = ctx+ .wait_for_scheduler_message("native resource tree response", |message| {+ message["id"] == 1203+ })+ .await;+ assert!(+ tree["result"]["frameTree"]["resources"]+ .as_array()+ .unwrap()+ .iter()+ .any(|resource| resource["url"] == script_url),+ "the renderer must already expose the loaded source: {tree}"+ );+ assert!(+ ctx.conn+ .loaded_page_mut_for_protocol_access(None)+ .unwrap()+ .subresource_network_records()+ .is_empty(),+ "native publication must not settle the held adapter Page cache"+ );++ ctx.process_async(json!({+ "id": 1204,+ "method": "Page.searchInResource",+ "params": {+ "frameId": tree["result"]["frameTree"]["frame"]["id"],+ "url": script_url,+ "query": "freshSourceNeedle"+ }+ }))+ .await;+ let search = take_response_by_id(&mut ctx, 1204);+ assert_eq!(+ search["result"]["result"],+ json!([{+ "lineNumber": 0,+ "lineContent": "const freshSourceNeedle = 'script source';",+ }]),+ "resource search must use the renderer source before adapter settlement: {search}"+ );++ assert!(guard.cancel_or_published().await.is_some());+ drop(held_insertion);+ drop(held_tree);+ server.abort();+}++#[tokio::test(flavor = "multi_thread")]+async fn search_in_resource_rejects_snapshot_from_replaced_document() {+ use crate::conn::{Cmd, CommandDispatchContext};+ use crate::domains::page::{+ PageCommandTaskStep, complete_pending_page_command, try_start_page_command_dispatch,+ };++ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let server = tokio::spawn(async move {+ axum::serve(+ listener,+ axum::Router::new()+ .route("/original", axum::routing::get(resource_search_document))+ .route("/search.js", axum::routing::get(resource_search_script))+ .route(+ "/replacement",+ axum::routing::get(|| async {+ (+ [(axum::http::header::CONTENT_TYPE.as_str(), "text/html")],+ "<!doctype html><script src='/replacement.js'></script><body>replacement document</body>",+ )+ }),+ )+ .route(+ "/replacement.js",+ axum::routing::get(|| async {+ (+ [(+ axum::http::header::CONTENT_TYPE.as_str(),+ "application/javascript; charset=utf-8",+ )],+ "globalThis.replacementSourceNeedle = 'replacement source';",+ )+ }),+ ),+ )+ .await+ .unwrap();+ });+ let original_url = format!("http://{addr}/original");+ let replacement_url = format!("http://{addr}/replacement");+ let replacement_script_url = format!("http://{addr}/replacement.js");+ let mut ctx = TestContext::new();+ load_bc_with_target(+ &mut ctx,+ "BID-RESOURCE-SEARCH-REPLACEMENT",+ "TID-RESOURCE-SEARCH-REPLACEMENT",+ &original_url,+ );+ ctx.install_navigation_fixture_for_session_owner(&original_url, None)+ .await;+ ctx.process_async(json!({"id": 1300, "method": "Page.enable"}))+ .await;+ take_response_by_id(&mut ctx, 1300);+ let original_attachment = ctx+ .conn+ .loaded_page_mut_for_protocol_access(None)+ .unwrap()+ .renderer_agent_attachment_id()+ .expect("original Document should have a renderer attachment");+ let params = json!({+ "frameId": "TID-RESOURCE-SEARCH-REPLACEMENT",+ "url": format!("http://{addr}/search.js"),+ "query": "benchmarkNeedle"+ });+ let raw_search = json!({+ "id": 1301,+ "method": "Page.searchInResource",+ "params": params+ })+ .to_string();+ let cmd = Cmd::for_test(+ Some(1301),+ "Page.searchInResource",+ ¶ms,+ None,+ &raw_search,+ );+ let PageCommandTaskStep::Pending(pending) =+ try_start_page_command_dispatch(&mut ctx.conn, &cmd).unwrap()+ else {+ panic!("subresource search should first capture a renderer snapshot");+ };+ // Complete the old snapshot in the renderer, then replace its Document+ // before protocol consumes or installs the captured Page state.+ let held_snapshot = pending.wait().await;++ ctx.process_async(json!({+ "id": 1302,+ "method": "Page.navigate",+ "params": {"url": replacement_url}+ }))+ .await;+ let navigation = take_response_by_id(&mut ctx, 1302);+ let replacement_loader = navigation["result"]["loaderId"].as_str().unwrap();+ wait_until_renderer_document_load(+ &mut ctx,+ None,+ "TID-RESOURCE-SEARCH-REPLACEMENT",+ replacement_loader,+ )+ .await;+ ctx.process_async(json!({"id": 1303, "method": "Page.getResourceTree"}))+ .await;+ let tree = take_response_by_id(&mut ctx, 1303);+ assert_eq!(tree["result"]["frameTree"]["frame"]["url"], replacement_url);+ let replacement_resources = {+ let page = ctx.conn.loaded_page_mut_for_protocol_access(None).unwrap();+ assert_ne!(+ page.renderer_agent_attachment_id(),+ Some(original_attachment)+ );+ assert_eq!(page.final_url().as_str(), replacement_url);+ let resources = page+ .subresource_network_records()+ .iter()+ .map(|record| record.url().as_str().to_owned())+ .collect::<Vec<_>>();+ assert!(resources.contains(&replacement_script_url));+ resources+ };++ let mut command_context = CommandDispatchContext::default();+ let PageCommandTaskStep::Complete(plan) =+ complete_pending_page_command(&mut ctx.conn, held_snapshot, &mut command_context).await+ else {+ panic!("retired snapshot should settle as an error without another search turn");+ };+ let mut out = Vec::new();+ plan.emit_into(&mut out, cmd.id, cmd.session_id);+ assert_eq!(out[0]["error"]["code"], json!(-32000));+ assert_eq!(+ out[0]["error"]["message"],+ json!("Content unavailable. Resource was not cached")+ );+ // Read the adapter cache directly before any command could refresh it.+ let page = ctx.conn.loaded_page_mut_for_protocol_access(None).unwrap();+ assert_eq!(page.final_url().as_str(), replacement_url);+ assert_eq!(+ page.subresource_network_records()+ .iter()+ .map(|record| record.url().as_str().to_owned())+ .collect::<Vec<_>>(),+ replacement_resources,+ "an old snapshot must not replace the new Document's resource records"+ );++ server.abort();+}+#[tokio::test(flavor = "multi_thread")]async fn search_in_resource_uses_original_document_and_subresource_sources() {let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();diff --git a/moli-renderer-v8/src/devtools/command/nested.rs b/moli-renderer-v8/src/devtools/command/nested.rsindex a52d78a31..ce7184924 100644--- a/moli-renderer-v8/src/devtools/command/nested.rs+++ b/moli-renderer-v8/src/devtools/command/nested.rs@@ -84,6 +84,7 @@ impl RendererPageCommand {| Self::BlobBytesForUuid { .. }| Self::DocumentFrontendNodeIdsForBackendNodeIds { .. }| Self::DocumentStorageKeySnapshot+ | Self::ResourceSearchSnapshot| Self::ChildFrameTreeSnapshot| Self::ChildFrameOwnerNodeReference { .. }| Self::ChildFrameDocumentRootNodeReference { .. }diff --git a/moli-renderer-v8/src/runtime/page_commands.rs b/moli-renderer-v8/src/runtime/page_commands.rsindex 65ccfbf99..e684f708f 100644--- a/moli-renderer-v8/src/runtime/page_commands.rs+++ b/moli-renderer-v8/src/runtime/page_commands.rs@@ -424,6 +424,7 @@ impl PageVm {self.stop_document_lifecycle()?;Ok(RendererPageReply::Unit)}+ RendererPageCommand::ResourceSearchSnapshot => Ok(RendererPageReply::Unit),RendererPageCommand::SearchTextByLines {text,query,diff --git a/moli-renderer-v8/src/runtime/page_surface.rs b/moli-renderer-v8/src/runtime/page_surface.rsindex dff5576f4..91a613203 100644--- a/moli-renderer-v8/src/runtime/page_surface.rs+++ b/moli-renderer-v8/src/runtime/page_surface.rs@@ -5116,6 +5116,7 @@ pub enum RendererPageCommand {backend_node_ids: Vec<u32>,},DocumentStorageKeySnapshot,+ ResourceSearchSnapshot,SearchTextByLines {text: String,query: String,@@ -5622,6 +5623,7 @@ impl RendererPageCommand {| Self::LiveChildDefaultRuntimeRealmInventory| Self::RuntimeConsoleMessagesWithContext| Self::ChildFrameTreeSnapshot+ | Self::ResourceSearchSnapshot| Self::PendingSubresourceRequestCount)}@@ -5631,6 +5633,7 @@ impl RendererPageCommand {Self::Native(command) => command.operation.command.cdp_nav_timing_label(),Self::Inspector(envelope) => envelope.cdp_nav_timing_label(),Self::DocumentStorageKeySnapshot => Some("DocumentStorageKeySnapshot"),+ Self::ResourceSearchSnapshot => Some("ResourceSearchSnapshot"),Self::CreateIsolatedWorldRuntimeActivity { .. } => {Some("CreateIsolatedWorldRuntimeActivity")}