Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(cdp): refresh resource sources before searching

ldm070b4b1eb1544
diff --git a/moli-core/src/page/resource_search_support.rs b/moli-core/src/page/resource_search_support.rs
index 013276ace..5e4ec90de 100644
--- a/moli-core/src/page/resource_search_support.rs
+++ b/moli-core/src/page/resource_search_support.rs
@@ -6,6 +6,11 @@ use super::{
};
impl Page {
+ /// Capture current resource records before selecting the immutable source.
+ pub fn start_resource_search_snapshot(&self) -> Result<PendingPageCommand> {
+ self.start_page_command(RendererPageCommand::ResourceSearchSnapshot)
+ }
+
pub fn start_text_search_by_lines(
&self,
text: String,
diff --git a/moli-protocol/src/domains/page/resource_search.rs b/moli-protocol/src/domains/page/resource_search.rs
index 767fc2fae..420387221 100644
--- a/moli-protocol/src/domains/page/resource_search.rs
+++ b/moli-protocol/src/domains/page/resource_search.rs
@@ -39,6 +39,7 @@ struct SearchInResourceParams {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ResourceSearchSource {
+ SubresourceSnapshot,
SelectedText,
ChildDocument,
}
@@ -129,11 +130,11 @@ pub(super) fn try_start_search_in_resource_command(
if is_root_frame {
let main_document = conn.current_main_document_resource_for_session_owner(cmd.session_id);
- let selected = if main_document.as_ref().is_some_and(|resource| {
+ if main_document.as_ref().is_some_and(|resource| {
resource.frame_id == root_frame_id
&& resource_urls_match(resource.url.as_str(), &params.url)
}) {
- match main_document.and_then(|resource| {
+ let selected = match main_document.and_then(|resource| {
resource
.body
.map(|body| (body, resource.response_headers, resource.from_cache))
@@ -152,14 +153,27 @@ pub(super) fn try_start_search_in_resource_command(
}
Some(_) => SelectedResource::Unavailable,
None => SelectedResource::Unavailable,
- }
- } else {
- let Some(page) = loaded_page(conn, &owner) else {
- return complete_error(CONTENT_UNAVAILABLE);
};
- select_subresource(page, &root_frame_id, true, &params.url, materialize_limit)
+ return start_selected_resource_search(conn, cmd.id, owner, params, selected);
+ }
+
+ let Some(page) = loaded_page(conn, &owner) else {
+ return complete_error(CONTENT_UNAVAILABLE);
+ };
+ // Native replies can arrive before their adapter updates the Page cache.
+ // Select from a fresh renderer snapshot, even after getResourceTree.
+ return match page.start_resource_search_snapshot() {
+ Ok(pending) => pending_step(
+ cmd.id,
+ owner,
+ PendingSearchInResourceCommand {
+ params,
+ source: ResourceSearchSource::SubresourceSnapshot,
+ pending,
+ },
+ ),
+ Err(error) => complete_error(format!("Failed to search resource: {error}")),
};
- return start_selected_resource_search(conn, cmd.id, owner, params, selected);
}
let Some(page) = loaded_page(conn, &owner) else {
@@ -199,6 +213,30 @@ pub(super) fn complete_search_in_resource_command(
Ok(completion) => completion,
Err(message) => return complete_error(format!("Failed to search resource: {message}")),
};
+ if page.renderer_agent_attachment_id() != completion.renderer_agent_attachment_id() {
+ return complete_error(CONTENT_UNAVAILABLE);
+ }
+ if completed.source == ResourceSearchSource::SubresourceSnapshot {
+ if let Err(error) =
+ page.finish_unit_runtime_page_command(completion, "resource search snapshot")
+ {
+ return complete_error(format!("Failed to search resource: {error}"));
+ }
+ let selected = select_subresource(
+ page,
+ &completed.params.frame_id,
+ true,
+ &completed.params.url,
+ materialize_limit,
+ );
+ return start_selected_resource_search(
+ conn,
+ command_id,
+ owner.clone(),
+ completed.params,
+ selected,
+ );
+ }
let outcome = match page.finish_resource_search_by_lines(completion) {
Ok(outcome) => outcome,
Err(error) => return complete_error(format!("Failed to search resource: {error}")),
@@ -210,10 +248,6 @@ pub(super) fn complete_search_in_resource_command(
(_, RendererResourceTextSearchOutcome::ContentUnavailable) => {
complete_error(CONTENT_UNAVAILABLE)
}
- (
- ResourceSearchSource::SelectedText,
- RendererResourceTextSearchOutcome::ResourceNotFound,
- ) => complete_error(RESOURCE_NOT_FOUND),
(
ResourceSearchSource::ChildDocument,
RendererResourceTextSearchOutcome::ResourceNotFound,
@@ -233,6 +267,9 @@ pub(super) fn complete_search_in_resource_command(
selected,
)
}
+ (_, RendererResourceTextSearchOutcome::ResourceNotFound) => {
+ complete_error(RESOURCE_NOT_FOUND)
+ }
}
}
diff --git a/moli-protocol/src/domains/page/tests/resources.rs b/moli-protocol/src/domains/page/tests/resources.rs
index 2ab4e5eb5..2a77fbf0a 100644
--- a/moli-protocol/src/domains/page/tests/resources.rs
+++ b/moli-protocol/src/domains/page/tests/resources.rs
@@ -602,6 +602,287 @@ async fn frame_and_resource_trees_report_main_document_response_mime() {
server.abort();
}
+#[tokio::test(flavor = "multi_thread")]
+async fn search_in_resource_reads_renderer_sources_before_native_adapter_settlement() {
+ use moli_core::{RendererNativeOperation, RendererNativeProtocolResponse, RendererPageCommand};
+
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ axum::Router::new().route(
+ "/source.js",
+ axum::routing::get(|| async {
+ (
+ [(
+ axum::http::header::CONTENT_TYPE.as_str(),
+ "application/javascript; charset=utf-8",
+ )],
+ "const freshSourceNeedle = 'script source';\nconsole.log('resource-search-ready');",
+ )
+ }),
+ ),
+ )
+ .await
+ .unwrap();
+ });
+ let document_url = format!("http://{addr}/document");
+ let script_url = format!("http://{addr}/source.js");
+ let mut ctx = TestContext::new();
+ load_bc_with_target(
+ &mut ctx,
+ "BID-RESOURCE-SEARCH-SETTLEMENT",
+ "TID-RESOURCE-SEARCH-SETTLEMENT",
+ &document_url,
+ );
+ ctx.install_buffered_navigation_fixture_for_session_owner(
+ url::Url::parse(&document_url).unwrap(),
+ "<!doctype html><body>resource search</body>".to_owned(),
+ None,
+ )
+ .await;
+ for (id, method) in [(1200, "Runtime.enable"), (1201, "Page.enable")] {
+ ctx.process_async(json!({"id": id, "method": method})).await;
+ take_response_by_id(&mut ctx, id);
+ }
+
+ let (pending, guard) = ctx
+ .conn
+ .loaded_page_mut_for_protocol_access(None)
+ .unwrap()
+ .start_cdp_call(
+ 1202,
+ None,
+ RendererNativeOperation::new(
+ RendererPageCommand::EvaluateExpression {
+ expression: format!(
+ "const script = document.createElement('script'); script.src = {}; document.head.appendChild(script);",
+ json!(script_url),
+ ),
+ await_promise: false,
+ },
+ |reply| {
+ reply.expect("dynamic script insertion should succeed");
+ RendererNativeProtocolResponse::success(json!({}))
+ },
+ ),
+ )
+ .unwrap();
+ // Keep renderer completion separate from adapter cache settlement.
+ let held_insertion = pending.wait().await.unwrap();
+ ctx.wait_for_scheduler_message("external script execution", |message| {
+ message["method"] == "Runtime.consoleAPICalled"
+ && message["params"]["args"][0]["value"] == "resource-search-ready"
+ })
+ .await;
+
+ let raw_tree = json!({"id": 1203, "method": "Page.getResourceTree"}).to_string();
+ let pending_tree = ctx
+ .conn
+ .try_start_pending_command_dispatch(&raw_tree)
+ .unwrap();
+ assert_eq!(pending_tree.kind_name(), "Native");
+ let held_tree = pending_tree.wait().await;
+ let tree = ctx
+ .wait_for_scheduler_message("native resource tree response", |message| {
+ message["id"] == 1203
+ })
+ .await;
+ assert!(
+ tree["result"]["frameTree"]["resources"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .any(|resource| resource["url"] == script_url),
+ "the renderer must already expose the loaded source: {tree}"
+ );
+ assert!(
+ ctx.conn
+ .loaded_page_mut_for_protocol_access(None)
+ .unwrap()
+ .subresource_network_records()
+ .is_empty(),
+ "native publication must not settle the held adapter Page cache"
+ );
+
+ ctx.process_async(json!({
+ "id": 1204,
+ "method": "Page.searchInResource",
+ "params": {
+ "frameId": tree["result"]["frameTree"]["frame"]["id"],
+ "url": script_url,
+ "query": "freshSourceNeedle"
+ }
+ }))
+ .await;
+ let search = take_response_by_id(&mut ctx, 1204);
+ assert_eq!(
+ search["result"]["result"],
+ json!([{
+ "lineNumber": 0,
+ "lineContent": "const freshSourceNeedle = 'script source';",
+ }]),
+ "resource search must use the renderer source before adapter settlement: {search}"
+ );
+
+ assert!(guard.cancel_or_published().await.is_some());
+ drop(held_insertion);
+ drop(held_tree);
+ server.abort();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn search_in_resource_rejects_snapshot_from_replaced_document() {
+ use crate::conn::{Cmd, CommandDispatchContext};
+ use crate::domains::page::{
+ PageCommandTaskStep, complete_pending_page_command, try_start_page_command_dispatch,
+ };
+
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ axum::Router::new()
+ .route("/original", axum::routing::get(resource_search_document))
+ .route("/search.js", axum::routing::get(resource_search_script))
+ .route(
+ "/replacement",
+ axum::routing::get(|| async {
+ (
+ [(axum::http::header::CONTENT_TYPE.as_str(), "text/html")],
+ "<!doctype html><script src='/replacement.js'></script><body>replacement document</body>",
+ )
+ }),
+ )
+ .route(
+ "/replacement.js",
+ axum::routing::get(|| async {
+ (
+ [(
+ axum::http::header::CONTENT_TYPE.as_str(),
+ "application/javascript; charset=utf-8",
+ )],
+ "globalThis.replacementSourceNeedle = 'replacement source';",
+ )
+ }),
+ ),
+ )
+ .await
+ .unwrap();
+ });
+ let original_url = format!("http://{addr}/original");
+ let replacement_url = format!("http://{addr}/replacement");
+ let replacement_script_url = format!("http://{addr}/replacement.js");
+ let mut ctx = TestContext::new();
+ load_bc_with_target(
+ &mut ctx,
+ "BID-RESOURCE-SEARCH-REPLACEMENT",
+ "TID-RESOURCE-SEARCH-REPLACEMENT",
+ &original_url,
+ );
+ ctx.install_navigation_fixture_for_session_owner(&original_url, None)
+ .await;
+ ctx.process_async(json!({"id": 1300, "method": "Page.enable"}))
+ .await;
+ take_response_by_id(&mut ctx, 1300);
+ let original_attachment = ctx
+ .conn
+ .loaded_page_mut_for_protocol_access(None)
+ .unwrap()
+ .renderer_agent_attachment_id()
+ .expect("original Document should have a renderer attachment");
+ let params = json!({
+ "frameId": "TID-RESOURCE-SEARCH-REPLACEMENT",
+ "url": format!("http://{addr}/search.js"),
+ "query": "benchmarkNeedle"
+ });
+ let raw_search = json!({
+ "id": 1301,
+ "method": "Page.searchInResource",
+ "params": params
+ })
+ .to_string();
+ let cmd = Cmd::for_test(
+ Some(1301),
+ "Page.searchInResource",
+ &params,
+ None,
+ &raw_search,
+ );
+ let PageCommandTaskStep::Pending(pending) =
+ try_start_page_command_dispatch(&mut ctx.conn, &cmd).unwrap()
+ else {
+ panic!("subresource search should first capture a renderer snapshot");
+ };
+ // Complete the old snapshot in the renderer, then replace its Document
+ // before protocol consumes or installs the captured Page state.
+ let held_snapshot = pending.wait().await;
+
+ ctx.process_async(json!({
+ "id": 1302,
+ "method": "Page.navigate",
+ "params": {"url": replacement_url}
+ }))
+ .await;
+ let navigation = take_response_by_id(&mut ctx, 1302);
+ let replacement_loader = navigation["result"]["loaderId"].as_str().unwrap();
+ wait_until_renderer_document_load(
+ &mut ctx,
+ None,
+ "TID-RESOURCE-SEARCH-REPLACEMENT",
+ replacement_loader,
+ )
+ .await;
+ ctx.process_async(json!({"id": 1303, "method": "Page.getResourceTree"}))
+ .await;
+ let tree = take_response_by_id(&mut ctx, 1303);
+ assert_eq!(tree["result"]["frameTree"]["frame"]["url"], replacement_url);
+ let replacement_resources = {
+ let page = ctx.conn.loaded_page_mut_for_protocol_access(None).unwrap();
+ assert_ne!(
+ page.renderer_agent_attachment_id(),
+ Some(original_attachment)
+ );
+ assert_eq!(page.final_url().as_str(), replacement_url);
+ let resources = page
+ .subresource_network_records()
+ .iter()
+ .map(|record| record.url().as_str().to_owned())
+ .collect::<Vec<_>>();
+ assert!(resources.contains(&replacement_script_url));
+ resources
+ };
+
+ let mut command_context = CommandDispatchContext::default();
+ let PageCommandTaskStep::Complete(plan) =
+ complete_pending_page_command(&mut ctx.conn, held_snapshot, &mut command_context).await
+ else {
+ panic!("retired snapshot should settle as an error without another search turn");
+ };
+ let mut out = Vec::new();
+ plan.emit_into(&mut out, cmd.id, cmd.session_id);
+ assert_eq!(out[0]["error"]["code"], json!(-32000));
+ assert_eq!(
+ out[0]["error"]["message"],
+ json!("Content unavailable. Resource was not cached")
+ );
+ // Read the adapter cache directly before any command could refresh it.
+ let page = ctx.conn.loaded_page_mut_for_protocol_access(None).unwrap();
+ assert_eq!(page.final_url().as_str(), replacement_url);
+ assert_eq!(
+ page.subresource_network_records()
+ .iter()
+ .map(|record| record.url().as_str().to_owned())
+ .collect::<Vec<_>>(),
+ replacement_resources,
+ "an old snapshot must not replace the new Document's resource records"
+ );
+
+ server.abort();
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn search_in_resource_uses_original_document_and_subresource_sources() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
diff --git a/moli-renderer-v8/src/devtools/command/nested.rs b/moli-renderer-v8/src/devtools/command/nested.rs
index a52d78a31..ce7184924 100644
--- a/moli-renderer-v8/src/devtools/command/nested.rs
+++ b/moli-renderer-v8/src/devtools/command/nested.rs
@@ -84,6 +84,7 @@ impl RendererPageCommand {
| Self::BlobBytesForUuid { .. }
| Self::DocumentFrontendNodeIdsForBackendNodeIds { .. }
| Self::DocumentStorageKeySnapshot
+ | Self::ResourceSearchSnapshot
| Self::ChildFrameTreeSnapshot
| Self::ChildFrameOwnerNodeReference { .. }
| Self::ChildFrameDocumentRootNodeReference { .. }
diff --git a/moli-renderer-v8/src/runtime/page_commands.rs b/moli-renderer-v8/src/runtime/page_commands.rs
index 65ccfbf99..e684f708f 100644
--- a/moli-renderer-v8/src/runtime/page_commands.rs
+++ b/moli-renderer-v8/src/runtime/page_commands.rs
@@ -424,6 +424,7 @@ impl PageVm {
self.stop_document_lifecycle()?;
Ok(RendererPageReply::Unit)
}
+ RendererPageCommand::ResourceSearchSnapshot => Ok(RendererPageReply::Unit),
RendererPageCommand::SearchTextByLines {
text,
query,
diff --git a/moli-renderer-v8/src/runtime/page_surface.rs b/moli-renderer-v8/src/runtime/page_surface.rs
index dff5576f4..91a613203 100644
--- a/moli-renderer-v8/src/runtime/page_surface.rs
+++ b/moli-renderer-v8/src/runtime/page_surface.rs
@@ -5116,6 +5116,7 @@ pub enum RendererPageCommand {
backend_node_ids: Vec<u32>,
},
DocumentStorageKeySnapshot,
+ ResourceSearchSnapshot,
SearchTextByLines {
text: String,
query: String,
@@ -5622,6 +5623,7 @@ impl RendererPageCommand {
| Self::LiveChildDefaultRuntimeRealmInventory
| Self::RuntimeConsoleMessagesWithContext
| Self::ChildFrameTreeSnapshot
+ | Self::ResourceSearchSnapshot
| Self::PendingSubresourceRequestCount
)
}
@@ -5631,6 +5633,7 @@ impl RendererPageCommand {
Self::Native(command) => command.operation.command.cdp_nav_timing_label(),
Self::Inspector(envelope) => envelope.cdp_nav_timing_label(),
Self::DocumentStorageKeySnapshot => Some("DocumentStorageKeySnapshot"),
+ Self::ResourceSearchSnapshot => Some("ResourceSearchSnapshot"),
Self::CreateIsolatedWorldRuntimeActivity { .. } => {
Some("CreateIsolatedWorldRuntimeActivity")
}