Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
perf(network): retire cumulative raw output history
ldm00c05739324fb
diff --git a/moli-core/src/page/mod.rs b/moli-core/src/page/mod.rsindex 16c758ab9..e0b7f2dea 100644--- a/moli-core/src/page/mod.rs+++ b/moli-core/src/page/mod.rs@@ -167,8 +167,7 @@ pub use protocol_support::{};pub use renderer_command_support::DocumentNodeClientRectResolution;pub use renderer_command_support::{- DocumentNodeRuntimeObjectResolution, PageNetworkOutputUpdate, PageObservableOutputUpdate,- TestingOutcome,+ DocumentNodeRuntimeObjectResolution, PageObservableOutputUpdate, TestingOutcome,};#[cfg(test)]diff --git a/moli-core/src/page/renderer_command_support.rs b/moli-core/src/page/renderer_command_support.rsindex e2057f303..61f68e78e 100644--- a/moli-core/src/page/renderer_command_support.rs+++ b/moli-core/src/page/renderer_command_support.rs@@ -5,9 +5,8 @@ use url::Url;use super::dom_protocol_support::DocumentNodeObjectSnapshot;use super::protocol_support::{- ChildFrameTreeSnapshot, ScriptExecutionReport, ScriptNetworkOutputItem,- ScriptObservableOutputItem, SubresourceNetworkRecord, WebSocketLifecycleEvent,- WebSocketNetworkEvent,+ ChildFrameTreeSnapshot, ScriptExecutionReport, ScriptObservableOutputItem,+ SubresourceNetworkRecord, WebSocketLifecycleEvent, WebSocketNetworkEvent,};use super::{CompletedPageCommand, Page, PendingPageCommand};use crate::renderer::{@@ -34,11 +33,6 @@ use crate::renderer::{// Types// ----------------------------------------------------------------------------#[derive(Debug, Clone, Copy)]-pub struct PageNetworkOutputUpdate<'a> {- network_output_items: &'a [ScriptNetworkOutputItem],-}-#[derive(Debug)]pub struct PageObservableOutputUpdate<'a> {observable_output_items: &'a [ScriptObservableOutputItem],@@ -66,21 +60,9 @@ pub enum DocumentNodeClientRectResolution {}// ----------------------------------------------------------------------------// PageNetworkOutputUpdate / PageObservableOutputUpdate / TestingOutcome impls+// PageObservableOutputUpdate / TestingOutcome impls// ----------------------------------------------------------------------------impl<'a> PageNetworkOutputUpdate<'a> {- pub fn append(network_output_items: &'a [ScriptNetworkOutputItem]) -> Self {- Self {- network_output_items,- }- }-- pub fn network_output_items(&self) -> &'a [ScriptNetworkOutputItem] {- self.network_output_items- }-}-impl<'a> PageObservableOutputUpdate<'a> {pub fn append(observable_output_items: &'a [ScriptObservableOutputItem]) -> Self {Self {@@ -230,11 +212,6 @@ impl Page {)}- pub fn take_network_output_update(&mut self) -> PageNetworkOutputUpdate<'_> {- let network_output_items = self.page_state.script_execution().network_output_items();- PageNetworkOutputUpdate::append(network_output_items)- }-pub fn take_observable_output_update(&mut self) -> PageObservableOutputUpdate<'_> {let observable_output_items = self.page_state.script_execution().observable_output_items();PageObservableOutputUpdate::append(observable_output_items)diff --git a/moli-core/tests/fetch_behaviors.rs b/moli-core/tests/fetch_behaviors.rsindex 2bfb078f8..7181aa9b9 100644--- a/moli-core/tests/fetch_behaviors.rs+++ b/moli-core/tests/fetch_behaviors.rs@@ -2,9 +2,7 @@ use moli_test_support as support;use anyhow::{Context, Result, anyhow};use moli_core::{- page::{- Page, ScriptNetworkOutputItem, SubresourceRequestInitiatorType, SubresourceResourceType,- },+ page::{Page, SubresourceResourceType},runtime::{Browser, BrowserConfig as AppConfig, FetchDeadline, RenderedDomWaitUntil},testing::{JsValueSnapshot, ScriptRunOutcome, ScriptSkipReason},};@@ -16,41 +14,21 @@ fn diagnostic_global<'a>(page: &'a Page, name: &str) -> Option<&'a JsValueSnapshpage.script_execution().global(name)}-fn page_started_image_request(- page: &Page,- token: &str,- source: &str,- initiator_type: SubresourceRequestInitiatorType,-) -> bool {- page.script_execution()- .network_output_items()- .iter()- .any(|item| match item {- ScriptNetworkOutputItem::SubresourceRequestStarted(request) => {- request.resource_type() == SubresourceResourceType::Image- && request.request_initiator_type() == initiator_type- && request.url().path() == "/assets/parser-image-fetch-policy.svg"- && request.url().query().is_some_and(|query| {- query.contains(token) && query.contains(&format!("source={source}"))- })- }- _ => false,- })-}-async fn wait_for_parser_image_fetch_policy_asset_requests(token: &str,+ source: &str,expected_count: usize,) -> Result<()> {let deadline = Instant::now() + Duration::from_secs(5);loop {- if support::parser_image_fetch_policy_asset_request_count(token) >= expected_count {+ if support::parser_image_fetch_policy_asset_request_count(token, source) >= expected_count {return Ok(());}if Instant::now() >= deadline {- let actual_count = support::parser_image_fetch_policy_asset_request_count(token);+ let actual_count =+ support::parser_image_fetch_policy_asset_request_count(token, source);return Err(anyhow!(- "timed out waiting for {expected_count} parser image fixture request(s) token `{token}`; observed {actual_count}"+ "timed out waiting for {expected_count} parser image fixture request(s) token `{token}`, source `{source}`; observed {actual_count}"));}tokio::time::sleep(Duration::from_millis(10)).await;@@ -1155,20 +1133,12 @@ async fn fetch_with_load_skips_parser_image_fetch_by_default() -> Result<()> {).await?;- assert!(!page_started_image_request(- &page,- &token,- "html",- SubresourceRequestInitiatorType::Parser- ));- assert!(!page_started_image_request(- &page,- &token,- "css",- SubresourceRequestInitiatorType::Css- ));assert_eq!(- support::parser_image_fetch_policy_asset_request_count(&token),+ support::parser_image_fetch_policy_asset_request_count(&token, "html"),+ 0+ );+ assert_eq!(+ support::parser_image_fetch_policy_asset_request_count(&token, "css"),0);assert!(@@ -1198,34 +1168,24 @@ async fn fetch_with_load_defers_css_images_until_a_paint_layout() -> Result<()>Duration::from_secs(5),).await?;- wait_for_parser_image_fetch_policy_asset_requests(&token, 1).await?;+ wait_for_parser_image_fetch_policy_asset_requests(&token, "html", 1).await?;assert!(- page_started_image_request(- &page,- &token,- "html",- SubresourceRequestInitiatorType::Parser- ),- "parser-discovered HTML image requests must preserve their parser initiator; network output: {:#?}",- page.script_execution().network_output_items()+ support::parser_image_fetch_policy_asset_request_count(&token, "html") >= 1,+ "the parser-discovered HTML image request must reach the network");- assert!(- !page_started_image_request(&page, &token, "css", SubresourceRequestInitiatorType::Css),+ assert_eq!(+ support::parser_image_fetch_policy_asset_request_count(&token, "css"),+ 0,"a load-only observation must not scan CSS text or start an unrendered CSS image");let pending = page.start_capture_screenshot()?;let completion = pending.wait().await?;let _ = page.finish_capture_screenshot(completion)?;- wait_for_parser_image_fetch_policy_asset_requests(&token, 2).await?;-- assert!(page_started_image_request(- &page,- &token,- "css",- SubresourceRequestInitiatorType::Css- ));+ wait_for_parser_image_fetch_policy_asset_requests(&token, "css", 1).await?;++ assert!(support::parser_image_fetch_policy_asset_request_count(&token, "css") >= 1);server.shutdown().await;Ok(())diff --git a/moli-page-types/src/lib.rs b/moli-page-types/src/lib.rsindex 68390e89c..b9feeab3c 100644--- a/moli-page-types/src/lib.rs+++ b/moli-page-types/src/lib.rs@@ -545,7 +545,6 @@ pub struct ScriptExecutionReport {console_messages: Vec<String>,lifecycle_errors: Vec<String>,inspector_issues: Vec<InspectorIssueSnapshot>,- network_output_items: Vec<ScriptNetworkOutputItem>,subresource_network_records: Vec<SubresourceNetworkRecord>,staged_subresource_lifecycle: Option<Box<StagedSubresourceReportState>>,websocket_network_events: Vec<WebSocketNetworkEvent>,@@ -563,9 +562,6 @@ struct StagedSubresourceReportState {#[derive(Debug, Clone, Default, PartialEq, Eq)]pub struct ScriptNetworkOutput {items: Vec<ScriptNetworkOutputItem>,- subresource_network_records: Vec<SubresourceNetworkRecord>,- websocket_network_events: Vec<WebSocketNetworkEvent>,- websocket_lifecycle_events: Vec<WebSocketLifecycleEvent>,}#[derive(Debug, Clone, PartialEq, Eq)]@@ -757,23 +753,7 @@ impl ScriptNetworkOutput {}pub fn push_item(&mut self, item: ScriptNetworkOutputItem) {- self.items.push(item.clone());- match item {- ScriptNetworkOutputItem::SubresourceNetworkRecord(record) => {- self.subresource_network_records.push(*record);- }- ScriptNetworkOutputItem::SubresourceRequestStarted(_)- | ScriptNetworkOutputItem::SubresourceResponseStarted(_)- | ScriptNetworkOutputItem::SubresourceDataReceived(_)- | ScriptNetworkOutputItem::SubresourceEventSourceMessageReceived(_)- | ScriptNetworkOutputItem::SubresourceBodyFinished(_) => {}- ScriptNetworkOutputItem::WebSocketNetworkEvent(event) => {- self.websocket_network_events.push(event);- }- ScriptNetworkOutputItem::WebSocketLifecycleEvent(event) => {- self.websocket_lifecycle_events.push(event);- }- }+ self.items.push(item);}}@@ -890,10 +870,6 @@ impl ScriptExecutionReport {&self.websocket_lifecycle_events}- pub fn network_output_items(&self) -> &[ScriptNetworkOutputItem] {- &self.network_output_items- }-pub fn observable_output_items(&self) -> &[ScriptObservableOutputItem] {&self.observable_output_items}@@ -926,7 +902,6 @@ impl ScriptExecutionReport {}fn push_network_output_item(&mut self, item: ScriptNetworkOutputItem) {- self.network_output_items.push(item.clone());match item {ScriptNetworkOutputItem::SubresourceNetworkRecord(record) => {if let Some(handle) = record.request_handle() {@@ -3752,14 +3727,6 @@ mod tests {report.websocket_lifecycle_events(),std::slice::from_ref(&lifecycle_event));- assert_eq!(- report.network_output_items(),- &[- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record.clone())),- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event.clone()),- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event.clone()),- ],- );}#[test]@@ -3820,14 +3787,6 @@ mod tests {};assert_eq!(*status, 200);assert_eq!(response_body.try_bytes().unwrap().as_ref(), &[1, 2, 3]);- assert_eq!(- report.network_output_items(),- &[- ScriptNetworkOutputItem::SubresourceRequestStarted(Box::new(request)),- ScriptNetworkOutputItem::SubresourceResponseStarted(Box::new(response)),- ScriptNetworkOutputItem::SubresourceBodyFinished(Box::new(body)),- ],- );}#[test]@@ -3872,102 +3831,6 @@ mod tests {);}- #[test]- fn script_network_output_from_items_materializes_grouped_report_shape() {- let document_url = test_url("/");- let record = SubresourceNetworkRecord::failure(- Some("FRAME".to_owned()),- document_url.clone(),- test_url("/api"),- "GET".to_owned(),- Vec::new(),- None,- SubresourceResourceType::Fetch,- "network failed".to_owned(),- );- let websocket_event = WebSocketNetworkEvent::new(- 7,- document_url.clone(),- test_url("/socket"),- WebSocketFrameDirection::Received,- WebSocketFrameOpcode::Text,- 5,- );- let lifecycle_event =- WebSocketLifecycleEvent::open(7, document_url.clone(), test_url("/socket"));-- let output = ScriptNetworkOutput::from_items([- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event.clone()),- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record.clone())),- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event.clone()),- ]);-- assert_eq!(- output.subresource_network_records.as_slice(),- std::slice::from_ref(&record)- );- assert_eq!(- output.websocket_network_events.as_slice(),- std::slice::from_ref(&websocket_event)- );- assert_eq!(- output.websocket_lifecycle_events.as_slice(),- std::slice::from_ref(&lifecycle_event)- );- assert_eq!(- output.items.as_slice(),- &[- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event.clone()),- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record.clone())),- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event.clone()),- ],- "producer item order should survive materialization into grouped views"- );- }-- #[test]- fn script_execution_report_preserves_network_output_item_order_from_producer_batch() {- let document_url = test_url("/");- let record = SubresourceNetworkRecord::failure(- Some("FRAME".to_owned()),- document_url.clone(),- test_url("/api"),- "GET".to_owned(),- Vec::new(),- None,- SubresourceResourceType::Fetch,- "network failed".to_owned(),- );- let websocket_event = WebSocketNetworkEvent::new(- 7,- document_url.clone(),- test_url("/socket"),- WebSocketFrameDirection::Received,- WebSocketFrameOpcode::Text,- 5,- );- let lifecycle_event =- WebSocketLifecycleEvent::open(7, document_url.clone(), test_url("/socket"));- let output = ScriptNetworkOutput::from_items([- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event.clone()),- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record.clone())),- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event.clone()),- ]);-- let mut report = ScriptExecutionReport::default();- report.extend_network_output(output);-- assert_eq!(- report.network_output_items(),- &[- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event),- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record)),- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event),- ],- "report should keep producer item order for later core/CDP output queues"- );- }-#[test]fn script_observable_output_items_preserve_batch_append_order() {let output = ScriptObservableOutput::from_items([diff --git a/moli-renderer-v8/src/runtime/page_vm/mod.rs b/moli-renderer-v8/src/runtime/page_vm/mod.rsindex 01920fd9c..f4502186e 100644--- a/moli-renderer-v8/src/runtime/page_vm/mod.rs+++ b/moli-renderer-v8/src/runtime/page_vm/mod.rs@@ -438,7 +438,9 @@ struct ScriptExecutionReportSnapshotSignature {globals: usize,globals_snapshot_state: crate::types::ScriptGlobalsSnapshotState,observable_outputs: usize,- network_outputs: usize,+ subresource_network_records: usize,+ websocket_network_events: usize,+ websocket_lifecycle_events: usize,}impl ScriptExecutionReportSnapshotSignature {@@ -448,7 +450,9 @@ impl ScriptExecutionReportSnapshotSignature {globals: report.globals().len(),globals_snapshot_state: report.globals_snapshot_state(),observable_outputs: report.observable_output_items().len(),- network_outputs: report.network_output_items().len(),+ subresource_network_records: report.subresource_network_records().len(),+ websocket_network_events: report.websocket_network_events().len(),+ websocket_lifecycle_events: report.websocket_lifecycle_events().len(),}}}diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/lifecycle.rs b/moli-renderer-v8/src/runtime/page_vm/tests/lifecycle.rsindex 7d1ffb255..f4800607e 100644--- a/moli-renderer-v8/src/runtime/page_vm/tests/lifecycle.rs+++ b/moli-renderer-v8/src/runtime/page_vm/tests/lifecycle.rs@@ -3778,7 +3778,7 @@ fn page_report_capture_reuses_unchanged_network_report() {let first = page_vm.capture_page_state().expect("first page state capture");- assert_eq!(first.report.network_output_items().len(), 1);+ assert_eq!(first.report.subresource_network_records().len(), 1);let second = page_vm.capture_page_state().expect("second page state capture");diff --git a/moli-test-support/src/lib.rs b/moli-test-support/src/lib.rsindex ff4f8f7af..70976adbe 100644--- a/moli-test-support/src/lib.rs+++ b/moli-test-support/src/lib.rs@@ -1715,8 +1715,8 @@ pub fn next_parser_image_fetch_policy_token() -> String {routes_core::next_parser_image_fetch_policy_token()}-pub fn parser_image_fetch_policy_asset_request_count(token: &str) -> usize {- routes_core::parser_image_fetch_policy_asset_request_count(token)+pub fn parser_image_fetch_policy_asset_request_count(token: &str, source: &str) -> usize {+ routes_core::parser_image_fetch_policy_asset_request_count(token, source)}pub fn scrapling_dynamic_fetcher_smoke_script() -> PathBuf {diff --git a/moli-test-support/src/routes_core.rs b/moli-test-support/src/routes_core.rsindex 02264ee16..1bd5bbf04 100644--- a/moli-test-support/src/routes_core.rs+++ b/moli-test-support/src/routes_core.rs@@ -7,7 +7,7 @@ use std::sync::{Arc, OnceLock};static NET_UPSTREAM_XHR_404_THEN_200_REQUESTS: AtomicUsize = AtomicUsize::new(0);static CONCURRENT_SHARED_STATE_REQUESTS_IN_FLIGHT: AtomicUsize = AtomicUsize::new(0);static PARSER_IMAGE_FETCH_POLICY_TOKEN_COUNTER: AtomicUsize = AtomicUsize::new(0);-static PARSER_IMAGE_FETCH_POLICY_ASSET_REQUESTS: OnceLock<Mutex<HashMap<String, usize>>> =+static PARSER_IMAGE_FETCH_POLICY_ASSET_REQUESTS: OnceLock<Mutex<HashMap<(String, String), usize>>> =OnceLock::new();static PARSE_TIME_ASYNC_CHUNKED_TAIL_GATES: OnceLock<Mutex<HashMap<String, Arc<tokio::sync::Notify>>>,@@ -39,19 +39,21 @@ pub(super) fn next_parser_image_fetch_policy_token() -> String {format!("parser-image-fetch-policy-{token}")}-fn parser_image_fetch_policy_asset_requests() -> &'static Mutex<HashMap<String, usize>> {+fn parser_image_fetch_policy_asset_requests() -> &'static Mutex<HashMap<(String, String), usize>> {PARSER_IMAGE_FETCH_POLICY_ASSET_REQUESTS.get_or_init(|| Mutex::new(HashMap::new()))}-fn record_parser_image_fetch_policy_asset_request(token: &str) {+fn record_parser_image_fetch_policy_asset_request(token: &str, source: &str) {let mut requests = parser_image_fetch_policy_asset_requests().lock();- *requests.entry(token.to_owned()).or_default() += 1;+ *requests+ .entry((token.to_owned(), source.to_owned()))+ .or_default() += 1;}-pub(super) fn parser_image_fetch_policy_asset_request_count(token: &str) -> usize {+pub(super) fn parser_image_fetch_policy_asset_request_count(token: &str, source: &str) -> usize {parser_image_fetch_policy_asset_requests().lock()- .get(token)+ .get(&(token.to_owned(), source.to_owned())).copied().unwrap_or(0)}@@ -4114,8 +4116,8 @@ pub(super) async fn detached_eager_images_delay_load_page() -> Html<&'static strpub(super) async fn asset_parser_image_fetch_policy_svg(Query(params): Query<HashMap<String, String>>,) -> Response {- if let Some(token) = params.get("token") {- record_parser_image_fetch_policy_asset_request(token);+ if let (Some(token), Some(source)) = (params.get("token"), params.get("source")) {+ record_parser_image_fetch_policy_asset_request(token, source);}Response::builder().header(CONTENT_TYPE, HeaderValue::from_static("image/svg+xml"))