Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

perf(network): retire cumulative raw output history

ldm00c05739324fb
diff --git a/moli-core/src/page/mod.rs b/moli-core/src/page/mod.rs
index 16c758ab9..e0b7f2dea 100644
--- a/moli-core/src/page/mod.rs
+++ b/moli-core/src/page/mod.rs
@@ -167,8 +167,7 @@ pub use protocol_support::{
};
pub use renderer_command_support::DocumentNodeClientRectResolution;
pub use renderer_command_support::{
- DocumentNodeRuntimeObjectResolution, PageNetworkOutputUpdate, PageObservableOutputUpdate,
- TestingOutcome,
+ DocumentNodeRuntimeObjectResolution, PageObservableOutputUpdate, TestingOutcome,
};
#[cfg(test)]
diff --git a/moli-core/src/page/renderer_command_support.rs b/moli-core/src/page/renderer_command_support.rs
index e2057f303..61f68e78e 100644
--- a/moli-core/src/page/renderer_command_support.rs
+++ b/moli-core/src/page/renderer_command_support.rs
@@ -5,9 +5,8 @@ use url::Url;
use super::dom_protocol_support::DocumentNodeObjectSnapshot;
use super::protocol_support::{
- ChildFrameTreeSnapshot, ScriptExecutionReport, ScriptNetworkOutputItem,
- ScriptObservableOutputItem, SubresourceNetworkRecord, WebSocketLifecycleEvent,
- WebSocketNetworkEvent,
+ ChildFrameTreeSnapshot, ScriptExecutionReport, ScriptObservableOutputItem,
+ SubresourceNetworkRecord, WebSocketLifecycleEvent, WebSocketNetworkEvent,
};
use super::{CompletedPageCommand, Page, PendingPageCommand};
use crate::renderer::{
@@ -34,11 +33,6 @@ use crate::renderer::{
// Types
// ---------------------------------------------------------------------------
-#[derive(Debug, Clone, Copy)]
-pub struct PageNetworkOutputUpdate<'a> {
- network_output_items: &'a [ScriptNetworkOutputItem],
-}
-
#[derive(Debug)]
pub struct PageObservableOutputUpdate<'a> {
observable_output_items: &'a [ScriptObservableOutputItem],
@@ -66,21 +60,9 @@ pub enum DocumentNodeClientRectResolution {
}
// ---------------------------------------------------------------------------
-// PageNetworkOutputUpdate / PageObservableOutputUpdate / TestingOutcome impls
+// PageObservableOutputUpdate / TestingOutcome impls
// ---------------------------------------------------------------------------
-impl<'a> PageNetworkOutputUpdate<'a> {
- pub fn append(network_output_items: &'a [ScriptNetworkOutputItem]) -> Self {
- Self {
- network_output_items,
- }
- }
-
- pub fn network_output_items(&self) -> &'a [ScriptNetworkOutputItem] {
- self.network_output_items
- }
-}
-
impl<'a> PageObservableOutputUpdate<'a> {
pub fn append(observable_output_items: &'a [ScriptObservableOutputItem]) -> Self {
Self {
@@ -230,11 +212,6 @@ impl Page {
)
}
- pub fn take_network_output_update(&mut self) -> PageNetworkOutputUpdate<'_> {
- let network_output_items = self.page_state.script_execution().network_output_items();
- PageNetworkOutputUpdate::append(network_output_items)
- }
-
pub fn take_observable_output_update(&mut self) -> PageObservableOutputUpdate<'_> {
let observable_output_items = self.page_state.script_execution().observable_output_items();
PageObservableOutputUpdate::append(observable_output_items)
diff --git a/moli-core/tests/fetch_behaviors.rs b/moli-core/tests/fetch_behaviors.rs
index 2bfb078f8..7181aa9b9 100644
--- a/moli-core/tests/fetch_behaviors.rs
+++ b/moli-core/tests/fetch_behaviors.rs
@@ -2,9 +2,7 @@ use moli_test_support as support;
use anyhow::{Context, Result, anyhow};
use moli_core::{
- page::{
- Page, ScriptNetworkOutputItem, SubresourceRequestInitiatorType, SubresourceResourceType,
- },
+ page::{Page, SubresourceResourceType},
runtime::{Browser, BrowserConfig as AppConfig, FetchDeadline, RenderedDomWaitUntil},
testing::{JsValueSnapshot, ScriptRunOutcome, ScriptSkipReason},
};
@@ -16,41 +14,21 @@ fn diagnostic_global<'a>(page: &'a Page, name: &str) -> Option<&'a JsValueSnapsh
page.script_execution().global(name)
}
-fn page_started_image_request(
- page: &Page,
- token: &str,
- source: &str,
- initiator_type: SubresourceRequestInitiatorType,
-) -> bool {
- page.script_execution()
- .network_output_items()
- .iter()
- .any(|item| match item {
- ScriptNetworkOutputItem::SubresourceRequestStarted(request) => {
- request.resource_type() == SubresourceResourceType::Image
- && request.request_initiator_type() == initiator_type
- && request.url().path() == "/assets/parser-image-fetch-policy.svg"
- && request.url().query().is_some_and(|query| {
- query.contains(token) && query.contains(&format!("source={source}"))
- })
- }
- _ => false,
- })
-}
-
async fn wait_for_parser_image_fetch_policy_asset_requests(
token: &str,
+ source: &str,
expected_count: usize,
) -> Result<()> {
let deadline = Instant::now() + Duration::from_secs(5);
loop {
- if support::parser_image_fetch_policy_asset_request_count(token) >= expected_count {
+ if support::parser_image_fetch_policy_asset_request_count(token, source) >= expected_count {
return Ok(());
}
if Instant::now() >= deadline {
- let actual_count = support::parser_image_fetch_policy_asset_request_count(token);
+ let actual_count =
+ support::parser_image_fetch_policy_asset_request_count(token, source);
return Err(anyhow!(
- "timed out waiting for {expected_count} parser image fixture request(s) token `{token}`; observed {actual_count}"
+ "timed out waiting for {expected_count} parser image fixture request(s) token `{token}`, source `{source}`; observed {actual_count}"
));
}
tokio::time::sleep(Duration::from_millis(10)).await;
@@ -1155,20 +1133,12 @@ async fn fetch_with_load_skips_parser_image_fetch_by_default() -> Result<()> {
)
.await?;
- assert!(!page_started_image_request(
- &page,
- &token,
- "html",
- SubresourceRequestInitiatorType::Parser
- ));
- assert!(!page_started_image_request(
- &page,
- &token,
- "css",
- SubresourceRequestInitiatorType::Css
- ));
assert_eq!(
- support::parser_image_fetch_policy_asset_request_count(&token),
+ support::parser_image_fetch_policy_asset_request_count(&token, "html"),
+ 0
+ );
+ assert_eq!(
+ support::parser_image_fetch_policy_asset_request_count(&token, "css"),
0
);
assert!(
@@ -1198,34 +1168,24 @@ async fn fetch_with_load_defers_css_images_until_a_paint_layout() -> Result<()>
Duration::from_secs(5),
)
.await?;
- wait_for_parser_image_fetch_policy_asset_requests(&token, 1).await?;
+ wait_for_parser_image_fetch_policy_asset_requests(&token, "html", 1).await?;
assert!(
- page_started_image_request(
- &page,
- &token,
- "html",
- SubresourceRequestInitiatorType::Parser
- ),
- "parser-discovered HTML image requests must preserve their parser initiator; network output: {:#?}",
- page.script_execution().network_output_items()
+ support::parser_image_fetch_policy_asset_request_count(&token, "html") >= 1,
+ "the parser-discovered HTML image request must reach the network"
);
- assert!(
- !page_started_image_request(&page, &token, "css", SubresourceRequestInitiatorType::Css),
+ assert_eq!(
+ support::parser_image_fetch_policy_asset_request_count(&token, "css"),
+ 0,
"a load-only observation must not scan CSS text or start an unrendered CSS image"
);
let pending = page.start_capture_screenshot()?;
let completion = pending.wait().await?;
let _ = page.finish_capture_screenshot(completion)?;
- wait_for_parser_image_fetch_policy_asset_requests(&token, 2).await?;
-
- assert!(page_started_image_request(
- &page,
- &token,
- "css",
- SubresourceRequestInitiatorType::Css
- ));
+ wait_for_parser_image_fetch_policy_asset_requests(&token, "css", 1).await?;
+
+ assert!(support::parser_image_fetch_policy_asset_request_count(&token, "css") >= 1);
server.shutdown().await;
Ok(())
diff --git a/moli-page-types/src/lib.rs b/moli-page-types/src/lib.rs
index 68390e89c..b9feeab3c 100644
--- a/moli-page-types/src/lib.rs
+++ b/moli-page-types/src/lib.rs
@@ -545,7 +545,6 @@ pub struct ScriptExecutionReport {
console_messages: Vec<String>,
lifecycle_errors: Vec<String>,
inspector_issues: Vec<InspectorIssueSnapshot>,
- network_output_items: Vec<ScriptNetworkOutputItem>,
subresource_network_records: Vec<SubresourceNetworkRecord>,
staged_subresource_lifecycle: Option<Box<StagedSubresourceReportState>>,
websocket_network_events: Vec<WebSocketNetworkEvent>,
@@ -563,9 +562,6 @@ struct StagedSubresourceReportState {
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ScriptNetworkOutput {
items: Vec<ScriptNetworkOutputItem>,
- subresource_network_records: Vec<SubresourceNetworkRecord>,
- websocket_network_events: Vec<WebSocketNetworkEvent>,
- websocket_lifecycle_events: Vec<WebSocketLifecycleEvent>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -757,23 +753,7 @@ impl ScriptNetworkOutput {
}
pub fn push_item(&mut self, item: ScriptNetworkOutputItem) {
- self.items.push(item.clone());
- match item {
- ScriptNetworkOutputItem::SubresourceNetworkRecord(record) => {
- self.subresource_network_records.push(*record);
- }
- ScriptNetworkOutputItem::SubresourceRequestStarted(_)
- | ScriptNetworkOutputItem::SubresourceResponseStarted(_)
- | ScriptNetworkOutputItem::SubresourceDataReceived(_)
- | ScriptNetworkOutputItem::SubresourceEventSourceMessageReceived(_)
- | ScriptNetworkOutputItem::SubresourceBodyFinished(_) => {}
- ScriptNetworkOutputItem::WebSocketNetworkEvent(event) => {
- self.websocket_network_events.push(event);
- }
- ScriptNetworkOutputItem::WebSocketLifecycleEvent(event) => {
- self.websocket_lifecycle_events.push(event);
- }
- }
+ self.items.push(item);
}
}
@@ -890,10 +870,6 @@ impl ScriptExecutionReport {
&self.websocket_lifecycle_events
}
- pub fn network_output_items(&self) -> &[ScriptNetworkOutputItem] {
- &self.network_output_items
- }
-
pub fn observable_output_items(&self) -> &[ScriptObservableOutputItem] {
&self.observable_output_items
}
@@ -926,7 +902,6 @@ impl ScriptExecutionReport {
}
fn push_network_output_item(&mut self, item: ScriptNetworkOutputItem) {
- self.network_output_items.push(item.clone());
match item {
ScriptNetworkOutputItem::SubresourceNetworkRecord(record) => {
if let Some(handle) = record.request_handle() {
@@ -3752,14 +3727,6 @@ mod tests {
report.websocket_lifecycle_events(),
std::slice::from_ref(&lifecycle_event)
);
- assert_eq!(
- report.network_output_items(),
- &[
- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record.clone())),
- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event.clone()),
- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event.clone()),
- ],
- );
}
#[test]
@@ -3820,14 +3787,6 @@ mod tests {
};
assert_eq!(*status, 200);
assert_eq!(response_body.try_bytes().unwrap().as_ref(), &[1, 2, 3]);
- assert_eq!(
- report.network_output_items(),
- &[
- ScriptNetworkOutputItem::SubresourceRequestStarted(Box::new(request)),
- ScriptNetworkOutputItem::SubresourceResponseStarted(Box::new(response)),
- ScriptNetworkOutputItem::SubresourceBodyFinished(Box::new(body)),
- ],
- );
}
#[test]
@@ -3872,102 +3831,6 @@ mod tests {
);
}
- #[test]
- fn script_network_output_from_items_materializes_grouped_report_shape() {
- let document_url = test_url("/");
- let record = SubresourceNetworkRecord::failure(
- Some("FRAME".to_owned()),
- document_url.clone(),
- test_url("/api"),
- "GET".to_owned(),
- Vec::new(),
- None,
- SubresourceResourceType::Fetch,
- "network failed".to_owned(),
- );
- let websocket_event = WebSocketNetworkEvent::new(
- 7,
- document_url.clone(),
- test_url("/socket"),
- WebSocketFrameDirection::Received,
- WebSocketFrameOpcode::Text,
- 5,
- );
- let lifecycle_event =
- WebSocketLifecycleEvent::open(7, document_url.clone(), test_url("/socket"));
-
- let output = ScriptNetworkOutput::from_items([
- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event.clone()),
- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record.clone())),
- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event.clone()),
- ]);
-
- assert_eq!(
- output.subresource_network_records.as_slice(),
- std::slice::from_ref(&record)
- );
- assert_eq!(
- output.websocket_network_events.as_slice(),
- std::slice::from_ref(&websocket_event)
- );
- assert_eq!(
- output.websocket_lifecycle_events.as_slice(),
- std::slice::from_ref(&lifecycle_event)
- );
- assert_eq!(
- output.items.as_slice(),
- &[
- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event.clone()),
- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record.clone())),
- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event.clone()),
- ],
- "producer item order should survive materialization into grouped views"
- );
- }
-
- #[test]
- fn script_execution_report_preserves_network_output_item_order_from_producer_batch() {
- let document_url = test_url("/");
- let record = SubresourceNetworkRecord::failure(
- Some("FRAME".to_owned()),
- document_url.clone(),
- test_url("/api"),
- "GET".to_owned(),
- Vec::new(),
- None,
- SubresourceResourceType::Fetch,
- "network failed".to_owned(),
- );
- let websocket_event = WebSocketNetworkEvent::new(
- 7,
- document_url.clone(),
- test_url("/socket"),
- WebSocketFrameDirection::Received,
- WebSocketFrameOpcode::Text,
- 5,
- );
- let lifecycle_event =
- WebSocketLifecycleEvent::open(7, document_url.clone(), test_url("/socket"));
- let output = ScriptNetworkOutput::from_items([
- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event.clone()),
- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record.clone())),
- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event.clone()),
- ]);
-
- let mut report = ScriptExecutionReport::default();
- report.extend_network_output(output);
-
- assert_eq!(
- report.network_output_items(),
- &[
- ScriptNetworkOutputItem::WebSocketLifecycleEvent(lifecycle_event),
- ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(record)),
- ScriptNetworkOutputItem::WebSocketNetworkEvent(websocket_event),
- ],
- "report should keep producer item order for later core/CDP output queues"
- );
- }
-
#[test]
fn script_observable_output_items_preserve_batch_append_order() {
let output = ScriptObservableOutput::from_items([
diff --git a/moli-renderer-v8/src/runtime/page_vm/mod.rs b/moli-renderer-v8/src/runtime/page_vm/mod.rs
index 01920fd9c..f4502186e 100644
--- a/moli-renderer-v8/src/runtime/page_vm/mod.rs
+++ b/moli-renderer-v8/src/runtime/page_vm/mod.rs
@@ -438,7 +438,9 @@ struct ScriptExecutionReportSnapshotSignature {
globals: usize,
globals_snapshot_state: crate::types::ScriptGlobalsSnapshotState,
observable_outputs: usize,
- network_outputs: usize,
+ subresource_network_records: usize,
+ websocket_network_events: usize,
+ websocket_lifecycle_events: usize,
}
impl ScriptExecutionReportSnapshotSignature {
@@ -448,7 +450,9 @@ impl ScriptExecutionReportSnapshotSignature {
globals: report.globals().len(),
globals_snapshot_state: report.globals_snapshot_state(),
observable_outputs: report.observable_output_items().len(),
- network_outputs: report.network_output_items().len(),
+ subresource_network_records: report.subresource_network_records().len(),
+ websocket_network_events: report.websocket_network_events().len(),
+ websocket_lifecycle_events: report.websocket_lifecycle_events().len(),
}
}
}
diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/lifecycle.rs b/moli-renderer-v8/src/runtime/page_vm/tests/lifecycle.rs
index 7d1ffb255..f4800607e 100644
--- a/moli-renderer-v8/src/runtime/page_vm/tests/lifecycle.rs
+++ b/moli-renderer-v8/src/runtime/page_vm/tests/lifecycle.rs
@@ -3778,7 +3778,7 @@ fn page_report_capture_reuses_unchanged_network_report() {
let first = page_vm
.capture_page_state()
.expect("first page state capture");
- assert_eq!(first.report.network_output_items().len(), 1);
+ assert_eq!(first.report.subresource_network_records().len(), 1);
let second = page_vm
.capture_page_state()
.expect("second page state capture");
diff --git a/moli-test-support/src/lib.rs b/moli-test-support/src/lib.rs
index ff4f8f7af..70976adbe 100644
--- a/moli-test-support/src/lib.rs
+++ b/moli-test-support/src/lib.rs
@@ -1715,8 +1715,8 @@ pub fn next_parser_image_fetch_policy_token() -> String {
routes_core::next_parser_image_fetch_policy_token()
}
-pub fn parser_image_fetch_policy_asset_request_count(token: &str) -> usize {
- routes_core::parser_image_fetch_policy_asset_request_count(token)
+pub fn parser_image_fetch_policy_asset_request_count(token: &str, source: &str) -> usize {
+ routes_core::parser_image_fetch_policy_asset_request_count(token, source)
}
pub fn scrapling_dynamic_fetcher_smoke_script() -> PathBuf {
diff --git a/moli-test-support/src/routes_core.rs b/moli-test-support/src/routes_core.rs
index 02264ee16..1bd5bbf04 100644
--- a/moli-test-support/src/routes_core.rs
+++ b/moli-test-support/src/routes_core.rs
@@ -7,7 +7,7 @@ use std::sync::{Arc, OnceLock};
static NET_UPSTREAM_XHR_404_THEN_200_REQUESTS: AtomicUsize = AtomicUsize::new(0);
static CONCURRENT_SHARED_STATE_REQUESTS_IN_FLIGHT: AtomicUsize = AtomicUsize::new(0);
static PARSER_IMAGE_FETCH_POLICY_TOKEN_COUNTER: AtomicUsize = AtomicUsize::new(0);
-static PARSER_IMAGE_FETCH_POLICY_ASSET_REQUESTS: OnceLock<Mutex<HashMap<String, usize>>> =
+static PARSER_IMAGE_FETCH_POLICY_ASSET_REQUESTS: OnceLock<Mutex<HashMap<(String, String), usize>>> =
OnceLock::new();
static PARSE_TIME_ASYNC_CHUNKED_TAIL_GATES: OnceLock<
Mutex<HashMap<String, Arc<tokio::sync::Notify>>>,
@@ -39,19 +39,21 @@ pub(super) fn next_parser_image_fetch_policy_token() -> String {
format!("parser-image-fetch-policy-{token}")
}
-fn parser_image_fetch_policy_asset_requests() -> &'static Mutex<HashMap<String, usize>> {
+fn parser_image_fetch_policy_asset_requests() -> &'static Mutex<HashMap<(String, String), usize>> {
PARSER_IMAGE_FETCH_POLICY_ASSET_REQUESTS.get_or_init(|| Mutex::new(HashMap::new()))
}
-fn record_parser_image_fetch_policy_asset_request(token: &str) {
+fn record_parser_image_fetch_policy_asset_request(token: &str, source: &str) {
let mut requests = parser_image_fetch_policy_asset_requests().lock();
- *requests.entry(token.to_owned()).or_default() += 1;
+ *requests
+ .entry((token.to_owned(), source.to_owned()))
+ .or_default() += 1;
}
-pub(super) fn parser_image_fetch_policy_asset_request_count(token: &str) -> usize {
+pub(super) fn parser_image_fetch_policy_asset_request_count(token: &str, source: &str) -> usize {
parser_image_fetch_policy_asset_requests()
.lock()
- .get(token)
+ .get(&(token.to_owned(), source.to_owned()))
.copied()
.unwrap_or(0)
}
@@ -4114,8 +4116,8 @@ pub(super) async fn detached_eager_images_delay_load_page() -> Html<&'static str
pub(super) async fn asset_parser_image_fetch_policy_svg(
Query(params): Query<HashMap<String, String>>,
) -> Response {
- if let Some(token) = params.get("token") {
- record_parser_image_fetch_policy_asset_request(token);
+ if let (Some(token), Some(source)) = (params.get("token"), params.get("source")) {
+ record_parser_image_fetch_policy_asset_request(token, source);
}
Response::builder()
.header(CONTENT_TYPE, HeaderValue::from_static("image/svg+xml"))