Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

perf(cdp): retire observed network delivery outputs

ldm0d703d229487d
diff --git a/moli-protocol/src/domains/network/agent.rs b/moli-protocol/src/domains/network/agent.rs
index ffb16cea0..760605933 100644
--- a/moli-protocol/src/domains/network/agent.rs
+++ b/moli-protocol/src/domains/network/agent.rs
@@ -296,11 +296,15 @@ impl TargetNetworkAgentState {
preferred_request_id,
request_id_allocator,
);
- self.output_queue.backlog_prepared_delivery_for_activity(
+ let delivery = self.output_queue.backlog_prepared_delivery_for_activity(
subresource_activity,
websocket_activity,
&mut request_ids,
- )
+ );
+ if !self.has_event_listeners() {
+ self.discard_observed_delivery_outputs();
+ }
+ delivery
}
pub(crate) fn renderer_subresources_are_idle(&self) -> bool {
@@ -471,6 +475,7 @@ impl TargetNetworkAgentState {
self.artifacts
.websocket_network_artifacts
.set_session_cursors(session_id, subresource_record_count, websocket_event_count);
+ self.discard_observed_delivery_outputs();
}
pub(crate) fn remove_session_observation_cursor(&mut self, session_id: Option<&str>) {
@@ -480,6 +485,7 @@ impl TargetNetworkAgentState {
self.artifacts
.websocket_network_artifacts
.remove_session_cursors(session_id);
+ self.discard_observed_delivery_outputs();
}
pub(crate) fn collected_network_data_artifacts(&self) -> Vec<CollectedNetworkDataArtifact> {
@@ -802,6 +808,38 @@ impl TargetNetworkAgentState {
cursor.event_count(),
);
}
+ self.discard_observed_delivery_outputs();
+ }
+
+ fn discard_observed_delivery_outputs(&mut self) {
+ let (subresource_record_cursor, websocket_record_cursor, websocket_event_cursor) =
+ if self.has_event_listeners() {
+ (
+ self.artifacts
+ .subresource_network_artifacts
+ .minimum_emitted_record_count()
+ .unwrap_or(0),
+ self.artifacts
+ .websocket_network_artifacts
+ .minimum_emitted_record_count()
+ .unwrap_or(0),
+ self.artifacts
+ .websocket_network_artifacts
+ .minimum_emitted_event_count()
+ .unwrap_or(0),
+ )
+ } else {
+ (
+ self.output_queue_subresource_record_count(),
+ self.output_queue_subresource_record_count(),
+ self.output_queue_websocket_event_count(),
+ )
+ };
+ self.output_queue.discard_observed_delivery_outputs(
+ subresource_record_cursor,
+ websocket_record_cursor,
+ websocket_event_cursor,
+ );
}
pub(crate) fn clear_websocket_request_ids(&mut self) {
@@ -1803,6 +1841,13 @@ impl SubresourceNetworkArtifacts {
.unwrap_or(0)
}
+ fn minimum_emitted_record_count(&self) -> Option<usize> {
+ self.emitted_record_counts_by_session
+ .values()
+ .copied()
+ .min()
+ }
+
pub(crate) fn mark_emitted(
&mut self,
session_id: Option<&str>,
@@ -2516,6 +2561,7 @@ mod tests {
None,
&mut request_id_allocator,
);
+ assert_eq!(agent.output_queue.retained_delivery_output_count(), 2);
let first_snapshot = agent
.pending_network_backlog_delivery_snapshot_from_backlog(&mut first_delivery)
@@ -2525,6 +2571,7 @@ mod tests {
vec!["https://example.com/first.js"]
);
agent.mark_network_backlog_delivery_snapshot_emitted(&first_snapshot);
+ assert_eq!(agent.output_queue.retained_delivery_output_count(), 1);
let second_snapshot = agent
.pending_network_backlog_delivery_snapshot_from_backlog(&mut second_delivery)
@@ -2535,6 +2582,7 @@ mod tests {
"a later ingress token must not rediscover an earlier unprojected fact from the session cursor"
);
agent.mark_network_backlog_delivery_snapshot_emitted(&second_snapshot);
+ assert_eq!(agent.output_queue.retained_delivery_output_count(), 0);
assert_eq!(
agent
.artifacts
@@ -2543,6 +2591,28 @@ mod tests {
);
}
+ #[test]
+ fn renderer_outputs_without_network_listeners_are_not_retained() {
+ let mut agent = TargetNetworkAgentState::default();
+ let mut request_id_allocator = ConnectionNetworkRequestIdAllocator::default();
+ let item = ScriptNetworkOutputItem::SubresourceNetworkRecord(Box::new(subresource_record(
+ "https://example.com/before-enable.js",
+ )));
+
+ let delivery = agent.ingest_renderer_output_item_and_prepare_live_delivery(
+ &item,
+ "LOADER-1",
+ None,
+ None,
+ None,
+ &mut request_id_allocator,
+ );
+
+ assert!(!delivery.has_output());
+ assert_eq!(agent.output_queue.subresource_record_count(), 1);
+ assert_eq!(agent.output_queue.retained_delivery_output_count(), 0);
+ }
+
#[test]
fn websocket_observation_cursors_and_request_ids_are_separate_artifacts() {
let mut artifacts = WebSocketNetworkArtifacts::default();
@@ -3192,6 +3262,22 @@ impl WebSocketNetworkArtifacts {
self.observation.emitted_event_count_for_session(session_id)
}
+ fn minimum_emitted_record_count(&self) -> Option<usize> {
+ self.observation
+ .emitted_record_counts_by_session
+ .values()
+ .copied()
+ .min()
+ }
+
+ fn minimum_emitted_event_count(&self) -> Option<usize> {
+ self.observation
+ .emitted_event_counts_by_session
+ .values()
+ .copied()
+ .min()
+ }
+
pub(crate) fn mark_emitted(
&mut self,
session_id: Option<&str>,
diff --git a/moli-protocol/src/domains/network/output_queue.rs b/moli-protocol/src/domains/network/output_queue.rs
index 53944312d..c7639e8f9 100644
--- a/moli-protocol/src/domains/network/output_queue.rs
+++ b/moli-protocol/src/domains/network/output_queue.rs
@@ -8,7 +8,7 @@ use moli_core::page::{
WebSocketFrameOpcode, WebSocketLifecycleEvent, WebSocketLifecycleKind, WebSocketNetworkEvent,
};
use moli_fetch::NegotiatedHttpVersion;
-use std::collections::{HashMap, HashSet};
+use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::Arc;
use url::Url;
@@ -64,7 +64,7 @@ pub(crate) struct TargetNetworkBacklogActivityCursor {
#[derive(Clone, Debug, Default, Eq, PartialEq)]
struct TargetNetworkDeliveryOutputQueue {
- outputs: Vec<TargetNetworkDeliveryOutputItem>,
+ outputs: VecDeque<TargetNetworkDeliveryOutputItem>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
@@ -481,15 +481,12 @@ impl TargetNetworkDeliveryOutputQueue {
let mut batch = TargetNetworkBacklogPreparedDeliveryBatch::default();
for item in self
.outputs
- .get(
- self.first_output_position_for_activity(
- subresource_record_start_index,
- websocket_record_start_index,
- websocket_event_start_index,
- )..,
- )
- .unwrap_or_default()
.iter()
+ .skip(self.first_output_position_for_activity(
+ subresource_record_start_index,
+ websocket_record_start_index,
+ websocket_event_start_index,
+ ))
{
match item.output() {
TargetNetworkDeliveryOutput::Subresource(output) => {
@@ -621,7 +618,7 @@ impl TargetNetworkDeliveryOutputQueue {
let websocket_event_tail_after_item = self
.websocket_event_tail()
.max(output.emitted_websocket_event_end());
- self.outputs.push(TargetNetworkDeliveryOutputItem::new(
+ self.outputs.push_back(TargetNetworkDeliveryOutputItem::new(
subresource_record_tail_after_item,
websocket_record_tail_after_item,
websocket_event_tail_after_item,
@@ -629,6 +626,34 @@ impl TargetNetworkDeliveryOutputQueue {
));
}
+ fn discard_observed_outputs(
+ &mut self,
+ subresource_record_cursor: usize,
+ websocket_record_cursor: usize,
+ websocket_event_cursor: usize,
+ ) {
+ while self
+ .outputs
+ .front()
+ .is_some_and(|item| match item.output() {
+ TargetNetworkDeliveryOutput::Subresource(output) => {
+ output.index() < subresource_record_cursor
+ }
+ TargetNetworkDeliveryOutput::WebSocket(output) => match output.source() {
+ TargetWebSocketDeliveryOutputSource::Handshake { record_index } => {
+ record_index < websocket_record_cursor
+ }
+ TargetWebSocketDeliveryOutputSource::Frame { event_index }
+ | TargetWebSocketDeliveryOutputSource::Lifecycle { event_index } => {
+ event_index < websocket_event_cursor
+ }
+ },
+ })
+ {
+ self.outputs.pop_front();
+ }
+ }
+
fn first_output_position_for_activity(
&self,
subresource_record_start_index: Option<usize>,
@@ -648,21 +673,21 @@ impl TargetNetworkDeliveryOutputQueue {
fn subresource_record_tail(&self) -> usize {
self.outputs
- .last()
+ .back()
.map(TargetNetworkDeliveryOutputItem::subresource_record_tail_after_item)
.unwrap_or(0)
}
fn websocket_record_tail(&self) -> usize {
self.outputs
- .last()
+ .back()
.map(TargetNetworkDeliveryOutputItem::websocket_record_tail_after_item)
.unwrap_or(0)
}
fn websocket_event_tail(&self) -> usize {
self.outputs
- .last()
+ .back()
.map(TargetNetworkDeliveryOutputItem::websocket_event_tail_after_item)
.unwrap_or(0)
}
@@ -670,9 +695,8 @@ impl TargetNetworkDeliveryOutputQueue {
#[cfg(test)]
fn subresource_outputs_from(&self, start_index: usize) -> Vec<TargetSubresourceMetadataOutput> {
self.outputs
- .get(self.first_output_position_for_activity(Some(start_index), None, None)..)
- .unwrap_or_default()
.iter()
+ .skip(self.first_output_position_for_activity(Some(start_index), None, None))
.filter_map(TargetNetworkDeliveryOutputItem::subresource_output)
.filter(|output| output.index() >= start_index)
.cloned()
@@ -2553,6 +2577,24 @@ impl TargetNetworkOutputQueue {
};
}
+ pub(crate) fn discard_observed_delivery_outputs(
+ &mut self,
+ subresource_record_cursor: usize,
+ websocket_record_cursor: usize,
+ websocket_event_cursor: usize,
+ ) {
+ self.delivery_outputs.discard_observed_outputs(
+ subresource_record_cursor,
+ websocket_record_cursor,
+ websocket_event_cursor,
+ );
+ }
+
+ #[cfg(test)]
+ pub(crate) fn retained_delivery_output_count(&self) -> usize {
+ self.delivery_outputs.outputs.len()
+ }
+
#[cfg(test)]
fn items(&self) -> Vec<TargetSubresourceMetadataOutput> {
self.delivery_outputs.subresource_outputs_from(0)