Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use crate::automation::{AutomationEvent, DevToolsNetworkResourceType};
use moli_cookie_jar::{
StoredCookieQueryReport, StoredCookieSetRejectionReason, StoredCookieSetStatus,
};
use moli_fetch::{
NegotiatedHttpVersion, NetworkExchangeObservation, NetworkObservationJournal,
NetworkRequestObservation, NetworkResponseObservation, RedirectInfo,
};
use serde_json::{Value, json};
use tokio::sync::mpsc::unbounded_channel;
use url::Url;
use super::gate::{
MainDocumentFailedNavigationProgressSource, MainDocumentProgressDrain,
MainDocumentProgressEventBatch, MainDocumentProgressOutputBoundary,
MainDocumentProgressOutputTarget, MainDocumentProgressPhase, MainDocumentProgressSource,
MainDocumentProgressSourceKind,
};
use super::*;
#[test]
fn aborted_navigation_preserves_the_document_without_hiding_other_failures() {
use crate::automation::DevToolsRequestFailure;
for error in [
anyhow::Error::new(moli_fetch::FetchCancelled),
anyhow::Error::new(DevToolsRequestFailure::Aborted),
anyhow::anyhow!("transport detail").context(moli_fetch::FetchCancelled),
anyhow::anyhow!("interception detail").context(DevToolsRequestFailure::Aborted),
] {
assert_eq!(
FailedNavigationDocumentPolicy::for_navigation_error(&error),
FailedNavigationDocumentPolicy::PreserveCommittedDocument,
);
let error = error
.context("failed to prepare page")
.context("failed to continue intercepted navigation");
assert_eq!(
FailedNavigationDocumentPolicy::for_navigation_error(&error),
FailedNavigationDocumentPolicy::PreserveCommittedDocument,
);
}
for error_text in [
"Aborted",
"net::ERR_ABORTED",
"Failed",
"net::ERR_CONNECTION_RESET",
"net::ERR_BLOCKED_BY_CLIENT",
] {
for error in [
anyhow::anyhow!(error_text),
anyhow::Error::new(DevToolsRequestFailure::Failed(error_text.to_owned())),
] {
assert_eq!(
FailedNavigationDocumentPolicy::for_navigation_error(&error),
FailedNavigationDocumentPolicy::InvalidateCommittedDocument,
);
assert_eq!(
FailedNavigationDocumentPolicy::for_navigation_error(
&error.context("net::ERR_ABORTED")
),
FailedNavigationDocumentPolicy::InvalidateCommittedDocument,
);
}
}
}
fn completed_events() -> CompletedMainDocumentNetworkEvents {
CompletedMainDocumentNetworkEvents::new(
"GET".to_owned(),
vec![("Accept".to_owned(), "text/html".to_owned())],
None,
200,
vec![("Content-Type".to_owned(), b"text/html".to_vec())],
Vec::new(),
Vec::new(),
false,
false,
)
}
fn observation_journal(
exchanges: Vec<(Vec<(String, String)>, u16, Vec<(String, String)>)>,
) -> NetworkObservationJournal {
NetworkObservationJournal::from_exchanges(
exchanges
.into_iter()
.map(|(request_headers, status, response_headers)| {
NetworkExchangeObservation::new(
NetworkRequestObservation::new(request_headers),
Some(NetworkResponseObservation::new(
status,
moli_fetch::headers_from_byte_strings(&response_headers).unwrap(),
)),
)
})
.collect(),
)
}
fn completed_progress_context() -> CompletedMainDocumentProgressContext {
CompletedMainDocumentProgressContext::new(
vec![Some("SID-1".to_owned())],
Some("REQ-1".to_owned()),
false,
Url::parse("http://example.test/start").unwrap(),
"GET".to_owned(),
None,
vec![("Accept".to_owned(), "text/html".to_owned())],
"LOADER-1".to_owned(),
"FRAME-1".to_owned(),
12.5,
)
}
fn live_progress_source() -> (
MainDocumentLiveNetworkProgressSource,
tokio::sync::mpsc::UnboundedReceiver<crate::conn::BackgroundProtocolEvent>,
) {
let (sender, receiver) = unbounded_channel();
(
MainDocumentLiveNetworkProgressSource {
sender: Some(sender),
progress_queue: super::gate::MainDocumentProgressQueueHandle::from_source(
MainDocumentProgressSource::streaming(),
),
session_ids: vec![Some("SID-1".to_owned())],
request_id: "REQ-1".to_owned(),
loader_id: "LOADER-1".to_owned(),
frame_id: "FRAME-1".to_owned(),
timestamp: 12.5,
initial_request_headers: vec![("Accept".to_owned(), "text/html".to_owned())],
initial_request_cookie_report: None,
},
receiver,
)
}
#[test]
fn failed_initial_transport_emits_request_extra_info_from_observed_headers() {
let (live_source, mut receiver) = live_progress_source();
let source = MainDocumentBodyProgressSource::from_live_source(
Some(live_source),
MainDocumentResponseVisibility::Immediate,
);
let journal =
NetworkObservationJournal::from_request_observation(NetworkRequestObservation::new(vec![
("Host".to_owned(), "example.test".to_owned()),
("Accept-Encoding".to_owned(), "gzip".to_owned()),
]));
source.emit_failed_initial_request_extra_info(&journal);
let event = receiver
.try_recv()
.expect("failed request ExtraInfo should be emitted")
.into_protocol_message();
assert_eq!(event["method"], json!("Network.requestWillBeSentExtraInfo"));
assert_eq!(event["params"]["requestId"], json!("REQ-1"));
assert_eq!(event["params"]["headers"]["Host"], json!("example.test"));
assert_eq!(event["params"]["headers"]["Accept-Encoding"], json!("gzip"));
assert_eq!(event["params"]["associatedCookies"], json!([]));
assert!(receiver.try_recv().is_err());
}
#[test]
fn failed_redirect_transport_emits_completed_hop_and_final_request_extra_info() {
let (live_source, mut receiver) = live_progress_source();
let source = MainDocumentBodyProgressSource::from_live_source(
Some(live_source),
MainDocumentResponseVisibility::Immediate,
);
let final_url = Url::parse("http://example.test/reset").unwrap();
let redirect = RedirectInfo {
source: moli_fetch::RedirectSource::Network,
from_url: Url::parse("http://example.test/start").unwrap(),
to_url: final_url.clone(),
status: 302,
headers: vec![(
"Location".to_owned(),
final_url.as_str().as_bytes().to_vec(),
)],
network_extra_info_available: true,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: true,
request_cookie_report: None,
cookie_set_reports: Vec::new(),
from_cache: false,
negotiated_http_version: None,
};
let journal = NetworkObservationJournal::from_exchanges(vec![
NetworkExchangeObservation::new(
NetworkRequestObservation::new(vec![(
"X-Request-Hop".to_owned(),
"initial".to_owned(),
)]),
Some(NetworkResponseObservation::new(
302,
vec![("X-Response-Hop".to_owned(), b"redirect".to_vec())],
)),
),
NetworkExchangeObservation::new(
NetworkRequestObservation::new(vec![("X-Request-Hop".to_owned(), "final".to_owned())]),
None,
),
]);
source.emit_failed_request_progress(
"GET",
None,
&[("Accept".to_owned(), "text/html".to_owned())],
&[redirect],
&journal,
);
let messages = std::iter::from_fn(|| receiver.try_recv().ok())
.map(crate::conn::BackgroundProtocolEvent::into_protocol_message)
.collect::<Vec<_>>();
assert_eq!(
messages
.iter()
.map(|message| message["method"].as_str().unwrap())
.collect::<Vec<_>>(),
[
"Network.requestWillBeSentExtraInfo",
"Network.requestWillBeSent",
"Network.responseReceivedExtraInfo",
"Network.requestWillBeSentExtraInfo",
]
);
assert_eq!(
messages[0]["params"]["headers"]["X-Request-Hop"],
json!("initial")
);
assert_eq!(messages[1]["params"]["request"]["url"], json!(final_url));
assert_eq!(
messages[1]["params"]["redirectResponse"]["status"],
json!(302)
);
assert_eq!(messages[1]["params"]["redirectHasExtraInfo"], json!(true));
assert_eq!(messages[2]["params"]["statusCode"], json!(302));
assert_eq!(
messages[2]["params"]["headers"]["X-Response-Hop"],
json!("redirect")
);
assert_eq!(
messages[3]["params"]["headers"]["X-Request-Hop"],
json!("final")
);
}
fn protocol_messages_from_background_events(
events: Vec<crate::conn::BackgroundProtocolEvent>,
) -> Vec<Value> {
events
.into_iter()
.map(crate::conn::BackgroundProtocolEvent::into_protocol_message)
.collect()
}
fn drain_into_background_events(
drain: &mut MainDocumentProgressDrain,
) -> Vec<crate::conn::BackgroundProtocolEvent> {
let mut events = Vec::new();
{
let mut output = MainDocumentProgressOutputTarget::background_events(&mut events);
drain.drain_into_output_target(&mut output);
}
events
}
fn drain_into_protocol_messages(drain: &mut MainDocumentProgressDrain) -> Vec<Value> {
protocol_messages_from_background_events(drain_into_background_events(drain))
}
fn drain_gate_until_response_metadata_visible_into_protocol_messages(
gate: &mut MainDocumentProgressGate,
) -> Vec<Value> {
let mut events = Vec::new();
MainDocumentProgressBackgroundEventBarrier::drain_until_response_metadata_visible(
&mut events,
gate,
);
protocol_messages_from_background_events(events)
}
fn drain_gate_until_body_finished_visible_into_protocol_messages(
gate: &mut MainDocumentProgressGate,
) -> Vec<Value> {
let mut events = Vec::new();
MainDocumentProgressBackgroundEventBarrier::drain_until_body_finished_visible(
&mut events,
gate,
);
protocol_messages_from_background_events(events)
}
fn response_progress_batch_for_output_target() -> MainDocumentProgressEventBatch {
MainDocumentProgressEventBatch::from_events(vec![
MainDocumentNavigationProgressEvent::ResponseReceived {
target: MainDocumentProgressEventTarget {
session_ids: vec![None, Some("SID-2".to_owned())],
request_id: "REQ-2".to_owned(),
loader_id: "LOADER-2".to_owned(),
frame_id: "FRAME-2".to_owned(),
timestamp: 21.0,
},
final_url: Url::parse("http://example.test/target").unwrap(),
status: 200,
headers: vec![("Content-Type".to_owned(), b"text/html".to_vec())],
cookie_set_reports: Vec::new(),
extra_info_status: 200,
extra_info_headers: vec![("Content-Type".to_owned(), b"text/html".to_vec())],
network_extra_info_available: false,
emit_extra_info: false,
encoded_data_length: 29,
from_cache: false,
negotiated_http_version: None,
has_extra_info: false,
},
])
}
#[test]
fn pending_document_progress_transfer_keeps_body_unmaterialized() {
let transfer = CompletedDocumentProgressTransfer::new_pending_body(
MainDocumentBodyNetworkProgress::StreamingBody,
);
let (body, progress) = transfer.into_parts();
assert!(matches!(body, CompletedDocumentProgressBody::Pending));
assert!(matches!(
progress,
MainDocumentBodyNetworkProgress::StreamingBody
));
}
#[test]
fn progress_output_target_serializes_background_targets_consistently() {
let (sender, mut receiver) = unbounded_channel();
let mut background = MainDocumentProgressOutputTarget::background_sender(&sender);
background.emit_batch(response_progress_batch_for_output_target());
drop(sender);
let mut background_events = Vec::new();
MainDocumentProgressOutputTarget::background_events(&mut background_events)
.emit_batch(response_progress_batch_for_output_target());
let mut background_parts = Vec::new();
while let Ok(value) = receiver.try_recv() {
background_parts.push(value.into_parts());
}
let background_event_parts = background_events
.into_iter()
.map(|event| event.into_parts())
.collect::<Vec<_>>();
let background_out = background_parts
.iter()
.map(|(message, _)| message.clone())
.collect::<Vec<_>>();
let background_event_out = background_event_parts
.iter()
.map(|(message, _)| message.clone())
.collect::<Vec<_>>();
assert_eq!(background_event_out, background_out);
assert_eq!(background_out.len(), 2);
assert!(matches!(
background_parts[0].1,
Some(AutomationEvent::NetworkResponseStarted(_))
));
assert!(matches!(
background_parts[1].1,
Some(AutomationEvent::NetworkResponseStarted(_))
));
assert!(matches!(
background_event_parts[0].1,
Some(AutomationEvent::NetworkResponseStarted(_))
));
assert!(matches!(
background_event_parts[1].1,
Some(AutomationEvent::NetworkResponseStarted(_))
));
assert_eq!(
background_out[0]["method"],
json!("Network.responseReceived")
);
assert_eq!(
background_out[0]["params"]["response"]["encodedDataLength"],
json!(29)
);
assert_eq!(background_out[1]["sessionId"], json!("SID-2"));
}
#[test]
fn progress_output_target_background_raw_events_stay_sidecar_free_without_classifier_bridge() {
let report = StoredCookieQueryReport::default();
let (sender, mut receiver) = unbounded_channel();
let mut background = MainDocumentProgressOutputTarget::background_sender(&sender);
super::emit::emit_request_will_be_sent_extra_info(
&mut background,
Some("SID-extra"),
"REQ-extra",
&[("Accept".to_owned(), "text/html".to_owned())],
&report,
1.25,
);
drop(sender);
let mut background_events = Vec::new();
let mut background_event_output =
MainDocumentProgressOutputTarget::background_events(&mut background_events);
super::emit::emit_request_will_be_sent_extra_info(
&mut background_event_output,
Some("SID-extra"),
"REQ-extra",
&[("Accept".to_owned(), "text/html".to_owned())],
&report,
1.25,
);
let event = receiver
.try_recv()
.expect("background extraInfo event should be sent");
let (message, automation_event) = event.into_parts();
let (background_event_message, background_event_sidecar) = background_events
.pop()
.expect("background event target should collect extraInfo event")
.into_parts();
assert_eq!(background_event_message, message);
assert_eq!(
message["method"],
json!("Network.requestWillBeSentExtraInfo")
);
assert_eq!(message["sessionId"], json!("SID-extra"));
assert!(
automation_event.is_none(),
"extraInfo raw protocol events should stay sidecar-free without a classifier bridge"
);
assert!(
background_event_sidecar.is_none(),
"background event target should keep the sender target sidecar semantics"
);
}
#[test]
fn live_progress_source_serializes_through_progress_emissions() {
let (source, mut receiver) = live_progress_source();
let final_url = Url::parse("http://example.test/final").unwrap();
let cookie_set_reports = vec![moli_cookie_jar::StoredCookieSetReport {
status: StoredCookieSetStatus::Rejected(StoredCookieSetRejectionReason::Parse),
rejection_reasons: vec![StoredCookieSetRejectionReason::Parse],
warning_reasons: Vec::new(),
effective_same_site: None,
}];
let redirect = RedirectInfo {
source: moli_fetch::RedirectSource::Network,
from_url: Url::parse("http://example.test/start").unwrap(),
to_url: final_url.clone(),
status: 302,
headers: vec![(
"Location".to_owned(),
final_url.as_str().as_bytes().to_vec(),
)],
network_extra_info_available: true,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: true,
request_cookie_report: None,
cookie_set_reports,
from_cache: false,
negotiated_http_version: None,
};
let journal = observation_journal(vec![
(
vec![("Host".to_owned(), "example.test".to_owned())],
302,
vec![("Location".to_owned(), final_url.to_string())],
),
(
vec![("Host".to_owned(), "example.test".to_owned())],
200,
vec![("Content-Type".to_owned(), "text/html".to_owned())],
),
]);
source.emit_redirect_requests(
"GET",
None,
&[("Accept".to_owned(), "text/html".to_owned())],
None,
&[redirect],
&journal,
true,
);
source.emit_response_received(
&final_url,
200,
&[("Content-Type".to_owned(), b"text/html".to_vec())],
&[],
&journal,
1,
true,
false,
None,
);
source.emit_body_finished(17);
let initial_request_extra = receiver
.try_recv()
.expect("initial request extra info event");
let (initial_request_extra, initial_request_extra_sidecar) = initial_request_extra.into_parts();
assert!(initial_request_extra_sidecar.is_none());
assert_eq!(
initial_request_extra["method"],
json!("Network.requestWillBeSentExtraInfo")
);
let redirect_request = receiver.try_recv().expect("redirect request event");
let (redirect_request, redirect_request_automation_event) = redirect_request.into_parts();
assert!(matches!(
redirect_request_automation_event,
Some(AutomationEvent::NetworkBeforeRequestSent(_))
));
assert_eq!(
redirect_request["method"],
json!("Network.requestWillBeSent")
);
assert_eq!(redirect_request["params"]["requestId"], json!("REQ-1"));
assert_eq!(
redirect_request["params"]["redirectResponse"]["status"],
json!(302)
);
assert_eq!(
redirect_request["params"]["redirectHasExtraInfo"],
json!(true)
);
let redirect_extra_info = receiver.try_recv().expect("redirect extra info event");
let (redirect_extra_info, redirect_extra_info_automation_event) =
redirect_extra_info.into_parts();
assert!(redirect_extra_info_automation_event.is_none());
assert_eq!(
redirect_extra_info["method"],
json!("Network.responseReceivedExtraInfo")
);
assert_eq!(redirect_extra_info["params"]["statusCode"], json!(302));
let final_request_extra = receiver.try_recv().expect("final request extra info event");
let (final_request_extra, final_request_extra_sidecar) = final_request_extra.into_parts();
assert!(final_request_extra_sidecar.is_none());
assert_eq!(
final_request_extra["method"],
json!("Network.requestWillBeSentExtraInfo")
);
let response_extra = receiver
.try_recv()
.expect("final response extra info event");
let (response_extra, response_extra_sidecar) = response_extra.into_parts();
assert!(response_extra_sidecar.is_none());
assert_eq!(
response_extra["method"],
json!("Network.responseReceivedExtraInfo")
);
assert_eq!(response_extra["params"]["statusCode"], json!(200));
let response = receiver.try_recv().expect("response event");
let (response, response_automation_event) = response.into_parts();
assert!(matches!(
response_automation_event,
Some(AutomationEvent::NetworkResponseStarted(_))
));
assert_eq!(response["method"], json!("Network.responseReceived"));
assert_eq!(response["params"]["hasExtraInfo"], json!(true));
assert_eq!(
response["params"]["response"]["url"],
json!(final_url.as_str())
);
let data = receiver.try_recv().expect("data received event");
let (data, data_automation_event) = data.into_parts();
assert!(data_automation_event.is_none());
assert_eq!(data["method"], json!("Network.dataReceived"));
assert_eq!(data["params"]["requestId"], json!("REQ-1"));
assert_eq!(data["params"]["dataLength"], json!(17));
assert_eq!(data["params"]["encodedDataLength"], json!(17));
let finished = receiver.try_recv().expect("loading finished event");
let (finished, finished_automation_event) = finished.into_parts();
assert!(matches!(
finished_automation_event,
Some(AutomationEvent::NetworkResponseCompleted(event))
if event.request_id.as_str() == "REQ-1"
&& event.encoded_data_length == Some(17)
));
assert_eq!(finished["method"], json!("Network.loadingFinished"));
assert_eq!(finished["params"]["encodedDataLength"], json!(17));
}
#[test]
fn completed_body_http_response_emits_correlated_empty_cookie_extra_info() {
let final_url = Url::parse("http://example.test/final").unwrap();
let mut events = completed_events();
events.network_extra_info_available = true;
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
let network_methods = out
.iter()
.filter_map(|message| message["method"].as_str())
.filter(|method| {
matches!(
*method,
"Network.requestWillBeSent"
| "Network.requestWillBeSentExtraInfo"
| "Network.responseReceivedExtraInfo"
| "Network.responseReceived"
)
})
.collect::<Vec<_>>();
assert_eq!(
network_methods,
[
"Network.requestWillBeSent",
"Network.requestWillBeSentExtraInfo",
"Network.responseReceivedExtraInfo",
"Network.responseReceived",
]
);
let request_extra = out
.iter()
.find(|message| message["method"] == json!("Network.requestWillBeSentExtraInfo"))
.expect("request extra info");
assert_eq!(request_extra["params"]["associatedCookies"], json!([]));
let response = out
.iter()
.find(|message| message["method"] == json!("Network.responseReceived"))
.expect("response event");
assert_eq!(response["params"]["hasExtraInfo"], json!(true));
}
#[test]
fn completed_body_extra_info_uses_transport_observed_headers() {
let final_url = Url::parse("http://example.test/final").unwrap();
let mut events = completed_events();
events.network_extra_info_available = true;
events = events.with_network_observation_journal(observation_journal(vec![(
vec![
("Host".to_owned(), "example.test".to_owned()),
("Accept-Encoding".to_owned(), "gzip, deflate".to_owned()),
],
200,
vec![("X-Raw-Response".to_owned(), "observed".to_owned())],
)]));
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
let request_extra = out
.iter()
.find(|message| message["method"] == json!("Network.requestWillBeSentExtraInfo"))
.expect("request extra info");
assert_eq!(
request_extra["params"]["headers"]["Host"],
json!("example.test")
);
assert_eq!(
request_extra["params"]["headers"]["Accept-Encoding"],
json!("gzip, deflate")
);
let response_extra = out
.iter()
.find(|message| message["method"] == json!("Network.responseReceivedExtraInfo"))
.expect("response extra info");
assert_eq!(
response_extra["params"]["headers"]["X-Raw-Response"],
json!("observed")
);
}
#[test]
fn completed_body_auth_retry_uses_initial_request_and_final_response_observations() {
let final_url = Url::parse("http://example.test/auth").unwrap();
let mut events = completed_events();
events.network_extra_info_available = true;
events = events.with_network_observation_journal(observation_journal(vec![
(
vec![
("Host".to_owned(), "example.test".to_owned()),
("X-Auth-Attempt".to_owned(), "initial".to_owned()),
],
407,
vec![(
"Proxy-Authenticate".to_owned(),
"Basic realm=\"proxy\"".to_owned(),
)],
),
(
vec![
("Host".to_owned(), "example.test".to_owned()),
("Proxy-Authorization".to_owned(), "Basic secret".to_owned()),
("X-Auth-Attempt".to_owned(), "credential".to_owned()),
],
200,
vec![("X-Final-Response".to_owned(), "yes".to_owned())],
),
]));
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
let request_extra = out
.iter()
.find(|message| message["method"] == json!("Network.requestWillBeSentExtraInfo"))
.expect("request extra info");
assert_eq!(
request_extra["params"]["headers"]["X-Auth-Attempt"],
json!("initial")
);
assert!(
request_extra["params"]["headers"]
.get("Proxy-Authorization")
.is_none()
);
let response_extra = out
.iter()
.find(|message| message["method"] == json!("Network.responseReceivedExtraInfo"))
.expect("response extra info");
assert_eq!(
response_extra["params"]["headers"]["X-Final-Response"],
json!("yes")
);
assert_eq!(response_extra["params"]["statusCode"], json!(200));
}
#[test]
fn completed_body_revalidation_keeps_raw_304_extra_info() {
let final_url = Url::parse("http://example.test/revalidated").unwrap();
let mut events = completed_events();
events.network_extra_info_available = true;
events.response_status = 200;
events.response_headers = vec![("X-Merged-Response".to_owned(), b"cached".to_vec())];
events = events.with_network_observation_journal(observation_journal(vec![(
vec![("If-None-Match".to_owned(), "\"v1\"".to_owned())],
304,
vec![("ETag".to_owned(), "\"v1\"".to_owned())],
)]));
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
let response_extra = out
.iter()
.find(|message| message["method"] == json!("Network.responseReceivedExtraInfo"))
.expect("response extra info");
assert_eq!(response_extra["params"]["statusCode"], json!(304));
assert_eq!(response_extra["params"]["headers"]["ETag"], json!("\"v1\""));
let response = out
.iter()
.find(|message| message["method"] == json!("Network.responseReceived"))
.expect("response event");
assert_eq!(response["params"]["response"]["status"], json!(200));
assert_eq!(
response["params"]["response"]["headers"]["X-Merged-Response"],
json!("cached")
);
}
#[test]
fn completed_body_http_service_worker_response_does_not_infer_extra_info_from_url() {
let final_url = Url::parse("http://example.test/sw-controlled").unwrap();
let events = completed_events();
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
assert!(
!out.iter().any(|message| {
matches!(
message["method"].as_str(),
Some("Network.requestWillBeSentExtraInfo" | "Network.responseReceivedExtraInfo")
)
}),
"an HTTP URL alone must not imply network-service ExtraInfo"
);
let response = out
.iter()
.find(|message| message["method"] == json!("Network.responseReceived"))
.expect("response event");
assert_eq!(response["params"]["hasExtraInfo"], json!(false));
}
#[test]
fn completed_body_http_redirect_emits_correlated_no_cookie_extra_info() {
let final_url = Url::parse("http://example.test/final").unwrap();
let mut events = completed_events();
events.network_extra_info_available = true;
events.redirect_chain = vec![moli_core::page::NavigationRedirect {
source: moli_fetch::RedirectSource::Network,
from_url: Url::parse("http://example.test/start").unwrap(),
to_url: final_url.clone(),
status: 302,
headers: vec![(
"location".to_owned(),
final_url.as_str().as_bytes().to_vec(),
)],
network_extra_info_available: true,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: true,
request_cookie_report: None,
cookie_set_reports: Vec::new(),
from_cache: false,
negotiated_http_version: None,
}];
events = events.with_network_observation_journal(observation_journal(vec![
(
vec![("X-Request-Hop".to_owned(), "initial".to_owned())],
302,
vec![("X-Response-Hop".to_owned(), "redirect".to_owned())],
),
(
vec![("X-Request-Hop".to_owned(), "final".to_owned())],
200,
vec![("X-Response-Hop".to_owned(), "final".to_owned())],
),
]));
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
let network_methods = out
.iter()
.filter_map(|message| message["method"].as_str())
.filter(|method| {
matches!(
*method,
"Network.requestWillBeSent"
| "Network.requestWillBeSentExtraInfo"
| "Network.responseReceivedExtraInfo"
| "Network.responseReceived"
)
})
.collect::<Vec<_>>();
assert_eq!(
network_methods,
[
"Network.requestWillBeSent",
"Network.requestWillBeSentExtraInfo",
"Network.requestWillBeSent",
"Network.responseReceivedExtraInfo",
"Network.requestWillBeSentExtraInfo",
"Network.responseReceivedExtraInfo",
"Network.responseReceived",
]
);
let redirected_request = out
.iter()
.filter(|message| message["method"] == json!("Network.requestWillBeSent"))
.nth(1)
.expect("redirected request");
assert_eq!(
redirected_request["params"]["redirectHasExtraInfo"],
json!(true)
);
let request_extras = out
.iter()
.filter(|message| message["method"] == json!("Network.requestWillBeSentExtraInfo"))
.collect::<Vec<_>>();
assert_eq!(
request_extras
.iter()
.map(|message| message["params"]["headers"]["X-Request-Hop"].clone())
.collect::<Vec<_>>(),
[json!("initial"), json!("final")]
);
let response_extras = out
.iter()
.filter(|message| message["method"] == json!("Network.responseReceivedExtraInfo"))
.collect::<Vec<_>>();
assert_eq!(
response_extras
.iter()
.map(|message| message["params"]["headers"]["X-Response-Hop"].clone())
.collect::<Vec<_>>(),
[json!("redirect"), json!("final")]
);
}
#[test]
fn completed_body_redirect_without_transport_extra_info_keeps_flag_false() {
let final_url = Url::parse("http://example.test/final").unwrap();
let mut events = completed_events();
events.redirect_chain = vec![moli_core::page::NavigationRedirect {
source: moli_fetch::RedirectSource::Network,
from_url: Url::parse("http://example.test/start").unwrap(),
to_url: final_url.clone(),
status: 302,
headers: vec![(
"location".to_owned(),
final_url.as_str().as_bytes().to_vec(),
)],
network_extra_info_available: false,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: false,
request_cookie_report: None,
cookie_set_reports: Vec::new(),
from_cache: false,
negotiated_http_version: None,
}];
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
let redirect_request = out
.iter()
.find(|message| {
message["method"] == json!("Network.requestWillBeSent")
&& message["params"]["redirectResponse"].is_object()
})
.expect("redirect requestWillBeSent");
assert_eq!(
redirect_request["params"]["redirectHasExtraInfo"],
json!(false),
"redirectHasExtraInfo must stay false without an observable HTTP response"
);
assert!(
!out.iter().any(|message| {
message["method"] == json!("Network.responseReceivedExtraInfo")
&& message["params"]["statusCode"] == json!(302)
}),
"a redirect without transport ExtraInfo must not enqueue responseReceivedExtraInfo"
);
}
#[test]
fn critical_client_hint_restart_keeps_discarded_response_extra_info_separate_from_internal_307() {
let navigation_url = Url::parse("http://example.test/start").unwrap();
let journal = observation_journal(vec![
(
vec![("Sec-CH-UA".to_owned(), "\"Chromium\";v=\"145\"".to_owned())],
403,
vec![
("Accept-CH".to_owned(), "Sec-CH-UA-Arch".to_owned()),
("Critical-CH".to_owned(), "Sec-CH-UA-Arch".to_owned()),
],
),
(
vec![
("Sec-CH-UA".to_owned(), "\"Chromium\";v=\"145\"".to_owned()),
("Sec-CH-UA-Arch".to_owned(), "\"x86\"".to_owned()),
],
200,
vec![("Content-Type".to_owned(), "text/html".to_owned())],
),
]);
let mut events = completed_events()
.with_negotiated_http_version(Some(NegotiatedHttpVersion::Http10))
.with_network_observation_journal(journal);
events.network_extra_info_available = true;
events.redirect_chain = vec![moli_core::page::NavigationRedirect {
source: moli_fetch::RedirectSource::Internal,
from_url: navigation_url.clone(),
to_url: navigation_url.clone(),
status: 307,
headers: vec![(
"Location".to_owned(),
navigation_url.as_str().as_bytes().to_vec(),
)],
network_extra_info_available: false,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: false,
request_cookie_report: None,
cookie_set_reports: Vec::new(),
from_cache: false,
negotiated_http_version: Some(NegotiatedHttpVersion::Http11),
}];
let batches = completed_progress_context().event_batches(&events, &navigation_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
let relevant = out
.iter()
.filter(|message| {
matches!(
message["method"].as_str(),
Some(
"Network.requestWillBeSent"
| "Network.requestWillBeSentExtraInfo"
| "Network.responseReceivedExtraInfo"
| "Network.responseReceived"
)
)
})
.collect::<Vec<_>>();
assert_eq!(
relevant
.iter()
.map(|message| message["method"].as_str().unwrap())
.collect::<Vec<_>>(),
vec![
"Network.requestWillBeSent",
"Network.requestWillBeSentExtraInfo",
"Network.responseReceivedExtraInfo",
"Network.requestWillBeSent",
"Network.requestWillBeSentExtraInfo",
"Network.responseReceivedExtraInfo",
"Network.responseReceived",
]
);
assert_eq!(relevant[2]["params"]["statusCode"], json!(403));
assert_eq!(
relevant[3]["params"]["redirectResponse"]["status"],
json!(307)
);
assert_eq!(
relevant[3]["params"]["redirectResponse"]["statusText"],
json!("Internal Redirect")
);
assert_eq!(
relevant[3]["params"]["redirectResponse"]["protocol"],
json!("http/1.1")
);
assert_eq!(relevant[3]["params"]["redirectHasExtraInfo"], json!(false));
assert_eq!(
relevant[4]["params"]["headers"]["Sec-CH-UA-Arch"],
json!("\"x86\"")
);
assert_eq!(relevant[5]["params"]["statusCode"], json!(200));
assert_eq!(relevant[6]["params"]["hasExtraInfo"], json!(true));
}
#[test]
fn completed_body_uses_negotiated_protocol_for_redirect_and_final_response() {
let final_url = Url::parse("https://example.test/final").unwrap();
let mut events =
completed_events().with_negotiated_http_version(Some(NegotiatedHttpVersion::Http2));
events.redirect_chain = vec![moli_core::page::NavigationRedirect {
source: moli_fetch::RedirectSource::Network,
from_url: Url::parse("https://example.test/start").unwrap(),
to_url: final_url.clone(),
status: 302,
headers: vec![(
"location".to_owned(),
final_url.as_str().as_bytes().to_vec(),
)],
network_extra_info_available: false,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: false,
request_cookie_report: None,
cookie_set_reports: Vec::new(),
from_cache: false,
negotiated_http_version: Some(NegotiatedHttpVersion::Http10),
}];
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
let redirect = out
.iter()
.find(|message| message["params"]["redirectResponse"].is_object())
.expect("redirect request");
assert_eq!(
redirect["params"]["redirectResponse"]["protocol"],
json!("http/1.0")
);
let response = out
.iter()
.find(|message| message["method"] == json!("Network.responseReceived"))
.expect("final response");
assert_eq!(response["params"]["response"]["protocol"], json!("h2"));
}
#[test]
fn cached_completed_body_emits_served_from_cache_before_response() {
let final_url = Url::parse("http://example.test/final").unwrap();
let mut events = completed_events();
events.response_from_cache = true;
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
let request = drain_into_protocol_messages(&mut drain);
assert_eq!(request.len(), 1);
assert_eq!(request[0]["method"], json!("Network.requestWillBeSent"));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::ResponseMetadataVisible);
let mut response_events = drain_into_background_events(&mut drain);
assert_eq!(response_events.len(), 2);
let (cached, cached_sidecar) = response_events.remove(0).into_parts();
assert_eq!(
cached,
json!({
"method": "Network.requestServedFromCache",
"params": { "requestId": "REQ-1" },
"sessionId": "SID-1"
})
);
assert!(cached_sidecar.is_none());
let (response, response_sidecar) = response_events.remove(0).into_parts();
assert_eq!(response["method"], json!("Network.responseReceived"));
assert_eq!(response["params"]["response"]["fromDiskCache"], json!(true));
assert!(matches!(
response_sidecar,
Some(AutomationEvent::NetworkResponseStarted(event))
if event.request_id.as_str() == "REQ-1" && event.from_cache
));
}
#[test]
fn cached_completed_body_redirect_emits_cache_event_before_next_request() {
let final_url = Url::parse("http://example.test/final").unwrap();
let mut events = completed_events();
events.redirect_chain = vec![moli_core::page::NavigationRedirect {
source: moli_fetch::RedirectSource::Network,
from_url: Url::parse("http://example.test/start").unwrap(),
to_url: final_url.clone(),
status: 302,
headers: vec![(
"location".to_owned(),
final_url.as_str().as_bytes().to_vec(),
)],
network_extra_info_available: false,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: false,
request_cookie_report: None,
cookie_set_reports: Vec::new(),
from_cache: true,
negotiated_http_version: None,
}];
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
let out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 3);
assert_eq!(out[0]["method"], json!("Network.requestWillBeSent"));
assert_eq!(out[1]["method"], json!("Network.requestServedFromCache"));
assert_eq!(out[1]["params"]["requestId"], json!("REQ-1"));
assert_eq!(out[2]["method"], json!("Network.requestWillBeSent"));
assert_eq!(
out[2]["params"]["redirectResponse"]["fromDiskCache"],
json!(true)
);
}
#[test]
fn live_ready_progress_emissions_reuse_streaming_output_queue() {
let mut drain = MainDocumentProgressDrain::new();
drain.append_ready_emission(
MainDocumentProgressPhase::ResponseReceived,
super::gate::MainDocumentProgressEmission::new(
MainDocumentProgressPhase::ResponseReceived,
response_progress_batch_for_output_target(),
),
);
let out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 2);
assert_eq!(out[0]["method"], json!("Network.responseReceived"));
assert_eq!(out[0]["params"]["response"]["encodedDataLength"], json!(29));
assert_eq!(out[1]["sessionId"], json!("SID-2"));
drain.append_ready_emission(
MainDocumentProgressPhase::BodyFinished,
super::gate::MainDocumentProgressEmission::new(
MainDocumentProgressPhase::BodyFinished,
MainDocumentProgressEventBatch::from_events(vec![
MainDocumentNavigationProgressEvent::LoadingFinished {
target: MainDocumentProgressEventTarget {
session_ids: vec![Some("SID-2".to_owned())],
request_id: "REQ-2".to_owned(),
loader_id: "LOADER-2".to_owned(),
frame_id: "FRAME-2".to_owned(),
timestamp: 21.0,
},
encoded_data_length: 29,
},
]),
),
);
let out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 2);
assert_eq!(out[0]["method"], json!("Network.dataReceived"));
assert_eq!(out[0]["params"]["dataLength"], json!(29));
assert_eq!(out[0]["params"]["encodedDataLength"], json!(29));
assert_eq!(out[1]["method"], json!("Network.loadingFinished"));
assert_eq!(out[1]["params"]["encodedDataLength"], json!(29));
assert_eq!(
out[1]["params"],
json!({
"requestId": "REQ-2",
"timestamp": 21.0,
"encodedDataLength": 29,
})
);
}
#[test]
fn progress_output_queue_buffers_body_finished_until_response_received() {
let mut drain = MainDocumentProgressDrain::new();
drain.append_ready_emission(
MainDocumentProgressPhase::BodyFinished,
super::gate::MainDocumentProgressEmission::new(
MainDocumentProgressPhase::BodyFinished,
MainDocumentProgressEventBatch::from_events(vec![
MainDocumentNavigationProgressEvent::LoadingFinished {
target: MainDocumentProgressEventTarget {
session_ids: vec![Some("SID-2".to_owned())],
request_id: "REQ-2".to_owned(),
loader_id: "LOADER-2".to_owned(),
frame_id: "FRAME-2".to_owned(),
timestamp: 21.0,
},
encoded_data_length: 29,
},
]),
),
);
let out = drain_into_protocol_messages(&mut drain);
assert!(
out.is_empty(),
"main document terminal Network event must wait for responseReceived"
);
drain.append_ready_emission(
MainDocumentProgressPhase::ResponseReceived,
super::gate::MainDocumentProgressEmission::new(
MainDocumentProgressPhase::ResponseReceived,
response_progress_batch_for_output_target(),
),
);
let out = drain_into_background_events(&mut drain);
let parts = out
.into_iter()
.map(crate::conn::BackgroundProtocolEvent::into_parts)
.collect::<Vec<_>>();
assert_eq!(parts.len(), 4);
assert_eq!(parts[0].0["method"], json!("Network.responseReceived"));
assert_eq!(parts[1].0["method"], json!("Network.responseReceived"));
assert_eq!(parts[2].0["method"], json!("Network.dataReceived"));
assert_eq!(parts[2].0["params"]["requestId"], json!("REQ-2"));
assert!(parts[2].1.is_none());
assert_eq!(parts[3].0["method"], json!("Network.loadingFinished"));
assert_eq!(parts[3].0["params"]["requestId"], json!("REQ-2"));
assert!(
matches!(
parts[3].1.as_ref(),
Some(AutomationEvent::NetworkResponseCompleted(event))
if event.request_id.as_str() == "REQ-2"
&& event.encoded_data_length == Some(29)
),
"buffered terminal event must retain its typed automation sidecar"
);
let out = drain_into_protocol_messages(&mut drain);
assert!(
out.is_empty(),
"buffered terminal event must only be released once"
);
}
#[test]
fn completed_body_progress_queue_drains_network_events_by_milestone() {
let final_url = Url::parse("http://example.test/final").unwrap();
let events = completed_events();
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
let mut out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 1);
assert_eq!(out[0]["method"], json!("Network.requestWillBeSent"));
assert_eq!(out[0]["params"]["requestId"], json!("REQ-1"));
assert_eq!(out[0]["params"]["loaderId"], json!("LOADER-1"));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::ResponseMetadataVisible);
out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 1);
assert_eq!(out[0]["method"], json!("Network.responseReceived"));
assert_eq!(
out[0]["params"]["response"]["url"],
json!(final_url.as_str())
);
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 2);
assert_eq!(out[0]["method"], json!("Network.dataReceived"));
assert_eq!(out[0]["params"]["dataLength"], json!(17));
assert_eq!(out[1]["method"], json!("Network.loadingFinished"));
assert_eq!(out[1]["params"]["encodedDataLength"], json!(17));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
out = drain_into_protocol_messages(&mut drain);
assert!(
out.is_empty(),
"repeating the same milestone must not duplicate Network events"
);
}
#[test]
fn completed_body_progress_queue_can_release_all_materialized_events_at_body_finished() {
let final_url = Url::parse("http://example.test/final").unwrap();
let events = completed_events();
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 4);
assert_eq!(out[0]["method"], json!("Network.requestWillBeSent"));
assert_eq!(out[1]["method"], json!("Network.responseReceived"));
assert_eq!(out[2]["method"], json!("Network.dataReceived"));
assert_eq!(out[2]["params"]["dataLength"], json!(17));
assert_eq!(out[3]["method"], json!("Network.loadingFinished"));
assert_eq!(out[3]["params"]["encodedDataLength"], json!(17));
}
#[test]
fn completed_body_progress_drain_keeps_mark_ready_separate_from_output_drain() {
let final_url = Url::parse("http://example.test/final").unwrap();
let events = completed_events();
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut drain =
MainDocumentProgressDrain::from_source(MainDocumentProgressSource::completed_body(batches));
let mut out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 1);
assert_eq!(out[0]["method"], json!("Network.requestWillBeSent"));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::ResponseMetadataVisible);
out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 1);
assert_eq!(out[0]["method"], json!("Network.responseReceived"));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 2);
assert_eq!(out[0]["method"], json!("Network.dataReceived"));
assert_eq!(out[0]["params"]["dataLength"], json!(17));
assert_eq!(out[1]["method"], json!("Network.loadingFinished"));
assert_eq!(out[1]["params"]["encodedDataLength"], json!(17));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::ResponseMetadataVisible);
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
out = drain_into_protocol_messages(&mut drain);
assert!(
out.is_empty(),
"draining a second time must not replay queued progress"
);
}
#[test]
fn progress_output_barrier_drains_source_generated_progress_before_cdp_output() {
let final_url = Url::parse("http://example.test/final").unwrap();
let events = completed_events();
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut gate = MainDocumentProgressGate::new(MainDocumentProgressDrain::from_source(
MainDocumentProgressSource::completed_body(batches),
));
let mut out = drain_gate_until_response_metadata_visible_into_protocol_messages(&mut gate);
assert_eq!(out.len(), 2);
assert_eq!(out[0]["method"], json!("Network.requestWillBeSent"));
assert_eq!(out[1]["method"], json!("Network.responseReceived"));
out = drain_gate_until_body_finished_visible_into_protocol_messages(&mut gate);
assert_eq!(out.len(), 2);
assert_eq!(out[0]["method"], json!("Network.dataReceived"));
assert_eq!(out[0]["params"]["dataLength"], json!(17));
assert_eq!(out[1]["method"], json!("Network.loadingFinished"));
assert_eq!(out[1]["params"]["encodedDataLength"], json!(17));
out = drain_gate_until_body_finished_visible_into_protocol_messages(&mut gate);
assert!(
out.is_empty(),
"body-finished milestones must be idempotent after the body is released"
);
let events = completed_events();
let batches = completed_progress_context().event_batches(&events, &final_url, 17);
let mut gate = MainDocumentProgressGate::new(MainDocumentProgressDrain::from_source(
MainDocumentProgressSource::completed_body(batches),
));
let mut out = {
let mut events = Vec::new();
let mut barrier =
MainDocumentProgressBackgroundEventBarrier::background_events(&mut events, &mut gate);
barrier.drain_progress();
protocol_messages_from_background_events(events)
};
crate::domains::command_output::CommandOutputPlan::result(json!({"ok": true})).emit_into(
&mut out,
Some(99),
Some("SID-1"),
);
assert_eq!(out.len(), 2);
assert_eq!(out[0]["method"], json!("Network.requestWillBeSent"));
assert_eq!(out[1]["id"], json!(99));
let out = drain_gate_until_body_finished_visible_into_protocol_messages(&mut gate);
assert_eq!(out.len(), 3);
assert_eq!(out[0]["method"], json!("Network.responseReceived"));
assert_eq!(out[1]["method"], json!("Network.dataReceived"));
assert_eq!(out[2]["method"], json!("Network.loadingFinished"));
}
#[test]
fn failed_navigation_progress_drain_prequeues_loading_failed() {
let target = MainDocumentProgressEventTarget {
session_ids: vec![Some("SID-1".to_owned())],
request_id: "REQ-1".to_owned(),
loader_id: "LOADER-1".to_owned(),
frame_id: "FRAME-1".to_owned(),
timestamp: 12.5,
};
let source = MainDocumentProgressSource::new(MainDocumentProgressSourceKind::FailedNavigation(
Box::new(MainDocumentFailedNavigationProgressSource {
failure_event: Some(MainDocumentNavigationProgressEvent::LoadingFailed {
target,
error_text: "net::ERR_FAILED".to_owned(),
}),
}),
));
let mut drain = MainDocumentProgressDrain::from_source(source);
let mut events = drain_into_background_events(&mut drain);
assert_eq!(events.len(), 1);
let (failed, failed_sidecar) = events.remove(0).into_parts();
assert!(matches!(
failed_sidecar,
Some(AutomationEvent::NetworkFetchError(event))
if event.request_id.as_str() == "REQ-1"
&& event.resource_type == Some(DevToolsNetworkResourceType::Document)
&& event.error_text.as_deref() == Some("net::ERR_FAILED")
));
assert_eq!(failed["method"], json!("Network.loadingFailed"));
assert_eq!(
failed["params"],
json!({
"requestId": "REQ-1",
"timestamp": 12.5,
"type": "Document",
"errorText": "net::ERR_FAILED",
"canceled": false,
})
);
let out = drain_into_protocol_messages(&mut drain);
assert!(
out.is_empty(),
"failed navigation progress must not replay loadingFailed"
);
}
#[test]
fn error_page_progress_releases_failed_before_finished_at_separate_boundaries() {
let target = MainDocumentProgressEventTarget {
session_ids: vec![Some("SID-1".to_owned())],
request_id: "REQ-1".to_owned(),
loader_id: "LOADER-1".to_owned(),
frame_id: "FRAME-1".to_owned(),
timestamp: 12.5,
};
let source = MainDocumentProgressSource::error_page(
Some(MainDocumentNavigationProgressEvent::LoadingFailed {
target: target.clone(),
error_text: "net::ERR_CONNECTION_REFUSED".to_owned(),
}),
Some(MainDocumentNavigationProgressEvent::LoadingFinished {
target,
encoded_data_length: 0,
}),
);
let mut drain = MainDocumentProgressDrain::from_source(source);
assert!(drain_into_protocol_messages(&mut drain).is_empty());
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::ResponseMetadataVisible);
let failed = drain_into_protocol_messages(&mut drain);
assert_eq!(failed.len(), 1);
assert_eq!(failed[0]["method"], json!("Network.loadingFailed"));
assert_eq!(failed[0]["params"]["requestId"], json!("REQ-1"));
assert_eq!(failed[0]["params"]["canceled"], json!(false));
drain.mark_output_visible_until(MainDocumentProgressOutputBoundary::BodyFinishedVisible);
let finished = drain_into_protocol_messages(&mut drain);
assert_eq!(finished.len(), 1);
assert_eq!(finished[0]["method"], json!("Network.loadingFinished"));
assert_eq!(finished[0]["params"]["requestId"], json!("REQ-1"));
assert_eq!(finished[0]["params"]["encodedDataLength"], json!(0));
assert!(drain_into_protocol_messages(&mut drain).is_empty());
}
#[test]
fn aborted_navigation_progress_marks_loading_failed_canceled() {
let target = MainDocumentProgressEventTarget {
session_ids: vec![Some("SID-1".to_owned())],
request_id: "REQ-1".to_owned(),
loader_id: "LOADER-1".to_owned(),
frame_id: "FRAME-1".to_owned(),
timestamp: 12.5,
};
let source = MainDocumentProgressSource::new(MainDocumentProgressSourceKind::FailedNavigation(
Box::new(MainDocumentFailedNavigationProgressSource {
failure_event: Some(MainDocumentNavigationProgressEvent::LoadingFailed {
target,
error_text: "net::ERR_ABORTED".to_owned(),
}),
}),
));
let mut drain = MainDocumentProgressDrain::from_source(source);
let out = drain_into_protocol_messages(&mut drain);
assert_eq!(out.len(), 1);
assert_eq!(out[0]["method"], json!("Network.loadingFailed"));
assert_eq!(out[0]["params"]["errorText"], json!("net::ERR_ABORTED"));
assert_eq!(out[0]["params"]["canceled"], json!(true));
}
#[test]
fn no_content_navigation_retains_response_metadata_with_live_progress() {
use crate::conn::{
BrowserContext, CommandOwnerScope, NavigationRequestLoadPolicy, NavigationResultProjection,
};
for status in [204, 205] {
let mut conn = CdpConnection::new();
let mut browser_context = BrowserContext::new("BID-1".to_owned());
browser_context.set_active_target_id("FRAME-1");
browser_context.attach_active_session("SID-1");
browser_context
.active_page_target_mut()
.devtools_sessions
.primary_mut()
.network_session_state
.network_enabled = true;
browser_context
.active_page_target_mut()
.runtime_slot
.enable_primary_network_events();
conn.install_browser_context_fixture_for_test(browser_context);
assert!(main_document_network_observed(&conn, Some("SID-1")));
let state = NavigationDispatchState {
web_mcp_invocation: None,
navigation_initiator: None,
initial_document_environment: None,
auxiliary_document_response: None,
redirect_chain: Vec::new(),
redirect_headers: None,
navigate_id: Some(1),
owner: CommandOwnerScope::for_session("SID-1"),
result_projection: NavigationResultProjection::Cdp(json!({
"frameId": "FRAME-1", "loaderId": "LOADER-1",
})),
frame_id: "FRAME-1".to_owned(),
session_id: Some("SID-1".to_owned()),
request_id: Some("REQ-1".to_owned()),
loader_id: "LOADER-1".to_owned(),
request_announced: false,
requested_url: Url::parse("http://example.test/start").unwrap(),
request_method: "GET".to_owned(),
request_body: None,
request_body_bytes: None,
request_headers: Vec::new().into(),
request_load_policy: NavigationRequestLoadPolicy::BrowserInitiated,
timestamp: 12.5,
source_document_security: Default::default(),
};
let (live_source, _receiver) = live_progress_source();
let source = MainDocumentBodyProgressSource::from_live_source(
Some(live_source),
MainDocumentResponseVisibility::Immediate,
);
let mut events = completed_events();
events.response_status = status;
events.response_headers = vec![("X-Ignored-Response".to_owned(), b"retained".to_vec())];
let navigation = NoContentNavigation {
final_url: Url::parse("http://example.test/final").unwrap(),
network_events: source.completed_body_network_events(events),
};
let MaterializedNavigationLoadOutcome::Failed(mut failed) =
materialize_no_content_navigation_progress(&conn, &state, navigation)
else {
panic!("ignored responses must retain main's failed-navigation projection");
};
let out = drain_gate_until_body_finished_visible_into_protocol_messages(
&mut failed.progress_gate,
);
let response = out
.iter()
.find(|event| event["method"] == "Network.responseReceived")
.expect("live ignored navigation must still emit its response metadata");
assert_eq!(response["params"]["requestId"], json!("REQ-1"));
assert_eq!(response["params"]["response"]["status"], json!(status));
assert_eq!(
response["params"]["response"]["headers"]["X-Ignored-Response"],
json!("retained")
);
let failure = out
.iter()
.find(|event| event["method"] == "Network.loadingFailed")
.expect("ignored navigation must terminate without committing a Document");
assert_eq!(
failure["params"]["errorText"],
json!(moli_fetch::NET_ERR_ABORTED_ERROR_TEXT)
);
assert_eq!(
failed.document_policy,
FailedNavigationDocumentPolicy::PreserveCommittedDocument
);
assert!(
!out.iter()
.any(|event| event["method"] == "Network.loadingFinished")
);
}
}