Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

feat(fetch): observe native request and interim response timing

ldm0029d2d0889d3
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rs
index 53e2fd6f5..fcecd3949 100644
--- a/moli-fetch/src/blocking/collectors.rs
+++ b/moli-fetch/src/blocking/collectors.rs
@@ -133,17 +133,28 @@ pub(crate) struct ResponseCollector {
max_response_size: Option<usize>,
response_too_large: bool,
cancel_handle: Option<FetchCancelHandle>,
+ response_progress: Option<ResponseProgress>,
}
impl ResponseCollector {
+ pub(crate) fn response_progress(&self) -> Option<&ResponseProgress> {
+ self.response_progress.as_ref()
+ }
+
pub(crate) fn new(cancel_handle: Option<FetchCancelHandle>) -> Self {
Self {
+ response_progress: cancel_handle
+ .as_ref()
+ .map(FetchCancelHandle::response_progress),
cancel_handle,
..Self::default()
}
}
pub(crate) fn begin_request(&mut self, max_response_size: Option<usize>) {
+ if let Some(cancel) = &self.cancel_handle {
+ self.response_progress = Some(cancel.response_progress());
+ }
self.body.clear();
self.headers.clear();
self.status_text = None;
@@ -190,6 +201,7 @@ pub struct StreamingResponseCollector {
client_hint_response_policy: Option<ClientHintResponsePolicy>,
client_hint_restart_requested: bool,
cancel_handle: FetchCancelHandle,
+ response_progress: ResponseProgress,
utf8_pending: Vec<u8>,
negotiated_http_version: Option<NegotiatedHttpVersion>,
network_request_extra_info: Option<NetworkRequestExtraInfo>,
@@ -269,6 +281,10 @@ pub struct RawStreamingResponseCollector {
}
impl StreamingResponseCollector {
+ pub(crate) fn response_progress(&self) -> &ResponseProgress {
+ &self.response_progress
+ }
+
pub fn new(
cookie_store: SharedBrowserCookieStore,
start_tx: oneshot::Sender<Result<StreamingHtmlResponseStart>>,
@@ -299,6 +315,7 @@ impl StreamingResponseCollector {
callback_error: None,
client_hint_response_policy: None,
client_hint_restart_requested: false,
+ response_progress: cancel_handle.response_progress(),
cancel_handle,
utf8_pending: Vec::new(),
negotiated_http_version: None,
@@ -316,6 +333,7 @@ impl StreamingResponseCollector {
redirect_chain: Vec<RedirectInfo>,
cache_body_writer: Option<HttpCacheBodyWriter>,
) {
+ self.response_progress = self.cancel_handle.response_progress();
self.headers.clear();
self.status_text = None;
self.current_url = Some(current_url);
@@ -652,6 +670,10 @@ impl StreamingResponseCollector {
}
impl RawStreamingResponseCollector {
+ pub(crate) fn response_progress(&self) -> &ResponseProgress {
+ &self.response_progress
+ }
+
pub fn new(
cookie_store: SharedBrowserCookieStore,
start_tx: oneshot::Sender<Result<StreamingHtmlResponseStart>>,
@@ -1044,6 +1066,13 @@ impl Handler for ResponseCollector {
self.headers.clear();
self.status_text = Some(parsed.status_text);
self.body.clear();
+ if let Some(progress) = &self.response_progress {
+ if parsed.status < 200 {
+ progress.mark_interim_response_started();
+ } else {
+ progress.mark_response_started();
+ }
+ }
return true;
}
@@ -1110,6 +1139,11 @@ impl Handler for StreamingResponseCollector {
self.response_bytes_received = 0;
self.response_too_large = false;
self.cookie_set_reports.clear();
+ if self.status < 200 {
+ self.response_progress.mark_interim_response_started();
+ } else {
+ self.response_progress.mark_response_started();
+ }
return true;
}
@@ -1185,6 +1219,8 @@ impl Handler for RawStreamingResponseCollector {
self.cookie_set_reports.clear();
if self.status >= 200 {
self.response_progress.mark_response_started();
+ } else {
+ self.response_progress.mark_interim_response_started();
}
return true;
}
@@ -1364,13 +1400,19 @@ mod tests {
assert!(collector.header(b"HTTP/1.1 103 Early Hints\r\n"));
assert!(collector.header(b"\r\n"));
assert!(handle.response_timing().response_started_at().is_none());
+ let first = handle
+ .response_timing()
+ .first_interim_response_started_at()
+ .unwrap();
assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
let started = handle.response_timing().response_started_at().unwrap();
assert!(handle.response_timing().response_end_at().is_none());
assert!(collector.header(b"Content-Length: 0\r\n"));
assert!(collector.header(b"\r\n"));
let complete = handle.response_timing();
+ assert_eq!(complete.first_interim_response_started_at(), Some(first));
assert_eq!(complete.response_started_at(), Some(started));
+ assert!(started >= first);
assert!(complete.response_end_at().unwrap() >= started);
}
}
diff --git a/moli-fetch/src/cancellation.rs b/moli-fetch/src/cancellation.rs
index f0cf0b752..4cf4838c7 100644
--- a/moli-fetch/src/cancellation.rs
+++ b/moli-fetch/src/cancellation.rs
@@ -22,6 +22,24 @@ pub(crate) struct ResponseProgress {
}
impl ResponseProgress {
+ pub(crate) fn mark_request_started(&self) {
+ self.timing
+ .lock()
+ .request_started_at
+ .get_or_insert_with(Instant::now);
+ }
+
+ pub(crate) fn mark_interim_response_started(&self) {
+ self.timing
+ .lock()
+ .first_interim_response_started_at
+ .get_or_insert_with(Instant::now);
+ }
+
+ pub(crate) fn mark_response_body_received(&self) {
+ self.timing.lock().body_bytes_received = true;
+ }
+
pub(crate) fn commit(&self) {
self.finish_timing();
self.commit_state();
@@ -73,11 +91,23 @@ impl ResponseProgress {
/// Native response boundaries shared with observers of the selected transfer.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct FetchResponseTiming {
+ request_started_at: Option<Instant>,
+ first_interim_response_started_at: Option<Instant>,
+ body_bytes_received: bool,
response_started_at: Option<Instant>,
response_end_at: Option<Instant>,
}
impl FetchResponseTiming {
+ pub fn body_bytes_received(self) -> bool {
+ self.body_bytes_received
+ }
+ pub fn request_started_at(self) -> Option<Instant> {
+ self.request_started_at
+ }
+ pub fn first_interim_response_started_at(self) -> Option<Instant> {
+ self.first_interim_response_started_at
+ }
pub fn response_started_at(self) -> Option<Instant> {
self.response_started_at
}
@@ -108,6 +138,21 @@ pub struct FetchCancelHandle {
}
impl FetchCancelHandle {
+ #[cfg(test)]
+ fn mark_request_started(&self) {
+ self.response_progress().mark_request_started();
+ }
+
+ #[cfg(test)]
+ fn mark_interim_response_started(&self) {
+ self.response_progress().mark_interim_response_started();
+ }
+
+ #[cfg(test)]
+ fn mark_response_body_received(&self) {
+ self.response_progress().mark_response_body_received();
+ }
+
pub fn new() -> Self {
Self::default()
}
@@ -565,4 +610,46 @@ mod tests {
}
}
}
+ #[test]
+ fn interim_response_timing_is_first_only_and_isolated_by_response_generation() {
+ let parent = FetchCancelHandle::new();
+ let early = parent.child_for_subrequest();
+ early.mark_request_started();
+ let request_start = early.response_timing().request_started_at().unwrap();
+ early.mark_request_started();
+ assert_eq!(
+ early.response_timing().request_started_at(),
+ Some(request_start)
+ );
+ early.mark_interim_response_started();
+ early.mark_response_body_received();
+ let first = early
+ .response_timing()
+ .first_interim_response_started_at()
+ .unwrap();
+ early.mark_interim_response_started();
+ early.mark_response_started();
+ parent.adopt_response_progress(&early);
+ let timing = parent.response_timing();
+ assert_eq!(timing.first_interim_response_started_at(), Some(first));
+ assert!(timing.body_bytes_received());
+ assert!(request_start <= first);
+ assert!(timing.response_started_at().unwrap() >= first);
+
+ let final_response = parent.child_for_subrequest();
+ final_response.mark_response_started();
+ parent.adopt_response_progress(&final_response);
+ early.mark_interim_response_started();
+ assert!(
+ parent
+ .response_timing()
+ .first_interim_response_started_at()
+ .is_none()
+ );
+ assert!(!parent.response_timing().body_bytes_received());
+ assert!(parent.response_timing().request_started_at().is_none());
+ parent.begin_response_progress();
+ final_response.mark_interim_response_started();
+ assert_eq!(parent.response_timing(), Default::default());
+ }
}
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index 66879527b..d13cab77a 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -2728,6 +2728,14 @@ enum FetchResponseCollector {
}
impl FetchTransferHandler {
+ fn response_progress(&self) -> Option<&crate::cancellation::ResponseProgress> {
+ match &self.response {
+ FetchResponseCollector::Buffered(collector) => collector.response_progress(),
+ FetchResponseCollector::Streaming(collector) => Some(collector.response_progress()),
+ FetchResponseCollector::StreamingRaw(collector) => Some(collector.response_progress()),
+ }
+ }
+
fn new_buffered(collector: ResponseCollector) -> Self {
Self::new(FetchResponseCollector::Buffered(collector))
}
@@ -2837,6 +2845,11 @@ impl Handler for FetchTransferHandler {
}
fn progress(&mut self, dltotal: f64, dlnow: f64, ultotal: f64, ulnow: f64) -> bool {
+ if dlnow > 0.0
+ && let Some(timing) = self.response_progress()
+ {
+ timing.mark_response_body_received();
+ }
match &mut self.response {
FetchResponseCollector::Buffered(collector) => {
collector.progress(dltotal, dlnow, ultotal, ulnow)
@@ -2856,6 +2869,9 @@ impl Handler for FetchTransferHandler {
let is_proxy_connect = self
.proxy_connect_response_recorder
.record_outgoing_header_block(data);
+ if !is_proxy_connect && let Some(timing) = self.response_progress() {
+ timing.mark_request_started();
+ }
if !is_proxy_connect
&& let Some(recorder) = self.network_observation_recorder.as_ref()
{
@@ -2877,7 +2893,9 @@ fn configure_network_observation(
capture_proxy_connect_response: bool,
) -> Result<()> {
let recorder = request.network_observation_recorder().cloned();
- let verbose = recorder.is_some() || capture_proxy_connect_response;
+ let verbose = recorder.is_some()
+ || capture_proxy_connect_response
+ || easy.get_ref().response_progress().is_some();
if let Some(recorder) = recorder.as_ref() {
recorder.set_current_request_cookie_report(request_cookie_report.cloned());
}
diff --git a/moli-fetch/src/tests/response_timing.rs b/moli-fetch/src/tests/response_timing.rs
index 31765f1d0..d2e1172ea 100644
--- a/moli-fetch/src/tests/response_timing.rs
+++ b/moli-fetch/src/tests/response_timing.rs
@@ -1,6 +1,106 @@
use super::*;
use std::io::Write;
+#[tokio::test]
+async fn native_response_timing_distinguishes_encoded_empty_bodies_from_head_headers() -> Result<()>
+{
+ // A complete gzip member with an empty decoded payload.
+ const GZIP: &[u8] = &[
+ 31, 139, 8, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ ];
+ for method in ["GET", "HEAD"] {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/empty", listener.local_addr()?);
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ let (mut stream, _) = listener.accept()?;
+ let mut request = Vec::new();
+ let mut byte = [0];
+ while !request.ends_with(b"\r\n\r\n") {
+ stream.read_exact(&mut byte)?;
+ request.extend_from_slice(&byte);
+ }
+ assert!(request.starts_with(method.as_bytes()));
+ write!(
+ stream,
+ "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+ GZIP.len()
+ )?;
+ if method == "GET" {
+ stream.write_all(GZIP)?;
+ }
+ Ok(())
+ });
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ let cancel = FetchCancelHandle::new();
+ let request = Request::new(method, &url, None, crate::RequestHeaders::default())?;
+ let mut response = client
+ .fetch_raw_stream_with_cancel(request, cancel.clone())
+ .await?;
+ while let Some(chunk) = response.next_chunk().await {
+ assert!(chunk.is_empty(), "decoded payload must remain empty");
+ }
+ response.finish().await?;
+ assert_eq!(
+ cancel.response_timing().body_bytes_received(),
+ method == "GET"
+ );
+ server.join().expect("server thread")?;
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn native_response_timing_separates_first_interim_and_final_headers() -> Result<()> {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/interim", listener.local_addr()?);
+ let (release_final, ready) = std_mpsc::sync_channel(1);
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ let (mut stream, _) = listener.accept()?;
+ let mut request = Vec::new();
+ let mut byte = [0];
+ while !request.ends_with(b"\r\n\r\n") {
+ stream.read_exact(&mut byte)?;
+ request.extend_from_slice(&byte);
+ }
+ stream.write_all(b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 103 Early Hints\r\nLink: </a>;rel=preload\r\n\r\n")?;
+ ready.recv().expect("consumer releases final headers");
+ stream
+ .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody")?;
+ Ok(())
+ });
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ let cancel = FetchCancelHandle::new();
+ let fetch = client.fetch_raw_stream_with_cancel(Request::get(&url)?, cancel.clone());
+ let release = async {
+ let deadline = Instant::now() + Duration::from_secs(5);
+ while cancel
+ .response_timing()
+ .first_interim_response_started_at()
+ .is_none()
+ {
+ assert!(Instant::now() < deadline, "interim status was not observed");
+ tokio::time::sleep(Duration::from_millis(1)).await;
+ }
+ assert!(cancel.response_timing().response_started_at().is_none());
+ release_final.send(()).unwrap();
+ };
+ let (response, ()) = tokio::join!(fetch, release);
+ let mut response = response?;
+ while response.next_chunk().await.is_some() {}
+ response.finish().await?;
+ let timing = cancel.response_timing();
+ assert!(
+ timing.request_started_at().unwrap() <= timing.first_interim_response_started_at().unwrap()
+ );
+ assert!(
+ timing.first_interim_response_started_at().unwrap()
+ <= timing.response_started_at().unwrap()
+ );
+ assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());
+ server.join().expect("server thread")?;
+ Ok(())
+}
+
#[tokio::test]
async fn raw_response_timing_is_native_even_when_the_consumer_does_not_poll() -> Result<()> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;