Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat(fetch): observe native request and interim response timing
ldm0029d2d0889d3
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rsindex 53e2fd6f5..fcecd3949 100644--- a/moli-fetch/src/blocking/collectors.rs+++ b/moli-fetch/src/blocking/collectors.rs@@ -133,17 +133,28 @@ pub(crate) struct ResponseCollector {max_response_size: Option<usize>,response_too_large: bool,cancel_handle: Option<FetchCancelHandle>,+ response_progress: Option<ResponseProgress>,}impl ResponseCollector {+ pub(crate) fn response_progress(&self) -> Option<&ResponseProgress> {+ self.response_progress.as_ref()+ }+pub(crate) fn new(cancel_handle: Option<FetchCancelHandle>) -> Self {Self {+ response_progress: cancel_handle+ .as_ref()+ .map(FetchCancelHandle::response_progress),cancel_handle,..Self::default()}}pub(crate) fn begin_request(&mut self, max_response_size: Option<usize>) {+ if let Some(cancel) = &self.cancel_handle {+ self.response_progress = Some(cancel.response_progress());+ }self.body.clear();self.headers.clear();self.status_text = None;@@ -190,6 +201,7 @@ pub struct StreamingResponseCollector {client_hint_response_policy: Option<ClientHintResponsePolicy>,client_hint_restart_requested: bool,cancel_handle: FetchCancelHandle,+ response_progress: ResponseProgress,utf8_pending: Vec<u8>,negotiated_http_version: Option<NegotiatedHttpVersion>,network_request_extra_info: Option<NetworkRequestExtraInfo>,@@ -269,6 +281,10 @@ pub struct RawStreamingResponseCollector {}impl StreamingResponseCollector {+ pub(crate) fn response_progress(&self) -> &ResponseProgress {+ &self.response_progress+ }+pub fn new(cookie_store: SharedBrowserCookieStore,start_tx: oneshot::Sender<Result<StreamingHtmlResponseStart>>,@@ -299,6 +315,7 @@ impl StreamingResponseCollector {callback_error: None,client_hint_response_policy: None,client_hint_restart_requested: false,+ response_progress: cancel_handle.response_progress(),cancel_handle,utf8_pending: Vec::new(),negotiated_http_version: None,@@ -316,6 +333,7 @@ impl StreamingResponseCollector {redirect_chain: Vec<RedirectInfo>,cache_body_writer: Option<HttpCacheBodyWriter>,) {+ self.response_progress = self.cancel_handle.response_progress();self.headers.clear();self.status_text = None;self.current_url = Some(current_url);@@ -652,6 +670,10 @@ impl StreamingResponseCollector {}impl RawStreamingResponseCollector {+ pub(crate) fn response_progress(&self) -> &ResponseProgress {+ &self.response_progress+ }+pub fn new(cookie_store: SharedBrowserCookieStore,start_tx: oneshot::Sender<Result<StreamingHtmlResponseStart>>,@@ -1044,6 +1066,13 @@ impl Handler for ResponseCollector {self.headers.clear();self.status_text = Some(parsed.status_text);self.body.clear();+ if let Some(progress) = &self.response_progress {+ if parsed.status < 200 {+ progress.mark_interim_response_started();+ } else {+ progress.mark_response_started();+ }+ }return true;}@@ -1110,6 +1139,11 @@ impl Handler for StreamingResponseCollector {self.response_bytes_received = 0;self.response_too_large = false;self.cookie_set_reports.clear();+ if self.status < 200 {+ self.response_progress.mark_interim_response_started();+ } else {+ self.response_progress.mark_response_started();+ }return true;}@@ -1185,6 +1219,8 @@ impl Handler for RawStreamingResponseCollector {self.cookie_set_reports.clear();if self.status >= 200 {self.response_progress.mark_response_started();+ } else {+ self.response_progress.mark_interim_response_started();}return true;}@@ -1364,13 +1400,19 @@ mod tests {assert!(collector.header(b"HTTP/1.1 103 Early Hints\r\n"));assert!(collector.header(b"\r\n"));assert!(handle.response_timing().response_started_at().is_none());+ let first = handle+ .response_timing()+ .first_interim_response_started_at()+ .unwrap();assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));let started = handle.response_timing().response_started_at().unwrap();assert!(handle.response_timing().response_end_at().is_none());assert!(collector.header(b"Content-Length: 0\r\n"));assert!(collector.header(b"\r\n"));let complete = handle.response_timing();+ assert_eq!(complete.first_interim_response_started_at(), Some(first));assert_eq!(complete.response_started_at(), Some(started));+ assert!(started >= first);assert!(complete.response_end_at().unwrap() >= started);}}diff --git a/moli-fetch/src/cancellation.rs b/moli-fetch/src/cancellation.rsindex f0cf0b752..4cf4838c7 100644--- a/moli-fetch/src/cancellation.rs+++ b/moli-fetch/src/cancellation.rs@@ -22,6 +22,24 @@ pub(crate) struct ResponseProgress {}impl ResponseProgress {+ pub(crate) fn mark_request_started(&self) {+ self.timing+ .lock()+ .request_started_at+ .get_or_insert_with(Instant::now);+ }++ pub(crate) fn mark_interim_response_started(&self) {+ self.timing+ .lock()+ .first_interim_response_started_at+ .get_or_insert_with(Instant::now);+ }++ pub(crate) fn mark_response_body_received(&self) {+ self.timing.lock().body_bytes_received = true;+ }+pub(crate) fn commit(&self) {self.finish_timing();self.commit_state();@@ -73,11 +91,23 @@ impl ResponseProgress {/// Native response boundaries shared with observers of the selected transfer.#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]pub struct FetchResponseTiming {+ request_started_at: Option<Instant>,+ first_interim_response_started_at: Option<Instant>,+ body_bytes_received: bool,response_started_at: Option<Instant>,response_end_at: Option<Instant>,}impl FetchResponseTiming {+ pub fn body_bytes_received(self) -> bool {+ self.body_bytes_received+ }+ pub fn request_started_at(self) -> Option<Instant> {+ self.request_started_at+ }+ pub fn first_interim_response_started_at(self) -> Option<Instant> {+ self.first_interim_response_started_at+ }pub fn response_started_at(self) -> Option<Instant> {self.response_started_at}@@ -108,6 +138,21 @@ pub struct FetchCancelHandle {}impl FetchCancelHandle {+ #[cfg(test)]+ fn mark_request_started(&self) {+ self.response_progress().mark_request_started();+ }++ #[cfg(test)]+ fn mark_interim_response_started(&self) {+ self.response_progress().mark_interim_response_started();+ }++ #[cfg(test)]+ fn mark_response_body_received(&self) {+ self.response_progress().mark_response_body_received();+ }+pub fn new() -> Self {Self::default()}@@ -565,4 +610,46 @@ mod tests {}}}+ #[test]+ fn interim_response_timing_is_first_only_and_isolated_by_response_generation() {+ let parent = FetchCancelHandle::new();+ let early = parent.child_for_subrequest();+ early.mark_request_started();+ let request_start = early.response_timing().request_started_at().unwrap();+ early.mark_request_started();+ assert_eq!(+ early.response_timing().request_started_at(),+ Some(request_start)+ );+ early.mark_interim_response_started();+ early.mark_response_body_received();+ let first = early+ .response_timing()+ .first_interim_response_started_at()+ .unwrap();+ early.mark_interim_response_started();+ early.mark_response_started();+ parent.adopt_response_progress(&early);+ let timing = parent.response_timing();+ assert_eq!(timing.first_interim_response_started_at(), Some(first));+ assert!(timing.body_bytes_received());+ assert!(request_start <= first);+ assert!(timing.response_started_at().unwrap() >= first);++ let final_response = parent.child_for_subrequest();+ final_response.mark_response_started();+ parent.adopt_response_progress(&final_response);+ early.mark_interim_response_started();+ assert!(+ parent+ .response_timing()+ .first_interim_response_started_at()+ .is_none()+ );+ assert!(!parent.response_timing().body_bytes_received());+ assert!(parent.response_timing().request_started_at().is_none());+ parent.begin_response_progress();+ final_response.mark_interim_response_started();+ assert_eq!(parent.response_timing(), Default::default());+ }}diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex 66879527b..d13cab77a 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -2728,6 +2728,14 @@ enum FetchResponseCollector {}impl FetchTransferHandler {+ fn response_progress(&self) -> Option<&crate::cancellation::ResponseProgress> {+ match &self.response {+ FetchResponseCollector::Buffered(collector) => collector.response_progress(),+ FetchResponseCollector::Streaming(collector) => Some(collector.response_progress()),+ FetchResponseCollector::StreamingRaw(collector) => Some(collector.response_progress()),+ }+ }+fn new_buffered(collector: ResponseCollector) -> Self {Self::new(FetchResponseCollector::Buffered(collector))}@@ -2837,6 +2845,11 @@ impl Handler for FetchTransferHandler {}fn progress(&mut self, dltotal: f64, dlnow: f64, ultotal: f64, ulnow: f64) -> bool {+ if dlnow > 0.0+ && let Some(timing) = self.response_progress()+ {+ timing.mark_response_body_received();+ }match &mut self.response {FetchResponseCollector::Buffered(collector) => {collector.progress(dltotal, dlnow, ultotal, ulnow)@@ -2856,6 +2869,9 @@ impl Handler for FetchTransferHandler {let is_proxy_connect = self.proxy_connect_response_recorder.record_outgoing_header_block(data);+ if !is_proxy_connect && let Some(timing) = self.response_progress() {+ timing.mark_request_started();+ }if !is_proxy_connect&& let Some(recorder) = self.network_observation_recorder.as_ref(){@@ -2877,7 +2893,9 @@ fn configure_network_observation(capture_proxy_connect_response: bool,) -> Result<()> {let recorder = request.network_observation_recorder().cloned();- let verbose = recorder.is_some() || capture_proxy_connect_response;+ let verbose = recorder.is_some()+ || capture_proxy_connect_response+ || easy.get_ref().response_progress().is_some();if let Some(recorder) = recorder.as_ref() {recorder.set_current_request_cookie_report(request_cookie_report.cloned());}diff --git a/moli-fetch/src/tests/response_timing.rs b/moli-fetch/src/tests/response_timing.rsindex 31765f1d0..d2e1172ea 100644--- a/moli-fetch/src/tests/response_timing.rs+++ b/moli-fetch/src/tests/response_timing.rs@@ -1,6 +1,106 @@use super::*;use std::io::Write;+#[tokio::test]+async fn native_response_timing_distinguishes_encoded_empty_bodies_from_head_headers() -> Result<()>+{+ // A complete gzip member with an empty decoded payload.+ const GZIP: &[u8] = &[+ 31, 139, 8, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0,+ ];+ for method in ["GET", "HEAD"] {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/empty", listener.local_addr()?);+ let server = thread::spawn(move || -> std::io::Result<()> {+ let (mut stream, _) = listener.accept()?;+ let mut request = Vec::new();+ let mut byte = [0];+ while !request.ends_with(b"\r\n\r\n") {+ stream.read_exact(&mut byte)?;+ request.extend_from_slice(&byte);+ }+ assert!(request.starts_with(method.as_bytes()));+ write!(+ stream,+ "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",+ GZIP.len()+ )?;+ if method == "GET" {+ stream.write_all(GZIP)?;+ }+ Ok(())+ });+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ let cancel = FetchCancelHandle::new();+ let request = Request::new(method, &url, None, crate::RequestHeaders::default())?;+ let mut response = client+ .fetch_raw_stream_with_cancel(request, cancel.clone())+ .await?;+ while let Some(chunk) = response.next_chunk().await {+ assert!(chunk.is_empty(), "decoded payload must remain empty");+ }+ response.finish().await?;+ assert_eq!(+ cancel.response_timing().body_bytes_received(),+ method == "GET"+ );+ server.join().expect("server thread")?;+ }+ Ok(())+}++#[tokio::test]+async fn native_response_timing_separates_first_interim_and_final_headers() -> Result<()> {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/interim", listener.local_addr()?);+ let (release_final, ready) = std_mpsc::sync_channel(1);+ let server = thread::spawn(move || -> std::io::Result<()> {+ let (mut stream, _) = listener.accept()?;+ let mut request = Vec::new();+ let mut byte = [0];+ while !request.ends_with(b"\r\n\r\n") {+ stream.read_exact(&mut byte)?;+ request.extend_from_slice(&byte);+ }+ stream.write_all(b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 103 Early Hints\r\nLink: </a>;rel=preload\r\n\r\n")?;+ ready.recv().expect("consumer releases final headers");+ stream+ .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody")?;+ Ok(())+ });+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ let cancel = FetchCancelHandle::new();+ let fetch = client.fetch_raw_stream_with_cancel(Request::get(&url)?, cancel.clone());+ let release = async {+ let deadline = Instant::now() + Duration::from_secs(5);+ while cancel+ .response_timing()+ .first_interim_response_started_at()+ .is_none()+ {+ assert!(Instant::now() < deadline, "interim status was not observed");+ tokio::time::sleep(Duration::from_millis(1)).await;+ }+ assert!(cancel.response_timing().response_started_at().is_none());+ release_final.send(()).unwrap();+ };+ let (response, ()) = tokio::join!(fetch, release);+ let mut response = response?;+ while response.next_chunk().await.is_some() {}+ response.finish().await?;+ let timing = cancel.response_timing();+ assert!(+ timing.request_started_at().unwrap() <= timing.first_interim_response_started_at().unwrap()+ );+ assert!(+ timing.first_interim_response_started_at().unwrap()+ <= timing.response_started_at().unwrap()+ );+ assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());+ server.join().expect("server thread")?;+ Ok(())+}+#[tokio::test]async fn raw_response_timing_is_native_even_when_the_consumer_does_not_poll() -> Result<()> {let listener = std::net::TcpListener::bind("127.0.0.1:0")?;