Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat(fetch): preserve native response timing observations
ldm0a8def2344e7e
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rsindex ddaad5036..53e2fd6f5 100644--- a/moli-fetch/src/blocking/collectors.rs+++ b/moli-fetch/src/blocking/collectors.rs@@ -1183,6 +1183,9 @@ impl Handler for RawStreamingResponseCollector {self.response_bytes_received = 0;self.response_too_large = false;self.cookie_set_reports.clear();+ if self.status >= 200 {+ self.response_progress.mark_response_started();+ }return true;}@@ -1344,4 +1347,30 @@ mod tests {assert!(!cancel_handle.response_completion_is_committed());}}+ #[test]+ fn raw_streaming_timing_starts_at_final_status_before_headers_finish() {+ let handle = FetchCancelHandle::new();+ let (mut collector, _body) = raw_streaming_collector_with_headers(handle.clone(), &[]);+ collector.begin_request(+ None,+ Url::parse("https://example.test/empty").unwrap(),+ NetworkCookieRequestContext::top_level_navigation("GET"),+ None,+ true,+ Vec::new(),+ handle.begin_response_progress(),+ );+ assert!(handle.response_timing().response_started_at().is_none());+ assert!(collector.header(b"HTTP/1.1 103 Early Hints\r\n"));+ assert!(collector.header(b"\r\n"));+ assert!(handle.response_timing().response_started_at().is_none());+ assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));+ let started = handle.response_timing().response_started_at().unwrap();+ assert!(handle.response_timing().response_end_at().is_none());+ assert!(collector.header(b"Content-Length: 0\r\n"));+ assert!(collector.header(b"\r\n"));+ let complete = handle.response_timing();+ assert_eq!(complete.response_started_at(), Some(started));+ assert!(complete.response_end_at().unwrap() >= started);+ }}diff --git a/moli-fetch/src/cancellation.rs b/moli-fetch/src/cancellation.rsindex 280d30919..f0cf0b752 100644--- a/moli-fetch/src/cancellation.rs+++ b/moli-fetch/src/cancellation.rs@@ -1,6 +1,9 @@-use std::sync::{- Arc,- atomic::{AtomicU8, Ordering},+use std::{+ sync::{+ Arc,+ atomic::{AtomicU8, Ordering},+ },+ time::Instant,};use parking_lot::Mutex;@@ -15,10 +18,16 @@ const RESPONSE_CANCELLED: u8 = 2;#[derive(Debug, Clone, Default)]pub(crate) struct ResponseProgress {state: Arc<AtomicU8>,+ timing: Arc<Mutex<FetchResponseTiming>>,}impl ResponseProgress {pub(crate) fn commit(&self) {+ self.finish_timing();+ self.commit_state();+ }++ fn commit_state(&self) {let _ = self.state.compare_exchange(RESPONSE_PENDING,RESPONSE_COMMITTED,@@ -27,6 +36,26 @@ impl ResponseProgress {);}+ pub(crate) fn mark_response_started(&self) {+ self.timing+ .lock()+ .response_started_at+ .get_or_insert_with(Instant::now);+ }++ fn finish_timing(&self) {+ self.timing+ .lock()+ .response_end_at+ .get_or_insert_with(Instant::now);+ }++ pub(crate) fn mark_cached_response_available(&self) -> CachedResponseBodyTiming {+ self.mark_response_started();+ self.commit_state();+ CachedResponseBodyTiming(self.clone())+ }+fn is_committed(&self) -> bool {self.state.load(Ordering::Acquire) == RESPONSE_COMMITTED}@@ -41,6 +70,30 @@ impl ResponseProgress {}}+/// Native response boundaries shared with observers of the selected transfer.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct FetchResponseTiming {+ response_started_at: Option<Instant>,+ response_end_at: Option<Instant>,+}++impl FetchResponseTiming {+ pub fn response_started_at(self) -> Option<Instant> {+ self.response_started_at+ }+ pub fn response_end_at(self) -> Option<Instant> {+ self.response_end_at+ }+}++pub(crate) struct CachedResponseBodyTiming(ResponseProgress);++impl CachedResponseBodyTiming {+ pub(crate) fn finish(self) {+ self.0.finish_timing();+ }+}+#[derive(Debug, Default)]struct FetchLifecycleState {cancellation: CancellationToken,@@ -127,6 +180,30 @@ impl FetchCancelHandle {progress}+ pub fn response_timing(&self) -> FetchResponseTiming {+ *self.response_progress().timing.lock()+ }++ #[cfg(test)]+ fn mark_response_started(&self) {+ self.response_progress().mark_response_started();+ }++ #[cfg(test)]+ fn mark_declared_response_body_complete(&self) {+ self.response_progress().commit();+ }++ #[cfg(test)]+ fn mark_cached_response_available(&self) -> CachedResponseBodyTiming {+ self.response_progress().mark_cached_response_available()+ }++ #[cfg(test)]+ fn mark_response_terminal(&self) {+ self.response_progress().commit();+ }+pub(crate) fn response_progress(&self) -> ResponseProgress {self.state.response_progress.lock().clone()}@@ -135,6 +212,7 @@ impl FetchCancelHandle {#[cfg(test)]mod tests {use super::FetchCancelHandle;+ use std::time::Instant;#[test]fn response_completion_commit_is_shared_and_resettable() {@@ -408,4 +486,83 @@ mod tests {assert!(!child.response_completion_is_committed());}}+ #[test]+ fn response_timing_retains_native_boundaries_across_late_observation() {+ let handle = FetchCancelHandle::new();+ let before_headers = Instant::now();+ handle.mark_response_started();+ let headers = handle.response_timing().response_started_at().unwrap();+ assert!(headers >= before_headers && headers <= Instant::now());+ assert!(handle.response_timing().response_end_at().is_none());+ let before_body = Instant::now();+ handle.mark_declared_response_body_complete();+ let complete = handle.response_timing();+ assert!(complete.response_end_at().unwrap() >= before_body);+ assert!(complete.response_end_at().unwrap() <= Instant::now());+ handle.mark_response_started();+ handle.mark_response_terminal();+ assert_eq!(handle.response_timing(), complete);+ assert_eq!(complete.response_started_at(), Some(headers));+ }++ #[test]+ fn response_timing_tracks_only_the_selected_child_and_generation() {+ let parent = FetchCancelHandle::new();+ let discarded = parent.child_for_subrequest();+ discarded.mark_response_started();+ discarded.mark_response_terminal();+ assert!(parent.response_timing().response_started_at().is_none());+ for complete_before_selection in [false, true] {+ let accepted = parent.child_for_subrequest();+ accepted.mark_response_started();+ if complete_before_selection {+ accepted.mark_response_terminal();+ }+ parent.adopt_response_progress(&accepted);+ assert_eq!(parent.response_timing(), accepted.response_timing());+ accepted.mark_response_terminal();+ let complete = parent.response_timing();+ assert!(complete.response_started_at().is_some());+ assert!(complete.response_end_at().is_some());+ discarded.mark_response_terminal();+ assert_eq!(parent.response_timing(), complete);+ parent.begin_response_progress();+ accepted.mark_response_terminal();+ assert!(parent.response_timing().response_started_at().is_none());+ assert!(parent.response_timing().response_end_at().is_none());+ }+ }++ #[test]+ fn failed_response_timing_does_not_invent_a_header_boundary() {+ let handle = FetchCancelHandle::new();+ handle.mark_response_terminal();+ let timing = handle.response_timing();+ assert!(timing.response_started_at().is_none());+ assert!(timing.response_end_at().is_some());+ }++ #[test]+ fn cached_response_body_timing_is_separate_and_cannot_finish_a_new_generation() {+ for reset_before_body_end in [false, true] {+ let parent = FetchCancelHandle::new();+ let response = parent.child_for_subrequest();+ let recorder = response.mark_cached_response_available();+ parent.adopt_response_progress(&response);+ assert!(parent.response_completion_is_committed());+ assert!(parent.response_timing().response_started_at().is_some());+ assert!(parent.response_timing().response_end_at().is_none());+ if reset_before_body_end {+ parent.begin_response_progress();+ }+ recorder.finish();+ if reset_before_body_end {+ assert!(parent.response_timing().response_started_at().is_none());+ assert!(parent.response_timing().response_end_at().is_none());+ } else {+ let timing = parent.response_timing();+ assert!(timing.response_end_at().unwrap() >= timing.response_started_at().unwrap());+ }+ }+ }}diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rsindex 5060b1195..7317d56d6 100644--- a/moli-fetch/src/lib.rs+++ b/moli-fetch/src/lib.rs@@ -38,7 +38,7 @@ pub use blocking::{clear_http_cache_root_for_origin, cookie_header_for_request, http_cache_stats,observe_cookie_access_report_for_request, trim_http_cache,};-pub use cancellation::FetchCancelHandle;+pub use cancellation::{FetchCancelHandle, FetchResponseTiming};pub use client::{FetchClient, FetchClientHandle};pub use config::FetchConfig;pub use cors::validate_cors_response_for_origin;diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex b18a82ba1..66879527b 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -3069,7 +3069,6 @@ fn complete_cached_streaming_raw_job(cached: CachedStreamingResponseLookup,request_cookie_report: Option<StoredCookieQueryReport>,) {- job.response_progress.commit();let redirected = !job.request.redirect_chain.is_empty();let redirect_chain = job.request.redirect_chain.clone();let CachedStreamingResponseLookup {@@ -3083,6 +3082,7 @@ fn complete_cached_streaming_raw_job(let final_url = match Url::parse(&final_url) {Ok(final_url) => final_url,Err(error) => {+ job.response_progress.commit();if let Some(started_tx) = job.started_tx {let _ = started_tx.send(Err(anyhow!("failed to parse cached raw response final url: {error}"@@ -3095,6 +3095,8 @@ fn complete_cached_streaming_raw_job(}};+ let response_timing = job.response_progress.mark_cached_response_available();+if let Some(started_tx) = job.started_tx {let _ = started_tx.send(Ok(StreamingHtmlResponseStart {status_text: metadata.status_text,@@ -3117,9 +3119,11 @@ fn complete_cached_streaming_raw_job(// instead of rebuilding a full RawResponse body in memory.thread::spawn(move || {let completion = send_cached_raw_body_chunks(&mut body, &body_tx);+ response_timing.finish();let _ = completion_tx.send(completion);});} else {+ response_timing.finish();let _ = job.completion_tx.send(Ok(()));}}diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex bdff2b222..5183131f0 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -6,6 +6,7 @@ mod header_eof;mod mixed_transport;mod request_security;mod response_progress;+mod response_timing;mod support;mod tls_credentials;mod websocket_transport;diff --git a/moli-fetch/src/tests/response_timing.rs b/moli-fetch/src/tests/response_timing.rsnew file mode 100644index 000000000..31765f1d0--- /dev/null+++ b/moli-fetch/src/tests/response_timing.rs@@ -0,0 +1,55 @@+use super::*;+use std::io::Write;++#[tokio::test]+async fn raw_response_timing_is_native_even_when_the_consumer_does_not_poll() -> Result<()> {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/timed", listener.local_addr()?);+ let (release_body, ready) = std_mpsc::sync_channel(1);+ let server = thread::spawn(move || -> std::io::Result<()> {+ let (mut stream, _) = listener.accept()?;+ let mut request = Vec::new();+ let mut byte = [0];+ while !request.ends_with(b"\r\n\r\n") {+ stream.read_exact(&mut byte)?;+ request.extend_from_slice(&byte);+ }+ stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n")?;+ ready.recv().expect("consumer releases the body");+ stream.write_all(b"body")?;+ Ok(())+ });+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ let cancel = FetchCancelHandle::new();+ let mut response = client+ .fetch_raw_stream_with_cancel(Request::get(&url)?, cancel.clone())+ .await?;+ let headers = cancel.response_timing();+ assert!(headers.response_started_at().is_some());+ assert!(headers.response_end_at().is_none());+ release_body.send(())?;+ // Intentionally keep this Tokio thread occupied: the curl semantic owner+ // must capture completion without the response consumer being polled.+ let deadline = Instant::now() + Duration::from_secs(5);+ while cancel.response_timing().response_end_at().is_none() {+ assert!(+ Instant::now() < deadline,+ "native transfer did not complete"+ );+ thread::sleep(Duration::from_millis(1));+ }+ let native = cancel.response_timing();+ let observed_at = Instant::now();+ assert_eq!(native.response_started_at(), headers.response_started_at());+ assert!(native.response_end_at().unwrap() >= native.response_started_at().unwrap());+ assert!(native.response_end_at().unwrap() <= observed_at);+ let mut body = Vec::new();+ while let Some(chunk) = response.next_chunk().await {+ body.extend_from_slice(&chunk);+ }+ response.finish().await?;+ assert_eq!(body, b"body");+ assert_eq!(cancel.response_timing(), native);+ server.join().expect("server thread")?;+ Ok(())+}