Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

feat(fetch): preserve native response timing observations

ldm0a8def2344e7e
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rs
index ddaad5036..53e2fd6f5 100644
--- a/moli-fetch/src/blocking/collectors.rs
+++ b/moli-fetch/src/blocking/collectors.rs
@@ -1183,6 +1183,9 @@ impl Handler for RawStreamingResponseCollector {
self.response_bytes_received = 0;
self.response_too_large = false;
self.cookie_set_reports.clear();
+ if self.status >= 200 {
+ self.response_progress.mark_response_started();
+ }
return true;
}
@@ -1344,4 +1347,30 @@ mod tests {
assert!(!cancel_handle.response_completion_is_committed());
}
}
+ #[test]
+ fn raw_streaming_timing_starts_at_final_status_before_headers_finish() {
+ let handle = FetchCancelHandle::new();
+ let (mut collector, _body) = raw_streaming_collector_with_headers(handle.clone(), &[]);
+ collector.begin_request(
+ None,
+ Url::parse("https://example.test/empty").unwrap(),
+ NetworkCookieRequestContext::top_level_navigation("GET"),
+ None,
+ true,
+ Vec::new(),
+ handle.begin_response_progress(),
+ );
+ assert!(handle.response_timing().response_started_at().is_none());
+ assert!(collector.header(b"HTTP/1.1 103 Early Hints\r\n"));
+ assert!(collector.header(b"\r\n"));
+ assert!(handle.response_timing().response_started_at().is_none());
+ assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
+ let started = handle.response_timing().response_started_at().unwrap();
+ assert!(handle.response_timing().response_end_at().is_none());
+ assert!(collector.header(b"Content-Length: 0\r\n"));
+ assert!(collector.header(b"\r\n"));
+ let complete = handle.response_timing();
+ assert_eq!(complete.response_started_at(), Some(started));
+ assert!(complete.response_end_at().unwrap() >= started);
+ }
}
diff --git a/moli-fetch/src/cancellation.rs b/moli-fetch/src/cancellation.rs
index 280d30919..f0cf0b752 100644
--- a/moli-fetch/src/cancellation.rs
+++ b/moli-fetch/src/cancellation.rs
@@ -1,6 +1,9 @@
-use std::sync::{
- Arc,
- atomic::{AtomicU8, Ordering},
+use std::{
+ sync::{
+ Arc,
+ atomic::{AtomicU8, Ordering},
+ },
+ time::Instant,
};
use parking_lot::Mutex;
@@ -15,10 +18,16 @@ const RESPONSE_CANCELLED: u8 = 2;
#[derive(Debug, Clone, Default)]
pub(crate) struct ResponseProgress {
state: Arc<AtomicU8>,
+ timing: Arc<Mutex<FetchResponseTiming>>,
}
impl ResponseProgress {
pub(crate) fn commit(&self) {
+ self.finish_timing();
+ self.commit_state();
+ }
+
+ fn commit_state(&self) {
let _ = self.state.compare_exchange(
RESPONSE_PENDING,
RESPONSE_COMMITTED,
@@ -27,6 +36,26 @@ impl ResponseProgress {
);
}
+ pub(crate) fn mark_response_started(&self) {
+ self.timing
+ .lock()
+ .response_started_at
+ .get_or_insert_with(Instant::now);
+ }
+
+ fn finish_timing(&self) {
+ self.timing
+ .lock()
+ .response_end_at
+ .get_or_insert_with(Instant::now);
+ }
+
+ pub(crate) fn mark_cached_response_available(&self) -> CachedResponseBodyTiming {
+ self.mark_response_started();
+ self.commit_state();
+ CachedResponseBodyTiming(self.clone())
+ }
+
fn is_committed(&self) -> bool {
self.state.load(Ordering::Acquire) == RESPONSE_COMMITTED
}
@@ -41,6 +70,30 @@ impl ResponseProgress {
}
}
+/// Native response boundaries shared with observers of the selected transfer.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct FetchResponseTiming {
+ response_started_at: Option<Instant>,
+ response_end_at: Option<Instant>,
+}
+
+impl FetchResponseTiming {
+ pub fn response_started_at(self) -> Option<Instant> {
+ self.response_started_at
+ }
+ pub fn response_end_at(self) -> Option<Instant> {
+ self.response_end_at
+ }
+}
+
+pub(crate) struct CachedResponseBodyTiming(ResponseProgress);
+
+impl CachedResponseBodyTiming {
+ pub(crate) fn finish(self) {
+ self.0.finish_timing();
+ }
+}
+
#[derive(Debug, Default)]
struct FetchLifecycleState {
cancellation: CancellationToken,
@@ -127,6 +180,30 @@ impl FetchCancelHandle {
progress
}
+ pub fn response_timing(&self) -> FetchResponseTiming {
+ *self.response_progress().timing.lock()
+ }
+
+ #[cfg(test)]
+ fn mark_response_started(&self) {
+ self.response_progress().mark_response_started();
+ }
+
+ #[cfg(test)]
+ fn mark_declared_response_body_complete(&self) {
+ self.response_progress().commit();
+ }
+
+ #[cfg(test)]
+ fn mark_cached_response_available(&self) -> CachedResponseBodyTiming {
+ self.response_progress().mark_cached_response_available()
+ }
+
+ #[cfg(test)]
+ fn mark_response_terminal(&self) {
+ self.response_progress().commit();
+ }
+
pub(crate) fn response_progress(&self) -> ResponseProgress {
self.state.response_progress.lock().clone()
}
@@ -135,6 +212,7 @@ impl FetchCancelHandle {
#[cfg(test)]
mod tests {
use super::FetchCancelHandle;
+ use std::time::Instant;
#[test]
fn response_completion_commit_is_shared_and_resettable() {
@@ -408,4 +486,83 @@ mod tests {
assert!(!child.response_completion_is_committed());
}
}
+ #[test]
+ fn response_timing_retains_native_boundaries_across_late_observation() {
+ let handle = FetchCancelHandle::new();
+ let before_headers = Instant::now();
+ handle.mark_response_started();
+ let headers = handle.response_timing().response_started_at().unwrap();
+ assert!(headers >= before_headers && headers <= Instant::now());
+ assert!(handle.response_timing().response_end_at().is_none());
+ let before_body = Instant::now();
+ handle.mark_declared_response_body_complete();
+ let complete = handle.response_timing();
+ assert!(complete.response_end_at().unwrap() >= before_body);
+ assert!(complete.response_end_at().unwrap() <= Instant::now());
+ handle.mark_response_started();
+ handle.mark_response_terminal();
+ assert_eq!(handle.response_timing(), complete);
+ assert_eq!(complete.response_started_at(), Some(headers));
+ }
+
+ #[test]
+ fn response_timing_tracks_only_the_selected_child_and_generation() {
+ let parent = FetchCancelHandle::new();
+ let discarded = parent.child_for_subrequest();
+ discarded.mark_response_started();
+ discarded.mark_response_terminal();
+ assert!(parent.response_timing().response_started_at().is_none());
+ for complete_before_selection in [false, true] {
+ let accepted = parent.child_for_subrequest();
+ accepted.mark_response_started();
+ if complete_before_selection {
+ accepted.mark_response_terminal();
+ }
+ parent.adopt_response_progress(&accepted);
+ assert_eq!(parent.response_timing(), accepted.response_timing());
+ accepted.mark_response_terminal();
+ let complete = parent.response_timing();
+ assert!(complete.response_started_at().is_some());
+ assert!(complete.response_end_at().is_some());
+ discarded.mark_response_terminal();
+ assert_eq!(parent.response_timing(), complete);
+ parent.begin_response_progress();
+ accepted.mark_response_terminal();
+ assert!(parent.response_timing().response_started_at().is_none());
+ assert!(parent.response_timing().response_end_at().is_none());
+ }
+ }
+
+ #[test]
+ fn failed_response_timing_does_not_invent_a_header_boundary() {
+ let handle = FetchCancelHandle::new();
+ handle.mark_response_terminal();
+ let timing = handle.response_timing();
+ assert!(timing.response_started_at().is_none());
+ assert!(timing.response_end_at().is_some());
+ }
+
+ #[test]
+ fn cached_response_body_timing_is_separate_and_cannot_finish_a_new_generation() {
+ for reset_before_body_end in [false, true] {
+ let parent = FetchCancelHandle::new();
+ let response = parent.child_for_subrequest();
+ let recorder = response.mark_cached_response_available();
+ parent.adopt_response_progress(&response);
+ assert!(parent.response_completion_is_committed());
+ assert!(parent.response_timing().response_started_at().is_some());
+ assert!(parent.response_timing().response_end_at().is_none());
+ if reset_before_body_end {
+ parent.begin_response_progress();
+ }
+ recorder.finish();
+ if reset_before_body_end {
+ assert!(parent.response_timing().response_started_at().is_none());
+ assert!(parent.response_timing().response_end_at().is_none());
+ } else {
+ let timing = parent.response_timing();
+ assert!(timing.response_end_at().unwrap() >= timing.response_started_at().unwrap());
+ }
+ }
+ }
}
diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs
index 5060b1195..7317d56d6 100644
--- a/moli-fetch/src/lib.rs
+++ b/moli-fetch/src/lib.rs
@@ -38,7 +38,7 @@ pub use blocking::{
clear_http_cache_root_for_origin, cookie_header_for_request, http_cache_stats,
observe_cookie_access_report_for_request, trim_http_cache,
};
-pub use cancellation::FetchCancelHandle;
+pub use cancellation::{FetchCancelHandle, FetchResponseTiming};
pub use client::{FetchClient, FetchClientHandle};
pub use config::FetchConfig;
pub use cors::validate_cors_response_for_origin;
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index b18a82ba1..66879527b 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -3069,7 +3069,6 @@ fn complete_cached_streaming_raw_job(
cached: CachedStreamingResponseLookup,
request_cookie_report: Option<StoredCookieQueryReport>,
) {
- job.response_progress.commit();
let redirected = !job.request.redirect_chain.is_empty();
let redirect_chain = job.request.redirect_chain.clone();
let CachedStreamingResponseLookup {
@@ -3083,6 +3082,7 @@ fn complete_cached_streaming_raw_job(
let final_url = match Url::parse(&final_url) {
Ok(final_url) => final_url,
Err(error) => {
+ job.response_progress.commit();
if let Some(started_tx) = job.started_tx {
let _ = started_tx.send(Err(anyhow!(
"failed to parse cached raw response final url: {error}"
@@ -3095,6 +3095,8 @@ fn complete_cached_streaming_raw_job(
}
};
+ let response_timing = job.response_progress.mark_cached_response_available();
+
if let Some(started_tx) = job.started_tx {
let _ = started_tx.send(Ok(StreamingHtmlResponseStart {
status_text: metadata.status_text,
@@ -3117,9 +3119,11 @@ fn complete_cached_streaming_raw_job(
// instead of rebuilding a full RawResponse body in memory.
thread::spawn(move || {
let completion = send_cached_raw_body_chunks(&mut body, &body_tx);
+ response_timing.finish();
let _ = completion_tx.send(completion);
});
} else {
+ response_timing.finish();
let _ = job.completion_tx.send(Ok(()));
}
}
diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs
index bdff2b222..5183131f0 100644
--- a/moli-fetch/src/tests/mod.rs
+++ b/moli-fetch/src/tests/mod.rs
@@ -6,6 +6,7 @@ mod header_eof;
mod mixed_transport;
mod request_security;
mod response_progress;
+mod response_timing;
mod support;
mod tls_credentials;
mod websocket_transport;
diff --git a/moli-fetch/src/tests/response_timing.rs b/moli-fetch/src/tests/response_timing.rs
new file mode 100644
index 000000000..31765f1d0
--- /dev/null
+++ b/moli-fetch/src/tests/response_timing.rs
@@ -0,0 +1,55 @@
+use super::*;
+use std::io::Write;
+
+#[tokio::test]
+async fn raw_response_timing_is_native_even_when_the_consumer_does_not_poll() -> Result<()> {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/timed", listener.local_addr()?);
+ let (release_body, ready) = std_mpsc::sync_channel(1);
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ let (mut stream, _) = listener.accept()?;
+ let mut request = Vec::new();
+ let mut byte = [0];
+ while !request.ends_with(b"\r\n\r\n") {
+ stream.read_exact(&mut byte)?;
+ request.extend_from_slice(&byte);
+ }
+ stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n")?;
+ ready.recv().expect("consumer releases the body");
+ stream.write_all(b"body")?;
+ Ok(())
+ });
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ let cancel = FetchCancelHandle::new();
+ let mut response = client
+ .fetch_raw_stream_with_cancel(Request::get(&url)?, cancel.clone())
+ .await?;
+ let headers = cancel.response_timing();
+ assert!(headers.response_started_at().is_some());
+ assert!(headers.response_end_at().is_none());
+ release_body.send(())?;
+ // Intentionally keep this Tokio thread occupied: the curl semantic owner
+ // must capture completion without the response consumer being polled.
+ let deadline = Instant::now() + Duration::from_secs(5);
+ while cancel.response_timing().response_end_at().is_none() {
+ assert!(
+ Instant::now() < deadline,
+ "native transfer did not complete"
+ );
+ thread::sleep(Duration::from_millis(1));
+ }
+ let native = cancel.response_timing();
+ let observed_at = Instant::now();
+ assert_eq!(native.response_started_at(), headers.response_started_at());
+ assert!(native.response_end_at().unwrap() >= native.response_started_at().unwrap());
+ assert!(native.response_end_at().unwrap() <= observed_at);
+ let mut body = Vec::new();
+ while let Some(chunk) = response.next_chunk().await {
+ body.extend_from_slice(&chunk);
+ }
+ response.finish().await?;
+ assert_eq!(body, b"body");
+ assert_eq!(cancel.response_timing(), native);
+ server.join().expect("server thread")?;
+ Ok(())
+}