Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): isolate HTML timing for each transfer attempt
ldm087efd22bc61b
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rsindex fcecd3949..b2dd3ad28 100644--- a/moli-fetch/src/blocking/collectors.rs+++ b/moli-fetch/src/blocking/collectors.rs@@ -369,6 +369,7 @@ impl StreamingResponseCollector {redirect_chain: Vec<RedirectInfo>,network_request_extra_info: Option<NetworkRequestExtraInfo>,cache_plan: Option<StreamingCachePlan>,+ response_progress: ResponseProgress,) {self.begin_request(max_response_size,@@ -379,6 +380,7 @@ impl StreamingResponseCollector {redirect_chain,None,);+ self.response_progress = response_progress;self.network_request_extra_info = network_request_extra_info;self.cache_plan = cache_plan;}diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex d13cab77a..d7f609973 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -1012,6 +1012,7 @@ impl RuntimeOwner {anyhow::Error,),> {+ job.response_progress = job.cancel_handle.begin_response_progress();if let Err(error) = job.request.validate_request_mode_for_url(&job.current_url) {return Err((Box::new(job), None, error));}@@ -1138,6 +1139,7 @@ impl RuntimeOwner {job.request.redirect_chain.clone(),request_extra_info.clone(),cache_plan,+ job.response_progress.clone(),);collector.set_client_hint_response_policy(prepared_request.response_policy);@@ -3790,6 +3792,9 @@ fn request_panics_for_testing(request: &Request) -> bool {})}+#[cfg(test)]+mod response_timing_tests;+#[cfg(test)]mod tests {use super::*;diff --git a/moli-fetch/src/runtime/response_timing_tests.rs b/moli-fetch/src/runtime/response_timing_tests.rsnew file mode 100644index 000000000..176b684dc--- /dev/null+++ b/moli-fetch/src/runtime/response_timing_tests.rs@@ -0,0 +1,78 @@+use super::*;+use moli_cookie_jar::new_shared_browser_cookie_store;+use std::{io::Write, sync::mpsc as std_mpsc};++fn submit_html_request(+ config: &FetchConfig,+ request: Request,+) -> (+ FetchRuntimeOwner,+ FetchCancelHandle,+ PendingStreamingHtmlResponse,+) {+ let owner = FetchRuntimeOwner::new(config, new_shared_browser_cookie_store());+ let pending = owner.handle().submit_html_stream(request).unwrap();+ let cancel = pending.cancel_handle.clone();+ (owner, cancel, pending)+}++#[tokio::test]+async fn html_redirect_timing_belongs_to_the_final_transfer() -> Result<()> {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/redirect", listener.local_addr()?);+ let (release_redirect, ready) = std_mpsc::sync_channel(1);+ let server = thread::spawn(move || -> std::io::Result<()> {+ for path in ["/redirect", "/final"] {+ let (mut stream, _) = listener.accept()?;+ stream.set_read_timeout(Some(Duration::from_secs(5)))?;+ let mut request = Vec::new();+ let mut byte = [0];+ while !request.ends_with(b"\r\n\r\n") {+ stream.read_exact(&mut byte)?;+ request.extend_from_slice(&byte);+ }+ assert!(request.starts_with(format!("GET {path} ").as_bytes()));+ if path == "/redirect" {+ stream.write_all(+ b"HTTP/1.1 103 Early Hints\r\n\r\n\+ HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 8\r\nConnection: close\r\n\r\nold!",+ )?;+ ready.recv_timeout(Duration::from_secs(5)).unwrap();+ stream.write_all(b"tail")?;+ } else {+ stream.write_all(+ b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",+ )?;+ }+ }+ Ok(())+ });+ let (_owner, cancel, pending) =+ submit_html_request(&FetchConfig::default(), Request::get(&url)?);+ let deadline = Instant::now() + Duration::from_secs(5);+ while !cancel.response_timing().body_bytes_received() {+ anyhow::ensure!(Instant::now() < deadline, "redirect body was not observed");+ tokio::time::sleep(Duration::from_millis(1)).await;+ }+ let redirect_timing = cancel.response_timing();+ assert!(+ redirect_timing+ .first_interim_response_started_at()+ .is_some()+ );+ assert!(redirect_timing.response_started_at().is_some());+ let redirect_finished_after = Instant::now();+ release_redirect.send(())?;++ let mut response = pending.into_response().await?;+ assert_eq!(response.final_url.path(), "/final");+ assert!(response.next_chunk().await.is_none());+ response.finish().await?;+ let timing = cancel.response_timing();+ assert!(timing.request_started_at().unwrap() >= redirect_finished_after);+ assert!(timing.response_started_at().unwrap() >= redirect_finished_after);+ assert!(timing.first_interim_response_started_at().is_none());+ assert!(!timing.body_bytes_received());+ server.join().expect("server thread")?;+ Ok(())+}