Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): record native terminal timing for HTML streams
ldm0703765dcd754
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rsindex b2dd3ad28..acc40d2f6 100644--- a/moli-fetch/src/blocking/collectors.rs+++ b/moli-fetch/src/blocking/collectors.rs@@ -186,6 +186,7 @@ pub struct StreamingResponseCollector {max_response_size: Option<usize>,response_too_large: bool,response_bytes_received: usize,+ declared_identity_body_length: Option<usize>,cache_body_writer: Option<HttpCacheBodyWriter>,cache_plan: Option<StreamingCachePlan>,started: bool,@@ -301,6 +302,7 @@ impl StreamingResponseCollector {max_response_size: None,response_too_large: false,response_bytes_received: 0,+ declared_identity_body_length: None,cache_body_writer: None,cache_plan: None,started: false,@@ -342,6 +344,7 @@ impl StreamingResponseCollector {self.max_response_size = max_response_size;self.response_too_large = false;self.response_bytes_received = 0;+ self.declared_identity_body_length = None;self.cache_body_writer = cache_body_writer;self.cache_plan = None;self.started = false;@@ -503,6 +506,7 @@ impl StreamingResponseCollector {self.client_hint_restart_requested = true;return false;}+ self.declared_identity_body_length = identity_encoded_content_length(&self.headers);// Create the cache writer only after final response headers prove the// response is cacheable.self.maybe_create_cache_body_writer();@@ -548,6 +552,11 @@ impl StreamingResponseCollector {{return;}+ if self.declared_identity_body_length == Some(0)+ || response_body_ends_at_headers(self.status)+ {+ self.response_progress.commit();+ }self.started = true;if let Some(start_tx) = self.start_tx.take() {let _ = start_tx.send(Ok(StreamingHtmlResponseStart {@@ -1116,6 +1125,10 @@ impl Handler for StreamingResponseCollector {}self.response_bytes_received += data.len();+ if self.started && self.declared_identity_body_length == Some(self.response_bytes_received)+ {+ self.response_progress.commit();+ }self.write_cache_body_bytes(data);self.stream_body_bytes(data);if self.cancel_handle.is_cancelled() {@@ -1139,6 +1152,7 @@ impl Handler for StreamingResponseCollector {self.status_text = Some(parsed.status_text);self.negotiated_http_version = parsed.version;self.response_bytes_received = 0;+ self.declared_identity_body_length = None;self.response_too_large = false;self.cookie_set_reports.clear();if self.status < 200 {diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex d7f609973..5553b9116 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -1928,6 +1928,7 @@ impl RuntimeOwner {return;}+ job.response_progress.commit();let cache_body_writer = {let streaming = easy.get_mut()@@ -2828,6 +2829,11 @@ impl FetchTransferHandler {impl Handler for FetchTransferHandler {fn write(&mut self, data: &[u8]) -> std::result::Result<usize, WriteError> {+ if !data.is_empty()+ && let Some(timing) = self.response_progress()+ {+ timing.mark_response_body_received();+ }match &mut self.response {FetchResponseCollector::Buffered(collector) => collector.write(data),FetchResponseCollector::Streaming(collector) => collector.write(data),@@ -2981,6 +2987,7 @@ fn complete_cached_streaming_html_job(let final_url = match Url::parse(&final_url) {Ok(final_url) => final_url,Err(error) => {+ job.response_progress.commit();if let Some(started_tx) = job.started_tx {let _ = started_tx.send(Err(anyhow!("failed to parse cached response final url: {error}"@@ -2993,6 +3000,7 @@ fn complete_cached_streaming_html_job(}};+ let response_timing = job.response_progress.mark_cached_response_available();if let Some(started_tx) = job.started_tx {let _ = started_tx.send(Ok(StreamingHtmlResponseStart {status_text: metadata.status_text,@@ -3014,6 +3022,7 @@ fn complete_cached_streaming_html_job(} else {Ok(())};+ response_timing.finish();let _ = job.completion_tx.send(completion);}@@ -3698,6 +3707,7 @@ fn fail_streaming_job_with_easy(&job.request.redirect_chain,error,);+ job.response_progress.commit();if let Some(easy) = easy.as_mut()&& let Some(streaming) = easy.get_mut().streaming_mut(){diff --git a/moli-fetch/src/runtime/response_timing_tests.rs b/moli-fetch/src/runtime/response_timing_tests.rsindex 176b684dc..ac404bd75 100644--- a/moli-fetch/src/runtime/response_timing_tests.rs+++ b/moli-fetch/src/runtime/response_timing_tests.rs@@ -76,3 +76,255 @@ async fn html_redirect_timing_belongs_to_the_final_transfer() -> Result<()> {server.join().expect("server thread")?;Ok(())}++#[tokio::test]+async fn html_response_timing_finishes_without_consumer_polling() -> Result<()> {+ for (framing, body) in [+ ("Content-Length: 4", b"body".as_slice()),+ (+ "Transfer-Encoding: chunked",+ b"4\r\nbody\r\n0\r\n\r\n".as_slice(),+ ),+ ] {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/timed", listener.local_addr()?);+ let (release_body, ready) = std_mpsc::sync_channel(1);+ let server = thread::spawn(move || -> std::io::Result<()> {+ let (mut stream, _) = listener.accept()?;+ read_request(&mut stream)?;+ write!(+ stream,+ "HTTP/1.1 200 OK\r\n{framing}\r\nConnection: close\r\n\r\n"+ )?;+ ready.recv_timeout(Duration::from_secs(5)).unwrap();+ stream.write_all(body)?;+ Ok(())+ });+ let (_owner, cancel, pending) =+ submit_html_request(&FetchConfig::default(), Request::get(&url)?);+ let mut response = pending.into_response().await?;+ let headers = cancel.response_timing();+ assert!(headers.response_started_at().is_some());+ assert!(headers.response_end_at().is_none());+ release_body.send(())?;+ // Keep the consumer's executor occupied while native threads finish.+ let deadline = Instant::now() + Duration::from_secs(5);+ while !cancel.response_completion_is_committed() {+ anyhow::ensure!(+ Instant::now() < deadline,+ "HTML transfer did not record completion"+ );+ thread::sleep(Duration::from_millis(1));+ }+ let native = cancel.response_timing();+ assert!(native.body_bytes_received());+ assert!(native.response_started_at().unwrap() <= native.response_end_at().unwrap());+ assert!(cancel.response_completion_is_committed());+ assert!(!cancel.cancel_if_response_pending());+ let mut text = String::new();+ while let Some(chunk) = response.next_chunk().await {+ text.push_str(&chunk);+ }+ response.finish().await?;+ assert_eq!(text, "body");+ assert_eq!(cancel.response_timing(), native);+ server.join().expect("server thread")?;+ }+ Ok(())+}++fn read_request(stream: &mut std::net::TcpStream) -> std::io::Result<()> {+ stream.set_read_timeout(Some(Duration::from_secs(5)))?;+ let mut request = Vec::new();+ let mut byte = [0];+ while !request.ends_with(b"\r\n\r\n") {+ stream.read_exact(&mut byte)?;+ request.extend_from_slice(&byte);+ }+ Ok(())+}++#[tokio::test]+async fn html_response_timing_records_terminal_failures() -> Result<()> {+ for bytes in [+ b"".as_slice(),+ b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\npart".as_slice(),+ ] {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/failure", listener.local_addr()?);+ let server = thread::spawn(move || -> std::io::Result<()> {+ let (mut stream, _) = listener.accept()?;+ read_request(&mut stream)?;+ stream.write_all(bytes)?;+ Ok(())+ });+ let (_owner, cancel, pending) =+ submit_html_request(&FetchConfig::default(), Request::get(&url)?);+ match pending.into_response().await {+ Ok(mut response) => assert!(response.finish().await.is_err()),+ Err(_) => assert!(bytes.is_empty()),+ }+ let timing = cancel.response_timing();+ assert_eq!(timing.response_started_at().is_some(), !bytes.is_empty());+ assert!(timing.response_end_at().is_some());+ assert!(cancel.response_completion_is_committed());+ assert!(!cancel.cancel_if_response_pending());+ server.join().expect("server thread")?;+ }+ Ok(())+}++#[tokio::test]+async fn html_response_timing_finishes_bodyless_responses_at_headers() -> Result<()> {+ for (method, headers) in [+ ("GET", "HTTP/1.1 204 No Content\r\n"),+ ("GET", "HTTP/1.1 205 Reset Content\r\n"),+ ("GET", "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n"),+ ("HEAD", "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n"),+ ] {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/empty", listener.local_addr()?);+ let (release, ready) = std_mpsc::sync_channel(1);+ let server = thread::spawn(move || -> std::io::Result<()> {+ let (mut stream, _) = listener.accept()?;+ read_request(&mut stream)?;+ write!(stream, "{headers}Connection: close\r\n\r\n")?;+ ready.recv_timeout(Duration::from_secs(5)).unwrap();+ Ok(())+ });+ let request = Request::new(method, &url, None, Vec::new())?;+ let (_owner, cancel, pending) = submit_html_request(&FetchConfig::default(), request);+ let mut response = pending.into_response().await?;+ tokio::time::timeout(Duration::from_secs(5), response.finish()).await??;+ let timing = cancel.response_timing();+ assert!(timing.response_end_at().is_some(), "{method}: {headers}");+ assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());+ assert!(!timing.body_bytes_received());+ assert!(!cancel.cancel_if_response_pending());+ release.send(())?;+ server.join().expect("server thread")?;+ }+ Ok(())+}++#[tokio::test]+async fn html_response_timing_preserves_cancellation_before_body_completion() -> Result<()> {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/cancel", listener.local_addr()?);+ let (release, ready) = std_mpsc::sync_channel(1);+ let server = thread::spawn(move || -> std::io::Result<()> {+ let (mut stream, _) = listener.accept()?;+ read_request(&mut stream)?;+ stream+ .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\npart")?;+ ready.recv_timeout(Duration::from_secs(5)).unwrap();+ Ok(())+ });+ let (_owner, cancel, pending) =+ submit_html_request(&FetchConfig::default(), Request::get(&url)?);+ let mut response = pending.into_response().await?;+ assert_eq!(response.next_chunk().await.as_deref(), Some("part"));+ assert!(cancel.response_timing().response_end_at().is_none());+ assert!(cancel.cancel_if_response_pending());+ assert!(+ tokio::time::timeout(Duration::from_secs(5), response.finish())+ .await?+ .is_err()+ );+ assert!(cancel.response_timing().response_end_at().is_some());+ assert!(!cancel.response_completion_is_committed());+ release.send(())?;+ server.join().expect("server thread")?;+ Ok(())+}++#[tokio::test]+async fn html_cached_response_timing_does_not_invent_network_observations() -> Result<()> {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/cache", listener.local_addr()?);+ let cache_dir = std::env::temp_dir().join(format!(+ "moli-html-timing-{}-{}",+ std::process::id(),+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)?+ .as_nanos(),+ ));+ let server = thread::spawn(move || -> std::io::Result<()> {+ let (mut stream, _) = listener.accept()?;+ read_request(&mut stream)?;+ stream.write_all(b"HTTP/1.1 200 OK\r\nCache-Control: max-age=600\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody")?;+ Ok(())+ });+ let mut config = FetchConfig::default();+ config.set_http_cache_dir(Some(cache_dir.display().to_string()));+ let (owner, _, pending) = submit_html_request(&config, Request::get(&url)?);+ pending.into_response().await?.finish().await?;+ server.join().expect("server thread")?;++ let pending = owner.handle().submit_html_stream(Request::get(&url)?)?;+ let cancel = pending.cancel_handle.clone();+ let mut response = pending.into_response().await?;+ assert!(response.from_cache);+ let mut text = String::new();+ while let Some(chunk) = response.next_chunk().await {+ text.push_str(&chunk);+ }+ response.finish().await?;+ assert_eq!(text, "body");+ let timing = cancel.response_timing();+ assert!(timing.request_started_at().is_none());+ assert!(timing.first_interim_response_started_at().is_none());+ assert!(!timing.body_bytes_received());+ assert!(timing.response_started_at().is_some());+ assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());+ assert!(!cancel.cancel_if_response_pending());+ drop(owner);+ std::fs::remove_dir_all(cache_dir)?;+ Ok(())+}++#[tokio::test]+async fn html_declared_body_completion_precedes_delivery_of_the_last_chunk() -> Result<()> {+ for (encoding_headers, complete) in [+ (b"".as_slice(), true),+ (b"Content-Encoding: gzip\r\n".as_slice(), false),+ (b"Transfer-Encoding: chunked\r\n".as_slice(), false),+ ] {+ let cancel = FetchCancelHandle::new();+ let (start_tx, _start_rx) = oneshot::channel();+ let (body_tx, mut body_rx) = mpsc::unbounded_channel();+ let mut collector = StreamingResponseCollector::new(+ new_shared_browser_cookie_store(),+ start_tx,+ body_tx,+ cancel.clone(),+ );+ collector.begin_request(+ None,+ Url::parse("http://example.test/body")?,+ NetworkCookieRequestContext::top_level_navigation("GET"),+ None,+ true,+ Vec::new(),+ None,+ );+ assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));+ assert!(collector.header(b"Content-Length: 4\r\n"));+ if !encoding_headers.is_empty() {+ assert!(collector.header(encoding_headers));+ }+ assert!(collector.header(b"\r\n"));+ assert_eq!(collector.write(b"bo").unwrap(), 2);+ assert_eq!(body_rx.try_recv()?.as_str(), "bo");+ assert!(cancel.response_timing().response_end_at().is_none());+ assert_eq!(collector.write(b"dy").unwrap(), 2);+ assert_eq!(body_rx.try_recv()?.as_str(), "dy");+ // Decoded or chunked lengths cannot be compared with Content-Length.+ assert_eq!(+ cancel.response_timing().response_end_at().is_some(),+ complete+ );+ assert_eq!(cancel.cancel_if_response_pending(), !complete);+ }+ Ok(())+}