Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): record native terminal timing for HTML streams

ldm0703765dcd754
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rs
index b2dd3ad28..acc40d2f6 100644
--- a/moli-fetch/src/blocking/collectors.rs
+++ b/moli-fetch/src/blocking/collectors.rs
@@ -186,6 +186,7 @@ pub struct StreamingResponseCollector {
max_response_size: Option<usize>,
response_too_large: bool,
response_bytes_received: usize,
+ declared_identity_body_length: Option<usize>,
cache_body_writer: Option<HttpCacheBodyWriter>,
cache_plan: Option<StreamingCachePlan>,
started: bool,
@@ -301,6 +302,7 @@ impl StreamingResponseCollector {
max_response_size: None,
response_too_large: false,
response_bytes_received: 0,
+ declared_identity_body_length: None,
cache_body_writer: None,
cache_plan: None,
started: false,
@@ -342,6 +344,7 @@ impl StreamingResponseCollector {
self.max_response_size = max_response_size;
self.response_too_large = false;
self.response_bytes_received = 0;
+ self.declared_identity_body_length = None;
self.cache_body_writer = cache_body_writer;
self.cache_plan = None;
self.started = false;
@@ -503,6 +506,7 @@ impl StreamingResponseCollector {
self.client_hint_restart_requested = true;
return false;
}
+ self.declared_identity_body_length = identity_encoded_content_length(&self.headers);
// Create the cache writer only after final response headers prove the
// response is cacheable.
self.maybe_create_cache_body_writer();
@@ -548,6 +552,11 @@ impl StreamingResponseCollector {
{
return;
}
+ if self.declared_identity_body_length == Some(0)
+ || response_body_ends_at_headers(self.status)
+ {
+ self.response_progress.commit();
+ }
self.started = true;
if let Some(start_tx) = self.start_tx.take() {
let _ = start_tx.send(Ok(StreamingHtmlResponseStart {
@@ -1116,6 +1125,10 @@ impl Handler for StreamingResponseCollector {
}
self.response_bytes_received += data.len();
+ if self.started && self.declared_identity_body_length == Some(self.response_bytes_received)
+ {
+ self.response_progress.commit();
+ }
self.write_cache_body_bytes(data);
self.stream_body_bytes(data);
if self.cancel_handle.is_cancelled() {
@@ -1139,6 +1152,7 @@ impl Handler for StreamingResponseCollector {
self.status_text = Some(parsed.status_text);
self.negotiated_http_version = parsed.version;
self.response_bytes_received = 0;
+ self.declared_identity_body_length = None;
self.response_too_large = false;
self.cookie_set_reports.clear();
if self.status < 200 {
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index d7f609973..5553b9116 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -1928,6 +1928,7 @@ impl RuntimeOwner {
return;
}
+ job.response_progress.commit();
let cache_body_writer = {
let streaming = easy
.get_mut()
@@ -2828,6 +2829,11 @@ impl FetchTransferHandler {
impl Handler for FetchTransferHandler {
fn write(&mut self, data: &[u8]) -> std::result::Result<usize, WriteError> {
+ if !data.is_empty()
+ && let Some(timing) = self.response_progress()
+ {
+ timing.mark_response_body_received();
+ }
match &mut self.response {
FetchResponseCollector::Buffered(collector) => collector.write(data),
FetchResponseCollector::Streaming(collector) => collector.write(data),
@@ -2981,6 +2987,7 @@ fn complete_cached_streaming_html_job(
let final_url = match Url::parse(&final_url) {
Ok(final_url) => final_url,
Err(error) => {
+ job.response_progress.commit();
if let Some(started_tx) = job.started_tx {
let _ = started_tx.send(Err(anyhow!(
"failed to parse cached response final url: {error}"
@@ -2993,6 +3000,7 @@ fn complete_cached_streaming_html_job(
}
};
+ let response_timing = job.response_progress.mark_cached_response_available();
if let Some(started_tx) = job.started_tx {
let _ = started_tx.send(Ok(StreamingHtmlResponseStart {
status_text: metadata.status_text,
@@ -3014,6 +3022,7 @@ fn complete_cached_streaming_html_job(
} else {
Ok(())
};
+ response_timing.finish();
let _ = job.completion_tx.send(completion);
}
@@ -3698,6 +3707,7 @@ fn fail_streaming_job_with_easy(
&job.request.redirect_chain,
error,
);
+ job.response_progress.commit();
if let Some(easy) = easy.as_mut()
&& let Some(streaming) = easy.get_mut().streaming_mut()
{
diff --git a/moli-fetch/src/runtime/response_timing_tests.rs b/moli-fetch/src/runtime/response_timing_tests.rs
index 176b684dc..ac404bd75 100644
--- a/moli-fetch/src/runtime/response_timing_tests.rs
+++ b/moli-fetch/src/runtime/response_timing_tests.rs
@@ -76,3 +76,255 @@ async fn html_redirect_timing_belongs_to_the_final_transfer() -> Result<()> {
server.join().expect("server thread")?;
Ok(())
}
+
+#[tokio::test]
+async fn html_response_timing_finishes_without_consumer_polling() -> Result<()> {
+ for (framing, body) in [
+ ("Content-Length: 4", b"body".as_slice()),
+ (
+ "Transfer-Encoding: chunked",
+ b"4\r\nbody\r\n0\r\n\r\n".as_slice(),
+ ),
+ ] {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/timed", listener.local_addr()?);
+ let (release_body, ready) = std_mpsc::sync_channel(1);
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ let (mut stream, _) = listener.accept()?;
+ read_request(&mut stream)?;
+ write!(
+ stream,
+ "HTTP/1.1 200 OK\r\n{framing}\r\nConnection: close\r\n\r\n"
+ )?;
+ ready.recv_timeout(Duration::from_secs(5)).unwrap();
+ stream.write_all(body)?;
+ Ok(())
+ });
+ let (_owner, cancel, pending) =
+ submit_html_request(&FetchConfig::default(), Request::get(&url)?);
+ let mut response = pending.into_response().await?;
+ let headers = cancel.response_timing();
+ assert!(headers.response_started_at().is_some());
+ assert!(headers.response_end_at().is_none());
+ release_body.send(())?;
+ // Keep the consumer's executor occupied while native threads finish.
+ let deadline = Instant::now() + Duration::from_secs(5);
+ while !cancel.response_completion_is_committed() {
+ anyhow::ensure!(
+ Instant::now() < deadline,
+ "HTML transfer did not record completion"
+ );
+ thread::sleep(Duration::from_millis(1));
+ }
+ let native = cancel.response_timing();
+ assert!(native.body_bytes_received());
+ assert!(native.response_started_at().unwrap() <= native.response_end_at().unwrap());
+ assert!(cancel.response_completion_is_committed());
+ assert!(!cancel.cancel_if_response_pending());
+ let mut text = String::new();
+ while let Some(chunk) = response.next_chunk().await {
+ text.push_str(&chunk);
+ }
+ response.finish().await?;
+ assert_eq!(text, "body");
+ assert_eq!(cancel.response_timing(), native);
+ server.join().expect("server thread")?;
+ }
+ Ok(())
+}
+
+fn read_request(stream: &mut std::net::TcpStream) -> std::io::Result<()> {
+ stream.set_read_timeout(Some(Duration::from_secs(5)))?;
+ let mut request = Vec::new();
+ let mut byte = [0];
+ while !request.ends_with(b"\r\n\r\n") {
+ stream.read_exact(&mut byte)?;
+ request.extend_from_slice(&byte);
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn html_response_timing_records_terminal_failures() -> Result<()> {
+ for bytes in [
+ b"".as_slice(),
+ b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\npart".as_slice(),
+ ] {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/failure", listener.local_addr()?);
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ let (mut stream, _) = listener.accept()?;
+ read_request(&mut stream)?;
+ stream.write_all(bytes)?;
+ Ok(())
+ });
+ let (_owner, cancel, pending) =
+ submit_html_request(&FetchConfig::default(), Request::get(&url)?);
+ match pending.into_response().await {
+ Ok(mut response) => assert!(response.finish().await.is_err()),
+ Err(_) => assert!(bytes.is_empty()),
+ }
+ let timing = cancel.response_timing();
+ assert_eq!(timing.response_started_at().is_some(), !bytes.is_empty());
+ assert!(timing.response_end_at().is_some());
+ assert!(cancel.response_completion_is_committed());
+ assert!(!cancel.cancel_if_response_pending());
+ server.join().expect("server thread")?;
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn html_response_timing_finishes_bodyless_responses_at_headers() -> Result<()> {
+ for (method, headers) in [
+ ("GET", "HTTP/1.1 204 No Content\r\n"),
+ ("GET", "HTTP/1.1 205 Reset Content\r\n"),
+ ("GET", "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n"),
+ ("HEAD", "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n"),
+ ] {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/empty", listener.local_addr()?);
+ let (release, ready) = std_mpsc::sync_channel(1);
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ let (mut stream, _) = listener.accept()?;
+ read_request(&mut stream)?;
+ write!(stream, "{headers}Connection: close\r\n\r\n")?;
+ ready.recv_timeout(Duration::from_secs(5)).unwrap();
+ Ok(())
+ });
+ let request = Request::new(method, &url, None, Vec::new())?;
+ let (_owner, cancel, pending) = submit_html_request(&FetchConfig::default(), request);
+ let mut response = pending.into_response().await?;
+ tokio::time::timeout(Duration::from_secs(5), response.finish()).await??;
+ let timing = cancel.response_timing();
+ assert!(timing.response_end_at().is_some(), "{method}: {headers}");
+ assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());
+ assert!(!timing.body_bytes_received());
+ assert!(!cancel.cancel_if_response_pending());
+ release.send(())?;
+ server.join().expect("server thread")?;
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn html_response_timing_preserves_cancellation_before_body_completion() -> Result<()> {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/cancel", listener.local_addr()?);
+ let (release, ready) = std_mpsc::sync_channel(1);
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ let (mut stream, _) = listener.accept()?;
+ read_request(&mut stream)?;
+ stream
+ .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\npart")?;
+ ready.recv_timeout(Duration::from_secs(5)).unwrap();
+ Ok(())
+ });
+ let (_owner, cancel, pending) =
+ submit_html_request(&FetchConfig::default(), Request::get(&url)?);
+ let mut response = pending.into_response().await?;
+ assert_eq!(response.next_chunk().await.as_deref(), Some("part"));
+ assert!(cancel.response_timing().response_end_at().is_none());
+ assert!(cancel.cancel_if_response_pending());
+ assert!(
+ tokio::time::timeout(Duration::from_secs(5), response.finish())
+ .await?
+ .is_err()
+ );
+ assert!(cancel.response_timing().response_end_at().is_some());
+ assert!(!cancel.response_completion_is_committed());
+ release.send(())?;
+ server.join().expect("server thread")?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn html_cached_response_timing_does_not_invent_network_observations() -> Result<()> {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/cache", listener.local_addr()?);
+ let cache_dir = std::env::temp_dir().join(format!(
+ "moli-html-timing-{}-{}",
+ std::process::id(),
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)?
+ .as_nanos(),
+ ));
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ let (mut stream, _) = listener.accept()?;
+ read_request(&mut stream)?;
+ stream.write_all(b"HTTP/1.1 200 OK\r\nCache-Control: max-age=600\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody")?;
+ Ok(())
+ });
+ let mut config = FetchConfig::default();
+ config.set_http_cache_dir(Some(cache_dir.display().to_string()));
+ let (owner, _, pending) = submit_html_request(&config, Request::get(&url)?);
+ pending.into_response().await?.finish().await?;
+ server.join().expect("server thread")?;
+
+ let pending = owner.handle().submit_html_stream(Request::get(&url)?)?;
+ let cancel = pending.cancel_handle.clone();
+ let mut response = pending.into_response().await?;
+ assert!(response.from_cache);
+ let mut text = String::new();
+ while let Some(chunk) = response.next_chunk().await {
+ text.push_str(&chunk);
+ }
+ response.finish().await?;
+ assert_eq!(text, "body");
+ let timing = cancel.response_timing();
+ assert!(timing.request_started_at().is_none());
+ assert!(timing.first_interim_response_started_at().is_none());
+ assert!(!timing.body_bytes_received());
+ assert!(timing.response_started_at().is_some());
+ assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());
+ assert!(!cancel.cancel_if_response_pending());
+ drop(owner);
+ std::fs::remove_dir_all(cache_dir)?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn html_declared_body_completion_precedes_delivery_of_the_last_chunk() -> Result<()> {
+ for (encoding_headers, complete) in [
+ (b"".as_slice(), true),
+ (b"Content-Encoding: gzip\r\n".as_slice(), false),
+ (b"Transfer-Encoding: chunked\r\n".as_slice(), false),
+ ] {
+ let cancel = FetchCancelHandle::new();
+ let (start_tx, _start_rx) = oneshot::channel();
+ let (body_tx, mut body_rx) = mpsc::unbounded_channel();
+ let mut collector = StreamingResponseCollector::new(
+ new_shared_browser_cookie_store(),
+ start_tx,
+ body_tx,
+ cancel.clone(),
+ );
+ collector.begin_request(
+ None,
+ Url::parse("http://example.test/body")?,
+ NetworkCookieRequestContext::top_level_navigation("GET"),
+ None,
+ true,
+ Vec::new(),
+ None,
+ );
+ assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
+ assert!(collector.header(b"Content-Length: 4\r\n"));
+ if !encoding_headers.is_empty() {
+ assert!(collector.header(encoding_headers));
+ }
+ assert!(collector.header(b"\r\n"));
+ assert_eq!(collector.write(b"bo").unwrap(), 2);
+ assert_eq!(body_rx.try_recv()?.as_str(), "bo");
+ assert!(cancel.response_timing().response_end_at().is_none());
+ assert_eq!(collector.write(b"dy").unwrap(), 2);
+ assert_eq!(body_rx.try_recv()?.as_str(), "dy");
+ // Decoded or chunked lengths cannot be compared with Content-Length.
+ assert_eq!(
+ cancel.response_timing().response_end_at().is_some(),
+ complete
+ );
+ assert_eq!(cancel.cancel_if_response_pending(), !complete);
+ }
+ Ok(())
+}