Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): isolate HTML timing for each transfer attempt

ldm087efd22bc61b
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rs
index fcecd3949..b2dd3ad28 100644
--- a/moli-fetch/src/blocking/collectors.rs
+++ b/moli-fetch/src/blocking/collectors.rs
@@ -369,6 +369,7 @@ impl StreamingResponseCollector {
redirect_chain: Vec<RedirectInfo>,
network_request_extra_info: Option<NetworkRequestExtraInfo>,
cache_plan: Option<StreamingCachePlan>,
+ response_progress: ResponseProgress,
) {
self.begin_request(
max_response_size,
@@ -379,6 +380,7 @@ impl StreamingResponseCollector {
redirect_chain,
None,
);
+ self.response_progress = response_progress;
self.network_request_extra_info = network_request_extra_info;
self.cache_plan = cache_plan;
}
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index d13cab77a..d7f609973 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -1012,6 +1012,7 @@ impl RuntimeOwner {
anyhow::Error,
),
> {
+ job.response_progress = job.cancel_handle.begin_response_progress();
if let Err(error) = job.request.validate_request_mode_for_url(&job.current_url) {
return Err((Box::new(job), None, error));
}
@@ -1138,6 +1139,7 @@ impl RuntimeOwner {
job.request.redirect_chain.clone(),
request_extra_info.clone(),
cache_plan,
+ job.response_progress.clone(),
);
collector.set_client_hint_response_policy(prepared_request.response_policy);
@@ -3790,6 +3792,9 @@ fn request_panics_for_testing(request: &Request) -> bool {
})
}
+#[cfg(test)]
+mod response_timing_tests;
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/moli-fetch/src/runtime/response_timing_tests.rs b/moli-fetch/src/runtime/response_timing_tests.rs
new file mode 100644
index 000000000..176b684dc
--- /dev/null
+++ b/moli-fetch/src/runtime/response_timing_tests.rs
@@ -0,0 +1,78 @@
+use super::*;
+use moli_cookie_jar::new_shared_browser_cookie_store;
+use std::{io::Write, sync::mpsc as std_mpsc};
+
+fn submit_html_request(
+ config: &FetchConfig,
+ request: Request,
+) -> (
+ FetchRuntimeOwner,
+ FetchCancelHandle,
+ PendingStreamingHtmlResponse,
+) {
+ let owner = FetchRuntimeOwner::new(config, new_shared_browser_cookie_store());
+ let pending = owner.handle().submit_html_stream(request).unwrap();
+ let cancel = pending.cancel_handle.clone();
+ (owner, cancel, pending)
+}
+
+#[tokio::test]
+async fn html_redirect_timing_belongs_to_the_final_transfer() -> Result<()> {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let url = format!("http://{}/redirect", listener.local_addr()?);
+ let (release_redirect, ready) = std_mpsc::sync_channel(1);
+ let server = thread::spawn(move || -> std::io::Result<()> {
+ for path in ["/redirect", "/final"] {
+ let (mut stream, _) = listener.accept()?;
+ stream.set_read_timeout(Some(Duration::from_secs(5)))?;
+ let mut request = Vec::new();
+ let mut byte = [0];
+ while !request.ends_with(b"\r\n\r\n") {
+ stream.read_exact(&mut byte)?;
+ request.extend_from_slice(&byte);
+ }
+ assert!(request.starts_with(format!("GET {path} ").as_bytes()));
+ if path == "/redirect" {
+ stream.write_all(
+ b"HTTP/1.1 103 Early Hints\r\n\r\n\
+ HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 8\r\nConnection: close\r\n\r\nold!",
+ )?;
+ ready.recv_timeout(Duration::from_secs(5)).unwrap();
+ stream.write_all(b"tail")?;
+ } else {
+ stream.write_all(
+ b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
+ )?;
+ }
+ }
+ Ok(())
+ });
+ let (_owner, cancel, pending) =
+ submit_html_request(&FetchConfig::default(), Request::get(&url)?);
+ let deadline = Instant::now() + Duration::from_secs(5);
+ while !cancel.response_timing().body_bytes_received() {
+ anyhow::ensure!(Instant::now() < deadline, "redirect body was not observed");
+ tokio::time::sleep(Duration::from_millis(1)).await;
+ }
+ let redirect_timing = cancel.response_timing();
+ assert!(
+ redirect_timing
+ .first_interim_response_started_at()
+ .is_some()
+ );
+ assert!(redirect_timing.response_started_at().is_some());
+ let redirect_finished_after = Instant::now();
+ release_redirect.send(())?;
+
+ let mut response = pending.into_response().await?;
+ assert_eq!(response.final_url.path(), "/final");
+ assert!(response.next_chunk().await.is_none());
+ response.finish().await?;
+ let timing = cancel.response_timing();
+ assert!(timing.request_started_at().unwrap() >= redirect_finished_after);
+ assert!(timing.response_started_at().unwrap() >= redirect_finished_after);
+ assert!(timing.first_interim_response_started_at().is_none());
+ assert!(!timing.body_bytes_received());
+ server.join().expect("server thread")?;
+ Ok(())
+}