Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): finalize response headers at successful EOF

ldm04944ea15b2fd
diff --git a/moli-fetch/src/blocking/collectors.rs b/moli-fetch/src/blocking/collectors.rs
index 1db5bfcdc..0b8c5bbc1 100644
--- a/moli-fetch/src/blocking/collectors.rs
+++ b/moli-fetch/src/blocking/collectors.rs
@@ -168,6 +168,7 @@ pub struct StreamingResponseCollector {
cache_body_writer: Option<HttpCacheBodyWriter>,
cache_plan: Option<StreamingCachePlan>,
started: bool,
+ headers_finalized: bool,
header_terminated: bool,
start_tx: Option<oneshot::Sender<Result<StreamingHtmlResponseStart>>>,
body_tx: Option<mpsc::UnboundedSender<String>>,
@@ -237,6 +238,7 @@ pub struct RawStreamingResponseCollector {
cache_body_writer: Option<HttpCacheBodyWriter>,
cache_plan: Option<StreamingCachePlan>,
started: bool,
+ headers_finalized: bool,
header_terminated: bool,
start_tx: Option<oneshot::Sender<Result<StreamingHtmlResponseStart>>>,
body_tx: Option<mpsc::UnboundedSender<Vec<u8>>>,
@@ -272,6 +274,7 @@ impl StreamingResponseCollector {
cache_body_writer: None,
cache_plan: None,
started: false,
+ headers_finalized: false,
header_terminated: false,
start_tx: Some(start_tx),
body_tx: Some(body_tx),
@@ -309,6 +312,7 @@ impl StreamingResponseCollector {
self.cache_body_writer = cache_body_writer;
self.cache_plan = None;
self.started = false;
+ self.headers_finalized = false;
self.header_terminated = false;
self.request_cookie_report = request_cookie_report;
self.response_credentials_allowed = response_credentials_allowed;
@@ -394,6 +398,24 @@ impl StreamingResponseCollector {
self.client_hint_restart_requested
}
+ pub(crate) fn finish_headers_at_eof(&mut self) -> Result<()> {
+ if self.headers_finalized || self.cancel_handle.is_cancelled() {
+ return Ok(());
+ }
+ anyhow::ensure!(
+ self.status != 0 && !is_interim_response_status(self.status),
+ "response ended without final HTTP headers"
+ );
+ // Curl may accept EOF after complete header lines without delivering
+ // the empty separator. Run the usual cookie/cache/client-hint policy
+ // before the owner decides whether to publish or retry this response.
+ self.finalize_headers();
+ if let Some(error) = &self.callback_error {
+ return Err(Error::msg(error.clone()));
+ }
+ Ok(())
+ }
+
fn finalize_headers(&mut self) -> bool {
if self.response_too_large {
return false;
@@ -407,6 +429,7 @@ impl StreamingResponseCollector {
return true;
};
+ self.headers_finalized = true;
if self.response_credentials_allowed {
let Some(request_context) = self.current_cookie_context.as_ref() else {
self.callback_error =
@@ -628,6 +651,7 @@ impl RawStreamingResponseCollector {
cache_body_writer: None,
cache_plan: None,
started: false,
+ headers_finalized: false,
header_terminated: false,
start_tx: Some(start_tx),
body_tx: Some(body_tx),
@@ -666,6 +690,7 @@ impl RawStreamingResponseCollector {
self.cache_body_writer = None;
self.cache_plan = None;
self.started = false;
+ self.headers_finalized = false;
self.header_terminated = false;
self.request_cookie_report = request_cookie_report;
self.response_credentials_allowed = response_credentials_allowed;
@@ -752,6 +777,23 @@ impl RawStreamingResponseCollector {
self.client_hint_restart_requested
}
+ pub(crate) fn finish_headers_at_eof(&mut self) -> Result<()> {
+ if self.headers_finalized || self.cancel_handle.is_cancelled() {
+ return Ok(());
+ }
+ anyhow::ensure!(
+ self.status != 0 && !is_interim_response_status(self.status),
+ "response ended without final HTTP headers"
+ );
+ // A deferred redirect or 304 has not started either, so track header
+ // finalization separately to avoid applying its response policy twice.
+ self.finalize_headers();
+ if let Some(error) = &self.callback_error {
+ return Err(Error::msg(error.clone()));
+ }
+ Ok(())
+ }
+
fn finalize_headers(&mut self) -> bool {
if self.response_too_large {
return false;
@@ -765,6 +807,7 @@ impl RawStreamingResponseCollector {
return true;
};
+ self.headers_finalized = true;
if self.response_credentials_allowed {
let Some(request_context) = self.current_cookie_context.as_ref() else {
self.callback_error =
@@ -1023,6 +1066,7 @@ impl Handler for StreamingResponseCollector {
.and_then(|status| status.parse::<u16>().ok())
{
self.headers.clear();
+ self.headers_finalized = false;
self.status = status;
self.negotiated_http_version = NegotiatedHttpVersion::from_status_line(line);
self.response_bytes_received = 0;
@@ -1098,6 +1142,7 @@ impl Handler for RawStreamingResponseCollector {
.and_then(|status| status.parse::<u16>().ok())
{
self.headers.clear();
+ self.headers_finalized = false;
self.status = status;
self.negotiated_http_version = NegotiatedHttpVersion::from_status_line(line);
self.response_bytes_received = 0;
diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs
index 726ea1610..f8cf4acdf 100644
--- a/moli-fetch/src/runtime.rs
+++ b/moli-fetch/src/runtime.rs
@@ -1663,6 +1663,13 @@ impl RuntimeOwner {
return;
}
+ let result = result.and_then(|()| {
+ easy.get_mut()
+ .streaming_mut()
+ .expect("streaming request should use streaming collector")
+ .finish_headers_at_eof()
+ });
+
if !job.cancel_handle.is_cancelled()
&& easy
.get_ref()
@@ -1975,6 +1982,13 @@ impl RuntimeOwner {
return;
}
+ let result = result.and_then(|()| {
+ easy.get_mut()
+ .raw_streaming_mut()
+ .expect("raw streaming request should use raw streaming collector")
+ .finish_headers_at_eof()
+ });
+
if !job.cancel_handle.is_cancelled()
&& easy
.get_ref()
diff --git a/moli-fetch/src/tests/header_eof.rs b/moli-fetch/src/tests/header_eof.rs
new file mode 100644
index 000000000..7c5097228
--- /dev/null
+++ b/moli-fetch/src/tests/header_eof.rs
@@ -0,0 +1,355 @@
+use super::*;
+
+async fn serve_responses(listener: TcpListener, responses: Vec<Vec<u8>>) -> Result<Vec<String>> {
+ let mut requests = Vec::new();
+ for response in responses {
+ let (mut stream, _) =
+ tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;
+ let mut request = Vec::new();
+ while !request.windows(4).any(|part| part == b"\r\n\r\n") {
+ let mut chunk = [0; 1024];
+ let count =
+ tokio::time::timeout(Duration::from_secs(5), stream.read(&mut chunk)).await??;
+ anyhow::ensure!(
+ count != 0 && request.len() < 64 * 1024,
+ "incomplete request"
+ );
+ request.extend_from_slice(&chunk[..count]);
+ }
+ requests.push(String::from_utf8(request)?);
+ stream.write_all(&response).await?;
+ stream.shutdown().await?;
+ }
+ Ok(requests)
+}
+
+async fn fetch_in_mode(
+ client: &FetchClientHandle,
+ request: Request,
+ mode: &str,
+) -> Result<(ResponseHead, Vec<u8>)> {
+ let (head, body) = match mode {
+ "buffered" => client
+ .fetch(request.with_follow_redirects(false))
+ .await?
+ .into_body(),
+ "html" => client.fetch_html_stream(request).await?.into_body(),
+ "raw" => client
+ .fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
+ .await?
+ .into_body(),
+ _ => unreachable!(),
+ };
+ Ok((head, body.into_materialized_bytes().await?))
+}
+
+#[tokio::test]
+async fn header_eof_delivers_complete_fields_on_every_transport() -> Result<()> {
+ for (method, response, status, headers) in [
+ ("GET", b"HTTP/1.1 200 OK\r\nX-Test: first\r\nX-Test: second\r\nX-Bytes: \xff\r\n".as_slice(), 200,
+ vec![("x-test", "first"), ("x-test", "second"), ("x-bytes", "\u{ff}")]),
+ ("GET", b"HTTP/1.0 200 NANANA\nCONTENT-LENGTH: 0\ncontent-length:\t 0\n", 200,
+ vec![("content-length", "0"), ("content-length", "0")]),
+ ("GET", b"HTTP/1.1 280 HELLO\nfoo-test: 1\nfoo-test: 2\nfoo-test: 3\n", 280,
+ vec![("foo-test", "1"), ("foo-test", "2"), ("foo-test", "3")]),
+ ("GET", b"HTTP/1.1 200 OK\r\n", 200, vec![]),
+ ("GET", b"HTTP/1.1 204 No Content\r\nX-Test: empty\r\n", 204,
+ vec![("x-test", "empty")]),
+ ("GET", b"HTTP/1.1 304 Not Modified\r\nX-Test: empty\r\n", 304,
+ vec![("x-test", "empty")]),
+ ("GET", b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n", 200,
+ vec![("transfer-encoding", "chunked")]),
+ ("HEAD", b"HTTP/1.1 200 OK\r\nContent-Length: 40\r\nX-Test: head\r\n", 200,
+ vec![("content-length", "40"), ("x-test", "head")]),
+ ("GET", b"HTTP/1.1 103 Early Hints\r\nX-Interim: hidden\r\n\r\nHTTP/1.1 200 OK\r\nX-Test: final\r\n", 200,
+ vec![("x-test", "final")]),
+ ] {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/eof", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(listener, vec![response.to_vec(); 3]));
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ for mode in ["buffered", "html", "raw"] {
+ let (head, body) = fetch_in_mode(&client, Request::new(method, &url, None, vec![])?, mode)
+ .await.with_context(|| format!("{mode}: {response:?}"))?;
+ assert_eq!(head.status, status, "{mode}");
+ assert_eq!(head.headers, headers.iter().map(|(n, v)| (n.to_string(), v.to_string())).collect::<Vec<_>>(), "{mode}");
+ assert!(body.is_empty(), "{mode}: {body:?}");
+ }
+ assert_eq!(server.await??.len(), 3);
+ assert!(client.shutdown().is_clean());
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_does_not_recover_transport_failures() -> Result<()> {
+ for response in [
+ b"".as_slice(),
+ b"HTTP/1.1 103 Early Hints\r\nX-Interim: hidden\r\n\r\n",
+ b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 4\r\nX-Test: conflict\r\n",
+ b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nx",
+ b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nx",
+ ] {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/failure", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(listener, vec![response.to_vec(); 3]));
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ for mode in ["buffered", "html", "raw"] {
+ // Subresource failures must not trigger the navigation HTTPS-upgrade retry.
+ let request =
+ Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
+ assert!(
+ fetch_in_mode(&client, request, mode).await.is_err(),
+ "{mode}: {response:?}"
+ );
+ }
+ assert_eq!(server.await??.len(), 3);
+ assert!(client.shutdown().is_clean());
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_processes_cookies_before_following_redirects() -> Result<()> {
+ for mode in ["html", "raw"] {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/start", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(
+ listener,
+ vec![
+ b"HTTP/1.1 302 Found\r\nLocation: /final\r\nSet-Cookie: redirect=one; Path=/\r\n"
+ .to_vec(),
+ b"HTTP/1.1 200 OK\r\nSet-Cookie: final=two; Path=/\r\nX-Test: done\r\n".to_vec(),
+ ],
+ ));
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
+ assert_eq!(head.status, 200, "{mode}");
+ assert_eq!(head.final_url.path(), "/final", "{mode}");
+ assert!(head.redirected);
+ assert_eq!(head.redirect_chain.len(), 1);
+ assert_eq!(head.redirect_chain[0].cookie_set_reports.len(), 1);
+ assert_eq!(head.cookie_set_reports.len(), 1);
+ assert!(body.is_empty());
+ let requests = server.await??;
+ assert_eq!(requests.len(), 2);
+ assert!(requests[1].starts_with("GET /final "));
+ assert!(
+ requests[1].contains("Cookie: redirect=one\r\n"),
+ "{}",
+ requests[1]
+ );
+ assert!(client.shutdown().is_clean());
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_keeps_manual_raw_redirects_unfollowed() -> Result<()> {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/start", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(
+ listener,
+ vec![
+ b"HTTP/1.1 302 Found\r\nLocation: /final\r\nSet-Cookie: redirect=one; Path=/\r\n"
+ .to_vec(),
+ ],
+ ));
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ let (head, body) = fetch_in_mode(
+ &client,
+ Request::get(&url)?.with_follow_redirects(false),
+ "raw",
+ )
+ .await?;
+ assert_eq!(head.status, 302);
+ assert_eq!(head.final_url.path(), "/start");
+ assert!(!head.redirected);
+ assert_eq!(head.cookie_set_reports.len(), 1);
+ assert!(body.is_empty());
+ assert_eq!(server.await??.len(), 1);
+ assert!(client.shutdown().is_clean());
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_creates_reusable_empty_cache_entries() -> Result<()> {
+ for mode in ["html", "raw"] {
+ let cache_dir = unique_test_cache_dir();
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/cached", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(
+ listener,
+ vec![b"HTTP/1.1 200 OK\r\nCache-Control: max-age=60\r\nX-Test: cached\r\n".to_vec()],
+ ));
+ let mut config = FetchConfig::default();
+ config.set_http_cache_dir(Some(cache_dir.display().to_string()));
+ for from_cache in [false, true] {
+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());
+ let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
+ assert_eq!(head.from_cache, from_cache, "{mode}");
+ assert!(head.headers.contains(&("x-test".into(), "cached".into())));
+ assert!(body.is_empty());
+ assert!(client.shutdown().is_clean());
+ }
+ assert_eq!(server.await??.len(), 1);
+ fs::remove_dir_all(cache_dir)?;
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_revalidates_cached_raw_bodies() -> Result<()> {
+ let cache_dir = unique_test_cache_dir();
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/revalidate", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(listener, vec![
+ b"HTTP/1.1 200 OK\r\nCache-Control: no-cache\r\nETag: \"eof\"\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody".to_vec(),
+ b"HTTP/1.1 304 Not Modified\r\nCache-Control: max-age=60\r\nETag: \"eof\"\r\nX-Test: updated\r\nSet-Cookie: revalidated=one; Path=/\r\n".to_vec(),
+ b"HTTP/1.1 200 OK\r\nX-Test: cookie-check\r\n".to_vec(),
+ ]));
+ let mut config = FetchConfig::default();
+ config.set_http_cache_dir(Some(cache_dir.display().to_string()));
+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());
+ let (first, body) = fetch_in_mode(&client, Request::get(&url)?, "raw").await?;
+ assert!(!first.from_cache);
+ assert_eq!(body, b"body");
+ let (revalidated, body) = fetch_in_mode(&client, Request::get(&url)?, "raw").await?;
+ assert_eq!(revalidated.status, 200);
+ assert!(revalidated.from_cache);
+ assert_eq!(body, b"body");
+ assert!(
+ revalidated
+ .headers
+ .contains(&("x-test".into(), "updated".into()))
+ );
+ let (_, body) = fetch_in_mode(
+ &client,
+ Request::get(&format!("{url}/cookie-check"))?,
+ "raw",
+ )
+ .await?;
+ assert!(body.is_empty());
+ let requests = server.await??;
+ assert_eq!(requests.len(), 3);
+ assert!(
+ requests[1]
+ .to_ascii_lowercase()
+ .contains("if-none-match: \"eof\"\r\n")
+ );
+ assert!(requests[2].contains("Cookie: revalidated=one\r\n"));
+ assert!(client.shutdown().is_clean());
+ fs::remove_dir_all(cache_dir)?;
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_applies_critical_client_hints_before_publishing() -> Result<()> {
+ for mode in ["html", "raw"] {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/navigation", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(listener, vec![
+ b"HTTP/1.1 403 Challenge\r\nAccept-CH: Sec-CH-UA-Arch\r\nCritical-CH: Sec-CH-UA-Arch\r\nSet-Cookie: challenge=one; Path=/\r\n".to_vec(),
+ b"HTTP/1.1 200 OK\r\nX-Test: restarted\r\n".to_vec(),
+ ]));
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
+ assert_eq!(head.status, 200, "{mode}");
+ assert_eq!(head.redirect_chain.len(), 1);
+ let restart = &head.redirect_chain[0];
+ assert_eq!(restart.from_url, restart.to_url);
+ assert_eq!(restart.response_extra_info.as_ref().unwrap().status, 403);
+ assert_eq!(
+ restart
+ .response_extra_info
+ .as_ref()
+ .unwrap()
+ .cookie_set_reports
+ .len(),
+ 1
+ );
+ assert!(body.is_empty());
+ let requests = server.await??;
+ assert_eq!(requests.len(), 2);
+ assert!(!requests[0].to_ascii_lowercase().contains("sec-ch-ua-arch:"));
+ assert!(
+ requests[1]
+ .to_ascii_lowercase()
+ .contains("sec-ch-ua-arch: \"x86\"")
+ );
+ assert!(requests[1].contains("Cookie: challenge=one\r\n"));
+ assert!(client.shutdown().is_clean());
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_preserves_the_response_size_limit() -> Result<()> {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/limited", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(
+ listener,
+ vec![b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\nX-Test: oversized\r\n".to_vec(); 3],
+ ));
+ let mut config = FetchConfig::default();
+ config.set_connection_limits(None, None, Some(4));
+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());
+ for mode in ["buffered", "html", "raw"] {
+ let error = fetch_in_mode(&client, Request::get(&url)?, mode)
+ .await
+ .unwrap_err();
+ if mode != "buffered" {
+ assert!(
+ error
+ .to_string()
+ .contains("response exceeded configured limit of 4 bytes"),
+ "{mode}: {error:#}"
+ );
+ }
+ }
+ assert_eq!(server.await??.len(), 3);
+ assert!(client.shutdown().is_clean());
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_does_not_publish_a_cancelled_response() -> Result<()> {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/cancel", listener.local_addr()?);
+ let (headers_tx, headers_rx) = oneshot::channel();
+ let (close_tx, close_rx) = oneshot::channel();
+ let server = tokio::spawn(async move {
+ let (mut stream, _) = listener.accept().await?;
+ let mut request = Vec::new();
+ while !request.ends_with(b"\r\n\r\n") {
+ let mut byte = [0];
+ stream.read_exact(&mut byte).await?;
+ request.push(byte[0]);
+ }
+ stream
+ .write_all(b"HTTP/1.1 200 OK\r\nX-Test: pending\r\n")
+ .await?;
+ let _ = headers_tx.send(());
+ close_rx.await?;
+ stream.shutdown().await?;
+ Ok::<_, anyhow::Error>(())
+ });
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ let cancel = FetchCancelHandle::new();
+ let request = Request::get(&url)?;
+ let mut response = Box::pin(client.fetch_raw_stream_with_cancel(request, cancel.clone()));
+ tokio::select! {
+ result = &mut response => panic!("response published before headers ended: {result:?}"),
+ result = headers_rx => result?,
+ }
+ cancel.cancel();
+ let _ = close_tx.send(());
+ assert!(
+ tokio::time::timeout(Duration::from_secs(5), response)
+ .await?
+ .is_err()
+ );
+ server.await??;
+ assert!(client.shutdown().is_clean());
+ Ok(())
+}
diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs
index 104654acc..8b8dbbf22 100644
--- a/moli-fetch/src/tests/mod.rs
+++ b/moli-fetch/src/tests/mod.rs
@@ -1,6 +1,7 @@
mod cookie_context;
mod cookie_store;
mod header_bytes;
+mod header_eof;
mod mixed_transport;
mod request_security;
mod support;