Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(fetch): match Chromium for bare-CR header EOF

ldm0ce7f503f2804
diff --git a/Cargo.lock b/Cargo.lock
index ddb3a136a..753d1b28e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -946,7 +946,7 @@ dependencies = [
[[package]]
name = "curl"
version = "0.4.51"
-source = "git+https://github.com/lexmount/curl-rust?rev=79310cbd939be3548b78d0b23ab1d60d5facdeec#79310cbd939be3548b78d0b23ab1d60d5facdeec"
+source = "git+https://github.com/lexmount/curl-rust?rev=0392e12475d8d1160d4cae07c172cac02797cc25#0392e12475d8d1160d4cae07c172cac02797cc25"
dependencies = [
"curl-sys",
"libc",
@@ -960,7 +960,7 @@ dependencies = [
[[package]]
name = "curl-sys"
version = "0.4.91+curl-8.23.0"
-source = "git+https://github.com/lexmount/curl-rust?rev=79310cbd939be3548b78d0b23ab1d60d5facdeec#79310cbd939be3548b78d0b23ab1d60d5facdeec"
+source = "git+https://github.com/lexmount/curl-rust?rev=0392e12475d8d1160d4cae07c172cac02797cc25#0392e12475d8d1160d4cae07c172cac02797cc25"
dependencies = [
"brotlic-sys",
"cc",
diff --git a/Cargo.toml b/Cargo.toml
index 3cc6c4666..9b4144584 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -107,8 +107,8 @@ subsetter = { git = "https://github.com/lexmount/subsetter", rev = "8d3815245835
url = { git = "https://github.com/lexmount/rust-url", rev = "ca9fcad2b99f20c30c28a618ad78410f6fd31161" }
idna = { git = "https://github.com/lexmount/rust-url", rev = "ca9fcad2b99f20c30c28a618ad78410f6fd31161" }
xml5ever = { git = "https://github.com/lexmount/html5ever", rev = "4574311c63e79bba7d4ddb98cb0562d8dad499ee" }
-curl = { git = "https://github.com/lexmount/curl-rust", rev = "79310cbd939be3548b78d0b23ab1d60d5facdeec" }
-curl-sys = { git = "https://github.com/lexmount/curl-rust", rev = "79310cbd939be3548b78d0b23ab1d60d5facdeec" }
+curl = { git = "https://github.com/lexmount/curl-rust", rev = "0392e12475d8d1160d4cae07c172cac02797cc25" }
+curl-sys = { git = "https://github.com/lexmount/curl-rust", rev = "0392e12475d8d1160d4cae07c172cac02797cc25" }
cookie = { git = "https://github.com/ldm0/cookie-rs", branch = "priority" }
v8 = { path = "vendor/v8-152.2.0" }
deno_v8 = { path = "vendor/deno_v8-0.3.0" }
diff --git a/moli-fetch/src/tests/header_eof.rs b/moli-fetch/src/tests/header_eof.rs
index 00c22b96a..c34aab739 100644
--- a/moli-fetch/src/tests/header_eof.rs
+++ b/moli-fetch/src/tests/header_eof.rs
@@ -352,3 +352,161 @@ async fn header_eof_does_not_publish_a_cancelled_response() -> Result<()> {
assert!(client.shutdown().is_clean());
Ok(())
}
+
+#[tokio::test]
+async fn bare_cr_header_eof_matches_chromium_after_final_and_informational_headers() -> Result<()> {
+ for prefix in [b"".as_slice(), b"HTTP/1.1 100 Continue\r\n\r\n"] {
+ for byte in [b'X', b':', b'\t', b'\r', 0xff] {
+ let response = [
+ prefix,
+ b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nX-Test: retained\r\n\r",
+ &[byte, b'\n'],
+ b"abc",
+ ]
+ .concat();
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/bare-cr-eof", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(listener, vec![response; 3]));
+ let client =
+ FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ for mode in ["buffered", "html", "raw"] {
+ let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode)
+ .await
+ .with_context(|| format!("{mode}, prefix={prefix:?}, byte={byte:#x}"))?;
+ assert_eq!(head.status, 200, "{mode}");
+ assert!(
+ head.headers
+ .contains(&("content-length".into(), b"3".to_vec()))
+ );
+ assert!(
+ head.headers
+ .contains(&("x-test".into(), b"retained".to_vec()))
+ );
+ // Chromium consumes these bytes as an unterminated header block.
+ assert!(body.is_empty(), "{mode}: {body:?}");
+ }
+ assert_eq!(server.await??.len(), 3);
+ assert!(client.shutdown().is_clean());
+ }
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn header_eof_rejects_nul_in_a_pending_field_on_every_transport() -> Result<()> {
+ for suffix in [b"X-Test: one\0two".as_slice(), b"\r\0\n"] {
+ let response = [
+ b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n".as_slice(),
+ suffix,
+ ]
+ .concat();
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/nul-eof", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(listener, vec![response; 3]));
+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ for mode in ["buffered", "html", "raw"] {
+ let request =
+ Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
+ assert!(
+ fetch_in_mode(&client, request, mode).await.is_err(),
+ "{mode}"
+ );
+ }
+ assert_eq!(server.await??.len(), 3);
+ assert!(client.shutdown().is_clean());
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn bounded_leading_cr_status_lines_work_after_informational_responses() -> Result<()> {
+ for prefix in ["", "HTTP/1.1 100 Continue\r\n\r\n"] {
+ for count in 1..=5 {
+ let response = format!(
+ "{prefix}{}HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc",
+ "\r".repeat(count)
+ );
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("http://{}/status-slop", listener.local_addr()?);
+ let server = tokio::spawn(serve_responses(listener, vec![response.into_bytes(); 3]));
+ let client =
+ FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
+ for mode in ["buffered", "html", "raw"] {
+ let request = Request::get(&url)?
+ .with_browser_request_metadata(BrowserRequestMetadata::Fetch);
+ let result = fetch_in_mode(&client, request, mode).await;
+ if count <= 4 {
+ let (head, body) = result.with_context(|| format!("{mode}, count={count}"))?;
+ assert_eq!(head.status, 200);
+ assert_eq!(body, b"abc");
+ } else {
+ assert!(result.is_err(), "{mode}: status slop exceeds four bytes");
+ }
+ }
+ assert_eq!(server.await??.len(), 3);
+ assert!(client.shutdown().is_clean());
+ }
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn https_header_eof_is_rejected_while_bare_cr_body_bytes_are_preserved() -> Result<()> {
+ let cases = [
+ (
+ b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\n\rX\n".as_slice(),
+ true,
+ ),
+ (
+ b"HTTP/1.1 200 OK\r\nX-Test: truncated\r\n".as_slice(),
+ false,
+ ),
+ (
+ b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\rX\nabc".as_slice(),
+ false,
+ ),
+ ];
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let url = format!("https://{}/eof", listener.local_addr()?);
+ let server = tokio::spawn(async move {
+ let acceptor = super::support::http11_tls_acceptor();
+ for (response, _) in cases.iter().cycle().take(cases.len() * 3) {
+ let (stream, _) =
+ tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;
+ let mut stream = acceptor.accept(stream).await?;
+ let mut request = Vec::new();
+ while !request.ends_with(b"\r\n\r\n") {
+ let mut byte = [0];
+ stream.read_exact(&mut byte).await?;
+ request.push(byte[0]);
+ }
+ stream.write_all(response).await?;
+ stream.shutdown().await?;
+ }
+ Ok::<_, anyhow::Error>(())
+ });
+ let mut config = FetchConfig::default();
+ config.set_tls_verify_host(false);
+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());
+ for mode in ["buffered", "html", "raw"] {
+ for (_, succeeds) in cases {
+ let request =
+ Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
+ let result = fetch_in_mode(&client, request, mode).await;
+ if succeeds {
+ let (head, body) =
+ result.with_context(|| format!("{mode}: complete TLS response"))?;
+ assert_eq!(head.status, 200);
+ assert_eq!(body, b"\rX\n");
+ } else {
+ assert!(
+ result.is_err(),
+ "{mode}: HTTPS must reject incomplete headers"
+ );
+ }
+ }
+ }
+ server.await??;
+ assert!(client.shutdown().is_clean());
+ Ok(())
+}
diff --git a/moli-fetch/src/tests/support.rs b/moli-fetch/src/tests/support.rs
index a5d61d9a7..060158e20 100644
--- a/moli-fetch/src/tests/support.rs
+++ b/moli-fetch/src/tests/support.rs
@@ -844,7 +844,7 @@ fn http2_and_http11_tls_acceptor() -> tokio_rustls::TlsAcceptor {
tokio_rustls::TlsAcceptor::from(Arc::new(config))
}
-fn http11_tls_acceptor() -> tokio_rustls::TlsAcceptor {
+pub(super) fn http11_tls_acceptor() -> tokio_rustls::TlsAcceptor {
use tokio_rustls::rustls::{
ServerConfig,
pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer},