Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): match Chromium for bare-CR header EOF
ldm0ce7f503f2804
diff --git a/Cargo.lock b/Cargo.lockindex ddb3a136a..753d1b28e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -946,7 +946,7 @@ dependencies = [[[package]]name = "curl"version = "0.4.51"-source = "git+https://github.com/lexmount/curl-rust?rev=79310cbd939be3548b78d0b23ab1d60d5facdeec#79310cbd939be3548b78d0b23ab1d60d5facdeec"+source = "git+https://github.com/lexmount/curl-rust?rev=0392e12475d8d1160d4cae07c172cac02797cc25#0392e12475d8d1160d4cae07c172cac02797cc25"dependencies = ["curl-sys","libc",@@ -960,7 +960,7 @@ dependencies = [[[package]]name = "curl-sys"version = "0.4.91+curl-8.23.0"-source = "git+https://github.com/lexmount/curl-rust?rev=79310cbd939be3548b78d0b23ab1d60d5facdeec#79310cbd939be3548b78d0b23ab1d60d5facdeec"+source = "git+https://github.com/lexmount/curl-rust?rev=0392e12475d8d1160d4cae07c172cac02797cc25#0392e12475d8d1160d4cae07c172cac02797cc25"dependencies = ["brotlic-sys","cc",diff --git a/Cargo.toml b/Cargo.tomlindex 3cc6c4666..9b4144584 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -107,8 +107,8 @@ subsetter = { git = "https://github.com/lexmount/subsetter", rev = "8d3815245835url = { git = "https://github.com/lexmount/rust-url", rev = "ca9fcad2b99f20c30c28a618ad78410f6fd31161" }idna = { git = "https://github.com/lexmount/rust-url", rev = "ca9fcad2b99f20c30c28a618ad78410f6fd31161" }xml5ever = { git = "https://github.com/lexmount/html5ever", rev = "4574311c63e79bba7d4ddb98cb0562d8dad499ee" }-curl = { git = "https://github.com/lexmount/curl-rust", rev = "79310cbd939be3548b78d0b23ab1d60d5facdeec" }-curl-sys = { git = "https://github.com/lexmount/curl-rust", rev = "79310cbd939be3548b78d0b23ab1d60d5facdeec" }+curl = { git = "https://github.com/lexmount/curl-rust", rev = "0392e12475d8d1160d4cae07c172cac02797cc25" }+curl-sys = { git = "https://github.com/lexmount/curl-rust", rev = "0392e12475d8d1160d4cae07c172cac02797cc25" }cookie = { git = "https://github.com/ldm0/cookie-rs", branch = "priority" }v8 = { path = "vendor/v8-152.2.0" }deno_v8 = { path = "vendor/deno_v8-0.3.0" }diff --git a/moli-fetch/src/tests/header_eof.rs b/moli-fetch/src/tests/header_eof.rsindex 00c22b96a..c34aab739 100644--- a/moli-fetch/src/tests/header_eof.rs+++ b/moli-fetch/src/tests/header_eof.rs@@ -352,3 +352,161 @@ async fn header_eof_does_not_publish_a_cancelled_response() -> Result<()> {assert!(client.shutdown().is_clean());Ok(())}++#[tokio::test]+async fn bare_cr_header_eof_matches_chromium_after_final_and_informational_headers() -> Result<()> {+ for prefix in [b"".as_slice(), b"HTTP/1.1 100 Continue\r\n\r\n"] {+ for byte in [b'X', b':', b'\t', b'\r', 0xff] {+ let response = [+ prefix,+ b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nX-Test: retained\r\n\r",+ &[byte, b'\n'],+ b"abc",+ ]+ .concat();+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let url = format!("http://{}/bare-cr-eof", listener.local_addr()?);+ let server = tokio::spawn(serve_responses(listener, vec![response; 3]));+ let client =+ FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ for mode in ["buffered", "html", "raw"] {+ let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode)+ .await+ .with_context(|| format!("{mode}, prefix={prefix:?}, byte={byte:#x}"))?;+ assert_eq!(head.status, 200, "{mode}");+ assert!(+ head.headers+ .contains(&("content-length".into(), b"3".to_vec()))+ );+ assert!(+ head.headers+ .contains(&("x-test".into(), b"retained".to_vec()))+ );+ // Chromium consumes these bytes as an unterminated header block.+ assert!(body.is_empty(), "{mode}: {body:?}");+ }+ assert_eq!(server.await??.len(), 3);+ assert!(client.shutdown().is_clean());+ }+ }+ Ok(())+}++#[tokio::test]+async fn header_eof_rejects_nul_in_a_pending_field_on_every_transport() -> Result<()> {+ for suffix in [b"X-Test: one\0two".as_slice(), b"\r\0\n"] {+ let response = [+ b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n".as_slice(),+ suffix,+ ]+ .concat();+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let url = format!("http://{}/nul-eof", listener.local_addr()?);+ let server = tokio::spawn(serve_responses(listener, vec![response; 3]));+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ for mode in ["buffered", "html", "raw"] {+ let request =+ Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);+ assert!(+ fetch_in_mode(&client, request, mode).await.is_err(),+ "{mode}"+ );+ }+ assert_eq!(server.await??.len(), 3);+ assert!(client.shutdown().is_clean());+ }+ Ok(())+}++#[tokio::test]+async fn bounded_leading_cr_status_lines_work_after_informational_responses() -> Result<()> {+ for prefix in ["", "HTTP/1.1 100 Continue\r\n\r\n"] {+ for count in 1..=5 {+ let response = format!(+ "{prefix}{}HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc",+ "\r".repeat(count)+ );+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let url = format!("http://{}/status-slop", listener.local_addr()?);+ let server = tokio::spawn(serve_responses(listener, vec![response.into_bytes(); 3]));+ let client =+ FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ for mode in ["buffered", "html", "raw"] {+ let request = Request::get(&url)?+ .with_browser_request_metadata(BrowserRequestMetadata::Fetch);+ let result = fetch_in_mode(&client, request, mode).await;+ if count <= 4 {+ let (head, body) = result.with_context(|| format!("{mode}, count={count}"))?;+ assert_eq!(head.status, 200);+ assert_eq!(body, b"abc");+ } else {+ assert!(result.is_err(), "{mode}: status slop exceeds four bytes");+ }+ }+ assert_eq!(server.await??.len(), 3);+ assert!(client.shutdown().is_clean());+ }+ }+ Ok(())+}++#[tokio::test]+async fn https_header_eof_is_rejected_while_bare_cr_body_bytes_are_preserved() -> Result<()> {+ let cases = [+ (+ b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\n\rX\n".as_slice(),+ true,+ ),+ (+ b"HTTP/1.1 200 OK\r\nX-Test: truncated\r\n".as_slice(),+ false,+ ),+ (+ b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\rX\nabc".as_slice(),+ false,+ ),+ ];+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let url = format!("https://{}/eof", listener.local_addr()?);+ let server = tokio::spawn(async move {+ let acceptor = super::support::http11_tls_acceptor();+ for (response, _) in cases.iter().cycle().take(cases.len() * 3) {+ let (stream, _) =+ tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;+ let mut stream = acceptor.accept(stream).await?;+ let mut request = Vec::new();+ while !request.ends_with(b"\r\n\r\n") {+ let mut byte = [0];+ stream.read_exact(&mut byte).await?;+ request.push(byte[0]);+ }+ stream.write_all(response).await?;+ stream.shutdown().await?;+ }+ Ok::<_, anyhow::Error>(())+ });+ let mut config = FetchConfig::default();+ config.set_tls_verify_host(false);+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());+ for mode in ["buffered", "html", "raw"] {+ for (_, succeeds) in cases {+ let request =+ Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);+ let result = fetch_in_mode(&client, request, mode).await;+ if succeeds {+ let (head, body) =+ result.with_context(|| format!("{mode}: complete TLS response"))?;+ assert_eq!(head.status, 200);+ assert_eq!(body, b"\rX\n");+ } else {+ assert!(+ result.is_err(),+ "{mode}: HTTPS must reject incomplete headers"+ );+ }+ }+ }+ server.await??;+ assert!(client.shutdown().is_clean());+ Ok(())+}diff --git a/moli-fetch/src/tests/support.rs b/moli-fetch/src/tests/support.rsindex a5d61d9a7..060158e20 100644--- a/moli-fetch/src/tests/support.rs+++ b/moli-fetch/src/tests/support.rs@@ -844,7 +844,7 @@ fn http2_and_http11_tls_acceptor() -> tokio_rustls::TlsAcceptor {tokio_rustls::TlsAcceptor::from(Arc::new(config))}-fn http11_tls_acceptor() -> tokio_rustls::TlsAcceptor {+pub(super) fn http11_tls_acceptor() -> tokio_rustls::TlsAcceptor {use tokio_rustls::rustls::{ServerConfig,pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer},