Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use super::*;
async fn serve_responses(listener: TcpListener, responses: Vec<Vec<u8>>) -> Result<Vec<String>> {
let mut requests = Vec::new();
for response in responses {
let (mut stream, _) =
tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;
let mut request = Vec::new();
while !request.windows(4).any(|part| part == b"\r\n\r\n") {
let mut chunk = [0; 1024];
let count =
tokio::time::timeout(Duration::from_secs(5), stream.read(&mut chunk)).await??;
anyhow::ensure!(
count != 0 && request.len() < 64 * 1024,
"incomplete request"
);
request.extend_from_slice(&chunk[..count]);
}
requests.push(String::from_utf8(request)?);
stream.write_all(&response).await?;
stream.shutdown().await?;
}
Ok(requests)
}
async fn fetch_in_mode(
client: &FetchClientHandle,
request: Request,
mode: &str,
) -> Result<(ResponseHead, Vec<u8>)> {
let (head, body) = match mode {
"buffered" => client.fetch(request.with_single_hop()).await?.into_body(),
"html" => client.fetch_html_stream(request).await?.into_body(),
"raw" => client
.fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
.await?
.into_body(),
_ => unreachable!(),
};
Ok((head, body.into_materialized_bytes().await?))
}
#[tokio::test]
async fn header_eof_delivers_complete_fields_on_every_transport() -> Result<()> {
for (method, response, status, headers) in [
("GET", b"HTTP/1.1 200 OK\r\nX-Test: first\r\nX-Test: second\r\nX-Bytes: \xff\r\n".as_slice(), 200,
vec![("x-test", b"first".as_slice()), ("x-test", b"second".as_slice()), ("x-bytes", b"\xff".as_slice())]),
("GET", b"HTTP/1.0 200 NANANA\nCONTENT-LENGTH: 0\ncontent-length:\t 0\n", 200,
vec![("content-length", b"0".as_slice()), ("content-length", b"0".as_slice())]),
("GET", b"HTTP/1.1 280 HELLO\nfoo-test: 1\nfoo-test: 2\nfoo-test: 3\n", 280,
vec![("foo-test", b"1".as_slice()), ("foo-test", b"2".as_slice()), ("foo-test", b"3".as_slice())]),
("GET", b"HTTP/1.1 200 OK\r\n", 200, vec![]),
("GET", b"HTTP/1.1 204 No Content\r\nX-Test: empty\r\n", 204,
vec![("x-test", b"empty".as_slice())]),
("GET", b"HTTP/1.1 304 Not Modified\r\nX-Test: empty\r\n", 304,
vec![("x-test", b"empty".as_slice())]),
("GET", b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n", 200,
vec![("transfer-encoding", b"chunked".as_slice())]),
("HEAD", b"HTTP/1.1 200 OK\r\nContent-Length: 40\r\nX-Test: head\r\n", 200,
vec![("content-length", b"40".as_slice()), ("x-test", b"head".as_slice())]),
("GET", b"HTTP/1.1 103 Early Hints\r\nX-Interim: hidden\r\n\r\nHTTP/1.1 200 OK\r\nX-Test: final\r\n", 200,
vec![("x-test", b"final".as_slice())]),
] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/eof", listener.local_addr()?);
let server = tokio::spawn(serve_responses(listener, vec![response.to_vec(); 3]));
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for mode in ["buffered", "html", "raw"] {
let (head, body) = fetch_in_mode(&client, Request::new(method, &url, None, vec![])?, mode)
.await.with_context(|| format!("{mode}: {response:?}"))?;
assert_eq!(head.status, status, "{mode}");
assert_eq!(head.headers, headers.iter().map(|(n, v)| (n.to_string(), v.to_vec())).collect::<Vec<_>>(), "{mode}");
assert!(body.is_empty(), "{mode}: {body:?}");
}
assert_eq!(server.await??.len(), 3);
assert!(client.shutdown().is_clean());
}
Ok(())
}
#[tokio::test]
async fn header_eof_does_not_recover_transport_failures() -> Result<()> {
for response in [
b"".as_slice(),
b"HTTP/1.1 103 Early Hints\r\nX-Interim: hidden\r\n\r\n",
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 4\r\nX-Test: conflict\r\n",
b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nx",
b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nx",
] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/failure", listener.local_addr()?);
let server = tokio::spawn(serve_responses(listener, vec![response.to_vec(); 3]));
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for mode in ["buffered", "html", "raw"] {
// Subresource failures must not trigger the navigation HTTPS-upgrade retry.
let request =
Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
assert!(
fetch_in_mode(&client, request, mode).await.is_err(),
"{mode}: {response:?}"
);
}
assert_eq!(server.await??.len(), 3);
assert!(client.shutdown().is_clean());
}
Ok(())
}
#[tokio::test]
async fn header_eof_processes_cookies_before_following_redirects() -> Result<()> {
for mode in ["html", "raw"] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/start", listener.local_addr()?);
let server = tokio::spawn(serve_responses(
listener,
vec![
b"HTTP/1.1 302 Found\r\nLocation: /final\r\nSet-Cookie: redirect=one; Path=/\r\n"
.to_vec(),
b"HTTP/1.1 200 OK\r\nSet-Cookie: final=two; Path=/\r\nX-Test: done\r\n".to_vec(),
],
));
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
assert_eq!(head.status, 200, "{mode}");
assert_eq!(head.final_url.path(), "/final", "{mode}");
assert!(head.redirected);
assert_eq!(head.redirect_chain.len(), 1);
assert_eq!(head.redirect_chain[0].cookie_set_reports.len(), 1);
assert_eq!(head.cookie_set_reports.len(), 1);
assert!(body.is_empty());
let requests = server.await??;
assert_eq!(requests.len(), 2);
assert!(requests[1].starts_with("GET /final "));
assert!(
requests[1].contains("Cookie: redirect=one\r\n"),
"{}",
requests[1]
);
assert!(client.shutdown().is_clean());
}
Ok(())
}
#[tokio::test]
async fn header_eof_keeps_manual_raw_redirects_unfollowed() -> Result<()> {
for headers_terminated in [false, true] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/start", listener.local_addr()?);
let mut response =
b"HTTP/1.1 302 Found\r\nLocation: /final\r\nSet-Cookie: redirect=one; Path=/\r\n"
.to_vec();
if headers_terminated {
response.extend_from_slice(b"\r\n");
}
let server = tokio::spawn(serve_responses(listener, vec![response]));
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let (head, body) =
fetch_in_mode(&client, Request::get(&url)?.with_single_hop(), "raw").await?;
assert_eq!(head.status, 302);
assert_eq!(head.final_url.path(), "/start");
assert!(!head.redirected);
assert_eq!(
head.cookie_set_reports.len(),
1,
"headers_terminated={headers_terminated}"
);
assert!(body.is_empty());
assert_eq!(server.await??.len(), 1);
assert!(client.shutdown().is_clean());
}
Ok(())
}
#[tokio::test]
async fn header_eof_creates_reusable_empty_cache_entries() -> Result<()> {
for mode in ["html", "raw"] {
let cache_dir = unique_test_cache_dir();
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/cached", listener.local_addr()?);
let server = tokio::spawn(serve_responses(
listener,
vec![b"HTTP/1.1 200 OK\r\nCache-Control: max-age=60\r\nX-Test: cached\r\n".to_vec()],
));
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
for from_cache in [false, true] {
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
assert_eq!(head.from_cache, from_cache, "{mode}");
assert!(head.headers.contains(&("x-test".into(), "cached".into())));
assert!(body.is_empty());
assert!(client.shutdown().is_clean());
}
assert_eq!(server.await??.len(), 1);
fs::remove_dir_all(cache_dir)?;
}
Ok(())
}
#[tokio::test]
async fn header_eof_revalidates_cached_raw_bodies() -> Result<()> {
let cache_dir = unique_test_cache_dir();
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/revalidate", listener.local_addr()?);
let server = tokio::spawn(serve_responses(listener, vec![
b"HTTP/1.1 200 OK\r\nCache-Control: no-cache\r\nETag: \"eof\"\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody".to_vec(),
b"HTTP/1.1 304 Not Modified\r\nCache-Control: max-age=60\r\nETag: \"eof\"\r\nX-Test: updated\r\nSet-Cookie: revalidated=one; Path=/\r\n".to_vec(),
b"HTTP/1.1 200 OK\r\nX-Test: cookie-check\r\n".to_vec(),
]));
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
let (first, body) = fetch_in_mode(&client, Request::get(&url)?, "raw").await?;
assert!(!first.from_cache);
assert_eq!(body, b"body");
let (revalidated, body) = fetch_in_mode(&client, Request::get(&url)?, "raw").await?;
assert_eq!(revalidated.status, 200);
assert!(revalidated.from_cache);
assert_eq!(body, b"body");
assert!(
revalidated
.headers
.contains(&("x-test".into(), "updated".into()))
);
let (_, body) = fetch_in_mode(
&client,
Request::get(&format!("{url}/cookie-check"))?,
"raw",
)
.await?;
assert!(body.is_empty());
let requests = server.await??;
assert_eq!(requests.len(), 3);
assert!(
requests[1]
.to_ascii_lowercase()
.contains("if-none-match: \"eof\"\r\n")
);
assert!(requests[2].contains("Cookie: revalidated=one\r\n"));
assert!(client.shutdown().is_clean());
fs::remove_dir_all(cache_dir)?;
Ok(())
}
#[tokio::test]
async fn header_eof_applies_critical_client_hints_before_publishing() -> Result<()> {
for mode in ["html", "raw"] {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/navigation", listener.local_addr()?);
let server = tokio::spawn(serve_responses(listener, vec![
b"HTTP/1.1 403 Challenge\r\nAccept-CH: Sec-CH-UA-Arch\r\nCritical-CH: Sec-CH-UA-Arch\r\nSet-Cookie: challenge=one; Path=/\r\n".to_vec(),
b"HTTP/1.1 200 OK\r\nX-Test: restarted\r\n".to_vec(),
]));
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
assert_eq!(head.status, 200, "{mode}");
assert_eq!(head.redirect_chain.len(), 1);
let restart = &head.redirect_chain[0];
assert_eq!(restart.from_url, restart.to_url);
assert_eq!(restart.response_extra_info.as_ref().unwrap().status, 403);
assert_eq!(
restart
.response_extra_info
.as_ref()
.unwrap()
.cookie_set_reports
.len(),
1
);
assert!(body.is_empty());
let requests = server.await??;
assert_eq!(requests.len(), 2);
assert!(!requests[0].to_ascii_lowercase().contains("sec-ch-ua-arch:"));
assert!(
requests[1]
.to_ascii_lowercase()
.contains("sec-ch-ua-arch: \"x86\"")
);
assert!(requests[1].contains("Cookie: challenge=one\r\n"));
assert!(client.shutdown().is_clean());
}
Ok(())
}
#[tokio::test]
async fn header_eof_preserves_the_response_size_limit() -> Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/limited", listener.local_addr()?);
let server = tokio::spawn(serve_responses(
listener,
vec![b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\nX-Test: oversized\r\n".to_vec(); 3],
));
let mut config = FetchConfig::default();
config.set_connection_limits(None, None, Some(4));
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
for mode in ["buffered", "html", "raw"] {
let error = fetch_in_mode(&client, Request::get(&url)?, mode)
.await
.unwrap_err();
if mode != "buffered" {
assert!(
error
.to_string()
.contains("response exceeded configured limit of 4 bytes"),
"{mode}: {error:#}"
);
}
}
assert_eq!(server.await??.len(), 3);
assert!(client.shutdown().is_clean());
Ok(())
}
#[tokio::test]
async fn header_eof_does_not_publish_a_cancelled_response() -> Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/cancel", listener.local_addr()?);
let (headers_tx, headers_rx) = oneshot::channel();
let (close_tx, close_rx) = oneshot::channel();
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await?;
let mut request = Vec::new();
while !request.ends_with(b"\r\n\r\n") {
let mut byte = [0];
stream.read_exact(&mut byte).await?;
request.push(byte[0]);
}
stream
.write_all(b"HTTP/1.1 200 OK\r\nX-Test: pending\r\n")
.await?;
let _ = headers_tx.send(());
close_rx.await?;
stream.shutdown().await?;
Ok::<_, anyhow::Error>(())
});
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let cancel = FetchCancelHandle::new();
let request = Request::get(&url)?;
let mut response = Box::pin(client.fetch_raw_stream_with_cancel(request, cancel.clone()));
tokio::select! {
result = &mut response => panic!("response published before headers ended: {result:?}"),
result = headers_rx => result?,
}
cancel.cancel();
let _ = close_tx.send(());
assert!(
tokio::time::timeout(Duration::from_secs(5), response)
.await?
.is_err()
);
server.await??;
assert!(client.shutdown().is_clean());
Ok(())
}
#[tokio::test]
async fn bare_cr_header_eof_matches_chromium_after_final_and_informational_headers() -> Result<()> {
for prefix in [b"".as_slice(), b"HTTP/1.1 100 Continue\r\n\r\n"] {
for byte in [b'X', b':', b'\t', b'\r', 0xff] {
let response = [
prefix,
b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nX-Test: retained\r\n\r",
&[byte, b'\n'],
b"abc",
]
.concat();
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/bare-cr-eof", listener.local_addr()?);
let server = tokio::spawn(serve_responses(listener, vec![response; 3]));
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for mode in ["buffered", "html", "raw"] {
let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode)
.await
.with_context(|| format!("{mode}, prefix={prefix:?}, byte={byte:#x}"))?;
assert_eq!(head.status, 200, "{mode}");
assert!(
head.headers
.contains(&("content-length".into(), b"3".to_vec()))
);
assert!(
head.headers
.contains(&("x-test".into(), b"retained".to_vec()))
);
// Chromium consumes these bytes as an unterminated header block.
assert!(body.is_empty(), "{mode}: {body:?}");
}
assert_eq!(server.await??.len(), 3);
assert!(client.shutdown().is_clean());
}
}
Ok(())
}
#[tokio::test]
async fn header_eof_rejects_nul_in_a_pending_field_on_every_transport() -> Result<()> {
for suffix in [b"X-Test: one\0two".as_slice(), b"\r\0\n"] {
let response = [
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n".as_slice(),
suffix,
]
.concat();
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/nul-eof", listener.local_addr()?);
let server = tokio::spawn(serve_responses(listener, vec![response; 3]));
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for mode in ["buffered", "html", "raw"] {
let request =
Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
assert!(
fetch_in_mode(&client, request, mode).await.is_err(),
"{mode}"
);
}
assert_eq!(server.await??.len(), 3);
assert!(client.shutdown().is_clean());
}
Ok(())
}
#[tokio::test]
async fn bounded_leading_cr_status_lines_work_after_informational_responses() -> Result<()> {
for prefix in ["", "HTTP/1.1 100 Continue\r\n\r\n"] {
for count in 1..=5 {
let response = format!(
"{prefix}{}HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc",
"\r".repeat(count)
);
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/status-slop", listener.local_addr()?);
let server = tokio::spawn(serve_responses(listener, vec![response.into_bytes(); 3]));
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for mode in ["buffered", "html", "raw"] {
let request = Request::get(&url)?
.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
let result = fetch_in_mode(&client, request, mode).await;
if count <= 4 {
let (head, body) = result.with_context(|| format!("{mode}, count={count}"))?;
assert_eq!(head.status, 200);
assert_eq!(body, b"abc");
} else {
assert!(result.is_err(), "{mode}: status slop exceeds four bytes");
}
}
assert_eq!(server.await??.len(), 3);
assert!(client.shutdown().is_clean());
}
}
Ok(())
}
#[tokio::test]
async fn https_header_eof_is_rejected_while_bare_cr_body_bytes_are_preserved() -> Result<()> {
let cases = [
(
b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\n\rX\n".as_slice(),
true,
),
(
b"HTTP/1.1 200 OK\r\nX-Test: truncated\r\n".as_slice(),
false,
),
(
b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\rX\nabc".as_slice(),
false,
),
];
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("https://{}/eof", listener.local_addr()?);
let server = tokio::spawn(async move {
let acceptor = super::support::http11_tls_acceptor();
for (response, _) in cases.iter().cycle().take(cases.len() * 3) {
let (stream, _) =
tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;
let mut stream = acceptor.accept(stream).await?;
let mut request = Vec::new();
while !request.ends_with(b"\r\n\r\n") {
let mut byte = [0];
stream.read_exact(&mut byte).await?;
request.push(byte[0]);
}
stream.write_all(response).await?;
stream.shutdown().await?;
}
Ok::<_, anyhow::Error>(())
});
let mut config = FetchConfig::default();
config.set_tls_verify_host(false);
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
for mode in ["buffered", "html", "raw"] {
for (_, succeeds) in cases {
let request =
Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
let result = fetch_in_mode(&client, request, mode).await;
if succeeds {
let (head, body) =
result.with_context(|| format!("{mode}: complete TLS response"))?;
assert_eq!(head.status, 200);
assert_eq!(body, b"\rX\n");
} else {
assert!(
result.is_err(),
"{mode}: HTTPS must reject incomplete headers"
);
}
}
}
server.await??;
assert!(client.shutdown().is_clean());
Ok(())
}