Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use std::{
collections::VecDeque,
io::{Read, Write},
net::{TcpListener, TcpStream},
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
mpsc as std_mpsc,
},
thread,
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use crate::FetchClientHandle;
use parking_lot::Mutex;
pub(super) fn spawn_socks5_proxy(
upstream_addr: std::net::SocketAddr,
) -> (
String,
std_mpsc::Receiver<(String, u16)>,
thread::JoinHandle<()>,
) {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind fetch SOCKS proxy");
let addr = listener.local_addr().expect("fetch SOCKS proxy address");
let (request_tx, request_rx) = std_mpsc::channel();
let handle = thread::spawn(move || {
let (mut client, _) = listener.accept().expect("accept fetch SOCKS client");
let mut greeting = [0u8; 2];
client
.read_exact(&mut greeting)
.expect("read fetch SOCKS greeting");
assert_eq!(greeting[0], 5);
let mut methods = vec![0; usize::from(greeting[1])];
client
.read_exact(&mut methods)
.expect("read fetch SOCKS methods");
assert!(methods.contains(&0), "SOCKS client must offer no-auth");
client
.write_all(&[5, 0])
.expect("write fetch SOCKS method selection");
let mut request_head = [0u8; 4];
client
.read_exact(&mut request_head)
.expect("read fetch SOCKS request head");
assert_eq!(&request_head[..3], &[5, 1, 0]);
let host = match request_head[3] {
3 => {
let mut length = [0u8; 1];
client
.read_exact(&mut length)
.expect("read fetch SOCKS domain length");
let mut domain = vec![0; usize::from(length[0])];
client
.read_exact(&mut domain)
.expect("read fetch SOCKS domain");
String::from_utf8(domain).expect("fetch SOCKS domain should be UTF-8")
}
atyp => panic!("SOCKS5 must send a domain target, got address type {atyp}"),
};
let mut port = [0u8; 2];
client
.read_exact(&mut port)
.expect("read fetch SOCKS target port");
let port = u16::from_be_bytes(port);
let _ = request_tx.send((host, port));
let upstream = TcpStream::connect(upstream_addr).expect("connect fetch SOCKS upstream");
client
.write_all(&[5, 0, 0, 1, 127, 0, 0, 1, 0, 0])
.expect("write fetch SOCKS success response");
relay_proxy_streams(client, upstream);
});
(format!("socks5://{addr}"), request_rx, handle)
}
pub(super) fn spawn_socks4_proxy(
upstream_addr: std::net::SocketAddr,
) -> (
String,
std_mpsc::Receiver<(String, u16)>,
thread::JoinHandle<()>,
) {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind fetch SOCKS4 proxy");
let addr = listener.local_addr().expect("fetch SOCKS4 proxy address");
let (request_tx, request_rx) = std_mpsc::channel();
let handle = thread::spawn(move || {
let (mut client, _) = listener.accept().expect("accept fetch SOCKS4 client");
let mut request_head = [0u8; 8];
client
.read_exact(&mut request_head)
.expect("read fetch SOCKS4 request head");
assert_eq!(&request_head[..2], &[4, 1]);
assert_eq!(&request_head[4..7], &[0, 0, 0]);
assert_ne!(request_head[7], 0, "SOCKS4a must use a synthetic IP");
let port = u16::from_be_bytes([request_head[2], request_head[3]]);
let _user_id = read_socks4_string(&mut client, "user id");
let domain = read_socks4_string(&mut client, "domain");
let host = String::from_utf8(domain).expect("fetch SOCKS4 domain should be UTF-8");
let _ = request_tx.send((host, port));
let upstream = TcpStream::connect(upstream_addr).expect("connect fetch SOCKS4 upstream");
client
.write_all(&[0, 0x5a, 0, 0, 0, 0, 0, 0])
.expect("write fetch SOCKS4 success response");
relay_proxy_streams(client, upstream);
});
(format!("socks4://{addr}"), request_rx, handle)
}
fn read_socks4_string(stream: &mut TcpStream, label: &str) -> Vec<u8> {
let mut value = Vec::new();
loop {
let mut byte = [0u8; 1];
stream
.read_exact(&mut byte)
.unwrap_or_else(|error| panic!("read fetch SOCKS4 {label}: {error}"));
if byte[0] == 0 {
return value;
}
assert!(value.len() < 255, "fetch SOCKS4 {label} is too long");
value.push(byte[0]);
}
}
fn relay_proxy_streams(mut client: TcpStream, mut upstream: TcpStream) {
let mut client_read = client.try_clone().expect("clone fetch proxy client");
let mut upstream_write = upstream.try_clone().expect("clone fetch proxy upstream");
let upload = thread::spawn(move || {
let _ = std::io::copy(&mut client_read, &mut upstream_write);
});
let _ = std::io::copy(&mut upstream, &mut client);
let _ = client.shutdown(std::net::Shutdown::Both);
let _ = upstream.shutdown(std::net::Shutdown::Both);
let _ = upload.join();
}
pub(super) struct ScriptedH2Server {
addr: std::net::SocketAddr,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
connection_stream_counts: Arc<Mutex<Vec<usize>>>,
shutdown_tx: std_mpsc::Sender<()>,
join_handle: Option<thread::JoinHandle<()>>,
}
pub(super) struct Http2ProtocolFallbackServer {
addr: std::net::SocketAddr,
h2_hits: Arc<AtomicUsize>,
http11_hits: Arc<AtomicUsize>,
shutdown_tx: std_mpsc::Sender<()>,
join_handle: Option<thread::JoinHandle<()>>,
}
pub(super) struct EmptyHttpHttpsUpgradeServer {
addr: std::net::SocketAddr,
http_hits: Arc<AtomicUsize>,
https_hits: Arc<AtomicUsize>,
shutdown_tx: std_mpsc::Sender<()>,
join_handle: Option<thread::JoinHandle<()>>,
}
pub(super) struct ScriptedHttps11Server {
addr: std::net::SocketAddr,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
request_heads: Arc<Mutex<Vec<String>>>,
shutdown_tx: std_mpsc::Sender<()>,
join_handle: Option<thread::JoinHandle<()>>,
}
impl ScriptedHttps11Server {
pub(super) fn spawn(responses: Vec<ScriptedResponse>) -> Self {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener
.set_nonblocking(true)
.expect("test https listener should be nonblocking");
let addr = listener.local_addr().unwrap();
let hits = Arc::new(AtomicUsize::new(0));
let requests = Arc::new(Mutex::new(Vec::new()));
let request_heads = Arc::new(Mutex::new(Vec::new()));
let (shutdown_tx, shutdown_rx) = std_mpsc::channel();
let responses = Arc::new(Mutex::new(VecDeque::from(responses)));
let hits_for_thread = Arc::clone(&hits);
let requests_for_thread = Arc::clone(&requests);
let request_heads_for_thread = Arc::clone(&request_heads);
let join_handle = thread::spawn(move || {
run_https11_server(
listener,
shutdown_rx,
hits_for_thread,
requests_for_thread,
request_heads_for_thread,
responses,
);
});
Self {
addr,
hits,
requests,
request_heads,
shutdown_tx,
join_handle: Some(join_handle),
}
}
pub(super) fn url(&self) -> String {
format!("https://{}/cache", self.addr)
}
pub(super) fn url_path(&self, path: &str) -> String {
let path = path.strip_prefix('/').unwrap_or(path);
format!("https://{}/{}", self.addr, path)
}
pub(super) fn hits(&self) -> usize {
self.hits.load(Ordering::SeqCst)
}
pub(super) fn requests(&self) -> Vec<String> {
self.requests.lock().clone()
}
pub(super) fn request_heads(&self) -> Vec<String> {
self.request_heads.lock().clone()
}
pub(super) fn shutdown(mut self) {
let _ = self.shutdown_tx.send(());
if let Some(join_handle) = self.join_handle.take() {
let _ = join_handle.join();
}
}
}
impl ScriptedH2Server {
pub(super) fn spawn(responses: Vec<ScriptedResponse>) -> Self {
Self::spawn_with_gate(responses, None)
}
pub(super) fn spawn_with_response_barrier(
responses: Vec<ScriptedResponse>,
barrier: Arc<tokio::sync::Barrier>,
) -> Self {
Self::spawn_with_gate(responses, Some(barrier))
}
fn spawn_with_gate(
responses: Vec<ScriptedResponse>,
barrier: Option<Arc<tokio::sync::Barrier>>,
) -> Self {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener
.set_nonblocking(true)
.expect("test h2 listener should be nonblocking");
let addr = listener.local_addr().unwrap();
let hits = Arc::new(AtomicUsize::new(0));
let requests = Arc::new(Mutex::new(Vec::new()));
let connection_stream_counts = Arc::new(Mutex::new(Vec::new()));
let (shutdown_tx, shutdown_rx) = std_mpsc::channel();
let responses = Arc::new(Mutex::new(VecDeque::from(responses)));
let hits_for_thread = Arc::clone(&hits);
let requests_for_thread = Arc::clone(&requests);
let connection_stream_counts_for_thread = Arc::clone(&connection_stream_counts);
let join_handle = thread::spawn(move || {
run_h2_server(
listener,
shutdown_rx,
hits_for_thread,
requests_for_thread,
connection_stream_counts_for_thread,
responses,
barrier,
);
});
Self {
addr,
hits,
requests,
connection_stream_counts,
shutdown_tx,
join_handle: Some(join_handle),
}
}
pub(super) fn url_path(&self, path: &str) -> String {
let path = path.strip_prefix('/').unwrap_or(path);
format!("https://{}/{}", self.addr, path)
}
pub(super) fn hits(&self) -> usize {
self.hits.load(Ordering::SeqCst)
}
pub(super) fn requests(&self) -> Vec<String> {
self.requests.lock().clone()
}
pub(super) fn connection_stream_counts(&self) -> Vec<usize> {
self.connection_stream_counts.lock().clone()
}
pub(super) fn shutdown(mut self) {
let _ = self.shutdown_tx.send(());
if let Some(join_handle) = self.join_handle.take() {
let _ = join_handle.join();
}
}
}
impl Http2ProtocolFallbackServer {
pub(super) fn spawn() -> Self {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener
.set_nonblocking(true)
.expect("test protocol fallback listener should be nonblocking");
let addr = listener.local_addr().unwrap();
let h2_hits = Arc::new(AtomicUsize::new(0));
let http11_hits = Arc::new(AtomicUsize::new(0));
let (shutdown_tx, shutdown_rx) = std_mpsc::channel();
let h2_hits_for_thread = Arc::clone(&h2_hits);
let http11_hits_for_thread = Arc::clone(&http11_hits);
let join_handle = thread::spawn(move || {
run_http2_protocol_fallback_server(
listener,
shutdown_rx,
h2_hits_for_thread,
http11_hits_for_thread,
);
});
Self {
addr,
h2_hits,
http11_hits,
shutdown_tx,
join_handle: Some(join_handle),
}
}
pub(super) fn url(&self) -> String {
format!("https://{}/fallback", self.addr)
}
pub(super) fn h2_hits(&self) -> usize {
self.h2_hits.load(Ordering::SeqCst)
}
pub(super) fn http11_hits(&self) -> usize {
self.http11_hits.load(Ordering::SeqCst)
}
pub(super) fn shutdown(mut self) {
let _ = self.shutdown_tx.send(());
if let Some(join_handle) = self.join_handle.take() {
let _ = join_handle.join();
}
}
}
impl EmptyHttpHttpsUpgradeServer {
pub(super) fn spawn() -> Self {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener
.set_nonblocking(true)
.expect("test empty HTTP upgrade listener should be nonblocking");
let addr = listener.local_addr().unwrap();
let http_hits = Arc::new(AtomicUsize::new(0));
let https_hits = Arc::new(AtomicUsize::new(0));
let (shutdown_tx, shutdown_rx) = std_mpsc::channel();
let http_hits_for_thread = Arc::clone(&http_hits);
let https_hits_for_thread = Arc::clone(&https_hits);
let join_handle = thread::spawn(move || {
run_empty_http_https_upgrade_server(
listener,
shutdown_rx,
http_hits_for_thread,
https_hits_for_thread,
);
});
Self {
addr,
http_hits,
https_hits,
shutdown_tx,
join_handle: Some(join_handle),
}
}
pub(super) fn url(&self) -> String {
format!("http://upgrade.example.org:{}/fallback", self.addr.port())
}
pub(super) fn resolve_entry(&self) -> String {
format!("upgrade.example.org:{}:127.0.0.1", self.addr.port())
}
pub(super) fn http_hits(&self) -> usize {
self.http_hits.load(Ordering::SeqCst)
}
pub(super) fn https_hits(&self) -> usize {
self.https_hits.load(Ordering::SeqCst)
}
pub(super) fn shutdown(mut self) {
let _ = self.shutdown_tx.send(());
if let Some(join_handle) = self.join_handle.take() {
let _ = join_handle.join();
}
}
}
pub(super) struct ScriptedHttpServer {
addr: std::net::SocketAddr,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
shutdown_tx: std_mpsc::Sender<()>,
join_handle: Option<thread::JoinHandle<()>>,
handler_handles: Arc<Mutex<Vec<thread::JoinHandle<()>>>>,
}
fn run_h2_server(
listener: std::net::TcpListener,
shutdown_rx: std_mpsc::Receiver<()>,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
connection_stream_counts: Arc<Mutex<Vec<usize>>>,
responses: Arc<Mutex<VecDeque<ScriptedResponse>>>,
barrier: Option<Arc<tokio::sync::Barrier>>,
) {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.expect("h2 test runtime");
rt.block_on(async move {
let listener = tokio::net::TcpListener::from_std(listener).expect("tokio h2 listener");
let tls_acceptor = h2_tls_acceptor();
loop {
if shutdown_rx.try_recv().is_ok() {
break;
}
let accepted = tokio::time::timeout(Duration::from_millis(25), listener.accept()).await;
let Ok(Ok((stream, _))) = accepted else {
continue;
};
let tls_acceptor = tls_acceptor.clone();
let hits = Arc::clone(&hits);
let requests = Arc::clone(&requests);
let connection_stream_counts = Arc::clone(&connection_stream_counts);
let responses = Arc::clone(&responses);
let barrier = barrier.clone();
tokio::spawn(async move {
let Ok(stream) = tls_acceptor.accept(stream).await else {
return;
};
if stream.get_ref().1.alpn_protocol() != Some(b"h2") {
return;
}
let Ok(mut connection) = h2::server::handshake(stream).await else {
return;
};
let connection_index = {
let mut counts = connection_stream_counts.lock();
counts.push(0);
counts.len() - 1
};
while let Some(result) = connection.accept().await {
let Ok((request, mut respond)) = result else {
break;
};
{
let mut counts = connection_stream_counts.lock();
counts[connection_index] += 1;
}
requests.lock().push(request.uri().path().to_owned());
let _ = hits.fetch_add(1, Ordering::SeqCst) + 1;
let response_spec = {
let mut responses = responses.lock();
responses
.pop_front()
.or_else(|| responses.back().cloned())
.expect("scripted h2 responses should not be empty")
};
let barrier = barrier.clone();
tokio::spawn(async move {
if let Some(barrier) = barrier {
barrier.wait().await;
}
if response_spec.delay_ms > 0 {
tokio::time::sleep(Duration::from_millis(response_spec.delay_ms)).await;
}
let response = http::Response::builder()
.status(response_spec.status)
.header("content-type", "text/plain")
.body(())
.expect("h2 test response should build");
let Ok(mut send) = respond.send_response(response, false) else {
return;
};
let _ = send.send_data(response_spec.body.into(), true);
});
}
});
}
});
}
fn run_http2_protocol_fallback_server(
listener: std::net::TcpListener,
shutdown_rx: std_mpsc::Receiver<()>,
h2_hits: Arc<AtomicUsize>,
http11_hits: Arc<AtomicUsize>,
) {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.expect("protocol fallback test runtime");
rt.block_on(async move {
let listener =
tokio::net::TcpListener::from_std(listener).expect("tokio protocol fallback listener");
let tls_acceptor = http2_and_http11_tls_acceptor();
loop {
if shutdown_rx.try_recv().is_ok() {
break;
}
let accepted = tokio::time::timeout(Duration::from_millis(25), listener.accept()).await;
let Ok(Ok((stream, _))) = accepted else {
continue;
};
let tls_acceptor = tls_acceptor.clone();
let h2_hits = Arc::clone(&h2_hits);
let http11_hits = Arc::clone(&http11_hits);
tokio::spawn(async move {
let Ok(mut stream) = tls_acceptor.accept(stream).await else {
return;
};
match stream.get_ref().1.alpn_protocol() {
Some(b"h2") => {
h2_hits.fetch_add(1, Ordering::SeqCst);
send_malformed_http2_response(&mut stream).await;
}
Some(b"http/1.1") => {
http11_hits.fetch_add(1, Ordering::SeqCst);
send_http11_fallback_response(&mut stream).await;
}
_ => {}
}
});
}
});
}
fn run_empty_http_https_upgrade_server(
listener: std::net::TcpListener,
shutdown_rx: std_mpsc::Receiver<()>,
http_hits: Arc<AtomicUsize>,
https_hits: Arc<AtomicUsize>,
) {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.expect("empty HTTP upgrade test runtime");
rt.block_on(async move {
let listener = tokio::net::TcpListener::from_std(listener)
.expect("tokio empty HTTP upgrade listener");
let tls_acceptor = http11_tls_acceptor();
loop {
if shutdown_rx.try_recv().is_ok() {
break;
}
let accepted = tokio::time::timeout(Duration::from_millis(25), listener.accept()).await;
let Ok(Ok((mut stream, _))) = accepted else {
continue;
};
let tls_acceptor = tls_acceptor.clone();
let http_hits = Arc::clone(&http_hits);
let https_hits = Arc::clone(&https_hits);
tokio::spawn(async move {
let mut first_byte = [0; 1];
let Ok(1) = stream.peek(&mut first_byte).await else {
return;
};
if first_byte[0] == 0x16 {
https_hits.fetch_add(1, Ordering::SeqCst);
let Ok(mut stream) = tls_acceptor.accept(stream).await else {
return;
};
read_http_request_head(&mut stream).await;
let body = b"https upgrade fallback";
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
if tokio::io::AsyncWriteExt::write_all(&mut stream, response.as_bytes())
.await
.is_ok()
{
let _ = tokio::io::AsyncWriteExt::write_all(&mut stream, body).await;
let _ = tokio::io::AsyncWriteExt::flush(&mut stream).await;
}
} else {
http_hits.fetch_add(1, Ordering::SeqCst);
read_http_request_head(&mut stream).await;
let _ = tokio::io::AsyncWriteExt::shutdown(&mut stream).await;
}
});
}
});
}
async fn read_http_request_head<S>(stream: &mut S)
where
S: tokio::io::AsyncRead + Unpin,
{
let mut request = Vec::new();
let mut buf = [0; 1024];
loop {
let read = match tokio::io::AsyncReadExt::read(stream, &mut buf).await {
Ok(0) | Err(_) => return,
Ok(read) => read,
};
request.extend_from_slice(&buf[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
return;
}
}
}
async fn send_malformed_http2_response(
stream: &mut tokio_rustls::server::TlsStream<tokio::net::TcpStream>,
) {
const CLIENT_PREFACE: &[u8] = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n";
const SETTINGS: &[u8] = &[0, 0, 0, 4, 0, 0, 0, 0, 0];
const SETTINGS_ACK: &[u8] = &[0, 0, 0, 4, 1, 0, 0, 0, 0];
const INVALID_RESPONSE_HEADERS: &[u8] = &[
0x88, // :status: 200
0x00, 0x07, b'u', b'p', b'g', b'r', b'a', b'd', b'e', 0x02, b'h', b'2',
];
let mut preface = [0; 24];
if tokio::io::AsyncReadExt::read_exact(stream, &mut preface)
.await
.is_err()
|| preface != CLIENT_PREFACE
{
return;
}
if tokio::io::AsyncWriteExt::write_all(stream, SETTINGS)
.await
.is_err()
{
return;
}
let mut request_stream_id = None;
loop {
let mut frame_header = [0; 9];
if tokio::io::AsyncReadExt::read_exact(stream, &mut frame_header)
.await
.is_err()
{
return;
}
let payload_len = usize::from(frame_header[0]) << 16
| usize::from(frame_header[1]) << 8
| usize::from(frame_header[2]);
let frame_type = frame_header[3];
let flags = frame_header[4];
let stream_id = u32::from_be_bytes([
frame_header[5] & 0x7f,
frame_header[6],
frame_header[7],
frame_header[8],
]);
let mut payload = vec![0; payload_len];
if tokio::io::AsyncReadExt::read_exact(stream, &mut payload)
.await
.is_err()
{
return;
}
if frame_type == 4 && flags & 0x1 == 0 {
if tokio::io::AsyncWriteExt::write_all(stream, SETTINGS_ACK)
.await
.is_err()
{
return;
}
} else if frame_type == 1 && stream_id != 0 {
request_stream_id = Some(stream_id);
if flags & 0x4 != 0 {
break;
}
} else if frame_type == 9 && request_stream_id == Some(stream_id) && flags & 0x4 != 0 {
break;
}
}
let stream_id = request_stream_id.expect("request headers should identify a stream");
let payload_len = INVALID_RESPONSE_HEADERS.len();
let mut response = Vec::with_capacity(9 + payload_len);
response.extend_from_slice(&[
((payload_len >> 16) & 0xff) as u8,
((payload_len >> 8) & 0xff) as u8,
(payload_len & 0xff) as u8,
1,
0x5,
]);
response.extend_from_slice(&(stream_id & 0x7fff_ffff).to_be_bytes());
response.extend_from_slice(INVALID_RESPONSE_HEADERS);
let _ = tokio::io::AsyncWriteExt::write_all(stream, &response).await;
let _ = tokio::io::AsyncWriteExt::flush(stream).await;
// Keep the connection alive until libcurl has classified the malformed
// response. A wall-clock delay is racy under load and can turn the
// intended HTTP/2 protocol error into a peer-reset error instead.
wait_for_http2_client_error(stream, stream_id).await;
}
async fn wait_for_http2_client_error(
stream: &mut tokio_rustls::server::TlsStream<tokio::net::TcpStream>,
request_stream_id: u32,
) {
loop {
let mut frame_header = [0; 9];
if tokio::io::AsyncReadExt::read_exact(stream, &mut frame_header)
.await
.is_err()
{
break;
}
let payload_len = usize::from(frame_header[0]) << 16
| usize::from(frame_header[1]) << 8
| usize::from(frame_header[2]);
let frame_type = frame_header[3];
let stream_id = u32::from_be_bytes([
frame_header[5] & 0x7f,
frame_header[6],
frame_header[7],
frame_header[8],
]);
let mut payload = vec![0; payload_len];
if tokio::io::AsyncReadExt::read_exact(stream, &mut payload)
.await
.is_err()
{
break;
}
const RST_STREAM: u8 = 3;
const GOAWAY: u8 = 7;
if (frame_type == RST_STREAM && stream_id == request_stream_id) || frame_type == GOAWAY {
break;
}
}
let _ = tokio::io::AsyncWriteExt::shutdown(stream).await;
}
async fn send_http11_fallback_response(
stream: &mut tokio_rustls::server::TlsStream<tokio::net::TcpStream>,
) {
let mut request = Vec::new();
let mut buf = [0; 1024];
loop {
let read = match tokio::io::AsyncReadExt::read(stream, &mut buf).await {
Ok(0) | Err(_) => return,
Ok(read) => read,
};
request.extend_from_slice(&buf[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let body = b"http1 fallback";
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nUpgrade: h2\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
if tokio::io::AsyncWriteExt::write_all(stream, response.as_bytes())
.await
.is_ok()
{
let _ = tokio::io::AsyncWriteExt::write_all(stream, body).await;
let _ = tokio::io::AsyncWriteExt::flush(stream).await;
// Send close_notify, then wait for the peer's half of the TLS close
// handshake. Dropping a socket with unread peer data can turn this
// otherwise successful response into an observable TCP reset.
let _ = tokio::io::AsyncWriteExt::shutdown(stream).await;
let mut peer_tail = [0; 64];
let _ = tokio::time::timeout(Duration::from_secs(1), async {
loop {
match tokio::io::AsyncReadExt::read(stream, &mut peer_tail).await {
Ok(0) | Err(_) => break,
Ok(_) => {}
}
}
})
.await;
}
}
fn h2_tls_acceptor() -> tokio_rustls::TlsAcceptor {
use tokio_rustls::rustls::{
ServerConfig,
pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer},
};
let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()])
.expect("self-signed h2 certificate");
let cert_der = CertificateDer::from(cert.cert.der().to_vec());
let key_der = PrivateKeyDer::from(PrivatePkcs8KeyDer::from(cert.key_pair.serialize_der()));
let mut config = ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(vec![cert_der], key_der)
.expect("h2 tls config");
config.alpn_protocols = vec![b"h2".to_vec()];
tokio_rustls::TlsAcceptor::from(Arc::new(config))
}
fn http2_and_http11_tls_acceptor() -> tokio_rustls::TlsAcceptor {
use tokio_rustls::rustls::{
ServerConfig,
pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer},
};
let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()])
.expect("self-signed protocol fallback certificate");
let cert_der = CertificateDer::from(cert.cert.der().to_vec());
let key_der = PrivateKeyDer::from(PrivatePkcs8KeyDer::from(cert.key_pair.serialize_der()));
let mut config = ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(vec![cert_der], key_der)
.expect("protocol fallback tls config");
config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
tokio_rustls::TlsAcceptor::from(Arc::new(config))
}
pub(super) fn http11_tls_acceptor() -> tokio_rustls::TlsAcceptor {
use tokio_rustls::rustls::{
ServerConfig,
pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer},
};
let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()])
.expect("self-signed http/1.1 certificate");
let cert_der = CertificateDer::from(cert.cert.der().to_vec());
let key_der = PrivateKeyDer::from(PrivatePkcs8KeyDer::from(cert.key_pair.serialize_der()));
let mut config = ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(vec![cert_der], key_der)
.expect("http/1.1 tls config");
config.alpn_protocols = vec![b"http/1.1".to_vec()];
tokio_rustls::TlsAcceptor::from(Arc::new(config))
}
fn run_https11_server(
listener: std::net::TcpListener,
shutdown_rx: std_mpsc::Receiver<()>,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
request_heads: Arc<Mutex<Vec<String>>>,
responses: Arc<Mutex<VecDeque<ScriptedResponse>>>,
) {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_io()
.enable_time()
.build()
.expect("http/1.1 tls test runtime");
rt.block_on(async move {
let listener = tokio::net::TcpListener::from_std(listener).expect("tokio tls listener");
let tls_acceptor = http11_tls_acceptor();
loop {
if shutdown_rx.try_recv().is_ok() {
break;
}
let accepted = tokio::time::timeout(Duration::from_millis(25), listener.accept()).await;
let Ok(Ok((stream, _))) = accepted else {
continue;
};
let tls_acceptor = tls_acceptor.clone();
let hits = Arc::clone(&hits);
let requests = Arc::clone(&requests);
let request_heads = Arc::clone(&request_heads);
let responses = Arc::clone(&responses);
tokio::spawn(async move {
let Ok(mut stream) = tls_acceptor.accept(stream).await else {
return;
};
let mut request = Vec::new();
let mut buf = [0; 1024];
loop {
let read = match tokio::io::AsyncReadExt::read(&mut stream, &mut buf).await {
Ok(0) | Err(_) => return,
Ok(read) => read,
};
request.extend_from_slice(&buf[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
if let Some(path) = request_path_from_head(&request) {
requests.lock().push(path);
}
request_heads
.lock()
.push(String::from_utf8_lossy(&request).into_owned());
let _ = hits.fetch_add(1, Ordering::SeqCst) + 1;
let response_spec = {
let mut responses = responses.lock();
responses
.pop_front()
.or_else(|| responses.back().cloned())
.expect("scripted https responses should not be empty")
};
if response_spec.delay_ms > 0 {
tokio::time::sleep(Duration::from_millis(response_spec.delay_ms)).await;
}
let response = scripted_http_response_bytes(&response_spec);
let _ = tokio::io::AsyncWriteExt::write_all(&mut stream, &response).await;
let _ = tokio::io::AsyncWriteExt::flush(&mut stream).await;
if response_spec.hold_open_ms > 0 {
tokio::time::sleep(Duration::from_millis(response_spec.hold_open_ms)).await;
}
});
}
});
}
impl ScriptedHttpServer {
pub(super) fn spawn(responses: Vec<ScriptedResponse>) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener
.set_nonblocking(true)
.expect("test listener should be nonblocking");
let addr = listener.local_addr().unwrap();
let hits = Arc::new(AtomicUsize::new(0));
let hits_for_thread = Arc::clone(&hits);
let requests = Arc::new(Mutex::new(Vec::new()));
let requests_for_thread = Arc::clone(&requests);
let (shutdown_tx, shutdown_rx) = std_mpsc::channel();
let responses = Arc::new(Mutex::new(VecDeque::from(responses)));
let responses_for_thread = Arc::clone(&responses);
let handler_handles = Arc::new(Mutex::new(Vec::new()));
let handler_handles_for_thread = Arc::clone(&handler_handles);
let join_handle = thread::spawn(move || {
loop {
if shutdown_rx.try_recv().is_ok() {
break;
}
match listener.accept() {
Ok((stream, _)) => {
let hits_for_handler = Arc::clone(&hits_for_thread);
let requests_for_handler = Arc::clone(&requests_for_thread);
let responses_for_handler = Arc::clone(&responses_for_thread);
let handle = thread::spawn(move || {
handle_scripted_connection(
stream,
hits_for_handler,
requests_for_handler,
responses_for_handler,
);
});
handler_handles_for_thread.lock().push(handle);
}
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(10));
}
Err(_) => break,
}
}
});
Self {
addr,
hits,
requests,
shutdown_tx,
join_handle: Some(join_handle),
handler_handles,
}
}
pub(super) fn url(&self) -> String {
format!("http://{}/cache", self.addr)
}
pub(super) fn origin(&self) -> String {
format!("http://{}", self.addr)
}
pub(super) fn url_path(&self, path: &str) -> String {
let path = path.strip_prefix('/').unwrap_or(path);
format!("http://{}/{}", self.addr, path)
}
pub(super) fn hits(&self) -> usize {
self.hits.load(Ordering::SeqCst)
}
pub(super) fn requests(&self) -> Vec<String> {
self.requests.lock().clone()
}
pub(super) fn shutdown(mut self) {
let _ = self.shutdown_tx.send(());
if let Some(join_handle) = self.join_handle.take() {
let _ = join_handle.join();
}
let mut handler_handles = self.handler_handles.lock();
for handle in handler_handles.drain(..) {
let _ = handle.join();
}
}
}
#[test]
fn scripted_server_waits_for_request_on_nonblocking_accepted_socket() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
client
.set_read_timeout(Some(Duration::from_secs(2)))
.unwrap();
let (stream, _) = listener.accept().unwrap();
// macOS may inherit this mode from the listening socket.
stream.set_nonblocking(true).unwrap();
let hits = Arc::new(AtomicUsize::new(0));
let requests = Arc::new(Mutex::new(Vec::new()));
let responses = Arc::new(Mutex::new(VecDeque::from([ScriptedResponse::ok("ready")])));
let (done_tx, done_rx) = std_mpsc::channel();
let handler = {
let hits = Arc::clone(&hits);
let requests = Arc::clone(&requests);
thread::spawn(move || {
handle_scripted_connection(stream, hits, requests, responses);
done_tx.send(()).unwrap();
})
};
assert!(
matches!(
done_rx.recv_timeout(Duration::from_millis(100)),
Err(std_mpsc::RecvTimeoutError::Timeout)
),
"a connection without HTTP bytes must not receive a scripted response"
);
client
.write_all(b"GET /delayed HTTP/1.1\r\nHost: localhost\r\n\r\n")
.unwrap();
let mut response = String::new();
client.read_to_string(&mut response).unwrap();
handler.join().unwrap();
assert!(response.ends_with("ready"));
assert_eq!(hits.load(Ordering::SeqCst), 1);
assert!(requests.lock()[0].starts_with("GET /delayed HTTP/1.1\r\n"));
}
#[test]
fn scripted_server_does_not_consume_response_for_closed_unused_connection() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
let (stream, _) = listener.accept().unwrap();
drop(client);
let hits = Arc::new(AtomicUsize::new(0));
let requests = Arc::new(Mutex::new(Vec::new()));
let responses = Arc::new(Mutex::new(VecDeque::from([ScriptedResponse::ok("ready")])));
handle_scripted_connection(
stream,
Arc::clone(&hits),
Arc::clone(&requests),
Arc::clone(&responses),
);
assert_eq!(hits.load(Ordering::SeqCst), 0);
assert!(requests.lock().is_empty());
assert_eq!(responses.lock().len(), 1);
}
fn handle_scripted_connection(
mut stream: std::net::TcpStream,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
responses: Arc<Mutex<VecDeque<ScriptedResponse>>>,
) {
stream
.set_nonblocking(false)
.expect("scripted HTTP connection should use blocking reads");
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.expect("scripted HTTP connection should have a bounded read");
let mut request = [0; 1024];
let bytes_read = stream
.read(&mut request)
.expect("read scripted HTTP request");
if bytes_read == 0 {
return;
}
let request_text = String::from_utf8_lossy(&request[..bytes_read]).into_owned();
requests.lock().push(request_text);
let _ = hits.fetch_add(1, Ordering::SeqCst) + 1;
let response_spec = {
let mut responses = responses.lock();
responses
.pop_front()
.or_else(|| responses.back().cloned())
.expect("scripted responses should not be empty")
};
if response_spec.delay_ms > 0 {
thread::sleep(Duration::from_millis(response_spec.delay_ms));
}
let response = scripted_http_response_bytes(&response_spec);
let _ = stream.write_all(&response);
let _ = stream.flush();
if response_spec.hold_open_ms > 0 {
thread::sleep(Duration::from_millis(response_spec.hold_open_ms));
}
}
fn scripted_http_response_bytes(response_spec: &ScriptedResponse) -> Vec<u8> {
let body = response_spec.body.clone();
let mut extra_headers = String::new();
let has_connection_header = response_spec
.headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("connection"));
for (name, value) in &response_spec.headers {
extra_headers.push_str(name);
extra_headers.push_str(": ");
extra_headers.push_str(value);
extra_headers.push_str("\r\n");
}
if response_spec.close_connection && !has_connection_header {
extra_headers.push_str("Connection: close\r\n");
}
format!(
"HTTP/{} {} {}\r\nContent-Length: {}\r\nContent-Type: text/plain\r\n{}\r\n{}",
response_spec.http_version,
response_spec.status,
response_spec.reason,
body.len(),
extra_headers,
body
)
.into_bytes()
}
fn request_path_from_head(request: &[u8]) -> Option<String> {
let request = std::str::from_utf8(request).ok()?;
request
.lines()
.next()
.and_then(|line| line.split_whitespace().nth(1))
.map(str::to_owned)
}
#[derive(Clone)]
pub(super) struct ScriptedResponse {
http_version: &'static str,
status: u16,
reason: &'static str,
headers: Vec<(String, String)>,
body: String,
delay_ms: u64,
hold_open_ms: u64,
close_connection: bool,
}
impl ScriptedResponse {
pub(super) fn ok(body: &str) -> Self {
Self {
http_version: "1.1",
status: 200,
reason: "OK",
headers: Vec::new(),
body: body.to_owned(),
delay_ms: 0,
hold_open_ms: 0,
close_connection: true,
}
}
pub(super) fn status(status: u16, reason: &'static str) -> Self {
Self {
http_version: "1.1",
status,
reason,
headers: Vec::new(),
body: String::new(),
delay_ms: 0,
hold_open_ms: 0,
close_connection: true,
}
}
pub(super) fn with_body(mut self, body: &str) -> Self {
self.body = body.to_owned();
self
}
pub(super) fn with_http_version(mut self, http_version: &'static str) -> Self {
self.http_version = http_version;
self
}
pub(super) fn with_header(mut self, name: &str, value: &str) -> Self {
self.headers.push((name.to_owned(), value.to_owned()));
self
}
pub(super) fn with_delay_ms(mut self, delay_ms: u64) -> Self {
self.delay_ms = delay_ms;
self
}
pub(super) fn with_hold_open_ms(mut self, hold_open_ms: u64) -> Self {
self.hold_open_ms = hold_open_ms;
self.close_connection = false;
self
}
}
pub(super) fn wait_for_runtime_owner_count(client: &FetchClientHandle, expected: usize) {
let deadline = Instant::now() + Duration::from_secs(1);
while Instant::now() < deadline {
if client.runtime_owner_count_for_testing() == expected {
return;
}
thread::sleep(Duration::from_millis(10));
}
assert_eq!(client.runtime_owner_count_for_testing(), expected);
}
pub(super) fn unique_test_cache_dir() -> std::path::PathBuf {
let unique = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
std::env::temp_dir().join(format!(
"moli-fetch-cache-test-{}-{unique}",
std::process::id()
))
}