Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
test(curl): cover shared HTTP and WebSocket workloads
ldm04e63c1e8c69b
diff --git a/Cargo.lock b/Cargo.lockindex f9761ab17..f0baf4b97 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2548,6 +2548,7 @@ dependencies = ["strum","tokio","tokio-rustls",+ "tokio-tungstenite","tracing","url","wildcard",diff --git a/moli-curl/README.md b/moli-curl/README.mdindex e51f4222a..9a1cb1728 100644--- a/moli-curl/README.md+++ b/moli-curl/README.md@@ -77,3 +77,49 @@ and record several runs. Avoid concurrent builds or test suites while measuring.CPU/throughput measurements are observations, not CI pass thresholds. DeterministicCI regressions cover idle receive attempts, buffer allocation counts, retainedChunk contents, WSS recovery, peer EOF and send progress under backpressure.++## Shared HTTP and WebSocket runtime++`CurlMultiRuntime::websocket_connector()` admits WebSockets to the same native+owner, Multi and poll loop as HTTP. HTTP completions remove an easy handle;+WebSocket handshake completion leaves it attached until the connection ends.+`CurlWebSocketRuntime` is a standalone wrapper around this same driver.++The connector is a request capability: retaining it does not retain the runtime+owner, and an explicitly supplied closed connector fails without a standalone+fallback. Fetch, Page and Worker use the connector from their network runtime.+The Fetch semantic thread and WebSocket Tokio session thread remain separate.++WebSockets have an owner-local CURLSH connection cache, so resident sockets do+not consume HTTP host/total connection limits. HTTP retains the Multi's default+connection pool; WebSocket session admission bounds its separate cache. This+shares scheduling and ownership, not TCP/TLS connections. WebSocket transport+still uses HTTP/1.1 Upgrade; this change does not implement RFC 8441.++In shared mode, poll/turn counters include HTTP work and HTTP wakeups. A fast+poll with no WebSocket byte progress can therefore reflect useful HTTP work.++Compare a shared owner with two separate owners in the same release binary:++```sh+target/release/examples/websocket_owner_probe --http separate --scenario active --idle 32 --seconds 5+target/release/examples/websocket_owner_probe --http shared --scenario active --idle 32 --seconds 5+```++`--http` defaults to `off`. Both enabled modes use an HTTP/1.1 keepalive peer,+16 warmup requests and a sequential offered load of up to 100 requests/s.+They report HTTP request count and p50/p95 latency, WebSocket throughput,+native owner count and the sum of native owner CPU time. The HTTP producer,+local peer and Tokio executor CPU are excluded. This measures native scheduling+coexistence, not HTTP/2 throughput or end-to-end browser page performance.+The Fetch regression tests separately verify real TLS HTTP/2 multiplexing while+WS/WSS reads are paused and after the WebSocket close handshake.++HTTP uses `runtime.http_sender().submit(job)` and WebSocket uses+`runtime.websocket_connector().connect(request)`. Only the runtime owns shutdown+and join; neither request handle does, and the runtime itself is not cloneable.++The browser WebSocket API takes the connector as an explicit spawn argument.+`ConnectOptions` contains only request configuration. Callers without a network+owner must choose `spawn_standalone_connection` (or its handshake-pause variant)+explicitly; scoped connection failures never select another owner.diff --git a/moli-curl/examples/websocket_owner_probe.rs b/moli-curl/examples/websocket_owner_probe.rsindex 081181730..5fa9e80a4 100644--- a/moli-curl/examples/websocket_owner_probe.rs+++ b/moli-curl/examples/websocket_owner_probe.rs@@ -1,4 +1,7 @@//! Local, repeatable owner workload. See moli-curl/README.md for commands.+#[path = "websocket_owner_probe/http.rs"]+mod http;+use anyhow::{Context, Result, bail};use moli_curl::websocket::{CurlWebSocketConnection, CurlWebSocketEvent, CurlWebSocketRequest, CurlWebSocketRuntime,@@ -15,6 +18,7 @@ use tokio_tungstenite::tungstenite::{self, Message};struct Options {scenario: String,+ http: String,idle: usize,duration: Duration,}@@ -22,6 +26,7 @@ struct Options {fn options() -> Result<Options> {let mut options = Options {scenario: "active".into(),+ http: "off".into(),idle: 32,duration: Duration::from_secs(3),};@@ -29,9 +34,10 @@ fn options() -> Result<Options> {while let Some(arg) = args.next() {let value = args.next()- .context("expected a value after --scenario, --idle or --seconds")?;+ .context("expected a value after --scenario, --http, --idle or --seconds")?;match arg.as_str() {"--scenario" => options.scenario = value,+ "--http" => options.http = value,"--idle" => options.idle = value.parse()?,"--seconds" => options.duration = Duration::try_from_secs_f64(value.parse()?)?,_ => bail!("unknown option: {arg}"),@@ -43,6 +49,9 @@ fn options() -> Result<Options> {) {bail!("scenario must be idle, active, slow-read or wss");}+ if !matches!(options.http.as_str(), "off" | "shared" | "separate") {+ bail!("http must be off, shared or separate");+ }let count = options.idle.checked_add(usize::from(options.scenario != "idle"))@@ -118,22 +127,29 @@ fn tls_config() -> Result<Arc<rustls::ServerConfig>> {// Linux exposes scheduled CPU nanoseconds per thread. Other platforms still// report workload throughput; no process-wide CPU is substituted for the owner.-fn owner_cpu_ns() -> Option<u64> {+fn owner_cpu_ns() -> Option<(usize, u64)> {+ let mut count = 0;+ let mut cpu = 0;for task in std::fs::read_dir("/proc/self/task").ok()? {- let path = task.ok()?.path();- if std::fs::read_to_string(path.join("comm"))- .ok()?- .starts_with("moli-curl-web")- {- return std::fs::read_to_string(path.join("schedstat"))+ let Ok(task) = task else {+ continue;+ };+ let path = task.path();+ // An unrelated peer/producer may exit while /proc is being sampled.+ let Ok(name) = std::fs::read_to_string(path.join("comm")) else {+ continue;+ };+ if name.starts_with("moli-curl-web") || name.starts_with("moli-curl-http") {+ count += 1;+ cpu += std::fs::read_to_string(path.join("schedstat")).ok()?.split_whitespace().next()?- .parse()- .ok();+ .parse::<u64>()+ .ok()?;}}- None+ (count > 0).then_some((count, cpu))}async fn incoming(@@ -166,7 +182,20 @@ async fn main() -> Result<()> {.with_ansi(false).with_writer(std::io::stderr).init();- let runtime = CurlWebSocketRuntime::new()?;+ let http = (options.http != "off")+ .then(http::Workload::new)+ .transpose()?;+ let runtime = (options.http != "shared")+ .then(CurlWebSocketRuntime::new)+ .transpose()?;+ let connector = match &runtime {+ Some(runtime) => runtime.connector(),+ None => http+ .as_ref()+ .context("shared HTTP runtime missing")?+ .runtime+ .websocket_connector(),+ };let tls = (options.scenario == "wss").then(tls_config).transpose()?;let mut peers = Vec::new();let mut connections = Vec::new();@@ -182,7 +211,7 @@ async fn main() -> Result<()> {let mut request = CurlWebSocketRequest::new(url);// The WSS fixture owns this locally generated certificate.request.tls.verify = tls.is_none();- let mut connection = runtime.connect(request)?;+ let mut connection = connector.connect(request)?;match tokio::time::timeout(Duration::from_secs(10), connection.recv()).await? {Some(CurlWebSocketEvent::Handshake { result, .. }) => {result.map_err(anyhow::Error::msg)?@@ -204,7 +233,10 @@ async fn main() -> Result<()> {}let cpu_before = owner_cpu_ns();let started = Instant::now();- let end = tokio::time::Instant::now() + options.duration;+ let http_task = http+ .as_ref()+ .map(|http| http.start(started + options.duration));+ let end = tokio::time::Instant::from_std(started + options.duration);let bytes = match options.scenario.as_str() {"idle" => {tokio::time::sleep_until(end).await;@@ -241,17 +273,25 @@ async fn main() -> Result<()> {let elapsed = started.elapsed();let cpu_ms = cpu_before.zip(owner_cpu_ns())- .map(|(before, after)| after.saturating_sub(before) as f64 / 1_000_000.0);+ .map(|((_, before), (_, after))| after.saturating_sub(before) as f64 / 1_000_000.0);println!(- "scenario={} idle={} elapsed_s={:.3} bytes={} mib_per_s={:.3} owner_cpu_ms={cpu_ms:?}",+ "scenario={} http={} idle={} elapsed_s={:.3} bytes={} mib_per_s={:.3} owner_cpu_ms={cpu_ms:?} owner_threads={:?}",options.scenario,+ options.http,options.idle,elapsed.as_secs_f64(),bytes,- bytes as f64 / 1_048_576.0 / elapsed.as_secs_f64()+ bytes as f64 / 1_048_576.0 / elapsed.as_secs_f64(),+ cpu_before.map(|(count, _)| count));+ if let Some(task) = http_task {+ http::report(task)?;+ }drop(connections);drop(runtime);+ if let Some(http) = http {+ http.finish()?;+ }for task in peers {task.join().map_err(|_| anyhow::anyhow!("peer panicked"))??;diff --git a/moli-curl/examples/websocket_owner_probe/http.rs b/moli-curl/examples/websocket_owner_probe/http.rsnew file mode 100644index 000000000..68d06d289--- /dev/null+++ b/moli-curl/examples/websocket_owner_probe/http.rs@@ -0,0 +1,139 @@+//! Fixed HTTP/1.1 keepalive load alongside the WebSocket owner probe.+use anyhow::{Context, Result, ensure};+use curl::easy::{Easy2, Handler, WriteError};+use moli_curl::{+ CurlDnsResolution, CurlHttpSender, CurlMultiCompletion, CurlMultiJob, CurlMultiRuntime,+ CurlMultiRuntimeConfig,+};+use std::{+ io::{Read, Write},+ net::TcpListener,+ thread,+ time::{Duration, Instant},+};++#[derive(Debug, Default)]+pub struct Body(Vec<u8>);+impl Handler for Body {+ fn write(&mut self, data: &[u8]) -> Result<usize, WriteError> {+ self.0.extend_from_slice(data);+ Ok(data.len())+ }+}++pub struct Workload {+ pub runtime: CurlMultiRuntime<Body, ()>,+ completed: crossbeam_channel::Receiver<CurlMultiCompletion<Body, ()>>,+ url: String,+ peer: thread::JoinHandle<Result<()>>,+}++impl Workload {+ pub fn new() -> Result<Self> {+ let listener = TcpListener::bind("127.0.0.1:0")?;+ let url = format!("http://{}/probe", listener.local_addr()?);+ let peer = thread::spawn(move || {+ let (mut stream, _) = listener.accept()?;+ stream.set_read_timeout(Some(Duration::from_secs(30)))?;+ let mut header = Vec::new();+ loop {+ let mut byte = [0];+ if stream.read(&mut byte)? == 0 {+ break;+ }+ header.push(byte[0]);+ if header.ends_with(b"\r\n\r\n") {+ stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")?;+ header.clear();+ }+ }+ Ok(())+ });+ let (runtime, completed) = CurlMultiRuntime::new(CurlMultiRuntimeConfig {+ thread_name: "moli-curl-http".to_owned(),+ ..CurlMultiRuntimeConfig::default()+ })?;+ let workload = Self {+ runtime,+ completed,+ url,+ peer,+ };+ for _ in 0..16 {+ request(+ &workload.runtime.http_sender(),+ &workload.completed,+ &workload.url,+ )?;+ }+ Ok(workload)+ }++ pub fn start(&self, end: Instant) -> thread::JoinHandle<Result<Vec<f64>>> {+ let sender = self.runtime.http_sender();+ let url = self.url.clone();+ let completed = self.completed.clone();+ thread::spawn(move || {+ let mut latency = Vec::new();+ while Instant::now() < end {+ let started = Instant::now();+ request(&sender, &completed, &url)?;+ latency.push(started.elapsed().as_secs_f64() * 1000.0);+ // A fixed 100 requests/s offered load; this is a benchmark,+ // not synchronization for a correctness test.+ thread::sleep(Duration::from_millis(10).saturating_sub(started.elapsed()));+ }+ Ok(latency)+ })+ }++ pub fn finish(self) -> Result<()> {+ drop(self.runtime);+ self.peer+ .join()+ .map_err(|_| anyhow::anyhow!("HTTP peer panicked"))?+ }+}++fn request(+ sender: &CurlHttpSender<Body, ()>,+ completed: &crossbeam_channel::Receiver<CurlMultiCompletion<Body, ()>>,+ url: &str,+) -> Result<()> {+ let mut easy = Easy2::new(Body::default());+ easy.url(url)?;+ easy.proxy("")?;+ sender+ .submit(CurlMultiJob {+ easy,+ context: (),+ origin: None,+ deadline: Some(Instant::now() + Duration::from_secs(10)),+ dns_resolution: CurlDnsResolution::curl_managed(),+ priority: 1,+ label: "owner probe HTTP".into(),+ })+ .map_err(|error| error.error)?;+ let completion = completed.recv_timeout(Duration::from_secs(10))?;+ completion.result?;+ ensure!(+ completion.easy.context("HTTP easy missing")?.get_ref().0 == b"ok",+ "HTTP body mismatch"+ );+ Ok(())+}++pub fn report(task: thread::JoinHandle<Result<Vec<f64>>>) -> Result<()> {+ let mut samples = task+ .join()+ .map_err(|_| anyhow::anyhow!("HTTP producer panicked"))??;+ samples.sort_by(f64::total_cmp);+ ensure!(!samples.is_empty(), "no HTTP samples");+ println!(+ "http_version=1.1 http_requests={} http_p50_ms={:.3} http_p95_ms={:.3}",+ samples.len(),+ samples[(samples.len() - 1) / 2],+ samples[(samples.len() - 1) * 95 / 100]+ );+ Ok(())+}diff --git a/moli-fetch/Cargo.toml b/moli-fetch/Cargo.tomlindex bca36392e..bf7bd0917 100644--- a/moli-fetch/Cargo.toml+++ b/moli-fetch/Cargo.toml@@ -39,5 +39,7 @@ rcgen = { version = "0.13", default-features = false, features = ["aws_lc_rs", "tokio = { version = "1.51.0", features = ["io-util", "macros", "net", "rt", "sync", "time"] }tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "logging", "tls12"] }+tokio-tungstenite = { version = "0.28.0", default-features = false, features = ["handshake"] }+[lints]workspace = truediff --git a/moli-fetch/src/tests/mixed_transport.rs b/moli-fetch/src/tests/mixed_transport.rsnew file mode 100644index 000000000..8e98a6a11--- /dev/null+++ b/moli-fetch/src/tests/mixed_transport.rs@@ -0,0 +1,242 @@+use super::*;+use moli_curl::websocket::{+ CurlWebSocketConnection, CurlWebSocketEvent, CurlWebSocketRequest, CurlWebSocketSend, WsFlags,+};+use std::io::Write;+use tokio_rustls::rustls;+use tokio_tungstenite::tungstenite::{self, Message};++const DEADLINE: Duration = Duration::from_secs(10);++fn tls_config() -> Arc<rustls::ServerConfig> {+ use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};+ let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]).unwrap();+ let mut config = rustls::ServerConfig::builder()+ .with_no_client_auth()+ .with_single_cert(+ vec![CertificateDer::from(cert.cert.der().to_vec())],+ PrivateKeyDer::from(PrivatePkcs8KeyDer::from(cert.key_pair.serialize_der())),+ )+ .unwrap();+ config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];+ Arc::new(config)+}++fn ws_peer<S: Read + Write>(stream: S, sent: oneshot::Sender<()>, shutdown: bool) {+ let mut socket =+ tungstenite::accept(stream).unwrap_or_else(|error| panic!("WS handshake: {error}"));+ socket+ .send(Message::Binary(vec![7; 64 * 1024].into()))+ .unwrap();+ sent.send(()).unwrap();+ if shutdown {+ match socket.read() {+ Err(tungstenite::Error::Protocol(+ tungstenite::error::ProtocolError::ResetWithoutClosingHandshake,+ )) => {}+ Err(tungstenite::Error::Io(error))+ if matches!(+ error.kind(),+ std::io::ErrorKind::UnexpectedEof | std::io::ErrorKind::ConnectionReset+ ) => {}+ result => panic!("expected owner shutdown to release the peer: {result:?}"),+ }+ return;+ }+ assert_eq!(socket.read().unwrap(), Message::Text("client".into()));+ socket+ .close(Some(tungstenite::protocol::CloseFrame {+ code: tungstenite::protocol::frame::coding::CloseCode::Normal,+ reason: "".into(),+ }))+ .unwrap();+ assert!(matches!(socket.read().unwrap(), Message::Close(_)));+}++async fn event(connection: &mut CurlWebSocketConnection) -> CurlWebSocketEvent {+ tokio::time::timeout(DEADLINE, connection.recv())+ .await+ .unwrap()+ .unwrap()+}++async fn mixed_http2_and_websocket(tls: bool, shutdown: bool) -> Result<()> {+ let tls_config = tls_config();+ let h2_listener = TcpListener::bind("127.0.0.1:0").await?;+ let h2_url = format!("https://{}", h2_listener.local_addr()?);+ let (held_tx, held_rx) = oneshot::channel();+ let (release_tx, mut release_rx) = oneshot::channel();+ let acceptor = tokio_rustls::TlsAcceptor::from(tls_config.clone());+ let h2_task = tokio::spawn(async move {+ // Exactly one TCP connection must carry all three HTTP/2 streams.+ let (stream, _) = h2_listener.accept().await.unwrap();+ let stream = acceptor.accept(stream).await.unwrap();+ assert_eq!(stream.get_ref().1.alpn_protocol(), Some(b"h2".as_slice()));+ let mut connection = h2::server::handshake(stream).await.unwrap();+ let mut held = None;+ let mut held_tx = Some(held_tx);+ let mut paths = Vec::new();+ loop {+ tokio::select! {+ _ = &mut release_rx, if held.is_some() => {+ let mut send: h2::SendStream<_> = held.take().unwrap();+ send.send_data("held".into(), true).unwrap();+ }+ incoming = connection.accept() => {+ let Some(Ok((request, mut respond))) = incoming else { break; };+ let path = request.uri().path().to_owned();+ let response = http::Response::builder().status(200).body(()).unwrap();+ let mut send = respond.send_response(response, false).unwrap();+ if path == "/held" {+ held = Some(send);+ held_tx.take().unwrap().send(()).unwrap();+ } else {+ send.send_data("fast".into(), true).unwrap();+ }+ paths.push(path);+ }+ }+ }+ paths+ });++ let ws_listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let ws_url = format!(+ "{}://{}/native",+ if tls { "wss" } else { "ws" },+ ws_listener.local_addr()?+ );+ let (sent_tx, sent_rx) = oneshot::channel();+ let ws_task = thread::spawn(move || {+ let (stream, _) = ws_listener.accept().unwrap();+ stream.set_read_timeout(Some(DEADLINE)).unwrap();+ stream.set_write_timeout(Some(DEADLINE)).unwrap();+ if tls {+ ws_peer(+ rustls::StreamOwned::new(+ rustls::ServerConnection::new(tls_config).unwrap(),+ stream,+ ),+ sent_tx,+ shutdown,+ );+ } else {+ ws_peer(stream, sent_tx, shutdown);+ }+ });+ let mut config = FetchConfig::default();+ config.set_tls_verify_host(false);+ config.set_transport_connection_limits(Some(1), Some(1), Some(8));+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());+ let handle = client.handle();+ let held_url = format!("{h2_url}/held");+ let held = tokio::spawn(async move { handle.fetch(Request::get(&held_url)?).await });+ tokio::time::timeout(DEADLINE, held_rx).await??;++ let mut request = CurlWebSocketRequest::new(ws_url);+ request.tls.verify = !tls;+ let mut connection = client.handle().websocket_connector().connect(request)?;+ assert!(matches!(+ event(&mut connection).await,+ CurlWebSocketEvent::Handshake { result: Ok(()), .. }+ ));+ // The server has queued a payload, but this connection is still paused.+ tokio::time::timeout(DEADLINE, sent_rx).await??;+ let fast = tokio::time::timeout(+ DEADLINE,+ client.fetch(Request::get(&format!("{h2_url}/fast"))?),+ )+ .await??;+ assert_eq!(+ fast.negotiated_http_version,+ Some(NegotiatedHttpVersion::Http2)+ );+ assert_eq!(fast.body_text(), "fast");+ assert!(!held.is_finished(), "a separate stream remains blocked");++ if shutdown {+ drop(client);+ assert!(tokio::time::timeout(DEADLINE, held).await??.is_err());+ assert!(matches!(+ event(&mut connection).await,+ CurlWebSocketEvent::Closed { result: Err(_) }+ ));+ ws_task.join().unwrap();+ assert_eq!(+ tokio::time::timeout(DEADLINE, h2_task).await??,+ ["/held", "/fast"]+ );+ return Ok(());+ }++ connection.sender().set_reading(true);+ let mut payload = Vec::new();+ while payload.len() < 64 * 1024 {+ let CurlWebSocketEvent::Chunk { data, .. } = event(&mut connection).await else {+ panic!("expected data");+ };+ payload.extend(data);+ }+ assert_eq!(payload, vec![7; 64 * 1024]);+ connection+ .sender()+ .send_frame(CurlWebSocketSend {+ flags: WsFlags::TEXT,+ data: b"client".to_vec(),+ })+ .await?;+ let CurlWebSocketEvent::Chunk { data, frame } = event(&mut connection).await else {+ panic!("expected Close");+ };+ assert!(frame.flags().contains(WsFlags::CLOSE));+ assert_eq!(data, 1000_u16.to_be_bytes());+ connection+ .sender()+ .send_frame(CurlWebSocketSend {+ flags: WsFlags::CLOSE,+ data,+ })+ .await?;+ drop(connection);+ ws_task.join().unwrap();++ // WS close does not complete or cancel either HTTP/2 stream.+ release_tx.send(()).unwrap();+ let response = tokio::time::timeout(DEADLINE, held).await???;+ assert_eq!(+ response.negotiated_http_version,+ Some(NegotiatedHttpVersion::Http2)+ );+ assert_eq!(response.body_text(), "held");+ let after = tokio::time::timeout(+ DEADLINE,+ client.fetch(Request::get(&format!("{h2_url}/after"))?),+ )+ .await??;+ assert_eq!(+ after.negotiated_http_version,+ Some(NegotiatedHttpVersion::Http2)+ );+ assert_eq!(after.body_text(), "fast");+ drop(client);+ assert_eq!(+ tokio::time::timeout(DEADLINE, h2_task).await??,+ ["/held", "/fast", "/after"]+ );+ Ok(())+}++#[tokio::test]+async fn http2_multiplexing_survives_paused_websocket_and_close() -> Result<()> {+ mixed_http2_and_websocket(false, false).await+}++#[tokio::test]+async fn http2_multiplexing_survives_paused_wss_and_close() -> Result<()> {+ mixed_http2_and_websocket(true, false).await+}++#[tokio::test]+async fn shared_fetch_shutdown_cancels_pending_http2_and_paused_wss() -> Result<()> {+ mixed_http2_and_websocket(true, true).await+}diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex 98eeadf8c..1c949516e 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -1,5 +1,6 @@mod cookie_context;mod cookie_store;+mod mixed_transport;mod support;mod tls_credentials;mod websocket_transport;