Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use super::*;
use axum::{
Router,
body::{Body, to_bytes},
extract::{WebSocketUpgrade, ws::Message},
http::{Method, Request, StatusCode, header},
response::IntoResponse,
routing::{get, post},
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use futures_util::{SinkExt, StreamExt};
use moli_browser_profile::{BrowserProfileLock, BrowserProfilePaths};
use moli_cookie_jar::{StoredCookieSameSite, StoredCookieSourceScheme};
use moli_protocol_webdriver_classic::{
CLASSIC_ELEMENT_REFERENCE_KEY, CLASSIC_FRAME_REFERENCE_KEY, CLASSIC_SHADOW_ROOT_REFERENCE_KEY,
CLASSIC_WINDOW_REFERENCE_KEY,
};
use serde_json::json;
use std::{
fs,
io::{Read, Write},
path::PathBuf,
sync::Arc,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tokio::time::{sleep, timeout};
use tokio_tungstenite::{connect_async, tungstenite::Message as WsMessage};
use tower::util::ServiceExt;
struct DedicatedFixtureServer {
shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
thread: Option<std::thread::JoinHandle<()>>,
}
impl Drop for DedicatedFixtureServer {
fn drop(&mut self) {
if let Some(shutdown_tx) = self.shutdown_tx.take() {
let _ = shutdown_tx.send(());
}
if let Some(thread) = self.thread.take() {
let _ = thread.join();
}
}
}
fn spawn_dedicated_fixture_server(
app: Router,
name: &'static str,
) -> (std::net::SocketAddr, DedicatedFixtureServer) {
let app = app.route("/__moli_fixture_ready", get(|| async { "ok" }));
let listener = std::net::TcpListener::bind("127.0.0.1:0")
.unwrap_or_else(|error| panic!("bind {name} fixture listener: {error}"));
listener
.set_nonblocking(true)
.unwrap_or_else(|error| panic!("set {name} fixture listener nonblocking: {error}"));
let addr = listener
.local_addr()
.unwrap_or_else(|error| panic!("read {name} fixture listener addr: {error}"));
let (ready_tx, ready_rx) = std::sync::mpsc::channel();
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
let thread = std::thread::Builder::new()
.name(format!("moli-{name}-fixture"))
.spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap_or_else(|error| panic!("build {name} fixture runtime: {error}"));
runtime.block_on(async move {
let listener = TcpListener::from_std(listener)
.unwrap_or_else(|error| panic!("adopt {name} fixture listener: {error}"));
let _ = ready_tx.send(());
let server = axum::serve(listener, app);
tokio::select! {
result = server => {
result.unwrap_or_else(|error| panic!("{name} fixture server failed: {error}"));
}
_ = shutdown_rx => {}
}
});
})
.unwrap_or_else(|error| panic!("spawn {name} fixture thread: {error}"));
ready_rx
.recv()
.unwrap_or_else(|error| panic!("wait for {name} fixture listener: {error}"));
wait_for_dedicated_fixture_ready(addr, name);
(
addr,
DedicatedFixtureServer {
shutdown_tx: Some(shutdown_tx),
thread: Some(thread),
},
)
}
fn spawn_shared_worker_fixture_server(
name: &'static str,
) -> (std::net::SocketAddr, DedicatedFixtureServer) {
let app = Router::new()
.route(
"/",
get(|| async move {
(
[(header::CONTENT_TYPE.as_str(), "text/html")],
r#"<!doctype html><html><body>shared worker
<script>
globalThis.__sharedWorkerProbe = value => new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('shared worker timeout')), 1000);
const worker = new SharedWorker('/shared-worker.js', 'webdriver-shared-worker-smoke');
globalThis.__sharedWorkerSmoke = worker;
worker.port.onmessage = event => {
if (event.data && event.data.kind === 'probe-result') {
clearTimeout(timer);
resolve(event.data);
}
};
worker.port.start();
worker.port.postMessage({ kind: 'probe', value });
});
</script></body></html>"#,
)
}),
)
.route(
"/shared-worker.js",
get(|| async move {
(
[(header::CONTENT_TYPE.as_str(), "text/javascript")],
"globalThis.__sharedWorkerConnectCount = 0;\
self.onconnect = event => {\
globalThis.__sharedWorkerConnectCount += 1;\
const port = event.ports[0];\
port.onmessage = message => {\
const data = message.data;\
if (data && data.kind === 'probe') {\
port.postMessage({\
kind: 'probe-result',\
echoed: data.value,\
name,\
pathname: self.location.pathname,\
isSharedWorker: typeof SharedWorkerGlobalScope !== 'undefined' && self instanceof SharedWorkerGlobalScope,\
selfEqualsGlobal: self === globalThis,\
connectCount: globalThis.__sharedWorkerConnectCount\
});\
}\
};\
port.start();\
};",
)
}),
);
spawn_dedicated_fixture_server(app, name)
}
fn wait_for_dedicated_fixture_ready(addr: std::net::SocketAddr, name: &'static str) {
let mut stream = std::net::TcpStream::connect(addr)
.unwrap_or_else(|error| panic!("connect to {name} fixture readiness route: {error}"));
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap_or_else(|error| panic!("set {name} fixture readiness read timeout: {error}"));
stream
.set_write_timeout(Some(Duration::from_secs(5)))
.unwrap_or_else(|error| panic!("set {name} fixture readiness write timeout: {error}"));
stream
.write_all(
b"GET /__moli_fixture_ready HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n",
)
.unwrap_or_else(|error| panic!("write {name} fixture readiness request: {error}"));
let mut response = String::new();
stream
.read_to_string(&mut response)
.unwrap_or_else(|error| panic!("read {name} fixture readiness response: {error}"));
assert!(
response.starts_with("HTTP/1.1 200") || response.starts_with("HTTP/1.0 200"),
"{name} fixture readiness route returned unexpected response: {response:?}"
);
}
struct TempDir {
path: PathBuf,
}
struct TempPath {
path: PathBuf,
}
impl TempDir {
fn new(name: &str) -> Self {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock should be after epoch")
.as_nanos();
let path = std::env::temp_dir().join(format!(
"moli-protocol-{name}-{}-{nonce}",
std::process::id()
));
let _ = fs::remove_file(&path);
Self { path }
}
}
impl TempPath {
fn new(name: &str) -> Self {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock should be after epoch")
.as_nanos();
let path =
std::env::temp_dir().join(format!("moli-cdp-{name}-{}-{nonce}", std::process::id()));
Self { path }
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.path);
}
}
impl Drop for TempPath {
fn drop(&mut self) {
let _ = fs::remove_file(&self.path);
}
}
fn stored_cookie(name: &str, value: &str) -> StoredCookie {
StoredCookie {
name: name.to_owned(),
value: value.to_owned(),
domain: "example.com".to_owned(),
host_only: false,
path: "/".to_owned(),
secure: false,
http_only: false,
expires: None,
same_site: StoredCookieSameSite::Unspecified,
priority: None,
partition_key: None,
source_scheme: StoredCookieSourceScheme::NonSecure,
source_port: -1,
creation_index: 0,
last_access_index: 0,
}
}
fn test_state() -> AppState {
protocol_server_test_state(
"127.0.0.1:9222".parse().expect("test addr"),
FetchConfig::default(),
OptionalResourceFetchMask::NONE,
)
}
fn protocol_server_test_fetch_config(mut fetch_config: FetchConfig) -> FetchConfig {
if fetch_config.http_no_proxy().is_none() {
fetch_config.set_http_no_proxy(Some("*".to_owned()));
}
fetch_config
}
fn protocol_server_test_state(
addr: std::net::SocketAddr,
fetch_config: FetchConfig,
optional_resource_fetch_mask: OptionalResourceFetchMask,
) -> AppState {
let storage_partition = Arc::new(
StoragePartitionState::open(None).expect("test storage partition should initialize"),
);
AppState::new_with_storage_partition_and_runtime_config(
addr,
storage_partition,
protocol_server_test_runtime_config(
protocol_server_test_fetch_config(fetch_config),
optional_resource_fetch_mask,
),
crate::config::DEFAULT_SCREENCAST_INTERVAL_MS,
BrowserInstanceControl::new(),
)
.expect("test app state should initialize")
}
async fn request_json(path: &str) -> serde_json::Value {
request_json_with_method(Method::GET, path).await
}
async fn request_json_with_method(method: Method, path: &str) -> serde_json::Value {
let response = build_router(test_state())
.oneshot(
Request::builder()
.method(method)
.uri(path)
.body(Body::empty())
.expect("request"),
)
.await
.expect("router response");
assert_eq!(response.status(), StatusCode::OK, "path {path}");
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("response body");
serde_json::from_slice(&body).expect("json response")
}
async fn request_status(path: &str) -> StatusCode {
request_status_with_method(Method::GET, path).await
}
async fn request_status_with_method(method: Method, path: &str) -> StatusCode {
request_status_and_text_with_method(method, path).await.0
}
async fn request_status_and_text_with_method(method: Method, path: &str) -> (StatusCode, String) {
let response = build_router(test_state())
.oneshot(
Request::builder()
.method(method)
.uri(path)
.body(Body::empty())
.expect("request"),
)
.await
.expect("router response");
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("response body");
(
status,
String::from_utf8(body.to_vec()).expect("utf-8 body"),
)
}
async fn spawn_test_protocol_server() -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
spawn_test_protocol_server_with_fetch_config_and_optional_resource_fetch_mask(
FetchConfig::default(),
OptionalResourceFetchMask::NONE,
)
.await
}
async fn spawn_test_protocol_server_with_owner_registry() -> (
std::net::SocketAddr,
tokio::task::JoinHandle<()>,
SharedCdpOwnerRegistry,
) {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind test protocol server");
let addr = listener.local_addr().expect("test protocol server addr");
let state = protocol_server_test_state(
addr,
FetchConfig::default(),
OptionalResourceFetchMask::NONE,
);
let owner_registry = state.cdp_owner_registry.clone();
let server = tokio::spawn(async move {
let service = axum::ServiceExt::<Request<Body>>::into_make_service(build_router(state));
axum::serve(listener, service)
.await
.expect("test protocol server should serve");
});
(addr, server, owner_registry)
}
async fn spawn_test_protocol_server_with_fetch_config(
fetch_config: FetchConfig,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
spawn_test_protocol_server_with_fetch_config_and_optional_resource_fetch_mask(
fetch_config,
OptionalResourceFetchMask::NONE,
)
.await
}
async fn spawn_test_protocol_server_with_image_fetch_enabled(
image_fetch_enabled: bool,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
spawn_test_protocol_server_with_fetch_config_and_optional_resource_fetch_mask(
FetchConfig::default(),
if image_fetch_enabled {
OptionalResourceFetchMask::IMAGE
} else {
OptionalResourceFetchMask::NONE
},
)
.await
}
async fn spawn_test_protocol_server_with_fetch_config_and_optional_resource_fetch_mask(
fetch_config: FetchConfig,
optional_resource_fetch_mask: OptionalResourceFetchMask,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let fetch_config = protocol_server_test_fetch_config(fetch_config);
spawn_test_protocol_server_with_runtime_config(protocol_server_test_runtime_config(
fetch_config,
optional_resource_fetch_mask,
))
.await
}
fn protocol_server_test_runtime_config(
fetch_config: FetchConfig,
optional_resource_fetch_mask: OptionalResourceFetchMask,
) -> NavigationRuntimeConfig {
NavigationRuntimeConfig::new(
fetch_config,
optional_resource_fetch_mask,
true,
LayoutPolicy::OnDemand,
)
// Most protocol fixtures exercise Chromium-compatible debugging, including pauses.
.with_debugger_enabled(true)
}
async fn spawn_test_protocol_server_with_layout_policy(
layout_policy: LayoutPolicy,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
spawn_test_protocol_server_with_runtime_config(NavigationRuntimeConfig::new(
protocol_server_test_fetch_config(FetchConfig::default()),
OptionalResourceFetchMask::NONE,
true,
layout_policy,
))
.await
}
async fn spawn_test_protocol_server_with_runtime_config(
navigation_runtime_config: NavigationRuntimeConfig,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind test protocol server");
let addr = listener.local_addr().expect("test protocol server addr");
let storage_partition = Arc::new(
StoragePartitionState::open(None).expect("test storage partition should initialize"),
);
let state = AppState::new_with_storage_partition_and_runtime_config(
addr,
storage_partition,
navigation_runtime_config,
crate::config::DEFAULT_SCREENCAST_INTERVAL_MS,
BrowserInstanceControl::new(),
)
.expect("test app state should initialize");
let server = tokio::spawn(async move {
let service = axum::ServiceExt::<Request<Body>>::into_make_service(build_router(state));
axum::serve(listener, service)
.await
.expect("test protocol server should serve");
});
(addr, server)
}
async fn spawn_delayed_download_fixture_server(
body: &'static str,
delay: Duration,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind delayed download fixture server");
let addr = listener
.local_addr()
.expect("delayed download fixture server addr");
let server = tokio::spawn(async move {
let app = Router::new()
.route(
"/page",
get(|| async move {
(
[(axum::http::header::CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><html><body><a id=\"dl\" href=\"/download\" download=\"saved.txt\" style=\"display:inline-block;width:200px;height:200px\">download</a></body></html>",
)
}),
)
.route(
"/download",
get(move || async move {
sleep(delay).await;
(
[
(axum::http::header::CONTENT_TYPE.as_str(), "text/plain"),
(
axum::http::header::CONTENT_DISPOSITION.as_str(),
"attachment; filename=\"saved.txt\"",
),
],
body,
)
}),
);
axum::serve(listener, app)
.await
.expect("delayed download fixture server should serve");
});
(addr, server)
}
async fn spawn_delayed_plain_attachment_fixture_server(
body: &'static str,
delay: Duration,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind delayed plain attachment fixture server");
let addr = listener
.local_addr()
.expect("delayed plain attachment fixture server addr");
let server = tokio::spawn(async move {
let app = Router::new()
.route(
"/page",
get(|| async move {
(
[(axum::http::header::CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><html><body><a id=\"dl\" href=\"/download\" style=\"display:inline-block;width:200px;height:200px\">download</a></body></html>",
)
}),
)
.route(
"/download",
get(move || async move {
sleep(delay).await;
(
[
(axum::http::header::CONTENT_TYPE.as_str(), "text/plain"),
(
axum::http::header::CONTENT_DISPOSITION.as_str(),
"attachment; filename=\"saved.txt\"",
),
],
body,
)
}),
);
axum::serve(listener, app)
.await
.expect("delayed plain attachment fixture server should serve");
});
(addr, server)
}
async fn spawn_post_parse_location_download_fixture_server(
body: &'static str,
delay: Duration,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind post-parse location download fixture server");
let addr = listener
.local_addr()
.expect("post-parse location download fixture server addr");
let server = tokio::spawn(async move {
let app = Router::new()
.route(
"/page",
get(|| async move {
(
[(axum::http::header::CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><html><body><main id=\"source\">source</main><script>window.addEventListener('load', () => setTimeout(() => location.assign('/download'), 0));</script></body></html>",
)
}),
)
.route(
"/download",
get(move || async move {
sleep(delay).await;
(
[
(axum::http::header::CONTENT_TYPE.as_str(), "text/plain"),
(
axum::http::header::CONTENT_DISPOSITION.as_str(),
"attachment; filename=\"saved.txt\"",
),
],
body,
)
}),
);
axum::serve(listener, app)
.await
.expect("post-parse location download fixture server should serve");
});
(addr, server)
}
async fn spawn_delayed_content_disposition_download_fixture_server(
body: &'static str,
delay: Duration,
content_disposition: &'static str,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind delayed content disposition fixture server");
let addr = listener
.local_addr()
.expect("delayed content disposition fixture server addr");
let server = tokio::spawn(async move {
let app = Router::new()
.route(
"/page",
get(|| async move {
(
[(axum::http::header::CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><html><body><a id=\"dl\" href=\"/download\" download style=\"display:inline-block;width:200px;height:200px\">download</a></body></html>",
)
}),
)
.route(
"/download",
get(move || async move {
sleep(delay).await;
(
[
(axum::http::header::CONTENT_TYPE.as_str(), "text/plain"),
(
axum::http::header::CONTENT_DISPOSITION.as_str(),
content_disposition,
),
],
body,
)
}),
);
axum::serve(listener, app)
.await
.expect("delayed content disposition fixture server should serve");
});
(addr, server)
}
async fn spawn_local_storage_fixture_server() -> (std::net::SocketAddr, tokio::task::JoinHandle<()>)
{
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind localStorage fixture server");
let addr = listener
.local_addr()
.expect("localStorage fixture server addr");
let server = tokio::spawn(async move {
let app = Router::new().route(
"/page",
get(|| async move {
(
[(axum::http::header::CONTENT_TYPE.as_str(), "text/html")],
"<!doctype html><html><body><main>localstorage fixture</main></body></html>",
)
}),
);
axum::serve(listener, app)
.await
.expect("localStorage fixture server should serve");
});
(addr, server)
}
async fn spawn_response_stage_streaming_document_fixture_server(
release_tail: Arc<tokio::sync::Notify>,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
spawn_response_stage_streaming_document_fixture_server_with_head_signal(release_tail, None)
.await
}
async fn spawn_response_stage_streaming_document_fixture_server_with_head_signal(
release_tail: Arc<tokio::sync::Notify>,
response_head_sent: Option<Arc<tokio::sync::Notify>>,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind response-stage streaming fixture server");
let addr = listener
.local_addr()
.expect("response-stage streaming fixture addr");
let server = tokio::spawn(async move {
let (mut stream, _) = listener
.accept()
.await
.expect("accept response-stage streaming request");
let mut request = Vec::new();
let mut buf = [0_u8; 1024];
loop {
let read = stream
.read(&mut buf)
.await
.expect("read response-stage streaming request");
if read == 0 {
return;
}
request.extend_from_slice(&buf[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let response = concat!(
"HTTP/1.1 200 OK\r\n",
"Content-Type: text/html; charset=utf-8\r\n",
"Transfer-Encoding: chunked\r\n",
"Connection: close\r\n",
"\r\n"
);
let head = b"<!doctype html><html><body><main id=\"head\">head";
stream
.write_all(response.as_bytes())
.await
.expect("write response-stage streaming response head");
stream
.write_all(format!("{:x}\r\n", head.len()).as_bytes())
.await
.expect("write response-stage streaming head chunk size");
stream
.write_all(head)
.await
.expect("write response-stage streaming head chunk");
stream
.write_all(b"\r\n")
.await
.expect("write response-stage streaming head chunk terminator");
if let Some(response_head_sent) = response_head_sent {
response_head_sent.notify_one();
}
release_tail.notified().await;
let tail = b" tail</main></body></html>";
stream
.write_all(format!("{:x}\r\n", tail.len()).as_bytes())
.await
.expect("write response-stage streaming tail chunk size");
stream
.write_all(tail)
.await
.expect("write response-stage streaming tail chunk");
stream
.write_all(b"\r\n0\r\n\r\n")
.await
.expect("finish response-stage streaming body");
let _ = stream.shutdown().await;
});
(addr, server)
}
async fn spawn_profiled_test_protocol_server(
profile_dir: PathBuf,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
spawn_profiled_test_protocol_server_with_cookie_profile(profile_dir, Vec::new()).await
}
async fn spawn_profiled_test_protocol_server_with_cookie_profile(
profile_dir: PathBuf,
initial_cookies: Vec<StoredCookie>,
) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
let fetch_config = protocol_server_test_fetch_config(FetchConfig::default());
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind profiled test protocol server");
let addr = listener
.local_addr()
.expect("profiled test protocol server addr");
let storage_partition = Arc::new(
StoragePartitionState::open(Some(&profile_dir)).expect("profiled partition should open"),
);
storage_partition
.import_cookies(initial_cookies)
.expect("profiled initial cookies should import");
let state = AppState::new_with_storage_partition(
addr,
storage_partition,
fetch_config,
OptionalResourceFetchMask::NONE,
true,
)
.expect("profiled app state should initialize");
let server = tokio::spawn(async move {
serve_listener(listener, state)
.await
.expect("profiled test protocol server should serve");
});
(addr, server)
}
async fn abort_test_cdp_server(server: tokio::task::JoinHandle<()>) {
server.abort();
let _ = server.await;
}
async fn recv_ws_json(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
) -> serde_json::Value {
loop {
match socket.next().await.expect("websocket message") {
Ok(WsMessage::Text(text)) => {
return serde_json::from_str(&text).expect("json websocket message");
}
Ok(WsMessage::Binary(bytes)) => {
return serde_json::from_slice(&bytes).expect("json websocket message");
}
Ok(WsMessage::Ping(_)) | Ok(WsMessage::Pong(_)) => continue,
Ok(other) => panic!("unexpected websocket message: {other:?}"),
Err(error) => panic!("websocket recv failed: {error:?}"),
}
}
}
async fn recv_until_id(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
expected_id: u64,
) -> Vec<serde_json::Value> {
// Test websocket helpers should fail with the messages seen so far instead
// of letting a missing response hang the whole nextest run.
const RECV_UNTIL_ID_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
let mut messages = Vec::new();
let deadline = tokio::time::Instant::now() + RECV_UNTIL_ID_TIMEOUT;
loop {
let message = tokio::time::timeout_at(deadline, recv_ws_json(socket))
.await
.unwrap_or_else(|_| {
panic!(
"recv_until_id timed out after {:?} waiting for id {}; \
received messages so far: {messages:#?}",
RECV_UNTIL_ID_TIMEOUT, expected_id
)
});
let done = message["id"] == json!(expected_id);
messages.push(message);
if done {
return messages;
}
}
}
async fn recv_until_match(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
mut predicate: impl FnMut(&serde_json::Value) -> bool,
) -> Vec<serde_json::Value> {
// CDP websocket tests run alongside many in-process HTTP servers, V8
// runtimes, and renderer tasks under full nextest parallelism. This is a
// test harness budget, not a protocol timing expectation; the predicates
// below still assert the exact event or response contract.
const RECV_UNTIL_MATCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
let mut messages = Vec::new();
let deadline = tokio::time::Instant::now() + RECV_UNTIL_MATCH_TIMEOUT;
loop {
let message = tokio::time::timeout_at(deadline, recv_ws_json(socket))
.await
.unwrap_or_else(|_| {
panic!(
"recv_until_match timed out after {:?} waiting for predicate match; \
received messages so far: {messages:#?}",
RECV_UNTIL_MATCH_TIMEOUT
)
});
let done = predicate(&message);
messages.push(message);
if done {
return messages;
}
}
}
async fn recv_cdp_messages_for(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
duration: std::time::Duration,
) -> Vec<serde_json::Value> {
let mut messages = Vec::new();
let deadline = tokio::time::Instant::now() + duration;
while let Ok(message) = tokio::time::timeout_at(deadline, recv_ws_json(socket)).await {
messages.push(message);
}
messages
}
async fn send_cdp_command_without_wait(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
id: u64,
method: &str,
session_id: Option<&str>,
params: serde_json::Value,
) {
let mut command = json!({
"id": id,
"method": method,
"params": params,
});
if let Some(session_id) = session_id {
command["sessionId"] = json!(session_id);
}
socket
.send(WsMessage::Text(command.to_string().into()))
.await
.unwrap_or_else(|error| panic!("send {method}: {error}"));
}
async fn send_cdp_command(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
id: u64,
method: &str,
session_id: Option<&str>,
params: serde_json::Value,
) -> Vec<serde_json::Value> {
send_cdp_command_without_wait(socket, id, method, session_id, params).await;
recv_until_id(socket, id).await
}
async fn cdp_create_browser_context(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
id: u64,
) -> String {
send_cdp_command(socket, id, "Target.createBrowserContext", None, json!({}))
.await
.iter()
.find(|message| message["id"] == json!(id))
.and_then(|message| message["result"]["browserContextId"].as_str())
.expect("browserContextId")
.to_owned()
}
struct TestCdpTargetSession {
target_id: String,
session_id: String,
}
async fn cdp_create_attached_target(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
id_base: u64,
browser_context_id: &str,
) -> TestCdpTargetSession {
let create_target = send_cdp_command(
socket,
id_base,
"Target.createTarget",
None,
json!({
"browserContextId": browser_context_id,
"url": "about:blank",
}),
)
.await;
let target_id = create_target
.iter()
.find(|message| message["id"] == json!(id_base))
.and_then(|message| message["result"]["targetId"].as_str())
.expect("targetId")
.to_owned();
let attach = send_cdp_command(
socket,
id_base + 1,
"Target.attachToTarget",
None,
json!({ "targetId": target_id }),
)
.await;
let session_id = attach
.iter()
.find(|message| message["id"] == json!(id_base + 1))
.and_then(|message| message["result"]["sessionId"].as_str())
.expect("sessionId")
.to_owned();
TestCdpTargetSession {
target_id,
session_id,
}
}
async fn cdp_navigate_and_wait_for_load(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
id: u64,
session_id: &str,
url: &str,
) -> Vec<serde_json::Value> {
socket
.send(WsMessage::Text(
json!({
"id": id,
"method": "Page.navigate",
"sessionId": session_id,
"params": { "url": url }
})
.to_string()
.into(),
))
.await
.expect("send Page.navigate");
let mut messages = Vec::new();
let mut saw_navigate_response = false;
let mut saw_load_event = false;
while !(saw_navigate_response && saw_load_event) {
let message = recv_ws_json(socket).await;
if message["id"] == json!(id) {
saw_navigate_response = true;
}
if message["sessionId"].as_str() == Some(session_id)
&& message["method"] == json!("Page.loadEventFired")
{
saw_load_event = true;
}
messages.push(message);
}
messages
}
async fn cdp_create_session_and_navigate(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
url: &str,
) -> String {
socket
.send(WsMessage::Text(
json!({ "id": 1_u64, "method": "Target.createBrowserContext" })
.to_string()
.into(),
))
.await
.expect("send createBrowserContext");
let create_browser_context = recv_until_id(socket, 1).await;
let browser_context_id = create_browser_context
.iter()
.find(|message| message["id"] == json!(1_u64))
.and_then(|message| message["result"]["browserContextId"].as_str())
.expect("browserContextId")
.to_owned();
socket
.send(WsMessage::Text(
json!({
"id": 2_u64,
"method": "Target.createTarget",
"params": {
"browserContextId": browser_context_id,
"url": "about:blank"
}
})
.to_string()
.into(),
))
.await
.expect("send createTarget");
let create_target = recv_until_id(socket, 2).await;
let target_id = create_target
.iter()
.find(|message| message["id"] == json!(2_u64))
.and_then(|message| message["result"]["targetId"].as_str())
.expect("targetId")
.to_owned();
socket
.send(WsMessage::Text(
json!({
"id": 3_u64,
"method": "Target.attachToTarget",
"params": { "targetId": target_id }
})
.to_string()
.into(),
))
.await
.expect("send attachToTarget");
let attach = recv_until_id(socket, 3).await;
let session_id = attach
.iter()
.find(|message| message["id"] == json!(3_u64))
.and_then(|message| message["result"]["sessionId"].as_str())
.expect("sessionId")
.to_owned();
socket
.send(WsMessage::Text(
json!({
"id": 4_u64,
"method": "Runtime.enable",
"sessionId": session_id,
})
.to_string()
.into(),
))
.await
.expect("send Runtime.enable");
let _ = recv_until_id(socket, 4).await;
socket
.send(WsMessage::Text(
json!({
"id": 5_u64,
"method": "Page.navigate",
"sessionId": session_id,
"params": { "url": url }
})
.to_string()
.into(),
))
.await
.expect("send Page.navigate");
let mut saw_navigate_response = false;
let mut saw_load_event = false;
while !(saw_navigate_response && saw_load_event) {
let message = recv_ws_json(socket).await;
if message["id"] == json!(5_u64) {
saw_navigate_response = true;
}
if message["sessionId"].as_str() == Some(session_id.as_str())
&& message["method"] == json!("Page.loadEventFired")
{
saw_load_event = true;
}
}
session_id
}
async fn cdp_create_default_session_and_navigate(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
url: &str,
) -> String {
let create_target = send_cdp_command(
socket,
1,
"Target.createTarget",
None,
json!({
"browserContextId": "BID-default",
"url": "about:blank",
}),
)
.await;
let target_id = create_target
.iter()
.find(|message| message["id"] == json!(1_u64))
.and_then(|message| message["result"]["targetId"].as_str())
.expect("default targetId")
.to_owned();
let attach = send_cdp_command(
socket,
2,
"Target.attachToTarget",
None,
json!({ "targetId": target_id }),
)
.await;
let session_id = attach
.iter()
.find(|message| message["id"] == json!(2_u64))
.and_then(|message| message["result"]["sessionId"].as_str())
.expect("default sessionId")
.to_owned();
let _ = send_cdp_command(socket, 3, "Runtime.enable", Some(&session_id), json!({})).await;
cdp_navigate_and_wait_for_load(socket, 5, &session_id, url).await;
session_id
}
async fn cdp_runtime_evaluate_string(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
session_id: &str,
id: u64,
expression: &str,
) -> String {
socket
.send(WsMessage::Text(
json!({
"id": id,
"method": "Runtime.evaluate",
"sessionId": session_id,
"params": { "expression": expression }
})
.to_string()
.into(),
))
.await
.expect("send Runtime.evaluate");
let messages = recv_until_id(socket, id).await;
messages
.iter()
.find(|message| message["id"] == json!(id))
.and_then(|message| message["result"]["result"]["value"].as_str())
.expect("Runtime.evaluate string result")
.to_owned()
}
async fn wait_for_cdp_runtime_string(
socket: &mut tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>,
session_id: &str,
mut id: u64,
expression: &str,
expected: &str,
) {
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
loop {
let value = cdp_runtime_evaluate_string(socket, session_id, id, expression).await;
if value == expected {
return;
}
assert!(
tokio::time::Instant::now() < deadline,
"timed out waiting for Runtime.evaluate `{expression}` to become `{expected}`, last value `{value}`"
);
id += 1;
sleep(Duration::from_millis(25)).await;
}
}
async fn rejected_websocket_status(url: String) -> u16 {
match connect_async(url)
.await
.expect_err("websocket should be rejected")
{
tokio_tungstenite::tungstenite::Error::Http(response) => response.status().as_u16(),
error => panic!("expected websocket HTTP rejection, got {error:?}"),
}
}
mod bidi;
mod cdp_dynamic_page;
mod classic;
mod download;
mod misc;
mod p6_output_handoff;
mod screenshot;
mod websocket;