Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use super::*;
use crate::{RedirectInfo, RedirectSource};
#[derive(Clone, Copy, Debug)]
enum Transport {
Buffered,
Html,
Raw,
}
impl Transport {
const ALL: [Self; 3] = [Self::Buffered, Self::Html, Self::Raw];
async fn fetch(self, client: &FetchClientHandle, request: Request) -> Result<()> {
match self {
Self::Buffered => {
let (tx, rx) = oneshot::channel();
client.fetch_with_cancel_callback(
request,
FetchCancelHandle::new(),
move |result| {
let _ = tx.send(result);
},
)?;
rx.await??;
}
Self::Html => {
let mut response = client.fetch_html_stream(request).await?;
while response.next_chunk().await.is_some() {}
response.finish().await?;
}
Self::Raw => {
client.fetch(request).await?;
}
}
Ok(())
}
}
fn synthetic_redirect(from: &Url, to: &Url) -> RedirectInfo {
RedirectInfo {
source: RedirectSource::ServiceWorker,
from_url: from.clone(),
to_url: to.clone(),
status: 302,
headers: vec![("Location".to_owned(), to.as_str().as_bytes().to_vec())],
network_extra_info_available: false,
request_extra_info: None,
response_extra_info: None,
redirect_has_extra_info: false,
request_cookie_report: None,
cookie_set_reports: Vec::new(),
from_cache: false,
negotiated_http_version: None,
}
}
#[tokio::test]
async fn plain_http_requests_do_not_infer_browser_origin_from_referrer() -> Result<()> {
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("http"); 3]);
let url = Url::parse(&server.url())?;
let referrer = Url::parse("http://other.test/page")?;
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for transport in Transport::ALL {
let request = Request::new_bytes("GET", url.as_str(), None, Vec::new())?
.with_initiator_url(&referrer);
assert!(request.request_origin().is_none());
transport.fetch(&client, request).await?;
}
assert_eq!(server.hits(), 3);
for request in server.requests() {
assert!(!request.to_ascii_lowercase().contains("\r\norigin:"));
}
server.shutdown();
Ok(())
}
#[tokio::test]
async fn same_origin_mode_blocks_cross_origin_before_every_transport() -> Result<()> {
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::ok("forbidden").with_header("Access-Control-Allow-Origin", "*"),
]);
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for transport in Transport::ALL {
for origin in [Some(Url::parse("http://other.test/")?), None] {
let mut request =
Request::get(&server.url())?.with_request_mode(RequestMode::SameOrigin);
if let Some(origin) = origin {
request = request
.with_initiator_url(&origin)
.with_request_origin(moli_url::WebOrigin::from_url(&origin));
}
let error = transport.fetch(&client, request).await.unwrap_err();
assert!(
error.to_string().contains("same-origin request mode"),
"{transport:?}: {error:#}"
);
}
}
assert_eq!(
server.hits(),
0,
"rejected requests must not reach the server"
);
server.shutdown();
Ok(())
}
#[tokio::test]
async fn same_origin_mode_checks_redirects_in_every_transport() -> Result<()> {
let target = ScriptedHttpServer::spawn(vec![
ScriptedResponse::ok("forbidden").with_header("Access-Control-Allow-Origin", "*"),
]);
for transport in Transport::ALL {
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::status(302, "Found").with_header("Location", "/final"),
ScriptedResponse::ok("ok"),
ScriptedResponse::status(302, "Found").with_header("Location", &target.url()),
]);
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let request = Request::get(&server.url())?
.with_initiator_url(&Url::parse(&server.origin())?)
.with_request_origin(moli_url::WebOrigin::from_url(&Url::parse(
&server.origin(),
)?))
.with_request_mode(RequestMode::SameOrigin);
transport.fetch(&client, request.clone()).await?;
let error = transport.fetch(&client, request).await.unwrap_err();
assert!(
error.to_string().contains("same-origin request mode"),
"{transport:?}: {error:#}"
);
assert_eq!(server.hits(), 3);
server.shutdown();
}
assert_eq!(
target.hits(),
0,
"cross-origin redirect must not be followed"
);
target.shutdown();
Ok(())
}
#[tokio::test]
async fn same_origin_mode_rejects_inherited_cross_origin_history() -> Result<()> {
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("forbidden")]);
let url = Url::parse(&server.url())?;
let request = Request::get_with_url(url.clone())
.with_initiator_url(&url)
.with_request_origin(moli_url::WebOrigin::from_url(&url))
.with_request_mode(RequestMode::SameOrigin)
.with_redirect_chain(vec![synthetic_redirect(
&Url::parse("http://other.test/start")?,
&url,
)]);
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for transport in Transport::ALL {
let error = transport.fetch(&client, request.clone()).await.unwrap_err();
assert!(
error.to_string().contains("same-origin request mode"),
"{transport:?}: {error:#}"
);
}
assert_eq!(server.hits(), 0);
server.shutdown();
Ok(())
}
#[tokio::test]
async fn cors_redirect_rejection_never_contacts_next_hop() -> Result<()> {
let origin = Url::parse("http://origin.test/")?;
let target = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("forbidden")]);
for transport in Transport::ALL {
for (allow_origin, allow_credentials, credentials) in [
(None, false, RequestCredentialsMode::SameOrigin),
(None, false, RequestCredentialsMode::Include),
(
Some("http://wrong.test"),
false,
RequestCredentialsMode::SameOrigin,
),
(Some("*"), true, RequestCredentialsMode::Include),
(
Some("http://origin.test"),
false,
RequestCredentialsMode::Include,
),
] {
let mut redirect =
ScriptedResponse::status(302, "Found").with_header("Location", &target.url());
if let Some(value) = allow_origin {
redirect = redirect.with_header("Access-Control-Allow-Origin", value);
}
if allow_credentials {
redirect = redirect.with_header("Access-Control-Allow-Credentials", "true");
}
let server = ScriptedHttpServer::spawn(vec![redirect]);
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let request = Request::get(&server.url())?
.with_initiator_url(&origin)
.with_request_origin(moli_url::WebOrigin::from_url(&origin))
.with_request_mode(RequestMode::Cors)
.with_credentials_mode(credentials);
let error = transport.fetch(&client, request).await.unwrap_err();
assert!(
error.to_string().contains("CORS check failed:"),
"{transport:?}, {credentials:?}: {error:#}"
);
assert_eq!(server.hits(), 1);
assert_eq!(target.hits(), 0, "CORS must fail before the next request");
server.shutdown();
}
}
target.shutdown();
Ok(())
}
#[tokio::test]
async fn cors_redirect_validates_the_origin_before_recording_the_hop() -> Result<()> {
for transport in Transport::ALL {
for credentials in [
RequestCredentialsMode::SameOrigin,
RequestCredentialsMode::Include,
] {
let origin = Url::parse("http://origin.test/")?;
let target = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("ok")]);
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::status(302, "Found")
.with_header("Location", &target.url())
.with_header("Access-Control-Allow-Origin", "http://origin.test")
.with_header("Access-Control-Allow-Credentials", "true"),
]);
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let request = Request::get(&server.url())?
.with_initiator_url(&origin)
.with_request_origin(moli_url::WebOrigin::from_url(&origin))
.with_request_mode(RequestMode::Cors)
.with_credentials_mode(credentials);
transport.fetch(&client, request).await?;
assert_eq!(server.hits(), 1);
assert_eq!(target.hits(), 1);
assert!(
server.requests()[0]
.to_ascii_lowercase()
.contains("\r\norigin: http://origin.test\r\n")
);
assert!(
target.requests()[0]
.to_ascii_lowercase()
.contains("\r\norigin: null\r\n")
);
server.shutdown();
target.shutdown();
}
}
Ok(())
}
#[tokio::test]
async fn cors_redirect_inherits_synthetic_redirect_taint_without_checking_synthetic_headers()
-> Result<()> {
for transport in Transport::ALL {
for (allow_origin, allowed) in [("null", true), ("http://old.test", false)] {
let target = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("ok")]);
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::status(302, "Found")
.with_header("Location", &target.url())
.with_header("Access-Control-Allow-Origin", allow_origin),
]);
let url = Url::parse(&server.url())?;
let request = Request::get_with_url(url.clone())
.with_initiator_url(&url)
.with_request_origin(moli_url::WebOrigin::from_url(&url))
.with_request_mode(RequestMode::Cors)
.with_credentials_mode(RequestCredentialsMode::SameOrigin)
.with_redirect_chain(vec![synthetic_redirect(
&Url::parse("http://cross.test/sw")?,
&url,
)]);
let client =
FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
let result = transport.fetch(&client, request).await;
if allowed {
result?;
} else {
assert!(
result
.unwrap_err()
.to_string()
.contains("CORS check failed:")
);
}
assert_eq!(server.hits(), 1);
assert_eq!(target.hits(), usize::from(allowed));
server.shutdown();
target.shutdown();
}
}
Ok(())
}
#[tokio::test]
async fn cached_redirects_enforce_cors_and_same_origin_before_follow() -> Result<()> {
for transport in [Transport::Html, Transport::Raw] {
for mode in [RequestMode::Cors, RequestMode::SameOrigin] {
let cache_dir = unique_test_cache_dir();
let mut config = FetchConfig::default();
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
let target = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("ok")]);
let server = ScriptedHttpServer::spawn(vec![
ScriptedResponse::status(302, "Found")
.with_header("Location", &target.url())
.with_header("Cache-Control", "max-age=600"),
]);
let origin = if mode == RequestMode::SameOrigin {
Url::parse(&server.origin())?
} else {
Url::parse("http://other.test/")?
};
let request = Request::get(&server.url())?
.with_initiator_url(&origin)
.with_request_origin(moli_url::WebOrigin::from_url(&origin))
.with_request_mode(RequestMode::NoCors);
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
transport.fetch(&client, request.clone()).await?;
let error = transport
.fetch(&client, request.with_request_mode(mode))
.await
.unwrap_err();
let expected = if mode == RequestMode::SameOrigin {
"same-origin request mode"
} else {
"CORS check failed:"
};
assert!(
error.to_string().contains(expected),
"{transport:?}, {mode:?}: {error:#}"
);
assert_eq!(server.hits(), 1, "the second redirect must come from cache");
assert_eq!(
target.hits(),
1,
"the rejected cached redirect must not contact its target"
);
server.shutdown();
target.shutdown();
drop(client);
fs::remove_dir_all(cache_dir)?;
}
}
Ok(())
}
#[tokio::test]
async fn explicit_request_origin_controls_every_transport_independently_of_initiator_url()
-> Result<()> {
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("allowed"); 3]);
let url = Url::parse(&server.url())?;
let unrelated = Url::parse("http://unrelated.test/base/")?;
let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
for transport in Transport::ALL {
let opaque = Request::get_with_url(url.clone())
.with_initiator_url(&url)
.with_request_origin(moli_url::WebOrigin::Opaque)
.with_request_mode(RequestMode::SameOrigin)
.with_credentials_mode(RequestCredentialsMode::SameOrigin);
assert!(!opaque.allows_credentials_for_url(&url));
assert_eq!(opaque.serialized_origin(), "null");
let error = transport.fetch(&client, opaque).await.unwrap_err();
assert!(
error.to_string().contains("same-origin request mode"),
"{error:#}"
);
let allowed = Request::get_with_url(url.clone())
.with_initiator_url(&unrelated)
.with_request_origin((&url).into())
.with_request_mode(RequestMode::SameOrigin);
transport.fetch(&client, allowed).await?;
}
assert_eq!(
server.hits(),
3,
"only requests with the matching origin are dispatched"
);
server.shutdown();
Ok(())
}