Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

21.4 KB
Raw Blame
use super::*;

async fn serve_responses(listener: TcpListener, responses: Vec<Vec<u8>>) -> Result<Vec<String>> {
    let mut requests = Vec::new();
    for response in responses {
        let (mut stream, _) =
            tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;
        let mut request = Vec::new();
        while !request.windows(4).any(|part| part == b"\r\n\r\n") {
            let mut chunk = [0; 1024];
            let count =
                tokio::time::timeout(Duration::from_secs(5), stream.read(&mut chunk)).await??;
            anyhow::ensure!(
                count != 0 && request.len() < 64 * 1024,
                "incomplete request"
            );
            request.extend_from_slice(&chunk[..count]);
        }
        requests.push(String::from_utf8(request)?);
        stream.write_all(&response).await?;
        stream.shutdown().await?;
    }
    Ok(requests)
}

async fn fetch_in_mode(
    client: &FetchClientHandle,
    request: Request,
    mode: &str,
) -> Result<(ResponseHead, Vec<u8>)> {
    let (head, body) = match mode {
        "buffered" => client.fetch(request.with_single_hop()).await?.into_body(),
        "html" => client.fetch_html_stream(request).await?.into_body(),
        "raw" => client
            .fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
            .await?
            .into_body(),
        _ => unreachable!(),
    };
    Ok((head, body.into_materialized_bytes().await?))
}

#[tokio::test]
async fn header_eof_delivers_complete_fields_on_every_transport() -> Result<()> {
    for (method, response, status, headers) in [
        ("GET", b"HTTP/1.1 200 OK\r\nX-Test: first\r\nX-Test: second\r\nX-Bytes: \xff\r\n".as_slice(), 200,
            vec![("x-test", b"first".as_slice()), ("x-test", b"second".as_slice()), ("x-bytes", b"\xff".as_slice())]),
        ("GET", b"HTTP/1.0 200 NANANA\nCONTENT-LENGTH:  0\ncontent-length:\t 0\n", 200,
            vec![("content-length", b"0".as_slice()), ("content-length", b"0".as_slice())]),
        ("GET", b"HTTP/1.1 280 HELLO\nfoo-test: 1\nfoo-test: 2\nfoo-test: 3\n", 280,
            vec![("foo-test", b"1".as_slice()), ("foo-test", b"2".as_slice()), ("foo-test", b"3".as_slice())]),
        ("GET", b"HTTP/1.1 200 OK\r\n", 200, vec![]),
        ("GET", b"HTTP/1.1 204 No Content\r\nX-Test: empty\r\n", 204,
            vec![("x-test", b"empty".as_slice())]),
        ("GET", b"HTTP/1.1 304 Not Modified\r\nX-Test: empty\r\n", 304,
            vec![("x-test", b"empty".as_slice())]),
        ("GET", b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n", 200,
            vec![("transfer-encoding", b"chunked".as_slice())]),
        ("HEAD", b"HTTP/1.1 200 OK\r\nContent-Length: 40\r\nX-Test: head\r\n", 200,
            vec![("content-length", b"40".as_slice()), ("x-test", b"head".as_slice())]),
        ("GET", b"HTTP/1.1 103 Early Hints\r\nX-Interim: hidden\r\n\r\nHTTP/1.1 200 OK\r\nX-Test: final\r\n", 200,
            vec![("x-test", b"final".as_slice())]),
    ] {
        let listener = TcpListener::bind("127.0.0.1:0").await?;
        let url = format!("http://{}/eof", listener.local_addr()?);
        let server = tokio::spawn(serve_responses(listener, vec![response.to_vec(); 3]));
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        for mode in ["buffered", "html", "raw"] {
            let (head, body) = fetch_in_mode(&client, Request::new(method, &url, None, vec![])?, mode)
                .await.with_context(|| format!("{mode}: {response:?}"))?;
            assert_eq!(head.status, status, "{mode}");
            assert_eq!(head.headers, headers.iter().map(|(n, v)| (n.to_string(), v.to_vec())).collect::<Vec<_>>(), "{mode}");
            assert!(body.is_empty(), "{mode}: {body:?}");
        }
        assert_eq!(server.await??.len(), 3);
        assert!(client.shutdown().is_clean());
    }
    Ok(())
}

#[tokio::test]
async fn header_eof_does_not_recover_transport_failures() -> Result<()> {
    for response in [
        b"".as_slice(),
        b"HTTP/1.1 103 Early Hints\r\nX-Interim: hidden\r\n\r\n",
        b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 4\r\nX-Test: conflict\r\n",
        b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nx",
        b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nx",
    ] {
        let listener = TcpListener::bind("127.0.0.1:0").await?;
        let url = format!("http://{}/failure", listener.local_addr()?);
        let server = tokio::spawn(serve_responses(listener, vec![response.to_vec(); 3]));
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        for mode in ["buffered", "html", "raw"] {
            // Subresource failures must not trigger the navigation HTTPS-upgrade retry.
            let request =
                Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
            assert!(
                fetch_in_mode(&client, request, mode).await.is_err(),
                "{mode}: {response:?}"
            );
        }
        assert_eq!(server.await??.len(), 3);
        assert!(client.shutdown().is_clean());
    }
    Ok(())
}

#[tokio::test]
async fn header_eof_processes_cookies_before_following_redirects() -> Result<()> {
    for mode in ["html", "raw"] {
        let listener = TcpListener::bind("127.0.0.1:0").await?;
        let url = format!("http://{}/start", listener.local_addr()?);
        let server = tokio::spawn(serve_responses(
            listener,
            vec![
                b"HTTP/1.1 302 Found\r\nLocation: /final\r\nSet-Cookie: redirect=one; Path=/\r\n"
                    .to_vec(),
                b"HTTP/1.1 200 OK\r\nSet-Cookie: final=two; Path=/\r\nX-Test: done\r\n".to_vec(),
            ],
        ));
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
        assert_eq!(head.status, 200, "{mode}");
        assert_eq!(head.final_url.path(), "/final", "{mode}");
        assert!(head.redirected);
        assert_eq!(head.redirect_chain.len(), 1);
        assert_eq!(head.redirect_chain[0].cookie_set_reports.len(), 1);
        assert_eq!(head.cookie_set_reports.len(), 1);
        assert!(body.is_empty());
        let requests = server.await??;
        assert_eq!(requests.len(), 2);
        assert!(requests[1].starts_with("GET /final "));
        assert!(
            requests[1].contains("Cookie: redirect=one\r\n"),
            "{}",
            requests[1]
        );
        assert!(client.shutdown().is_clean());
    }
    Ok(())
}

#[tokio::test]
async fn header_eof_keeps_manual_raw_redirects_unfollowed() -> Result<()> {
    for headers_terminated in [false, true] {
        let listener = TcpListener::bind("127.0.0.1:0").await?;
        let url = format!("http://{}/start", listener.local_addr()?);
        let mut response =
            b"HTTP/1.1 302 Found\r\nLocation: /final\r\nSet-Cookie: redirect=one; Path=/\r\n"
                .to_vec();
        if headers_terminated {
            response.extend_from_slice(b"\r\n");
        }
        let server = tokio::spawn(serve_responses(listener, vec![response]));
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        let (head, body) =
            fetch_in_mode(&client, Request::get(&url)?.with_single_hop(), "raw").await?;
        assert_eq!(head.status, 302);
        assert_eq!(head.final_url.path(), "/start");
        assert!(!head.redirected);
        assert_eq!(
            head.cookie_set_reports.len(),
            1,
            "headers_terminated={headers_terminated}"
        );
        assert!(body.is_empty());
        assert_eq!(server.await??.len(), 1);
        assert!(client.shutdown().is_clean());
    }
    Ok(())
}

#[tokio::test]
async fn header_eof_creates_reusable_empty_cache_entries() -> Result<()> {
    for mode in ["html", "raw"] {
        let cache_dir = unique_test_cache_dir();
        let listener = TcpListener::bind("127.0.0.1:0").await?;
        let url = format!("http://{}/cached", listener.local_addr()?);
        let server = tokio::spawn(serve_responses(
            listener,
            vec![b"HTTP/1.1 200 OK\r\nCache-Control: max-age=60\r\nX-Test: cached\r\n".to_vec()],
        ));
        let mut config = FetchConfig::default();
        config.set_http_cache_dir(Some(cache_dir.display().to_string()));
        for from_cache in [false, true] {
            let client = FetchClient::new(&config, new_shared_browser_cookie_store());
            let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
            assert_eq!(head.from_cache, from_cache, "{mode}");
            assert!(head.headers.contains(&("x-test".into(), "cached".into())));
            assert!(body.is_empty());
            assert!(client.shutdown().is_clean());
        }
        assert_eq!(server.await??.len(), 1);
        fs::remove_dir_all(cache_dir)?;
    }
    Ok(())
}

#[tokio::test]
async fn header_eof_revalidates_cached_raw_bodies() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let url = format!("http://{}/revalidate", listener.local_addr()?);
    let server = tokio::spawn(serve_responses(listener, vec![
        b"HTTP/1.1 200 OK\r\nCache-Control: no-cache\r\nETag: \"eof\"\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody".to_vec(),
        b"HTTP/1.1 304 Not Modified\r\nCache-Control: max-age=60\r\nETag: \"eof\"\r\nX-Test: updated\r\nSet-Cookie: revalidated=one; Path=/\r\n".to_vec(),
        b"HTTP/1.1 200 OK\r\nX-Test: cookie-check\r\n".to_vec(),
    ]));
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let (first, body) = fetch_in_mode(&client, Request::get(&url)?, "raw").await?;
    assert!(!first.from_cache);
    assert_eq!(body, b"body");
    let (revalidated, body) = fetch_in_mode(&client, Request::get(&url)?, "raw").await?;
    assert_eq!(revalidated.status, 200);
    assert!(revalidated.from_cache);
    assert_eq!(body, b"body");
    assert!(
        revalidated
            .headers
            .contains(&("x-test".into(), "updated".into()))
    );
    let (_, body) = fetch_in_mode(
        &client,
        Request::get(&format!("{url}/cookie-check"))?,
        "raw",
    )
    .await?;
    assert!(body.is_empty());
    let requests = server.await??;
    assert_eq!(requests.len(), 3);
    assert!(
        requests[1]
            .to_ascii_lowercase()
            .contains("if-none-match: \"eof\"\r\n")
    );
    assert!(requests[2].contains("Cookie: revalidated=one\r\n"));
    assert!(client.shutdown().is_clean());
    fs::remove_dir_all(cache_dir)?;
    Ok(())
}

#[tokio::test]
async fn header_eof_applies_critical_client_hints_before_publishing() -> Result<()> {
    for mode in ["html", "raw"] {
        let listener = TcpListener::bind("127.0.0.1:0").await?;
        let url = format!("http://{}/navigation", listener.local_addr()?);
        let server = tokio::spawn(serve_responses(listener, vec![
            b"HTTP/1.1 403 Challenge\r\nAccept-CH: Sec-CH-UA-Arch\r\nCritical-CH: Sec-CH-UA-Arch\r\nSet-Cookie: challenge=one; Path=/\r\n".to_vec(),
            b"HTTP/1.1 200 OK\r\nX-Test: restarted\r\n".to_vec(),
        ]));
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode).await?;
        assert_eq!(head.status, 200, "{mode}");
        assert_eq!(head.redirect_chain.len(), 1);
        let restart = &head.redirect_chain[0];
        assert_eq!(restart.from_url, restart.to_url);
        assert_eq!(restart.response_extra_info.as_ref().unwrap().status, 403);
        assert_eq!(
            restart
                .response_extra_info
                .as_ref()
                .unwrap()
                .cookie_set_reports
                .len(),
            1
        );
        assert!(body.is_empty());
        let requests = server.await??;
        assert_eq!(requests.len(), 2);
        assert!(!requests[0].to_ascii_lowercase().contains("sec-ch-ua-arch:"));
        assert!(
            requests[1]
                .to_ascii_lowercase()
                .contains("sec-ch-ua-arch: \"x86\"")
        );
        assert!(requests[1].contains("Cookie: challenge=one\r\n"));
        assert!(client.shutdown().is_clean());
    }
    Ok(())
}

#[tokio::test]
async fn header_eof_preserves_the_response_size_limit() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let url = format!("http://{}/limited", listener.local_addr()?);
    let server = tokio::spawn(serve_responses(
        listener,
        vec![b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\nX-Test: oversized\r\n".to_vec(); 3],
    ));
    let mut config = FetchConfig::default();
    config.set_connection_limits(None, None, Some(4));
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    for mode in ["buffered", "html", "raw"] {
        let error = fetch_in_mode(&client, Request::get(&url)?, mode)
            .await
            .unwrap_err();
        if mode != "buffered" {
            assert!(
                error
                    .to_string()
                    .contains("response exceeded configured limit of 4 bytes"),
                "{mode}: {error:#}"
            );
        }
    }
    assert_eq!(server.await??.len(), 3);
    assert!(client.shutdown().is_clean());
    Ok(())
}

#[tokio::test]
async fn header_eof_does_not_publish_a_cancelled_response() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let url = format!("http://{}/cancel", listener.local_addr()?);
    let (headers_tx, headers_rx) = oneshot::channel();
    let (close_tx, close_rx) = oneshot::channel();
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await?;
        let mut request = Vec::new();
        while !request.ends_with(b"\r\n\r\n") {
            let mut byte = [0];
            stream.read_exact(&mut byte).await?;
            request.push(byte[0]);
        }
        stream
            .write_all(b"HTTP/1.1 200 OK\r\nX-Test: pending\r\n")
            .await?;
        let _ = headers_tx.send(());
        close_rx.await?;
        stream.shutdown().await?;
        Ok::<_, anyhow::Error>(())
    });
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let cancel = FetchCancelHandle::new();
    let request = Request::get(&url)?;
    let mut response = Box::pin(client.fetch_raw_stream_with_cancel(request, cancel.clone()));
    tokio::select! {
        result = &mut response => panic!("response published before headers ended: {result:?}"),
        result = headers_rx => result?,
    }
    cancel.cancel();
    let _ = close_tx.send(());
    assert!(
        tokio::time::timeout(Duration::from_secs(5), response)
            .await?
            .is_err()
    );
    server.await??;
    assert!(client.shutdown().is_clean());
    Ok(())
}

#[tokio::test]
async fn bare_cr_header_eof_matches_chromium_after_final_and_informational_headers() -> Result<()> {
    for prefix in [b"".as_slice(), b"HTTP/1.1 100 Continue\r\n\r\n"] {
        for byte in [b'X', b':', b'\t', b'\r', 0xff] {
            let response = [
                prefix,
                b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nX-Test: retained\r\n\r",
                &[byte, b'\n'],
                b"abc",
            ]
            .concat();
            let listener = TcpListener::bind("127.0.0.1:0").await?;
            let url = format!("http://{}/bare-cr-eof", listener.local_addr()?);
            let server = tokio::spawn(serve_responses(listener, vec![response; 3]));
            let client =
                FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
            for mode in ["buffered", "html", "raw"] {
                let (head, body) = fetch_in_mode(&client, Request::get(&url)?, mode)
                    .await
                    .with_context(|| format!("{mode}, prefix={prefix:?}, byte={byte:#x}"))?;
                assert_eq!(head.status, 200, "{mode}");
                assert!(
                    head.headers
                        .contains(&("content-length".into(), b"3".to_vec()))
                );
                assert!(
                    head.headers
                        .contains(&("x-test".into(), b"retained".to_vec()))
                );
                // Chromium consumes these bytes as an unterminated header block.
                assert!(body.is_empty(), "{mode}: {body:?}");
            }
            assert_eq!(server.await??.len(), 3);
            assert!(client.shutdown().is_clean());
        }
    }
    Ok(())
}

#[tokio::test]
async fn header_eof_rejects_nul_in_a_pending_field_on_every_transport() -> Result<()> {
    for suffix in [b"X-Test: one\0two".as_slice(), b"\r\0\n"] {
        let response = [
            b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n".as_slice(),
            suffix,
        ]
        .concat();
        let listener = TcpListener::bind("127.0.0.1:0").await?;
        let url = format!("http://{}/nul-eof", listener.local_addr()?);
        let server = tokio::spawn(serve_responses(listener, vec![response; 3]));
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        for mode in ["buffered", "html", "raw"] {
            let request =
                Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
            assert!(
                fetch_in_mode(&client, request, mode).await.is_err(),
                "{mode}"
            );
        }
        assert_eq!(server.await??.len(), 3);
        assert!(client.shutdown().is_clean());
    }
    Ok(())
}

#[tokio::test]
async fn bounded_leading_cr_status_lines_work_after_informational_responses() -> Result<()> {
    for prefix in ["", "HTTP/1.1 100 Continue\r\n\r\n"] {
        for count in 1..=5 {
            let response = format!(
                "{prefix}{}HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc",
                "\r".repeat(count)
            );
            let listener = TcpListener::bind("127.0.0.1:0").await?;
            let url = format!("http://{}/status-slop", listener.local_addr()?);
            let server = tokio::spawn(serve_responses(listener, vec![response.into_bytes(); 3]));
            let client =
                FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
            for mode in ["buffered", "html", "raw"] {
                let request = Request::get(&url)?
                    .with_browser_request_metadata(BrowserRequestMetadata::Fetch);
                let result = fetch_in_mode(&client, request, mode).await;
                if count <= 4 {
                    let (head, body) = result.with_context(|| format!("{mode}, count={count}"))?;
                    assert_eq!(head.status, 200);
                    assert_eq!(body, b"abc");
                } else {
                    assert!(result.is_err(), "{mode}: status slop exceeds four bytes");
                }
            }
            assert_eq!(server.await??.len(), 3);
            assert!(client.shutdown().is_clean());
        }
    }
    Ok(())
}

#[tokio::test]
async fn https_header_eof_is_rejected_while_bare_cr_body_bytes_are_preserved() -> Result<()> {
    let cases = [
        (
            b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\n\rX\n".as_slice(),
            true,
        ),
        (
            b"HTTP/1.1 200 OK\r\nX-Test: truncated\r\n".as_slice(),
            false,
        ),
        (
            b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\rX\nabc".as_slice(),
            false,
        ),
    ];
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let url = format!("https://{}/eof", listener.local_addr()?);
    let server = tokio::spawn(async move {
        let acceptor = super::support::http11_tls_acceptor();
        for (response, _) in cases.iter().cycle().take(cases.len() * 3) {
            let (stream, _) =
                tokio::time::timeout(Duration::from_secs(5), listener.accept()).await??;
            let mut stream = acceptor.accept(stream).await?;
            let mut request = Vec::new();
            while !request.ends_with(b"\r\n\r\n") {
                let mut byte = [0];
                stream.read_exact(&mut byte).await?;
                request.push(byte[0]);
            }
            stream.write_all(response).await?;
            stream.shutdown().await?;
        }
        Ok::<_, anyhow::Error>(())
    });
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    for mode in ["buffered", "html", "raw"] {
        for (_, succeeds) in cases {
            let request =
                Request::get(&url)?.with_browser_request_metadata(BrowserRequestMetadata::Fetch);
            let result = fetch_in_mode(&client, request, mode).await;
            if succeeds {
                let (head, body) =
                    result.with_context(|| format!("{mode}: complete TLS response"))?;
                assert_eq!(head.status, 200);
                assert_eq!(body, b"\rX\n");
            } else {
                assert!(
                    result.is_err(),
                    "{mode}: HTTPS must reject incomplete headers"
                );
            }
        }
    }
    server.await??;
    assert!(client.shutdown().is_clean());
    Ok(())
}