Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

20.1 KB
Raw Blame
use super::*;
use moli_cookie_jar::new_shared_browser_cookie_store;
use std::{io::Write, sync::mpsc as std_mpsc};

fn submit_html_request(
    config: &FetchConfig,
    request: Request,
) -> (
    FetchRuntimeOwner,
    FetchCancelHandle,
    PendingStreamingHtmlResponse,
) {
    let owner = FetchRuntimeOwner::new(config, new_shared_browser_cookie_store());
    let pending = owner.handle().submit_html_stream(request).unwrap();
    let cancel = pending.cancel_handle.clone();
    (owner, cancel, pending)
}

#[tokio::test]
async fn html_redirect_timing_belongs_to_the_final_transfer() -> Result<()> {
    let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
    let url = format!("http://{}/redirect", listener.local_addr()?);
    let (release_redirect, ready) = std_mpsc::sync_channel(1);
    let server = thread::spawn(move || -> std::io::Result<()> {
        for path in ["/redirect", "/final"] {
            let (mut stream, _) = listener.accept()?;
            stream.set_read_timeout(Some(Duration::from_secs(5)))?;
            let mut request = Vec::new();
            let mut byte = [0];
            while !request.ends_with(b"\r\n\r\n") {
                stream.read_exact(&mut byte)?;
                request.extend_from_slice(&byte);
            }
            assert!(request.starts_with(format!("GET {path} ").as_bytes()));
            if path == "/redirect" {
                stream.write_all(
                    b"HTTP/1.1 103 Early Hints\r\n\r\n\
                      HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 8\r\nConnection: close\r\n\r\nold!",
                )?;
                ready.recv_timeout(Duration::from_secs(5)).unwrap();
                stream.write_all(b"tail")?;
            } else {
                stream.write_all(
                    b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
                )?;
            }
        }
        Ok(())
    });
    let (_owner, cancel, pending) =
        submit_html_request(&FetchConfig::default(), Request::get(&url)?);
    let deadline = Instant::now() + Duration::from_secs(5);
    while !cancel.response_timing().body_bytes_received() {
        anyhow::ensure!(Instant::now() < deadline, "redirect body was not observed");
        tokio::time::sleep(Duration::from_millis(1)).await;
    }
    let redirect_timing = cancel.response_timing();
    assert!(
        redirect_timing
            .first_interim_response_started_at()
            .is_some()
    );
    assert!(redirect_timing.response_started_at().is_some());
    let redirect_finished_after = Instant::now();
    release_redirect.send(())?;

    let mut response = pending.into_response().await?;
    assert_eq!(response.final_url.path(), "/final");
    assert!(response.next_chunk().await.is_none());
    response.finish().await?;
    let timing = cancel.response_timing();
    assert!(timing.request_started_at().unwrap() >= redirect_finished_after);
    assert!(timing.response_started_at().unwrap() >= redirect_finished_after);
    assert!(timing.first_interim_response_started_at().is_none());
    assert!(!timing.body_bytes_received());
    server.join().expect("server thread")?;
    Ok(())
}

#[tokio::test]
async fn html_response_timing_finishes_without_consumer_polling() -> Result<()> {
    for (framing, body) in [
        ("Content-Length: 4", b"body".as_slice()),
        (
            "Transfer-Encoding: chunked",
            b"4\r\nbody\r\n0\r\n\r\n".as_slice(),
        ),
    ] {
        let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
        let url = format!("http://{}/timed", listener.local_addr()?);
        let (release_body, ready) = std_mpsc::sync_channel(1);
        let server = thread::spawn(move || -> std::io::Result<()> {
            let (mut stream, _) = listener.accept()?;
            read_request(&mut stream)?;
            write!(
                stream,
                "HTTP/1.1 200 OK\r\n{framing}\r\nConnection: close\r\n\r\n"
            )?;
            ready.recv_timeout(Duration::from_secs(5)).unwrap();
            stream.write_all(body)?;
            Ok(())
        });
        let (_owner, cancel, pending) =
            submit_html_request(&FetchConfig::default(), Request::get(&url)?);
        let mut response = pending.into_response().await?;
        let headers = cancel.response_timing();
        assert!(headers.response_started_at().is_some());
        assert!(headers.response_end_at().is_none());
        release_body.send(())?;
        // Keep the consumer's executor occupied while native threads finish.
        let deadline = Instant::now() + Duration::from_secs(5);
        while !cancel.response_completion_is_committed() {
            anyhow::ensure!(
                Instant::now() < deadline,
                "HTML transfer did not record completion"
            );
            thread::sleep(Duration::from_millis(1));
        }
        let native = cancel.response_timing();
        assert!(native.body_bytes_received());
        assert!(native.response_started_at().unwrap() <= native.response_end_at().unwrap());
        assert!(cancel.response_completion_is_committed());
        assert!(!cancel.cancel_if_response_pending());
        let mut text = String::new();
        while let Some(chunk) = response.next_chunk().await {
            text.push_str(&chunk);
        }
        response.finish().await?;
        assert_eq!(text, "body");
        assert_eq!(cancel.response_timing(), native);
        server.join().expect("server thread")?;
    }
    Ok(())
}

fn read_request(stream: &mut std::net::TcpStream) -> std::io::Result<()> {
    stream.set_read_timeout(Some(Duration::from_secs(5)))?;
    let mut request = Vec::new();
    let mut byte = [0];
    while !request.ends_with(b"\r\n\r\n") {
        stream.read_exact(&mut byte)?;
        request.extend_from_slice(&byte);
    }
    Ok(())
}

#[tokio::test]
async fn html_response_timing_records_terminal_failures() -> Result<()> {
    for bytes in [
        b"".as_slice(),
        b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\npart".as_slice(),
    ] {
        let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
        let url = format!("http://{}/failure", listener.local_addr()?);
        let server = thread::spawn(move || -> std::io::Result<()> {
            let (mut stream, _) = listener.accept()?;
            read_request(&mut stream)?;
            stream.write_all(bytes)?;
            Ok(())
        });
        let (_owner, cancel, pending) =
            submit_html_request(&FetchConfig::default(), Request::get(&url)?);
        match pending.into_response().await {
            Ok(mut response) => assert!(response.finish().await.is_err()),
            Err(_) => assert!(bytes.is_empty()),
        }
        let timing = cancel.response_timing();
        assert_eq!(timing.response_started_at().is_some(), !bytes.is_empty());
        assert!(timing.response_end_at().is_some());
        assert!(cancel.response_completion_is_committed());
        assert!(!cancel.cancel_if_response_pending());
        server.join().expect("server thread")?;
    }
    Ok(())
}

#[tokio::test]
async fn html_response_timing_finishes_bodyless_responses_at_headers() -> Result<()> {
    for (method, headers) in [
        ("GET", "HTTP/1.1 101 Switching Protocols\r\n"),
        ("GET", "HTTP/1.1 204 No Content\r\n"),
        ("GET", "HTTP/1.1 205 Reset Content\r\n"),
        ("GET", "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n"),
        ("HEAD", "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n"),
    ] {
        let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
        let url = format!("http://{}/empty", listener.local_addr()?);
        let (release, ready) = std_mpsc::sync_channel(1);
        let server = thread::spawn(move || -> std::io::Result<()> {
            let (mut stream, _) = listener.accept()?;
            read_request(&mut stream)?;
            write!(stream, "{headers}Connection: close\r\n\r\n")?;
            ready.recv_timeout(Duration::from_secs(5)).unwrap();
            Ok(())
        });
        let request = Request::new(method, &url, None, Vec::new())?;
        let (_owner, cancel, pending) = submit_html_request(&FetchConfig::default(), request);
        let mut response = pending.into_response().await?;
        tokio::time::timeout(Duration::from_secs(5), response.finish()).await??;
        let timing = cancel.response_timing();
        assert!(timing.response_end_at().is_some(), "{method}: {headers}");
        assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());
        assert!(!timing.body_bytes_received());
        assert!(!cancel.cancel_if_response_pending());
        release.send(())?;
        server.join().expect("server thread")?;
    }
    Ok(())
}

#[tokio::test]
async fn html_response_timing_preserves_cancellation_before_body_completion() -> Result<()> {
    let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
    let url = format!("http://{}/cancel", listener.local_addr()?);
    let (release, ready) = std_mpsc::sync_channel(1);
    let server = thread::spawn(move || -> std::io::Result<()> {
        let (mut stream, _) = listener.accept()?;
        read_request(&mut stream)?;
        stream
            .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\npart")?;
        ready.recv_timeout(Duration::from_secs(5)).unwrap();
        Ok(())
    });
    let (_owner, cancel, pending) =
        submit_html_request(&FetchConfig::default(), Request::get(&url)?);
    let mut response = pending.into_response().await?;
    assert_eq!(response.next_chunk().await.as_deref(), Some("part"));
    assert!(cancel.response_timing().response_end_at().is_none());
    assert!(cancel.cancel_if_response_pending());
    assert!(
        tokio::time::timeout(Duration::from_secs(5), response.finish())
            .await?
            .is_err()
    );
    assert!(cancel.response_timing().response_end_at().is_some());
    assert!(!cancel.response_completion_is_committed());
    release.send(())?;
    server.join().expect("server thread")?;
    Ok(())
}

#[tokio::test]
async fn html_cached_response_timing_does_not_invent_network_observations() -> Result<()> {
    let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
    let url = format!("http://{}/cache", listener.local_addr()?);
    let cache_dir = std::env::temp_dir().join(format!(
        "moli-html-timing-{}-{}",
        std::process::id(),
        std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)?
            .as_nanos(),
    ));
    let server = thread::spawn(move || -> std::io::Result<()> {
        let (mut stream, _) = listener.accept()?;
        read_request(&mut stream)?;
        stream.write_all(b"HTTP/1.1 200 OK\r\nCache-Control: max-age=600\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody")?;
        Ok(())
    });
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let (owner, _, pending) = submit_html_request(&config, Request::get(&url)?);
    pending.into_response().await?.finish().await?;
    server.join().expect("server thread")?;

    let pending = owner.handle().submit_html_stream(Request::get(&url)?)?;
    let cancel = pending.cancel_handle.clone();
    let mut response = pending.into_response().await?;
    assert!(response.from_cache);
    let mut text = String::new();
    while let Some(chunk) = response.next_chunk().await {
        text.push_str(&chunk);
    }
    response.finish().await?;
    assert_eq!(text, "body");
    let timing = cancel.response_timing();
    assert!(timing.request_started_at().is_none());
    assert!(timing.first_interim_response_started_at().is_none());
    assert!(!timing.body_bytes_received());
    assert!(timing.response_started_at().is_some());
    assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());
    assert!(!cancel.cancel_if_response_pending());
    drop(owner);
    std::fs::remove_dir_all(cache_dir)?;
    Ok(())
}

#[tokio::test]
async fn html_declared_body_completion_precedes_delivery_of_the_last_chunk() -> Result<()> {
    for (encoding_headers, complete) in [
        (b"".as_slice(), true),
        (b"Content-Encoding: gzip\r\n".as_slice(), false),
        (b"Transfer-Encoding: chunked\r\n".as_slice(), false),
    ] {
        let cancel = FetchCancelHandle::new();
        let (start_tx, _start_rx) = oneshot::channel();
        let (body_tx, mut body_rx) = mpsc::unbounded_channel();
        let mut collector = StreamingResponseCollector::new(
            new_shared_browser_cookie_store(),
            start_tx,
            body_tx,
            cancel.clone(),
        );
        collector.begin_request(
            None,
            Url::parse("http://example.test/body")?,
            NetworkCookieRequestContext::top_level_navigation("GET"),
            None,
            true,
            Vec::new(),
            None,
        );
        let mut handler = FetchTransferHandler::new_streaming(collector);
        assert!(handler.header(b"HTTP/1.1 200 OK\r\n"));
        assert!(handler.header(b"Content-Length: 4\r\n"));
        if !encoding_headers.is_empty() {
            assert!(handler.header(encoding_headers));
        }
        assert!(handler.header(b"\r\n"));
        assert_eq!(handler.write(b"bo").unwrap(), 2);
        assert_eq!(body_rx.try_recv()?.as_str(), "bo");
        assert!(cancel.response_timing().body_bytes_received());
        assert!(cancel.response_timing().response_end_at().is_none());
        assert_eq!(handler.write(b"dy").unwrap(), 2);
        assert_eq!(body_rx.try_recv()?.as_str(), "dy");
        // Decoded or chunked lengths cannot be compared with Content-Length.
        assert_eq!(
            cancel.response_timing().response_end_at().is_some(),
            complete
        );
        assert_eq!(cancel.cancel_if_response_pending(), !complete);
    }
    Ok(())
}

#[test]
fn switching_protocols_timing_is_final_in_every_collector() {
    for interim in [false, true] {
        for mode in ["buffered", "html", "raw"] {
            let cancel = FetchCancelHandle::new();
            let mut handler = match mode {
                "buffered" => {
                    FetchTransferHandler::new_buffered(ResponseCollector::new(Some(cancel.clone())))
                }
                "html" => {
                    let (start_tx, _start_rx) = oneshot::channel();
                    let (body_tx, _body_rx) = mpsc::unbounded_channel();
                    FetchTransferHandler::new_streaming(StreamingResponseCollector::new(
                        new_shared_browser_cookie_store(),
                        start_tx,
                        body_tx,
                        cancel.clone(),
                    ))
                }
                "raw" => {
                    let (start_tx, _start_rx) = oneshot::channel();
                    let (body_tx, _body_rx) = mpsc::unbounded_channel();
                    FetchTransferHandler::new_raw_streaming(RawStreamingResponseCollector::new(
                        new_shared_browser_cookie_store(),
                        start_tx,
                        body_tx,
                        cancel.clone(),
                    ))
                }
                _ => unreachable!(),
            };
            if interim {
                assert!(handler.header(b"HTTP/1.1 103 Early Hints\r\n"));
                assert!(cancel.response_timing().response_started_at().is_none());
            }
            assert!(handler.header(b"HTTP/1.1 101 Switching Protocols\r\n"));
            let timing = cancel.response_timing();
            assert!(
                timing.response_started_at().is_some(),
                "{mode}: 101 must be final"
            );
            assert_eq!(
                timing.first_interim_response_started_at().is_some(),
                interim,
                "{mode}"
            );
            if let Some(first_interim) = timing.first_interim_response_started_at() {
                assert!(first_interim <= timing.response_started_at().unwrap());
            }
        }
    }
}

#[test]
fn transfer_progress_preserves_cancellation_and_body_observations_on_reuse() {
    let cancel = FetchCancelHandle::new();
    let mut handler =
        FetchTransferHandler::new_buffered(ResponseCollector::new(Some(cancel.clone())));
    for _ in 0..2 {
        cancel.begin_response_progress();
        handler.buffered_mut().unwrap().begin_request(None);
        handler.begin_transfer(None, false);
        assert!(handler.progress(4.0, 0.0, 0.0, 0.0));
        assert!(!cancel.response_timing().body_bytes_received());
        assert!(handler.progress(4.0, 1.0, 0.0, 0.0));
        assert!(cancel.response_timing().body_bytes_received());
        assert!(handler.progress(4.0, 4.0, 0.0, 0.0));
    }
    cancel.cancel();
    assert!(!handler.progress(4.0, 4.0, 0.0, 0.0));
}

#[tokio::test]
async fn html_redirect_to_a_cached_response_preserves_channels_and_timing() -> Result<()> {
    let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
    let address = format!("http://{}", listener.local_addr()?);
    let cache_dir = std::env::temp_dir().join(format!(
        "moli-html-redirect-timing-{}-{}",
        std::process::id(),
        std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)?
            .as_nanos(),
    ));
    let server = thread::spawn(move || -> std::io::Result<()> {
        for response in [
            b"HTTP/1.1 200 OK\r\nCache-Control: max-age=600\r\nContent-Length: 4\r\nConnection: close\r\n\r\nbody".as_slice(),
            b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 302 Found\r\nLocation: /cache\r\nContent-Length: 4\r\nConnection: close\r\n\r\njunk".as_slice(),
        ] {
            let (mut stream, _) = listener.accept()?;
            read_request(&mut stream)?;
            stream.write_all(response)?;
        }
        Ok(())
    });
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let (owner, _, pending) =
        submit_html_request(&config, Request::get(&format!("{address}/cache"))?);
    pending.into_response().await?.finish().await?;

    let pending = owner
        .handle()
        .submit_html_stream(Request::get(&format!("{address}/redirect"))?)?;
    let cancel = pending.cancel_handle.clone();
    let mut response = pending.into_response().await?;
    assert!(response.from_cache);
    assert!(response.redirected);
    assert_eq!(response.final_url.path(), "/cache");
    assert_eq!(response.redirect_chain.len(), 1);
    assert!(!response.redirect_chain[0].from_cache);
    let mut body = String::new();
    while let Some(chunk) = response.next_chunk().await {
        body.push_str(&chunk);
    }
    response.finish().await?;
    assert_eq!(body, "body");
    let timing = cancel.response_timing();
    assert!(timing.request_started_at().is_none());
    assert!(timing.first_interim_response_started_at().is_none());
    assert!(!timing.body_bytes_received());
    assert!(timing.response_started_at().unwrap() <= timing.response_end_at().unwrap());
    assert!(!cancel.cancel_if_response_pending());
    server.join().expect("server thread")?;
    drop(owner);
    std::fs::remove_dir_all(cache_dir)?;
    Ok(())
}

#[tokio::test]
async fn html_failure_before_reusing_an_easy_preserves_the_original_error() -> Result<()> {
    let cancel = FetchCancelHandle::new();
    let (started_tx, started_rx) = oneshot::channel();
    let (body_tx, mut body_rx) = mpsc::unbounded_channel();
    let (completion_tx, completion_rx) = oneshot::channel();
    let mut job = StreamingRuntimeJob::new(
        Request::get("http://example.test/cache")?,
        started_tx,
        body_tx,
        completion_tx,
        cancel.clone(),
    );
    job.easy = Some(Easy2::new(FetchTransferHandler::new_streaming(
        StreamingResponseCollector::new(
            new_shared_browser_cookie_store(),
            job.started_tx.take().unwrap(),
            job.body_tx.take().unwrap(),
            cancel.clone(),
        ),
    )));
    fail_streaming_job_with_easy(job, None, anyhow!("cached redirect rejected"));
    let error = started_rx.await?.unwrap_err();
    assert!(format!("{error:#}").contains("cached redirect rejected"));
    assert!(body_rx.recv().await.is_none());
    assert!(completion_rx.await?.is_err());
    assert!(cancel.response_timing().response_end_at().is_some());
    Ok(())
}