Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

229 KB
Raw Blame
mod cache_request_controls;
mod cookie_context;
mod cookie_store;
mod header_bytes;
mod header_eof;
mod mixed_transport;
mod request_security;
mod response_progress;
mod response_timing;
mod support;
mod tls_credentials;
mod websocket_transport;

use anyhow::{Context, Result};
use curl::easy::Handler;
use moli_browser_profile::DEFAULT_ACCEPT_LANGUAGE;
use moli_cookie_jar::{NetworkCookieRequestContext, new_shared_browser_cookie_store};
use moli_http_cache::{HttpCacheEntryMetadata, HttpCacheStore};
use std::{
    collections::BTreeSet,
    fs,
    io::Read,
    num::NonZeroU32,
    process::Command,
    sync::{Arc, mpsc as std_mpsc},
    thread,
    time::{Duration, Instant},
};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tokio::sync::{mpsc, oneshot};
use url::Url;

use crate::{
    BrowserNavigationRequestKind, BrowserRequestMetadata, FetchCancelHandle, FetchClient,
    FetchClientHandle, FetchConfig, FetchPriorityHint, FetchRuntimeJoinStatus,
    NegotiatedHttpVersion, NetworkFetchFailureContext, RawResponse, Request, RequestAuth,
    RequestAuthScheme, RequestAuthTarget, RequestCacheMode, RequestCredentialsMode, RequestMode,
    RequestRedirectHandling, RequestRedirectMode, RequestResourceType, Response, ResponseBody,
    ResponseHead, ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority,
    StreamingResponseCollector, SubresourceRequestMetadata, WebBotAuthProfile, WebBotAuthSigner,
    http_cache_stats, runtime::FetchRuntimeOwner,
};

use self::support::{
    EmptyHttpHttpsUpgradeServer, Http2ProtocolFallbackServer, ScriptedH2Server, ScriptedHttpServer,
    ScriptedHttps11Server, ScriptedResponse, spawn_socks4_proxy, spawn_socks5_proxy,
    unique_test_cache_dir, wait_for_runtime_owner_count,
};

const ENV_PROXY_CHILD_TEST: &str = "MOLI_FETCH_ENV_PROXY_CHILD";
const ENV_PROXY_URL: &str = "MOLI_FETCH_ENV_PROXY_URL";
const TEST_HIGH_ENTROPY_CLIENT_HINTS: &str = "Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Platform-Version, Sec-CH-UA-Model, Sec-CH-UA-WoW64";
const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\
MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\
-----END PRIVATE KEY-----\n";

fn test_web_bot_auth_signer() -> WebBotAuthSigner {
    WebBotAuthSigner::from_pem(
        RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
        "bot.example",
        Some("poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U"),
        WebBotAuthProfile::Cloudflare,
    )
    .unwrap()
}

fn sample_response_head() -> ResponseHead {
    ResponseHead {
        status_text: None,
        final_url: Url::parse("http://example.test/final").unwrap(),
        status: 203,
        headers: vec![("content-type".to_owned(), b"text/plain".to_vec())],
        request_cookie_report: None,
        cookie_set_reports: Vec::new(),
        redirected: false,
        redirect_chain: Vec::new(),
        from_cache: false,
        negotiated_http_version: None,
    }
}

fn load_test_cache_body(store: &HttpCacheStore, key: &str) -> Result<Option<Vec<u8>>> {
    let Some(mut entry) = store.load_reader(key)? else {
        return Ok(None);
    };
    let mut body = Vec::new();
    entry.body.read_to_end(&mut body)?;
    Ok(Some(body))
}

fn store_test_cache_body(
    store: &HttpCacheStore,
    key: &str,
    metadata: HttpCacheEntryMetadata,
    body: &[u8],
) -> Result<()> {
    let mut writer = store.create_body_writer(key)?;
    writer.write_all(body)?;
    writer.finish(metadata)
}

fn fetch_response_for_test(client: &FetchClientHandle, request: Request) -> Result<Response> {
    tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()?
        .block_on(client.fetch(request))
}

fn fetch_raw_with_network_metadata_for_test(
    client: &FetchClient,
    request: Request,
) -> Result<crate::NetworkFetchResult<RawResponse>> {
    tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()?
        .block_on(client.fetch_raw_with_network_metadata(request))
}

#[test]
fn response_head_round_trips_through_text_and_raw_materialized_responses() {
    for status_text in [None, Some("Custom message"), Some("")] {
        let mut head = sample_response_head();
        head.status_text = status_text.map(str::to_owned);
        let response =
            Response::from_head_and_body(head.clone(), "hello".to_owned(), b"hello".to_vec());
        assert_eq!(response.clone().head().status_text, head.status_text);
        assert_eq!(
            response.clone().into_parts().0.status_text,
            head.status_text
        );

        let raw = response.into_materialized_raw_response();
        assert_eq!(raw.final_url, head.final_url);
        assert_eq!(raw.status, head.status);
        assert_eq!(raw.clone().into_parts().0.status_text, head.status_text);
        assert_eq!(raw.headers, head.headers);
        assert_eq!(raw.body_bytes(), b"hello");

        let text = raw.into_lossy_materialized_text_response();
        assert_eq!(text.final_url, head.final_url);
        assert_eq!(text.status, head.status);
        assert_eq!(text.head().status_text, head.status_text);
        assert_eq!(text.headers, head.headers);
        assert_eq!(text.body_text(), "hello");
        assert_eq!(text.body_bytes(), b"hello");
    }
}

#[test]
fn http_status_messages_preserve_bytes_and_explicit_empty_values() {
    for (line, status, expected) in [
        (b"HTTP/1.0 402 FIVE BUCKS\r\n".as_slice(), 402, "FIVE BUCKS"),
        (b"HTTP/1.1 699 WAY OUTTA RANGE\r\n", 699, "WAY OUTTA RANGE"),
        (b"HTTP/1.1 200\r\n", 200, ""),
        (
            b"HTTP/1.1 200   Keep  inner spaces   \r\n",
            200,
            "Keep  inner spaces",
        ),
        (b"HTTP/1.1 200 \ttab\tinside\t \r\n", 200, "\ttab\tinside\t"),
        (
            b"HTTP/1.1 200 \xa0caf\xe9\xff\r\n",
            200,
            "\u{a0}caf\u{e9}\u{ff}",
        ),
        (b"HTTP/2 200 \r\n", 200, ""),
        (b"HTTP/3 200 \r\n", 200, ""),
    ] {
        let parsed = crate::response::parse_http_response_status_line(line).unwrap();
        assert_eq!(parsed.status, status);
        assert_eq!(parsed.status_text, expected);
    }

    let mut head = sample_response_head();
    head.status = 200;
    assert_eq!(head.status_text(), "OK");
    head.negotiated_http_version = Some(NegotiatedHttpVersion::Http2);
    assert_eq!(head.status_text(), "");
    head.status_text = Some("Explicit override".to_owned());
    assert_eq!(head.status_text(), "Explicit override");
    head.negotiated_http_version = Some(NegotiatedHttpVersion::Http11);
    head.status_text = Some(String::new());
    assert_eq!(head.status_text(), "");
}

#[tokio::test]
async fn http_status_messages_survive_buffered_and_streamed_transport() -> Result<()> {
    for (status, received, expected) in [
        (200, "Different OK", "Different OK"),
        (200, "", ""),
        (200, "  Keep  inner spaces  ", "Keep  inner spaces"),
        (200, "tab\tinside", "tab\tinside"),
        (699, "WAY OUTTA RANGE", "WAY OUTTA RANGE"),
    ] {
        let server = ScriptedHttpServer::spawn(vec![
            ScriptedResponse::status(status, received)
                .with_body("body");
            3
        ]);
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        for mode in ["buffered", "html", "raw"] {
            let request = Request::get(&server.url())?;
            let (head, body) = match mode {
                "buffered" => client.fetch(request.with_single_hop()).await?.into_body(),
                "html" => client.fetch_html_stream(request).await?.into_body(),
                _ => client
                    .fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
                    .await?
                    .into_body(),
            };
            assert_eq!(head.status, status, "{mode}");
            assert_eq!(head.status_text.as_deref(), Some(expected), "{mode}");
            assert_eq!(body.into_materialized_bytes().await?, b"body", "{mode}");
        }
        assert_eq!(server.hits(), 3);
        assert!(client.shutdown().is_clean());
        server.shutdown();
    }
    Ok(())
}

#[tokio::test]
async fn http_status_messages_survive_disk_cache_across_streaming_modes() -> Result<()> {
    for first_mode in ["html", "raw"] {
        for status_text in ["Cached message", ""] {
            let cache_dir = unique_test_cache_dir();
            let server = ScriptedHttpServer::spawn(vec![
                ScriptedResponse::status(200, status_text)
                    .with_header("Cache-Control", "max-age=60")
                    .with_body("cached body"),
            ]);
            let mut config = FetchConfig::default();
            config.set_http_cache_dir(Some(cache_dir.display().to_string()));
            for (index, mode) in [first_mode, "html", "raw", "materialized"]
                .into_iter()
                .enumerate()
            {
                // Reopen the client so every subsequent read comes from disk.
                let client = FetchClient::new(&config, new_shared_browser_cookie_store());
                let request = Request::get(&server.url())?;
                let (head, body) = match mode {
                    "html" => client.fetch_html_stream(request).await?.into_body(),
                    "raw" => client
                        .fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
                        .await?
                        .into_body(),
                    _ => client.fetch(request).await?.into_body(),
                };
                assert_eq!(head.from_cache, index != 0, "{first_mode} to {mode}");
                assert_eq!(head.status_text.as_deref(), Some(status_text), "{mode}");
                assert_eq!(body.into_materialized_bytes().await?, b"cached body");
                assert!(client.shutdown().is_clean());
            }
            assert_eq!(server.hits(), 1);
            server.shutdown();
            fs::remove_dir_all(cache_dir)?;
        }
    }
    Ok(())
}

#[test]
fn single_hop_requests_preserve_redirect_mode() {
    for mode in [
        RequestRedirectMode::Follow,
        RequestRedirectMode::Manual,
        RequestRedirectMode::Error,
    ] {
        let request = Request::get("https://example.test/start")
            .unwrap()
            .with_redirect_mode(mode);
        let hop = request.with_single_hop();

        assert_eq!(hop.redirect_mode(), mode);
        assert!(!hop.transport_follows_redirects());
    }
}

#[tokio::test]
async fn http_transport_rejects_file_urls_in_every_response_mode() -> Result<()> {
    const FILE_URL: &str = "file:///moli-policy-must-not-open";
    const EXPECTED: &str = "URL scheme \"file\" is not supported by the HTTP network transport.";

    let mut config = FetchConfig::default();
    config.set_network_blocking(true, Vec::new());
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let buffered_error = client
        .fetch(Request::get(FILE_URL)?.with_single_hop())
        .await
        .expect_err("buffered transport must reject file URL");
    assert!(format!("{buffered_error:#}").contains(EXPECTED));

    let html_stream_error = client
        .fetch_html_stream(Request::get(FILE_URL)?)
        .await
        .expect_err("HTML streaming transport must reject file URL");
    assert!(format!("{html_stream_error:#}").contains(EXPECTED));

    let raw_stream_error = client
        .fetch_raw_stream_with_cancel(Request::get(FILE_URL)?, FetchCancelHandle::new())
        .await
        .expect_err("raw streaming transport must reject file URL");
    assert!(format!("{raw_stream_error:#}").contains(EXPECTED));
    Ok(())
}

#[test]
fn cloned_requests_and_preserving_redirects_share_upload_storage() -> Result<()> {
    let upload = vec![0, 255, 195, 40, 128, 65];
    let original_storage = upload.as_ptr().wrapping_add(1);
    let body = bytes::Bytes::from(upload).slice(1..5);
    let request = Request::new_bytes(
        "POST",
        "https://example.test/upload",
        Some(body.clone()),
        Vec::new(),
    )?;

    for status in [307, 308] {
        let mut redirected = request.clone();
        redirected.apply_redirect_status(status);
        let retained = redirected.body.as_ref().unwrap();
        assert_eq!(retained.as_ptr(), original_storage);
        assert_eq!(retained, &body);
        assert_eq!(redirected.method, "POST");
    }

    let mut rewritten = request.clone();
    rewritten.apply_redirect_status(303);
    assert_eq!(rewritten.method, "GET");
    assert!(rewritten.body.is_none());
    assert_eq!(request.body.as_ref().unwrap().as_ptr(), original_storage);
    assert_eq!(request.body.as_ref().unwrap(), &body);
    Ok(())
}

#[test]
fn request_apply_redirect_status_rewrites_post_to_get_and_drops_body_headers() {
    let mut request = Request::new(
        "POST",
        "https://example.test/target",
        Some("payload".to_owned()),
        vec![
            ("Content-Type".to_owned(), "text/plain".to_owned()),
            ("X-Keep".to_owned(), "1".to_owned()),
        ],
    )
    .unwrap();

    request.apply_redirect_status(303);

    assert_eq!(request.method, "GET");
    assert!(request.body.is_none());
    assert!(
        !request
            .request_headers
            .iter()
            .any(|(name, _)| name.eq_ignore_ascii_case("content-type"))
    );
    assert!(
        request
            .request_headers
            .iter()
            .any(|(name, _)| name.eq_ignore_ascii_case("x-keep"))
    );
}

#[test]
fn request_with_request_mode_keeps_expected_mode() {
    let request = Request::new("GET", "https://example.test/script.js", None, Vec::new())
        .unwrap()
        .with_request_mode(RequestMode::NoCors);

    assert_eq!(request.request_mode, RequestMode::NoCors);
}

#[test]
fn raw_response_head_helpers_keep_body_separate() {
    let head = sample_response_head();
    let raw = RawResponse::from_head_and_body(head.clone(), vec![0, 255, b'a']);
    let (split_head, body) = raw.into_parts();

    assert_eq!(split_head.final_url, head.final_url);
    assert_eq!(split_head.status, head.status);
    assert_eq!(split_head.headers, head.headers);
    assert_eq!(
        body.try_into_materialized_bytes()
            .expect("raw body should remain materialized"),
        vec![0, 255, b'a']
    );
}

#[test]
fn response_body_marks_materialized_and_streaming_shapes() {
    let body = ResponseBody::materialized_text("hello".to_owned(), Vec::new());
    assert!(!body.is_streaming());
    assert_eq!(
        body.try_into_materialized_bytes()
            .expect("materialized body should convert to bytes"),
        b"hello"
    );

    let (body_tx, body_rx) = mpsc::unbounded_channel();
    drop(body_tx);
    let (_completion_tx, completion_rx) = oneshot::channel();
    let stream = crate::StreamingHtmlResponse::new(
        Url::parse("http://example.test/stream").unwrap(),
        200,
        Vec::new(),
        body_rx,
        FetchCancelHandle::new(),
        completion_rx,
    );
    let (_head, streaming_body) = stream.into_body();
    assert!(streaming_body.is_streaming());
    assert!(streaming_body.try_into_materialized_bytes().is_err());
}

#[test]
fn response_body_reuses_valid_utf8_bytes_as_text_storage() {
    let body = ResponseBody::lossy_text_from_bytes(b"hello".to_vec());
    assert_eq!(body.as_materialized_text(), Some("hello"));
    assert_eq!(body.as_materialized_bytes(), Some(b"hello".as_slice()));
    assert!(matches!(
        body,
        ResponseBody::MaterializedText {
            exact_bytes: None,
            ..
        }
    ));

    let (text, bytes) = body
        .try_into_lossy_materialized_text()
        .expect("materialized text should expose exact parts");
    assert_eq!(text, "hello");
    assert_eq!(bytes, b"hello");
}

#[test]
fn response_body_keeps_invalid_utf8_bytes_beside_lossy_text() {
    let body = ResponseBody::lossy_text_from_bytes(vec![b'a', 0xff, b'b']);
    assert_eq!(body.as_materialized_text(), Some("a\u{fffd}b"));
    assert_eq!(
        body.as_materialized_bytes(),
        Some([b'a', 0xff, b'b'].as_slice())
    );
    assert!(matches!(
        body,
        ResponseBody::MaterializedText {
            exact_bytes: Some(ref bytes),
            ..
        } if bytes == &[b'a', 0xff, b'b']
    ));
}

#[test]
fn response_byte_parts_transfer_utf8_storage_without_copying() {
    let head = sample_response_head();
    let bytes = b"module source".to_vec();
    let storage = bytes.as_ptr();
    let response = Response::from_head_and_lossy_body_bytes(head.clone(), bytes);

    let (split_head, bytes) = response.into_byte_parts();

    assert_eq!(split_head.final_url, head.final_url);
    assert_eq!(split_head.status, head.status);
    assert_eq!(split_head.headers, head.headers);
    assert_eq!(bytes, b"module source");
    assert_eq!(bytes.as_ptr(), storage, "UTF-8 storage must be transferred");
}

#[test]
fn response_byte_parts_preserve_exact_non_utf8_storage() {
    let bytes = vec![b'a', 0xff, b'b'];
    let storage = bytes.as_ptr();
    let response = Response::from_head_and_lossy_body_bytes(sample_response_head(), bytes);
    assert_eq!(response.body_text(), "a\u{fffd}b");

    let (_, bytes) = response.into_byte_parts();

    assert_eq!(bytes, [b'a', 0xff, b'b']);
    assert_eq!(bytes.as_ptr(), storage, "exact bytes must be transferred");
}

#[tokio::test]
async fn response_body_materializes_streaming_text_source() {
    let (body_tx, body_rx) = mpsc::unbounded_channel();
    body_tx.send("hel".to_owned()).unwrap();
    body_tx.send("lo".to_owned()).unwrap();
    drop(body_tx);
    let (completion_tx, completion_rx) = oneshot::channel();
    completion_tx.send(Ok(())).unwrap();
    let cancel_handle = FetchCancelHandle::new();

    let stream = crate::StreamingHtmlResponse::new_with_head(
        sample_response_head(),
        body_rx,
        cancel_handle.clone(),
        completion_rx,
    );
    let (text, bytes) = stream
        .into_body()
        .1
        .into_lossy_materialized_text()
        .await
        .expect("streaming text body should materialize");

    assert_eq!(text, "hello");
    assert_eq!(bytes, b"hello");
    assert!(
        !cancel_handle.is_cancelled(),
        "observing a successful streaming completion must disarm Drop cancellation"
    );
}

#[tokio::test]
async fn streaming_raw_response_materializes_through_response_body_source() {
    let (body_tx, body_rx) = mpsc::unbounded_channel();
    body_tx.send(vec![0, 255]).unwrap();
    body_tx.send(vec![b'a']).unwrap();
    drop(body_tx);
    let (completion_tx, completion_rx) = oneshot::channel();
    completion_tx.send(Ok(())).unwrap();
    let cancel_handle = FetchCancelHandle::new();

    let raw = crate::StreamingRawResponse::new(
        sample_response_head().final_url,
        200,
        Vec::new(),
        None,
        Vec::new(),
        false,
        Vec::new(),
        body_rx,
        cancel_handle.clone(),
        completion_rx,
    )
    .into_materialized_raw_response()
    .await
    .expect("streaming raw body should materialize through ResponseBody");

    assert_eq!(raw.body_bytes(), vec![0, 255, b'a']);
    assert!(
        !cancel_handle.is_cancelled(),
        "observing a successful streaming completion must disarm Drop cancellation"
    );
}

#[test]
fn request_credentials_mode_uses_standard_webidl_labels() {
    use std::str::FromStr;

    assert_eq!(
        RequestCredentialsMode::from_str("include"),
        Ok(RequestCredentialsMode::Include)
    );
    assert_eq!(
        RequestCredentialsMode::from_str("same-origin"),
        Ok(RequestCredentialsMode::SameOrigin)
    );
    assert_eq!(
        RequestCredentialsMode::from_str("omit"),
        Ok(RequestCredentialsMode::Omit)
    );
    assert_eq!(RequestCredentialsMode::Include.as_ref(), "include");
    assert_eq!(RequestCredentialsMode::SameOrigin.as_ref(), "same-origin");
    assert_eq!(RequestCredentialsMode::Omit.as_ref(), "omit");
    assert!(RequestCredentialsMode::from_str("credentialless").is_err());
}

fn fetch_with_config_for_test(config: &FetchConfig, request: Request) -> Result<Response> {
    let client = FetchClient::new(config, new_shared_browser_cookie_store());
    fetch_response_for_test(&client, request)
}

#[test]
fn request_min_timeout_can_extend_short_configured_deadline() {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("slow").with_delay_ms(150)]);
    let mut config = FetchConfig::default();
    config.set_request_timeout_ms(50);

    let response = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_min_request_timeout(Duration::from_secs(1)),
    )
    .unwrap();

    assert_eq!(response.body_text(), "slow");
    assert_eq!(server.hits(), 1);
    server.shutdown();
}

async fn read_http_request_head(
    stream: &mut tokio::net::TcpStream,
) -> Result<String, std::io::Error> {
    let mut buffer = Vec::new();
    let mut byte = [0u8; 1];
    while !buffer.ends_with(b"\r\n\r\n") {
        let read = stream.read(&mut byte).await?;
        if read == 0 {
            break;
        }
        buffer.push(byte[0]);
    }
    Ok(String::from_utf8_lossy(&buffer).into_owned())
}

async fn spawn_raw_body_server(
    body: &'static [u8],
    content_type: &'static str,
) -> Result<(String, tokio::task::JoinHandle<()>)> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.expect("accept raw fetch client");
        let _ = read_http_request_head(&mut stream)
            .await
            .expect("read raw fetch request");
        let headers = format!(
            "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: {content_type}\r\nConnection: close\r\n\r\n",
            body.len()
        );
        stream
            .write_all(headers.as_bytes())
            .await
            .expect("write raw fetch headers");
        stream.write_all(body).await.expect("write raw fetch body");
    });
    Ok((format!("http://{addr}/raw.bin"), server))
}

async fn spawn_delayed_raw_body_server(
    body: &'static [u8],
    content_type: &'static str,
    delay: Duration,
) -> Result<(String, tokio::task::JoinHandle<()>)> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener
            .accept()
            .await
            .expect("accept delayed raw fetch client");
        let _ = read_http_request_head(&mut stream)
            .await
            .expect("read delayed raw fetch request");
        let headers = format!(
            "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: {content_type}\r\nContent-Disposition: attachment; filename=\"streamed.bin\"\r\nConnection: close\r\n\r\n",
            body.len()
        );
        stream
            .write_all(headers.as_bytes())
            .await
            .expect("write delayed raw fetch headers");
        tokio::time::sleep(delay).await;
        stream
            .write_all(body)
            .await
            .expect("write delayed raw fetch body");
    });
    Ok((format!("http://{addr}/raw-stream.bin"), server))
}

#[test]
fn request_lowers_script_fetch_metadata_into_owned_request_fields() -> Result<()> {
    let metadata = ScriptFetchRequestMetadata {
        cross_origin: Some("anonymous".to_owned()),
        referrer_policy: Some("no-referrer".to_owned()),
        document_referrer_policy: Some("strict-origin".to_owned()),
        charset: Some("utf-8".to_owned()),
        integrity: Some("sha256-test".to_owned()),
        nonce: Some("nonce-1".to_owned()),
        fetch_priority: Some(FetchPriorityHint::High),
        scheduler_priority: Some(ScriptFetchSchedulerPriority::High),
    };
    let request = Request::new("GET", "https://example.test/app.js", None, Vec::new())?
        .with_script_fetch_metadata(metadata);

    assert_eq!(
        request.subresource_request_metadata(),
        Some(&SubresourceRequestMetadata {
            referrer_policy: Some("no-referrer".to_owned()),
            document_referrer_policy: Some("strict-origin".to_owned()),
            integrity: Some("sha256-test".to_owned()),
        })
    );
    assert_eq!(
        request.priority_hints.fetch_priority,
        Some(FetchPriorityHint::High)
    );
    assert_eq!(
        request.script_scheduler_priority(),
        Some(ScriptFetchSchedulerPriority::High)
    );
    assert_eq!(request.resource_type, RequestResourceType::Script);
    Ok(())
}

#[test]
fn fetch_priority_hint_parsing_keeps_webidl_strict_and_html_case_insensitive() {
    assert_eq!(
        "high".parse::<FetchPriorityHint>().ok(),
        Some(FetchPriorityHint::High)
    );
    assert!(
        "HIGH".parse::<FetchPriorityHint>().is_err(),
        "RequestInit priority is a WebIDL enum and must stay case-sensitive"
    );
    assert_eq!(
        FetchPriorityHint::from_attribute(Some(" HIGH ")),
        Some(FetchPriorityHint::High),
        "HTML fetchpriority attribute keywords are ASCII case-insensitive"
    );
}

#[tokio::test]
async fn streaming_response_collector_ignores_interim_headers_before_final_start() -> Result<()> {
    let (start_tx, mut start_rx) = oneshot::channel();
    let (body_tx, _body_rx) = mpsc::unbounded_channel();
    let cancel_handle = FetchCancelHandle::new();
    let mut collector = StreamingResponseCollector::new(
        new_shared_browser_cookie_store(),
        start_tx,
        body_tx,
        cancel_handle,
    );
    let current_url = Url::parse("http://example.test/stream")?;
    collector.begin_request(
        None,
        current_url.clone(),
        NetworkCookieRequestContext::top_level_navigation("GET"),
        None,
        true,
        vec![],
        None,
    );

    assert!(collector.header(b"HTTP/1.1 103 Early Hints\r\n"));
    assert!(collector.header(b"Link: </style.css>; rel=preload; as=style\r\n"));
    assert!(collector.header(b"\r\n"));
    assert!(!collector.started());
    assert!(matches!(
        start_rx.try_recv(),
        Err(tokio::sync::oneshot::error::TryRecvError::Empty)
    ));

    assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
    assert!(collector.header(b"Content-Type: text/html; charset=utf-8\r\n"));
    assert!(collector.header(b"\r\n"));
    assert!(collector.started());

    let started = start_rx.await??;
    assert_eq!(started.status, 200);
    assert_eq!(started.final_url, current_url);
    assert_eq!(
        started.headers,
        vec![(
            "content-type".to_owned(),
            b"text/html; charset=utf-8".to_vec()
        )]
    );
    Ok(())
}

#[tokio::test]
async fn streaming_response_collector_drops_redirect_body_before_final_response() -> Result<()> {
    let (start_tx, mut start_rx) = oneshot::channel();
    let (body_tx, mut body_rx) = mpsc::unbounded_channel();
    let cancel_handle = FetchCancelHandle::new();
    let mut collector = StreamingResponseCollector::new(
        new_shared_browser_cookie_store(),
        start_tx,
        body_tx,
        cancel_handle,
    );
    let redirected_url = Url::parse("http://example.test/redirect")?;
    let final_url = Url::parse("http://example.test/final")?;

    collector.begin_request(
        None,
        redirected_url.clone(),
        NetworkCookieRequestContext::top_level_navigation("GET"),
        None,
        true,
        vec![],
        None,
    );
    assert!(collector.header(b"HTTP/1.1 302 Found\r\n"));
    assert!(collector.header(b"Location: /final\r\n"));
    assert!(collector.header(b"Content-Type: text/html; charset=utf-8\r\n"));
    assert!(collector.header(b"\r\n"));
    assert!(!collector.started());
    assert_eq!(
        collector
            .write(b"<a href=\"/final\">Found</a>")
            .expect("redirect body write should succeed"),
        26
    );
    assert!(matches!(
        start_rx.try_recv(),
        Err(tokio::sync::oneshot::error::TryRecvError::Empty)
    ));
    assert!(matches!(
        body_rx.try_recv(),
        Err(tokio::sync::mpsc::error::TryRecvError::Empty)
    ));

    collector.begin_request(
        None,
        final_url.clone(),
        NetworkCookieRequestContext::top_level_navigation("GET"),
        None,
        true,
        vec![],
        None,
    );
    assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
    assert!(collector.header(b"Content-Type: text/html; charset=utf-8\r\n"));
    assert!(collector.header(b"\r\n"));
    assert!(collector.started());
    assert_eq!(
        collector
            .write(b"<!doctype html><html><body>ok</body></html>")
            .expect("final body write should succeed"),
        43
    );
    collector.finish_streaming_body();

    let started = start_rx.await??;
    assert_eq!(started.status, 200);
    assert_eq!(started.final_url, final_url);
    assert_eq!(
        body_rx.recv().await.as_deref(),
        Some("<!doctype html><html><body>ok</body></html>")
    );
    assert!(body_rx.recv().await.is_none());
    Ok(())
}

#[tokio::test]
async fn streaming_response_collector_treats_connection_established_reason_as_normal_response()
-> Result<()> {
    let (start_tx, start_rx) = oneshot::channel();
    let (body_tx, mut body_rx) = mpsc::unbounded_channel();
    let cancel_handle = FetchCancelHandle::new();
    let mut collector = StreamingResponseCollector::new(
        new_shared_browser_cookie_store(),
        start_tx,
        body_tx,
        cancel_handle,
    );
    let current_url = Url::parse("https://example.test/final")?;

    collector.begin_request(
        None,
        current_url.clone(),
        NetworkCookieRequestContext::top_level_navigation("GET"),
        None,
        true,
        vec![],
        None,
    );
    assert!(collector.header(b"HTTP/1.1 200 Connection established\r\n"));
    assert!(collector.header(b"Content-Type: text/html; charset=utf-8\r\n"));
    assert!(collector.header(b"\r\n"));
    assert!(collector.started());
    assert_eq!(
        collector
            .write(b"<!doctype html><html><body>ok</body></html>")
            .expect("final body write should succeed"),
        43
    );
    collector.finish_streaming_body();

    let started = start_rx.await??;
    assert_eq!(started.status, 200);
    assert_eq!(started.final_url, current_url);
    assert_eq!(
        body_rx.recv().await.as_deref(),
        Some("<!doctype html><html><body>ok</body></html>")
    );
    assert!(body_rx.recv().await.is_none());
    Ok(())
}

#[tokio::test]
async fn streaming_response_collector_respects_response_credentials_gate() -> Result<()> {
    let (start_tx, start_rx) = oneshot::channel();
    let (body_tx, _body_rx) = mpsc::unbounded_channel();
    let cookie_store = new_shared_browser_cookie_store();
    let mut collector = StreamingResponseCollector::new(
        cookie_store.clone(),
        start_tx,
        body_tx,
        FetchCancelHandle::new(),
    );
    let current_url = Url::parse("http://example.test/stream")?;

    collector.begin_request(
        None,
        current_url.clone(),
        NetworkCookieRequestContext::top_level_navigation("GET"),
        None,
        false,
        vec![],
        None,
    );
    assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
    assert!(collector.header(b"Set-Cookie: blocked=1; Path=/\r\n"));
    assert!(collector.header(b"\r\n"));

    let started = start_rx.await??;
    assert!(started.cookie_set_reports.is_empty());
    let cookie_header = crate::cookie_header_for_request(
        &cookie_store,
        &current_url,
        Request::get(current_url.as_str())?.cookie_context,
    )?;
    assert!(cookie_header.is_none());
    Ok(())
}

#[tokio::test]
async fn streaming_response_collector_writes_cache_body_only_with_writer() -> Result<()> {
    let (start_tx, _start_rx) = oneshot::channel();
    let (body_tx, mut body_rx) = mpsc::unbounded_channel();
    let cancel_handle = FetchCancelHandle::new();
    let mut collector = StreamingResponseCollector::new(
        new_shared_browser_cookie_store(),
        start_tx,
        body_tx,
        cancel_handle,
    );
    let current_url = Url::parse("http://example.test/stream")?;

    collector.begin_request(
        None,
        current_url.clone(),
        NetworkCookieRequestContext::top_level_navigation("GET"),
        None,
        true,
        vec![],
        None,
    );
    assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
    assert!(collector.header(b"Content-Type: text/html; charset=utf-8\r\n"));
    assert!(collector.header(b"\r\n"));
    assert_eq!(
        collector
            .write(b"<!doctype html><html><body>ok</body></html>")
            .expect("streaming body write should succeed"),
        43
    );
    assert_eq!(
        body_rx.recv().await.as_deref(),
        Some("<!doctype html><html><body>ok</body></html>")
    );
    assert!(collector.take_cache_body_writer().is_none());

    let (start_tx, _start_rx) = oneshot::channel();
    let (body_tx, _body_rx) = mpsc::unbounded_channel();
    let mut collector = StreamingResponseCollector::new(
        new_shared_browser_cookie_store(),
        start_tx,
        body_tx,
        FetchCancelHandle::new(),
    );
    let cache_dir = unique_test_cache_dir();
    let store = HttpCacheStore::new(&cache_dir);
    let cache_key = HttpCacheStore::key_for_url(current_url.as_str());
    let cache_body_writer = store.create_body_writer(&cache_key)?;
    collector.begin_request(
        None,
        current_url.clone(),
        NetworkCookieRequestContext::top_level_navigation("GET"),
        None,
        true,
        vec![],
        Some(cache_body_writer),
    );
    assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
    assert!(collector.header(b"Content-Type: text/html; charset=utf-8\r\n"));
    assert!(collector.header(b"\r\n"));
    assert_eq!(
        collector
            .write(b"<!doctype html><html><body>cached</body></html>")
            .expect("cacheable streaming body write should succeed"),
        47
    );
    let cache_body_writer = collector
        .take_cache_body_writer()
        .expect("cache body writer should remain attached");
    cache_body_writer.finish(HttpCacheEntryMetadata::new(
        current_url.to_string(),
        current_url.to_string(),
        200,
        Vec::new(),
        1,
        Some(2),
        Vec::new(),
    ))?;
    let cached = load_test_cache_body(&store, &cache_key)?.expect("cache body should be published");
    assert_eq!(cached, b"<!doctype html><html><body>cached</body></html>");
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn streaming_response_collector_caches_original_body_bytes() -> Result<()> {
    let (start_tx, _start_rx) = oneshot::channel();
    let (body_tx, mut body_rx) = mpsc::unbounded_channel();
    let cancel_handle = FetchCancelHandle::new();
    let mut collector = StreamingResponseCollector::new(
        new_shared_browser_cookie_store(),
        start_tx,
        body_tx,
        cancel_handle,
    );
    let current_url = Url::parse("http://example.test/binary-stream")?;
    let cache_dir = unique_test_cache_dir();
    let store = HttpCacheStore::new(&cache_dir);
    let cache_key = HttpCacheStore::key_for_url(current_url.as_str());
    let cache_body_writer = store.create_body_writer(&cache_key)?;
    let body = [b'a', 0xff, 0x80, b'b'];

    collector.begin_request(
        None,
        current_url.clone(),
        NetworkCookieRequestContext::top_level_navigation("GET"),
        None,
        true,
        vec![],
        Some(cache_body_writer),
    );
    assert!(collector.header(b"HTTP/1.1 200 OK\r\n"));
    assert!(collector.header(b"Content-Type: application/octet-stream\r\n"));
    assert!(collector.header(b"\r\n"));
    assert_eq!(
        collector
            .write(&body)
            .expect("binary streaming body write should succeed"),
        body.len()
    );
    collector.finish_streaming_body();

    let cache_body_writer = collector
        .take_cache_body_writer()
        .expect("cache body writer should remain attached");
    cache_body_writer.finish(HttpCacheEntryMetadata::new(
        current_url.to_string(),
        current_url.to_string(),
        200,
        Vec::new(),
        1,
        Some(2),
        Vec::new(),
    ))?;
    let cached = load_test_cache_body(&store, &cache_key)?.expect("cache body should be published");
    assert_eq!(cached, body);
    assert_eq!(body_rx.recv().await.as_deref(), Some("a"));
    assert_eq!(body_rx.recv().await.as_deref(), Some("\u{FFFD}"));
    assert_eq!(body_rx.recv().await.as_deref(), Some("\u{FFFD}"));
    assert_eq!(body_rx.recv().await.as_deref(), Some("b"));
    assert!(body_rx.recv().await.is_none());

    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn fetch_html_stream_drops_redirect_body_chunks() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(302, "<a href=\"/final\">Found</a>")
            .with_header("Location", "/final")
            .with_header("Content-Type", "text/html; charset=utf-8"),
        ScriptedResponse::ok("<!doctype html><html><body>ok</body></html>")
            .with_header("Content-Type", "text/html; charset=utf-8"),
    ]);

    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let mut response = client
        .fetch_html_stream(Request::get(&server.url()).unwrap())
        .await?;

    let mut body = String::new();
    while let Some(chunk) = response.next_chunk().await {
        body.push_str(&chunk);
    }
    response.finish().await?;

    assert_eq!(response.status, 200);
    assert!(response.redirected);
    assert_eq!(response.redirect_chain.len(), 1);
    let head = response.head();
    assert_eq!(head.status, 200);
    assert!(head.redirected);
    assert_eq!(head.redirect_chain.len(), 1);
    assert_eq!(body, "<!doctype html><html><body>ok</body></html>");

    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn fetch_html_stream_uses_disk_cache_for_safe_gets() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("<!doctype html><html><body>hit-1</body></html>")
            .with_header("Content-Type", "text/html; charset=utf-8")
            .with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut first = client
        .fetch_html_stream(Request::get(&server.url()).unwrap())
        .await?;
    let mut first_body = String::new();
    while let Some(chunk) = first.next_chunk().await {
        first_body.push_str(&chunk);
    }
    first.finish().await?;

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut second = client
        .fetch_html_stream(Request::get(&server.url()).unwrap())
        .await?;
    let mut second_body = String::new();
    while let Some(chunk) = second.next_chunk().await {
        second_body.push_str(&chunk);
    }
    second.finish().await?;

    assert_eq!(first_body, "<!doctype html><html><body>hit-1</body></html>");
    assert_eq!(
        second_body,
        "<!doctype html><html><body>hit-1</body></html>"
    );
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn fetch_html_stream_does_not_create_cache_entry_for_no_store() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("<!doctype html><html><body>hit-1</body></html>")
            .with_header("Content-Type", "text/html; charset=utf-8")
            .with_header("Cache-Control", "no-store"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut response = client
        .fetch_html_stream(Request::get(&server.url()).unwrap())
        .await?;
    while response.next_chunk().await.is_some() {}
    response.finish().await?;

    assert_eq!(server.hits(), 1);
    assert!(
        !cache_dir.exists() || fs::read_dir(&cache_dir)?.next().is_none(),
        "uncacheable streaming responses should not create temp entry dirs"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn fetch_raw_preserves_binary_body_bytes() -> Result<()> {
    const BODY: &[u8] = b"\x00\xffraw-download\x80\xfe";
    let (url, server) = spawn_raw_body_server(BODY, "application/octet-stream").await?;
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());

    let response = client.fetch_raw(Request::get(&url)?).await?;

    assert_eq!(response.status, 200);
    assert!(
        response.headers.iter().any(|(name, value)| {
            name == "content-type" && value == b"application/octet-stream"
        })
    );
    assert_eq!(response.body_bytes(), BODY);

    server.await.expect("raw fetch server should finish");
    Ok(())
}

#[tokio::test]
async fn fetch_preserves_binary_body_bytes_on_text_response() -> Result<()> {
    const BODY: &[u8] = b"\x00\xfffetch-download\x80\xfe";
    let (url, server) = spawn_raw_body_server(BODY, "application/octet-stream").await?;
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());

    let response = client.fetch(Request::get(&url)?).await?;

    assert_eq!(response.status, 200);
    assert_eq!(response.body_bytes(), BODY);
    assert_ne!(
        response.body_text().as_bytes(),
        BODY,
        "lossy text preview should not be used as the byte source"
    );

    server
        .await
        .expect("binary text response server should finish");
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_yields_headers_before_delayed_body_and_preserves_bytes() -> Result<()> {
    const BODY: &[u8] = b"\x00\xffstreamed-download\x80\xfe";
    let (url, server) =
        spawn_delayed_raw_body_server(BODY, "application/octet-stream", Duration::from_millis(150))
            .await?;
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let start = Instant::now();
    let mut response = client
        .fetch_raw_stream_with_cancel(Request::get(&url)?, FetchCancelHandle::new())
        .await?;

    assert!(
        start.elapsed() < Duration::from_millis(120),
        "streaming raw response should become available from headers before the delayed body arrives"
    );
    assert_eq!(response.status, 200);
    assert_eq!(response.final_url.as_str(), url);
    assert!(!response.redirected);
    assert!(response.redirect_chain.is_empty());
    assert!(response.cookie_set_reports.is_empty());
    assert!(response.headers.iter().any(|(name, value)| {
        name == "content-disposition" && value == b"attachment; filename=\"streamed.bin\""
    }));

    let mut body = Vec::new();
    while let Some(chunk) = response.next_chunk().await {
        body.extend_from_slice(&chunk);
    }
    response.finish().await?;
    assert_eq!(body, BODY);

    server
        .await
        .expect("delayed raw streaming fetch server should finish");
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_preserves_redirect_metadata() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(302, "Found").with_header("Location", "/final.bin"),
        ScriptedResponse::ok("final-body").with_header("Content-Type", "application/octet-stream"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let mut response = client
        .fetch_raw_stream_with_cancel(
            Request::get(&server.url_path("/redirect.bin"))?,
            FetchCancelHandle::new(),
        )
        .await?;

    assert_eq!(response.status, 200);
    assert_eq!(response.final_url.as_str(), server.url_path("/final.bin"));
    assert!(response.redirected);
    assert_eq!(response.redirect_chain.len(), 1);
    assert_eq!(response.redirect_chain[0].status, 302);
    assert_eq!(
        response.redirect_chain[0].to_url.as_str(),
        server.url_path("/final.bin")
    );

    let mut body = Vec::new();
    while let Some(chunk) = response.next_chunk().await {
        body.extend_from_slice(&chunk);
    }
    response.finish().await?;
    assert_eq!(body, b"final-body");
    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn followed_http_redirect_to_file_is_rejected_without_opening_the_target() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(302, "Found")
            .with_header("Location", "file:///moli-policy-must-not-open"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());

    let error = client
        .fetch_raw_stream_with_cancel(
            Request::get(&server.url_path("/redirect-to-file"))?,
            FetchCancelHandle::new(),
        )
        .await
        .expect_err("followed redirect to file must fail");

    assert!(
        error
            .to_string()
            .contains("URL scheme \"file\" is not supported by the HTTP network transport.")
    );
    assert_eq!(server.hits(), 1);
    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn manual_http_redirect_to_file_remains_observable_without_being_followed() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(302, "Found")
            .with_header("Location", "file:///moli-policy-must-not-open"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());

    let response = client
        .fetch(
            Request::get(&server.url_path("/manual-redirect-to-file"))?
                .with_redirect_mode(RequestRedirectMode::Manual),
        )
        .await?;

    assert_eq!(response.status, 302);
    assert!(response.headers.iter().any(|(name, value)| {
        name.eq_ignore_ascii_case("location") && value == b"file:///moli-policy-must-not-open"
    }));
    assert_eq!(server.hits(), 1);
    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_obeys_redirect_handling_for_each_hop() -> Result<()> {
    for handling in [
        RequestRedirectHandling::Follow,
        RequestRedirectHandling::FollowByCaller,
        RequestRedirectHandling::Manual,
        RequestRedirectHandling::Error,
    ] {
        let server = ScriptedHttpServer::spawn(vec![
            ScriptedResponse::status(302, "Found")
                .with_header("Location", "/final.bin")
                .with_body("redirect-body"),
            ScriptedResponse::ok("final-body")
                .with_header("Content-Type", "application/octet-stream"),
        ]);
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        let mut response = client
            .fetch_raw_stream_with_cancel(
                Request::get(&server.url_path("/redirect.bin"))?.with_redirect_handling(handling),
                FetchCancelHandle::new(),
            )
            .await?;

        if handling == RequestRedirectHandling::Follow {
            assert_eq!(response.status, 200);
            assert_eq!(response.final_url.as_str(), server.url_path("/final.bin"));
            assert!(response.redirected);
            assert_eq!(response.redirect_chain.len(), 1);
        } else {
            // This is the raw transport: manual/error policy decisions and
            // caller-owned redirect loops consume the returned redirect head.
            assert_eq!(response.status, 302);
            assert_eq!(
                response.final_url.as_str(),
                server.url_path("/redirect.bin")
            );
            assert!(!response.redirected);
            assert!(response.redirect_chain.is_empty());
            assert!(response.headers.iter().any(|(name, value)| {
                name.eq_ignore_ascii_case("location") && value == b"/final.bin"
            }));
        }

        let mut body = Vec::new();
        while let Some(chunk) = response.next_chunk().await {
            body.extend_from_slice(&chunk);
        }
        response.finish().await?;
        if handling == RequestRedirectHandling::Follow {
            assert_eq!(body, b"final-body");
            assert_eq!(server.requests().len(), 2);
        } else {
            assert!(body.is_empty());
            assert_eq!(server.requests().len(), 1);
        }
        server.shutdown();
    }
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_treats_switching_protocols_as_final_response() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(101, "Switching Protocols").with_body("HTTP Response Status"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let mut response = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;

    assert_eq!(response.status, 101);
    assert_eq!(response.final_url.as_str(), server.url());
    assert!(!response.redirected);
    assert!(response.redirect_chain.is_empty());

    let mut body = Vec::new();
    while let Some(chunk) = response.next_chunk().await {
        body.extend_from_slice(&chunk);
    }
    response.finish().await?;
    assert!(body.is_empty());
    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_treats_https11_switching_protocols_as_final_response() -> Result<()> {
    let server = ScriptedHttps11Server::spawn(vec![
        ScriptedResponse::status(101, "Switching Protocols").with_body("HTTP Response Status"),
    ]);
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut response = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;

    assert_eq!(response.status, 101);
    assert_eq!(response.final_url.as_str(), server.url());
    assert!(!response.redirected);
    assert!(response.redirect_chain.is_empty());

    let mut body = Vec::new();
    while let Some(chunk) = response.next_chunk().await {
        body.extend_from_slice(&chunk);
    }
    response.finish().await?;
    assert!(body.is_empty());
    assert_eq!(server.hits(), 1);
    assert_eq!(server.requests(), ["/cache".to_owned()]);
    server.shutdown();
    Ok(())
}

#[test]
fn web_bot_auth_resigns_restarts_redirects_and_subresources() -> Result<()> {
    let server = ScriptedHttps11Server::spawn(vec![
        ScriptedResponse::status(403, "Challenge")
            .with_header("Accept-CH", "Sec-CH-UA-Arch")
            .with_header("Critical-CH", "Sec-CH-UA-Arch"),
        ScriptedResponse::status(302, "Found").with_header("Location", "/final"),
        ScriptedResponse::ok("final"),
        ScriptedResponse::ok("asset"),
    ]);
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    config.set_web_bot_auth(Some(test_web_bot_auth_signer()));
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let response = fetch_response_for_test(&client, Request::get(&server.url_path("/start"))?)?;
    assert_eq!(response.body_text(), "final");
    assert_eq!(response.redirect_chain.len(), 2);
    assert_eq!(response.redirect_chain[0].status, 307);
    assert_eq!(response.redirect_chain[1].status, 302);

    let final_url = Url::parse(&server.url_path("/final"))?;
    let subresource = Request::new(
        "POST",
        &server.url_path("/asset"),
        Some("probe".to_owned()),
        Vec::new(),
    )?
    .with_browser_request_metadata(BrowserRequestMetadata::Fetch)
    .with_initiator_url(&final_url)
    .with_request_origin(moli_url::WebOrigin::from_url(&final_url));
    assert_eq!(
        fetch_response_for_test(&client, subresource)?.body_text(),
        "asset"
    );

    assert_eq!(
        server.requests(),
        [
            "/start".to_owned(),
            "/start".to_owned(),
            "/final".to_owned(),
            "/asset".to_owned(),
        ]
    );
    let request_heads = server.request_heads();
    assert_eq!(request_heads.len(), 4);
    for request_head in &request_heads {
        assert_eq!(
            request_head_header_value(request_head, "Signature-Agent"),
            Some("\"https://bot.example\"")
        );
        let signature_input = request_head_header_value(request_head, "Signature-Input")
            .expect("signed request should include Signature-Input");
        assert!(
            signature_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")")
        );
        assert!(request_head_header_value(request_head, "Signature").is_some());
    }
    assert!(request_heads[3].starts_with("POST /asset HTTP/1.1\r\n"));

    let nonces = request_heads
        .iter()
        .map(|request_head| {
            nonce_from_signature_input(
                request_head_header_value(request_head, "Signature-Input").unwrap(),
            )
            .to_owned()
        })
        .collect::<BTreeSet<_>>();
    assert_eq!(nonces.len(), request_heads.len());

    let restart = &response.redirect_chain[0];
    assert_request_extra_signature_matches_wire(
        &restart
            .response_extra_info
            .as_ref()
            .expect("restart response extra info")
            .request_extra_info,
        &request_heads[0],
    );
    assert_request_extra_signature_matches_wire(
        restart
            .request_extra_info
            .as_ref()
            .expect("restarted request extra info"),
        &request_heads[1],
    );
    let redirect = &response.redirect_chain[1];
    assert_request_extra_signature_matches_wire(
        &redirect
            .response_extra_info
            .as_ref()
            .expect("redirect response extra info")
            .request_extra_info,
        &request_heads[1],
    );
    assert_request_extra_signature_matches_wire(
        redirect
            .request_extra_info
            .as_ref()
            .expect("redirect target request extra info"),
        &request_heads[2],
    );
    assert_request_extra_signature_matches_wire(
        response
            .network_request_extra_info()
            .expect("final request extra info"),
        &request_heads[2],
    );

    server.shutdown();
    Ok(())
}

#[test]
fn web_bot_auth_is_not_sent_over_plain_http() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("plain")]);
    let mut config = FetchConfig::default();
    config.set_web_bot_auth(Some(test_web_bot_auth_signer()));

    let response = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
    assert_eq!(response.body_text(), "plain");
    let requests = server.requests();
    assert_eq!(requests.len(), 1);
    for name in ["Signature-Agent", "Signature-Input", "Signature"] {
        assert_eq!(request_head_header_value(&requests[0], name), None);
    }

    server.shutdown();
    Ok(())
}

#[test]
fn web_bot_auth_bypasses_shared_http_cache() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttps11Server::spawn(vec![
        ScriptedResponse::ok("first").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("second").with_header("Cache-Control", "max-age=60"),
    ]);
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    config.set_web_bot_auth(Some(test_web_bot_auth_signer()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
    let second = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
    assert_eq!(first.body_text(), "first");
    assert_eq!(second.body_text(), "second");
    assert!(!first.from_cache);
    assert!(!second.from_cache);
    assert_eq!(server.hits(), 2);
    let request_heads = server.request_heads();
    assert_ne!(
        request_head_header_value(&request_heads[0], "Signature-Input"),
        request_head_header_value(&request_heads[1], "Signature-Input")
    );
    assert!(
        !cache_dir.exists() || fs::read_dir(&cache_dir)?.next().is_none(),
        "authenticated responses must not enter the shared disk cache"
    );

    server.shutdown();
    let _ = fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_finishes_switching_protocols_body_without_connection_close() -> Result<()>
{
    let server = ScriptedHttps11Server::spawn(vec![
        ScriptedResponse::status(101, "Switching Protocols")
            .with_body("HTTP Response Status")
            .with_hold_open_ms(2_000),
    ]);
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut response = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;

    assert_eq!(response.status, 101);
    assert_eq!(response.final_url.as_str(), server.url());

    let body = tokio::time::timeout(Duration::from_millis(500), async {
        let mut body = Vec::new();
        while let Some(chunk) = response.next_chunk().await {
            body.extend_from_slice(&chunk);
        }
        response.finish().await?;
        anyhow::Ok(body)
    })
    .await
    .expect("101 streaming body should finish before the keep-alive connection closes")?;
    assert!(body.is_empty());
    assert_eq!(server.hits(), 1);
    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_finishes_null_body_status_without_connection_close() -> Result<()> {
    for (status, reason) in [(204, "No Content"), (205, "Reset Content")] {
        let server = ScriptedHttps11Server::spawn(vec![
            ScriptedResponse::status(status, reason)
                .with_body("HTTP Response Status")
                .with_hold_open_ms(2_000),
        ]);
        let mut config = FetchConfig::default();
        config.set_tls_verify_host(false);
        let client = FetchClient::new(&config, new_shared_browser_cookie_store());
        let mut response = client
            .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
            .await?;

        assert_eq!(response.status, status);

        let body = tokio::time::timeout(Duration::from_millis(500), async {
            let mut body = Vec::new();
            while let Some(chunk) = response.next_chunk().await {
                body.extend_from_slice(&chunk);
            }
            response.finish().await?;
            anyhow::Ok(body)
        })
        .await
        .with_context(|| format!("{status} streaming body should finish at headers"))??;
        assert!(body.is_empty());
        assert_eq!(server.hits(), 1);
        server.shutdown();
    }
    Ok(())
}

#[tokio::test]
async fn fetch_redirects_recompute_bodyless_put_content_length() -> Result<()> {
    for status in [301, 302, 303, 307, 308] {
        let server = ScriptedHttpServer::spawn(vec![
            ScriptedResponse::status(status, "Redirect").with_header("Location", "/final"),
            ScriptedResponse::ok("done"),
        ]);
        let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
        let response = client
            .fetch_raw(Request::new(
                "PUT",
                &server.url_path("/start"),
                None,
                vec![],
            )?)
            .await?;
        assert_eq!(response.status, 200);
        assert!(response.redirected);
        let requests = server.requests();
        server.shutdown();
        assert_eq!(requests.len(), 2, "{status}: {requests:?}");
        let final_method = if status == 303 { "GET" } else { "PUT" };
        for (request, method, path) in [
            (&requests[0], "PUT", "/start"),
            (&requests[1], final_method, "/final"),
        ] {
            assert!(request.starts_with(&format!("{method} {path} HTTP/1.1\r\n")));
            assert_eq!(
                request_head_header_value(request, "content-length"),
                if method == "PUT" { Some("0") } else { None },
                "{status}: {request}"
            );
            assert_eq!(request_head_header_value(request, "content-type"), None);
            assert_eq!(
                request_head_header_value(request, "transfer-encoding"),
                None
            );
        }
    }
    Ok(())
}

#[tokio::test]
async fn request_headers_distinguish_empty_values_from_absent_upload_content_type() -> Result<()> {
    let mut cases = Vec::new();
    for buffered in [false, true] {
        for method in [
            "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "YO",
        ] {
            for body in [
                None,
                Some(bytes::Bytes::new()),
                Some(bytes::Bytes::from_static(b"payload")),
            ] {
                if matches!(method, "GET" | "HEAD") && body.is_some() {
                    continue;
                }
                for content_type in [None, Some(""), Some("application/example")] {
                    cases.push((buffered, method, body.clone(), content_type));
                }
            }
        }
    }
    // Digest can probe with an empty upload before sending the body, even when
    // the fixture accepts the first request. Inspect both requests on the wire.
    let server = ScriptedHttps11Server::spawn(vec![ScriptedResponse::ok(""); cases.len() * 2]);
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    config.set_default_request_headers(vec![("X-Default-Empty".to_owned(), String::new())]);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    for (index, (buffered, method, body, content_type)) in cases.iter().enumerate() {
        let mut headers = vec![("X-Empty".to_owned(), String::new())];
        if let Some(content_type) = content_type {
            headers.push(("Content-Type".to_owned(), (*content_type).to_owned()));
        }
        let mut request = Request::new_bytes(
            method,
            &server.url_path(&format!("/empty-headers/{index}")),
            body.clone(),
            headers,
        )?;
        if *buffered {
            request = request.with_auth(RequestAuth {
                target: RequestAuthTarget::Server,
                scheme: RequestAuthScheme::Digest,
                username: "user".to_owned(),
                password: "pass".to_owned(),
            });
        }
        assert_eq!(request.auth_requires_buffered_transport(), *buffered);
        assert_eq!(client.fetch_raw(request).await?.status, 200);
    }

    let requests = server.request_heads();
    server.shutdown();
    let mut seen = BTreeSet::new();
    for request in &requests {
        let index = request_path(request)
            .strip_prefix("/empty-headers/")
            .expect("fixture request path")
            .parse::<usize>()?;
        seen.insert(index);
        let case = &cases[index];
        let (_, method, body, content_type) = case;
        assert!(request.starts_with(&format!("{method} /empty-headers/{index} HTTP/1.1\r\n")));
        let values = |name: &str| {
            request
                .split("\r\n")
                .skip(1)
                .take_while(|line| !line.is_empty())
                .filter_map(|line| line.split_once(':'))
                .filter(|(key, _)| key.eq_ignore_ascii_case(name))
                .map(|(_, value)| value.trim())
                .collect::<Vec<_>>()
        };
        assert_eq!(values("x-empty"), [""], "{case:?}: {request}");
        assert_eq!(values("x-default-empty"), [""], "{case:?}: {request}");
        assert_eq!(
            values("content-type"),
            content_type.iter().copied().collect::<Vec<_>>(),
            "{case:?}: {request}"
        );
        if body.as_ref().is_none_or(bytes::Bytes::is_empty) {
            let expected_length = if body.is_some() || matches!(*method, "POST" | "PUT") {
                vec!["0"]
            } else {
                vec![]
            };
            assert_eq!(
                values("content-length"),
                expected_length,
                "{case:?}: {request}"
            );
            assert!(
                values("transfer-encoding").is_empty(),
                "{case:?}: {request}"
            );
        }
    }
    assert_eq!(seen, (0..cases.len()).collect());
    Ok(())
}

#[tokio::test]
async fn fetch_redirect_303_rewrites_post_to_get_and_drops_body_headers() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(303, "See Other").with_header("Location", "/final"),
        ScriptedResponse::ok("final-body"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let response = client
        .fetch(Request::new(
            "POST",
            &server.url_path("/redirect"),
            Some("payload".to_owned()),
            vec![
                ("Content-Type".to_owned(), "text/plain".to_owned()),
                ("X-Keep".to_owned(), "1".to_owned()),
            ],
        )?)
        .await?;

    assert_eq!(response.status, 200);
    assert_eq!(response.body_text(), "final-body");
    assert!(response.redirected);

    let requests = server.requests();
    assert_eq!(requests.len(), 2);
    assert!(requests[0].starts_with("POST /redirect HTTP/1.1"));
    assert!(requests[1].starts_with("GET /final HTTP/1.1"));
    assert!(
        !requests[1]
            .to_ascii_lowercase()
            .contains("content-type: text/plain"),
        "rewritten GET request should not keep Content-Type: {}",
        requests[1]
    );
    assert!(
        !requests[1].contains("payload"),
        "rewritten GET request should not keep the POST body: {}",
        requests[1]
    );
    assert!(
        requests[1].contains("X-Keep: 1"),
        "non body header should be preserved: {}",
        requests[1]
    );
    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn fetch_redirect_307_preserves_post_method_and_body_headers() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(307, "Temporary Redirect").with_header("Location", "/final"),
        ScriptedResponse::ok("final-body"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let response = client
        .fetch(Request::new(
            "POST",
            &server.url_path("/redirect"),
            Some("payload".to_owned()),
            vec![("Content-Type".to_owned(), "text/plain".to_owned())],
        )?)
        .await?;

    assert_eq!(response.status, 200);
    assert_eq!(response.body_text(), "final-body");
    assert!(response.redirected);

    let requests = server.requests();
    assert_eq!(requests.len(), 2);
    assert!(requests[0].starts_with("POST /redirect HTTP/1.1"));
    assert!(requests[1].starts_with("POST /final HTTP/1.1"));
    assert!(
        requests[1]
            .to_ascii_lowercase()
            .contains("content-type: text/plain"),
        "307 redirect should preserve request body headers: {}",
        requests[1]
    );
    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_uses_disk_cache_for_safe_gets() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("cached-raw-stream")
            .with_header("Content-Type", "application/octet-stream")
            .with_header("Cache-Control", "max-age=60"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut first = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;
    assert!(!first.from_cache);
    let mut first_body = Vec::new();
    while let Some(chunk) = first.next_chunk().await {
        first_body.extend_from_slice(&chunk);
    }
    first.finish().await?;

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut second = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;
    assert!(second.from_cache);
    let mut second_body = Vec::new();
    while let Some(chunk) = second.next_chunk().await {
        second_body.extend_from_slice(&chunk);
    }
    second.finish().await?;

    assert_eq!(first_body, b"cached-raw-stream");
    assert_eq!(second_body, b"cached-raw-stream");
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn cache_mode_bypass_skips_fresh_cache_read_and_replaces_entry() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("cached-v1").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("network-v2").with_header("Cache-Control", "max-age=60"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let first = client.fetch_raw(Request::get(&server.url())?).await?;
    let bypassed = client
        .fetch_raw(Request::get(&server.url())?.with_cache_mode(crate::RequestCacheMode::Bypass))
        .await?;
    let after_bypass = client.fetch_raw(Request::get(&server.url())?).await?;

    assert_eq!(first.body_bytes(), b"cached-v1");
    assert_eq!(bypassed.body_bytes(), b"network-v2");
    assert!(!bypassed.from_cache);
    assert_eq!(after_bypass.body_bytes(), b"network-v2");
    assert!(after_bypass.from_cache);
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_uses_disk_cache_for_redirect_hops() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(301, "Moved Permanently")
            .with_header("Location", "/final")
            .with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("final-body")
            .with_header("Content-Type", "application/octet-stream")
            .with_header("Cache-Control", "max-age=60"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut first = client
        .fetch_raw_stream_with_cancel(
            Request::get(&server.url_path("/redirect"))?,
            FetchCancelHandle::new(),
        )
        .await?;
    assert!(!first.from_cache);
    assert_eq!(first.redirect_chain.len(), 1);
    assert!(!first.redirect_chain[0].from_cache);
    let mut first_body = Vec::new();
    while let Some(chunk) = first.next_chunk().await {
        first_body.extend_from_slice(&chunk);
    }
    first.finish().await?;

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut second = client
        .fetch_raw_stream_with_cancel(
            Request::get(&server.url_path("/redirect"))?,
            FetchCancelHandle::new(),
        )
        .await?;
    assert!(second.from_cache);
    assert_eq!(second.redirect_chain.len(), 1);
    assert!(second.redirect_chain[0].from_cache);
    let mut second_body = Vec::new();
    while let Some(chunk) = second.next_chunk().await {
        second_body.extend_from_slice(&chunk);
    }
    second.finish().await?;

    assert_eq!(first_body, b"final-body");
    assert_eq!(second_body, b"final-body");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_delivers_response_when_cache_entry_limit_is_exceeded() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("too-large-for-cache")
            .with_header("Content-Type", "application/octet-stream")
            .with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("network-refetch")
            .with_header("Content-Type", "application/octet-stream")
            .with_header("Cache-Control", "max-age=60"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    config.set_http_cache_max_bytes(Some(4));

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut first = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;
    let mut first_body = Vec::new();
    while let Some(chunk) = first.next_chunk().await {
        first_body.extend_from_slice(&chunk);
    }
    first.finish().await?;

    let mut second = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;
    let mut second_body = Vec::new();
    while let Some(chunk) = second.next_chunk().await {
        second_body.extend_from_slice(&chunk);
    }
    second.finish().await?;

    assert_eq!(first_body, b"too-large-for-cache");
    assert_eq!(second_body, b"network-refetch");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[test]
fn fetch_client_startup_trims_existing_disk_cache_to_quota() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let seed_store = HttpCacheStore::new(&cache_dir);
    let first_url = "http://example.test/first";
    let second_url = "http://example.test/second";
    let third_url = "http://example.test/third";
    let body = vec![b'x'; 1024];
    for (url, stored_at) in [(first_url, 1), (second_url, 2), (third_url, 3)] {
        store_test_cache_body(
            &seed_store,
            &HttpCacheStore::key_for_url(url),
            HttpCacheEntryMetadata::new(
                url.to_owned(),
                url.to_owned(),
                200,
                Vec::new(),
                stored_at,
                Some(stored_at + 60),
                Vec::new(),
            ),
            &body,
        )?;
    }
    assert_eq!(seed_store.entries()?.len(), 3);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    config.set_http_cache_max_bytes(Some(3_000));
    let _client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let trimming_store = HttpCacheStore::with_max_bytes(&cache_dir, Some(3_000));
    assert!(
        trimming_store
            .load_reader(&HttpCacheStore::key_for_url(first_url))?
            .is_none(),
        "oldest existing cache entry should be pruned during client startup"
    );
    assert!(
        trimming_store
            .load_reader(&HttpCacheStore::key_for_url(second_url))?
            .is_some()
    );
    assert!(
        trimming_store
            .load_reader(&HttpCacheStore::key_for_url(third_url))?
            .is_some()
    );

    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn fetch_raw_compat_uses_streaming_disk_cache_for_safe_gets() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("cached-raw-compat")
            .with_header("Content-Type", "application/octet-stream")
            .with_header("Cache-Control", "max-age=60"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let first = client.fetch_raw(Request::get(&server.url())?).await?;

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let second = client.fetch_raw(Request::get(&server.url())?).await?;

    assert_eq!(first.body_bytes(), b"cached-raw-compat");
    assert_eq!(second.body_bytes(), b"cached-raw-compat");
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[test]
fn fetch_client_rejects_private_network_targets() {
    let mut config = FetchConfig::default();
    config.set_network_blocking(true, vec![]);

    let error = fetch_with_config_for_test(&config, Request::get("http://127.0.0.1/").unwrap())
        .unwrap_err();
    let error_chain = format!("{error:#}");

    assert!(
        error_chain.contains("blocked private network address `127.0.0.1`"),
        "unexpected error: {error_chain}"
    );
}

#[test]
fn fetch_client_checks_the_shared_dns_answer_for_domains() {
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(String::new()));
    config.set_network_blocking(true, vec![]);

    let error = fetch_with_config_for_test(&config, Request::get("http://localhost/").unwrap())
        .expect_err("the shared localhost DNS answer must be rejected before curl connects");
    let error_chain = format!("{error:#}");

    assert!(
        error_chain.contains("blocked private network address"),
        "unexpected error: {error_chain}"
    );
}

#[test]
fn fetch_client_allows_proxy_resolved_target_under_address_policy() {
    let proxy = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("proxied-private-target")]);
    let proxy_port = Url::parse(&proxy.origin())
        .unwrap()
        .port()
        .expect("proxy URL should include a port");
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(format!("http://localhost:{proxy_port}")));
    config.set_network_blocking(true, vec![]);

    let response = fetch_with_config_for_test(
        &config,
        Request::get("http://127.0.0.1/proxy-resolved-target").unwrap(),
    )
    .expect("the trusted proxy owns request-target DNS and address admission");

    assert_eq!(response.body_text(), "proxied-private-target");
    assert_eq!(proxy.hits(), 1);
    proxy.shutdown();
}

#[test]
fn fetch_client_uses_host_resolve_for_proxy_endpoint() {
    let proxy = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("fixed-proxy")]);
    let proxy_port = Url::parse(&proxy.origin())
        .unwrap()
        .port()
        .expect("proxy URL should include a port");
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(format!("http://proxy.invalid:{proxy_port}")));
    config.set_http_host_resolve(vec![format!(
        "proxy.invalid:{proxy_port}:127.0.0.2,127.0.0.1"
    )]);

    let response = fetch_with_config_for_test(
        &config,
        Request::get("http://remote-target.invalid/through-fixed-proxy").unwrap(),
    )
    .expect("a fixed proxy endpoint must bypass DNS without fixing the remote target");

    assert_eq!(response.body_text(), "fixed-proxy");
    assert_eq!(proxy.hits(), 1);
    proxy.shutdown();
}

#[test]
fn fetch_client_sends_bearer_auth_only_as_http_proxy_header() {
    let proxy = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("authenticated-proxy")]);
    let mut config = FetchConfig::default();
    config.set_proxy_options(Some(proxy.origin()), Some("proxy-token".to_owned()));

    let response = fetch_with_config_for_test(
        &config,
        Request::get("http://proxy-auth-target.invalid/path").unwrap(),
    )
    .expect("HTTP proxy should receive bearer authentication");
    let requests = proxy.requests();

    assert_eq!(response.body_text(), "authenticated-proxy");
    assert_eq!(requests.len(), 1);
    assert!(
        requests[0].contains("\r\nProxy-Authorization: Bearer proxy-token\r\n"),
        "unexpected proxy request: {:?}",
        requests[0]
    );
    proxy.shutdown();
}

#[test]
fn fetch_client_rejects_host_resolve_for_remote_proxy_target() {
    let proxy = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("unexpected")]);
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(proxy.origin()));
    config.set_http_host_resolve(vec!["target.invalid:80:127.0.0.1".to_owned()]);

    let error = fetch_with_config_for_test(
        &config,
        Request::get("http://target.invalid/ignored-override").unwrap(),
    )
    .expect_err("a remote-DNS proxy cannot honor a target host-resolve override");

    assert!(error.to_string().contains("would be ignored"), "{error:#}");
    assert_eq!(proxy.hits(), 0);
    proxy.shutdown();
}

#[test]
fn fetch_client_rejects_http_header_auth_for_socks_proxy() {
    let mut config = FetchConfig::default();
    config.set_proxy_options(
        Some("socks5://127.0.0.1:1".to_owned()),
        Some("proxy-token".to_owned()),
    );

    let error = fetch_with_config_for_test(
        &config,
        Request::get("http://target.invalid/through-socks").unwrap(),
    )
    .expect_err("SOCKS proxy authentication must not use an HTTP header");

    let error_chain = format!("{error:#}");
    assert!(
        error_chain.contains("Proxy-Authorization headers require an HTTP(S) proxy"),
        "{error_chain}"
    );
}

#[test]
fn fetch_client_socks5_sends_target_hostname_to_proxy() {
    let upstream = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("through-socks")]);
    let upstream_url = Url::parse(&upstream.origin()).unwrap();
    let upstream_port = upstream_url
        .port()
        .expect("upstream URL should include a port");
    let upstream_addr = format!("127.0.0.1:{upstream_port}")
        .parse()
        .expect("upstream socket address");
    let (proxy_url, request_rx, proxy) = spawn_socks5_proxy(upstream_addr);
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(proxy_url));
    config.set_http_no_proxy(Some(String::new()));

    let response = fetch_with_config_for_test(
        &config,
        Request::get(&format!(
            "http://http-target.invalid:{upstream_port}/through-socks"
        ))
        .unwrap(),
    )
    .expect("socks5 must use remote DNS and relay the HTTP request");
    let (requested_host, requested_port) = request_rx
        .recv_timeout(Duration::from_secs(3))
        .expect("SOCKS request should arrive");
    proxy.join().expect("fetch SOCKS proxy should finish");

    assert_eq!(response.body_text(), "through-socks");
    assert_eq!(requested_host, "http-target.invalid");
    assert_eq!(requested_port, upstream_port);
    upstream.shutdown();
}

#[test]
fn fetch_client_socks4_sends_target_hostname_to_proxy() {
    let upstream = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("through-socks4")]);
    let upstream_url = Url::parse(&upstream.origin()).unwrap();
    let upstream_port = upstream_url
        .port()
        .expect("upstream URL should include a port");
    let upstream_addr = format!("127.0.0.1:{upstream_port}")
        .parse()
        .expect("upstream socket address");
    let (proxy_url, request_rx, proxy) = spawn_socks4_proxy(upstream_addr);
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(proxy_url));
    config.set_http_no_proxy(Some(String::new()));

    let response = fetch_with_config_for_test(
        &config,
        Request::get(&format!(
            "http://http-socks4-target.invalid:{upstream_port}/through-socks4"
        ))
        .unwrap(),
    )
    .expect("socks4 must use remote DNS and relay the HTTP request");
    let (requested_host, requested_port) = request_rx
        .recv_timeout(Duration::from_secs(3))
        .expect("SOCKS4 request should arrive");
    proxy.join().expect("fetch SOCKS4 proxy should finish");

    assert_eq!(response.body_text(), "through-socks4");
    assert_eq!(requested_host, "http-socks4-target.invalid");
    assert_eq!(requested_port, upstream_port);
    upstream.shutdown();
}

#[test]
fn fetch_redirect_reselects_direct_or_proxy_endpoint_per_hop() {
    let proxy = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("redirected-through-proxy")]);
    let proxy_port = Url::parse(&proxy.origin())
        .unwrap()
        .port()
        .expect("proxy URL should include a port");
    let direct = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(302, "Found")
            .with_header("Location", "http://redirect-target.invalid/final"),
    ]);
    let direct_url = Url::parse(&direct.url()).unwrap();
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(format!("http://localhost:{proxy_port}")));
    config.set_http_no_proxy(Some("127.0.0.1".to_owned()));

    let response = fetch_with_config_for_test(
        &config,
        Request::get(direct_url.as_str())
            .unwrap()
            .with_redirect_mode(crate::RequestRedirectMode::Follow),
    )
    .expect("redirect should switch from direct origin DNS to proxy endpoint DNS");

    assert_eq!(response.body_text(), "redirected-through-proxy");
    assert_eq!(direct.hits(), 1);
    assert_eq!(proxy.hits(), 1);
    direct.shutdown();
    proxy.shutdown();
}

#[test]
fn fetch_client_applies_selected_no_proxy_route_to_curl() {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("direct")]);
    let port = Url::parse(&server.url())
        .unwrap()
        .port()
        .expect("scripted server URL should include a port");
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some("http://127.0.0.1:1".to_owned()));
    config.set_http_no_proxy(Some("localhost".to_owned()));

    let response = fetch_with_config_for_test(
        &config,
        Request::get(&format!("http://localhost:{port}/direct")).unwrap(),
    )
    .expect(
        "the selected direct route must use shared origin DNS and disable curl proxy evaluation",
    );

    assert_eq!(response.body_text(), "direct");
    assert_eq!(server.hits(), 1);
    server.shutdown();
}

#[test]
fn fetch_client_rejects_private_network_targets_via_host_resolve_override() {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("should-not-be-reached")]);
    let port = Url::parse(&server.url())
        .unwrap()
        .port()
        .expect("scripted server URL should include a port");
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(String::new()));
    config.set_network_blocking(true, vec![]);
    config.set_http_host_resolve(vec![format!("example.com:{port}:127.0.0.1")]);

    let error = fetch_with_config_for_test(
        &config,
        Request::get(&format!("http://example.com:{port}/")).unwrap(),
    )
    .unwrap_err();
    let error_chain = format!("{error:#}");

    assert!(
        error_chain.contains("blocked private network address `127.0.0.1`"),
        "unexpected error: {error_chain}"
    );
    assert_eq!(server.hits(), 0);

    server.shutdown();
}

#[test]
fn fetch_client_rejects_ip_literal_private_target_via_host_resolve_override() {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("should-not-be-reached")]);
    let port = Url::parse(&server.url())
        .unwrap()
        .port()
        .expect("scripted server URL should include a port");
    let mut config = FetchConfig::default();
    config.set_http_proxy(Some(String::new()));
    config.set_network_blocking(true, vec![]);
    config.set_http_host_resolve(vec![format!("1.1.1.1:{port}:127.0.0.1")]);

    let error = fetch_with_config_for_test(
        &config,
        Request::get(&format!("http://1.1.1.1:{port}/")).unwrap(),
    )
    .unwrap_err();
    let error_chain = format!("{error:#}");

    assert!(
        error_chain.contains("blocked private network address `127.0.0.1`"),
        "unexpected error: {error_chain}"
    );
    assert_eq!(server.hits(), 0);

    server.shutdown();
}

#[test]
fn fetch_client_rejects_configured_cidrs() {
    let mut config = FetchConfig::default();
    config.set_network_blocking(false, vec!["198.18.0.0/15".parse().unwrap()]);

    let error = fetch_with_config_for_test(&config, Request::get("http://198.18.0.1/").unwrap())
        .unwrap_err();
    let error_chain = format!("{error:#}");

    assert!(
        error_chain.contains("matches `198.18.0.0/15`"),
        "unexpected error: {error_chain}"
    );
}

#[test]
fn fetch_client_rejects_http_bad_ports_before_network_io() {
    let error = fetch_with_config_for_test(
        &FetchConfig::default(),
        Request::get("http://example.test:25/").unwrap(),
    )
    .unwrap_err();
    let error_chain = format!("{error:#}");

    assert!(
        error_chain.contains("blocked bad port for `http://example.test:25/`"),
        "unexpected error: {error_chain}"
    );
}

#[test]
fn fetch_client_preserves_env_proxy_fallback() {
    let proxy = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("proxied-env")]);
    let mut proxy_url = Url::parse(&proxy.origin()).unwrap();
    proxy_url.set_host(Some("localhost")).unwrap();
    let proxy_origin = proxy_url.to_string();
    let output = Command::new(std::env::current_exe().expect("test binary path"))
        .arg("--exact")
        .arg("tests::env_proxy_child_uses_selected_env_proxy_route")
        .arg("--ignored")
        .arg("--nocapture")
        .env(ENV_PROXY_CHILD_TEST, "1")
        .env(ENV_PROXY_URL, &proxy_origin)
        .env("http_proxy", &proxy_origin)
        .env_remove("HTTP_PROXY")
        .env_remove("HTTPS_PROXY")
        .env_remove("https_proxy")
        .env_remove("ALL_PROXY")
        .env_remove("all_proxy")
        .env_remove("NO_PROXY")
        .env_remove("no_proxy")
        .output()
        .expect("child proxy fallback test should run");

    assert!(
        output.status.success(),
        "child proxy fallback test failed with status {:?}\nstdout:\n{}\nstderr:\n{}",
        output.status.code(),
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr)
    );
    assert_eq!(proxy.hits(), 1);
    let requests = proxy.requests();
    assert_eq!(requests.len(), 1);
    assert_eq!(
        request_path(&requests[0]),
        "http://env-proxy-fallback.invalid/from-env"
    );

    proxy.shutdown();
}

#[test]
#[ignore = "spawned by fetch_client_preserves_env_proxy_fallback"]
fn env_proxy_child_uses_selected_env_proxy_route() -> Result<()> {
    if std::env::var_os(ENV_PROXY_CHILD_TEST).is_none() {
        return Ok(());
    }

    let proxy_origin = std::env::var(ENV_PROXY_URL)?;
    assert_eq!(std::env::var("http_proxy")?, proxy_origin);
    let mut config = FetchConfig::default();
    config.set_request_timeout_ms(1_000);
    config.set_connect_timeout_ms(Some(1_000));

    let response = fetch_with_config_for_test(
        &config,
        Request::get("http://env-proxy-fallback.invalid/from-env")?,
    )?;
    assert_eq!(response.body_text(), "proxied-env");
    Ok(())
}

#[tokio::test]
async fn direct_localhost_uses_shared_dns_across_all_curl_transports() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("buffered-dns"),
        ScriptedResponse::ok("html-dns"),
        ScriptedResponse::ok("raw-dns"),
    ]);
    let port = Url::parse(&server.url())?
        .port()
        .context("scripted server URL should contain a port")?;
    let mut config = FetchConfig::default();
    // Disable ambient proxy environment variables so this test proves the
    // direct-origin DNS residence rather than curl's proxy resolver path.
    config.set_http_proxy(Some(String::new()));
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let buffered = client
        .fetch(Request::get(&format!("http://localhost:{port}/buffered"))?.with_single_hop())
        .await?;
    assert_eq!(buffered.body_text(), "buffered-dns");

    let mut html = client
        .fetch_html_stream(Request::get(&format!("http://localhost:{port}/html"))?)
        .await?;
    let mut html_body = String::new();
    while let Some(chunk) = html.next_chunk().await {
        html_body.push_str(&chunk);
    }
    html.finish().await?;
    assert_eq!(html_body, "html-dns");

    let mut raw = client
        .fetch_raw_stream_with_cancel(
            Request::get(&format!("http://localhost:{port}/raw"))?,
            FetchCancelHandle::new(),
        )
        .await?;
    let mut raw_body = Vec::new();
    while let Some(chunk) = raw.next_chunk().await {
        raw_body.extend_from_slice(&chunk);
    }
    raw.finish().await?;
    assert_eq!(raw_body, b"raw-dns");
    assert_eq!(server.hits(), 3);

    assert!(client.shutdown().is_clean());
    server.shutdown();
    Ok(())
}

#[test]
fn fetch_client_uses_disk_cache_for_safe_gets() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[tokio::test]
async fn fetch_raw_stream_reuses_https_cache_when_tls_verification_is_disabled() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttps11Server::spawn(vec![
        ScriptedResponse::ok("cached-https-raw-stream")
            .with_header("Content-Type", "image/png")
            .with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    config.set_tls_verify_host(false);

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut first = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;
    assert!(!first.from_cache);
    let mut first_body = Vec::new();
    while let Some(chunk) = first.next_chunk().await {
        first_body.extend_from_slice(&chunk);
    }
    first.finish().await?;

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut second = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;
    assert!(second.from_cache);
    let mut second_body = Vec::new();
    while let Some(chunk) = second.next_chunk().await {
        second_body.extend_from_slice(&chunk);
    }
    second.finish().await?;

    assert_eq!(first_body, b"cached-https-raw-stream");
    assert_eq!(second_body, b"cached-https-raw-stream");
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[test]
fn fetch_client_disk_cache_reuses_entries_across_fragment_variants() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let first_url = format!("{}#one", server.url());
    let second_url = format!("{}#two", server.url());

    let first = fetch_with_config_for_test(&config, Request::get(&first_url).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&second_url).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(second.final_url.as_str(), second_url);
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_treats_oversized_cached_body_as_miss() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("cached-body-too-large").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("ok").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut write_config = FetchConfig::default();
    write_config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let mut read_config = FetchConfig::default();
    read_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    read_config.set_connection_limits(None, None, Some(4));

    let first =
        fetch_with_config_for_test(&write_config, Request::get(&server.url()).unwrap()).unwrap();
    let second =
        fetch_with_config_for_test(&read_config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "cached-body-too-large");
    assert_eq!(second.body_text(), "ok");
    assert_eq!(
        server.hits(),
        2,
        "oversized buffered cache hits should fall through to the network"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_does_not_revalidate_oversized_cached_body() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("cached-body-too-large")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"large\""),
        ScriptedResponse::ok("ok").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut write_config = FetchConfig::default();
    write_config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let mut read_config = FetchConfig::default();
    read_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    read_config.set_connection_limits(None, None, Some(4));

    let first =
        fetch_with_config_for_test(&write_config, Request::get(&server.url()).unwrap()).unwrap();
    let second =
        fetch_with_config_for_test(&read_config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "cached-body-too-large");
    assert_eq!(second.body_text(), "ok");
    assert_eq!(server.hits(), 2);
    let requests = server.requests();
    assert!(
        !requests[1].to_ascii_lowercase().contains("if-none-match"),
        "oversized stale cache entries should miss instead of sending conditional validation"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_refetches_when_304_cached_body_replay_exceeds_limit() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("small")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"small\""),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "max-age=60")
            .with_delay_ms(200),
        ScriptedResponse::ok("ok").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut write_config = FetchConfig::default();
    write_config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let mut read_config = FetchConfig::default();
    read_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    read_config.set_connection_limits(None, None, Some(8));

    let first =
        fetch_with_config_for_test(&write_config, Request::get(&server.url()).unwrap()).unwrap();
    assert_eq!(first.body_text(), "small");

    let body_path = fs::read_dir(&cache_dir)
        .unwrap()
        .filter_map(|entry| entry.ok())
        .map(|entry| entry.path())
        .find(|path| path.is_dir() && path.extension().is_some_and(|ext| ext == "entry"))
        .and_then(|entry_dir| {
            fs::read_dir(entry_dir)
                .ok()?
                .filter_map(|entry| entry.ok())
                .find_map(|entry| {
                    let path = entry.path();
                    path.file_name()
                        .and_then(|name| name.to_str())
                        .is_some_and(|name| name.starts_with("body.") && name.ends_with(".bin"))
                        .then_some(path)
                })
        })
        .expect("cached body path should exist");

    let request_url = server.url();
    let fetch_handle = thread::spawn(move || {
        fetch_with_config_for_test(&read_config, Request::get(&request_url).unwrap())
            .expect("post-304 cached body replay failure should refetch")
    });

    let deadline = Instant::now() + Duration::from_secs(2);
    while server.hits() < 2 && Instant::now() < deadline {
        thread::sleep(Duration::from_millis(10));
    }
    assert_eq!(server.hits(), 2, "second request should be revalidating");
    fs::write(&body_path, b"cached-body-too-large").unwrap();

    let second = fetch_handle.join().expect("fetch thread should not panic");

    assert_eq!(second.body_text(), "ok");
    assert_eq!(server.hits(), 3);
    let requests = server.requests();
    assert!(
        requests[1].to_ascii_lowercase().contains("if-none-match"),
        "second request should revalidate the stale cache entry: {}",
        requests[1]
    );
    assert!(
        !requests[2].to_ascii_lowercase().contains("if-none-match"),
        "retry after failed 304 merge must be an unconditional network fetch: {}",
        requests[2]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_persists_disk_cache_files_to_configured_dir() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let response =
        fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(response.body_text(), "hit-1");
    let entries = std::fs::read_dir(&cache_dir)
        .unwrap()
        .map(|entry| entry.unwrap().path())
        .collect::<Vec<_>>();
    assert_eq!(entries.len(), 1, "cache dir should contain one entry dir");
    assert!(
        entries[0].is_dir(),
        "cache should use Chromium-like directory entries"
    );
    assert!(entries[0].join("meta.json").is_file());
    assert!(
        std::fs::read_dir(&entries[0]).unwrap().any(|entry| entry
            .unwrap()
            .file_name()
            .to_string_lossy()
            .starts_with("body.")),
        "entry dir should contain a separately stored body stream"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_rejects_disk_cache_entry_when_request_url_differs() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let entry_dir = std::fs::read_dir(&cache_dir)
        .unwrap()
        .next()
        .expect("cache entry should exist")
        .unwrap()
        .path();
    let meta_path = entry_dir.join("meta.json");
    let metadata = std::fs::read_to_string(&meta_path).unwrap();
    let rewritten_metadata = metadata.replace(
        &format!("\"request_url\":\"{}\"", server.url()),
        "\"request_url\":\"http://example.invalid/other\"",
    );
    assert_ne!(metadata, rewritten_metadata);
    std::fs::write(&meta_path, rewritten_metadata).unwrap();

    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_skips_disk_cache_for_vary_star() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "*"),
        ScriptedResponse::ok("hit-2")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "*"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_skips_disk_cache_when_any_vary_header_is_star() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Accept-Encoding")
            .with_header("Vary", "*"),
        ScriptedResponse::ok("hit-2")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Accept-Encoding")
            .with_header("Vary", "*"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(
        server.hits(),
        2,
        "all Vary header fields must be considered before storing"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_vary_user_agent_partitions_hits() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "User-Agent"),
        ScriptedResponse::ok("hit-2")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "User-Agent"),
    ]);

    let mut first_config = FetchConfig::default();
    first_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    first_config.set_user_agent("Moli/Test-UA-1");
    let mut second_config = FetchConfig::default();
    second_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    second_config.set_user_agent("Moli/Test-UA-2");

    let first =
        fetch_with_config_for_test(&first_config, Request::get(&server.url()).unwrap()).unwrap();
    let second =
        fetch_with_config_for_test(&second_config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_vary_accept_language_records_effective_header() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Accept-Language"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let response =
        fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(response.body_text(), "hit-1");
    let entry_dir = fs::read_dir(&cache_dir)
        .unwrap()
        .next()
        .expect("cache entry should exist")
        .unwrap()
        .path();
    let metadata = fs::read_to_string(entry_dir.join("meta.json")).unwrap();
    assert!(
        metadata.contains("\"name\":\"accept-language\""),
        "metadata should record Accept-Language vary key: {metadata}"
    );
    assert!(
        metadata.contains(DEFAULT_ACCEPT_LANGUAGE),
        "metadata should record effective Accept-Language value: {metadata}"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_vary_referer_partitions_hits() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Referer"),
        ScriptedResponse::ok("hit-2")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Referer"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let request_url = Url::parse(&server.url()).unwrap();
    let first_referrer = request_url.join("/page-one").unwrap();
    let second_referrer = request_url.join("/page-two").unwrap();
    let fetch_for_referrer = |referrer: &Url| {
        fetch_with_config_for_test(
            &config,
            Request::new("GET", &server.url(), None, Vec::new())
                .unwrap()
                .with_browser_request_metadata(BrowserRequestMetadata::Fetch)
                .with_initiator_url(referrer)
                .with_request_origin(moli_url::WebOrigin::from_url(referrer)),
        )
        .unwrap()
    };

    let first = fetch_for_referrer(&first_referrer);
    let second = fetch_for_referrer(&first_referrer);
    let third = fetch_for_referrer(&second_referrer);

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(third.body_text(), "hit-2");
    assert_eq!(
        server.hits(),
        2,
        "Vary: Referer should hit only when the effective Referer matches"
    );

    let entry_dir = fs::read_dir(&cache_dir)
        .unwrap()
        .next()
        .expect("cache entry should exist")
        .unwrap()
        .path();
    let metadata = fs::read_to_string(entry_dir.join("meta.json")).unwrap();
    assert!(
        metadata.contains("\"name\":\"referer\""),
        "metadata should record Referer vary key: {metadata}"
    );
    assert!(
        metadata.contains("/page-two"),
        "metadata should record the latest effective Referer value: {metadata}"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_vary_navigation_browser_headers_are_cacheable() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header(
                "Vary",
                "Accept, Sec-Fetch-Mode, Sec-Fetch-Dest, Sec-Fetch-Site, Sec-Fetch-User, Upgrade-Insecure-Requests",
            ),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(
        server.hits(),
        1,
        "Vary over synthesized navigation headers should remain cacheable"
    );

    let entry_dir = fs::read_dir(&cache_dir)
        .unwrap()
        .next()
        .expect("cache entry should exist")
        .unwrap()
        .path();
    let metadata = fs::read_to_string(entry_dir.join("meta.json")).unwrap();
    for expected in [
        "\"name\":\"accept\"",
        "\"name\":\"sec-fetch-mode\"",
        "\"name\":\"sec-fetch-dest\"",
        "\"name\":\"sec-fetch-site\"",
        "\"name\":\"sec-fetch-user\"",
        "\"name\":\"upgrade-insecure-requests\"",
    ] {
        assert!(
            metadata.contains(expected),
            "metadata should record supported Vary key {expected}: {metadata}"
        );
    }

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_vary_client_hints_are_cacheable() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Sec-CH-UA, Sec-CH-UA-Mobile, Sec-CH-UA-Platform"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(
        server.hits(),
        1,
        "Vary over synthesized client hints should remain cacheable"
    );

    let entry_dir = fs::read_dir(&cache_dir)
        .unwrap()
        .next()
        .expect("cache entry should exist")
        .unwrap()
        .path();
    let metadata = fs::read_to_string(entry_dir.join("meta.json")).unwrap();
    for expected in [
        "\"name\":\"sec-ch-ua\"",
        "\"name\":\"sec-ch-ua-mobile\"",
        "\"name\":\"sec-ch-ua-platform\"",
    ] {
        assert!(
            metadata.contains(expected),
            "metadata should record supported client-hint Vary key {expected}: {metadata}"
        );
    }

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn critical_client_hints_restart_navigation_before_exposing_the_first_response() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(403, "Challenge")
            .with_body("discarded challenge")
            .with_header("Accept-CH", TEST_HIGH_ENTROPY_CLIENT_HINTS)
            .with_header("Critical-CH", TEST_HIGH_ENTROPY_CLIENT_HINTS),
        ScriptedResponse::ok("restarted navigation"),
        ScriptedResponse::ok("subresource"),
    ]);
    let config = FetchConfig::default();
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let navigation = fetch_response_for_test(&client, Request::get(&server.url())?)?;
    assert_eq!(navigation.body_text(), "restarted navigation");
    assert_eq!(navigation.redirect_chain.len(), 1);
    let restart = &navigation.redirect_chain[0];
    assert_eq!(restart.from_url, restart.to_url);
    assert_eq!(restart.status, 307);
    assert_eq!(
        restart
            .headers
            .iter()
            .find(|(name, _)| name.eq_ignore_ascii_case("location"))
            .map(|(_, value)| value.as_slice()),
        Some(restart.to_url.as_str().as_bytes())
    );
    assert!(!restart.redirect_has_extra_info);
    assert!(restart.cookie_set_reports.is_empty());
    assert_eq!(
        restart
            .response_extra_info
            .as_ref()
            .map(|extra| extra.status),
        Some(403)
    );
    let initial_headers = &restart
        .response_extra_info
        .as_ref()
        .expect("discarded server response extra info")
        .request_extra_info
        .headers;
    assert!(
        !initial_headers
            .iter()
            .any(|(name, _)| name.eq_ignore_ascii_case("sec-ch-ua-arch"))
    );
    let restarted_headers = &restart
        .request_extra_info
        .as_ref()
        .expect("restarted request extra info")
        .headers;
    assert!(restarted_headers.iter().any(|(name, value)| {
        name.eq_ignore_ascii_case("sec-ch-ua-arch") && value == "\"x86\""
    }));
    assert!(navigation.network_request_extra_info().is_some());

    let navigation_url = Url::parse(&server.url())?;
    let subresource = Request::new("POST", &server.url(), Some("probe".to_owned()), Vec::new())?
        .with_browser_request_metadata(BrowserRequestMetadata::Fetch)
        .with_initiator_url(&navigation_url)
        .with_request_origin(moli_url::WebOrigin::from_url(&navigation_url));
    let subresource = fetch_response_for_test(&client, subresource)?;
    assert_eq!(subresource.body_text(), "subresource");

    let requests = server.requests();
    assert_eq!(requests.len(), 3, "unexpected request chain: {requests:#?}");
    let first = requests[0].to_ascii_lowercase();
    let restarted = requests[1].to_ascii_lowercase();
    let later = requests[2].to_ascii_lowercase();
    assert!(!first.contains("sec-ch-ua-arch:"));
    for expected in [
        "sec-ch-ua-full-version: \"145.0.0.0\"",
        "sec-ch-ua-full-version-list:",
        "sec-ch-ua-arch: \"x86\"",
        "sec-ch-ua-bitness: \"64\"",
        "sec-ch-ua-platform-version: \"19.0.0\"",
        "sec-ch-ua-model: \"\"",
        "sec-ch-ua-wow64: ?0",
    ] {
        assert!(
            restarted.contains(expected),
            "restarted navigation missing {expected}: {}",
            requests[1]
        );
        assert!(
            later.contains(expected),
            "later same-origin request missing persisted {expected}: {}",
            requests[2]
        );
    }

    server.shutdown();
    Ok(())
}

#[test]
fn critical_client_hints_restart_redirect_hop_before_following_location() -> Result<()> {
    let redirect = || {
        ScriptedResponse::status(302, "Found")
            .with_header("Location", "/final")
            .with_header("Accept-CH", "Sec-CH-UA-Arch")
            .with_header("Critical-CH", "Sec-CH-UA-Arch")
    };
    let server =
        ScriptedHttpServer::spawn(vec![redirect(), redirect(), ScriptedResponse::ok("ok")]);
    let config = FetchConfig::default();
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let response = fetch_response_for_test(&client, Request::get(&server.url())?)?;
    assert_eq!(response.body_text(), "ok");
    assert_eq!(response.redirect_chain.len(), 2);
    let restart = &response.redirect_chain[0];
    assert_eq!(restart.status, 307);
    assert_eq!(restart.from_url, restart.to_url);
    assert!(!restart.redirect_has_extra_info);
    assert_eq!(
        restart
            .response_extra_info
            .as_ref()
            .map(|extra| extra.status),
        Some(302)
    );
    assert!(restart.request_extra_info.is_some());
    let server_redirect = &response.redirect_chain[1];
    assert_eq!(server_redirect.status, 302);
    assert!(server_redirect.redirect_has_extra_info);
    assert_eq!(
        server_redirect
            .response_extra_info
            .as_ref()
            .map(|extra| extra.status),
        Some(302)
    );

    let requests = server.requests();
    assert_eq!(requests.len(), 3, "unexpected request chain: {requests:#?}");
    assert!(requests[0].starts_with("GET /cache "));
    assert!(requests[1].starts_with("GET /cache "));
    assert!(requests[2].starts_with("GET /final "));
    assert!(!requests[0].to_ascii_lowercase().contains("sec-ch-ua-arch:"));
    assert!(
        requests[1]
            .to_ascii_lowercase()
            .contains("sec-ch-ua-arch: \"x86\"")
    );
    assert!(
        requests[2]
            .to_ascii_lowercase()
            .contains("sec-ch-ua-arch: \"x86\"")
    );

    server.shutdown();
    Ok(())
}

#[test]
fn critical_client_hint_vary_remains_cacheable_after_restart() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(403, "Challenge")
            .with_header("Accept-CH", "Sec-CH-UA-Arch")
            .with_header("Critical-CH", "Sec-CH-UA-Arch"),
        ScriptedResponse::ok("cached")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Sec-CH-UA-Arch"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let first = fetch_response_for_test(&client, Request::get(&server.url())?)?;
    let second = fetch_response_for_test(&client, Request::get(&server.url())?)?;
    assert_eq!(first.body_text(), "cached");
    assert_eq!(second.body_text(), "cached");
    assert_eq!(
        server.hits(),
        2,
        "the post-restart response should be reused"
    );

    let entry_dir = fs::read_dir(&cache_dir)
        .unwrap()
        .next()
        .expect("cache entry should exist")
        .unwrap()
        .path();
    let metadata = fs::read_to_string(entry_dir.join("meta.json"))?;
    assert!(
        metadata.contains("\"name\":\"sec-ch-ua-arch\""),
        "{metadata}"
    );
    assert!(metadata.contains("\\\"x86\\\""), "{metadata}");

    server.shutdown();
    let _ = fs::remove_dir_all(cache_dir);
    Ok(())
}

#[test]
fn fetch_client_partitions_disk_cache_by_top_frame_origin() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=60"),
    ]);
    let first_top_frame = Url::parse("https://app.example.test/").unwrap();
    let second_top_frame = Url::parse("https://other.example.test/").unwrap();

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_top_frame_origin_url(&first_top_frame),
    )
    .unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_top_frame_origin_url(&second_top_frame),
    )
    .unwrap();
    let third = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_top_frame_origin_url(&first_top_frame),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(third.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_treats_disk_cache_store_failure_as_best_effort() {
    let cache_file = unique_test_cache_dir();
    fs::write(&cache_file, b"not a directory").unwrap();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_file.display().to_string()));

    let response =
        fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(response.body_text(), "hit-1");
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_file(cache_file);
}

#[test]
fn fetch_client_caches_script_referrer_and_priority_metadata_requests() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let script_metadata = ScriptFetchRequestMetadata {
        referrer_policy: Some("origin".to_owned()),
        fetch_priority: Some(FetchPriorityHint::High),
        ..ScriptFetchRequestMetadata::default()
    };

    let first = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_script_fetch_metadata(script_metadata.clone()),
    )
    .unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_script_fetch_metadata(script_metadata),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert!(second.head().from_cache);
    assert_eq!(
        server.hits(),
        1,
        "referrer policy and scheduling hints do not change response identity without Vary"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_uses_disk_cache_for_default_script_metadata_requests() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_script_fetch_metadata(ScriptFetchRequestMetadata::default()),
    )
    .unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_script_fetch_metadata(ScriptFetchRequestMetadata::default()),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(
        server.hits(),
        1,
        "default script metadata should remain eligible for HTTP cache reuse"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

fn stylesheet_cache_request(url: &str, metadata: SubresourceRequestMetadata) -> Request {
    Request::new("GET", url, None, Vec::new())
        .unwrap()
        .with_resource_type(RequestResourceType::CssStyleSheet)
        .with_request_mode(RequestMode::NoCors)
        .with_credentials_mode(RequestCredentialsMode::Include)
        .with_browser_request_metadata(BrowserRequestMetadata::Style)
        .with_subresource_request_metadata(metadata)
}

#[test]
fn fetch_client_uses_disk_cache_for_default_stylesheet_metadata() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("body { color: green; }")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(
        &config,
        stylesheet_cache_request(&server.url(), SubresourceRequestMetadata::default()),
    )
    .unwrap();
    let second = fetch_with_config_for_test(
        &config,
        stylesheet_cache_request(&server.url(), SubresourceRequestMetadata::default()),
    )
    .unwrap();

    assert_eq!(first.body_text(), "body { color: green; }");
    assert_eq!(second.body_text(), "body { color: green; }");
    assert!(second.head().from_cache);
    assert_eq!(server.hits(), 1);
    let request = server.requests()[0].to_ascii_lowercase().replace(' ', "");
    assert!(
        request.contains("sec-fetch-dest:style"),
        "stylesheet request should carry style destination: {request}"
    );
    assert!(
        request.contains("sec-fetch-mode:no-cors"),
        "stylesheet request should carry no-cors mode: {request}"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_excludes_unvalidated_stylesheet_integrity_from_cache() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("first")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css"),
        ScriptedResponse::ok("second")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let metadata = SubresourceRequestMetadata {
        integrity: Some("sha256-test".to_owned()),
        ..SubresourceRequestMetadata::default()
    };

    let first = fetch_with_config_for_test(
        &config,
        stylesheet_cache_request(&server.url(), metadata.clone()),
    )
    .unwrap();
    let second =
        fetch_with_config_for_test(&config, stylesheet_cache_request(&server.url(), metadata))
            .unwrap();

    assert_eq!(first.body_text(), "first");
    assert_eq!(second.body_text(), "second");
    assert_eq!(
        server.hits(),
        2,
        "integrity stays excluded until stylesheet cache hits revalidate each client"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_stylesheet_referrer_policy_uses_vary_referer_on_cache_hits() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("origin-policy")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css")
            .with_header("Vary", "Referer"),
        ScriptedResponse::ok("no-referrer-policy")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css")
            .with_header("Vary", "Referer"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let document_url = Url::parse("https://document.example/path/page.html").unwrap();
    let request = |policy: &str| {
        stylesheet_cache_request(
            &server.url(),
            SubresourceRequestMetadata {
                referrer_policy: Some(policy.to_owned()),
                ..SubresourceRequestMetadata::default()
            },
        )
        .with_initiator_url(&document_url)
        .with_request_origin(moli_url::WebOrigin::from_url(&document_url))
    };

    let first = fetch_with_config_for_test(&config, request("origin")).unwrap();
    let second = fetch_with_config_for_test(&config, request("origin")).unwrap();
    let third = fetch_with_config_for_test(&config, request("no-referrer")).unwrap();
    let fourth = fetch_with_config_for_test(&config, request("no-referrer")).unwrap();

    assert_eq!(first.body_text(), "origin-policy");
    assert_eq!(second.body_text(), "origin-policy");
    assert!(second.head().from_cache);
    assert_eq!(third.body_text(), "no-referrer-policy");
    assert_eq!(fourth.body_text(), "no-referrer-policy");
    assert!(fourth.head().from_cache);
    assert_eq!(server.hits(), 2);
    assert!(
        server.requests()[0]
            .to_ascii_lowercase()
            .contains("referer: https://document.example/")
    );
    assert!(
        !server.requests()[1]
            .to_ascii_lowercase()
            .contains("referer:")
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_ignores_script_producer_and_scheduling_only_metadata() {
    let cases = [
        (
            "cross-origin-attribute",
            ScriptFetchRequestMetadata {
                cross_origin: Some("anonymous".to_owned()),
                ..ScriptFetchRequestMetadata::default()
            },
        ),
        (
            "charset",
            ScriptFetchRequestMetadata {
                charset: Some("utf-8".to_owned()),
                ..ScriptFetchRequestMetadata::default()
            },
        ),
        (
            "nonce",
            ScriptFetchRequestMetadata {
                nonce: Some("nonce-test".to_owned()),
                ..ScriptFetchRequestMetadata::default()
            },
        ),
        (
            "fetch-priority",
            ScriptFetchRequestMetadata {
                fetch_priority: Some(FetchPriorityHint::High),
                ..ScriptFetchRequestMetadata::default()
            },
        ),
        (
            "scheduler-priority",
            ScriptFetchRequestMetadata {
                scheduler_priority: Some(ScriptFetchSchedulerPriority::High),
                ..ScriptFetchRequestMetadata::default()
            },
        ),
    ];

    for (label, metadata) in cases {
        let cache_dir = unique_test_cache_dir();
        let server = ScriptedHttpServer::spawn(vec![
            ScriptedResponse::ok("first")
                .with_header("Cache-Control", "max-age=60")
                .with_header("Content-Type", "text/css"),
        ]);
        let mut config = FetchConfig::default();
        config.set_http_cache_dir(Some(cache_dir.display().to_string()));
        let request = || {
            stylesheet_cache_request(&server.url(), SubresourceRequestMetadata::default())
                .with_script_fetch_metadata(metadata.clone())
                .with_resource_type(RequestResourceType::CssStyleSheet)
        };

        let first = fetch_with_config_for_test(&config, request()).unwrap();
        let second = fetch_with_config_for_test(&config, request()).unwrap();

        assert_eq!(first.body_text(), "first", "{label}");
        assert_eq!(second.body_text(), "first", "{label}");
        assert!(second.head().from_cache, "{label}");
        assert_eq!(
            server.hits(),
            1,
            "{label} must not split HTTP response identity"
        );

        server.shutdown();
        let _ = std::fs::remove_dir_all(cache_dir);
    }
}

#[test]
fn fetch_client_partitions_stylesheet_cache_by_credentials_mode() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("include")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css"),
        ScriptedResponse::ok("same-origin")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let request = |credentials_mode| {
        stylesheet_cache_request(&server.url(), SubresourceRequestMetadata::default())
            .with_credentials_mode(credentials_mode)
    };

    let first =
        fetch_with_config_for_test(&config, request(RequestCredentialsMode::Include)).unwrap();
    let second =
        fetch_with_config_for_test(&config, request(RequestCredentialsMode::SameOrigin)).unwrap();
    let third =
        fetch_with_config_for_test(&config, request(RequestCredentialsMode::Include)).unwrap();

    assert_eq!(first.body_text(), "include");
    assert_eq!(second.body_text(), "same-origin");
    assert_eq!(third.body_text(), "include");
    assert!(third.head().from_cache);
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_skips_stylesheet_cache_for_vary_origin() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("first")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css")
            .with_header("Vary", "Origin"),
        ScriptedResponse::ok("second")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Content-Type", "text/css")
            .with_header("Vary", "Origin"),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let initiator = Url::parse("https://document.example/page").unwrap();
    let request = || {
        stylesheet_cache_request(&server.url(), SubresourceRequestMetadata::default())
            .with_request_mode(RequestMode::Cors)
            .with_credentials_mode(RequestCredentialsMode::SameOrigin)
            .with_initiator_url(&initiator)
            .with_request_origin(moli_url::WebOrigin::from_url(&initiator))
    };

    let first = fetch_with_config_for_test(&config, request()).unwrap();
    let second = fetch_with_config_for_test(&config, request()).unwrap();

    assert_eq!(first.body_text(), "first");
    assert_eq!(second.body_text(), "second");
    assert_eq!(server.hits(), 2);
    assert!(
        server.requests()[0]
            .to_ascii_lowercase()
            .contains("origin: https://document.example")
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_mode_validate_revalidates_fresh_script_metadata_hit() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\""),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_script_fetch_metadata(ScriptFetchRequestMetadata::default()),
    )
    .unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_cache_mode(RequestCacheMode::Validate)
            .with_script_fetch_metadata(ScriptFetchRequestMetadata::default()),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(
        server.hits(),
        2,
        "Validate cache mode must revalidate a fresh script cache entry"
    );
    let second_request = server.requests()[1].to_ascii_lowercase();
    assert!(
        second_request.contains("if-none-match: \"v1\""),
        "Validate cache mode should attach cached validators: {}",
        server.requests()[1]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_matches_effective_explicit_user_agent() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "User-Agent"),
        ScriptedResponse::ok("hit-2")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "User-Agent"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    config.set_user_agent("Moli/Base-UA");
    let request_with_user_agent = |user_agent: &str| {
        Request::new(
            "GET",
            &server.url(),
            None,
            vec![("User-Agent".to_owned(), user_agent.to_owned())],
        )
        .unwrap()
    };

    let first =
        fetch_with_config_for_test(&config, request_with_user_agent("Moli/Protocol-UA-1")).unwrap();
    let second =
        fetch_with_config_for_test(&config, request_with_user_agent("Moli/Protocol-UA-1")).unwrap();
    let third =
        fetch_with_config_for_test(&config, request_with_user_agent("Moli/Protocol-UA-2")).unwrap();
    let fourth =
        fetch_with_config_for_test(&config, request_with_user_agent("Moli/Protocol-UA-2")).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(third.body_text(), "hit-2");
    assert_eq!(fourth.body_text(), "hit-2");
    assert_eq!(
        server.hits(),
        2,
        "Vary: User-Agent should match the effective explicit header"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

fn assert_explicit_request_headers_bypass_disk_cache(
    request_headers: Vec<(String, String)>,
    reason: &str,
) {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let request = || Request::new("GET", &server.url(), None, request_headers.clone()).unwrap();

    let first = fetch_with_config_for_test(&config, request()).unwrap();
    let second = fetch_with_config_for_test(&config, request()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2, "{reason}");

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_skips_disk_cache_for_unsupported_explicit_request_headers() {
    assert_explicit_request_headers_bypass_disk_cache(
        vec![("X-Custom-Cache-Input".to_owned(), "one".to_owned())],
        "unsupported custom request headers must remain outside the disk cache",
    );
}

#[test]
fn fetch_client_skips_disk_cache_for_request_cache_control_no_store() {
    assert_explicit_request_headers_bypass_disk_cache(
        vec![("Cache-Control".to_owned(), "no-store".to_owned())],
        "request Cache-Control: no-store must bypass disk cache storage and reuse",
    );
}

#[test]
fn fetch_client_skips_disk_cache_for_duplicate_supported_request_headers() {
    assert_explicit_request_headers_bypass_disk_cache(
        vec![
            ("Accept-Language".to_owned(), "en-US".to_owned()),
            ("accept-language".to_owned(), "fr".to_owned()),
        ],
        "duplicate supported headers must remain outside last-value Vary matching",
    );
}

#[test]
fn fetch_client_skips_disk_cache_for_default_request_headers() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    config.set_default_request_headers(vec![("X-Test".to_owned(), "one".to_owned())]);

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(
        server.hits(),
        2,
        "requests with configured default headers must not use the URL-only disk cache"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_skips_disk_cache_for_post_requests() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(
        &config,
        Request::new(
            "POST",
            &server.url(),
            Some("payload".to_owned()),
            Vec::new(),
        )
        .unwrap(),
    )
    .unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::new(
            "POST",
            &server.url(),
            Some("payload".to_owned()),
            Vec::new(),
        )
        .unwrap(),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_skips_disk_cache_for_get_requests_with_body() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(
        &config,
        Request::new("GET", &server.url(), Some("payload".to_owned()), Vec::new()).unwrap(),
    )
    .unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::new("GET", &server.url(), Some("payload".to_owned()), Vec::new()).unwrap(),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_skips_disk_cache_for_auth_requests() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let auth = RequestAuth {
        target: RequestAuthTarget::Server,
        scheme: RequestAuthScheme::Basic,
        username: "user".to_owned(),
        password: "pass".to_owned(),
    };

    let first = fetch_with_config_for_test(
        &config,
        Request::get(&server.url()).unwrap().with_auth(auth.clone()),
    )
    .unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::get(&server.url()).unwrap().with_auth(auth),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_respects_no_store() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "no-store"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "no-store"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_streams_preemptive_basic_server_auth() {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("basic-ok")]);
    let auth = RequestAuth {
        target: RequestAuthTarget::Server,
        scheme: RequestAuthScheme::Basic,
        username: "user".to_owned(),
        password: "pass".to_owned(),
    };

    let response = fetch_with_config_for_test(
        &FetchConfig::default(),
        Request::get(&server.url()).unwrap().with_auth(auth),
    )
    .unwrap();

    assert_eq!(response.body_text(), "basic-ok");
    assert_eq!(server.hits(), 1);
    assert!(
        server.requests()[0].to_ascii_lowercase().contains(
            "authorization: basic dxnlcjpwYXNz"
                .to_ascii_lowercase()
                .as_str()
        ),
        "basic auth should be sent as a preemptive header on the streaming path: {}",
        server.requests()[0]
    );

    server.shutdown();
}

#[test]
fn fetch_client_cache_respects_no_cache_response() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "no-cache"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "no-cache"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_revalidates_no_cache_response_with_etag() {
    // Cache-Control: no-cache permits storage, but every reuse must validate.
    // A 304 response should merge with the stored body instead of surfacing an
    // empty Not Modified response to the caller.
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "no-cache")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "no-cache")
            .with_header("ETag", "\"v1\""),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);
    assert!(
        server.requests()[1].contains("If-None-Match: \"v1\""),
        "no-cache reuse should validate with the stored ETag: {}",
        server.requests()[1]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn network_metadata_captures_transport_generated_request_headers() {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("wire headers").with_header("Cache-Control", "no-store"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let request = Request::new(
        "GET",
        &server.url(),
        None,
        vec![("X-Observed".to_owned(), "yes".to_owned())],
    )
    .unwrap();

    let observed = fetch_raw_with_network_metadata_for_test(&client, request).unwrap();
    let journal = observed.observation_journal();

    assert_eq!(journal.exchanges().len(), 1);
    let request_headers = journal.exchanges()[0].request().headers();
    assert_header_present(request_headers, "Host");
    assert_header_present(request_headers, "Accept-Encoding");
    assert_header_present(request_headers, "User-Agent");
    assert_eq!(header_value(request_headers, "X-Observed"), Some("yes"));
    let response = journal.exchanges()[0]
        .response()
        .expect("raw response observation");
    assert_eq!(response.status(), 200);
    assert_eq!(
        moli_web_mime::response_header_value(response.headers(), "Cache-Control").as_deref(),
        Some("no-store")
    );

    assert!(client.shutdown().is_clean());
    server.shutdown();
}

#[tokio::test]
async fn network_metadata_failure_preserves_request_sent_before_empty_response() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener
            .accept()
            .await
            .expect("empty response server should accept request");
        read_http_request_head(&mut stream)
            .await
            .expect("empty response server should read request");
    });
    let url = Url::parse(&format!("http://{addr}/empty"))?;
    let cookie_store = new_shared_browser_cookie_store();
    cookie_store
        .lock()
        .set_document_cookie(&url, "failedRequestCookie=present; Path=/");
    let client = FetchClient::new(&FetchConfig::default(), cookie_store);

    let error = client
        .fetch_raw_with_network_metadata(
            Request::get(url.as_str())?.with_top_level_navigation_cookie_context(),
        )
        .await
        .expect_err("empty response should fail");
    let failure = error
        .downcast_ref::<NetworkFetchFailureContext>()
        .expect("metadata fetch failure should retain its typed observation journal");
    let [exchange] = failure.observation_journal().exchanges() else {
        panic!("expected exactly one observed exchange");
    };

    assert!(exchange.response().is_none());
    assert_header_present(exchange.request().headers(), "Host");
    assert_header_present(exchange.request().headers(), "Accept-Encoding");
    let cookie_report = exchange
        .request()
        .cookie_report()
        .expect("actual request should retain its cookie access report");
    assert_eq!(cookie_report.included_cookies.len(), 1);

    assert!(client.shutdown().is_clean());
    server.await.expect("empty response server should finish");
    Ok(())
}

#[tokio::test]
async fn network_metadata_failure_preserves_redirect_chain_and_final_request() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut first, _) = listener
            .accept()
            .await
            .expect("redirect server should accept initial request");
        read_http_request_head(&mut first)
            .await
            .expect("redirect server should read initial request");
        first
            .write_all(
                b"HTTP/1.1 302 Found\r\nLocation: /reset\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
            )
            .await
            .expect("redirect server should write redirect response");
        first
            .shutdown()
            .await
            .expect("redirect server should close initial connection");

        let (mut second, _) = listener
            .accept()
            .await
            .expect("redirect server should accept final request");
        read_http_request_head(&mut second)
            .await
            .expect("redirect server should read final request");
    });
    let initial_url = Url::parse(&format!("http://{addr}/start"))?;
    let final_url = Url::parse(&format!("http://{addr}/reset"))?;
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());

    let error = client
        .fetch_raw_with_network_metadata(Request::get(initial_url.as_str())?)
        .await
        .expect_err("redirect final request should fail without a response");
    let failure = error
        .downcast_ref::<NetworkFetchFailureContext>()
        .expect("metadata fetch failure should retain typed request context");
    let context = failure
        .request_context()
        .expect("runtime failure should retain final request context");

    assert_eq!(context.current_url(), &final_url);
    assert_eq!(context.request_method(), "GET");
    assert_eq!(context.request_body(), None);
    assert_eq!(context.redirect_chain().len(), 1);
    assert_eq!(context.redirect_chain()[0].from_url, initial_url);
    assert_eq!(context.redirect_chain()[0].to_url, final_url);
    assert_eq!(context.redirect_chain()[0].status, 302);
    let [initial_exchange, final_exchange] = failure.observation_journal().exchanges() else {
        panic!("expected redirect response and request-only final exchange");
    };
    assert_eq!(
        initial_exchange
            .response()
            .expect("redirect response observation")
            .status(),
        302
    );
    assert!(final_exchange.response().is_none());
    assert_header_present(final_exchange.request().headers(), "Host");

    assert!(client.shutdown().is_clean());
    server.await.expect("redirect reset server should finish");
    Ok(())
}

#[test]
fn network_metadata_preserves_each_redirect_exchange() {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(302, "Found")
            .with_header("Location", "/final")
            .with_header("Cache-Control", "no-store"),
        ScriptedResponse::ok("redirected").with_header("Cache-Control", "no-store"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());

    let observed =
        fetch_raw_with_network_metadata_for_test(&client, Request::get(&server.url()).unwrap())
            .unwrap();
    let journal = observed.observation_journal();

    assert_eq!(journal.exchanges().len(), 2);
    assert_eq!(
        journal.exchanges()[0]
            .response()
            .expect("redirect response")
            .status(),
        302
    );
    assert_eq!(
        journal.exchanges()[1]
            .response()
            .expect("final response")
            .status(),
        200
    );
    assert_header_present(journal.exchanges()[0].request().headers(), "Host");
    assert_header_present(journal.exchanges()[1].request().headers(), "Host");

    assert!(client.shutdown().is_clean());
    server.shutdown();
}

#[test]
fn network_metadata_keeps_raw_304_separate_from_merged_cached_response() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("cached")
            .with_header("Cache-Control", "no-cache")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "no-cache")
            .with_header("ETag", "\"v1\""),
    ]);
    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let first =
        fetch_raw_with_network_metadata_for_test(&client, Request::get(&server.url()).unwrap())
            .unwrap();
    assert_eq!(first.response().status, 200);
    let second =
        fetch_raw_with_network_metadata_for_test(&client, Request::get(&server.url()).unwrap())
            .unwrap();

    assert_eq!(second.response().status, 200);
    assert_eq!(second.response().body_bytes(), b"cached");
    let exchange = second
        .observation_journal()
        .exchanges()
        .last()
        .expect("revalidation exchange");
    assert_eq!(
        header_value(exchange.request().headers(), "If-None-Match"),
        Some("\"v1\"")
    );
    assert_eq!(
        exchange
            .response()
            .expect("raw revalidation response")
            .status(),
        304
    );

    assert!(client.shutdown().is_clean());
    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

fn assert_header_present(headers: &[(String, String)], name: &str) {
    assert!(
        header_value(headers, name).is_some(),
        "missing {name} in observed headers: {headers:?}"
    );
}

fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> {
    headers
        .iter()
        .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name))
        .map(|(_, value)| value.as_str())
}

fn request_head_header_value<'a>(request_head: &'a str, name: &str) -> Option<&'a str> {
    request_head.lines().skip(1).find_map(|line| {
        let (header_name, value) = line.split_once(':')?;
        header_name.eq_ignore_ascii_case(name).then(|| value.trim())
    })
}

fn nonce_from_signature_input(signature_input: &str) -> &str {
    signature_input
        .split(";nonce=\"")
        .nth(1)
        .and_then(|value| value.split('"').next())
        .expect("Signature-Input should contain a nonce parameter")
}

fn assert_request_extra_signature_matches_wire(
    extra_info: &crate::NetworkRequestExtraInfo,
    request_head: &str,
) {
    for name in ["Signature-Agent", "Signature-Input", "Signature"] {
        assert_eq!(
            header_value(&extra_info.headers, name),
            request_head_header_value(request_head, name),
            "network extra info and wire request differ for {name}"
        );
    }
}

#[test]
fn fetch_client_cache_respects_pragma_no_cache_response() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Pragma", "no-cache"),
        ScriptedResponse::ok("hit-2").with_header("Pragma", "no-cache"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_respects_private_response() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "private, max-age=60"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "private, max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_respects_set_cookie_response() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Set-Cookie", "session=one; Path=/"),
        ScriptedResponse::ok("hit-2")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Set-Cookie", "session=two; Path=/"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_response_stores_partitioned_cookie_under_request_top_level_site() -> Result<()> {
    let server = ScriptedHttps11Server::spawn(vec![ScriptedResponse::ok("ok").with_header(
        "Set-Cookie",
        "chip=one; Path=/; Secure; SameSite=None; Partitioned",
    )]);
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let cookie_store = new_shared_browser_cookie_store();
    let client = FetchClient::new(&config, Arc::clone(&cookie_store));
    let request_url = Url::parse(&server.url())?;
    let first_top_level = Url::parse("https://first.example/page")?;
    let second_top_level = Url::parse("https://second.example/page")?;

    let response = fetch_response_for_test(
        &client,
        Request::new("GET", request_url.as_str(), None, Vec::new())?
            .with_initiator_url(&first_top_level)
            .with_request_origin(moli_url::WebOrigin::from_url(&first_top_level)),
    )?;

    assert_eq!(response.status, 200);
    let first_context = NetworkCookieRequestContext::subresource("GET")
        .with_initiator_url(&request_url, &first_top_level);
    let second_context = NetworkCookieRequestContext::subresource("GET")
        .with_initiator_url(&request_url, &second_top_level);
    assert_eq!(
        crate::cookie_header_for_request(&cookie_store, &request_url, first_context)?,
        Some("chip=one".to_owned())
    );
    assert_eq!(
        crate::cookie_header_for_request(&cookie_store, &request_url, second_context)?,
        None
    );

    server.shutdown();
    Ok(())
}

#[test]
fn fetch_client_skips_disk_cache_for_non_success_status() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(404, "Not Found").with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::status(404, "Not Found").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.status, 404);
    assert_eq!(second.status, 404);
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_caches_redirect_final_response_under_final_url() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(302, "Found")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Location", "/final"),
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first =
        fetch_with_config_for_test(&config, Request::get(&server.url_path("/start")).unwrap())
            .unwrap();
    let second =
        fetch_with_config_for_test(&config, Request::get(&server.url_path("/final")).unwrap())
            .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert!(first.redirected);
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);
    assert!(
        second.from_cache,
        "follow-up final URL request should reuse the cached final response"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_respects_max_age_expiry() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Cache-Control", "max-age=1"),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=1"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    std::thread::sleep(Duration::from_millis(1_100));
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_age_header_can_make_max_age_stale() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Age", "120")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);
    assert!(
        server.requests()[1]
            .to_ascii_lowercase()
            .contains("if-none-match: \"v1\""),
        "Age should make the cached entry stale enough to validate: {}",
        server.requests()[1]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_date_header_can_make_max_age_stale() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Date", "Wed, 21 Oct 2015 07:28:00 GMT")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);
    assert!(
        server.requests()[1]
            .to_ascii_lowercase()
            .contains("if-none-match: \"v1\""),
        "old Date should make the cached entry stale enough to validate: {}",
        server.requests()[1]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_expired_expires_header_revalidates() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Expires", "Wed, 21 Oct 2015 07:28:00 GMT")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Expires", "Wed, 07 Sep 2033 21:46:42 GMT"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let third = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(third.body_text(), "hit-1");
    assert_eq!(
        server.hits(),
        2,
        "304 Expires update should make the third request a fresh cache hit"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_revalidates_with_etag() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);
    assert!(
        server.requests()[1]
            .to_ascii_lowercase()
            .contains("if-none-match: \"v1\""),
        "unexpected request headers: {}",
        server.requests()[1]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_updates_freshness_after_not_modified() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::status(200, "Original cached message")
            .with_body("hit-1")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::ok("hit-3").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let third = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(third.body_text(), "hit-1");
    for response in [&first, &second, &third] {
        assert_eq!(response.status, 200);
        assert_eq!(
            response.status_text.as_deref(),
            Some("Original cached message")
        );
    }
    assert_eq!(
        server.hits(),
        2,
        "304 metadata update should make the next cache lookup fresh"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_revalidates_same_document_reload() {
    // Mirrors Chromium net/http/http_cache_unittest.cc LoadValidateCacheImplicit:
    // a request-level max-age=0 directive must validate an otherwise fresh hit.
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\""),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let reload_url = Url::parse(&server.url()).unwrap();

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_browser_navigation_kind(BrowserNavigationRequestKind::Reload)
            .with_initiator_url(&reload_url)
            .with_request_origin(moli_url::WebOrigin::from_url(&reload_url)),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);
    let reload_request = server.requests()[1].to_ascii_lowercase();
    assert!(
        reload_request.contains("cache-control: max-age=0"),
        "reload should synthesize Cache-Control validation: {}",
        server.requests()[1]
    );
    assert!(
        reload_request.contains("if-none-match: \"v1\""),
        "reload should revalidate a fresh cached entry instead of serving it directly: {}",
        server.requests()[1]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_immutable_ignores_same_document_reload_validation_when_fresh() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60, immutable")
            .with_header("ETag", "\"v1\""),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let reload_url = Url::parse(&server.url()).unwrap();

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_browser_navigation_kind(BrowserNavigationRequestKind::Reload)
            .with_initiator_url(&reload_url)
            .with_request_origin(moli_url::WebOrigin::from_url(&reload_url)),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(
        server.hits(),
        1,
        "fresh immutable cache hits should satisfy reload without revalidation"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_immutable_revalidates_after_expiry() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=0, immutable")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::ok("hit-2").with_header("Cache-Control", "max-age=60, immutable"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let reload_url = Url::parse(&server.url()).unwrap();

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(
        &config,
        Request::get(&server.url())
            .unwrap()
            .with_browser_navigation_kind(BrowserNavigationRequestKind::Reload)
            .with_initiator_url(&reload_url)
            .with_request_origin(moli_url::WebOrigin::from_url(&reload_url)),
    )
    .unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(
        server.hits(),
        2,
        "immutable must not allow stale entries to bypass network validation"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_removes_entry_when_304_returns_no_store() {
    // Mirrors Chromium CacheControlNoStore3 / ConditionalRequest304NoStore:
    // a 304 that says no-store invalidates the previously stored entry.
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified").with_header("Cache-Control", "no-store"),
        ScriptedResponse::ok("hit-3").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let third = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(third.body_text(), "hit-3");
    assert_eq!(server.hits(), 3);
    assert!(
        !server.requests()[2]
            .to_ascii_lowercase()
            .contains("if-none-match"),
        "304 no-store should evict the old cache entry before the next request: {}",
        server.requests()[2]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_304_updates_vary_header_metadata() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"v1\"")
            .with_header("Vary", "User-Agent"),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\"")
            .with_header("Vary", "Accept-Language"),
        ScriptedResponse::ok("hit-3").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut first_config = FetchConfig::default();
    first_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    first_config.set_user_agent("Moli/Test-UA-1");
    let mut second_config = FetchConfig::default();
    second_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    second_config.set_user_agent("Moli/Test-UA-2");

    let first =
        fetch_with_config_for_test(&first_config, Request::get(&server.url()).unwrap()).unwrap();
    let second =
        fetch_with_config_for_test(&first_config, Request::get(&server.url()).unwrap()).unwrap();
    let third =
        fetch_with_config_for_test(&second_config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(third.body_text(), "hit-1");
    assert_eq!(
        server.hits(),
        2,
        "304 Vary update should replace the cached Vary key for later lookups"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_304_without_vary_preserves_existing_vary_metadata() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"v1\"")
            .with_header("Vary", "User-Agent"),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::ok("hit-3").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut first_config = FetchConfig::default();
    first_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    first_config.set_user_agent("Moli/Test-UA-1");
    let mut second_config = FetchConfig::default();
    second_config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    second_config.set_user_agent("Moli/Test-UA-2");

    let first =
        fetch_with_config_for_test(&first_config, Request::get(&server.url()).unwrap()).unwrap();
    let second =
        fetch_with_config_for_test(&first_config, Request::get(&server.url()).unwrap()).unwrap();
    let third =
        fetch_with_config_for_test(&second_config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(third.body_text(), "hit-3");
    assert_eq!(
        server.hits(),
        3,
        "304 without Vary should not delete the original cached Vary key"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_preserves_content_length_when_merging_304() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"v1\""),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\""),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(
        second
            .headers
            .iter()
            .find(|(name, _)| name.eq_ignore_ascii_case("content-length"))
            .map(|(_, value)| value.as_slice()),
        Some(b"5".as_slice()),
        "304 Content-Length must not replace the cached 200 Content-Length"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_skips_connection_nominated_headers_when_merging_304() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=0")
            .with_header("ETag", "\"v1\"")
            .with_header("X-Transient", "cached-value"),
        ScriptedResponse::status(304, "Not Modified")
            .with_header("Cache-Control", "max-age=60")
            .with_header("ETag", "\"v1\"")
            .with_header("Connection", "X-Transient")
            .with_header("X-Transient", "not-modified-value"),
        ScriptedResponse::ok("hit-3").with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let third = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(third.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);
    assert_eq!(
        third
            .headers
            .iter()
            .find(|(name, _)| name.eq_ignore_ascii_case("x-transient"))
            .map(|(_, value)| value.as_slice()),
        Some(b"cached-value".as_slice()),
        "304 Connection-nominated fields must not replace cached metadata"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_uses_expires_freshness_without_max_age() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_header("Expires", "Wed, 07 Sep 2033 21:46:42 GMT"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_cache_skips_unknown_vary_header() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Foo"),
        ScriptedResponse::ok("hit-2")
            .with_header("Cache-Control", "max-age=60")
            .with_header("Vary", "Foo"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_prunes_disk_cache_to_configured_quota() {
    let cache_dir = unique_test_cache_dir();
    let large_a = "a".repeat(1024);
    let large_b = "b".repeat(1024);
    let large_c = "c".repeat(1024);
    let large_d = "d".repeat(1024);
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok(&large_a).with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok(&large_b).with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok(&large_c).with_header("Cache-Control", "max-age=60"),
        ScriptedResponse::ok(&large_d).with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    config.set_http_cache_max_bytes(Some(3_000));

    let first =
        fetch_with_config_for_test(&config, Request::get(&server.url_path("/one")).unwrap())
            .unwrap();
    let second =
        fetch_with_config_for_test(&config, Request::get(&server.url_path("/two")).unwrap())
            .unwrap();
    let third =
        fetch_with_config_for_test(&config, Request::get(&server.url_path("/three")).unwrap())
            .unwrap();
    let first_again =
        fetch_with_config_for_test(&config, Request::get(&server.url_path("/one")).unwrap())
            .unwrap();

    assert_eq!(first.body_text(), large_a);
    assert_eq!(second.body_text(), large_b);
    assert_eq!(third.body_text(), large_c);
    assert_eq!(first_again.body_text(), large_d);
    assert_eq!(
        server.hits(),
        4,
        "small quota should evict the oldest cached entry"
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn http_cache_stats_reports_profile_cache_growth() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let store = HttpCacheStore::new(&cache_dir);
    let readable_url = "http://example.test/readable";
    let orphan_url = "http://example.test/orphan";
    store_test_cache_body(
        &store,
        &HttpCacheStore::key_for_url(readable_url),
        HttpCacheEntryMetadata::new(
            readable_url.to_owned(),
            readable_url.to_owned(),
            200,
            Vec::new(),
            1,
            Some(2),
            Vec::new(),
        ),
        b"readable",
    )?;
    let orphan_dir = cache_dir.join(format!("{}.entry", HttpCacheStore::key_for_url(orphan_url)));
    fs::create_dir_all(&orphan_dir)?;
    fs::write(orphan_dir.join("body.orphan.bin"), b"orphaned")?;

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let stats = http_cache_stats(&config)?;

    assert_eq!(stats.entry_count, 1);
    assert_eq!(stats.unreadable_entry_count, 1);
    assert_eq!(stats.readable_body_bytes, 8);
    assert!(stats.total_bytes >= 16);

    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test]
async fn fetch_html_stream_treats_disk_cache_store_failure_as_best_effort() -> Result<()> {
    let cache_file = unique_test_cache_dir();
    fs::write(&cache_file, b"not a directory").unwrap();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("<!doctype html><html><body>hit-1</body></html>")
            .with_header("Content-Type", "text/html; charset=utf-8")
            .with_header("Cache-Control", "max-age=60"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_file.display().to_string()));

    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut response = client
        .fetch_html_stream(Request::get(&server.url()).unwrap())
        .await?;
    let mut body = String::new();
    while let Some(chunk) = response.next_chunk().await {
        body.push_str(&chunk);
    }
    response.finish().await?;

    assert_eq!(body, "<!doctype html><html><body>hit-1</body></html>");
    assert_eq!(server.hits(), 1);

    server.shutdown();
    let _ = std::fs::remove_file(cache_file);
    Ok(())
}

#[test]
fn fetch_client_cache_revalidates_with_last_modified() {
    let cache_dir = unique_test_cache_dir();
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1")
            .with_header("Cache-Control", "max-age=0")
            .with_header("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"),
        ScriptedResponse::status(304, "Not Modified"),
    ]);

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));

    let first = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_with_config_for_test(&config, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-1");
    assert_eq!(server.hits(), 2);
    assert!(
        server.requests()[1]
            .to_ascii_lowercase()
            .contains("if-modified-since: wed, 21 oct 2015 07:28:00 gmt"),
        "unexpected request headers: {}",
        server.requests()[1]
    );

    server.shutdown();
    let _ = std::fs::remove_dir_all(cache_dir);
}

#[test]
fn fetch_client_reuses_existing_runtime_owner_for_sequential_requests() {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1"),
        ScriptedResponse::ok("hit-2"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());

    let first = fetch_response_for_test(&client, Request::get(&server.url()).unwrap()).unwrap();
    let second = fetch_response_for_test(&client, Request::get(&server.url()).unwrap()).unwrap();

    assert_eq!(first.body_text(), "hit-1");
    assert_eq!(second.body_text(), "hit-2");
    assert_eq!(server.hits(), 2);
    assert_eq!(client.runtime_owner_count_for_testing(), 1);

    server.shutdown();
}

#[tokio::test]
async fn fetch_with_cancel_aborts_inflight_streaming_transfer() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let (disconnect_tx, disconnect_rx) = oneshot::channel();
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.expect("cancel test accept");
        let _request = read_http_request_head(&mut stream)
            .await
            .expect("cancel test should read request");
        stream
            .write_all(
                b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\nhello",
            )
            .await
            .expect("cancel test should write initial response bytes");
        tokio::time::sleep(Duration::from_millis(150)).await;
        let mut tail = [0u8; 1];
        let disconnected = matches!(
            tokio::time::timeout(Duration::from_secs(2), stream.read(&mut tail)).await,
            Ok(Ok(0))
        );
        let _ = disconnect_tx.send(disconnected);
    });

    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let cancel_handle = FetchCancelHandle::new();
    let request = Request::get(&format!("http://{addr}/slow"))?;
    let fetch_task = {
        let client = client.clone();
        let cancel_handle = cancel_handle.clone();
        tokio::spawn(async move { client.fetch_with_cancel(request, cancel_handle).await })
    };

    tokio::time::sleep(Duration::from_millis(20)).await;
    cancel_handle.cancel();

    let error = fetch_task
        .await
        .expect("fetch task should join")
        .unwrap_err();
    let error_chain = format!("{error:#}");
    assert!(
        error_chain.contains("fetch runtime request cancelled")
            || error_chain.contains("Callback aborted"),
        "unexpected cancel error: {error_chain}"
    );

    let disconnected = tokio::time::timeout(Duration::from_secs(3), disconnect_rx)
        .await
        .expect("server disconnect observation timed out")
        .expect("server disconnect observation channel closed");
    assert!(
        disconnected,
        "expected client transport to close after fetch cancellation"
    );

    server.await.expect("cancel test server should finish");
    Ok(())
}

#[tokio::test]
async fn fetch_raw_stream_with_cancel_aborts_inflight_raw_transfer() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let (disconnect_tx, disconnect_rx) = oneshot::channel();
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.expect("raw cancel test accept");
        let _request = read_http_request_head(&mut stream)
            .await
            .expect("raw cancel test should read request");
        stream
            .write_all(
                b"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nConnection: close\r\n\r\nhello",
            )
            .await
            .expect("raw cancel test should write initial response bytes");
        tokio::time::sleep(Duration::from_millis(150)).await;
        let mut tail = [0u8; 1];
        let disconnected = matches!(
            tokio::time::timeout(Duration::from_secs(2), stream.read(&mut tail)).await,
            Ok(Ok(0))
        );
        let _ = disconnect_tx.send(disconnected);
    });

    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let cancel_handle = FetchCancelHandle::new();
    let request = Request::get(&format!("http://{addr}/slow-raw"))?;
    let mut response = client
        .fetch_raw_stream_with_cancel(request, cancel_handle.clone())
        .await?;
    assert_eq!(response.status, 200);

    tokio::time::sleep(Duration::from_millis(20)).await;
    cancel_handle.cancel();

    let error = response.finish().await.unwrap_err();
    let error_chain = format!("{error:#}");
    assert!(
        error_chain.contains("fetch runtime request cancelled")
            || error_chain.contains("Callback aborted"),
        "unexpected cancel error: {error_chain}"
    );

    let disconnected = tokio::time::timeout(Duration::from_secs(3), disconnect_rx)
        .await
        .expect("raw server disconnect observation timed out")
        .expect("raw server disconnect observation channel closed");
    assert!(
        disconnected,
        "expected client transport to close after raw fetch cancellation"
    );

    server.await.expect("raw cancel test server should finish");
    Ok(())
}

#[test]
fn fetch_client_runs_parallel_requests_on_one_runtime_owner() {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_delay_ms(250),
        ScriptedResponse::ok("hit-2").with_delay_ms(250),
    ]);
    let mut config = FetchConfig::default();
    config.set_connection_limits(NonZeroU32::new(2), None, None);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let start = Arc::new(std::sync::Barrier::new(3));

    let left_client = client.clone();
    let left_start = Arc::clone(&start);
    let left_url = server.url();
    let left = thread::spawn(move || {
        left_start.wait();
        fetch_response_for_test(&left_client, Request::get(&left_url).unwrap())
    });

    let right_client = client.clone();
    let right_start = Arc::clone(&start);
    let right_url = server.url();
    let right = thread::spawn(move || {
        right_start.wait();
        fetch_response_for_test(&right_client, Request::get(&right_url).unwrap())
    });

    start.wait();
    wait_for_runtime_owner_count(&client, 1);
    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline && server.hits() < 2 {
        thread::sleep(Duration::from_millis(10));
    }
    assert_eq!(
        server.hits(),
        2,
        "multi owner should start both transfers concurrently"
    );

    let mut bodies = [
        left.join().unwrap().unwrap().body_text().to_owned(),
        right.join().unwrap().unwrap().body_text().to_owned(),
    ];
    bodies.sort();
    assert_eq!(bodies, ["hit-1".to_owned(), "hit-2".to_owned()]);

    server.shutdown();
}

#[test]
fn fetch_runtime_wakes_active_owner_for_new_parallel_request() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("slow").with_delay_ms(300),
        ScriptedResponse::ok("fast"),
    ]);
    let mut config = FetchConfig::default();
    config.set_connection_limits(NonZeroU32::new(2), None, None);
    let runtime = FetchRuntimeOwner::new(&config, new_shared_browser_cookie_store());

    let slow_rx = runtime.submit(Request::get(&server.url_path("/slow"))?)?;
    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline && server.hits() < 1 {
        thread::sleep(Duration::from_millis(10));
    }
    assert_eq!(server.hits(), 1, "slow transfer should be active");

    let fast_rx = runtime.submit(Request::get(&server.url_path("/fast"))?)?;
    let deadline = Instant::now() + Duration::from_millis(150);
    while Instant::now() < deadline && server.hits() < 2 {
        thread::sleep(Duration::from_millis(10));
    }
    assert_eq!(
        server.hits(),
        2,
        "active curl wait should wake and start newly submitted transfer"
    );

    assert_eq!(fast_rx.blocking_recv()??.body_text(), "fast");
    assert_eq!(slow_rx.blocking_recv()??.body_text(), "slow");

    server.shutdown();
    Ok(())
}

#[test]
fn fetch_client_respects_host_transfer_cap_for_parallel_same_host_requests() {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("hit-1").with_delay_ms(250),
        ScriptedResponse::ok("hit-2").with_delay_ms(250),
    ]);
    let mut config = FetchConfig::default();
    config.set_connection_limits(NonZeroU32::new(2), NonZeroU32::new(1), None);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let start = Arc::new(std::sync::Barrier::new(3));

    let left_client = client.clone();
    let left_start = Arc::clone(&start);
    let left_url = server.url();
    let left = thread::spawn(move || {
        left_start.wait();
        fetch_response_for_test(&left_client, Request::get(&left_url).unwrap())
    });

    let right_client = client.clone();
    let right_start = Arc::clone(&start);
    let right_url = server.url();
    let right = thread::spawn(move || {
        right_start.wait();
        fetch_response_for_test(&right_client, Request::get(&right_url).unwrap())
    });

    start.wait();
    wait_for_runtime_owner_count(&client, 1);
    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline && server.hits() < 1 {
        thread::sleep(Duration::from_millis(10));
    }
    assert_eq!(server.hits(), 1, "first same-host transfer should start");
    thread::sleep(Duration::from_millis(100));
    assert_eq!(
        server.hits(),
        1,
        "host cap should keep the second same-host transfer pending"
    );

    let mut bodies = [
        left.join().unwrap().unwrap().body_text().to_owned(),
        right.join().unwrap().unwrap().body_text().to_owned(),
    ];
    bodies.sort();
    assert_eq!(bodies, ["hit-1".to_owned(), "hit-2".to_owned()]);

    server.shutdown();
}

#[test]
fn fetch_runtime_starts_queued_high_priority_request_before_low_priority_request() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("first").with_delay_ms(250),
        ScriptedResponse::ok("high"),
        ScriptedResponse::ok("low"),
    ]);
    let mut config = FetchConfig::default();
    config.set_connection_limits(NonZeroU32::new(1), None, None);
    let runtime = FetchRuntimeOwner::new(&config, new_shared_browser_cookie_store());

    let first_rx = runtime.submit(Request::get(&server.url_path("/first"))?)?;
    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline && server.hits() < 1 {
        thread::sleep(Duration::from_millis(10));
    }
    assert_eq!(server.hits(), 1, "first transfer should be active");

    let low_request = Request::new("GET", &server.url_path("/low"), None, Vec::new())?
        .with_script_fetch_metadata(ScriptFetchRequestMetadata {
            fetch_priority: Some(FetchPriorityHint::Low),
            ..ScriptFetchRequestMetadata::default()
        });
    let high_request = Request::new("GET", &server.url_path("/high"), None, Vec::new())?
        .with_script_fetch_metadata(ScriptFetchRequestMetadata {
            fetch_priority: Some(FetchPriorityHint::High),
            ..ScriptFetchRequestMetadata::default()
        });
    let low_rx = runtime.submit(low_request)?;
    let high_rx = runtime.submit(high_request)?;

    let first = first_rx
        .blocking_recv()
        .expect("first response channel should complete")?;
    let high = high_rx
        .blocking_recv()
        .expect("high response channel should complete")?;
    let low = low_rx
        .blocking_recv()
        .expect("low response channel should complete")?;

    assert_eq!(first.body_text(), "first");
    assert_eq!(high.body_text(), "high");
    assert_eq!(low.body_text(), "low");
    let requests = server.requests();
    let request_paths = requests
        .iter()
        .map(|request| request_path(request))
        .collect::<Vec<_>>();
    assert_eq!(request_paths, ["/first", "/high", "/low"]);

    server.shutdown();
    Ok(())
}

#[test]
fn fetch_runtime_keeps_browser_fetch_and_auto_script_at_same_default_priority() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("first").with_delay_ms(250),
        ScriptedResponse::ok("second"),
        ScriptedResponse::ok("third"),
    ]);
    let mut config = FetchConfig::default();
    config.set_connection_limits(NonZeroU32::new(1), None, None);
    let runtime = FetchRuntimeOwner::new(&config, new_shared_browser_cookie_store());

    let first_rx = runtime.submit(Request::get(&server.url_path("/first"))?)?;
    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline && server.hits() < 1 {
        thread::sleep(Duration::from_millis(10));
    }
    assert_eq!(server.hits(), 1, "first transfer should be active");

    let script_request = Request::new("GET", &server.url_path("/script"), None, Vec::new())?
        .with_script_fetch_metadata(ScriptFetchRequestMetadata::default());
    let fetch_request = Request::new("GET", &server.url_path("/fetch"), None, Vec::new())?
        .with_browser_request_metadata(BrowserRequestMetadata::Fetch);
    let script_rx = runtime.submit(script_request)?;
    let fetch_rx = runtime.submit(fetch_request)?;

    let first = first_rx
        .blocking_recv()
        .expect("first response channel should complete")?;
    let fetch = fetch_rx
        .blocking_recv()
        .expect("fetch response channel should complete")?;
    let script = script_rx
        .blocking_recv()
        .expect("script response channel should complete")?;

    assert_eq!(first.body_text(), "first");
    assert_eq!(script.body_text(), "second");
    assert_eq!(fetch.body_text(), "third");
    let requests = server.requests();
    let request_paths = requests
        .iter()
        .map(|request| request_path(request))
        .collect::<Vec<_>>();
    assert_eq!(request_paths, ["/first", "/script", "/fetch"]);

    server.shutdown();
    Ok(())
}

#[test]
fn fetch_runtime_negotiates_http2_over_tls() {
    let server = ScriptedH2Server::spawn(vec![ScriptedResponse::ok("h2-ok")]);
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let response =
        fetch_response_for_test(&client, Request::get(&server.url_path("/h2")).unwrap()).unwrap();

    assert_eq!(response.body_text(), "h2-ok");
    assert_eq!(response.status_text.as_deref(), Some(""));
    assert_eq!(response.head().status_text(), "");
    assert_eq!(
        response.negotiated_http_version,
        Some(NegotiatedHttpVersion::Http2)
    );
    assert_eq!(server.hits(), 1);
    assert_eq!(server.requests(), ["/h2".to_owned()]);

    server.shutdown();
}

#[test]
fn fetch_runtime_reports_http10_from_wire_response() {
    let server = ScriptedHttpServer::spawn(vec![
        ScriptedResponse::ok("http10-ok").with_http_version("1.0"),
    ]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());

    let response =
        fetch_response_for_test(&client, Request::get(&server.url_path("/http10")).unwrap())
            .unwrap();

    assert_eq!(response.body_text(), "http10-ok");
    assert_eq!(
        response.negotiated_http_version,
        Some(NegotiatedHttpVersion::Http10)
    );
    server.shutdown();
}

#[tokio::test]
async fn fetch_runtime_raw_stream_reports_negotiated_http2() -> Result<()> {
    let server = ScriptedH2Server::spawn(vec![ScriptedResponse::ok("h2-stream")]);
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let mut response = client
        .fetch_raw_stream_with_cancel(
            Request::get(&server.url_path("/h2-stream"))?,
            FetchCancelHandle::new(),
        )
        .await?;
    assert_eq!(response.status_text.as_deref(), Some(""));
    assert_eq!(response.head().status_text(), "");
    assert_eq!(
        response.negotiated_http_version,
        Some(NegotiatedHttpVersion::Http2)
    );
    let mut body = Vec::new();
    while let Some(chunk) = response.next_chunk().await {
        body.extend_from_slice(&chunk);
    }
    response.finish().await?;

    assert_eq!(body, b"h2-stream");
    server.shutdown();
    Ok(())
}

#[test]
fn fetch_runtime_retries_streaming_html_get_over_http11_after_http2_protocol_error() -> Result<()> {
    let server = Http2ProtocolFallbackServer::spawn();
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let body = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()?
        .block_on(async {
            let mut response = client
                .fetch_html_stream(Request::get(&server.url())?)
                .await?;
            let mut body = String::new();
            while let Some(chunk) = response.next_chunk().await {
                body.push_str(&chunk);
            }
            response.finish().await?;
            Ok::<_, anyhow::Error>(body)
        })?;

    assert_eq!(body, "http1 fallback");
    assert_eq!(server.h2_hits(), 1);
    assert_eq!(server.http11_hits(), 1);

    server.shutdown();
    Ok(())
}

#[test]
fn fetch_runtime_retries_raw_streaming_get_over_http11_after_http2_protocol_error() -> Result<()> {
    let server = Http2ProtocolFallbackServer::spawn();
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());

    let response = fetch_response_for_test(&client, Request::get(&server.url())?)?;

    assert_eq!(response.body_text(), "http1 fallback");
    assert_eq!(server.h2_hits(), 1);
    assert_eq!(server.http11_hits(), 1);

    server.shutdown();
    Ok(())
}

#[test]
fn fetch_runtime_retries_buffered_get_over_http11_after_http2_protocol_error() -> Result<()> {
    let server = Http2ProtocolFallbackServer::spawn();
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let request = Request::get(&server.url())?.with_single_hop();

    let response = fetch_response_for_test(&client, request)?;

    assert_eq!(response.body_text(), "http1 fallback");
    assert_eq!(server.h2_hits(), 1);
    assert_eq!(server.http11_hits(), 1);

    server.shutdown();
    Ok(())
}

#[test]
fn fetch_runtime_does_not_replay_post_after_http2_protocol_error() -> Result<()> {
    let server = Http2ProtocolFallbackServer::spawn();
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let request = Request::new("POST", &server.url(), Some("body".to_owned()), Vec::new())?;

    fetch_response_for_test(&client, request)
        .expect_err("POST should not be replayed after an HTTP/2 protocol error");
    assert_eq!(server.h2_hits(), 1);
    assert_eq!(server.http11_hits(), 0);

    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn fetch_runtime_retries_streaming_html_empty_http_navigation_over_https() -> Result<()> {
    let server = EmptyHttpHttpsUpgradeServer::spawn();
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    config.set_http_host_resolve(vec![server.resolve_entry()]);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut response = client
        .fetch_html_stream(Request::get(&server.url())?)
        .await?;
    let mut body = String::new();
    while let Some(chunk) = response.next_chunk().await {
        body.push_str(&chunk);
    }
    response.finish().await?;

    assert_eq!(body, "https upgrade fallback");
    assert_eq!(response.final_url.scheme(), "https");
    assert!(response.redirected);
    assert_https_upgrade_redirect(&response.redirect_chain);
    assert_eq!(server.http_hits(), 1);
    assert_eq!(server.https_hits(), 1);

    server.shutdown();
    Ok(())
}

#[tokio::test]
async fn fetch_runtime_retries_raw_streaming_empty_http_navigation_over_https() -> Result<()> {
    let server = EmptyHttpHttpsUpgradeServer::spawn();
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    config.set_http_host_resolve(vec![server.resolve_entry()]);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let mut response = client
        .fetch_raw_stream_with_cancel(Request::get(&server.url())?, FetchCancelHandle::new())
        .await?;
    let mut body = Vec::new();
    while let Some(chunk) = response.next_chunk().await {
        body.extend_from_slice(&chunk);
    }
    response.finish().await?;

    assert_eq!(body, b"https upgrade fallback");
    assert_eq!(response.final_url.scheme(), "https");
    assert!(response.redirected);
    assert_https_upgrade_redirect(&response.redirect_chain);
    assert_eq!(server.http_hits(), 1);
    assert_eq!(server.https_hits(), 1);

    server.shutdown();
    Ok(())
}

#[test]
fn fetch_runtime_retries_buffered_empty_http_navigation_over_https() -> Result<()> {
    let server = EmptyHttpHttpsUpgradeServer::spawn();
    let mut config = FetchConfig::default();
    config.set_tls_verify_host(false);
    config.set_http_host_resolve(vec![server.resolve_entry()]);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let request = Request::get(&server.url())?.with_single_hop();

    let response = fetch_response_for_test(&client, request)?;

    assert_eq!(response.body_text(), "https upgrade fallback");
    assert_eq!(response.final_url.scheme(), "https");
    assert!(response.redirected);
    assert_https_upgrade_redirect(&response.redirect_chain);
    assert_eq!(server.http_hits(), 1);
    assert_eq!(server.https_hits(), 1);

    server.shutdown();
    Ok(())
}

fn assert_https_upgrade_redirect(redirect_chain: &[crate::RedirectInfo]) {
    assert_eq!(redirect_chain.len(), 1);
    let redirect = &redirect_chain[0];
    assert_eq!(redirect.status, 307);
    assert_eq!(redirect.from_url.scheme(), "http");
    assert_eq!(redirect.to_url.scheme(), "https");
    assert!(
        redirect.headers.iter().any(|(name, value)| {
            name == "non-authoritative-reason" && value == b"HttpsUpgrades"
        })
    );
}

#[test]
fn fetch_runtime_starts_concurrent_http2_requests_without_waiting_for_reuse() {
    let server = ScriptedH2Server::spawn(vec![
        ScriptedResponse::ok("h2-a").with_delay_ms(250),
        ScriptedResponse::ok("h2-b").with_delay_ms(250),
    ]);
    let mut config = FetchConfig::default();
    config.set_connection_limits(NonZeroU32::new(2), None, None);
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let start = Arc::new(std::sync::Barrier::new(3));

    let left_client = client.clone();
    let left_start = Arc::clone(&start);
    let left_url = server.url_path("/h2-a");
    let left = thread::spawn(move || {
        left_start.wait();
        fetch_response_for_test(&left_client, Request::get(&left_url).unwrap())
    });

    let right_client = client.clone();
    let right_start = Arc::clone(&start);
    let right_url = server.url_path("/h2-b");
    let right = thread::spawn(move || {
        right_start.wait();
        fetch_response_for_test(&right_client, Request::get(&right_url).unwrap())
    });

    start.wait();
    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline && server.hits() < 2 {
        thread::sleep(Duration::from_millis(10));
    }

    assert_eq!(
        server.hits(),
        2,
        "concurrent H2 requests should start without waiting for a reusable TLS connection"
    );
    let mut bodies = [
        left.join().unwrap().unwrap().body_text().to_owned(),
        right.join().unwrap().unwrap().body_text().to_owned(),
    ];
    bodies.sort();
    assert_eq!(bodies, ["h2-a".to_owned(), "h2-b".to_owned()]);
    let mut requests = server.requests();
    requests.sort();
    assert_eq!(requests, ["/h2-a".to_owned(), "/h2-b".to_owned()]);
    assert_eq!(
        server.connection_stream_counts().iter().sum::<usize>(),
        2,
        "server should observe both requests as H2 streams"
    );

    server.shutdown();
}

#[test]
fn fetch_runtime_starts_streaming_html_and_fetch_without_waiting_for_h2_reuse() {
    let server = ScriptedH2Server::spawn(vec![
        ScriptedResponse::ok("<!doctype html><p>stream</p>").with_delay_ms(250),
        ScriptedResponse::ok("h2-fetch").with_delay_ms(250),
    ]);
    let mut config = FetchConfig::default();
    config.set_connection_limits(NonZeroU32::new(2), None, None);
    config.set_tls_verify_host(false);
    let client = FetchClient::new(&config, new_shared_browser_cookie_store());
    let start = Arc::new(std::sync::Barrier::new(3));

    let stream_client = client.clone();
    let stream_start = Arc::clone(&start);
    let stream_url = server.url_path("/stream");
    let stream = thread::spawn(move || -> Result<String> {
        stream_start.wait();
        tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()?
            .block_on(async move {
                let mut response = stream_client
                    .fetch_html_stream(Request::get(&stream_url).unwrap())
                    .await?;
                let mut body = String::new();
                while let Some(chunk) = response.next_chunk().await {
                    body.push_str(&chunk);
                }
                response.finish().await?;
                Ok(body)
            })
    });

    let fetch_client = client.clone();
    let fetch_start = Arc::clone(&start);
    let fetch_url = server.url_path("/fetch");
    let fetch = thread::spawn(move || {
        fetch_start.wait();
        fetch_response_for_test(&fetch_client, Request::get(&fetch_url).unwrap())
            .map(|response| response.body_text().to_owned())
    });

    start.wait();
    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline && server.hits() < 2 {
        thread::sleep(Duration::from_millis(10));
    }

    assert_eq!(
        server.hits(),
        2,
        "streaming HTML and ordinary fetch should start without waiting for a reusable TLS connection"
    );
    let mut bodies = [
        stream.join().unwrap().unwrap(),
        fetch.join().unwrap().unwrap(),
    ];
    bodies.sort();
    assert_eq!(
        bodies,
        [
            "<!doctype html><p>stream</p>".to_owned(),
            "h2-fetch".to_owned()
        ]
    );
    let mut requests = server.requests();
    requests.sort();
    assert_eq!(requests, ["/fetch".to_owned(), "/stream".to_owned()]);
    assert_eq!(
        server.connection_stream_counts().iter().sum::<usize>(),
        2,
        "server should observe both requests as H2 streams"
    );

    server.shutdown();
}

#[test]
fn fetch_client_recovers_after_runtime_owner_panic() -> Result<()> {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("hit-1")]);
    let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let panic_request = Request::new(
        "GET",
        &server.url(),
        None,
        vec![("x-moli-test-panic".to_owned(), "runtime-worker".to_owned())],
    )?;

    let panic_error = fetch_response_for_test(&client, panic_request)
        .unwrap_err()
        .to_string();
    assert!(
        panic_error.contains("fetch runtime owner panicked while handling GET"),
        "unexpected panic error: {panic_error}"
    );

    let follow_up_client = client.clone();
    let follow_up_url = server.url();
    let (result_tx, result_rx) = std_mpsc::channel();
    let follow_up = thread::spawn(move || {
        let result =
            fetch_response_for_test(&follow_up_client, Request::get(&follow_up_url).unwrap());
        let _ = result_tx.send(result);
    });
    let response = result_rx
        .recv_timeout(Duration::from_secs(1))
        .expect("follow-up request should not hang")?;
    assert_eq!(response.body_text(), "hit-1");
    follow_up
        .join()
        .expect("follow-up request should join cleanly");
    assert_eq!(client.runtime_owner_count_for_testing(), 1);

    server.shutdown();
    Ok(())
}

fn request_path(request: &str) -> &str {
    request
        .lines()
        .next()
        .and_then(|line| line.split_whitespace().nth(1))
        .expect("scripted test request should have a request target")
}

#[test]
fn curl_build_reports_brotli_and_http2_support() {
    let version = curl::Version::get();

    assert!(
        version.feature_http2(),
        "libcurl is missing HTTP/2 support: {:?}",
        version
    );
    assert!(
        version.feature_brotli(),
        "libcurl is missing Brotli support: {:?}",
        version
    );
    assert!(
        version.brotli_version().is_some(),
        "libcurl reports Brotli feature but no Brotli version: {:?}",
        version
    );
}

#[test]
fn dropping_fetch_runtime_cancels_inflight_requests_without_waiting_for_request_timeout() {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("slow").with_delay_ms(2_000)]);
    let mut config = FetchConfig::default();
    config.set_request_timeout_ms(10_000);
    let runtime = FetchRuntimeOwner::new(&config, new_shared_browser_cookie_store());
    let response_rx = runtime
        .submit(Request::get(&server.url()).unwrap())
        .unwrap();

    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline {
        if runtime.owner_count_for_testing() == 1 && server.hits() >= 1 {
            break;
        }
        thread::sleep(Duration::from_millis(10));
    }

    assert_eq!(runtime.owner_count_for_testing(), 1);
    assert!(
        server.hits() >= 1,
        "fetch runtime owner never started the in-flight request"
    );

    // Dropping the runtime while a request is still running should signal curl
    // cancellation and then join the owner, rather than waiting for network
    // completion or the configured request timeout.
    let drop_started = Instant::now();
    drop(runtime);
    assert!(
        drop_started.elapsed() < Duration::from_secs(1),
        "runtime drop waited too long for in-flight request cancellation"
    );

    let error = response_rx
        .blocking_recv()
        .expect("in-flight request should still respond after shutdown")
        .expect_err("in-flight request should be cancelled during shutdown");
    assert!(
        error.to_string().contains("cancelled during shutdown"),
        "unexpected shutdown error: {error:#}"
    );

    server.shutdown();
}

#[test]
fn explicit_fetch_runtime_shutdown_cancels_inflight_requests_with_live_clones() {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("slow").with_delay_ms(2_000)]);
    let mut config = FetchConfig::default();
    config.set_request_timeout_ms(10_000);
    let runtime = FetchRuntimeOwner::new(&config, new_shared_browser_cookie_store());
    let runtime_clone = runtime.clone();
    let response_rx = runtime
        .submit(Request::get(&server.url()).unwrap())
        .unwrap();

    let deadline = Instant::now() + Duration::from_secs(1);
    while Instant::now() < deadline {
        if runtime.owner_count_for_testing() == 1 && server.hits() >= 1 {
            break;
        }
        thread::sleep(Duration::from_millis(10));
    }

    assert_eq!(runtime.owner_count_for_testing(), 1);
    assert!(
        server.hits() >= 1,
        "fetch runtime owner never started the in-flight request"
    );

    let shutdown_started = Instant::now();
    assert!(runtime.shutdown().is_clean());
    assert!(
        shutdown_started.elapsed() < Duration::from_secs(1),
        "explicit runtime shutdown waited too long for in-flight request cancellation"
    );

    let error = response_rx
        .blocking_recv()
        .expect("in-flight request should still respond after shutdown")
        .expect_err("in-flight request should be cancelled during shutdown");
    assert!(
        error.to_string().contains("cancelled during shutdown"),
        "unexpected shutdown error: {error:#}"
    );

    let submit_after_shutdown = runtime_clone.submit(Request::get(&server.url()).unwrap());
    assert!(
        submit_after_shutdown.is_err(),
        "live clone should reject requests after explicit shutdown"
    );
    server.shutdown();
}

#[test]
fn owner_thread_can_release_request_handle_before_external_owner_join() {
    let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("complete")]);
    let runtime =
        FetchRuntimeOwner::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let callback_runtime = runtime.clone();
    let (callback_entered_tx, callback_entered_rx) = std_mpsc::channel();
    let (release_callback_tx, release_callback_rx) = std_mpsc::channel();
    let (callback_exited_tx, callback_exited_rx) = std_mpsc::channel();

    runtime
        .submit_with_cancel_callback(
            Request::get(&server.url()).unwrap(),
            FetchCancelHandle::new(),
            Box::new(move |result| {
                callback_entered_tx
                    .send(result.map(|response| response.body_text().to_owned()))
                    .expect("test should still be waiting for the callback");
                release_callback_rx
                    .recv()
                    .expect("test should release the owner-thread callback");
                // This is a request handle, not the JoinHandle-owning runtime
                // owner. Releasing it on lm-fetch-semantics must be inert.
                drop(callback_runtime);
                callback_exited_tx
                    .send(())
                    .expect("test should still be waiting for callback exit");
            }),
        )
        .unwrap();

    let body = callback_entered_rx
        .recv_timeout(Duration::from_secs(1))
        .expect("fetch callback should run on the runtime owner")
        .expect("fetch request should complete successfully");
    assert_eq!(body, "complete");
    release_callback_tx
        .send(())
        .expect("owner callback should still be waiting");
    callback_exited_rx
        .recv_timeout(Duration::from_secs(1))
        .expect("owner callback should release its request handle without self-joining");

    assert!(runtime.shutdown().is_clean());

    server.shutdown();
}

#[test]
fn concurrent_request_handles_can_request_shutdown_before_external_join() {
    let runtime =
        FetchRuntimeOwner::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let post_shutdown_handle = runtime.clone();

    let start = Arc::new(std::sync::Barrier::new(3));
    let shutdowns = [runtime.clone(), runtime.clone()].map(|handle| {
        let start = Arc::clone(&start);
        thread::spawn(move || {
            start.wait();
            handle.request_shutdown();
        })
    });
    start.wait();
    for shutdown in shutdowns {
        shutdown
            .join()
            .expect("shutdown requester should not panic");
    }

    let external_join_started = Instant::now();
    assert!(runtime.shutdown().is_clean());
    assert!(
        external_join_started.elapsed() < Duration::from_secs(1),
        "external owner join waited too long after concurrent shutdown requests"
    );
    assert!(
        post_shutdown_handle
            .submit(Request::get("https://example.test/").unwrap())
            .is_err(),
        "all request handles should observe the concurrent shutdown"
    );
}

#[test]
fn fetch_runtime_owner_panic_returns_payload_and_identity_and_other_owners_still_join() {
    let mut panicking_owner =
        FetchRuntimeOwner::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let healthy_owner =
        FetchRuntimeOwner::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let panic_log_count = panicking_owner.panic_log_count_for_testing();

    panicking_owner.panic_owner_for_testing();
    let report = panicking_owner.join();
    assert_eq!(report, panicking_owner.join(), "join must be idempotent");
    assert_eq!(report.identity().thread_name(), "lm-fetch-semantics");
    assert!(!report.identity().thread_id().is_empty());
    let FetchRuntimeJoinStatus::Panicked(panic) = report.status() else {
        panic!("expected the deterministic owner panic, got {report:?}");
    };
    assert_eq!(panic.payload(), "deterministic fetch runtime panic");
    let location = panic
        .location()
        .expect("semantic owner panic must retain its panic-site location");
    assert!(
        location.contains("moli-fetch/src/runtime.rs"),
        "unexpected panic location: {location}"
    );
    let backtrace = panic
        .backtrace()
        .expect("semantic owner panic must retain its panic-site backtrace");
    assert!(!backtrace.trim().is_empty());
    // Toolchains can demangle inherent methods as Type::method or <Type>::method.
    // Require the same fully qualified owner and method in a single frame.
    assert!(
        backtrace.lines().any(|frame| {
            frame.contains("moli_fetch::runtime::RuntimeOwner::handle_command")
                || frame.contains("<moli_fetch::runtime::RuntimeOwner>::handle_command")
        }),
        "panic backtrace should identify the semantic owner command boundary: {backtrace}"
    );
    assert_eq!(
        panic_log_count.load(std::sync::atomic::Ordering::SeqCst),
        0,
        "explicit join returns evidence without pre-empting the Drop fallback log"
    );

    let healthy_report = healthy_owner.shutdown();
    assert!(healthy_report.is_clean());
    assert_ne!(
        report.identity().runtime_id(),
        healthy_report.identity().runtime_id(),
        "join reports must identify the exact runtime"
    );

    drop(panicking_owner);
    assert_eq!(
        panic_log_count.load(std::sync::atomic::Ordering::SeqCst),
        1,
        "one owner panic must be logged exactly once"
    );
}

#[test]
fn panicking_fetch_runtime_owner_drop_does_not_replace_an_outer_unwind() {
    let runtime =
        FetchRuntimeOwner::new(&FetchConfig::default(), new_shared_browser_cookie_store());
    let panic_log_count = runtime.panic_log_count_for_testing();

    let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
        runtime.panic_owner_for_testing();
        panic!("outer unwind sentinel");
    }));

    let payload = unwind.expect_err("the outer panic should remain observable");
    assert_eq!(
        payload.downcast_ref::<&'static str>().copied(),
        Some("outer unwind sentinel"),
        "Drop join must not replace the original panic"
    );
    assert_eq!(
        panic_log_count.load(std::sync::atomic::Ordering::SeqCst),
        1,
        "Drop during unwind must log the owner panic once without panicking"
    );
}

mod range_encoding;

mod referrer_redirects;