Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

62.8 KB
Raw Blame
mod websocket;

use anyhow::Result;
use std::{
    path::PathBuf,
    sync::Arc,
    time::{SystemTime, UNIX_EPOCH},
};
use tokio::{
    io::{AsyncReadExt, AsyncWriteExt},
    net::TcpListener,
    sync::oneshot,
    time::{Duration, sleep, timeout},
};
use url::Url;

use moli_cookie_jar::{
    BrowserCookieFacadeContext, NetworkCookieRequestContext, new_shared_browser_cookie_store,
};
use moli_fetch::{
    BrowserRequestMetadata, FetchCancelHandle, RawResponse, RedirectInfo, RequestCredentialsMode,
    RequestResourceType, ResponseHead, ScriptFetchRequestMetadata, StreamingRawResponse,
    cookie_header_for_request,
};

use super::{
    FetchConfig, Request, ResourceRequestClient, raw_subresource_memory_cache_key,
    streaming_raw_response_from_cached_subresource,
};
use crate::network::{BrowserResourceRuntimeOwner, BrowserResourceRuntimeOwnerRoot};
use crate::protocol_types::OptionalResourceFetchMask;
use crate::types::{SubresourceResourceType, SubresourceResponseBodyWriter};

// These transport/cache tests use browser navigation semantics; origin policy
// tests below construct their own explicit origin.
fn browser_navigation_request(url: &str) -> Result<Request> {
    Ok(Request::get(url)?.with_request_origin(moli_url::WebOrigin::Opaque))
}

#[test]
fn browser_client_rejects_http_requests_even_with_an_initiator() -> Result<()> {
    let owner = ResourceRequestClient::new(&FetchConfig::default())?;
    let url = Url::parse("https://example.test/")?;
    let http = Request::get_with_url(url.clone()).with_initiator_url(&url);
    assert!(http.request_origin().is_none());
    let error = owner.handle().apply_network_policy(http).unwrap_err();
    assert!(error.to_string().contains("explicit environment origin"));

    let browser = Request::new_browser("GET", url.clone(), None, Vec::new(), (&url).into());
    owner.handle().apply_network_policy(browser)?;
    Ok(())
}

#[test]
fn loader_clones_share_one_browser_resource_runtime() {
    let loader = ResourceRequestClient::new(&FetchConfig::default()).expect("loader");
    let clone = loader.clone();

    assert!(loader.shares_resource_runtime_with(&clone));
    assert!(loader.shares_page_network_policy_with(&clone));
    assert_eq!(
        loader.resource_runtime_diagnostics().runtime_id,
        clone.resource_runtime_diagnostics().runtime_id
    );
}

#[test]
fn loader_clones_share_one_resource_disk_pool() {
    let loader = ResourceRequestClient::new(&FetchConfig::default()).expect("loader");
    let clone = loader.clone();
    let pool = loader
        .disk_pool()
        .expect("resource runtime should create one disk pool");
    let mut writer =
        SubresourceResponseBodyWriter::with_memory_limit_and_disk_pool(2, clone.disk_pool());
    writer.append(b"shared");
    let body = writer.finish();

    assert_eq!(body.materialize_bytes().unwrap(), b"shared");
    assert_eq!(pool.diagnostics().disk_footprint_bytes, 6);
    assert_eq!(
        loader
            .resource_runtime_diagnostics()
            .disk_pool
            .expect("disk pool diagnostics should be present")
            .disk_footprint_bytes,
        6
    );
    drop(body);
    assert_eq!(pool.diagnostics().free_bytes, 6);
}

#[test]
fn loader_clones_share_one_parkable_image_manager() {
    let loader = ResourceRequestClient::new(&FetchConfig::default()).expect("loader");
    let clone = loader.clone();
    // Test shared ownership without a concurrent sweep temporarily retaining
    // the image. Background scheduling has its own live-runtime coverage.
    let runtime = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .expect("image-manager ownership test runtime");
    let runner =
        crate::network::RendererResourceTaskRunner::from_tokio_handle(runtime.handle().clone());
    let image = loader
        .parkable_image_manager(&runner)
        .from_frozen_bytes(vec![7; 2 * 1024]);

    assert_eq!(
        clone
            .resource_runtime_diagnostics()
            .parkable_images
            .image_count,
        1
    );
    assert_eq!(
        clone
            .resource_runtime_diagnostics()
            .parkable_images
            .retained_memory_bytes,
        2 * 1024
    );
    drop(image);
    assert_eq!(
        loader
            .resource_runtime_diagnostics()
            .parkable_images
            .image_count,
        0
    );
}

#[test]
fn page_loader_fork_shares_backend_but_isolates_mutable_policy() {
    let parent_owner = ResourceRequestClient::new(&FetchConfig::default()).expect("parent loader");
    let parent = parent_owner.handle().with_browser_site_context(
        BrowserCookieFacadeContext::default()
            .with_site_for_cookies_url(&Url::parse("https://parent.test/").unwrap())
            .with_top_frame_origin_url(&Url::parse("https://parent.test/").unwrap()),
    );
    parent.set_extra_http_headers(&vec![("x-page".to_owned(), "parent".to_owned())].into());
    let page = parent.fork_with_isolated_page_network_policy();

    assert!(parent.shares_resource_runtime_with(&page));
    assert!(!parent.shares_page_network_policy_with(&page));
    assert_ne!(
        parent.page_network_policy().memory_cache_partition_id(),
        page.page_network_policy().memory_cache_partition_id(),
        "a new Page target must own a distinct renderer memory-cache partition"
    );
    assert!(page.browser_site_context().is_none());

    page.set_network_offline(true);
    page.set_extra_http_headers(&vec![("x-page".to_owned(), "child".to_owned())].into());

    assert!(!parent.page_network_policy().snapshot().network_offline());
    assert!(page.page_network_policy().snapshot().network_offline());
}

#[test]
fn document_loader_fork_keeps_page_memory_cache_ownership() {
    let owner = ResourceRequestClient::new(&FetchConfig::default()).expect("Page loader");
    let document = owner.fork_with_isolated_document_network_policy();

    assert!(owner.shares_resource_runtime_with(&document));
    assert!(!owner.shares_page_network_policy_with(&document));
    assert_eq!(
        owner.page_network_policy().memory_cache_partition_id(),
        document.page_network_policy().memory_cache_partition_id(),
        "replacement Documents must retain their Page's memory-cache partition"
    );
}

#[test]
fn worker_loader_fork_preserves_creator_browser_site_context() {
    let context = BrowserCookieFacadeContext::default()
        .with_site_for_cookies_url(&Url::parse("https://top.test/").unwrap())
        .with_top_frame_origin_url(&Url::parse("https://top.test/").unwrap());
    let parent_owner = ResourceRequestClient::new(&FetchConfig::default()).expect("parent loader");
    let parent = parent_owner
        .handle()
        .with_browser_site_context(context.clone());

    let worker = parent.fork_with_isolated_worker_network_policy();

    assert!(parent.shares_resource_runtime_with(&worker));
    assert!(!parent.shares_page_network_policy_with(&worker));
    assert!(parent.shares_memory_cache_partition_with(&worker));
    assert_eq!(worker.browser_site_context(), Some(&context));
}

#[test]
fn independently_created_loaders_use_isolated_browser_resource_runtimes() {
    let left = ResourceRequestClient::new(&FetchConfig::default()).expect("left loader");
    let right = ResourceRequestClient::new(&FetchConfig::default()).expect("right loader");

    assert!(!left.shares_resource_runtime_with(&right));
    assert_ne!(
        left.resource_runtime_diagnostics().runtime_id,
        right.resource_runtime_diagnostics().runtime_id
    );

    let runner = crate::network::RendererResourceTaskRunner::for_test();
    let _left_image = left
        .parkable_image_manager(&runner)
        .from_frozen_bytes(vec![1; 1024]);
    assert_eq!(
        left.resource_runtime_diagnostics()
            .parkable_images
            .image_count,
        1
    );
    assert_eq!(
        right
            .resource_runtime_diagnostics()
            .parkable_images
            .image_count,
        0
    );

    let left_data = left
        .disk_pool()
        .expect("left runtime should have a disk pool")
        .store(b"left")
        .unwrap()
        .unwrap();
    assert_eq!(
        left.resource_runtime_diagnostics()
            .disk_pool
            .unwrap()
            .disk_footprint_bytes,
        4
    );
    assert_eq!(
        right
            .resource_runtime_diagnostics()
            .disk_pool
            .unwrap()
            .disk_footprint_bytes,
        0
    );
    drop(left_data);
}

fn unique_test_cache_dir() -> PathBuf {
    let unique = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap()
        .as_nanos();
    std::env::temp_dir().join(format!(
        "moli-renderer-v8-cache-test-{}-{unique}",
        std::process::id()
    ))
}

#[tokio::test(flavor = "multi_thread")]
async fn memory_cache_tee_drop_after_body_eof_cancels_pending_completion_and_releases_exact_runtime()
-> Result<()> {
    let registration = BrowserResourceRuntimeOwner::new(
        &FetchConfig::default(),
        new_shared_browser_cookie_store(),
    );
    let (root, binding) = BrowserResourceRuntimeOwnerRoot::new(registration);
    let registrar = root.registrar();
    let old_runtime = binding.current();
    let client = ResourceRequestClient::from_browser_resource_runtime(old_runtime.clone());
    let request = browser_navigation_request("https://cache.test/last-chunk.css")?
        .with_resource_type(RequestResourceType::CssStyleSheet);
    let cache_key = raw_subresource_memory_cache_key(&request)
        .expect("stylesheet request should use the raw subresource memory cache");

    let (body_tx, body_rx) = tokio::sync::mpsc::unbounded_channel();
    body_tx
        .send(b"last".to_vec())
        .expect("fake upstream body receiver should be live");
    drop(body_tx);
    let cancel_handle = FetchCancelHandle::new();
    let (mut completion_tx, completion_rx) = oneshot::channel();
    let inner = StreamingRawResponse::new_with_head(
        ResponseHead {
            status_text: None,
            final_url: request.url.clone(),
            status: 200,
            headers: vec![("cache-control".to_owned(), b"max-age=60".to_vec())],
            request_cookie_report: None,
            cookie_set_reports: Vec::new(),
            redirected: false,
            redirect_chain: Vec::new(),
            from_cache: false,
            negotiated_http_version: None,
        },
        body_rx,
        cancel_handle.clone(),
        completion_rx,
    )
    .with_lifetime_lease(old_runtime.clone());
    let (body_eof_tx, body_eof_rx) = oneshot::channel();
    let mut outward = client.tee_raw_subresource_response_for_memory_cache_with_body_eof_observer(
        request,
        cache_key,
        inner,
        Some(body_eof_tx),
    );

    let replacement = registrar
        .replace_owned(BrowserResourceRuntimeOwner::new(
            &FetchConfig::default(),
            new_shared_browser_cookie_store(),
        ))
        .expect("replacement runtime should register on the same root");
    drop(replacement);
    drop(client);
    drop(old_runtime);
    assert_eq!(root.owner_count_for_testing(), 2);

    assert_eq!(outward.next_chunk().await.as_deref(), Some(&b"last"[..]));
    body_eof_rx
        .await
        .expect("tee should observe upstream body EOF before completion");
    assert!(
        !completion_tx.is_closed(),
        "upstream completion must remain pending at the test barrier"
    );

    drop(outward);
    completion_tx.closed().await;
    assert!(
        cancel_handle.is_cancelled(),
        "dropping the outward response must cancel the pending inner response"
    );

    let retired_reports = root.reap_retired();
    assert_eq!(retired_reports.len(), 1);
    assert!(retired_reports[0].is_clean());
    assert_eq!(root.owner_count_for_testing(), 1);
    let active_reports = root.shutdown_and_join_reports_for_testing();
    assert_eq!(active_reports.len(), 1);
    assert!(active_reports[0].is_clean());
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn dropping_cacheable_stream_stalled_between_chunks_cancels_real_fetch_and_reaps_exact_owner()
-> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let (disconnected_tx, disconnected_rx) = oneshot::channel();
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await?;
        read_http_request_head(&mut stream).await?;
        stream
            .write_all(
                concat!(
                    "HTTP/1.1 200 OK\r\n",
                    "Content-Type: text/css\r\n",
                    "Cache-Control: max-age=60\r\n",
                    "Transfer-Encoding: chunked\r\n",
                    "\r\n",
                    "4\r\n",
                    "a{}b\r\n",
                )
                .as_bytes(),
            )
            .await?;

        // The next response chunk is deliberately never published. A read is
        // an exact cancellation witness: no further HTTP request bytes are
        // expected, so EOF means the client closed the in-progress transfer.
        let mut byte = [0_u8; 1];
        let disconnected = stream.read(&mut byte).await? == 0;
        let _ = disconnected_tx.send(disconnected);
        Ok::<_, std::io::Error>(())
    });

    let registration = BrowserResourceRuntimeOwner::new(
        &FetchConfig::default(),
        new_shared_browser_cookie_store(),
    );
    let (root, binding) = BrowserResourceRuntimeOwnerRoot::new(registration);
    let registrar = root.registrar();
    let old_runtime = binding.current();
    let client = ResourceRequestClient::from_browser_resource_runtime(old_runtime.clone());
    let cancel_handle = FetchCancelHandle::new();
    let request = browser_navigation_request(&format!("http://{addr}/stalled.css"))?
        .with_resource_type(RequestResourceType::CssStyleSheet);
    assert!(
        raw_subresource_memory_cache_key(&request).is_some(),
        "the request must exercise the renderer memory-cache tee"
    );
    let mut outward = client
        .fetch_raw_stream_with_cancel(request, cancel_handle.clone())
        .await?;
    assert_eq!(outward.next_chunk().await.as_deref(), Some(&b"a{}b"[..]));

    let replacement = registrar
        .replace_owned(BrowserResourceRuntimeOwner::new(
            &FetchConfig::default(),
            new_shared_browser_cookie_store(),
        ))
        .expect("replacement runtime should register on the same root");
    drop(replacement);
    drop(client);
    drop(old_runtime);
    assert_eq!(root.owner_count_for_testing(), 2);

    drop(outward);
    assert!(
        timeout(Duration::from_secs(3), disconnected_rx).await??,
        "the server must observe cancellation while stalled between chunks"
    );
    assert!(
        cancel_handle.is_cancelled(),
        "dropping the outward tee must cancel its exact inner fetch"
    );

    let retired_reports = root.reap_retired();
    assert_eq!(retired_reports.len(), 1);
    assert!(
        retired_reports[0].is_clean(),
        "the exact retired fetch owner must join cleanly: {:?}",
        retired_reports[0]
    );
    let active_reports = root.shutdown_and_join_reports_for_testing();
    assert_eq!(active_reports.len(), 1);
    assert!(active_reports[0].is_clean());
    server.await??;
    Ok(())
}

async fn read_http_request_head(stream: &mut tokio::net::TcpStream) -> std::io::Result<()> {
    let mut request = Vec::new();
    let mut byte = [0_u8; 1];
    loop {
        let read = stream.read(&mut byte).await?;
        if read == 0 {
            return Err(std::io::Error::new(
                std::io::ErrorKind::UnexpectedEof,
                "client closed before sending complete request",
            ));
        }
        request.push(byte[0]);
        if request.ends_with(b"\r\n\r\n") {
            return Ok(());
        }
    }
}

async fn read_http_request_text(stream: &mut tokio::net::TcpStream) -> std::io::Result<String> {
    let mut request = Vec::new();
    let mut byte = [0_u8; 1];
    loop {
        let read = stream.read(&mut byte).await?;
        if read == 0 {
            return Err(std::io::Error::new(
                std::io::ErrorKind::UnexpectedEof,
                "client closed before sending complete request",
            ));
        }
        request.push(byte[0]);
        if request.ends_with(b"\r\n\r\n") {
            return String::from_utf8(request).map_err(|error| {
                std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string())
            });
        }
    }
}

#[test]
fn merge_loader_network_policy_headers_uses_header_name_keys_and_request_order() {
    let policy = super::PageNetworkPolicy::default();
    policy.set_extra_http_headers(
        &vec![
            ("X-Test".to_owned(), "context".to_owned()),
            ("Accept".to_owned(), "text/html".to_owned()),
        ]
        .into(),
    );
    let mut request = browser_navigation_request("https://example.test/headers")
        .unwrap()
        .with_page_network_policy();
    request.request_headers = vec![
        ("x-test".to_owned(), "request".to_owned()),
        ("X-New".to_owned(), "new".to_owned()),
    ]
    .into();
    let merged = policy
        .snapshot()
        .apply_to_request(request)
        .unwrap()
        .request_headers;

    assert_eq!(
        merged.to_byte_strings(),
        vec![
            ("Accept".to_owned(), "text/html".to_owned()),
            ("x-test".to_owned(), "request".to_owned()),
            ("X-New".to_owned(), "new".to_owned()),
        ]
    );
}

#[test]
fn csp_report_subresource_maps_to_csp_report_request_type() {
    assert_eq!(
        crate::network::request_resource_type_for_subresource(SubresourceResourceType::CspReport),
        Some(RequestResourceType::CspReport)
    );
    assert_eq!(
        SubresourceResourceType::CspReport.as_cdp_type(),
        "CSPViolationReport"
    );
}

#[test]
fn dictionary_subresource_maps_to_dictionary_request_type_and_other_cdp_type() {
    assert_eq!(
        crate::network::request_resource_type_for_subresource(SubresourceResourceType::Dictionary),
        Some(RequestResourceType::Dictionary)
    );
    assert_eq!(SubresourceResourceType::Dictionary.as_cdp_type(), "Other");
}

#[test]
fn audio_video_subresources_keep_media_request_type_and_cdp_type() {
    assert_eq!(
        crate::network::request_resource_type_for_subresource(SubresourceResourceType::Audio),
        Some(RequestResourceType::Media)
    );
    assert_eq!(
        crate::network::request_resource_type_for_subresource(SubresourceResourceType::Video),
        Some(RequestResourceType::Media)
    );
    assert_eq!(SubresourceResourceType::Audio.as_cdp_type(), "Media");
    assert_eq!(SubresourceResourceType::Video.as_cdp_type(), "Media");
}

#[test]
fn loader_disables_every_optional_resource_family_by_default() {
    let loader = ResourceRequestClient::new(&FetchConfig::default()).expect("loader");
    let optional = [
        SubresourceResourceType::Image,
        SubresourceResourceType::Font,
        SubresourceResourceType::Audio,
        SubresourceResourceType::Video,
        SubresourceResourceType::Media,
        SubresourceResourceType::TextTrack,
    ];

    assert_eq!(
        loader.optional_resource_fetch_mask(),
        OptionalResourceFetchMask::NONE
    );
    for resource_type in optional {
        assert!(
            !loader.optional_resource_fetch_enabled(resource_type),
            "{resource_type:?} must be disabled on a default loader"
        );
    }
    for resource_type in [
        SubresourceResourceType::Script,
        SubresourceResourceType::Stylesheet,
        SubresourceResourceType::Fetch,
        SubresourceResourceType::Xhr,
    ] {
        assert!(
            loader.optional_resource_fetch_enabled(resource_type),
            "{resource_type:?} must remain outside this policy"
        );
    }
}

#[test]
fn loader_preserves_each_optional_resource_bit_without_aliasing() {
    let resources = [
        (
            SubresourceResourceType::Image,
            OptionalResourceFetchMask::IMAGE,
        ),
        (
            SubresourceResourceType::Font,
            OptionalResourceFetchMask::FONT,
        ),
        (
            SubresourceResourceType::Audio,
            OptionalResourceFetchMask::AUDIO,
        ),
        (
            SubresourceResourceType::Video,
            OptionalResourceFetchMask::VIDEO,
        ),
        (
            SubresourceResourceType::Media,
            OptionalResourceFetchMask::MEDIA,
        ),
        (
            SubresourceResourceType::TextTrack,
            OptionalResourceFetchMask::TEXT_TRACK,
        ),
    ];
    let loader = ResourceRequestClient::new(&FetchConfig::default()).expect("loader");

    for (enabled_type, enabled_bit) in resources {
        loader.set_optional_resource_fetch_mask(enabled_bit);
        assert_eq!(loader.optional_resource_fetch_mask(), enabled_bit);
        for (observed_type, _) in resources {
            assert_eq!(
                loader.optional_resource_fetch_enabled(observed_type),
                observed_type == enabled_type,
                "enabling {enabled_type:?} changed {observed_type:?}"
            );
        }
    }
}

#[test]
fn loader_image_compatibility_switch_preserves_other_resource_bits() {
    let loader = ResourceRequestClient::new(&FetchConfig::default()).expect("loader");
    let preserved = OptionalResourceFetchMask::AUDIO | OptionalResourceFetchMask::FONT;
    loader.set_optional_resource_fetch_mask(preserved);

    loader.set_image_fetch_enabled(true);
    assert_eq!(
        loader.optional_resource_fetch_mask(),
        preserved | OptionalResourceFetchMask::IMAGE
    );

    loader.set_image_fetch_enabled(false);
    assert_eq!(loader.optional_resource_fetch_mask(), preserved);
}

#[test]
fn memory_caches_partition_explicit_origin_independently_of_referrer() -> Result<()> {
    let url = Url::parse("https://scripts.test/app.js")?;
    let origin = moli_url::WebOrigin::from_url(&url);
    let request = Request::new_browser("GET", url.clone(), None, Vec::new(), origin)
        .with_initiator_url(&url)
        .with_resource_type(RequestResourceType::CssStyleSheet)
        .with_script_fetch_metadata(ScriptFetchRequestMetadata::default());
    let opaque = request
        .clone()
        .with_request_origin(moli_url::WebOrigin::Opaque);
    let no_cors = request
        .clone()
        .with_request_mode(moli_fetch::RequestMode::NoCors);
    for other in [opaque, no_cors] {
        assert_ne!(
            super::script_text_cache_key(&request),
            super::script_text_cache_key(&other)
        );
        assert_ne!(
            raw_subresource_memory_cache_key(&request),
            raw_subresource_memory_cache_key(&other)
        );
    }
    Ok(())
}

#[test]
fn script_text_memory_cache_key_partitions_credentials_and_cookie_context() -> Result<()> {
    let url = Url::parse("https://scripts.test/app.js")?;
    let base_request = || {
        browser_navigation_request(url.as_str())
            .unwrap()
            .with_script_fetch_metadata(ScriptFetchRequestMetadata::default())
    };

    let include_key = super::script_text_cache_key(
        &base_request().with_credentials_mode(RequestCredentialsMode::Include),
    );
    let omit_key = super::script_text_cache_key(
        &base_request().with_credentials_mode(RequestCredentialsMode::Omit),
    );
    assert_ne!(
        include_key, omit_key,
        "script text cache must not share responses across credentials modes"
    );

    let same_origin = Url::parse("https://scripts.test/page")?;
    let cross_origin = Url::parse("https://other.test/page")?;
    let mut same_site_request = base_request();
    same_site_request.cookie_context =
        NetworkCookieRequestContext::subresource("GET").with_initiator_url(&url, &same_origin);
    let mut cross_site_request = base_request();
    cross_site_request.cookie_context =
        NetworkCookieRequestContext::subresource("GET").with_initiator_url(&url, &cross_origin);

    assert_ne!(
        super::script_text_cache_key(&same_site_request),
        super::script_text_cache_key(&cross_site_request),
        "script text cache must not share responses across cookie contexts"
    );

    let first_page = Url::parse("https://scripts.test/first-page")?;
    let second_page = Url::parse("https://scripts.test/second-page")?;
    let mut first_page_request = base_request();
    first_page_request.cookie_context =
        NetworkCookieRequestContext::subresource("GET").with_initiator_url(&url, &first_page);
    let mut second_page_request = base_request();
    second_page_request.cookie_context =
        NetworkCookieRequestContext::subresource("GET").with_initiator_url(&url, &second_page);
    assert_eq!(
        super::script_text_cache_key(&first_page_request),
        super::script_text_cache_key(&second_page_request),
        "same-site page URL changes should not partition reusable script/module responses"
    );

    let first_partition_key = base_request().with_network_partition_key(Some(
        "storage-key:v1;origin=https://scripts.test;top-level-site=https://app.example.test;opaque-nonce=1"
            .to_owned(),
    ));
    let second_partition_key = base_request().with_network_partition_key(Some(
        "storage-key:v1;origin=https://scripts.test;top-level-site=https://app.example.test;opaque-nonce=2"
            .to_owned(),
    ));
    assert_ne!(
        super::script_text_cache_key(&first_partition_key),
        super::script_text_cache_key(&second_partition_key),
        "script text cache must not share responses across network partition keys"
    );

    let no_integrity_key = super::script_text_cache_key(&base_request());
    let integrity_key = super::script_text_cache_key(&base_request().with_script_fetch_metadata(
        ScriptFetchRequestMetadata {
            integrity: Some("sha384-integrity".to_owned()),
            ..ScriptFetchRequestMetadata::default()
        },
    ));
    assert_ne!(
        no_integrity_key, integrity_key,
        "script text cache must not share responses across integrity metadata"
    );
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn streaming_fetch_commits_set_cookie_before_body_completion() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        stream
            .write_all(
                concat!(
                    "HTTP/1.1 200 OK\r\n",
                    "Content-Type: text/html; charset=utf-8\r\n",
                    "Set-Cookie: sid=server; Path=/\r\n",
                    "Transfer-Encoding: chunked\r\n",
                    "\r\n",
                    "c\r\n",
                    "hello world!\r\n",
                )
                .as_bytes(),
            )
            .await
            .unwrap();
        sleep(Duration::from_millis(100)).await;
        stream.write_all(b"0\r\n\r\n").await.unwrap();
    });

    let cookie_store = new_shared_browser_cookie_store();
    let loader = ResourceRequestClient::new_with_cookie_store(
        &FetchConfig::default(),
        Arc::clone(&cookie_store),
    )?;
    let request = browser_navigation_request(&format!("http://{addr}/stream"))?;
    let mut response = loader
        .fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
        .await?;

    assert_eq!(
        cookie_header_for_request(
            &cookie_store,
            &Url::parse(&format!("http://{addr}/subresource"))?,
            NetworkCookieRequestContext::subresource("GET"),
        )?,
        Some("sid=server".to_owned())
    );
    assert_eq!(
        response.next_chunk().await.as_deref(),
        Some(&b"hello world!"[..])
    );

    response.finish().await?;
    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn dropping_streaming_response_cancels_the_source_transfer() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let (disconnect_tx, disconnect_rx) = oneshot::channel();
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        stream
            .write_all(
                concat!(
                    "HTTP/1.1 200 OK\r\n",
                    "Content-Type: text/html; charset=utf-8\r\n",
                    "Transfer-Encoding: chunked\r\n",
                    "\r\n",
                    "1\r\n",
                    "a\r\n",
                )
                .as_bytes(),
            )
            .await
            .unwrap();
        let mut disconnected = false;
        for _ in 0..200 {
            sleep(Duration::from_millis(10)).await;
            if stream.write_all(b"1\r\na\r\n").await.is_err() {
                disconnected = true;
                break;
            }
        }
        let _ = disconnect_tx.send(disconnected);
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let request = browser_navigation_request(&format!("http://{addr}/stream"))?;
    let mut response = loader
        .fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())
        .await?;

    assert_eq!(response.next_chunk().await.as_deref(), Some(&b"a"[..]));
    drop(response);

    assert!(
        timeout(Duration::from_secs(3), disconnect_rx).await??,
        "server never observed the client cancelling the streaming response"
    );

    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn text_stream_fetch_uses_disk_cache_for_safe_gets() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        let body = "cached-renderer-text-stream";
        let response = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            body.len(),
            body
        );
        stream.write_all(response.as_bytes()).await.unwrap();
    });

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let loader = ResourceRequestClient::new(&config)?;
    let url = format!("http://{addr}/cacheable.txt");

    let first = loader
        .fetch_text_stream(browser_navigation_request(&url)?.with_page_network_policy())
        .await?;
    let second = loader
        .fetch_text_stream(browser_navigation_request(&url)?.with_page_network_policy())
        .await?;

    assert_eq!(first.body_text(), "cached-renderer-text-stream");
    assert_eq!(second.body_text(), "cached-renderer-text-stream");

    server.await?;
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn script_text_fetch_uses_shared_memory_resource_cache_with_fresh_cache_headers() -> Result<()>
{
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        let body = "window.scriptMemoryCacheHit = true;";
        let response = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: application/javascript\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            body.len(),
            body
        );
        stream.write_all(response.as_bytes()).await.unwrap();

        assert!(
            timeout(Duration::from_millis(100), listener.accept())
                .await
                .is_err(),
            "second script fetch should be served from the loader memory cache"
        );
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let url = format!("http://{addr}/app.js");
    let request = || {
        browser_navigation_request(&url).map(|request| {
            request.with_script_fetch_metadata(ScriptFetchRequestMetadata::default())
        })
    };

    let first = loader
        .fetch_cacheable_script_text_stream(request()?)
        .await?;
    let second = loader
        .fetch_cacheable_script_text_stream(request()?)
        .await?;

    assert_eq!(first.body_text(), "window.scriptMemoryCacheHit = true;");
    assert_eq!(second.body_text(), "window.scriptMemoryCacheHit = true;");
    assert!(!first.from_cache);
    assert!(second.from_cache);

    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn cache_bypass_replaces_script_text_memory_entry() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        for body in [
            "window.cacheGeneration = 1;",
            "window.cacheGeneration = 1;",
            "window.cacheGeneration = 2;",
        ] {
            let (mut stream, _) = listener.accept().await.unwrap();
            read_http_request_head(&mut stream).await.unwrap();
            let response = format!(
                "HTTP/1.1 200 OK\r\nContent-Type: application/javascript\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
                body.len(),
                body
            );
            stream.write_all(response.as_bytes()).await.unwrap();
        }
        assert!(
            timeout(Duration::from_millis(100), listener.accept())
                .await
                .is_err(),
            "normal loading after bypass should retain both Page-local script entries"
        );
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let peer = loader.fork_with_isolated_page_network_policy();
    let url = format!("http://{addr}/app.js");
    let request = || {
        browser_navigation_request(&url).map(|request| {
            request
                .with_page_network_policy()
                .with_script_fetch_metadata(ScriptFetchRequestMetadata::default())
        })
    };

    let first = loader
        .fetch_cacheable_script_text_stream(request()?)
        .await?;
    let peer_first = peer.fetch_cacheable_script_text_stream(request()?).await?;
    loader.set_cache_disabled(true);
    let bypassed = loader
        .fetch_cacheable_script_text_stream(request()?)
        .await?;
    loader.set_cache_disabled(false);
    let restored = loader
        .fetch_cacheable_script_text_stream(request()?)
        .await?;
    let peer_restored = peer.fetch_cacheable_script_text_stream(request()?).await?;

    assert_eq!(first.body_text(), "window.cacheGeneration = 1;");
    assert_eq!(peer_first.body_text(), "window.cacheGeneration = 1;");
    assert_eq!(bypassed.body_text(), "window.cacheGeneration = 2;");
    assert_eq!(restored.body_text(), "window.cacheGeneration = 2;");
    assert_eq!(peer_restored.body_text(), "window.cacheGeneration = 1;");
    assert!(!first.from_cache);
    assert!(!peer_first.from_cache);
    assert!(!bypassed.from_cache);
    assert!(restored.from_cache);
    assert!(peer_restored.from_cache);

    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn unique_script_text_fetches_stay_within_one_loader_memory_budget() -> Result<()> {
    const SCRIPT_COUNT: usize = 40;
    const SCRIPT_BYTES: usize = 256 * 1024;

    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let body = format!("/*{}*/", "x".repeat(SCRIPT_BYTES - 4));
        for _ in 0..SCRIPT_COUNT {
            let (mut stream, _) = listener.accept().await.unwrap();
            read_http_request_head(&mut stream).await.unwrap();
            let response_head = format!(
                "HTTP/1.1 200 OK\r\nContent-Type: application/javascript\r\nCache-Control: max-age=3600\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
                body.len()
            );
            stream.write_all(response_head.as_bytes()).await.unwrap();
            stream.write_all(body.as_bytes()).await.unwrap();
        }
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    for index in 0..SCRIPT_COUNT {
        let url = format!("http://{addr}/script-{index}.js");
        let request = browser_navigation_request(&url)?
            .with_script_fetch_metadata(ScriptFetchRequestMetadata::default());
        let response = loader.fetch_cacheable_script_text_stream(request).await?;
        assert_eq!(response.body_bytes().len(), SCRIPT_BYTES);
    }

    server.await?;
    let diagnostics = loader.memory_cache_diagnostics();
    assert!(
        diagnostics.retained_bytes <= diagnostics.retained_bytes_limit,
        "loader memory cache exceeded its shared budget: {diagnostics:?}"
    );
    assert!(
        diagnostics.completed_script_entry_count < SCRIPT_COUNT,
        "unique completed scripts should evict older strong references: {diagnostics:?}"
    );
    assert_eq!(diagnostics.pending_script_entry_count, 0);
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn stylesheet_text_stream_uses_shared_memory_resource_cache_without_http_cache_dir()
-> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        let body = "body { color: rgb(1, 2, 3); }";
        let response = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: text/css\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            body.len(),
            body
        );
        stream.write_all(response.as_bytes()).await.unwrap();

        assert!(
            timeout(Duration::from_millis(100), listener.accept())
                .await
                .is_err(),
            "second stylesheet text stream should be served from the loader memory cache"
        );
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let url = format!("http://{addr}/cached.css");
    let request = || {
        browser_navigation_request(&url).map(|request| {
            request
                .with_page_network_policy()
                .with_resource_type(RequestResourceType::CssStyleSheet)
        })
    };

    let first = loader.fetch_text_stream(request()?).await?;
    let second = loader.fetch_text_stream(request()?).await?;

    assert_eq!(first.body_text(), "body { color: rgb(1, 2, 3); }");
    assert_eq!(second.body_text(), "body { color: rgb(1, 2, 3); }");
    assert!(!first.from_cache);
    assert!(second.from_cache);

    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn text_stream_fetch_handles_local_data_stylesheet_urls() -> Result<()> {
    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let response = loader
        .fetch_text_stream(
            browser_navigation_request("data:text/css,:root%7Bbackground:green%7D")?
                .with_page_network_policy()
                .with_resource_type(RequestResourceType::CssStyleSheet),
        )
        .await?;

    assert_eq!(response.status, 200);
    assert_eq!(
        response.final_url.as_str(),
        "data:text/css,:root%7Bbackground:green%7D"
    );
    assert!(
        response
            .headers
            .iter()
            .any(|(name, value)| name.eq_ignore_ascii_case("content-type") && value == b"text/css")
    );
    assert_eq!(response.body_text(), ":root{background:green}");
    assert!(!response.from_cache);
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn local_blob_method_errors_reach_streaming_and_callback_consumers() -> Result<()> {
    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let origin = moli_url::WebOrigin::from_url(&Url::parse("https://method.test/page")?);
    for method in [
        "HEAD", "POST", "PUT", "DELETE", "OPTIONS", "PATCH", "CUSTOM",
    ] {
        let request = Request::new_browser_bytes(
            method,
            "blob:https://method.test/unregistered",
            None,
            Vec::new(),
            origin.clone(),
        )?;
        let expected = format!("blob URL fetch requires GET, got `{method}`");
        let error = loader.fetch_text_stream(request.clone()).await.unwrap_err();
        assert_eq!(error.to_string(), expected);
        let error = loader
            .fetch_cacheable_script_text_stream(request.clone())
            .await
            .unwrap_err();
        assert_eq!(error.to_string(), expected);
        let error = loader
            .fetch_raw_stream_with_cancel(request.clone(), FetchCancelHandle::new())
            .await
            .expect_err("raw stream must reject non-GET blob requests");
        assert_eq!(error.to_string(), expected);
        let error = loader
            .fetch_raw_stream_with_cancel_and_network_metadata(
                request.clone(),
                FetchCancelHandle::new(),
            )
            .await
            .expect_err("observed raw stream must reject non-GET blob requests");
        assert_eq!(error.to_string(), expected);
        let (tx, rx) = oneshot::channel();
        loader.fetch_text_callback(request, move |result| {
            tx.send(result).expect("callback receiver is live");
        })?;
        let error = rx.await?.unwrap_err();
        assert_eq!(error.to_string(), expected);

        let data = loader
            .fetch_text_stream(Request::new_browser_bytes(
                method,
                "data:text/plain,payload",
                None,
                Vec::new(),
                origin.clone(),
            )?)
            .await?;
        assert_eq!(
            data.status, 200,
            "data: must not inherit the blob method restriction"
        );
        assert_eq!(data.body_text(), "payload");
    }
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn concurrent_script_text_waiter_preserves_owner_cache_state() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let (release_tx, release_rx) = oneshot::channel();
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        let body = "export default function fromCacheCoalescing() {}";
        let response_head = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: text/javascript\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
            body.len()
        );
        stream.write_all(response_head.as_bytes()).await.unwrap();
        let _ = release_rx.await;
        stream.write_all(body.as_bytes()).await.unwrap();

        assert!(
            timeout(Duration::from_millis(100), listener.accept())
                .await
                .is_err(),
            "coalesced and completed-cache script loads should share the single network request"
        );
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let url = format!("http://{addr}/module.js");
    let request = || {
        browser_navigation_request(&url).map(|request| {
            request.with_script_fetch_metadata(ScriptFetchRequestMetadata::default())
        })
    };

    let first_request = request()?;
    let second_request = request()?;
    let first = loader.fetch_cacheable_script_text_stream(first_request);
    let second = loader.fetch_cacheable_script_text_stream(second_request);
    let release = async move {
        sleep(Duration::from_millis(50)).await;
        let _ = release_tx.send(());
    };

    let (first, second, _) = tokio::join!(first, second, release);
    let first = first?;
    let second = second?;
    assert_eq!(
        first.body_text(),
        "export default function fromCacheCoalescing() {}"
    );
    assert_eq!(
        second.body_text(),
        "export default function fromCacheCoalescing() {}"
    );
    assert!(!first.from_cache);
    assert!(
        !second.from_cache,
        "in-flight coalescing should preserve the owner's network provenance"
    );

    let third = loader
        .fetch_cacheable_script_text_stream(request()?)
        .await?;
    assert_eq!(
        third.body_text(),
        "export default function fromCacheCoalescing() {}"
    );
    assert!(
        third.from_cache,
        "completed script text cache hits should still report memory-cache provenance"
    );

    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn script_text_fetch_respects_configured_request_timeout() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        stream
            .write_all(
                concat!(
                    "HTTP/1.1 200 OK\r\n",
                    "Content-Type: application/javascript\r\n",
                    "Content-Length: 32\r\n",
                    "Connection: close\r\n",
                    "\r\n",
                )
                .as_bytes(),
            )
            .await
            .unwrap();
        sleep(Duration::from_millis(500)).await;
        let _ = stream.write_all(b"window.scriptLoaded = true;\n").await;
    });

    let mut config = FetchConfig::default();
    config.set_request_timeout_ms(100);
    let loader = ResourceRequestClient::new(&config)?;
    let request = browser_navigation_request(&format!("http://{addr}/slow-script.js"))?
        .with_script_fetch_metadata(ScriptFetchRequestMetadata::default());
    let error = timeout(
        Duration::from_secs(2),
        loader.fetch_cacheable_script_text_stream(request),
    )
    .await
    .expect("script fetch should complete with the configured request timeout")
    .unwrap_err();

    assert!(
        error
            .chain()
            .any(|cause| cause.to_string().contains("Timeout was reached")),
        "expected configured curl request timeout, got: {error:#}"
    );
    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn compatibility_fetch_uses_streaming_disk_cache_for_safe_gets() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        let body = "cached-renderer-compat-fetch";
        let response = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            body.len(),
            body
        );
        stream.write_all(response.as_bytes()).await.unwrap();
    });

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let loader = ResourceRequestClient::new(&config)?;
    let url = format!("http://{addr}/cacheable.txt");

    let first = loader
        .fetch(browser_navigation_request(&url)?.with_page_network_policy())
        .await?;
    let second = loader
        .fetch(browser_navigation_request(&url)?.with_page_network_policy())
        .await?;

    assert_eq!(first.body_text(), "cached-renderer-compat-fetch");
    assert_eq!(second.body_text(), "cached-renderer-compat-fetch");

    server.await?;
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn compatibility_raw_fetch_uses_streaming_disk_cache_for_safe_gets() -> Result<()> {
    let cache_dir = unique_test_cache_dir();
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        let body = b"cached-renderer-compat-raw\xff";
        let response_head = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
            body.len()
        );
        stream.write_all(response_head.as_bytes()).await.unwrap();
        stream.write_all(body).await.unwrap();
    });

    let mut config = FetchConfig::default();
    config.set_http_cache_dir(Some(cache_dir.display().to_string()));
    let loader = ResourceRequestClient::new(&config)?;
    let url = format!("http://{addr}/cacheable.bin");

    let first = loader
        .fetch_raw(browser_navigation_request(&url)?.with_page_network_policy())
        .await?;
    let second = loader
        .fetch_raw(browser_navigation_request(&url)?.with_page_network_policy())
        .await?;

    assert_eq!(first.body_bytes(), b"cached-renderer-compat-raw\xff");
    assert_eq!(second.body_bytes(), b"cached-renderer-compat-raw\xff");

    server.await?;
    let _ = std::fs::remove_dir_all(cache_dir);
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn image_raw_stream_uses_shared_memory_resource_cache_without_http_cache_dir() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        let body = b"cached-image-body";
        let response_head = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
            body.len()
        );
        stream.write_all(response_head.as_bytes()).await.unwrap();
        stream.write_all(body).await.unwrap();

        assert!(
            timeout(Duration::from_millis(100), listener.accept())
                .await
                .is_err(),
            "second image fetch should be served from the loader memory cache"
        );
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let url = format!("http://{addr}/cached.png");
    let request = || {
        browser_navigation_request(&url).map(|request| {
            request
                .with_page_network_policy()
                .with_resource_type(RequestResourceType::Image)
        })
    };

    let mut first = loader
        .fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
        .await?;
    assert!(!first.from_cache);
    let mut first_body = Vec::new();
    while let Some(chunk) = first.next_chunk().await {
        first_body.extend_from_slice(&chunk);
    }
    first.finish().await?;

    let mut second = loader
        .fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
        .await?;
    assert!(second.from_cache);
    let mut second_body = Vec::new();
    while let Some(chunk) = second.next_chunk().await {
        second_body.extend_from_slice(&chunk);
    }
    second.finish().await?;

    assert_eq!(first_body, b"cached-image-body");
    assert_eq!(second_body, b"cached-image-body");

    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn cache_bypass_replaces_raw_subresource_memory_entry() -> Result<()> {
    async fn body(mut response: StreamingRawResponse) -> Result<(Vec<u8>, bool)> {
        let from_cache = response.from_cache;
        let mut bytes = Vec::new();
        while let Some(chunk) = response.next_chunk().await {
            bytes.extend_from_slice(&chunk);
        }
        response.finish().await?;
        Ok((bytes, from_cache))
    }

    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        for payload in [
            b"image-v1".as_slice(),
            b"image-v1".as_slice(),
            b"image-v2".as_slice(),
        ] {
            let (mut stream, _) = listener.accept().await.unwrap();
            read_http_request_head(&mut stream).await.unwrap();
            let response_head = format!(
                "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
                payload.len()
            );
            stream.write_all(response_head.as_bytes()).await.unwrap();
            stream.write_all(payload).await.unwrap();
        }
        assert!(
            timeout(Duration::from_millis(100), listener.accept())
                .await
                .is_err(),
            "normal loading after bypass should retain both Page-local image entries"
        );
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let peer = loader.fork_with_isolated_page_network_policy();
    let url = format!("http://{addr}/cached.png");
    let request = || {
        browser_navigation_request(&url).map(|request| {
            request
                .with_page_network_policy()
                .with_resource_type(RequestResourceType::Image)
        })
    };

    let first = body(
        loader
            .fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
            .await?,
    )
    .await?;
    let peer_first = body(
        peer.fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
            .await?,
    )
    .await?;
    loader.set_cache_disabled(true);
    let bypassed = body(
        loader
            .fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
            .await?,
    )
    .await?;
    loader.set_cache_disabled(false);
    let restored = body(
        loader
            .fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
            .await?,
    )
    .await?;
    let peer_restored = body(
        peer.fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
            .await?,
    )
    .await?;

    assert_eq!(first, (b"image-v1".to_vec(), false));
    assert_eq!(peer_first, (b"image-v1".to_vec(), false));
    assert_eq!(bypassed, (b"image-v2".to_vec(), false));
    assert_eq!(restored, (b"image-v2".to_vec(), true));
    assert_eq!(peer_restored, (b"image-v1".to_vec(), true));

    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn browser_fetch_raw_stream_uses_shared_memory_resource_cache_without_http_cache_dir()
-> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        read_http_request_head(&mut stream).await.unwrap();
        let body = b"cached-fetch-body";
        let response_head = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCache-Control: max-age=60\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
            body.len()
        );
        stream.write_all(response_head.as_bytes()).await.unwrap();
        stream.write_all(body).await.unwrap();

        assert!(
            timeout(Duration::from_millis(100), listener.accept())
                .await
                .is_err(),
            "second browser fetch raw stream should be served from the loader memory cache"
        );
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let url = format!("http://{addr}/cached-fetch.txt");
    let request = || {
        browser_navigation_request(&url).map(|request| {
            request
                .with_page_network_policy()
                .with_browser_request_metadata(BrowserRequestMetadata::Fetch)
        })
    };

    let mut first = loader
        .fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
        .await?;
    assert!(!first.from_cache);
    let mut first_body = Vec::new();
    while let Some(chunk) = first.next_chunk().await {
        first_body.extend_from_slice(&chunk);
    }
    first.finish().await?;

    let mut second = loader
        .fetch_raw_stream_with_cancel(request()?, FetchCancelHandle::new())
        .await?;
    assert!(second.from_cache);
    let mut second_body = Vec::new();
    while let Some(chunk) = second.next_chunk().await {
        second_body.extend_from_slice(&chunk);
    }
    second.finish().await?;

    assert_eq!(first_body, b"cached-fetch-body");
    assert_eq!(second_body, b"cached-fetch-body");

    server.await?;
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn cached_raw_subresource_marks_redirect_hops_from_cache() -> Result<()> {
    let start_url = Url::parse("http://example.test/redirect.txt")?;
    let final_url = Url::parse("http://example.test/final.txt")?;
    let response = RawResponse::from_head_and_body(
        ResponseHead {
            status_text: None,
            final_url: final_url.clone(),
            status: 200,
            headers: vec![("content-type".to_owned(), b"text/plain".to_vec())],
            request_cookie_report: None,
            cookie_set_reports: Vec::new(),
            redirected: true,
            redirect_chain: vec![RedirectInfo {
                source: moli_fetch::RedirectSource::Network,
                from_url: start_url,
                to_url: final_url,
                status: 301,
                headers: vec![("location".to_owned(), b"/final.txt".to_vec())],
                network_extra_info_available: true,
                request_extra_info: None,
                response_extra_info: None,
                redirect_has_extra_info: true,
                request_cookie_report: None,
                cookie_set_reports: Vec::new(),
                from_cache: false,
                negotiated_http_version: None,
            }],
            from_cache: false,
            negotiated_http_version: None,
        },
        b"cached-final-body".to_vec(),
    );

    let mut response = streaming_raw_response_from_cached_subresource(response)?;

    assert!(response.from_cache);
    assert_eq!(response.redirect_chain.len(), 1);
    assert!(response.redirect_chain[0].from_cache);
    assert!(!response.redirect_chain[0].network_extra_info_available);
    let mut body = Vec::new();
    while let Some(chunk) = response.next_chunk().await {
        body.extend_from_slice(&chunk);
    }
    response.finish().await?;
    assert_eq!(body, b"cached-final-body");
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn loader_applies_network_policy_only_to_opt_in_requests() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut first, _) = listener.accept().await.unwrap();
        let first_request = read_http_request_text(&mut first).await.unwrap();
        let first_seen = first_request
            .lines()
            .find_map(|line| {
                let (name, value) = line.split_once(':')?;
                name.eq_ignore_ascii_case("x-cdp-test")
                    .then(|| value.trim().to_owned())
            })
            .unwrap_or_default();
        let first_body = format!("plain:{first_seen}");
        let first_response = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            first_body.len(),
            first_body
        );
        first.write_all(first_response.as_bytes()).await.unwrap();

        let (mut second, _) = listener.accept().await.unwrap();
        let second_request = read_http_request_text(&mut second).await.unwrap();
        let second_seen = second_request
            .lines()
            .find_map(|line| {
                let (name, value) = line.split_once(':')?;
                name.eq_ignore_ascii_case("x-cdp-test")
                    .then(|| value.trim().to_owned())
            })
            .unwrap_or_default();
        let second_body = format!("optin:{second_seen}");
        let second_response = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            second_body.len(),
            second_body
        );
        second.write_all(second_response.as_bytes()).await.unwrap();
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    loader.set_extra_http_headers(
        &vec![("x-cdp-test".to_owned(), "loader-policy".to_owned())].into(),
    );

    let plain = loader
        .fetch(browser_navigation_request(&format!("http://{addr}/plain"))?)
        .await?;
    assert_eq!(plain.body_text(), "plain:");

    let opt_in = loader
        .fetch(
            browser_navigation_request(&format!("http://{addr}/optin"))?.with_page_network_policy(),
        )
        .await?;
    assert_eq!(opt_in.body_text(), "optin:loader-policy");

    server.await?;
    Ok(())
}

#[test]
fn service_worker_bypass_isolated_between_page_request_clients() -> Result<()> {
    let shared = ResourceRequestClient::new(&FetchConfig::default())?;
    let first_page = shared.fork_with_isolated_page_network_policy();
    let second_page = shared.fork_with_isolated_page_network_policy();
    first_page.set_bypass_service_worker(true);
    second_page.set_bypass_service_worker(false);

    assert!(!shared.bypass_service_worker());
    assert!(first_page.bypass_service_worker());
    assert!(!second_page.bypass_service_worker());

    first_page.set_bypass_service_worker(false);
    second_page.set_bypass_service_worker(true);

    assert!(!shared.bypass_service_worker());
    assert!(!first_page.bypass_service_worker());
    assert!(second_page.bypass_service_worker());
    Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn loader_opt_in_network_policy_enforces_blocked_and_offline() -> Result<()> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let addr = listener.local_addr()?;
    let server = tokio::spawn(async move {
        let (mut stream, _) = listener.accept().await.unwrap();
        let _ = read_http_request_head(&mut stream).await;
        let body = "reachable";
        let response = format!(
            "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
            body.len(),
            body
        );
        stream.write_all(response.as_bytes()).await.unwrap();
    });

    let loader = ResourceRequestClient::new(&FetchConfig::default())?;
    let blocked_url = format!("http://{addr}/blocked");
    loader.set_blocked_url_patterns(std::slice::from_ref(&blocked_url));
    let blocked = loader
        .fetch(browser_navigation_request(&blocked_url)?.with_page_network_policy())
        .await
        .unwrap_err()
        .to_string();
    assert!(blocked.contains("net::ERR_BLOCKED_BY_CLIENT"));

    let plain = loader
        .fetch(browser_navigation_request(&format!("http://{addr}/plain"))?)
        .await?;
    assert_eq!(plain.body_text(), "reachable");

    loader.set_network_offline(true);
    let offline = loader
        .fetch(
            browser_navigation_request(&format!("http://{addr}/offline"))?
                .with_page_network_policy(),
        )
        .await
        .unwrap_err()
        .to_string();
    assert!(offline.contains("Network emulation offline"));

    server.await?;
    Ok(())
}