Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

10.5 KB
Raw Blame
mod key;
mod lookup;
mod metadata;
mod revalidation;
mod vary;

use anyhow::{Context, Result, anyhow, bail};
use moli_http_cache::{
    HttpCacheEntryMetadata, HttpCacheStats, HttpCacheStore, cached_response_is_stale,
    request_header_requires_validation,
};
use moli_url_policy::ensure_http_network_transport_url;
use url::Url;

use std::path::Path;

use crate::{FetchConfig, Request};

use super::outgoing_request_headers_for_url;

pub fn clear_http_cache(config: &FetchConfig) -> Result<()> {
    let Some(cache_dir) = config.http_cache_dir() else {
        return Ok(());
    };
    clear_http_cache_root(cache_dir, config.http_cache_max_bytes())
}

pub fn clear_http_cache_for_origin(config: &FetchConfig, origin: &Url) -> Result<usize> {
    let Some(cache_dir) = config.http_cache_dir() else {
        return Ok(0);
    };
    clear_http_cache_root_for_origin(cache_dir, config.http_cache_max_bytes(), origin)
}

pub fn clear_http_cache_root(cache_dir: impl AsRef<Path>, max_bytes: Option<u64>) -> Result<()> {
    HttpCacheStore::with_max_bytes(cache_dir.as_ref(), max_bytes).clear()
}

pub fn clear_http_cache_root_for_origin(
    cache_dir: impl AsRef<Path>,
    max_bytes: Option<u64>,
    origin: &Url,
) -> Result<usize> {
    HttpCacheStore::with_max_bytes(cache_dir.as_ref(), max_bytes).remove_entries_for_origin(origin)
}

pub fn trim_http_cache(config: &FetchConfig) -> Result<()> {
    let Some(cache_dir) = config.http_cache_dir() else {
        return Ok(());
    };
    HttpCacheStore::with_max_bytes(
        std::path::Path::new(cache_dir),
        config.http_cache_max_bytes(),
    )
    .trim_to_max_bytes();
    Ok(())
}

pub fn http_cache_stats(config: &FetchConfig) -> Result<HttpCacheStats> {
    let Some(cache_dir) = config.http_cache_dir() else {
        return Ok(HttpCacheStats::default());
    };
    HttpCacheStore::with_max_bytes(
        std::path::Path::new(cache_dir),
        config.http_cache_max_bytes(),
    )
    .stats()
}

pub(crate) use self::{
    lookup::{
        cached_streaming_response_body_exceeds_response_limit,
        create_streaming_cache_body_writer_for_response_parts, finish_streaming_cached_response,
        load_cached_streaming_response_lookup, remove_cached_response,
    },
    metadata::response_headers_forbid_cache_storage,
    revalidation::{
        merge_cached_not_modified_streaming_response_lookup,
        validation_headers_for_cached_streaming_response_lookup,
    },
};

#[derive(Debug)]
pub(crate) struct CachedStreamingResponseLookup {
    pub(crate) key: String,
    pub(crate) metadata: HttpCacheEntryMetadata,
    pub(crate) final_url: String,
    pub(crate) status: u16,
    pub(crate) headers: Vec<(String, Vec<u8>)>,
    pub(crate) body: std::io::BufReader<std::fs::File>,
    pub(crate) expires_at_unix_ms: Option<u64>,
    pub(crate) force_validate: bool,
}

pub(crate) fn cached_streaming_response_is_stale(record: &CachedStreamingResponseLookup) -> bool {
    cached_response_is_stale(record.expires_at_unix_ms, record.force_validate)
}

pub(super) fn request_cache_control_requires_validation(
    config: &FetchConfig,
    request: &Request,
    request_url: &Url,
) -> bool {
    outgoing_request_headers_for_url(config, request, request_url, None)
        .iter()
        .any(|(name, value)| request_header_requires_validation(name, value))
}

pub(crate) fn next_redirect_url_from_parts(
    final_url: &Url,
    status: u16,
    headers: &[(String, Vec<u8>)],
    redirect_count: usize,
) -> Result<Option<Url>> {
    if !matches!(status, 301 | 302 | 303 | 307 | 308) {
        return Ok(None);
    }

    let Some(location) = headers
        .iter()
        .find(|(name, _)| name == "location")
        .map(|(_, value)| crate::decode_header_value(value))
    else {
        return Ok(None);
    };

    if redirect_count >= super::MAX_REDIRECTS {
        bail!("redirect limit exceeded for {}", final_url);
    }

    final_url
        .join(&location)
        .or_else(|_| Url::parse(&location))
        .map(Some)
        .with_context(|| {
            anyhow!(
                "failed to resolve redirect location `{location}` from {}",
                final_url
            )
        })
}

pub(crate) fn next_followed_redirect_url_from_parts(
    request: &Request,
    final_url: &Url,
    status: u16,
    headers: &[(String, Vec<u8>)],
    redirect_count: usize,
) -> Result<Option<Url>> {
    // HTTP fetch checks the network response before HTTP-redirect fetch. This
    // also applies to cached redirects and manual/error redirect modes.
    if matches!(status, 301 | 302 | 303 | 307 | 308) {
        request
            .validate_cors_response_for_url(final_url, headers)
            .map_err(anyhow::Error::msg)?;
    }
    // A browser may follow redirects itself, preserving the full URL list for
    // CORS and Fetch Metadata while asking this transport for only one hop.
    // Only automatically followed transport requests consume this layer's limit.
    let transport_redirect_count = if request.transport_follows_redirects() {
        redirect_count
    } else {
        0
    };
    let next_url =
        next_redirect_url_from_parts(final_url, status, headers, transport_redirect_count)?;
    if request.transport_follows_redirects()
        && let Some(next_url) = next_url.as_ref()
    {
        ensure_http_network_transport_url(next_url)?;
        request.validate_request_mode_for_url(next_url)?;
    }
    Ok(next_url)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn cache_key_includes_browser_partition_context() -> Result<()> {
        let request_url = Url::parse("https://cdn.example.test/app.css")?;
        let first_top_frame = Url::parse("https://app.example.test/")?;
        let second_top_frame = Url::parse("https://other.example.test/")?;
        let first =
            Request::get_with_url(request_url.clone()).with_top_frame_origin_url(&first_top_frame);
        let second =
            Request::get_with_url(request_url.clone()).with_top_frame_origin_url(&second_top_frame);

        assert_ne!(
            key::cache_key_for_request(&first, &request_url),
            key::cache_key_for_request(&second, &request_url)
        );
        Ok(())
    }

    #[test]
    fn followed_redirect_rejects_non_http_target_before_request_state_changes() -> Result<()> {
        let current = Url::parse("https://example.test/start")?;
        let headers: Vec<(String, Vec<u8>)> = vec![(
            "location".to_owned(),
            b"file:///moli-policy-must-not-open".to_vec(),
        )];

        let request = Request::get_with_url(current.clone());
        let error = next_followed_redirect_url_from_parts(&request, &current, 302, &headers, 0)
            .expect_err("followed file redirect must be rejected");
        assert_eq!(
            error.to_string(),
            "URL scheme \"file\" is not supported by the HTTP network transport."
        );

        let request = request.with_redirect_mode(crate::RequestRedirectMode::Manual);
        let manual = next_followed_redirect_url_from_parts(&request, &current, 302, &headers, 0)?
            .expect("manual redirect URL should remain observable");
        assert_eq!(manual.as_str(), "file:///moli-policy-must-not-open");
        Ok(())
    }

    #[test]
    fn browser_followed_hops_preserve_cors_without_reapplying_transport_redirect_limit()
    -> Result<()> {
        let origin_url = Url::parse("https://origin.test/page")?;
        let current = Url::parse("https://other.test/redirect")?;
        let request = Request::new_browser(
            "GET",
            current.clone(),
            None,
            Vec::new(),
            moli_url::WebOrigin::from_url(&origin_url),
        )
        .with_single_hop();
        let headers = vec![
            ("location".to_owned(), b"/next".to_vec()),
            (
                "access-control-allow-origin".to_owned(),
                b"https://origin.test".to_vec(),
            ),
            (
                "access-control-allow-credentials".to_owned(),
                b"true".to_vec(),
            ),
        ];
        let next = next_followed_redirect_url_from_parts(
            &request,
            &current,
            302,
            &headers,
            super::super::MAX_REDIRECTS,
        )?;
        assert_eq!(next, Some(current.join("/next")?));
        let automatic = request
            .clone()
            .with_redirect_mode(crate::RequestRedirectMode::Follow);
        let error = next_followed_redirect_url_from_parts(
            &automatic,
            &current,
            302,
            &headers,
            super::super::MAX_REDIRECTS,
        )
        .expect_err("automatically followed transport requests retain their redirect limit");
        assert!(error.to_string().contains("redirect limit exceeded"));
        let denied = next_followed_redirect_url_from_parts(
            &request,
            &current,
            302,
            &headers[..1],
            super::super::MAX_REDIRECTS,
        )
        .expect_err("a manually followed browser hop still requires CORS permission");
        assert!(denied.to_string().contains("CORS"));
        Ok(())
    }

    #[test]
    fn cache_key_includes_network_partition_key() -> Result<()> {
        let request_url = Url::parse("https://cdn.example.test/app.css")?;
        let first = Request::get_with_url(request_url.clone()).with_network_partition_key(Some(
            "storage-key:v1;origin=https://cdn.example.test;top-level-site=https://app.example.test;opaque-nonce=1"
                .to_owned(),
        ));
        let second = Request::get_with_url(request_url.clone()).with_network_partition_key(Some(
            "storage-key:v1;origin=https://cdn.example.test;top-level-site=https://app.example.test;opaque-nonce=2"
                .to_owned(),
        ));

        assert_ne!(
            key::cache_key_for_request(&first, &request_url),
            key::cache_key_for_request(&second, &request_url)
        );
        Ok(())
    }

    #[test]
    fn cache_key_ignores_fragment_and_userinfo() -> Result<()> {
        let first_url = Url::parse("https://user:[email protected]/app.css#one")?;
        let second_url = Url::parse("https://cdn.example.test/app.css#two")?;
        let first = Request::get_with_url(first_url.clone());
        let second = Request::get_with_url(second_url.clone());

        assert_eq!(
            key::cache_key_for_request(&first, &first_url),
            key::cache_key_for_request(&second, &second_url)
        );
        assert_eq!(
            key::normalized_cache_url_string(&first_url),
            "https://cdn.example.test/app.css"
        );
        Ok(())
    }
}