Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

3.7 KB
Raw Blame
use std::str;

use moli_http_cache::{
    HttpCacheVaryHeader, request_cache_control_forbids_storage, request_header_requires_validation,
    response_vary_header_names,
};
use url::Url;

use crate::{FetchConfig, Request};

use super::super::outgoing_request_headers_for_url;

const SUPPORTED_CACHE_REQUEST_HEADERS: &[&str] = &[
    "accept",
    "accept-encoding",
    "accept-language",
    "referer",
    "sec-ch-ua",
    "sec-ch-ua-arch",
    "sec-ch-ua-bitness",
    "sec-ch-ua-form-factors",
    "sec-ch-ua-full-version",
    "sec-ch-ua-full-version-list",
    "sec-ch-ua-mobile",
    "sec-ch-ua-model",
    "sec-ch-ua-platform",
    "sec-ch-ua-platform-version",
    "sec-ch-ua-wow64",
    "sec-fetch-dest",
    "sec-fetch-mode",
    "sec-fetch-site",
    "sec-fetch-user",
    "upgrade-insecure-requests",
    "user-agent",
];

pub(super) fn vary_headers_for_response(
    config: &FetchConfig,
    request: &Request,
    request_url: &Url,
    response_headers: &[(String, Vec<u8>)],
) -> Option<Vec<HttpCacheVaryHeader>> {
    let mut out = Vec::new();
    for normalized_name in response_vary_header_names(response_headers)? {
        if !supported_vary_header(&normalized_name) {
            return None;
        }
        out.push(HttpCacheVaryHeader {
            value: vary_request_header_value(config, request, request_url, &normalized_name),
            name: normalized_name,
        });
    }
    Some(out)
}

pub(super) fn vary_headers_match(
    config: &FetchConfig,
    request: &Request,
    request_url: &Url,
    vary_headers: &[HttpCacheVaryHeader],
) -> bool {
    vary_headers.iter().all(|header| {
        vary_request_header_value(config, request, request_url, &header.name) == header.value
    })
}

pub(super) fn request_headers_allow_http_cache(headers: &[(String, String)]) -> bool {
    headers.iter().enumerate().all(|(index, (name, value))| {
        // Revalidation controls select the existing conditional-cache path;
        // they must not turn a fresh entry into an unconditional cache miss.
        // Other unsupported controls keep the conservative bypass policy.
        (supported_cache_request_header(name) || request_header_requires_validation(name, value))
            && !(name.eq_ignore_ascii_case("cache-control")
                && request_cache_control_forbids_storage(value))
            && !headers[..index]
                .iter()
                .any(|(previous, _)| previous.eq_ignore_ascii_case(name))
    })
}

fn vary_request_header_value(
    config: &FetchConfig,
    request: &Request,
    request_url: &Url,
    normalized_name: &str,
) -> Option<String> {
    let explicit_value = || {
        outgoing_request_headers_for_url(config, request, request_url, None)
            .into_iter()
            .rev()
            .find(|(name, _)| name.eq_ignore_ascii_case(normalized_name))
            .map(|(_, value)| value)
    };
    if normalized_name == "user-agent" {
        return explicit_value()
            .or_else(|| Some(request.browser_identity(config).user_agent().to_owned()));
    }
    if normalized_name == "accept-encoding" {
        // libcurl synthesizes this header from `easy.accept_encoding("")`.
        return explicit_value().or_else(|| Some("libcurl-auto".to_owned()));
    }
    explicit_value()
}

fn supported_vary_header(normalized_name: &str) -> bool {
    supported_cache_request_header(normalized_name)
}

fn supported_cache_request_header(name: &str) -> bool {
    // Keep the allowlist to non-sensitive browser headers whose effective
    // values this module can reproduce for Vary matching. Arbitrary custom,
    // cookie, authorization, and range headers remain cache-ineligible.
    SUPPORTED_CACHE_REQUEST_HEADERS
        .iter()
        .any(|supported| name.eq_ignore_ascii_case(supported))
}