Skip to content

misterdownloadmanager: The ultimate open-source, clean-room download manager written from scratch in pure Rust for Windows, Mac, and Linux

15.4 KB
Raw Blame
//! #668: a site whose bot check refuses MDM's own connection (`Error::BotCheck`) is asked once
//! more the way a browser connects, and the download goes on through that connection only. The
//! lab server's `browser_only` fault stands in for the bot check: it refuses, with Cloudflare's
//! `cf-mitigated: challenge`, every request without the headers browsers send (`Sec-Fetch-Mode`).
#![cfg(feature = "browser-connection")]

use std::time::{Duration, Instant};

use mdm_engine::{Destination, Download, Options, Request};
use mdm_labserver::{Config, Faults, LabServer, LogEntry, content};
use reqwest::header::{COOKIE, HeaderMap, REFERER, USER_AGENT};

const SETTLE: Duration = Duration::from_secs(20);
const CHROME: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36";

async fn server(faults: &str) -> LabServer {
    LabServer::start(Config {
        faults: Faults::parse(faults).unwrap(),
        ..Config::default()
    })
    .await
    .unwrap()
}

fn opts() -> Options {
    Options {
        retry_delay: Duration::from_millis(50),
        user_agent: CHROME.to_owned(),
        ..Options::default()
    }
}

/// The browser's hand-over: its session cookie and the page as Referer.
fn handed_over() -> HeaderMap {
    let mut h = HeaderMap::new();
    h.insert(COOKIE, "sid=1".parse().unwrap());
    h.insert(REFERER, "http://127.0.0.1/app/".parse().unwrap());
    h
}

fn same_content(path: &std::path::Path, seed: u64, len: usize) {
    let data = std::fs::read(path).unwrap();
    let mut want = vec![0; len];
    content::fill(seed, 0, &mut want);
    assert!(data == want, "content mismatch");
}

async fn settled_log(s: &LabServer) -> Vec<LogEntry> {
    tokio::time::timeout(SETTLE, s.settled())
        .await
        .expect("every request ends");
    s.log()
}

#[tokio::test]
async fn a_bot_check_is_passed_the_way_a_browser_connects() {
    let s = server("browser_only=1&require_cookie=sid=1").await;
    let dir = tempfile::tempdir().unwrap();
    let dl = Download::start(
        Request {
            url: s.url("/gen/3M/app.apk?seed=668").parse().unwrap(),
            headers: handed_over(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        opts(),
    );
    let out = dl
        .wait()
        .await
        .expect("the browser-like connection gets through");
    same_content(&out.path, 668, 3 << 20);
    let log = settled_log(&s).await;
    // MDM's own request first, refused; then the browser-like connection's, one at a time.
    assert_eq!(log[0].outcome, "fault:browser_only", "{log:?}");
    assert!(log[0].sec_fetch_mode.is_none());
    let through: Vec<_> = log[1..].iter().collect();
    assert!(!through.is_empty());
    for e in &through {
        assert!(e.sec_fetch_mode.is_some(), "{e:?}");
        // The download's own User-Agent, the browser's cookie for its site, its Referer.
        assert_eq!(e.user_agent.as_deref(), Some(CHROME), "{e:?}");
        assert_eq!(e.cookie.as_deref(), Some("sid=1"), "{e:?}");
        assert_eq!(e.referer.as_deref(), Some("http://127.0.0.1/app/"), "{e:?}");
        assert!(e.status == 200 || e.status == 206, "{e:?}");
    }
    assert_eq!(
        log.iter()
            .filter(|e| e.outcome == "fault:browser_only")
            .count(),
        1,
        "asked once more, not again and again: {log:?}"
    );
}

#[tokio::test]
async fn the_user_agent_handed_over_is_the_one_sent() {
    let s = server("browser_only=1").await;
    let dir = tempfile::tempdir().unwrap();
    let firefox = "Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0";
    let mut headers = HeaderMap::new();
    headers.insert(USER_AGENT, firefox.parse().unwrap());
    let out = Download::start(
        Request {
            url: s.url("/gen/256K/f.bin?seed=2").parse().unwrap(),
            headers,
            dest: Destination::Dir(dir.path().to_owned()),
        },
        opts(),
    )
    .wait()
    .await
    .unwrap();
    same_content(&out.path, 2, 256 << 10);
    let log = settled_log(&s).await;
    assert!(
        log.iter()
            .filter(|e| e.sec_fetch_mode.is_some())
            .all(|e| e.user_agent.as_deref() == Some(firefox)),
        "{log:?}"
    );
}

#[tokio::test]
async fn cookies_stay_with_their_site_through_redirects() {
    let s = server("browser_only=1").await;
    let port = s.addr().port();
    // 127.0.0.1 redirects to localhost: another host, as when a site hands off to a CDN.
    let target = format!("http://localhost:{port}/gen/1M/r.bin?seed=3");
    let enc: String = target
        .chars()
        .map(|c| match c {
            ':' => "%3A".to_owned(),
            '/' => "%2F".to_owned(),
            '?' => "%3F".to_owned(),
            '=' => "%3D".to_owned(),
            c => c.to_string(),
        })
        .collect();
    let dir = tempfile::tempdir().unwrap();
    let out = Download::start(
        Request {
            url: s.url(&format!("/redirect?to={enc}")).parse().unwrap(),
            headers: handed_over(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        opts(),
    )
    .wait()
    .await
    .unwrap();
    same_content(&out.path, 3, 1 << 20);
    let log = settled_log(&s).await;
    assert!(log.iter().any(|e| e.sec_fetch_mode.is_some()), "{log:?}");
    assert!(
        log.iter().all(|e| e.cookie.is_none()),
        "the cookie for 127.0.0.1 reached localhost: {log:?}"
    );
}

#[tokio::test]
async fn a_stopped_download_resumes_through_the_same_connection() {
    let s = server("browser_only=1&rate=1M&stall_at=1M").await;
    let dir = tempfile::tempdir().unwrap();
    let dl = Download::start(
        Request {
            url: s.url("/gen/3M/r.bin?seed=7").parse().unwrap(),
            headers: handed_over(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        Options {
            save_interval: Duration::from_millis(100),
            ..opts()
        },
    );
    let deadline = Instant::now() + Duration::from_secs(20);
    let file = loop {
        tokio::time::sleep(Duration::from_millis(50)).await;
        let snap = dl.snapshot();
        if let Some(f) = snap.file.filter(|_| snap.downloaded >= 256 << 10) {
            break f;
        }
        assert!(Instant::now() < deadline, "the download never started");
    };
    dl.cancel();
    assert!(matches!(dl.wait().await, Err(mdm_engine::Error::Cancelled)));
    s.set_faults(Faults::parse("browser_only=1&rate=4M").unwrap());
    tokio::time::timeout(SETTLE, s.settled()).await.unwrap();
    let state = mdm_engine::state::state_path(&file);
    let saved = mdm_engine::state::ResumeState::load(&state).unwrap();
    assert!(saved.browser_like, "the state says how it got through");
    s.clear_log();
    let out = Download::resume(state.clone(), handed_over(), opts())
        .unwrap()
        .wait()
        .await
        .expect("the resume goes on");
    same_content(&out.path, 7, 3 << 20);
    let log = settled_log(&s).await;
    // Straight through the browser-like connection, from where it stopped, its bytes as they
    // are (no compression asked for a range).
    assert!(!log.is_empty());
    for e in &log {
        assert!(e.sec_fetch_mode.is_some(), "{e:?}");
        assert_eq!(e.status, 206, "{e:?}");
        assert!(
            e.range
                .as_deref()
                .is_some_and(|r| !r.starts_with("bytes=0-")),
            "{e:?}"
        );
        assert_eq!(e.accept_encoding.as_deref(), Some("identity"), "{e:?}");
    }
}

#[tokio::test]
async fn the_speed_limit_holds() {
    let s = server("browser_only=1").await;
    let dir = tempfile::tempdir().unwrap();
    let started = Instant::now();
    let out = Download::start(
        Request {
            url: s.url("/gen/1M/s.bin?seed=9").parse().unwrap(),
            headers: Default::default(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        Options {
            speed_limit: Some(512 << 10),
            ..opts()
        },
    )
    .wait()
    .await
    .unwrap();
    same_content(&out.path, 9, 1 << 20);
    // 1 MiB at 512 KiB/s: two seconds, one at least with the limiter's first second of burst.
    assert!(
        started.elapsed() >= Duration::from_millis(900),
        "{:?}",
        started.elapsed()
    );
}

#[tokio::test]
async fn a_look_gets_past_the_bot_check_too() {
    let s = server("browser_only=1").await;
    let info = mdm_engine::probe(
        s.url("/gen/2M/look.apk?seed=1").parse().unwrap(),
        Default::default(),
        &opts(),
    )
    .await
    .expect("the look gets through");
    assert_eq!(info.size, Some(2 << 20));
    assert_eq!(info.file_name, "look.apk");
}

#[tokio::test]
async fn stop_ends_it() {
    let s = server("browser_only=1&rate=256K").await;
    let dir = tempfile::tempdir().unwrap();
    let dl = Download::start(
        Request {
            url: s.url("/gen/8M/c.bin?seed=4").parse().unwrap(),
            headers: Default::default(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        opts(),
    );
    let deadline = Instant::now() + Duration::from_secs(20);
    while dl.snapshot().downloaded == 0 {
        assert!(Instant::now() < deadline, "never started");
        tokio::time::sleep(Duration::from_millis(20)).await;
    }
    dl.cancel();
    let stopped = tokio::time::timeout(Duration::from_secs(5), dl.wait())
        .await
        .expect("stop is prompt");
    assert!(matches!(stopped, Err(mdm_engine::Error::Cancelled)));
}

/// Over HTTPS the lab's bot check looks at the TLS handshake itself, as real ones do: a plain
/// client that sends a browser's headers is still refused; MDM's own request is refused, and the
/// browser-like connection, whose handshake is Chrome's, gets the file.
#[tokio::test]
async fn over_https_the_handshake_itself_is_what_gets_through() {
    let s = LabServer::start(Config {
        faults: Faults::parse("browser_only=1").unwrap(),
        tls: true,
        ..Config::default()
    })
    .await
    .unwrap();
    let ca = s.ca_der().unwrap();
    // A program's TLS with every header a browser sends: refused all the same.
    let plain = reqwest::Client::builder()
        .tls_certs_merge([reqwest::Certificate::from_der(&ca).unwrap()])
        .build()
        .unwrap();
    let refused = plain
        .get(s.url("/gen/64K/x.bin?seed=1"))
        .header(USER_AGENT, CHROME)
        .header("Sec-Fetch-Mode", "navigate")
        .send()
        .await
        .unwrap();
    assert_eq!(refused.status(), 403);
    assert_eq!(refused.headers()["cf-mitigated"], "challenge");
    s.clear_log();

    let dir = tempfile::tempdir().unwrap();
    let out = Download::start(
        Request {
            url: s.url("/gen/2M/app.apk?seed=21").parse().unwrap(),
            headers: handed_over(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        Options {
            extra_roots: vec![ca],
            ..opts()
        },
    )
    .wait()
    .await
    .expect("the browser-like connection gets through");
    same_content(&out.path, 21, 2 << 20);
    let log = settled_log(&s).await;
    assert_eq!(log[0].outcome, "fault:browser_only", "{log:?}");
    assert_eq!(log[0].browser_tls, Some(false), "{log:?}");
    assert!(log.len() >= 2, "{log:?}");
    for e in &log[1..] {
        assert_eq!(e.browser_tls, Some(true), "{e:?}");
        assert!(e.status == 200 || e.status == 206, "{e:?}");
    }
}

/// The file's information window looks first (`probe_kept`): its browser-like answer is kept
/// for the download, so a click costs one refused request and one that gets through, no more.
#[tokio::test]
async fn a_look_and_its_download_ask_once_more_in_all() {
    let s = server("browser_only=1").await;
    let url: url::Url = s.url("/gen/1M/kept.apk?seed=5").parse().unwrap();
    let (info, kept) = mdm_engine::probe_kept(url.clone(), handed_over(), &opts())
        .await
        .expect("the look gets through");
    assert_eq!(info.size, Some(1 << 20));
    assert!(kept.is_some(), "the browser-like answer is kept");
    let dir = tempfile::tempdir().unwrap();
    let out = Download::start_with(
        Request {
            url,
            headers: handed_over(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        opts(),
        kept,
    )
    .wait()
    .await
    .unwrap();
    same_content(&out.path, 5, 1 << 20);
    let log = settled_log(&s).await;
    assert_eq!(log.len(), 2, "{log:?}");
    assert_eq!(log[0].outcome, "fault:browser_only");
    assert!(
        log[1].sec_fetch_mode.is_some() && log[1].status == 200,
        "{log:?}"
    );
}

/// Options → Proxy: the browser-like connection goes through the proxy too. The lab server
/// stands in for the proxy (it answers by path), so a host that can't exist only works there.
#[tokio::test]
async fn it_goes_through_the_proxy() {
    let s = server("browser_only=1").await;
    let dir = tempfile::tempdir().unwrap();
    let out = Download::start(
        Request {
            url: "http://nowhere.invalid/gen/512K/p.bin?seed=8"
                .parse()
                .unwrap(),
            headers: Default::default(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        Options {
            proxy: mdm_engine::Proxy::Manual {
                url: s.url(""),
                login: None,
                http: true,
                https: true,
            },
            ..opts()
        },
    )
    .wait()
    .await
    .expect("downloaded through the proxy");
    same_content(&out.path, 8, 512 << 10);
    let log = settled_log(&s).await;
    assert_eq!(log[0].outcome, "fault:browser_only", "{log:?}");
    assert!(
        log[1..].iter().all(|e| e.sec_fetch_mode.is_some()),
        "{log:?}"
    );
}

/// A site that refuses the browser-like connection too (a challenge to solve, say): the
/// download fails with the bot check's refusal, as before, after one more request only.
#[tokio::test]
async fn a_second_refusal_stays_the_bot_check() {
    let s = server("browser_only=1&refuse_agent=Chrome").await;
    let dir = tempfile::tempdir().unwrap();
    let err = Download::start(
        Request {
            url: s.url("/gen/64K/no.bin?seed=1").parse().unwrap(),
            headers: Default::default(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        opts(),
    )
    .wait()
    .await
    .unwrap_err();
    assert!(matches!(err, mdm_engine::Error::BotCheck), "{err:?}");
    let log = settled_log(&s).await;
    assert_eq!(log.len(), 2, "{log:?}");
}

/// FileHippo's way (#630): 412 Precondition Failed, empty, to whatever isn't a browser. MDM's
/// first request carries no precondition, so that 412 can only refuse the program: it is asked
/// once more the way a browser connects too, and the file comes.
#[tokio::test]
async fn a_412_to_a_request_without_preconditions_is_asked_once_more_too() {
    let s = server("refuse_programs=412").await;
    let dir = tempfile::tempdir().unwrap();
    let out = Download::start(
        Request {
            url: s.url("/gen/1M/winrar-x64.exe?seed=630").parse().unwrap(),
            headers: handed_over(),
            dest: Destination::Dir(dir.path().to_owned()),
        },
        opts(),
    )
    .wait()
    .await
    .expect("the browser-like connection gets through");
    same_content(&out.path, 630, 1 << 20);
    let log = settled_log(&s).await;
    assert_eq!(log[0].status, 412, "{log:?}");
    assert_eq!(log.iter().filter(|e| e.status == 412).count(), 1, "{log:?}");
    assert!(
        log[1..].iter().all(|e| e.sec_fetch_mode.is_some()),
        "{log:?}"
    );
}