Skip to content

misterdownloadmanager: The ultimate open-source, clean-room download manager written from scratch in pure Rust for Windows, Mac, and Linux

9.5 KB
Raw Blame
//! Whole-file checksums (beyond IDM, D-015): verify a finished download when its hash is known —
//! given by the user, or announced by the server (`Digest`, `Repr-Digest`, `Content-MD5`,
//! `x-goog-hash`). The overlap check catches a *changed* file; this catches a *corrupt* one.

use std::io::Read;
use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};

use base64::Engine as _;
use reqwest::header::HeaderMap;
use serde::{Deserialize, Serialize};
use sha2::Digest as _;

#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum Algo {
    Crc32,
    Md5,
    Sha1,
    Sha256,
    Sha512,
}

impl Algo {
    pub fn name(self) -> &'static str {
        match self {
            Algo::Crc32 => "CRC32",
            Algo::Md5 => "MD5",
            Algo::Sha1 => "SHA-1",
            Algo::Sha256 => "SHA-256",
            Algo::Sha512 => "SHA-512",
        }
    }
}

/// An expected hash.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Hash {
    pub algo: Algo,
    #[serde(with = "hex_bytes")]
    pub value: Vec<u8>,
}

impl Hash {
    /// A hash typed or pasted by the user: hex, optionally `algo:` / `algo=` first
    /// (`sha256:…`, `md5=…`); without a name the length decides (8 CRC32, 32 MD5, 40 SHA-1,
    /// 64 SHA-256, 128 SHA-512). Spaces and a trailing file name (`… *file.iso`) are ignored.
    pub fn parse(text: &str) -> Option<Hash> {
        let text = text.trim();
        let (name, rest) = match text.split_once([':', '=']) {
            Some((n, r)) if n.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') => {
                (Some(n.to_ascii_lowercase().replace('-', "")), r)
            }
            _ => (None, text),
        };
        let digits: String = rest
            .split_whitespace()
            .next()?
            .chars()
            .filter(|c| !c.is_whitespace())
            .collect();
        let value = hex::decode(&digits).ok()?;
        let algo = match name.as_deref() {
            Some("crc32") => Algo::Crc32,
            Some("md5") => Algo::Md5,
            Some("sha1") => Algo::Sha1,
            Some("sha256") => Algo::Sha256,
            Some("sha512") => Algo::Sha512,
            Some(_) => return None,
            None => match value.len() {
                4 => Algo::Crc32,
                16 => Algo::Md5,
                20 => Algo::Sha1,
                32 => Algo::Sha256,
                64 => Algo::Sha512,
                _ => return None,
            },
        };
        (value.len() == digest_len(algo)).then_some(Hash { algo, value })
    }

    /// The strongest hash a full (200) response announces for its content, if any.
    pub fn from_headers(h: &HeaderMap) -> Option<Hash> {
        let b64 = |s: &str| {
            base64::engine::general_purpose::STANDARD
                .decode(s.trim().trim_matches(':'))
                .ok()
        };
        let mut found: Vec<Hash> = Vec::new();
        // RFC 9530 `Repr-Digest: sha-256=:…:`, RFC 3230 `Digest: SHA-256=…`.
        for name in ["repr-digest", "digest"] {
            for v in h.get_all(name).iter().filter_map(|v| v.to_str().ok()) {
                for part in v.split(',') {
                    let Some((algo, value)) = part.trim().split_once('=') else {
                        continue;
                    };
                    let algo = match algo.trim().to_ascii_lowercase().as_str() {
                        "sha-512" => Algo::Sha512,
                        "sha-256" => Algo::Sha256,
                        "sha" | "sha-1" => Algo::Sha1,
                        "md5" => Algo::Md5,
                        _ => continue,
                    };
                    if let Some(value) = b64(value).filter(|v| v.len() == digest_len(algo)) {
                        found.push(Hash { algo, value });
                    }
                }
            }
        }
        if let Some(value) = h
            .get("content-md5")
            .and_then(|v| v.to_str().ok())
            .and_then(b64)
            .filter(|v| v.len() == 16)
        {
            found.push(Hash {
                algo: Algo::Md5,
                value,
            });
        }
        // Google Cloud Storage: `x-goog-hash: crc32c=…, md5=…` (crc32c is not CRC32).
        for v in h
            .get_all("x-goog-hash")
            .iter()
            .filter_map(|v| v.to_str().ok())
        {
            for part in v.split(',') {
                if let Some(value) = part.trim().strip_prefix("md5=").and_then(b64)
                    && value.len() == 16
                {
                    found.push(Hash {
                        algo: Algo::Md5,
                        value,
                    });
                }
            }
        }
        found.into_iter().max_by_key(|h| strength(h.algo))
    }

    /// The hash as text (`sha256:…`).
    pub fn display(&self) -> String {
        format!(
            "{}:{}",
            self.algo.name().to_ascii_lowercase().replace('-', ""),
            hex::encode(&self.value)
        )
    }
}

fn digest_len(a: Algo) -> usize {
    match a {
        Algo::Crc32 => 4,
        Algo::Md5 => 16,
        Algo::Sha1 => 20,
        Algo::Sha256 => 32,
        Algo::Sha512 => 64,
    }
}

fn strength(a: Algo) -> u8 {
    match a {
        Algo::Crc32 => 0,
        Algo::Md5 => 1,
        Algo::Sha1 => 2,
        Algo::Sha256 => 3,
        Algo::Sha512 => 4,
    }
}

/// The file's hash with `algo` (reads the whole file; call off the async threads).
pub fn digest(path: &Path, algo: Algo) -> std::io::Result<Vec<u8>> {
    digest_counted(path, algo, &AtomicU64::new(0))
}

/// [`digest`], adding each part read to `read` as it goes (#736: how far a check has come).
pub fn digest_counted(path: &Path, algo: Algo, read: &AtomicU64) -> std::io::Result<Vec<u8>> {
    let mut f = std::fs::File::open(path)?;
    let mut buf = vec![0u8; 1 << 20];
    macro_rules! run {
        ($h:expr) => {{
            let mut h = $h;
            loop {
                let n = f.read(&mut buf)?;
                if n == 0 {
                    break;
                }
                h.update(&buf[..n]);
                read.fetch_add(n as u64, Ordering::Relaxed);
            }
            h.finalize().to_vec()
        }};
    }
    Ok(match algo {
        Algo::Crc32 => {
            let mut h = crc32fast::Hasher::new();
            loop {
                let n = f.read(&mut buf)?;
                if n == 0 {
                    break;
                }
                h.update(&buf[..n]);
                read.fetch_add(n as u64, Ordering::Relaxed);
            }
            h.finalize().to_be_bytes().to_vec()
        }
        Algo::Md5 => run!(md5::Md5::new()),
        Algo::Sha1 => run!(sha1::Sha1::new()),
        Algo::Sha256 => run!(sha2::Sha256::new()),
        Algo::Sha512 => run!(sha2::Sha512::new()),
    })
}

/// Whether the file matches `hash`.
pub fn matches(path: &Path, hash: &Hash) -> std::io::Result<bool> {
    matches_counted(path, hash, &AtomicU64::new(0))
}

/// [`matches`], counting the bytes checked into `read` as it goes.
pub fn matches_counted(path: &Path, hash: &Hash, read: &AtomicU64) -> std::io::Result<bool> {
    Ok(digest_counted(path, hash.algo, read)? == hash.value)
}

mod hex_bytes {
    use serde::{Deserialize, Deserializer, Serializer};

    pub fn serialize<S: Serializer>(v: &[u8], s: S) -> Result<S::Ok, S::Error> {
        s.serialize_str(&hex::encode(v))
    }

    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
        let s = String::deserialize(d)?;
        hex::decode(s).map_err(serde::de::Error::custom)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn user_hashes_by_length_or_name() {
        let md5 = Hash::parse("9e107d9d372bb6826bd81d3542a419d6").unwrap();
        assert_eq!(md5.algo, Algo::Md5);
        let sha = Hash::parse(
            "SHA256: d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592  *x.iso",
        )
        .unwrap();
        assert_eq!(sha.algo, Algo::Sha256);
        assert!(Hash::parse("md5:abc").is_none());
        assert!(Hash::parse("not a hash").is_none());
    }

    #[test]
    fn server_announced_hashes() {
        let mut h = HeaderMap::new();
        h.insert("content-md5", "nhB9nTcrtoJr2B01QqQZ1g==".parse().unwrap());
        h.insert(
            "repr-digest",
            "sha-256=:16j7swfXgJRpypq8sAguT41WUeRtPNt2LQLQvzfJ5ZI=:"
                .parse()
                .unwrap(),
        );
        assert_eq!(
            Hash::from_headers(&h).unwrap().algo,
            Algo::Sha256,
            "the strongest"
        );
        h.remove("repr-digest");
        assert_eq!(Hash::from_headers(&h).unwrap().algo, Algo::Md5);
    }

    #[test]
    fn digests_of_a_file() {
        let dir = tempfile::tempdir().unwrap();
        let p = dir.path().join("fox.txt");
        std::fs::write(&p, b"The quick brown fox jumps over the lazy dog").unwrap();
        let sha = Hash::parse("d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592")
            .unwrap();
        assert!(matches(&p, &sha).unwrap());
        let md5 = Hash::parse("9e107d9d372bb6826bd81d3542a419d6").unwrap();
        assert!(matches(&p, &md5).unwrap());
        let crc = Hash::parse("crc32:414fa339").unwrap();
        assert!(matches(&p, &crc).unwrap());
        let wrong = Hash::parse("00000000000000000000000000000000").unwrap();
        assert!(!matches(&p, &wrong).unwrap());
        // #736: every byte checked is counted.
        let read = AtomicU64::new(0);
        assert!(matches_counted(&p, &sha, &read).unwrap());
        assert_eq!(read.load(Ordering::Relaxed), 43);
    }
}