Skip to content

misterdownloadmanager: The ultimate open-source, clean-room download manager written from scratch in pure Rust for Windows, Mac, and Linux

4.8 KB
Raw Blame
//! Sparse part files (D-008 amendment, 2026-10-29).
//!
//! Extending a file with `set_len` and then writing far into it makes NTFS zero-fill everything
//! before the write, synchronously: one 1 MiB write at 2 GiB took 4.3 s on the user's D: drive
//! (and doubles the bytes written). The planner writes at the middle of the file within the
//! first second, so every large download paid it. A sparse file has no such fill. Elsewhere
//! (`ftruncate` already makes sparse files) this is a no-op.
//!
//! The engine's unsafe code: one `DeviceIoControl` call here (and, in its test, one
//! `GetCompressedFileSizeW`), and the free-space probe's calls in `crate::disk`: two on Windows,
//! one on macOS (with its CoreFoundation key, and the release of what it returns).
#![allow(unsafe_code)]

use std::fs::File;

/// Smaller files aren't worth it: the fill is short.
pub const MIN_SIZE: u64 = 5 << 20;

/// Mark `file` sparse (before `set_len`) or, once every byte is written, normal again (clearing
/// allocates any holes still there). False where the file system can't (FAT, exFAT): the
/// download then works as before.
pub fn set(file: &File, sparse: bool) -> bool {
    imp::set(file, sparse)
}

#[cfg(windows)]
mod imp {
    use std::os::windows::io::AsRawHandle;

    use windows_sys::Win32::System::IO::DeviceIoControl;
    use windows_sys::Win32::System::Ioctl::{FILE_SET_SPARSE_BUFFER, FSCTL_SET_SPARSE};

    pub fn set(file: &std::fs::File, sparse: bool) -> bool {
        let input = FILE_SET_SPARSE_BUFFER { SetSparse: sparse };
        let mut returned = 0u32;
        // SAFETY: `file` owns the handle and outlives the call; the input pointer and size
        // describe `input`, a FILE_SET_SPARSE_BUFFER on the stack; there is no output buffer;
        // `returned` is a valid u32; no OVERLAPPED, so the call completes synchronously.
        let ok = unsafe {
            DeviceIoControl(
                file.as_raw_handle(),
                FSCTL_SET_SPARSE,
                (&raw const input).cast(),
                size_of::<FILE_SET_SPARSE_BUFFER>() as u32,
                std::ptr::null_mut(),
                0,
                &mut returned,
                std::ptr::null_mut(),
            )
        };
        ok != 0
    }
}

#[cfg(not(windows))]
mod imp {
    pub fn set(_: &std::fs::File, _: bool) -> bool {
        false
    }
}

#[cfg(all(test, windows))]
mod tests {
    use std::io::{Seek, SeekFrom, Write};
    use std::os::windows::fs::MetadataExt;

    use super::*;

    const FILE_ATTRIBUTE_SPARSE_FILE: u32 = 0x200;

    fn is_sparse(f: &File) -> bool {
        f.metadata().unwrap().file_attributes() & FILE_ATTRIBUTE_SPARSE_FILE != 0
    }

    /// The bytes the file takes on the drive (for a sparse file, only its written ranges).
    fn on_drive(path: &std::path::Path) -> u64 {
        use std::os::windows::ffi::OsStrExt;
        use windows_sys::Win32::Storage::FileSystem::GetCompressedFileSizeW;
        let wide: Vec<u16> = path.as_os_str().encode_wide().chain([0]).collect();
        let mut high = 0u32;
        // SAFETY: `wide` is a NUL-terminated UTF-16 path that outlives the call; `high` is a
        // valid u32 for the size's upper half.
        let low = unsafe { GetCompressedFileSizeW(wide.as_ptr(), &mut high) };
        if low == u32::MAX {
            // Also a valid low half: only the error code tells.
            let e = std::io::Error::last_os_error();
            assert_eq!(e.raw_os_error(), Some(0), "GetCompressedFileSizeW: {e}");
        }
        (u64::from(high) << 32) | u64::from(low)
    }

    #[test]
    fn a_sparse_part_file_is_written_far_in_without_a_fill() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("p.bin");
        let mut f = File::options()
            .read(true)
            .write(true)
            .create(true)
            .truncate(true)
            .open(&path)
            .unwrap();
        assert!(set(&f, true), "NTFS temp folder");
        assert!(is_sparse(&f));
        f.set_len(4 << 30).unwrap();
        f.seek(SeekFrom::Start(2 << 30)).unwrap();
        f.write_all(&[7u8; 1 << 20]).unwrap();
        f.sync_data().unwrap();
        // A fill would have written the 2 GiB before the write (0.5–4 s on this machine), and
        // a file that isn't sparse takes its whole 4 GiB on the drive. This one holds only what
        // was written: measured on the drive, not timed (#201: a 400 ms budget failed on a busy
        // CI runner and on a starved CPU).
        let used = on_drive(&path);
        assert!(
            ((1 << 20)..(64 << 20)).contains(&used),
            "{used} bytes on the drive"
        );
        // Clearing allocates any holes left, so only a fully written file is cleared: shrink
        // this mostly empty one first.
        f.set_len(0).unwrap();
        assert!(set(&f, false));
        assert!(!is_sparse(&f));
    }
}