misterdownloadmanager: The ultimate open-source, clean-room download manager written from scratch in pure Rust for Windows, Mac, and Linux
Rust bittorrent-clientdebriddownload-acceleratordownload-managerdownloadmanagereguifile-downloadergallery-dlidmidm-alternativeinternet-download-managerm3u8-downloadermagnetplaylist-downloaderreal-debridrusttorrent-clientvideo-downloaderyoutube-downloaderyt-dlp
//! Sparse part files (D-008 amendment, 2026-10-29).
//!
//! Extending a file with `set_len` and then writing far into it makes NTFS zero-fill everything
//! before the write, synchronously: one 1 MiB write at 2 GiB took 4.3 s on the user's D: drive
//! (and doubles the bytes written). The planner writes at the middle of the file within the
//! first second, so every large download paid it. A sparse file has no such fill. Elsewhere
//! (`ftruncate` already makes sparse files) this is a no-op.
//!
//! The engine's unsafe code: one `DeviceIoControl` call here (and, in its test, one
//! `GetCompressedFileSizeW`), and the free-space probe's calls in `crate::disk`: two on Windows,
//! one on macOS (with its CoreFoundation key, and the release of what it returns).
#![allow(unsafe_code)]
use std::fs::File;
/// Smaller files aren't worth it: the fill is short.
pub const MIN_SIZE: u64 = 5 << 20;
/// Mark `file` sparse (before `set_len`) or, once every byte is written, normal again (clearing
/// allocates any holes still there). False where the file system can't (FAT, exFAT): the
/// download then works as before.
pub fn set(file: &File, sparse: bool) -> bool {
imp::set(file, sparse)
}
#[cfg(windows)]
mod imp {
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::System::IO::DeviceIoControl;
use windows_sys::Win32::System::Ioctl::{FILE_SET_SPARSE_BUFFER, FSCTL_SET_SPARSE};
pub fn set(file: &std::fs::File, sparse: bool) -> bool {
let input = FILE_SET_SPARSE_BUFFER { SetSparse: sparse };
let mut returned = 0u32;
// SAFETY: `file` owns the handle and outlives the call; the input pointer and size
// describe `input`, a FILE_SET_SPARSE_BUFFER on the stack; there is no output buffer;
// `returned` is a valid u32; no OVERLAPPED, so the call completes synchronously.
let ok = unsafe {
DeviceIoControl(
file.as_raw_handle(),
FSCTL_SET_SPARSE,
(&raw const input).cast(),
size_of::<FILE_SET_SPARSE_BUFFER>() as u32,
std::ptr::null_mut(),
0,
&mut returned,
std::ptr::null_mut(),
)
};
ok != 0
}
}
#[cfg(not(windows))]
mod imp {
pub fn set(_: &std::fs::File, _: bool) -> bool {
false
}
}
#[cfg(all(test, windows))]
mod tests {
use std::io::{Seek, SeekFrom, Write};
use std::os::windows::fs::MetadataExt;
use super::*;
const FILE_ATTRIBUTE_SPARSE_FILE: u32 = 0x200;
fn is_sparse(f: &File) -> bool {
f.metadata().unwrap().file_attributes() & FILE_ATTRIBUTE_SPARSE_FILE != 0
}
/// The bytes the file takes on the drive (for a sparse file, only its written ranges).
fn on_drive(path: &std::path::Path) -> u64 {
use std::os::windows::ffi::OsStrExt;
use windows_sys::Win32::Storage::FileSystem::GetCompressedFileSizeW;
let wide: Vec<u16> = path.as_os_str().encode_wide().chain([0]).collect();
let mut high = 0u32;
// SAFETY: `wide` is a NUL-terminated UTF-16 path that outlives the call; `high` is a
// valid u32 for the size's upper half.
let low = unsafe { GetCompressedFileSizeW(wide.as_ptr(), &mut high) };
if low == u32::MAX {
// Also a valid low half: only the error code tells.
let e = std::io::Error::last_os_error();
assert_eq!(e.raw_os_error(), Some(0), "GetCompressedFileSizeW: {e}");
}
(u64::from(high) << 32) | u64::from(low)
}
#[test]
fn a_sparse_part_file_is_written_far_in_without_a_fill() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("p.bin");
let mut f = File::options()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(&path)
.unwrap();
assert!(set(&f, true), "NTFS temp folder");
assert!(is_sparse(&f));
f.set_len(4 << 30).unwrap();
f.seek(SeekFrom::Start(2 << 30)).unwrap();
f.write_all(&[7u8; 1 << 20]).unwrap();
f.sync_data().unwrap();
// A fill would have written the 2 GiB before the write (0.5–4 s on this machine), and
// a file that isn't sparse takes its whole 4 GiB on the drive. This one holds only what
// was written: measured on the drive, not timed (#201: a 400 ms budget failed on a busy
// CI runner and on a starved CPU).
let used = on_drive(&path);
assert!(
((1 << 20)..(64 << 20)).contains(&used),
"{used} bytes on the drive"
);
// Clearing allocates any holes left, so only a fully written file is cleared: shrink
// this mostly empty one first.
f.set_len(0).unwrap();
assert!(set(&f, false));
assert!(!is_sparse(&f));
}
}