Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

Fetch: validate caller-supplied names, sizes and hashes; FileSpec borrows its strings; download_url for an exact address

Daniel Voshart8dd3afce51d2
diff --git a/crates/fetch/README.md b/crates/fetch/README.md
new file mode 100644
index 000000000..60ac1d329
--- /dev/null
+++ b/crates/fetch/README.md
@@ -0,0 +1,7 @@
+# lightcraft-fetch
+
+Native model downloads in pure Rust (L0, Apache-2.0): HTTP/1.1 over std sockets and rustls with the RustCrypto provider, without ring, aws-lc, C or assembly build dependencies. The wasm32 crate is empty.
+
+Call `download` with bounded file specifications and ordered mirrors, or `download_url` for an exact catalog URL (including its query). Each file has a local filename independent of the URL, a size cap and optional pinned size/SHA-256. Transfers resume pinned `.part` files, verify before renaming, report progress and check cancellation during reads. Run on a worker thread. No proxy support; DNS resolution uses the platform resolver. Tokens are off by default and never forwarded to a different redirect host.
+
+Tests use local synthetic HTTP servers for redirects, resume, truncation, cancellation, stalls, hashes, size limits and hostile specifications; they make no internet requests.
diff --git a/crates/fetch/src/http.rs b/crates/fetch/src/http.rs
index 332f01477..e900850ac 100644
--- a/crates/fetch/src/http.rs
+++ b/crates/fetch/src/http.rs
@@ -60,7 +60,10 @@ pub struct Url {
impl Url {
pub fn parse(s: &str) -> Result<Url, HttpError> {
- let bad = |why: &str| HttpError::BadUrl(format!("{s}: {why}"));
+ let bad = |why: &str| HttpError::BadUrl(why.to_string());
+ if s.len() > 16 * 1024 {
+ return Err(bad("URL exceeds 16 KiB"));
+ }
let s = s.trim();
if s.chars().any(|c| c.is_control() || c == ' ') {
return Err(bad("contains spaces or control characters"));
@@ -509,6 +512,12 @@ pub fn get(url: &Url, headers: &[(&str, String)], limits: &Limits) -> Result<Res
mod tests {
use super::*;
+ #[test]
+ fn rejects_oversized_and_control_character_urls() {
+ assert!(Url::parse(&format!("https://example.com/{}", "x".repeat(16 * 1024))).is_err());
+ assert!(Url::parse("https://example.com/x?secret=abc\r\nInjected: 1").is_err());
+ }
+
#[test]
fn parses_urls() {
let u = Url::parse("https://cdn.example.com/models/sam3/model.safetensors?sig=1#x").unwrap();
diff --git a/crates/fetch/src/lib.rs b/crates/fetch/src/lib.rs
index 12ad639c7..241725294 100644
--- a/crates/fetch/src/lib.rs
+++ b/crates/fetch/src/lib.rs
@@ -33,12 +33,12 @@ use http::{HttpError, Limits, Url};
/// One file of the model.
#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct FileSpec {
- pub name: &'static str,
+pub struct FileSpec<'a> {
+ pub name: &'a str,
/// Exact size in bytes, when pinned (then the download resumes and is checked against it).
pub size: Option<u64>,
/// SHA-256 (lower-case hex), when pinned.
- pub sha256: Option<&'static str>,
+ pub sha256: Option<&'a str>,
/// Largest size accepted when `size` is not pinned.
pub max: u64,
}
@@ -106,6 +106,8 @@ pub struct Progress {
pub enum DownloadError {
/// No mirror is configured.
NoMirrors,
+ /// Invalid filename, size, hash or aggregate bound supplied by the caller.
+ Invalid(String),
Cancelled,
/// A local file problem (folder not writable, disk full…).
Disk(String),
@@ -119,6 +121,7 @@ pub enum DownloadError {
impl std::fmt::Display for DownloadError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
+ DownloadError::Invalid(e) => write!(f, "invalid download: {e}"),
DownloadError::NoMirrors => write!(f, "no download location is configured for this model in this build"),
DownloadError::Cancelled => write!(f, "the download was cancelled"),
DownloadError::Disk(e) => write!(f, "{e}"),
@@ -135,16 +138,45 @@ enum Fail {
Disk(String),
}
-/// Download `files` into `dir` from `mirrors` (in order; see [`mirrors`]). Files already there
-/// and verified are kept. `progress` is called as bytes arrive (often: throttle in it).
+/// Fetch a single file from its exact URL, preserving any query string. The local filename
+/// remains `file.name`; it never comes from redirects or response headers.
+pub fn download_url(
+ file: &FileSpec<'_>,
+ url: &str,
+ dir: &Path,
+ opts: &Options,
+ cancel: &AtomicBool,
+ progress: &mut dyn FnMut(&Progress),
+) -> Result<(), DownloadError> {
+ download_source(std::slice::from_ref(file), &[url.to_string()], dir, opts, cancel, progress, true)
+}
+
+/// Download `files` into `dir` from `mirrors` (in order; see [`mirrors`]). Verified files
+/// are kept. `progress` is called as bytes arrive (often: throttle in it).
pub fn download(
- files: &[FileSpec],
+ files: &[FileSpec<'_>],
mirrors: &[String],
dir: &Path,
opts: &Options,
cancel: &AtomicBool,
progress: &mut dyn FnMut(&Progress),
) -> Result<(), DownloadError> {
+ download_source(files, mirrors, dir, opts, cancel, progress, false)
+}
+
+fn download_source(
+ files: &[FileSpec<'_>],
+ mirrors: &[String],
+ dir: &Path,
+ opts: &Options,
+ cancel: &AtomicBool,
+ progress: &mut dyn FnMut(&Progress),
+ exact_urls: bool,
+) -> Result<(), DownloadError> {
+ let total = validate_files(files)?;
+ if mirrors.len() > MAX_MIRRORS {
+ return Err(DownloadError::Invalid("too many mirrors".into()));
+ }
std::fs::create_dir_all(dir).map_err(|e| DownloadError::Disk(format!("can't create {}: {e}", dir.display())))?;
// what's left to fetch
let mut todo = Vec::new();
@@ -159,14 +191,13 @@ pub fn download(
if mirrors.is_empty() {
return Err(DownloadError::NoMirrors);
}
- let total: u64 = files.iter().map(|f| f.size.unwrap_or(0)).sum();
let mut before: u64 = files.iter().filter(|f| !todo.contains(f)).filter_map(|f| f.size).sum();
for f in todo {
let mut errors = Vec::new();
let mut ok = false;
'mirrors: for m in mirrors {
- let url = format!("{m}/{}", f.name);
- for _ in 0..opts.attempts.max(1) {
+ let url = if exact_urls { m.clone() } else { format!("{m}/{}", f.name) };
+ for _ in 0..opts.attempts.clamp(1, 16) {
if cancel.load(Ordering::Relaxed) {
return Err(DownloadError::Cancelled);
}
@@ -187,11 +218,11 @@ pub fn download(
Err(Fail::Cancelled) => return Err(DownloadError::Cancelled),
Err(Fail::Disk(e)) => return Err(DownloadError::Disk(e)),
Err(Fail::Retry(e)) => {
- log::warn!("download of {url}: {e} (retrying)");
+ log::warn!("download of {}: {e} (retrying)", short(&url));
errors.push(format!("{}: {e}", short(m)));
}
Err(Fail::NextMirror(e)) => {
- log::warn!("download of {url}: {e}");
+ log::warn!("download of {}: {e}", short(&url));
errors.push(format!("{}: {e}", short(m)));
continue 'mirrors;
}
@@ -206,6 +237,43 @@ pub fn download(
Ok(())
}
+/// Validate before touching disk, including names supplied by a user model catalog.
+fn validate_files(files: &[FileSpec<'_>]) -> Result<u64, DownloadError> {
+ if files.len() > 256 {
+ return Err(DownloadError::Invalid("too many files".into()));
+ }
+ let mut seen = std::collections::HashSet::new();
+ let mut total = 0u64;
+ for f in files {
+ let stem = f.name.split('.').next().unwrap_or_default().to_ascii_uppercase();
+ let reserved = matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL")
+ || stem
+ .strip_prefix("COM")
+ .or_else(|| stem.strip_prefix("LPT"))
+ .is_some_and(|n| matches!(n, "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"));
+ if reserved
+ || f.name.ends_with(['.', ' '])
+ || f.name.ends_with(".part")
+ || f.name.is_empty()
+ || f.name.len() > 255
+ || f.name.starts_with('.')
+ || f.name.contains(['/', '\\', ':'])
+ || f.name.chars().any(char::is_control)
+ || !seen.insert(f.name)
+ {
+ return Err(DownloadError::Invalid("a filename must be a unique, ordinary filename".into()));
+ }
+ if f.max == 0
+ || f.size.is_some_and(|n| n == 0 || n > f.max)
+ || f.sha256.is_some_and(|h| h.len() != 64 || !h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
+ {
+ return Err(DownloadError::Invalid("invalid file size or SHA-256".into()));
+ }
+ total = total.checked_add(f.size.unwrap_or(0)).ok_or_else(|| DownloadError::Invalid("total size overflow".into()))?;
+ }
+ Ok(total)
+}
+
/// The bearer token to send to `url`: only a non-empty one, only over https, and only to the host the download began
/// at (`origin`), never to the host a redirect leads to.
fn bearer<'a>(token: Option<&'a str>, url: &Url, origin: &str) -> Option<&'a str> {
@@ -218,7 +286,7 @@ fn short(m: &str) -> String {
}
/// Whether `path` is already the right file (exact size and hash when pinned).
-fn verified(f: &FileSpec, path: &Path) -> Result<bool, DownloadError> {
+fn verified(f: &FileSpec<'_>, path: &Path) -> Result<bool, DownloadError> {
let Ok(meta) = std::fs::metadata(path) else { return Ok(false) };
if !meta.is_file() || meta.len() == 0 {
return Ok(false);
@@ -252,7 +320,7 @@ fn part_path(dir: &Path, name: &str) -> PathBuf {
/// Fetch one file from one mirror into `<name>.part` (resuming it when its size is pinned),
/// check it and move it into place.
-fn fetch_file(f: &FileSpec, url: &str, dir: &Path, opts: &Options, cancel: &AtomicBool, report: &mut dyn FnMut(u64, u64)) -> Result<(), Fail> {
+fn fetch_file(f: &FileSpec<'_>, url: &str, dir: &Path, opts: &Options, cancel: &AtomicBool, report: &mut dyn FnMut(u64, u64)) -> Result<(), Fail> {
let part = part_path(dir, f.name);
let limit = f.size.unwrap_or(f.max);
let disk = |e: std::io::Error| Fail::Disk(format!("{}: {e}", part.display()));
diff --git a/crates/fetch/src/tests.rs b/crates/fetch/src/tests.rs
index ad3f1063d..4a828c6d6 100644
--- a/crates/fetch/src/tests.rs
+++ b/crates/fetch/src/tests.rs
@@ -22,7 +22,7 @@ fn leak(s: String) -> &'static str {
Box::leak(s.into_boxed_str())
}
-fn pinned() -> FileSpec {
+fn pinned() -> FileSpec<'static> {
let p = payload();
FileSpec { name: "model.bin", size: Some(p.len() as u64), sha256: Some(leak(sha_hex(&p))), max: p.len() as u64 }
}
@@ -140,7 +140,7 @@ fn quick() -> Options {
Options { connect_timeout: Duration::from_secs(2), stall_timeout: Duration::from_millis(600), attempts: 2, token_env: None }
}
-fn run(files: &[FileSpec], mirrors: &[String], dir: &Path, cancel: &AtomicBool) -> (Result<(), DownloadError>, Vec<Progress>) {
+fn run(files: &[FileSpec<'_>], mirrors: &[String], dir: &Path, cancel: &AtomicBool) -> (Result<(), DownloadError>, Vec<Progress>) {
let mut seen = Vec::new();
let r = download(files, mirrors, dir, &quick(), cancel, &mut |p| seen.push(p.clone()));
(r, seen)
@@ -329,3 +329,60 @@ fn a_token_is_sent_only_over_https_to_the_first_host() {
// a caller that names no variable sends nothing, whatever the environment holds
assert!(Options::default().token_env.is_none());
}
+
+#[test]
+fn exact_url_keeps_query_and_owned_catalog_values() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let url = format!("http://{}/download?file=weights&sig=secret", listener.local_addr().unwrap());
+ let worker = std::thread::spawn(move || {
+ let (mut socket, _) = listener.accept().unwrap();
+ let mut reader = BufReader::new(socket.try_clone().unwrap());
+ let mut request = String::new();
+ reader.read_line(&mut request).unwrap();
+ assert_eq!(request.trim(), "GET /download?file=weights&sig=secret HTTP/1.1");
+ loop {
+ let mut line = String::new();
+ if reader.read_line(&mut line).unwrap() == 0 || line.trim().is_empty() {
+ break;
+ }
+ }
+ socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc").unwrap();
+ });
+ let name = String::from("catalog.onnx");
+ let hash = sha_hex(b"abc");
+ let spec = FileSpec { name: &name, size: Some(3), sha256: Some(&hash), max: 3 };
+ let dir = tmp("exact");
+ let mut progress = Vec::new();
+ download_url(&spec, &url, &dir, &quick(), &AtomicBool::new(false), &mut |p| progress.push(p.clone())).unwrap();
+ worker.join().unwrap();
+ assert_eq!(std::fs::read(dir.join(name)).unwrap(), b"abc");
+ assert!(progress.iter().all(|p| !p.mirror.contains("secret")));
+ let _ = std::fs::remove_dir_all(dir);
+}
+
+#[test]
+fn hostile_file_specs_are_rejected_before_touching_disk() {
+ let dir = tmp("invalid");
+ let base = FileSpec { name: "valid.onnx", size: Some(1), sha256: None, max: 1 };
+ let cancel = AtomicBool::new(false);
+ for name in ["", ".hidden", "../escape", "a/b", "a\\b", "C:escape", "bad\nname", "trailing.", "CON.onnx", "a.part"] {
+ let spec = FileSpec { name, ..base.clone() };
+ assert!(matches!(run(&[spec], &[], &dir, &cancel).0, Err(DownloadError::Invalid(_))), "{name}");
+ assert!(!dir.exists());
+ }
+ for spec in [
+ FileSpec { max: 0, ..base.clone() },
+ FileSpec { size: Some(0), ..base.clone() },
+ FileSpec { size: Some(2), ..base.clone() },
+ FileSpec { sha256: Some("not a SHA"), ..base.clone() },
+ ] {
+ assert!(matches!(run(&[spec], &[], &dir, &cancel).0, Err(DownloadError::Invalid(_))));
+ assert!(!dir.exists());
+ }
+ let huge = FileSpec { size: Some(u64::MAX), max: u64::MAX, ..base.clone() };
+ let other = FileSpec { name: "other.onnx", ..huge.clone() };
+ for specs in [vec![huge, other], vec![base.clone(); 257], vec![base.clone(), base]] {
+ assert!(matches!(run(&specs, &[], &dir, &cancel).0, Err(DownloadError::Invalid(_))));
+ assert!(!dir.exists());
+ }
+}