LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
Fetch: the model downloader is its own crate, lightcraft-fetch
Daniel Voshart7fa7506db46d
diff --git a/Cargo.lock b/Cargo.lockindex 3f0861610..8de1ee494 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2900,6 +2900,17 @@ dependencies = ["web-time",]+[[package]]+name = "lightcraft-fetch"+version = "0.2.1"+dependencies = [+ "log",+ "rustls",+ "rustls-rustcrypto",+ "sha2",+ "webpki-roots",+]+[[package]]name = "lightcraft-geom"version = "0.2.1"@@ -3039,14 +3050,11 @@ version = "0.2.1"dependencies = ["candle-core","candle-nn",+ "lightcraft-fetch","log",- "rustls",- "rustls-rustcrypto","serde","serde_json",- "sha2","thiserror 2.0.21",- "webpki-roots",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex d59f8f45e..7af2f898b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -29,6 +29,7 @@ lightcraft-merge = { path = "crates/merge" }lightcraft-engine = { path = "crates/engine" }lightcraft-ui-egui = { path = "crates/ui-egui" }lightcraft-mcp = { path = "crates/mcp" }+lightcraft-fetch = { path = "crates/fetch" }lightcraft-segment = { path = "crates/segment" }# foundationdiff --git a/crates/engine/src/segment/mod.rs b/crates/engine/src/segment/mod.rsindex 84397042d..3cf14712e 100644--- a/crates/engine/src/segment/mod.rs+++ b/crates/engine/src/segment/mod.rs@@ -260,7 +260,7 @@ impl Segmenter {{let mirrors = self.mirrors();if mirrors.is_empty() {- return Err(lightcraft_segment::fetch::DownloadError::NoMirrors.to_string());+ return Err(lightcraft_segment::fetch::no_mirrors_message());}self.download.start(lightcraft_segment::fetch::SAM3_FILES, mirrors, dir, lightcraft_segment::fetch::Options::default())}diff --git a/crates/fetch/Cargo.toml b/crates/fetch/Cargo.tomlnew file mode 100644index 000000000..2e65583fb--- /dev/null+++ b/crates/fetch/Cargo.toml@@ -0,0 +1,17 @@+[package]+name = "lightcraft-fetch"+version.workspace = true+edition.workspace = true+license = "Apache-2.0"+description = "LightCraft model downloads: mirrors, resume, pinned SHA-256, over HTTPS in pure Rust (L0)"++[lints]+workspace = true++# downloads run natively only (the web build fetches no models)+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]+log = { workspace = true }+rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] }+rustls-rustcrypto = "0.0.2-alpha"+webpki-roots = "1"+sha2 = { version = "0.10", default-features = false, features = ["std"] }diff --git a/crates/segment/src/fetch/http.rs b/crates/fetch/src/http.rssimilarity index 100%rename from crates/segment/src/fetch/http.rsrename to crates/fetch/src/http.rsdiff --git a/crates/fetch/src/lib.rs b/crates/fetch/src/lib.rsnew file mode 100644index 000000000..b13d4c343--- /dev/null+++ b/crates/fetch/src/lib.rs@@ -0,0 +1,392 @@+//! Downloading a model's files, only when the user asked for it: from an ordered list of mirrors+//! (the next one when a mirror fails), resuming partial files, with connect and stall timeouts,+//! a size cap, progress and cancellation, and checked (exact size, SHA-256 where pinned) before+//! each file is moved into the model folder. Runs on whatever thread calls [`download`] (the+//! engine uses a background thread; the UI thread never waits for the network). Pure Rust: the+//! HTTPS client is rustls with the RustCrypto provider (see [`http`]).+//!+//! Files are written to `<name>.part` next to their final place and renamed when verified, so a+//! model folder never holds a half-written file under a real name, and an interrupted download+//! resumes from where it stopped. A file that fails its check is deleted, never kept.+//!+//! What to download and where from belongs to the caller: it passes the [`FileSpec`]s and the+//! default mirrors (see [`mirrors`]). `lightcraft-segment` does so for the SAM 3 weights, which+//! are never part of LightCraft (SAM License, see docs/ai-masks.md).+//!+//! Native only: on wasm32 this crate is empty (the web build downloads no models).++#![cfg(not(target_arch = "wasm32"))]++pub mod http;++use std::fs::{File, OpenOptions};+use std::io::{Read, Write};+use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicBool, Ordering};+use std::time::Duration;++use sha2::{Digest, Sha256};++use http::{HttpError, Limits, Url};++/// One file of the model.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct FileSpec {+ pub name: &'static str,+ /// Exact size in bytes, when pinned (then the download resumes and is checked against it).+ pub size: Option<u64>,+ /// SHA-256 (lower-case hex), when pinned.+ pub sha256: Option<&'static str>,+ /// Largest size accepted when `size` is not pinned.+ pub max: u64,+}++/// Most mirrors used (a hostile list can't make a download loop forever).+const MAX_MIRRORS: usize = 16;+/// Redirects followed per request.+const MAX_REDIRECTS: usize = 8;++/// The mirrors to try, in order: `env`'s (base URLs separated by commas, spaces or newlines), then+/// the mirrors file's (one base URL per line, `#` comments), then `defaults`; duplicates and+/// unusable URLs dropped, at most 16.+pub fn mirrors(env: Option<&str>, file: Option<&Path>, defaults: &[&str]) -> Vec<String> {+ let mut out: Vec<String> = Vec::new();+ let mut add = |s: &str| {+ let s = s.trim().trim_end_matches('/');+ if !s.is_empty() && Url::parse(s).is_ok() && !out.iter().any(|m| m == s) && out.len() < MAX_MIRRORS {+ out.push(s.to_string());+ }+ };+ if let Some(env) = env {+ env.split([',', ' ', '\n', '\t']).for_each(&mut add);+ }+ // a small text file; anything larger isn't a mirror list+ if let Some(text) = file.and_then(|f| std::fs::metadata(f).ok().filter(|m| m.len() < 64 * 1024).and_then(|_| std::fs::read_to_string(f).ok())) {+ text.lines().map(|l| l.split('#').next().unwrap_or_default()).for_each(&mut add);+ }+ defaults.iter().for_each(|m| add(m));+ out+}++/// Download settings.+#[derive(Clone, Debug)]+pub struct Options {+ /// Longest wait to connect to a mirror.+ pub connect_timeout: Duration,+ /// Longest wait for the next bytes once connected.+ pub stall_timeout: Duration,+ /// Tries per mirror and file (each resumes where the last one stopped).+ pub attempts: u32,+}++impl Default for Options {+ fn default() -> Self {+ Options { connect_timeout: Duration::from_secs(15), stall_timeout: Duration::from_secs(30), attempts: 3 }+ }+}++/// Where a download is.+#[derive(Clone, Debug, Default, PartialEq, Eq)]+pub struct Progress {+ pub file: String,+ /// Bytes of all files on disk so far, and the total expected (pinned sizes + announced ones).+ pub done: u64,+ pub total: u64,+ /// The mirror in use (without query).+ pub mirror: String,+}++/// Why a download failed.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum DownloadError {+ /// No mirror is configured.+ NoMirrors,+ Cancelled,+ /// A local file problem (folder not writable, disk full…).+ Disk(String),+ /// Every mirror failed for `file`; the reason for each.+ AllMirrorsFailed {+ file: String,+ errors: Vec<String>,+ },+}++impl std::fmt::Display for DownloadError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ DownloadError::NoMirrors => write!(f, "no download location is configured for this model in this build"),+ DownloadError::Cancelled => write!(f, "the download was cancelled"),+ DownloadError::Disk(e) => write!(f, "{e}"),+ DownloadError::AllMirrorsFailed { file, errors } => write!(f, "could not download {file}: {}", errors.join("; ")),+ }+ }+}++/// One mirror's failure for one file; `Retry` resumes on the same mirror.+enum Fail {+ Retry(String),+ NextMirror(String),+ Cancelled,+ Disk(String),+}++/// Download `files` into `dir` from `mirrors` (in order; see [`mirrors`]). Files already there+/// and verified are kept. `progress` is called as bytes arrive (often: throttle in it).+pub fn download(+ files: &[FileSpec],+ mirrors: &[String],+ dir: &Path,+ opts: &Options,+ cancel: &AtomicBool,+ progress: &mut dyn FnMut(&Progress),+) -> Result<(), DownloadError> {+ std::fs::create_dir_all(dir).map_err(|e| DownloadError::Disk(format!("can't create {}: {e}", dir.display())))?;+ // what's left to fetch+ let mut todo = Vec::new();+ for f in files {+ if !verified(f, &dir.join(f.name))? {+ todo.push(f);+ }+ }+ if todo.is_empty() {+ return Ok(());+ }+ if mirrors.is_empty() {+ return Err(DownloadError::NoMirrors);+ }+ let total: u64 = files.iter().map(|f| f.size.unwrap_or(0)).sum();+ let mut before: u64 = files.iter().filter(|f| !todo.contains(f)).filter_map(|f| f.size).sum();+ for f in todo {+ let mut errors = Vec::new();+ let mut ok = false;+ 'mirrors: for m in mirrors {+ let url = format!("{m}/{}", f.name);+ for _ in 0..opts.attempts.max(1) {+ if cancel.load(Ordering::Relaxed) {+ return Err(DownloadError::Cancelled);+ }+ let mut report = |n: u64, announced: u64| {+ let mirror = Url::parse(m).map(|u| u.display()).unwrap_or_default();+ progress(&Progress {+ file: f.name.to_string(),+ done: before.saturating_add(n),+ total: total.max(before.saturating_add(announced)),+ mirror,+ })+ };+ match fetch_file(f, &url, dir, opts, cancel, &mut report) {+ Ok(()) => {+ ok = true;+ break 'mirrors;+ }+ Err(Fail::Cancelled) => return Err(DownloadError::Cancelled),+ Err(Fail::Disk(e)) => return Err(DownloadError::Disk(e)),+ Err(Fail::Retry(e)) => {+ log::warn!("download of {url}: {e} (retrying)");+ errors.push(format!("{}: {e}", short(m)));+ }+ Err(Fail::NextMirror(e)) => {+ log::warn!("download of {url}: {e}");+ errors.push(format!("{}: {e}", short(m)));+ continue 'mirrors;+ }+ }+ }+ }+ if !ok {+ return Err(DownloadError::AllMirrorsFailed { file: f.name.to_string(), errors });+ }+ before = before.saturating_add(std::fs::metadata(dir.join(f.name)).map(|m| m.len()).unwrap_or(0));+ }+ Ok(())+}++/// A mirror for messages (no query string: it may hold a token).+fn short(m: &str) -> String {+ Url::parse(m).map(|u| u.display()).unwrap_or_else(|_| "mirror".into())+}++/// Whether `path` is already the right file (exact size and hash when pinned).+fn verified(f: &FileSpec, path: &Path) -> Result<bool, DownloadError> {+ let Ok(meta) = std::fs::metadata(path) else { return Ok(false) };+ if !meta.is_file() || meta.len() == 0 {+ return Ok(false);+ }+ if f.size.is_some_and(|s| s != meta.len()) || meta.len() > f.max {+ return Ok(false);+ }+ match f.sha256 {+ Some(want) => Ok(sha256_file(path).map_err(DownloadError::Disk)? == want),+ None => Ok(true),+ }+}++fn sha256_file(path: &Path) -> Result<String, String> {+ let mut file = File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;+ let mut h = Sha256::new();+ let mut buf = vec![0u8; 1 << 20];+ loop {+ let n = file.read(&mut buf).map_err(|e| format!("{}: {e}", path.display()))?;+ if n == 0 {+ break;+ }+ h.update(buf.get(..n).unwrap_or_default());+ }+ Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect())+}++fn part_path(dir: &Path, name: &str) -> PathBuf {+ dir.join(format!("{name}.part"))+}++/// Fetch one file from one mirror into `<name>.part` (resuming it when its size is pinned),+/// check it and move it into place.+fn fetch_file(f: &FileSpec, url: &str, dir: &Path, opts: &Options, cancel: &AtomicBool, report: &mut dyn FnMut(u64, u64)) -> Result<(), Fail> {+ let part = part_path(dir, f.name);+ let limit = f.size.unwrap_or(f.max);+ let disk = |e: std::io::Error| Fail::Disk(format!("{}: {e}", part.display()));+ // resume only a pinned file (an unpinned one can't be checked after resuming)+ let mut have = match std::fs::metadata(&part) {+ Ok(m) if f.size.is_some() && m.len() <= limit => m.len(),+ Ok(_) => {+ std::fs::remove_file(&part).map_err(disk)?;+ 0+ }+ Err(_) => 0,+ };+ let fetched = f.size != Some(have) || have == 0;+ if fetched {+ let limits = Limits { connect: opts.connect_timeout, stall: opts.stall_timeout, cancel };+ let mut url = Url::parse(url).map_err(|e| Fail::NextMirror(e.to_string()))?;+ let origin = url.host.clone();+ let mut redirects = 0;+ let mut resp = loop {+ let mut headers = vec![];+ if have > 0 {+ headers.push(("Range", format!("bytes={have}-")));+ }+ // a token for the configured host only (never sent on to a redirect's host)+ if let Ok(t) = std::env::var("LIGHTCRAFT_SAM3_TOKEN")+ && !t.trim().is_empty()+ && url.tls+ && url.host == origin+ {+ headers.push(("Authorization", format!("Bearer {}", t.trim())));+ }+ let r = http::get(&url, &headers, &limits).map_err(http_fail)?;+ match r.status {+ 301 | 302 | 303 | 307 | 308 => {+ redirects += 1;+ if redirects > MAX_REDIRECTS {+ return Err(Fail::NextMirror("too many redirects".into()));+ }+ let loc = r.header("location").ok_or_else(|| Fail::NextMirror("redirect without a location".into()))?;+ url = url.join(loc).map_err(|e| Fail::NextMirror(e.to_string()))?;+ }+ _ => break r,+ }+ };+ let mut file = match resp.status {+ 200 => {+ // the whole file (from the start, also when a range was asked for)+ if resp.content_length().is_some_and(|n| n > limit || f.size.is_some_and(|s| s != n)) {+ return Err(Fail::NextMirror(format!("the server's file has the wrong size ({:?} bytes)", resp.content_length())));+ }+ have = 0;+ File::create(&part).map_err(disk)?+ }+ 206 if have > 0 => {+ let range = resp.header("content-range").unwrap_or_default();+ let (start, total) = parse_content_range(range).ok_or_else(|| Fail::NextMirror(format!("bad Content-Range `{range}`")))?;+ if start != have || total.is_some_and(|t| Some(t) != f.size) {+ return Err(Fail::NextMirror(format!("the server resumed at the wrong place ({range})")));+ }+ OpenOptions::new().append(true).open(&part).map_err(disk)?+ }+ 416 => {+ // the partial file doesn't fit what the server has: start over+ std::fs::remove_file(&part).map_err(disk)?;+ return Err(Fail::Retry("the partial download didn't match; starting over".into()));+ }+ 401 | 403 => return Err(Fail::NextMirror(format!("access denied (HTTP {})", resp.status))),+ 404 | 410 => return Err(Fail::NextMirror(format!("not found (HTTP {})", resp.status))),+ s if s >= 500 => return Err(Fail::Retry(format!("server error (HTTP {s})"))),+ s => return Err(Fail::NextMirror(format!("HTTP {s}"))),+ };+ let announced = f.size.unwrap_or_else(|| resp.content_length().unwrap_or(0));+ report(have, announced);+ let mut buf = vec![0u8; 256 * 1024];+ loop {+ let n = match resp.read(&mut buf, &limits) {+ Ok(n) => n,+ Err(e) => {+ let _ = file.flush();+ if f.size.is_none() {+ drop(file);+ let _ = std::fs::remove_file(&part);+ }+ return Err(http_fail(e));+ }+ };+ if n == 0 {+ break;+ }+ if have.saturating_add(n as u64) > limit {+ drop(file);+ let _ = std::fs::remove_file(&part);+ return Err(Fail::NextMirror(format!("the server sent more than the expected {limit} bytes")));+ }+ file.write_all(buf.get(..n).unwrap_or_default()).map_err(disk)?;+ have = have.saturating_add(n as u64);+ report(have, announced);+ }+ file.sync_all().map_err(disk)?;+ }+ // checks: exact size, hash+ let len = std::fs::metadata(&part).map_err(disk)?.len();+ if f.size.is_some_and(|s| s != len) {+ return Err(Fail::Retry(format!("incomplete ({len} of {} bytes)", f.size.unwrap_or(0))));+ }+ if len == 0 {+ let _ = std::fs::remove_file(&part);+ return Err(Fail::NextMirror("the server sent an empty file".into()));+ }+ if let Some(want) = f.sha256 {+ let got = sha256_file(&part).map_err(Fail::Disk)?;+ if got != want {+ std::fs::remove_file(&part).map_err(disk)?;+ let why = format!("the file is damaged or not the official one (SHA-256 {got})");+ // a damaged partial file from an earlier run isn't this mirror's fault+ return Err(if fetched { Fail::NextMirror(why) } else { Fail::Retry(why) });+ }+ }+ std::fs::rename(&part, dir.join(f.name)).map_err(disk)?;+ Ok(())+}++fn http_fail(e: HttpError) -> Fail {+ match e {+ HttpError::Cancelled => Fail::Cancelled,+ HttpError::Stalled | HttpError::Io(_) | HttpError::Protocol(_) => Fail::Retry(e.to_string()),+ HttpError::BadUrl(_) | HttpError::Connect(_) | HttpError::Tls(_) => Fail::NextMirror(e.to_string()),+ }+}++/// `bytes a-b/total` → (a, total) (`*` total → None).+fn parse_content_range(v: &str) -> Option<(u64, Option<u64>)> {+ let v = v.trim().strip_prefix("bytes")?.trim();+ let (range, total) = v.split_once('/')?;+ let (a, b) = range.split_once('-')?;+ let (a, b) = (a.trim().parse::<u64>().ok()?, b.trim().parse::<u64>().ok()?);+ if b < a {+ return None;+ }+ let total = match total.trim() {+ "*" => None,+ t => Some(t.parse::<u64>().ok()?),+ };+ Some((a, total))+}++#[cfg(test)]+mod tests;diff --git a/crates/segment/src/fetch/tests.rs b/crates/fetch/src/tests.rssimilarity index 97%rename from crates/segment/src/fetch/tests.rsrename to crates/fetch/src/tests.rsindex 81e38f018..57c168a2d 100644--- a/crates/segment/src/fetch/tests.rs+++ b/crates/fetch/src/tests.rs@@ -294,13 +294,15 @@ fn no_mirrors_is_a_clear_error_and_mirror_lists_are_parsed() {let dir = tmp("nomirror");let e = run(&[pinned()], &[], &dir, &AtomicBool::new(false)).0.unwrap_err();assert_eq!(e, DownloadError::NoMirrors);- assert!(e.to_string().contains(MIRRORS_ENV));+ assert!(e.to_string().contains("no download location"));let file = std::env::temp_dir().join(format!("lc-sam3-mirrors-{}.txt", std::process::id()));std::fs::write(&file, "# mine\nhttps://b.example/m/ # second\n\nnot a url\nhttps://a.example/x\n").unwrap();- let m = mirrors(Some("https://a.example/x/, ftp://no"), Some(&file));- assert_eq!(m[..2], ["https://a.example/x".to_string(), "https://b.example/m".to_string()]);- assert_eq!(m.len(), 2 + DEFAULT_MIRRORS.len());- assert_eq!(mirrors(None, Some(Path::new("/nonexistent/mirrors.txt"))).len(), DEFAULT_MIRRORS.len());+ let defaults = ["https://c.example/d", "https://a.example/x"];+ let m = mirrors(Some("https://a.example/x/, ftp://no"), Some(&file), &defaults);+ // the environment's, then the file's, then the defaults; a repeated one is kept once+ assert_eq!(m, ["https://a.example/x", "https://b.example/m", "https://c.example/d"]);+ assert_eq!(mirrors(None, Some(Path::new("/nonexistent/mirrors.txt")), &defaults).len(), 2);+ assert!(mirrors(None, None, &[]).is_empty());let _ = std::fs::remove_file(&file);let _ = std::fs::remove_dir_all(&dir);}diff --git a/crates/segment/Cargo.toml b/crates/segment/Cargo.tomlindex bd7b0d665..11dff5c77 100644--- a/crates/segment/Cargo.toml+++ b/crates/segment/Cargo.toml@@ -18,10 +18,7 @@ log = { workspace = true }[target.'cfg(not(target_arch = "wasm32"))'.dependencies]candle-core = { workspace = true }candle-nn = { workspace = true }-rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] }-rustls-rustcrypto = "0.0.2-alpha"-webpki-roots = "1"-sha2 = { version = "0.10", default-features = false, features = ["std"] }+lightcraft-fetch = { workspace = true }[target.'cfg(target_os = "macos")'.dependencies]candle-core = { workspace = true, features = ["metal"] }diff --git a/crates/segment/src/fetch/mod.rs b/crates/segment/src/fetch/mod.rsindex 92c5b6ea0..55c083ccf 100644--- a/crates/segment/src/fetch/mod.rs+++ b/crates/segment/src/fetch/mod.rs@@ -1,39 +1,13 @@-//! Downloading the SAM 3 model, only when the user asked for it: from an ordered list of mirrors-//! (the next one when a mirror fails), resuming partial files, with connect and stall timeouts,-//! a size cap, progress and cancellation, and checked (exact size, SHA-256 where pinned) before-//! each file is moved into the model folder. Runs on whatever thread calls [`download`] (the-//! engine uses a background thread; the UI thread never waits for the network).-//!-//! Files are written to `<name>.part` next to their final place and renamed when verified, so a-//! model folder never holds a half-written file under a real name, and an interrupted download-//! resumes from where it stopped. A file that fails its check is deleted, never kept.+//! Downloading the SAM 3 model, only when the user asked for it. The downloader itself (mirrors,+//! resuming, checks, the pure-Rust HTTPS client) is the `lightcraft-fetch` crate; this module+//! says what SAM 3 needs and where it may come from.//!//! The weights are never part of LightCraft (SAM License, see docs/ai-masks.md); where they are//! downloaded from is configured by [`DEFAULT_MIRRORS`] and the user's own list.-pub mod http;--use std::fs::{File, OpenOptions};-use std::io::{Read, Write};-use std::path::{Path, PathBuf};-use std::sync::atomic::{AtomicBool, Ordering};-use std::time::Duration;+pub use lightcraft_fetch::*;-use sha2::{Digest, Sha256};--use http::{HttpError, Limits, Url};--/// One file of the model.-#[derive(Clone, Debug, PartialEq, Eq)]-pub struct FileSpec {- pub name: &'static str,- /// Exact size in bytes, when pinned (then the download resumes and is checked against it).- pub size: Option<u64>,- /// SHA-256 (lower-case hex), when pinned.- pub sha256: Option<&'static str>,- /// Largest size accepted when `size` is not pinned.- pub max: u64,-}+use std::path::Path;/// The `facebook/sam3` checkpoint files [`crate::Sam3::load`] needs.///@@ -71,354 +45,27 @@ pub const DEFAULT_MIRRORS: &[&str] = &[];/// tried before the defaults.pub const MIRRORS_ENV: &str = "LIGHTCRAFT_SAM3_MIRRORS";-/// Most mirrors used (a hostile list can't make a download loop forever).-const MAX_MIRRORS: usize = 16;-/// Redirects followed per request.-const MAX_REDIRECTS: usize = 8;--/// The mirrors to try, in order: the environment variable's, then the mirrors file's (one base-/// URL per line, `#` comments), then [`DEFAULT_MIRRORS`]; duplicates and unusable URLs dropped.+/// The mirrors to try for SAM 3, in order: the environment variable's, then the mirrors file's+/// (one base URL per line, `#` comments), then [`DEFAULT_MIRRORS`].pub fn mirrors(env: Option<&str>, file: Option<&Path>) -> Vec<String> {- let mut out: Vec<String> = Vec::new();- let mut add = |s: &str| {- let s = s.trim().trim_end_matches('/');- if !s.is_empty() && Url::parse(s).is_ok() && !out.iter().any(|m| m == s) && out.len() < MAX_MIRRORS {- out.push(s.to_string());- }- };- if let Some(env) = env {- env.split([',', ' ', '\n', '\t']).for_each(&mut add);- }- // a small text file; anything larger isn't a mirror list- if let Some(text) = file.and_then(|f| std::fs::metadata(f).ok().filter(|m| m.len() < 64 * 1024).and_then(|_| std::fs::read_to_string(f).ok())) {- text.lines().map(|l| l.split('#').next().unwrap_or_default()).for_each(&mut add);- }- DEFAULT_MIRRORS.iter().for_each(|m| add(m));- out-}--/// Download settings.-#[derive(Clone, Debug)]-pub struct Options {- /// Longest wait to connect to a mirror.- pub connect_timeout: Duration,- /// Longest wait for the next bytes once connected.- pub stall_timeout: Duration,- /// Tries per mirror and file (each resumes where the last one stopped).- pub attempts: u32,-}--impl Default for Options {- fn default() -> Self {- Options { connect_timeout: Duration::from_secs(15), stall_timeout: Duration::from_secs(30), attempts: 3 }- }-}--/// Where a download is.-#[derive(Clone, Debug, Default, PartialEq, Eq)]-pub struct Progress {- pub file: String,- /// Bytes of all files on disk so far, and the total expected (pinned sizes + announced ones).- pub done: u64,- pub total: u64,- /// The mirror in use (without query).- pub mirror: String,-}--/// Why a download failed.-#[derive(Debug, Clone, PartialEq, Eq)]-pub enum DownloadError {- /// No mirror is configured.- NoMirrors,- Cancelled,- /// A local file problem (folder not writable, disk full…).- Disk(String),- /// Every mirror failed for `file`; the reason for each.- AllMirrorsFailed {- file: String,- errors: Vec<String>,- },-}--impl std::fmt::Display for DownloadError {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- match self {- DownloadError::NoMirrors => write!(- f,- "no download location is configured for the SAM 3 model in this build (set {MIRRORS_ENV}, or put the files in the model folder yourself; see docs/ai-masks.md)"- ),- DownloadError::Cancelled => write!(f, "the download was cancelled"),- DownloadError::Disk(e) => write!(f, "{e}"),- DownloadError::AllMirrorsFailed { file, errors } => write!(f, "could not download {file}: {}", errors.join("; ")),- }- }-}--/// One mirror's failure for one file; `Retry` resumes on the same mirror.-enum Fail {- Retry(String),- NextMirror(String),- Cancelled,- Disk(String),-}--/// Download `files` into `dir` from `mirrors` (in order; see [`mirrors`]). Files already there-/// and verified are kept. `progress` is called as bytes arrive (often: throttle in it).-pub fn download(- files: &[FileSpec],- mirrors: &[String],- dir: &Path,- opts: &Options,- cancel: &AtomicBool,- progress: &mut dyn FnMut(&Progress),-) -> Result<(), DownloadError> {- std::fs::create_dir_all(dir).map_err(|e| DownloadError::Disk(format!("can't create {}: {e}", dir.display())))?;- // what's left to fetch- let mut todo = Vec::new();- for f in files {- if !verified(f, &dir.join(f.name))? {- todo.push(f);- }- }- if todo.is_empty() {- return Ok(());- }- if mirrors.is_empty() {- return Err(DownloadError::NoMirrors);- }- let total: u64 = files.iter().map(|f| f.size.unwrap_or(0)).sum();- let mut before: u64 = files.iter().filter(|f| !todo.contains(f)).filter_map(|f| f.size).sum();- for f in todo {- let mut errors = Vec::new();- let mut ok = false;- 'mirrors: for m in mirrors {- let url = format!("{m}/{}", f.name);- for _ in 0..opts.attempts.max(1) {- if cancel.load(Ordering::Relaxed) {- return Err(DownloadError::Cancelled);- }- let mut report = |n: u64, announced: u64| {- let mirror = Url::parse(m).map(|u| u.display()).unwrap_or_default();- progress(&Progress {- file: f.name.to_string(),- done: before.saturating_add(n),- total: total.max(before.saturating_add(announced)),- mirror,- })- };- match fetch_file(f, &url, dir, opts, cancel, &mut report) {- Ok(()) => {- ok = true;- break 'mirrors;- }- Err(Fail::Cancelled) => return Err(DownloadError::Cancelled),- Err(Fail::Disk(e)) => return Err(DownloadError::Disk(e)),- Err(Fail::Retry(e)) => {- log::warn!("SAM 3 download of {url}: {e} (retrying)");- errors.push(format!("{}: {e}", short(m)));- }- Err(Fail::NextMirror(e)) => {- log::warn!("SAM 3 download of {url}: {e}");- errors.push(format!("{}: {e}", short(m)));- continue 'mirrors;- }- }- }- }- if !ok {- return Err(DownloadError::AllMirrorsFailed { file: f.name.to_string(), errors });- }- before = before.saturating_add(std::fs::metadata(dir.join(f.name)).map(|m| m.len()).unwrap_or(0));- }- Ok(())+ lightcraft_fetch::mirrors(env, file, DEFAULT_MIRRORS)}-/// A mirror for messages (no query string: it may hold a token).-fn short(m: &str) -> String {- Url::parse(m).map(|u| u.display()).unwrap_or_else(|_| "mirror".into())+/// What to tell the user when there is no mirror for the SAM 3 model.+pub fn no_mirrors_message() -> String {+ format!(+ "no download location is configured for the SAM 3 model in this build (set {MIRRORS_ENV}, or put the files in the model folder yourself; see docs/ai-masks.md)"+ )}-/// Whether `path` is already the right file (exact size and hash when pinned).-fn verified(f: &FileSpec, path: &Path) -> Result<bool, DownloadError> {- let Ok(meta) = std::fs::metadata(path) else { return Ok(false) };- if !meta.is_file() || meta.len() == 0 {- return Ok(false);- }- if f.size.is_some_and(|s| s != meta.len()) || meta.len() > f.max {- return Ok(false);- }- match f.sha256 {- Some(want) => Ok(sha256_file(path).map_err(DownloadError::Disk)? == want),- None => Ok(true),- }-}--fn sha256_file(path: &Path) -> Result<String, String> {- let mut file = File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;- let mut h = Sha256::new();- let mut buf = vec![0u8; 1 << 20];- loop {- let n = file.read(&mut buf).map_err(|e| format!("{}: {e}", path.display()))?;- if n == 0 {- break;- }- h.update(buf.get(..n).unwrap_or_default());- }- Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect())-}--fn part_path(dir: &Path, name: &str) -> PathBuf {- dir.join(format!("{name}.part"))-}--/// Fetch one file from one mirror into `<name>.part` (resuming it when its size is pinned),-/// check it and move it into place.-fn fetch_file(f: &FileSpec, url: &str, dir: &Path, opts: &Options, cancel: &AtomicBool, report: &mut dyn FnMut(u64, u64)) -> Result<(), Fail> {- let part = part_path(dir, f.name);- let limit = f.size.unwrap_or(f.max);- let disk = |e: std::io::Error| Fail::Disk(format!("{}: {e}", part.display()));- // resume only a pinned file (an unpinned one can't be checked after resuming)- let mut have = match std::fs::metadata(&part) {- Ok(m) if f.size.is_some() && m.len() <= limit => m.len(),- Ok(_) => {- std::fs::remove_file(&part).map_err(disk)?;- 0- }- Err(_) => 0,- };- let fetched = f.size != Some(have) || have == 0;- if fetched {- let limits = Limits { connect: opts.connect_timeout, stall: opts.stall_timeout, cancel };- let mut url = Url::parse(url).map_err(|e| Fail::NextMirror(e.to_string()))?;- let origin = url.host.clone();- let mut redirects = 0;- let mut resp = loop {- let mut headers = vec![];- if have > 0 {- headers.push(("Range", format!("bytes={have}-")));- }- // a token for the configured host only (never sent on to a redirect's host)- if let Ok(t) = std::env::var("LIGHTCRAFT_SAM3_TOKEN")- && !t.trim().is_empty()- && url.tls- && url.host == origin- {- headers.push(("Authorization", format!("Bearer {}", t.trim())));- }- let r = http::get(&url, &headers, &limits).map_err(http_fail)?;- match r.status {- 301 | 302 | 303 | 307 | 308 => {- redirects += 1;- if redirects > MAX_REDIRECTS {- return Err(Fail::NextMirror("too many redirects".into()));- }- let loc = r.header("location").ok_or_else(|| Fail::NextMirror("redirect without a location".into()))?;- url = url.join(loc).map_err(|e| Fail::NextMirror(e.to_string()))?;- }- _ => break r,- }- };- let mut file = match resp.status {- 200 => {- // the whole file (from the start, also when a range was asked for)- if resp.content_length().is_some_and(|n| n > limit || f.size.is_some_and(|s| s != n)) {- return Err(Fail::NextMirror(format!("the server's file has the wrong size ({:?} bytes)", resp.content_length())));- }- have = 0;- File::create(&part).map_err(disk)?- }- 206 if have > 0 => {- let range = resp.header("content-range").unwrap_or_default();- let (start, total) = parse_content_range(range).ok_or_else(|| Fail::NextMirror(format!("bad Content-Range `{range}`")))?;- if start != have || total.is_some_and(|t| Some(t) != f.size) {- return Err(Fail::NextMirror(format!("the server resumed at the wrong place ({range})")));- }- OpenOptions::new().append(true).open(&part).map_err(disk)?- }- 416 => {- // the partial file doesn't fit what the server has: start over- std::fs::remove_file(&part).map_err(disk)?;- return Err(Fail::Retry("the partial download didn't match; starting over".into()));- }- 401 | 403 => return Err(Fail::NextMirror(format!("access denied (HTTP {})", resp.status))),- 404 | 410 => return Err(Fail::NextMirror(format!("not found (HTTP {})", resp.status))),- s if s >= 500 => return Err(Fail::Retry(format!("server error (HTTP {s})"))),- s => return Err(Fail::NextMirror(format!("HTTP {s}"))),- };- let announced = f.size.unwrap_or_else(|| resp.content_length().unwrap_or(0));- report(have, announced);- let mut buf = vec![0u8; 256 * 1024];- loop {- let n = match resp.read(&mut buf, &limits) {- Ok(n) => n,- Err(e) => {- let _ = file.flush();- if f.size.is_none() {- drop(file);- let _ = std::fs::remove_file(&part);- }- return Err(http_fail(e));- }- };- if n == 0 {- break;- }- if have.saturating_add(n as u64) > limit {- drop(file);- let _ = std::fs::remove_file(&part);- return Err(Fail::NextMirror(format!("the server sent more than the expected {limit} bytes")));- }- file.write_all(buf.get(..n).unwrap_or_default()).map_err(disk)?;- have = have.saturating_add(n as u64);- report(have, announced);- }- file.sync_all().map_err(disk)?;- }- // checks: exact size, hash- let len = std::fs::metadata(&part).map_err(disk)?.len();- if f.size.is_some_and(|s| s != len) {- return Err(Fail::Retry(format!("incomplete ({len} of {} bytes)", f.size.unwrap_or(0))));- }- if len == 0 {- let _ = std::fs::remove_file(&part);- return Err(Fail::NextMirror("the server sent an empty file".into()));- }- if let Some(want) = f.sha256 {- let got = sha256_file(&part).map_err(Fail::Disk)?;- if got != want {- std::fs::remove_file(&part).map_err(disk)?;- let why = format!("the file is damaged or not the official one (SHA-256 {got})");- // a damaged partial file from an earlier run isn't this mirror's fault- return Err(if fetched { Fail::NextMirror(why) } else { Fail::Retry(why) });- }- }- std::fs::rename(&part, dir.join(f.name)).map_err(disk)?;- Ok(())-}--fn http_fail(e: HttpError) -> Fail {- match e {- HttpError::Cancelled => Fail::Cancelled,- HttpError::Stalled | HttpError::Io(_) | HttpError::Protocol(_) => Fail::Retry(e.to_string()),- HttpError::BadUrl(_) | HttpError::Connect(_) | HttpError::Tls(_) => Fail::NextMirror(e.to_string()),- }-}+#[cfg(test)]+mod tests {+ use super::*;-/// `bytes a-b/total` → (a, total) (`*` total → None).-fn parse_content_range(v: &str) -> Option<(u64, Option<u64>)> {- let v = v.trim().strip_prefix("bytes")?.trim();- let (range, total) = v.split_once('/')?;- let (a, b) = range.split_once('-')?;- let (a, b) = (a.trim().parse::<u64>().ok()?, b.trim().parse::<u64>().ok()?);- if b < a {- return None;+ #[test]+ fn sam3_mirrors_come_from_the_environment_the_file_and_the_defaults() {+ let m = mirrors(Some("https://a.example/x/"), Some(Path::new("/nonexistent/mirrors.txt")));+ assert_eq!(m.len(), 1 + DEFAULT_MIRRORS.len());+ assert!(no_mirrors_message().contains(MIRRORS_ENV));}- let total = match total.trim() {- "*" => None,- t => Some(t.parse::<u64>().ok()?),- };- Some((a, total))}--#[cfg(test)]-mod tests;diff --git a/docs/ai-masks.md b/docs/ai-masks.mdindex 26d4f717f..5a32d6902 100644--- a/docs/ai-masks.md+++ b/docs/ai-masks.md@@ -170,7 +170,7 @@ wait and return the result (when built with the `sam` feature; the default CLI hwith 2-D RoPE and windowed attention, the feature pyramids, the SAM 2-style prompt encoder andtwo-way mask decoder for clicks, and the CLIP text encoder, DETR encoder/decoder (boxrelative-position bias, presence token) and pixel decoder for text. The CLIP tokenizer is a- small BPE in `tokenizer.rs`. `fetch/` is the downloader (original code).+ small BPE in `tokenizer.rs`. The downloader (original code) is the `lightcraft-fetch` crate, shared by any model that is downloaded; `fetch/` here says what SAM 3 needs.- candle is pinned at 0.9.2: later releases make `candle-core` depend on `tokenizers` with theOniguruma C library, and the product is pure Rust. Weights are read with positional reads (nomemory map: `unsafe` stays in `lightcraft-sysmem`), only the tensors a path needs, and everydiff --git a/xtask/src/layers.rs b/xtask/src/layers.rsindex 28c994622..28d52677c 100644--- a/xtask/src/layers.rs+++ b/xtask/src/layers.rs@@ -38,6 +38,7 @@ pub const TABLE: &[(&str, Class)] = &[("raster", Class::Layer(0)),("tiff", Class::Layer(0)),("sysmem", Class::Layer(0)),+ ("fetch", Class::Layer(0)),("raw", Class::Layer(1)),("codecs", Class::Layer(1)),("meta", Class::Layer(1)),