EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
#1539: keep SAM2 weights install-only until mirrored
Kristian Myram33f545cf6475
diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rsindex a64e880d1..5d5cf436d 100644--- a/crates/engine/src/analysis/models.rs+++ b/crates/engine/src/analysis/models.rs@@ -635,6 +635,9 @@ fn download(s: &mut Session, p: &Value, task: Task) -> Result<Value> {let k = kind(task);let cmd = format!("{}.model.download", k.prefix);let m = model_p(p, &cmd, task)?;+ if m.url.is_empty() {+ return Err(bad(&cmd, format!("{} has no app download available: use Install from File with the SHA-256-verified weights", m.name)));+ }let dir = s.models_dir().ok_or_else(|| bad(&cmd, "no settings folder to install into"))?;if s.models.status(task).busy.is_some() {return Err(bad(&cmd, "a model is already downloading or loading"));@@ -748,6 +751,19 @@ pub fn specs() -> Vec<CommandSpec> {mod availability_tests {use super::*;+ /// SAM 2 cannot start a network operation; file installation still verifies the published hash.+ #[cfg(not(target_arch = "wasm32"))]+ #[test]+ fn sam2_is_install_from_file_only() {+ let mut session = Session::default();+ for model in [seg::SAM2_TINY, seg::SAM2_SMALL, seg::SAM2_BASE_PLUS, seg::SAM2_LARGE] {+ let error = session.execute("roto.model.download", json!({"id": model.id})).unwrap_err();+ assert!(error.to_string().contains("Install from File"));+ assert!(session.models.status(Task::Mask).busy.is_none());+ assert!(install_bytes(Path::new("unused"), &model, b"invalid weights").unwrap_err().contains("SHA-256 differs"));+ }+ }+#[cfg(not(target_arch = "wasm32"))]#[test]fn stale_model_worker_completions_do_not_poison_the_current_choice() {diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rsindex e0347ffe1..c34d02ed7 100644--- a/crates/engine/src/tests/models.rs+++ b/crates/engine/src/tests/models.rs@@ -249,19 +249,23 @@ fn reloading_for_a_stroke_does_not_block_or_retry_a_failed_load() {let _ = std::fs::remove_dir_all(d);}-/// With `EFFECTCRAFT_DOWNLOAD_SIZE=sam2.1-hiera-small` (or another registry id): the in-app download (Settings ▸ Roto-/// Brush ▸ Download) fetches the pinned file over the network, verifies it, installs it with its notice and loads it.-/// Skipped without the variable: it needs the network and hundreds of megabytes.+/// With `EFFECTCRAFT_DOWNLOAD_SIZE=mobilesam` (or another downloadable registry id): the in-app+/// download verifies and loads the file. SAM 2 must refuse until a storytold mirror is available.+/// Skipped without the variable: downloadable entries need the network and large files.#[cfg(not(target_arch = "wasm32"))]#[test]fn downloading_a_size_installs_and_loads_it() {let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return };crate::tests_roto::hold_roto_cache_test_lock();let (mut s, d) = session("download");+ let info = effectcraft_segment::info(&id).unwrap();+ if info.url.is_empty() {+ assert!(s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap_err().to_string().contains("Install from File"));+ return;+ }let r = s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap();assert_eq!(r["id"], id.as_str(), "{r}");let dir = s.models_dir().unwrap();- let info = effectcraft_segment::info(&id).unwrap();let file = dir.join(info.file_name);assert_eq!(std::fs::metadata(&file).unwrap().len(), info.size, "the whole file arrived");assert!(dir.join(format!("{}.NOTICE.txt", info.file_name)).exists());diff --git a/crates/segment/README.md b/crates/segment/README.mdindex e6e4c6c98..06ffce7ad 100644--- a/crates/segment/README.md+++ b/crates/segment/README.md@@ -12,7 +12,8 @@ Trained models for Roto Brush and face tracking, each kind behind one swappabledepends on one model's layout. The classical face tracker stays built in as the fallback.- **`MODELS`**: the registry. Every entry must be open source under a licence compatible withEffectCraft's MIT OR Apache-2.0 (`ALLOWED_LICENCES`), with its task, authors, homepage, licence- URL, the official weights URL, size and SHA-256. `load` refuses anything else.+ URL, an app download URL when available, size and SHA-256. `load` refuses anything else.+ SAM 2 is Install from File only until a storytold-controlled mirror is available.- **Weights are never bundled.** They are downloaded on demand (or installed from a file) by theengine (`roto.model.*` / `face.model.*`, Settings ▸ Roto Brush / Face Tracking), verifiedagainst the registry's SHA-256 (`sha256`, dependency-free) and kept with a notice naming theirdiff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rsindex 04894cb2b..0d9cc3134 100644--- a/crates/segment/src/lib.rs+++ b/crates/segment/src/lib.rs@@ -141,7 +141,7 @@ pub struct ModelInfo {pub licence_url: &'static str,/// The project's page.pub homepage: &'static str,- /// The official weights file.+ /// App download URL; empty when only Install from File is available.pub url: &'static str,pub file_name: &'static str,pub size: u64,@@ -168,8 +168,8 @@ pub const MOBILE_SAM: ModelInfo = ModelInfo {};/// SAM 2.1 Tiny (Meta AI, Apache-2.0): a video object tracker with streaming memory. The weights are-/// the Hugging Face `transformers` checkpoint, pinned to a commit so the SHA-256 stays valid; they-/// are loaded by `effectcraft-sam2` (native only), not by [`load`].+/// the Hugging Face `transformers` checkpoint, identified by SHA-256. Until a storytold mirror+/// is available, install it from a file. `effectcraft-sam2` (native only) loads it, not [`load`].pub const SAM2_TINY: ModelInfo = ModelInfo {id: "sam2.1-hiera-tiny",task: Task::Mask,@@ -179,7 +179,7 @@ pub const SAM2_TINY: ModelInfo = ModelInfo {licence: "Apache-2.0",licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",homepage: "https://github.com/facebookresearch/sam2",- url: "https://huggingface.co/facebook/sam2.1-hiera-tiny/resolve/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors",+ url: "",file_name: "sam2.1-hiera-tiny.safetensors",size: 155_908_064,sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7",@@ -195,7 +195,7 @@ pub const SAM2_SMALL: ModelInfo = ModelInfo {licence: "Apache-2.0",licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",homepage: "https://github.com/facebookresearch/sam2",- url: "https://huggingface.co/facebook/sam2.1-hiera-small/resolve/ee5bba1d82bb8749febdf90f45e84b687142ba03/model.safetensors",+ url: "",file_name: "sam2.1-hiera-small.safetensors",size: 184_305_280,sha256: "0a4067b11ce1e23d5229203f11c718a823060d15a4b23fa2372a7d4b77cbbc60",@@ -211,7 +211,7 @@ pub const SAM2_BASE_PLUS: ModelInfo = ModelInfo {licence: "Apache-2.0",licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",homepage: "https://github.com/facebookresearch/sam2",- url: "https://huggingface.co/facebook/sam2.1-hiera-base-plus/resolve/b7320756a13354e7530a63935656d35b2f91a290/model.safetensors",+ url: "",file_name: "sam2.1-hiera-base-plus.safetensors",size: 323_476_296,sha256: "2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780",@@ -227,7 +227,7 @@ pub const SAM2_LARGE: ModelInfo = ModelInfo {licence: "Apache-2.0",licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",homepage: "https://github.com/facebookresearch/sam2",- url: "https://huggingface.co/facebook/sam2.1-hiera-large/resolve/665f8e2ad61cf5f53d65644ff27c8ee525124610/model.safetensors",+ url: "",file_name: "sam2.1-hiera-large.safetensors",size: 897_897_416,sha256: "dc407dce21301fd94abb395c5099b4f2c455fdc8a8f261ac3d0ea6d4cd197230",@@ -314,9 +314,10 @@ fn loader(id: &str) -> Option<Loader> {/// The notice kept next to installed weights: what they are, who made them, their licence.pub fn notice(m: &ModelInfo) -> String {+ let source = if m.url.is_empty() { m.homepage } else { m.url };format!(- "{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nDownloaded unmodified from the source above. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",- m.name, m.authors, m.licence, m.licence, m.licence_url, m.url, m.homepage+ "{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",+ m.name, m.authors, m.licence, m.licence, m.licence_url, source, m.homepage)}@@ -418,14 +419,19 @@ mod tests {for m in MODELS {assert!(ALLOWED_LICENCES.contains(&m.licence), "{}", m.id);assert_eq!(m.sha256.len(), 64);- assert!(m.url.starts_with("https://") && m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));+ assert!(m.url.is_empty() || m.url.starts_with("https://"));+ assert!(m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));assert!(!m.authors.is_empty() && notice(m).contains(m.authors));}assert_eq!(models(Task::Mask).map(|m| m.id).collect::<Vec<_>>(),["mobilesam", "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"]);- assert!(SAM2_TINY.url.contains("/resolve/") && SAM2_TINY.file_name.ends_with(".safetensors"), "pinned to a commit, not a branch");+ for model in [SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE] {+ assert!(model.url.is_empty(), "SAM 2 has no app download until a storytold mirror is available");+ assert!(model.file_name.ends_with(".safetensors"));+ assert!(notice(&model).contains(&format!("Source: {}", model.homepage)));+ }assert_eq!(models(Task::Face).map(|m| m.id).collect::<Vec<_>>(), ["mediapipe-face"]);assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match")));// A file that isn't the published one is refused before anything is parsed.diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rsindex 8b4133fbc..5fdbf1c50 100644--- a/crates/ui-egui/src/panels/settings.rs+++ b/crates/ui-egui/src/panels/settings.rs@@ -513,7 +513,7 @@ fn models_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, t: &Tokens, acts: &mutreturn;}ui.horizontal(|ui| {- if !installed && !web {+ if !installed && !web && m["url"].as_str().is_some_and(|url| !url.is_empty()) {let r = ui.add_enabled(busy.is_none(), egui::Button::new(tr("Download")));reg(app, &format!("settings.{prefix}.download.{id}"), &r, "Download");if r.clicked() {@@ -924,3 +924,27 @@ pub fn recovery(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) {app.ui.status = e;}}++#[cfg(all(test, not(target_arch = "wasm32")))]+mod tests {+ use super::*;++ /// Install-only models must not offer a download that the command will reject.+ #[test]+ fn sam2_settings_offer_file_installation_without_downloads() {+ let mut app = EffectcraftApp::new(effectcraft_engine::Session::default());+ let ctx = egui::Context::default();+ let tokens = app.tokens;+ let mut actions = vec![];+ let mut output = ctx+ .run_ui(egui::RawInput { screen_rect: Some(egui::Rect::from_min_size(egui::Pos2::ZERO, vec2(900.0, 2200.0))), ..Default::default() }, |ui| {+ models_ui(&mut app, ui, &tokens, &mut actions, Task::Mask)+ });+ output.textures_delta.clear();+ for id in ["sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] {+ assert!(app.auto.find(&format!("settings.roto.download.{id}")).is_none());+ assert!(app.auto.find(&format!("settings.roto.install.{id}")).is_some());+ }+ assert!(actions.is_empty());+ }+}diff --git a/docs/agents.md b/docs/agents.mdindex ef88acccb..1a5035f8f 100644--- a/docs/agents.md+++ b/docs/agents.md@@ -504,8 +504,9 @@ reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/seTrained model (Roto Brush 2.0 / 3.0):- `roto.models` lists the built-in engine and every registered model, with its authors,licence, size, URL, SHA-256 and whether it is installed, selected and active.-- `roto.model.download {id}` fetches and verifies the official weights in the background;- `roto.model.install {path, id?}` installs a file you already have.+- `roto.model.download {id}` fetches and verifies weights when the registry provides an app download URL;+ `roto.model.install {path, id?}` installs a file you already have. SAM 2 is Install from File only until+ a storytold-controlled mirror is available; its size and SHA-256 are verified before installation.- `roto.model.select {id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large}` chooses which one Roto Brush uses (SAM 2 also tracks the object through the clip, see [roto-brush-sam2.md](roto-brush-sam2.md); `roto.status` then reports `source` and per-frame `tracker.frames` with `confidence` and `occluded`). Loading runsin the background; `"wait": true` (also on download and install) returns once it is in use.- `roto.propagate` never falls back to the classic engine silently: a chosen model that is not installed, or faileddiff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.mdindex f2a2cb20d..5b8038552 100644--- a/docs/roto-brush-sam2.md+++ b/docs/roto-brush-sam2.md@@ -4,10 +4,10 @@ Roto Brush can follow an object through a clip with Meta's **SAM 2.1 Tiny** (HieApache-2.0). It is optional: nothing changes until you install the model and choose it.## Using it-1. Settings ▸ Roto Brush ▸ SAM 2.1 Tiny ▸ **Download** (downloads `model.safetensors`, 155.9 MB, from the pinned- Hugging Face commit; the SHA-256 is checked and the licence notice is kept next to the file). Or install from a- file you downloaded. Agents: `roto.model.download {id: "sam2.1-hiera-tiny", wait: true}`, or- `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}` for a file you already have.+1. Settings ▸ Roto Brush ▸ SAM 2.1 Tiny ▸ **Install from File…** with the published `model.safetensors`+ (155.9 MB). The file's size and SHA-256 are checked and the licence notice is kept next to it.+ Agents: `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}`.+ SAM 2 has no in-app download until the weights are mirrored under storytold control.2. Choose it (`roto.model.select {id: "sam2.1-hiera-tiny"}`).3. Paint strokes on the base frame as usual, then **Propagate**. One tracking pass follows the object through thewhole span in both directions, using SAM 2's streaming memory, so it keeps tracking through occlusion and when the@@ -59,7 +59,7 @@ instances can see different pixels. Objects inside one instance always see the sFour checkpoints are in the registry (Settings ▸ Roto Brush); the size is detected from the file, and each ispinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bigger backbone costs time and memory.-| Model | Download | GPU encode (RX 7900 GRE) | Agreement with PyTorch |+| Model | File size | GPU encode (RX 7900 GRE) | Agreement with PyTorch ||---|---|---|---|| SAM 2.1 Tiny (`sam2.1-hiera-tiny`) | 155.9 MB | 0.24 s | encoder 6.8e-6 || SAM 2.1 Small (`sam2.1-hiera-small`) | 184.3 MB | 0.20 s | encoder 1.0e-5 |