Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

#1539: keep SAM2 weights install-only until mirrored

Kristian Myram33f545cf6475
diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rs
index a64e880d1..5d5cf436d 100644
--- a/crates/engine/src/analysis/models.rs
+++ b/crates/engine/src/analysis/models.rs
@@ -635,6 +635,9 @@ fn download(s: &mut Session, p: &Value, task: Task) -> Result<Value> {
let k = kind(task);
let cmd = format!("{}.model.download", k.prefix);
let m = model_p(p, &cmd, task)?;
+ if m.url.is_empty() {
+ return Err(bad(&cmd, format!("{} has no app download available: use Install from File with the SHA-256-verified weights", m.name)));
+ }
let dir = s.models_dir().ok_or_else(|| bad(&cmd, "no settings folder to install into"))?;
if s.models.status(task).busy.is_some() {
return Err(bad(&cmd, "a model is already downloading or loading"));
@@ -748,6 +751,19 @@ pub fn specs() -> Vec<CommandSpec> {
mod availability_tests {
use super::*;
+ /// SAM 2 cannot start a network operation; file installation still verifies the published hash.
+ #[cfg(not(target_arch = "wasm32"))]
+ #[test]
+ fn sam2_is_install_from_file_only() {
+ let mut session = Session::default();
+ for model in [seg::SAM2_TINY, seg::SAM2_SMALL, seg::SAM2_BASE_PLUS, seg::SAM2_LARGE] {
+ let error = session.execute("roto.model.download", json!({"id": model.id})).unwrap_err();
+ assert!(error.to_string().contains("Install from File"));
+ assert!(session.models.status(Task::Mask).busy.is_none());
+ assert!(install_bytes(Path::new("unused"), &model, b"invalid weights").unwrap_err().contains("SHA-256 differs"));
+ }
+ }
+
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn stale_model_worker_completions_do_not_poison_the_current_choice() {
diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs
index e0347ffe1..c34d02ed7 100644
--- a/crates/engine/src/tests/models.rs
+++ b/crates/engine/src/tests/models.rs
@@ -249,19 +249,23 @@ fn reloading_for_a_stroke_does_not_block_or_retry_a_failed_load() {
let _ = std::fs::remove_dir_all(d);
}
-/// With `EFFECTCRAFT_DOWNLOAD_SIZE=sam2.1-hiera-small` (or another registry id): the in-app download (Settings ▸ Roto
-/// Brush ▸ Download) fetches the pinned file over the network, verifies it, installs it with its notice and loads it.
-/// Skipped without the variable: it needs the network and hundreds of megabytes.
+/// With `EFFECTCRAFT_DOWNLOAD_SIZE=mobilesam` (or another downloadable registry id): the in-app
+/// download verifies and loads the file. SAM 2 must refuse until a storytold mirror is available.
+/// Skipped without the variable: downloadable entries need the network and large files.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn downloading_a_size_installs_and_loads_it() {
let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return };
crate::tests_roto::hold_roto_cache_test_lock();
let (mut s, d) = session("download");
+ let info = effectcraft_segment::info(&id).unwrap();
+ if info.url.is_empty() {
+ assert!(s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap_err().to_string().contains("Install from File"));
+ return;
+ }
let r = s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap();
assert_eq!(r["id"], id.as_str(), "{r}");
let dir = s.models_dir().unwrap();
- let info = effectcraft_segment::info(&id).unwrap();
let file = dir.join(info.file_name);
assert_eq!(std::fs::metadata(&file).unwrap().len(), info.size, "the whole file arrived");
assert!(dir.join(format!("{}.NOTICE.txt", info.file_name)).exists());
diff --git a/crates/segment/README.md b/crates/segment/README.md
index e6e4c6c98..06ffce7ad 100644
--- a/crates/segment/README.md
+++ b/crates/segment/README.md
@@ -12,7 +12,8 @@ Trained models for Roto Brush and face tracking, each kind behind one swappable
depends on one model's layout. The classical face tracker stays built in as the fallback.
- **`MODELS`**: the registry. Every entry must be open source under a licence compatible with
EffectCraft's MIT OR Apache-2.0 (`ALLOWED_LICENCES`), with its task, authors, homepage, licence
- URL, the official weights URL, size and SHA-256. `load` refuses anything else.
+ URL, an app download URL when available, size and SHA-256. `load` refuses anything else.
+ SAM 2 is Install from File only until a storytold-controlled mirror is available.
- **Weights are never bundled.** They are downloaded on demand (or installed from a file) by the
engine (`roto.model.*` / `face.model.*`, Settings ▸ Roto Brush / Face Tracking), verified
against the registry's SHA-256 (`sha256`, dependency-free) and kept with a notice naming their
diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs
index 04894cb2b..0d9cc3134 100644
--- a/crates/segment/src/lib.rs
+++ b/crates/segment/src/lib.rs
@@ -141,7 +141,7 @@ pub struct ModelInfo {
pub licence_url: &'static str,
/// The project's page.
pub homepage: &'static str,
- /// The official weights file.
+ /// App download URL; empty when only Install from File is available.
pub url: &'static str,
pub file_name: &'static str,
pub size: u64,
@@ -168,8 +168,8 @@ pub const MOBILE_SAM: ModelInfo = ModelInfo {
};
/// SAM 2.1 Tiny (Meta AI, Apache-2.0): a video object tracker with streaming memory. The weights are
-/// the Hugging Face `transformers` checkpoint, pinned to a commit so the SHA-256 stays valid; they
-/// are loaded by `effectcraft-sam2` (native only), not by [`load`].
+/// the Hugging Face `transformers` checkpoint, identified by SHA-256. Until a storytold mirror
+/// is available, install it from a file. `effectcraft-sam2` (native only) loads it, not [`load`].
pub const SAM2_TINY: ModelInfo = ModelInfo {
id: "sam2.1-hiera-tiny",
task: Task::Mask,
@@ -179,7 +179,7 @@ pub const SAM2_TINY: ModelInfo = ModelInfo {
licence: "Apache-2.0",
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
homepage: "https://github.com/facebookresearch/sam2",
- url: "https://huggingface.co/facebook/sam2.1-hiera-tiny/resolve/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors",
+ url: "",
file_name: "sam2.1-hiera-tiny.safetensors",
size: 155_908_064,
sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7",
@@ -195,7 +195,7 @@ pub const SAM2_SMALL: ModelInfo = ModelInfo {
licence: "Apache-2.0",
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
homepage: "https://github.com/facebookresearch/sam2",
- url: "https://huggingface.co/facebook/sam2.1-hiera-small/resolve/ee5bba1d82bb8749febdf90f45e84b687142ba03/model.safetensors",
+ url: "",
file_name: "sam2.1-hiera-small.safetensors",
size: 184_305_280,
sha256: "0a4067b11ce1e23d5229203f11c718a823060d15a4b23fa2372a7d4b77cbbc60",
@@ -211,7 +211,7 @@ pub const SAM2_BASE_PLUS: ModelInfo = ModelInfo {
licence: "Apache-2.0",
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
homepage: "https://github.com/facebookresearch/sam2",
- url: "https://huggingface.co/facebook/sam2.1-hiera-base-plus/resolve/b7320756a13354e7530a63935656d35b2f91a290/model.safetensors",
+ url: "",
file_name: "sam2.1-hiera-base-plus.safetensors",
size: 323_476_296,
sha256: "2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780",
@@ -227,7 +227,7 @@ pub const SAM2_LARGE: ModelInfo = ModelInfo {
licence: "Apache-2.0",
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
homepage: "https://github.com/facebookresearch/sam2",
- url: "https://huggingface.co/facebook/sam2.1-hiera-large/resolve/665f8e2ad61cf5f53d65644ff27c8ee525124610/model.safetensors",
+ url: "",
file_name: "sam2.1-hiera-large.safetensors",
size: 897_897_416,
sha256: "dc407dce21301fd94abb395c5099b4f2c455fdc8a8f261ac3d0ea6d4cd197230",
@@ -314,9 +314,10 @@ fn loader(id: &str) -> Option<Loader> {
/// The notice kept next to installed weights: what they are, who made them, their licence.
pub fn notice(m: &ModelInfo) -> String {
+ let source = if m.url.is_empty() { m.homepage } else { m.url };
format!(
- "{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nDownloaded unmodified from the source above. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",
- m.name, m.authors, m.licence, m.licence, m.licence_url, m.url, m.homepage
+ "{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",
+ m.name, m.authors, m.licence, m.licence, m.licence_url, source, m.homepage
)
}
@@ -418,14 +419,19 @@ mod tests {
for m in MODELS {
assert!(ALLOWED_LICENCES.contains(&m.licence), "{}", m.id);
assert_eq!(m.sha256.len(), 64);
- assert!(m.url.starts_with("https://") && m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));
+ assert!(m.url.is_empty() || m.url.starts_with("https://"));
+ assert!(m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));
assert!(!m.authors.is_empty() && notice(m).contains(m.authors));
}
assert_eq!(
models(Task::Mask).map(|m| m.id).collect::<Vec<_>>(),
["mobilesam", "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"]
);
- assert!(SAM2_TINY.url.contains("/resolve/") && SAM2_TINY.file_name.ends_with(".safetensors"), "pinned to a commit, not a branch");
+ for model in [SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE] {
+ assert!(model.url.is_empty(), "SAM 2 has no app download until a storytold mirror is available");
+ assert!(model.file_name.ends_with(".safetensors"));
+ assert!(notice(&model).contains(&format!("Source: {}", model.homepage)));
+ }
assert_eq!(models(Task::Face).map(|m| m.id).collect::<Vec<_>>(), ["mediapipe-face"]);
assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match")));
// A file that isn't the published one is refused before anything is parsed.
diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs
index 8b4133fbc..5fdbf1c50 100644
--- a/crates/ui-egui/src/panels/settings.rs
+++ b/crates/ui-egui/src/panels/settings.rs
@@ -513,7 +513,7 @@ fn models_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, t: &Tokens, acts: &mut
return;
}
ui.horizontal(|ui| {
- if !installed && !web {
+ if !installed && !web && m["url"].as_str().is_some_and(|url| !url.is_empty()) {
let r = ui.add_enabled(busy.is_none(), egui::Button::new(tr("Download")));
reg(app, &format!("settings.{prefix}.download.{id}"), &r, "Download");
if r.clicked() {
@@ -924,3 +924,27 @@ pub fn recovery(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) {
app.ui.status = e;
}
}
+
+#[cfg(all(test, not(target_arch = "wasm32")))]
+mod tests {
+ use super::*;
+
+ /// Install-only models must not offer a download that the command will reject.
+ #[test]
+ fn sam2_settings_offer_file_installation_without_downloads() {
+ let mut app = EffectcraftApp::new(effectcraft_engine::Session::default());
+ let ctx = egui::Context::default();
+ let tokens = app.tokens;
+ let mut actions = vec![];
+ let mut output = ctx
+ .run_ui(egui::RawInput { screen_rect: Some(egui::Rect::from_min_size(egui::Pos2::ZERO, vec2(900.0, 2200.0))), ..Default::default() }, |ui| {
+ models_ui(&mut app, ui, &tokens, &mut actions, Task::Mask)
+ });
+ output.textures_delta.clear();
+ for id in ["sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] {
+ assert!(app.auto.find(&format!("settings.roto.download.{id}")).is_none());
+ assert!(app.auto.find(&format!("settings.roto.install.{id}")).is_some());
+ }
+ assert!(actions.is_empty());
+ }
+}
diff --git a/docs/agents.md b/docs/agents.md
index ef88acccb..1a5035f8f 100644
--- a/docs/agents.md
+++ b/docs/agents.md
@@ -504,8 +504,9 @@ reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/se
Trained model (Roto Brush 2.0 / 3.0):
- `roto.models` lists the built-in engine and every registered model, with its authors,
licence, size, URL, SHA-256 and whether it is installed, selected and active.
-- `roto.model.download {id}` fetches and verifies the official weights in the background;
- `roto.model.install {path, id?}` installs a file you already have.
+- `roto.model.download {id}` fetches and verifies weights when the registry provides an app download URL;
+ `roto.model.install {path, id?}` installs a file you already have. SAM 2 is Install from File only until
+ a storytold-controlled mirror is available; its size and SHA-256 are verified before installation.
- `roto.model.select {id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large}` chooses which one Roto Brush uses (SAM 2 also tracks the object through the clip, see [roto-brush-sam2.md](roto-brush-sam2.md); `roto.status` then reports `source` and per-frame `tracker.frames` with `confidence` and `occluded`). Loading runs
in the background; `"wait": true` (also on download and install) returns once it is in use.
- `roto.propagate` never falls back to the classic engine silently: a chosen model that is not installed, or failed
diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md
index f2a2cb20d..5b8038552 100644
--- a/docs/roto-brush-sam2.md
+++ b/docs/roto-brush-sam2.md
@@ -4,10 +4,10 @@ Roto Brush can follow an object through a clip with Meta's **SAM 2.1 Tiny** (Hie
Apache-2.0). It is optional: nothing changes until you install the model and choose it.
## Using it
-1. Settings ▸ Roto Brush ▸ SAM 2.1 Tiny ▸ **Download** (downloads `model.safetensors`, 155.9 MB, from the pinned
- Hugging Face commit; the SHA-256 is checked and the licence notice is kept next to the file). Or install from a
- file you downloaded. Agents: `roto.model.download {id: "sam2.1-hiera-tiny", wait: true}`, or
- `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}` for a file you already have.
+1. Settings ▸ Roto Brush ▸ SAM 2.1 Tiny ▸ **Install from File…** with the published `model.safetensors`
+ (155.9 MB). The file's size and SHA-256 are checked and the licence notice is kept next to it.
+ Agents: `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}`.
+ SAM 2 has no in-app download until the weights are mirrored under storytold control.
2. Choose it (`roto.model.select {id: "sam2.1-hiera-tiny"}`).
3. Paint strokes on the base frame as usual, then **Propagate**. One tracking pass follows the object through the
whole span in both directions, using SAM 2's streaming memory, so it keeps tracking through occlusion and when the
@@ -59,7 +59,7 @@ instances can see different pixels. Objects inside one instance always see the s
Four checkpoints are in the registry (Settings ▸ Roto Brush); the size is detected from the file, and each is
pinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bigger backbone costs time and memory.
-| Model | Download | GPU encode (RX 7900 GRE) | Agreement with PyTorch |
+| Model | File size | GPU encode (RX 7900 GRE) | Agreement with PyTorch |
|---|---|---|---|
| SAM 2.1 Tiny (`sam2.1-hiera-tiny`) | 155.9 MB | 0.24 s | encoder 6.8e-6 |
| SAM 2.1 Small (`sam2.1-hiera-small`) | 184.3 MB | 0.20 s | encoder 1.0e-5 |