Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

S2.30: SAM 2.1 Tiny in the model registry

Kristian Myrame5842c0c9236
diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs
index 21d316e75..786770ec5 100644
--- a/crates/engine/src/tests/models.rs
+++ b/crates/engine/src/tests/models.rs
@@ -36,7 +36,9 @@ fn models_are_listed_chosen_and_verified() {
for (prefix, model, task) in [("roto", "mobilesam", Task::Mask), ("face", "mediapipe-face", Task::Face)] {
let list = s.execute_checked(&format!("{prefix}.models"), json!({})).unwrap();
let ids: Vec<&str> = list["models"].as_array().unwrap().iter().map(|m| m["id"].as_str().unwrap()).collect();
- assert_eq!(ids, ["classical", model]);
+ // Roto Brush also offers SAM 2.1 Tiny (a video tracker; see crates/sam2).
+ let expect: Vec<&str> = if task == Task::Mask { vec!["classical", model, "sam2.1-hiera-tiny"] } else { vec!["classical", model] };
+ assert_eq!(ids, expect);
let m = &list["models"][1];
assert_eq!((m["licence"].as_str(), m["installed"].as_bool()), (Some("Apache-2.0"), Some(false)));
assert!(m["url"].as_str().unwrap().starts_with("https://") && !m["authors"].as_str().unwrap().is_empty());
diff --git a/crates/segment/README.md b/crates/segment/README.md
index 79b1d996b..4962f8ef8 100644
--- a/crates/segment/README.md
+++ b/crates/segment/README.md
@@ -36,6 +36,7 @@ kernels at run time). No ML framework, no unsafe code.
| id | Task | Model | Authors | Licence | File | Size |
|---|---|---|---|---|---|---|
| `mobilesam` | Roto Brush | [MobileSAM](https://github.com/ChaoningZhang/MobileSAM): Segment Anything with a 5M-parameter TinyViT encoder | Chaoning Zhang et al., building on Meta AI's Segment Anything and Microsoft's TinyViT | Apache-2.0 | `mobile_sam.pt` from the official repository | 40.7 MB |
+| `sam2.1-hiera-tiny` | Roto Brush | [SAM 2.1 Tiny](https://github.com/facebookresearch/sam2): Hiera-Tiny video object tracker with streaming memory | Meta AI (Nikhila Ravi et al.) | Apache-2.0 | `model.safetensors` of `facebook/sam2.1-hiera-tiny` (Hugging Face, pinned to a commit); loaded by `effectcraft-sam2` | 155.9 MB |
| `mediapipe-face` | Face tracking | [MediaPipe Face Landmarker](https://developers.google.com/edge/mediapipe/solutions/vision/face_landmarker): BlazeFace detector and Face Mesh V2 | Google LLC | Apache-2.0 ([model card](https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Face%20Mesh%20V2.pdf)) | `face_landmarker.task` (float16, version 1) from Google's model storage | 3.8 MB |
How they were checked:
diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs
index adeb44881..dbfe07c1f 100644
--- a/crates/segment/src/lib.rs
+++ b/crates/segment/src/lib.rs
@@ -96,6 +96,24 @@ pub const MOBILE_SAM: ModelInfo = ModelInfo {
sha256: "6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f",
};
+/// SAM 2.1 Tiny (Meta AI, Apache-2.0): a video object tracker with streaming memory. The weights are
+/// the Hugging Face `transformers` checkpoint, pinned to a commit so the SHA-256 stays valid; they
+/// are loaded by `effectcraft-sam2` (native only), not by [`load`].
+pub const SAM2_TINY: ModelInfo = ModelInfo {
+ id: "sam2.1-hiera-tiny",
+ task: Task::Mask,
+ name: "SAM 2.1 Tiny",
+ description: "Meta's Segment Anything Model 2 (Hiera-Tiny, 39M parameters). Follows the object through the clip with a streaming memory: it keeps tracking through occlusion and when the object leaves and re-enters the frame. Slower than MobileSAM on a CPU.",
+ authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint",
+ licence: "Apache-2.0",
+ licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
+ homepage: "https://github.com/facebookresearch/sam2",
+ url: "https://huggingface.co/facebook/sam2.1-hiera-tiny/resolve/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors",
+ file_name: "sam2.1-hiera-tiny.safetensors",
+ size: 155_908_064,
+ sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7",
+};
+
pub const FACE_LANDMARKER: ModelInfo = ModelInfo {
id: "mediapipe-face",
task: Task::Face,
@@ -112,7 +130,7 @@ pub const FACE_LANDMARKER: ModelInfo = ModelInfo {
};
/// The trained models, besides the built-in classical engines.
-pub const MODELS: &[ModelInfo] = &[MOBILE_SAM, FACE_LANDMARKER];
+pub const MODELS: &[ModelInfo] = &[MOBILE_SAM, SAM2_TINY, FACE_LANDMARKER];
/// The id of the built-in classical engines (graph cut segmenter, shape-model face tracker): no
/// weights, always available.
@@ -150,6 +168,7 @@ pub fn load(id: &str, bytes: &[u8]) -> Result<Loaded> {
Ok(Loaded::Mask(Arc::new(Sam { net: mobilesam::MobileSam::from_weights(&w)? })))
}
"mediapipe-face" => Ok(Loaded::Face(Arc::new(mediapipe::FaceLandmarker::from_task(bytes)?))),
+ "sam2.1-hiera-tiny" => Err(format!("{} is loaded by the desktop application (effectcraft-sam2)", info.name)),
_ => Err(format!("no loader for `{id}`")),
}
}
@@ -263,6 +282,8 @@ mod tests {
assert!(m.url.starts_with("https://") && m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));
assert!(!m.authors.is_empty() && notice(m).contains(m.authors));
}
+ assert_eq!(models(Task::Mask).map(|m| m.id).collect::<Vec<_>>(), ["mobilesam", "sam2.1-hiera-tiny"]);
+ assert!(SAM2_TINY.url.contains("/resolve/") && SAM2_TINY.file_name.ends_with(".safetensors"), "pinned to a commit, not a branch");
assert_eq!(models(Task::Face).map(|m| m.id).collect::<Vec<_>>(), ["mediapipe-face"]);
assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match")));
// A file that isn't the published one is refused before anything is parsed.
diff --git a/docs/agents.md b/docs/agents.md
index d801fcec6..864e7cf59 100644
--- a/docs/agents.md
+++ b/docs/agents.md
@@ -506,7 +506,7 @@ Trained model (Roto Brush 2.0 / 3.0):
licence, size, URL, SHA-256 and whether it is installed, selected and active.
- `roto.model.download {id}` fetches and verifies the official weights in the background;
`roto.model.install {path, id?}` installs a file you already have.
-- `roto.model.select {id: classical|mobilesam}` chooses which one Roto Brush uses. Loading runs
+- `roto.model.select {id: classical|mobilesam|sam2.1-hiera-tiny}` chooses which one Roto Brush uses. Loading runs
in the background; `"wait": true` (also on download and install) returns once it is in use.
- `roto.model.remove {id}` deletes the installed weights.
- The `version` property (`effects/#1/version`: 0 = 1.0, 1 = 2.0, 2 = 3.0) picks between the