Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

S2.73: Tiny is the recommended size (benchmarked per size); stroke count and remove button per object; download test

Kristian Myrama0ec3e2da4d0
diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs
index 1dbc507aa..ed0eecf9b 100644
--- a/crates/engine/src/tests/models.rs
+++ b/crates/engine/src/tests/models.rs
@@ -160,3 +160,26 @@ fn an_idle_roto_model_is_unloaded() {
crate::effects::roto::set_model(None);
let _ = std::fs::remove_dir_all(d);
}
+
+/// With `EFFECTCRAFT_DOWNLOAD_SIZE=sam2.1-hiera-small` (or another registry id): the in-app download (Settings ▸ Roto
+/// Brush ▸ Download) fetches the pinned file over the network, verifies it, installs it with its notice and loads it.
+/// Skipped without the variable: it needs the network and hundreds of megabytes.
+#[cfg(not(target_arch = "wasm32"))]
+#[test]
+fn downloading_a_size_installs_and_loads_it() {
+ let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return };
+ crate::tests_roto::hold_roto_cache_test_lock();
+ let (mut s, d) = session("download");
+ let r = s.execute_checked("roto.model.download", json!({"id": id, "wait": true})).unwrap();
+ assert_eq!(r["id"], id.as_str(), "{r}");
+ let dir = s.models_dir().unwrap();
+ let info = effectcraft_segment::info(&id).unwrap();
+ let file = dir.join(info.file_name);
+ assert_eq!(std::fs::metadata(&file).unwrap().len(), info.size, "the whole file arrived");
+ assert!(dir.join(format!("{}.NOTICE.txt", info.file_name)).exists());
+ assert!(!dir.join(format!("{}.part", info.file_name)).exists(), "no partial file is left");
+ s.execute_checked("roto.model.select", json!({"id": id})).unwrap();
+ wait_loaded(&mut s, Task::Mask);
+ assert_eq!(s.models.status(Task::Mask).active.as_deref(), Some(id.as_str()));
+ let _ = std::fs::remove_dir_all(d);
+}
diff --git a/crates/sam2/src/mask.rs b/crates/sam2/src/mask.rs
index d840d4052..8470bd4f3 100644
--- a/crates/sam2/src/mask.rs
+++ b/crates/sam2/src/mask.rs
@@ -148,25 +148,29 @@ pub fn load(bytes: &[u8]) -> std::result::Result<Arc<Sam2Mask>, String> {
Ok(Arc::new(Sam2Mask::with_info(Arc::new(model), info)))
}
-/// The size to suggest on this computer: Small when a GPU is usable (no slower to run than Tiny there, and more accurate),
-/// Tiny otherwise (the bigger backbones take several seconds a frame on a CPU).
+/// The size to suggest: Tiny. On the benchmark clips (`bench --roto`, 24 frames, a GPU) no larger size tracked better, they
+/// cost 1.0x (Small), 1.15x (Base+) and 1.7x (Large) the time, and Large lost the object after it re-entered the frame.
pub fn recommended() -> &'static ModelInfo {
- if crate::gpu::Gpu::shared().is_ok() { &effectcraft_segment::SAM2_SMALL } else { &SAM2_TINY }
+ &SAM2_TINY
}
-/// A one-line note for the Settings list: what suits this computer, and what the bigger sizes cost without a GPU.
+/// A one-line note for the Settings list: what was measured for this size, and what the bigger ones cost without a GPU.
pub fn hint(id: &str) -> Option<String> {
- if !SIZES.iter().any(|m| m.id == id) {
- return None;
- }
let gpu = crate::gpu::Gpu::shared().is_ok();
- Some(match (id == recommended().id, gpu, id) {
- (true, true, _) => "Recommended: a GPU was found, and this size is no slower than Tiny on it.".to_string(),
- (true, false, _) => "Recommended: no GPU was found, and this is the quickest on a CPU.".to_string(),
- (false, false, "sam2.1-hiera-base-plus" | "sam2.1-hiera-large") => "Slow without a GPU (several seconds a frame); none was found.".to_string(),
- (false, true, "sam2.1-hiera-large") => "For difficult footage; about 0.7 s of GPU time to look at each frame.".to_string(),
- _ => return None,
- })
+ Some(
+ match id {
+ "sam2.1-hiera-tiny" => "Recommended: the smallest download, and no larger size tracked better on the benchmark clips.",
+ "sam2.1-hiera-small" => "As fast as Tiny on a GPU and no better on the benchmark clips; worth a try on difficult footage.",
+ "sam2.1-hiera-base-plus" if !gpu => "Slow without a GPU (several seconds a frame); none was found.",
+ "sam2.1-hiera-base-plus" => "About 15% slower than Tiny on a GPU and no better on the benchmark clips.",
+ "sam2.1-hiera-large" if !gpu => "Slow without a GPU (many seconds a frame); none was found.",
+ "sam2.1-hiera-large" => {
+ "About 1.7 times Tiny's time on a GPU; it lost the object after it re-entered in one benchmark clip. Try it on difficult footage."
+ }
+ _ => return None,
+ }
+ .to_string(),
+ )
}
/// The registry entries this crate loads.
diff --git a/crates/ui-egui/src/panels/roto_tool.rs b/crates/ui-egui/src/panels/roto_tool.rs
index 56fa39b0a..0b42befbe 100644
--- a/crates/ui-egui/src/panels/roto_tool.rs
+++ b/crates/ui-egui/src/panels/roto_tool.rs
@@ -417,8 +417,9 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
let objects = fx::data(&params).objects();
let current = app.session.state.roto.object;
let known = objects.contains(&current);
- let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(86.0, 20.0));
- let label = tr_args("Object {}", &[&(current + 1)]);
+ let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(104.0, 20.0));
+ let strokes = fx::data(&params).strokes.iter().filter(|st| st.object == current && st.kind.is_segmentation()).count();
+ let label = if strokes == 0 { tr_args("Object {}", &[&(current + 1)]) } else { tr_args("Object {} ({})", &[&(current + 1), &strokes]) };
if widgets::text_button(ui, r, &label, false, &t, egui::Id::new(("roto-object", uid))).clicked() {
// cycle through the objects that have strokes (and the empty one being drawn)
let mut all = objects.clone();
@@ -431,14 +432,26 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
}
p.circle_filled(pos2(r.min.x + 9.0, r.center().y), 4.0, object_color(current, app.session.state.roto.boundary_color));
app.auto.add("layerPanel.roto.object", r, &label);
- x += 90.0;
+ x += 108.0;
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));
if widgets::text_button(ui, r, "+", false, &t, egui::Id::new(("roto-object-new", uid))).clicked() {
let next = objects.iter().copied().chain([current]).max().map_or(0, |m| m + 1);
let _ = app.session.execute("roto.options", json!({"object": next}));
}
app.auto.add("layerPanel.roto.newObject", r, "New object");
- x += 34.0;
+ x += 30.0;
+ // remove the current object's strokes (the object stays selected; with none left it is simply empty)
+ if strokes > 0 {
+ let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));
+ let c = widgets::text_button(ui, r, "×", false, &t, egui::Id::new(("roto-object-clear", uid))).on_hover_text(tr("Remove this object's strokes"));
+ if c.clicked()
+ && let Err(e) = app.session.execute("roto.clearStrokes", json!({"layer": layer.id.0, "effect": uid, "object": current}))
+ {
+ app.ui.status = e.to_string();
+ }
+ app.auto.add("layerPanel.roto.clearObject", r, "Remove object strokes");
+ x += 30.0;
+ }
if running {
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(60.0, 20.0));
if widgets::text_button(ui, r, tr("Stop"), false, &t, egui::Id::unique("roto-stop")).clicked() {
diff --git a/crates/ui-egui/tests/ui_roto_objects.rs b/crates/ui-egui/tests/ui_roto_objects.rs
index ee5efc020..5d16b0ada 100644
--- a/crates/ui-egui/tests/ui_roto_objects.rs
+++ b/crates/ui-egui/tests/ui_roto_objects.rs
@@ -84,6 +84,9 @@ fn the_boundary_view_outlines_each_object_and_show_object_picks_one() {
if let Some(path) = std::env::var_os("ROTO_OBJECTS_SNAPSHOT") {
both.save(path).unwrap();
}
+ // the bar offers to remove the current object's strokes, and says how many it has
+ assert!(h.state().auto.find("layerPanel.roto.clearObject").is_some(), "the remove button is on screen");
+ assert!(h.state().auto.find("layerPanel.roto.object").is_some_and(|e| e.label.contains("(1)")), "the object's stroke count is on the button");
let (a, b) = (count(&both, pink), count(&both, cyan));
assert!(a > 50 && b > 50, "both objects are outlined: {a} pink, {b} cyan pixels");
diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md
index 4e1792404..72c48e4e1 100644
--- a/docs/roto-brush-sam2.md
+++ b/docs/roto-brush-sam2.md
@@ -65,9 +65,27 @@ pinned to a Hugging Face commit with its SHA-256. All run the same tracker; a bi
| SAM 2.1 Base+ (`sam2.1-hiera-base-plus`) | 323.5 MB | 0.37 s | encoder 2.3e-5 |
| SAM 2.1 Large (`sam2.1-hiera-large`) | 897.9 MB | 0.72 s | encoder 5.2e-4 |
-Settings marks the size that suits the computer: **Small** when a usable GPU is found (it encodes no slower than
-Tiny there and is more accurate), **Tiny** otherwise; Base+ and Large say so when they would be slow on a CPU. The list
-carries `recommended` and `hint` per model (`roto.models`).
+### How the sizes compare on the benchmark clips
+`effectcraft-cli bench --roto --frames 24 --fixes --sam2 FILE`, release build, SAM 2 on the GPU (RX 7900 GRE), one run each.
+Mean IoU (after the object returns, in brackets) and seconds per tracked frame:
+
+| Clip | Tiny | Small | Base+ | Large |
+|---|---|---|---|---|
+| move | 0.992 | 0.988 | 0.986 | 0.994 |
+| light | 0.990 | 0.985 | 0.987 | 0.993 |
+| occlusion | 0.989 (0.987) | 0.988 (0.984) | 0.987 (0.984) | 0.989 (0.987) |
+| reenter | 0.982 (0.975) | 0.981 (0.970) | 0.978 (0.966) | **0.694 (0.110)** |
+| s / tracked frame | 0.69 | 0.70 | 0.79 | 1.17 |
+
+The classic engine scores 0.98 / 0.98 / 0.49 / 0.62 on the same clips. Peak tracker state is 88 MB for every size (the
+weights are not counted). The clips are synthetic and easy, so they cannot tell the sizes apart on accuracy: **no larger
+size tracked better here, and Large lost the object after it left and came back** (IoU 0.11 after the return, where
+Tiny, Small and Base+ found it again). That is the model's behaviour on this clip, not the port:
+Large matches PyTorch stage by stage. Whether a larger size helps on real footage is not measured.
+
+Settings therefore marks **Tiny** as recommended (the smallest download, and nothing larger did better), and says what was
+measured for the others; Base+ and Large also warn when no GPU was found (they take several seconds a frame on a CPU).
+The list carries `recommended` and `hint` per model (`roto.models`).
### Half-precision weights (optional)
`EFFECTCRAFT_SAM2_F16=1` keeps the weight matrices on the GPU as half precision, halving their memory (Large: about