Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

Smart brushes name a thing: SlimSAM reads the region under a stroke

Jareer9884ddeb91c0
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 49cfafd5..7776bb2c 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -438,6 +438,17 @@ either, because `apply::Mapping` is the one place a decoded pixel is
walked back to a source fraction. A mask the model left blank shows the
picture as shot rather than nothing.
+**The smart brushes name a thing.** A stroke records the source instant
+it was painted at; for a smart brush or smart eraser, `host::brush`
+decodes that frame, runs SlimSAM's encoder on it once (the embedding is
+kept for the next stroke on the same frame) and its decoder on sixteen
+points along the stroke, and writes the region it answers - the whole
+thing the stroke touched - beside the media's masks, named by a hash of
+the stroke. `strokes::paint` keeps or drops that region whole; until it
+has been read, or when the model is not to hand, the smart tools fall
+back to the mask's own confidence under a disc. The plain brush and
+eraser are discs and nothing more.
+
**The store is the project's.** Each media file's masks are PNGs in
`cache/masks/<hash>-<model>/`, one per analysed source instant, named by
that instant in milliseconds, ten a second of footage and one for a
diff --git a/engine/crates/concat-host/src/brush.rs b/engine/crates/concat-host/src/brush.rs
new file mode 100644
index 00000000..faa5bf13
--- /dev/null
+++ b/engine/crates/concat-host/src/brush.rs
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Reading what a smart stroke lies on.
+//!
+//! A smart brush or smart eraser names a thing in one frame, and the
+//! whole of it is kept or dropped. This is the job that finds the thing:
+//! it decodes the frame the stroke was painted at, runs the brush model's
+//! encoder on it - once per frame, the embedding kept for the next stroke
+//! on the same one - and its decoder on the stroke's points, and writes
+//! the region beside the media's masks, where `concat-vision` paints it
+//! in. The model's two files download on first use, like the segmenters'.
+//!
+//! One region at a time through a [`SingleFlight`]; the window queues the
+//! rest behind it.
+
+use std::collections::VecDeque;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+
+use concat_core::time::Rational;
+use concat_media::{DecodeOptions, Decoder, FrameSource};
+use concat_project::model::{Stroke, Subject};
+use concat_vision::{Brush, Embedding, ModelId, mask_dir, models, region_file};
+
+use crate::cutout::Progress;
+use crate::jobs::SingleFlight;
+
+/// One stroke to read.
+#[derive(Clone, Debug)]
+pub struct RegionRequest {
+ /// The project folder the masks are cached under.
+ pub project: PathBuf,
+ /// The media file the stroke was painted on.
+ pub media_path: String,
+ /// A still: its one frame, whatever the stroke's instant.
+ pub still: bool,
+ /// The cutout's subject, which names the masks' directory.
+ pub subject: Subject,
+ /// The stroke, with its instant.
+ pub stroke: Stroke,
+}
+
+impl RegionRequest {
+ /// Where the region goes.
+ pub fn file(&self) -> PathBuf {
+ region_file(
+ &mask_dir(&self.project, &self.media_path, self.subject),
+ &self.stroke,
+ )
+ }
+
+ /// Whether the region is still to be read.
+ pub fn outstanding(&self) -> bool {
+ self.stroke.is_smart() && self.stroke.at.is_some() && !self.file().is_file()
+ }
+}
+
+/// How many frames' embeddings stay in memory. Each is a few megabytes;
+/// a few frames is a session of strokes on one shot.
+const EMBEDDINGS: usize = 6;
+
+/// A frame read lately: its media and source millisecond, and what the
+/// encoder made of it.
+type Read = ((String, u64), Arc<Embedding>);
+
+/// The brush service: the model, loaded once, and the one-job slot.
+pub struct Brushes {
+ gate: Arc<SingleFlight>,
+ data: PathBuf,
+ brush: Mutex<Option<Arc<Brush>>>,
+ /// Frames read lately, by media and source millisecond.
+ embeddings: Mutex<VecDeque<Read>>,
+}
+
+impl Brushes {
+ /// A service with nothing loaded yet, keeping its models under `data`.
+ pub fn new(data: &Path) -> Brushes {
+ Brushes {
+ gate: Arc::new(SingleFlight::new()),
+ data: data.to_path_buf(),
+ brush: Mutex::new(None),
+ embeddings: Mutex::new(VecDeque::new()),
+ }
+ }
+
+ /// Whether a region is being read.
+ pub fn is_busy(&self) -> bool {
+ self.gate.is_busy()
+ }
+
+ /// Asks the running job to stop.
+ pub fn cancel(&self) {
+ self.gate.cancel();
+ }
+
+ fn brush(
+ &self,
+ cancel: &AtomicBool,
+ progress: &mut dyn FnMut(Progress),
+ ) -> Result<Arc<Brush>, String> {
+ if let Some(loaded) = self.brush.lock().map_err(|_| "brush slot poisoned")?.as_ref() {
+ return Ok(Arc::clone(loaded));
+ }
+ let encoder = models::model_file(&self.data, ModelId::BrushEncoder);
+ let decoder = models::model_file(&self.data, ModelId::BrushDecoder);
+ if !models::installed(&self.data, ModelId::BrushEncoder) {
+ crate::cutout::fetch(ModelId::BrushEncoder, &encoder, cancel, progress)?;
+ }
+ if !models::installed(&self.data, ModelId::BrushDecoder) {
+ crate::cutout::fetch(ModelId::BrushDecoder, &decoder, cancel, progress)?;
+ }
+ let brush = Arc::new(Brush::load(&encoder, &decoder)?);
+ *self.brush.lock().map_err(|_| "brush slot poisoned")? = Some(Arc::clone(&brush));
+ Ok(brush)
+ }
+
+ /// Reads the region under `request`'s stroke and writes it. Blocks
+ /// for the whole run, so run it on its own thread.
+ pub fn read(
+ &self,
+ request: &RegionRequest,
+ progress: &mut dyn FnMut(Progress),
+ ) -> Result<(), String> {
+ let job = self.gate.begin("brush reading")?;
+ let cancel = job.cancel_handle();
+ let Some(at) = request.stroke.at else {
+ return Err("the stroke has no instant".to_owned());
+ };
+ let file = request.file();
+ if file.is_file() {
+ return Ok(());
+ }
+ let brush = self.brush(&cancel, progress)?;
+ progress(Progress::Analysing(0.0));
+
+ let millis = if request.still {
+ 0
+ } else {
+ (at * 1000.0).round() as u64
+ };
+ let key = (request.media_path.clone(), millis);
+ let held = self
+ .embeddings
+ .lock()
+ .map_err(|_| "embeddings poisoned")?
+ .iter()
+ .find(|(k, _)| *k == key)
+ .map(|(_, embedding)| Arc::clone(embedding));
+ let embedding = match held {
+ Some(embedding) => embedding,
+ None => {
+ let facts = concat_media::probe(&request.media_path)
+ .map_err(|error| error.to_string())?;
+ let video = facts.require_video().map_err(|error| error.to_string())?;
+ let (w, h) = Brush::input_size(video.width, video.height);
+ let mut options = DecodeOptions::default().scaled_to(w, h).limited_to(1);
+ if !request.still {
+ options = options.starting_at(Rational::new(millis as i64, 1000));
+ }
+ let mut decoder = Decoder::open(&request.media_path, &options)
+ .map_err(|error| error.to_string())?;
+ let frame = decoder
+ .next_frame()
+ .map_err(|error| error.to_string())?
+ .ok_or_else(|| format!("{}: no picture at the stroke", request.media_path))?;
+ if cancel.load(Ordering::Relaxed) {
+ return Err("brush reading cancelled".to_owned());
+ }
+ let embedding = Arc::new(brush.embed(&frame)?);
+ if let Ok(mut held) = self.embeddings.lock() {
+ held.push_back((key, Arc::clone(&embedding)));
+ while held.len() > EMBEDDINGS {
+ held.pop_front();
+ }
+ }
+ embedding
+ }
+ };
+ progress(Progress::Analysing(0.5));
+ let region = brush.region(&embedding, &request.stroke.points)?;
+ if let Some(parent) = file.parent() {
+ std::fs::create_dir_all(parent)
+ .map_err(|error| format!("could not create {}: {error}", parent.display()))?;
+ }
+ std::fs::write(&file, region.to_png())
+ .map_err(|error| format!("could not write {}: {error}", file.display()))?;
+ progress(Progress::Analysing(1.0));
+ Ok(())
+ }
+}
diff --git a/engine/crates/concat-host/src/cutout.rs b/engine/crates/concat-host/src/cutout.rs
index 2ce6ea70..557ba5df 100644
--- a/engine/crates/concat-host/src/cutout.rs
+++ b/engine/crates/concat-host/src/cutout.rs
@@ -322,7 +322,7 @@ impl Cutouts {
/// Streams a model into `file`, by way of a `.part` beside it, reporting
/// every couple of megabytes and stopping when `cancel` is set.
-fn fetch(
+pub(crate) fn fetch(
id: ModelId,
file: &Path,
cancel: &AtomicBool,
diff --git a/engine/crates/concat-host/src/lib.rs b/engine/crates/concat-host/src/lib.rs
index 5e59c15f..855c131f 100644
--- a/engine/crates/concat-host/src/lib.rs
+++ b/engine/crates/concat-host/src/lib.rs
@@ -18,6 +18,7 @@
//! Nothing here knows about a window. Long work reports through callbacks
//! and cancels through flags, and the caller decides which thread it runs on.
+pub mod brush;
pub mod cutout;
pub mod dirs;
pub mod export;
@@ -30,6 +31,7 @@ pub mod session;
pub mod templates;
pub mod titles;
+pub use brush::{Brushes, RegionRequest};
pub use cutout::{AnalyseRequest, Cutouts};
pub use dirs::AppDirs;
pub use jobs::{Job, SingleFlight};
diff --git a/engine/crates/concat-project/src/model.rs b/engine/crates/concat-project/src/model.rs
index b6a673cc..bdd019f2 100644
--- a/engine/crates/concat-project/src/model.rs
+++ b/engine/crates/concat-project/src/model.rs
@@ -219,6 +219,11 @@ pub struct Stroke {
pub size: f64,
/// The path, as `[x, y]` fractions.
pub points: Vec<[f64; 2]>,
+ /// The source instant, in seconds, the stroke was painted at: the
+ /// frame a smart brush reads the thing under it from. Absent on
+ /// strokes from before it was recorded, which paint as plain discs.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub at: Option<f64>,
}
/// The four brushes of a custom cutout.
@@ -273,6 +278,12 @@ impl Cutout {
}
impl Stroke {
+ /// Whether the stroke's tool reads the picture rather than painting
+ /// a disc: the smart brush and the smart eraser.
+ pub fn is_smart(&self) -> bool {
+ matches!(self.tool, BrushTool::SmartBrush | BrushTool::SmartEraser)
+ }
+
/// The size held to its range and the points to the picture; `None`
/// for a stroke with no points left.
pub fn tidy(mut self) -> Option<Stroke> {
@@ -281,6 +292,7 @@ impl Stroke {
} else {
MIN_BRUSH
};
+ self.at = self.at.filter(|at| at.is_finite()).map(|at| at.max(0.0));
self.points.retain(|[x, y]| x.is_finite() && y.is_finite());
for point in &mut self.points {
point[0] = point[0].clamp(-0.5, 1.5);
diff --git a/engine/crates/concat-vision/src/brush.rs b/engine/crates/concat-vision/src/brush.rs
new file mode 100644
index 00000000..f0dc37fe
--- /dev/null
+++ b/engine/crates/concat-vision/src/brush.rs
@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! The thing under a brush stroke.
+//!
+//! The smart brush and smart eraser do not paint a disc: they name a
+//! thing, and the whole of it is kept or dropped. The model that reads
+//! the thing is SlimSAM, a small Segment Anything, in two halves. The
+//! encoder turns a frame into embeddings once, a few hundred milliseconds
+//! on a laptop's cores and a few tens with an accelerator; the decoder
+//! turns the stroke's points into the mask of what they lie on, in a few
+//! milliseconds, so a second stroke on the same frame costs almost
+//! nothing. A frame's embedding is therefore worth keeping between
+//! strokes, and [`Embedding`] is what a host caches.
+//!
+//! The model works on a 1024 square: the frame is scaled to fit by its
+//! long edge and the rest is padding. The mask it answers with covers
+//! that square; [`Brush::region`] reads it back as a mask over the source
+//! picture alone, in the fractions every other mask here uses.
+
+use std::path::Path;
+use std::sync::Mutex;
+
+use concat_core::frame::{BYTES_PER_PIXEL, Frame};
+
+use crate::Mask;
+use crate::runtime::{Input, Model, Output};
+
+/// The model's square.
+const SAM_SIZE: usize = 1024;
+/// The decoder's answer is a quarter the size of the square.
+const SAM_MASK: usize = 256;
+/// The region is stored at this size over the source picture.
+const REGION_SIZE: u32 = 256;
+/// ImageNet's mean and deviation per channel, the model's normalisation.
+const MEAN: [f32; 3] = [0.485, 0.456, 0.406];
+const STD: [f32; 3] = [0.229, 0.224, 0.225];
+
+/// A frame as the encoder saw it: its embeddings, and how much of the
+/// square the picture filled.
+pub struct Embedding {
+ image: Output,
+ positional: Output,
+ /// The picture's width and height inside the square, in pixels.
+ scaled: (f32, f32),
+}
+
+/// The two halves of the brush model, loaded.
+pub struct Brush {
+ encoder: Mutex<Model>,
+ decoder: Mutex<Model>,
+}
+
+impl Brush {
+ /// Loads the encoder and decoder from their downloaded files.
+ pub fn load(encoder: &Path, decoder: &Path) -> Result<Brush, String> {
+ Ok(Brush {
+ encoder: Mutex::new(Model::from_file(encoder)?),
+ decoder: Mutex::new(Model::from_file(decoder)?),
+ })
+ }
+
+ /// The size to decode a source of `width` × `height` to for the
+ /// encoder: fitted inside the square by its long edge.
+ pub fn input_size(width: u32, height: u32) -> (u32, u32) {
+ let (w, h) = (width.max(1) as f64, height.max(1) as f64);
+ let scale = SAM_SIZE as f64 / w.max(h);
+ (
+ ((w * scale).round() as u32).clamp(1, SAM_SIZE as u32),
+ ((h * scale).round() as u32).clamp(1, SAM_SIZE as u32),
+ )
+ }
+
+ /// Reads a frame. The expensive half.
+ pub fn embed(&self, frame: &Frame) -> Result<Embedding, String> {
+ if frame.width() == 0 || frame.height() == 0 {
+ return Err("brush: an empty frame".to_owned());
+ }
+ let (width, height) = Brush::input_size(frame.width(), frame.height());
+ let (width, height) = (width as usize, height as usize);
+ let (fw, fh) = (frame.width() as usize, frame.height() as usize);
+ let pixels = frame.pixels();
+ // Normalised picture in the top-left of the square, zero beyond.
+ let mut data = vec![0f32; 3 * SAM_SIZE * SAM_SIZE];
+ for c in 0..3 {
+ for y in 0..height {
+ let sy = (y * fh / height).min(fh - 1);
+ for x in 0..width {
+ let sx = (x * fw / width).min(fw - 1);
+ let value = f32::from(pixels[(sy * fw + sx) * BYTES_PER_PIXEL + c]) / 255.0;
+ data[(c * SAM_SIZE + y) * SAM_SIZE + x] = (value - MEAN[c]) / STD[c];
+ }
+ }
+ }
+ let input = Input {
+ name: "pixel_values",
+ dims: vec![1, 3, SAM_SIZE, SAM_SIZE],
+ data: data.into(),
+ };
+ let mut outputs = self
+ .encoder
+ .lock()
+ .map_err(|_| "brush encoder poisoned".to_owned())?
+ .run(vec![input], &["image_embeddings", "image_positional_embeddings"])?;
+ let positional = outputs.pop().ok_or("brush: no positional embeddings")?;
+ let image = outputs.pop().ok_or("brush: no image embeddings")?;
+ Ok(Embedding {
+ image,
+ positional,
+ scaled: (width as f32, height as f32),
+ })
+ }
+
+ /// The thing under `points`, which are source fractions along the
+ /// stroke, as a mask over the source picture. The cheap half.
+ pub fn region(&self, embedding: &Embedding, points: &[[f64; 2]]) -> Result<Mask, String> {
+ // A stroke has as many points as the pointer reported; the model
+ // wants a handful. Sixteen spread along the path say where the
+ // stroke went without drowning the prompt.
+ const PROMPTS: usize = 16;
+ let picked: Vec<[f64; 2]> = if points.len() <= PROMPTS {
+ points.to_vec()
+ } else {
+ (0..PROMPTS)
+ .map(|i| points[i * (points.len() - 1) / (PROMPTS - 1)])
+ .collect()
+ };
+ if picked.is_empty() {
+ return Err("brush: a stroke with no points".to_owned());
+ }
+ let (sw, sh) = embedding.scaled;
+ let coords: Vec<f32> = picked
+ .iter()
+ .flat_map(|[x, y]| {
+ [
+ (*x as f32).clamp(0.0, 1.0) * sw,
+ (*y as f32).clamp(0.0, 1.0) * sh,
+ ]
+ })
+ .collect();
+ let labels = vec![1i64; picked.len()];
+ let inputs = vec![
+ Input {
+ name: "input_points",
+ dims: vec![1, 1, picked.len(), 2],
+ data: coords.into(),
+ },
+ Input {
+ name: "input_labels",
+ dims: vec![1, 1, picked.len()],
+ data: labels.into(),
+ },
+ Input {
+ name: "image_embeddings",
+ dims: embedding.image.dims.clone(),
+ data: embedding.image.data.clone().into(),
+ },
+ Input {
+ name: "image_positional_embeddings",
+ dims: embedding.positional.dims.clone(),
+ data: embedding.positional.data.clone().into(),
+ },
+ ];
+ let outputs = self
+ .decoder
+ .lock()
+ .map_err(|_| "brush decoder poisoned".to_owned())?
+ .run(inputs, &["iou_scores", "pred_masks"])?;
+ // Three candidate masks, scored; the model's own best guess wins.
+ let best = outputs[0]
+ .data
+ .iter()
+ .enumerate()
+ .max_by(|a, b| a.1.total_cmp(b.1))
+ .map(|(index, _)| index)
+ .unwrap_or(0);
+ let masks = &outputs[1].data;
+ let plane = SAM_MASK * SAM_MASK;
+ if masks.len() < (best + 1) * plane {
+ return Err("brush: the model's answer is the wrong size".to_owned());
+ }
+ let logits = &masks[best * plane..(best + 1) * plane];
+ // The answer covers the square; the picture is its top-left
+ // corner. Read the region over the picture alone.
+ let size = REGION_SIZE as usize;
+ let mut bytes = Vec::with_capacity(size * size);
+ for y in 0..size {
+ let py = ((y as f32 + 0.5) / size as f32) * sh / SAM_SIZE as f32 * SAM_MASK as f32;
+ for x in 0..size {
+ let px = ((x as f32 + 0.5) / size as f32) * sw / SAM_SIZE as f32 * SAM_MASK as f32;
+ let logit = bilinear(logits, SAM_MASK, px - 0.5, py - 0.5);
+ let probability = 1.0 / (1.0 + (-logit).exp());
+ bytes.push((probability.clamp(0.0, 1.0) * 255.0 + 0.5) as u8);
+ }
+ }
+ Mask::from_bytes(REGION_SIZE, REGION_SIZE, bytes)
+ .ok_or_else(|| "brush: the region is the wrong size".to_owned())
+ }
+}
+
+/// A value from a square plane, bilinear between the four around it,
+/// clamped to the edge.
+fn bilinear(plane: &[f32], edge: usize, x: f32, y: f32) -> f32 {
+ let at = |x: i64, y: i64| {
+ let x = x.clamp(0, edge as i64 - 1) as usize;
+ let y = y.clamp(0, edge as i64 - 1) as usize;
+ plane[y * edge + x]
+ };
+ let (x0, y0) = (x.floor(), y.floor());
+ let (fx, fy) = (x - x0, y - y0);
+ let (x0, y0) = (x0 as i64, y0 as i64);
+ let top = at(x0, y0) * (1.0 - fx) + at(x0 + 1, y0) * fx;
+ let bottom = at(x0, y0 + 1) * (1.0 - fx) + at(x0 + 1, y0 + 1) * fx;
+ top * (1.0 - fy) + bottom * fy
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_picture_fits_the_square_by_its_long_edge() {
+ assert_eq!(Brush::input_size(1920, 1080), (1024, 576));
+ assert_eq!(Brush::input_size(1080, 1920), (576, 1024));
+ assert_eq!(Brush::input_size(500, 500), (1024, 1024));
+ }
+
+ #[test]
+ fn bilinear_reads_between_pixels_and_clamps_at_the_edge() {
+ let plane = [0.0, 1.0, 2.0, 3.0];
+ assert_eq!(bilinear(&plane, 2, 0.0, 0.0), 0.0);
+ assert_eq!(bilinear(&plane, 2, 0.5, 0.0), 0.5);
+ assert_eq!(bilinear(&plane, 2, 0.5, 0.5), 1.5);
+ assert_eq!(bilinear(&plane, 2, 5.0, 5.0), 3.0);
+ }
+}
diff --git a/engine/crates/concat-vision/src/lib.rs b/engine/crates/concat-vision/src/lib.rs
index 18cde303..05830bf0 100644
--- a/engine/crates/concat-vision/src/lib.rs
+++ b/engine/crates/concat-vision/src/lib.rs
@@ -11,7 +11,10 @@
//! - [`mask`]: the answer itself, an eight-bit probability picture, with
//! the sampling, softening and PNG form everything else uses.
//! - [`strokes`]: the corrections. A custom cutout paints brush strokes
-//! over the model's mask; this is where a stroke becomes pixels.
+//! over the model's mask; this is where a stroke becomes pixels. The
+//! smart brushes name a thing rather than paint a disc: [`brush`] reads
+//! the thing under a stroke with a small Segment Anything, and the
+//! region it answers is kept beside the masks.
//! - [`apply`]: the frame with its background gone. One function, called
//! by the exporter and the monitor alike, so the file and the screen
//! agree by construction.
@@ -37,6 +40,8 @@
//! are undone.
pub mod apply;
+#[cfg(feature = "infer")]
+pub mod brush;
pub mod mask;
pub mod models;
#[cfg(feature = "infer")]
@@ -47,11 +52,13 @@ pub mod store;
pub mod strokes;
pub use apply::{Mapping, cut};
+#[cfg(feature = "infer")]
+pub use brush::{Brush, Embedding};
pub use mask::Mask;
pub use models::ModelId;
#[cfg(feature = "infer")]
pub use segment::Segmenter;
-pub use store::{MaskStore, mask_dir};
+pub use store::{MaskStore, mask_dir, region_file};
/// Masks are found this many times a second of source. Ten is where a
/// person's outline stops visibly lagging their movement, and where a
diff --git a/engine/crates/concat-vision/src/runtime.rs b/engine/crates/concat-vision/src/runtime.rs
index a4cb4f42..bfd78917 100644
--- a/engine/crates/concat-vision/src/runtime.rs
+++ b/engine/crates/concat-vision/src/runtime.rs
@@ -13,14 +13,35 @@
use ort::session::Session;
use ort::value::Tensor;
-/// An `f32` tensor by name: its dimensions and its values, row-major.
+/// A tensor by name: its dimensions and its values, row-major.
pub struct Input<'a> {
/// The graph's input name.
pub name: &'a str,
/// The tensor's shape.
pub dims: Vec<usize>,
/// The values, `dims` product of them.
- pub data: Vec<f32>,
+ pub data: Data,
+}
+
+/// A tensor's values. Pictures and probabilities are `f32`; the brush
+/// model's point labels are `i64`.
+pub enum Data {
+ /// Floating point values.
+ F32(Vec<f32>),
+ /// Integer values.
+ I64(Vec<i64>),
+}
+
+impl From<Vec<f32>> for Data {
+ fn from(values: Vec<f32>) -> Data {
+ Data::F32(values)
+ }
+}
+
+impl From<Vec<i64>> for Data {
+ fn from(values: Vec<i64>) -> Data {
+ Data::I64(values)
+ }
}
/// An `f32` tensor a model answered with.
@@ -72,9 +93,15 @@ impl Model {
let mut fed: Vec<(std::borrow::Cow<'static, str>, ort::session::SessionInputValue<'_>)> =
Vec::with_capacity(inputs.len());
for input in inputs {
- let tensor = Tensor::from_array((input.dims, input.data))
- .map_err(|error| format!("cutout input {}: {error}", input.name))?;
- fed.push((input.name.to_owned().into(), tensor.into()));
+ let value: ort::session::SessionInputValue<'_> = match input.data {
+ Data::F32(data) => Tensor::from_array((input.dims, data))
+ .map_err(|error| format!("cutout input {}: {error}", input.name))?
+ .into(),
+ Data::I64(data) => Tensor::from_array((input.dims, data))
+ .map_err(|error| format!("cutout input {}: {error}", input.name))?
+ .into(),
+ };
+ fed.push((input.name.to_owned().into(), value));
}
let answered = self
.session
diff --git a/engine/crates/concat-vision/src/segment.rs b/engine/crates/concat-vision/src/segment.rs
index cf46910c..75747747 100644
--- a/engine/crates/concat-vision/src/segment.rs
+++ b/engine/crates/concat-vision/src/segment.rs
@@ -163,7 +163,7 @@ impl Segmenter {
let input = Input {
name: "pixel_values",
dims: vec![1, 3, height as usize, width as usize],
- data: planes,
+ data: planes.into(),
};
let outputs = self.run(vec![input], &["alphas"])?;
probabilities(&outputs[0].data, size, width, height)
@@ -180,7 +180,7 @@ impl Segmenter {
let input = Input {
name: "input_image",
dims: vec![1, 3, height as usize, width as usize],
- data: planes,
+ data: planes.into(),
};
let outputs = self.run(vec![input], &["output_image"])?;
// The model answers in its own range; the span of the answer is
@@ -203,7 +203,7 @@ impl Segmenter {
let mut inputs = vec![Input {
name: "src",
dims: vec![1, 3, height as usize, width as usize],
- data: planes,
+ data: planes.into(),
}];
let carried = self
.state
@@ -217,7 +217,7 @@ impl Segmenter {
inputs.push(Input {
name,
dims: output.dims,
- data: output.data,
+ data: output.data.into(),
});
}
}
@@ -226,7 +226,7 @@ impl Segmenter {
inputs.push(Input {
name,
dims: vec![1, 1, 1, 1],
- data: vec![0.0],
+ data: vec![0.0f32].into(),
});
}
}
@@ -236,7 +236,7 @@ impl Segmenter {
inputs.push(Input {
name: "downsample_ratio",
dims: vec![1],
- data: vec![1.0],
+ data: vec![1.0f32].into(),
});
let mut outputs = self.run(inputs, &["pha", "r1o", "r2o", "r3o", "r4o"])?;
let alpha = outputs.remove(0);
diff --git a/engine/crates/concat-vision/src/store.rs b/engine/crates/concat-vision/src/store.rs
index 8c49214a..551a5d1b 100644
--- a/engine/crates/concat-vision/src/store.rs
+++ b/engine/crates/concat-vision/src/store.rs
@@ -19,7 +19,7 @@ use std::collections::{HashMap, VecDeque};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
-use concat_project::model::{Cutout, CutoutMode, Subject};
+use concat_project::model::{Cutout, CutoutMode, Stroke, Subject};
use crate::{MASK_RATE, Mask, strokes};
@@ -51,6 +51,13 @@ fn model_file(dir: &Path) -> PathBuf {
dir.join("model")
}
+/// The file a smart stroke's region is kept in, under the masks' own
+/// directory: the thing the brush model read under that stroke.
+pub fn region_file(dir: &Path, stroke: &Stroke) -> PathBuf {
+ dir.join("strokes")
+ .join(format!("{:016x}.png", strokes::stroke_key(stroke)))
+}
+
/// The file for the mask at `millis` of source.
pub fn mask_file(dir: &Path, millis: u64) -> PathBuf {
dir.join(format!("{millis:09}.png"))
@@ -178,7 +185,7 @@ impl MaskStore {
let file = mask_file(&self.dir, millis);
let key = ResolvedKey {
file: file.clone(),
- settings: settings_key(cutout, aspect),
+ settings: settings_key(&self.dir, cutout, aspect),
};
if let Some(hit) = resolved_cache()
.lock()
@@ -194,7 +201,9 @@ impl MaskStore {
}
let painted = match cutout.mode {
CutoutMode::Custom if !cutout.strokes.is_empty() => {
- strokes::paint(&auto, &cutout.strokes, aspect)
+ let dir = self.dir.clone();
+ let regions = move |stroke: &Stroke| load(&region_file(&dir, stroke));
+ strokes::paint(&auto, &cutout.strokes, aspect, &regions)
}
_ => (*auto).clone(),
};
@@ -258,13 +267,17 @@ impl MaskStore {
self.times.clear();
let _ = std::fs::remove_file(model_file(&self.dir));
self.model = None;
+ // The regions the brushes read are the same footage's; they go
+ // with the masks, and any resolved mask that used one.
+ let _ = std::fs::remove_dir_all(self.dir.join("strokes"));
let _ = std::fs::remove_dir(&self.dir);
}
}
/// What besides the file decides a resolved mask: the mode, the feather
-/// and every stroke, hashed, with the aspect the brushes were sized by.
-fn settings_key(cutout: &Cutout, aspect: f32) -> u64 {
+/// and every stroke, hashed, with the aspect the brushes were sized by,
+/// and for each smart stroke whether its region has been read yet.
+fn settings_key(dir: &Path, cutout: &Cutout, aspect: f32) -> u64 {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
(cutout.mode == CutoutMode::Custom).hash(&mut hasher);
@@ -272,11 +285,9 @@ fn settings_key(cutout: &Cutout, aspect: f32) -> u64 {
aspect.to_bits().hash(&mut hasher);
if cutout.mode == CutoutMode::Custom {
for stroke in &cutout.strokes {
- (stroke.tool as u8).hash(&mut hasher);
- stroke.size.to_bits().hash(&mut hasher);
- for [x, y] in &stroke.points {
- x.to_bits().hash(&mut hasher);
- y.to_bits().hash(&mut hasher);
+ strokes::stroke_key(stroke).hash(&mut hasher);
+ if stroke.is_smart() {
+ region_file(dir, stroke).is_file().hash(&mut hasher);
}
}
}
@@ -432,6 +443,7 @@ mod tests {
subject: Subject::Auto,
feather: 0.0,
strokes: vec![Stroke {
+ at: None,
tool: BrushTool::Brush,
size: 0.5,
points: vec![[0.5, 0.5]],
@@ -464,6 +476,7 @@ mod tests {
subject: Subject::Auto,
feather: 0.0,
strokes: vec![Stroke {
+ at: None,
tool: BrushTool::Brush,
size: 0.5,
points: vec![[0.5, 0.5]],
diff --git a/engine/crates/concat-vision/src/strokes.rs b/engine/crates/concat-vision/src/strokes.rs
index 679c045b..75a33bff 100644
--- a/engine/crates/concat-vision/src/strokes.rs
+++ b/engine/crates/concat-vision/src/strokes.rs
@@ -9,34 +9,87 @@
//! and each disc changes the pixels beneath it the way its tool says:
//!
//! - **Brush** keeps everything under it, **Eraser** removes everything.
-//! - **Smart brush** keeps only what the model gave some chance of being
-//! the subject - a sleeve it was unsure of comes back, the wall behind
-//! does not. **Smart eraser** removes only what the model was not sure
-//! of - a bit of chair it half-kept goes, the shoulder beside it stays.
+//! - **Smart brush** keeps the thing under it, whole: the brush model
+//! (see `brush`) reads what the stroke lies on and the region it answers
+//! is kept. **Smart eraser** removes that thing. Until the region has
+//! been read - or when it cannot be, with the model not to hand - the
+//! smart tools fall back to the mask's own confidence: the brush keeps
+//! only what the model gave some chance of being the subject, the
+//! eraser removes only what it was not sure of.
//!
-//! The mask is square whatever the picture's shape, so a round brush is an
-//! ellipse in mask pixels; `aspect` (width over height of the source) is
-//! what makes it round again on screen.
+//! A mask may be any shape, so a round brush is an ellipse in mask
+//! pixels; `aspect` (width over height of the source) is what makes it
+//! round again on screen.
+
+use std::sync::Arc;
use concat_project::model::{BrushTool, Stroke};
use crate::Mask;
+/// Where a smart stroke's region is found, by the stroke: `None` when it
+/// has not been read yet.
+pub type Regions<'a> = &'a dyn Fn(&Stroke) -> Option<Arc<Mask>>;
+
+/// A region pixel at least this likely is the thing.
+const REGION_IS: f32 = 0.5;
+
+/// Names a stroke for the file its region is kept in: everything about
+/// it, hashed.
+pub fn stroke_key(stroke: &Stroke) -> u64 {
+ let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
+ let mut feed = |bytes: &[u8]| {
+ for &byte in bytes {
+ hash ^= u64::from(byte);
+ hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
+ }
+ };
+ feed(&[stroke.tool as u8]);
+ feed(&stroke.size.to_bits().to_le_bytes());
+ feed(&stroke.at.unwrap_or(-1.0).to_bits().to_le_bytes());
+ for [x, y] in &stroke.points {
+ feed(&x.to_bits().to_le_bytes());
+ feed(&y.to_bits().to_le_bytes());
+ }
+ hash
+}
+
/// Below this much confidence a smart brush leaves a pixel alone.
const SMART_KEEP: u8 = 38;
/// Above this much confidence a smart eraser leaves a pixel alone.
const SMART_DROP: u8 = 217;
/// The model's mask with `strokes` painted over it, in order. `aspect` is
-/// the source's width over its height.
-pub fn paint(auto: &Mask, strokes: &[Stroke], aspect: f32) -> Mask {
+/// the source's width over its height; `regions` finds what the brush
+/// model read under each smart stroke.
+pub fn paint(auto: &Mask, strokes: &[Stroke], aspect: f32, regions: Regions<'_>) -> Mask {
let mut out = auto.clone();
for stroke in strokes {
- paint_one(&mut out, auto, stroke, aspect);
+ match regions(stroke).filter(|_| stroke.is_smart()) {
+ Some(region) => paint_region(&mut out, stroke.tool, &region),
+ None => paint_one(&mut out, auto, stroke, aspect),
+ }
}
out
}
+/// The thing a smart stroke named, kept or removed whole.
+fn paint_region(out: &mut Mask, tool: BrushTool, region: &Mask) {
+ let width = out.width();
+ let height = out.height();
+ let keep = tool == BrushTool::SmartBrush;
+ for y in 0..height {
+ let v = (y as f32 + 0.5) / height as f32;
+ for x in 0..width {
+ let u = (x as f32 + 0.5) / width as f32;
+ if region.sample(u, v) >= REGION_IS {
+ let index = (y * width + x) as usize;
+ out.bytes_mut()[index] = if keep { 255 } else { 0 };
+ }
+ }
+ }
+}
+
fn paint_one(out: &mut Mask, auto: &Mask, stroke: &Stroke, aspect: f32) {
let width = out.width() as f32;
let height = out.height() as f32;
@@ -109,6 +162,7 @@ mod tests {
fn stroke(tool: BrushTool, size: f64, points: &[[f64; 2]]) -> Stroke {
Stroke {
+ at: None,
tool,
size,
points: points.to_vec(),
@@ -122,6 +176,7 @@ mod tests {
&auto,
&[stroke(BrushTool::Brush, 0.25, &[[0.25, 0.5], [0.75, 0.5]])],
1.0,
+ &|_| None,
);
assert_eq!(kept.at(16, 16), 255);
assert_eq!(kept.at(16, 2), 0);
@@ -131,6 +186,7 @@ mod tests {
&kept,
&[stroke(BrushTool::Eraser, 0.25, &[[0.5, 0.5]])],
1.0,
+ &|_| None,
);
assert_eq!(gone.at(16, 16), 0);
assert_eq!(gone.at(8, 16), 255);
@@ -143,7 +199,7 @@ mod tests {
auto.bytes_mut()[1] = 0; // background: left alone
auto.bytes_mut()[2] = 250; // sure: kept by a smart eraser
let whole = [stroke(BrushTool::SmartBrush, 2.0, &[[0.5, 0.5]])];
- let brushed = paint(&auto, &whole, 1.0);
+ let brushed = paint(&auto, &whole, 1.0, &|_| None);
assert_eq!(brushed.at(0, 0), 255);
assert_eq!(brushed.at(1, 0), 0);
let erased = paint(
@@ -153,6 +209,7 @@ mod tests {
stroke(BrushTool::SmartEraser, 2.0, &[[0.5, 0.5]]),
],
1.0,
+ &|_| None,
);
// The eraser judges by the model's confidence, not by what an
// earlier stroke did: the rescued pixel was unsure, so it goes; the
@@ -161,12 +218,45 @@ mod tests {
assert_eq!(erased.at(2, 0), 255);
}
+ #[test]
+ fn a_smart_stroke_with_a_region_keeps_or_drops_the_whole_thing() {
+ let auto = Mask::filled(8, 8, 0);
+ // The thing is the left half of the picture.
+ let mut region = Mask::filled(4, 4, 0);
+ for y in 0..4 {
+ region.bytes_mut()[y * 4] = 255;
+ region.bytes_mut()[y * 4 + 1] = 255;
+ }
+ let region = Arc::new(region);
+ let lookup = move |_: &Stroke| Some(Arc::clone(&region));
+ let kept = paint(
+ &auto,
+ &[stroke(BrushTool::SmartBrush, 0.1, &[[0.9, 0.9]])],
+ 1.0,
+ &lookup,
+ );
+ assert_eq!(kept.at(1, 4), 255);
+ assert_eq!(kept.at(6, 4), 0);
+ let gone = paint(
+ &kept,
+ &[stroke(BrushTool::SmartEraser, 0.1, &[[0.9, 0.9]])],
+ 1.0,
+ &lookup,
+ );
+ assert_eq!(gone.at(1, 4), 0);
+ }
+
#[test]
fn the_brush_is_round_on_a_wide_picture() {
// A 2:1 source squashed into a square mask: a round brush must reach
// twice as far down the mask as across it.
let auto = Mask::filled(64, 64, 0);
- let out = paint(&auto, &[stroke(BrushTool::Brush, 0.25, &[[0.5, 0.5]])], 2.0);
+ let out = paint(
+ &auto,
+ &[stroke(BrushTool::Brush, 0.25, &[[0.5, 0.5]])],
+ 2.0,
+ &|_| None,
+ );
assert_eq!(out.at(32 + 7, 32), 255);
assert_eq!(out.at(32 + 9, 32), 0);
assert_eq!(out.at(32, 32 + 15), 255);
diff --git a/engine/crates/concat/locales/de.json b/engine/crates/concat/locales/de.json
index 4b1199f4..cb345b14 100644
--- a/engine/crates/concat/locales/de.json
+++ b/engine/crates/concat/locales/de.json
@@ -45,7 +45,7 @@
"Auto captions": "Automatische Untertitel",
"Auto captions from this clip's sound": "Automatische Untertitel aus dem Ton dieses Clips",
"Auto removal": "Automatisch entfernen",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "Automatisch entfernen behält die Person, die das Freistellmodell findet, und entfernt den Rest. Manuell entfernen beginnt mit derselben Maske und nimmt Korrekturen an, die du auf dem Monitor malst: Smart-Pinsel und Smart-Radierer folgen der Sicherheit des Modells, die einfachen tun genau, was du malst; Anwenden beendet das Malen. Chroma-Key stanzt stattdessen eine Farbe aus; Ähnlichkeit bestimmt, wie weit von der Farbe entfernt noch zählt, Weichheit federt die Kante.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "Automatisch entfernen behält, was das Modell findet, und entfernt den Rest: eine Person oder das eine Ding, das das Bild zeigt. Manuell entfernen beginnt mit derselben Maske und nimmt Korrekturen an, die du auf dem Monitor malst: Der Smart-Pinsel behält das ganze Ding, das er berührt, der Smart-Radierer entfernt es, die einfachen tun genau, was du malst; Anwenden beendet das Malen. Chroma-Key stanzt stattdessen eine Farbe aus; Ähnlichkeit bestimmt, wie weit von der Farbe entfernt noch zählt, Weichheit federt die Kante.",
"Balanced": "Ausgewogen",
"Basic": "Grundlagen",
"Bass": "Bass",
@@ -435,6 +435,7 @@
"Slower": "Langsamer",
"Small": "Klein",
"Smart brush": "Smart-Pinsel",
+ "Smart brush: {0}": "Smart-Pinsel: {0}",
"Smart eraser": "Smart-Radierer",
"Snap to edges": "An Kanten einrasten",
"Softness": "Weichheit",
diff --git a/engine/crates/concat/locales/en.json b/engine/crates/concat/locales/en.json
index 5d526879..bf1847fe 100644
--- a/engine/crates/concat/locales/en.json
+++ b/engine/crates/concat/locales/en.json
@@ -45,7 +45,7 @@
"Auto captions": "Auto captions",
"Auto captions from this clip's sound": "Auto captions from this clip's sound",
"Auto removal": "Auto removal",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.",
"Balanced": "Balanced",
"Basic": "Basic",
"Bass": "Bass",
@@ -435,6 +435,7 @@
"Slower": "Slower",
"Small": "Small",
"Smart brush": "Smart brush",
+ "Smart brush: {0}": "Smart brush: {0}",
"Smart eraser": "Smart eraser",
"Snap to edges": "Snap to edges",
"Softness": "Softness",
diff --git a/engine/crates/concat/locales/es.json b/engine/crates/concat/locales/es.json
index f951c183..6a05661d 100644
--- a/engine/crates/concat/locales/es.json
+++ b/engine/crates/concat/locales/es.json
@@ -45,7 +45,7 @@
"Auto captions": "Subtítulos automáticos",
"Auto captions from this clip's sound": "Subtítulos automáticos a partir del sonido de este clip",
"Auto removal": "Eliminación automática",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "La eliminación automática conserva a la persona que encuentra el modelo de recorte y elimina el resto. La eliminación manual parte de la misma máscara y acepta correcciones pintadas en el monitor: el pincel y el borrador inteligentes siguen la confianza del modelo; los simples hacen exactamente lo que pintas; Aplicar termina el pintado. El croma, en cambio, elimina un color; la similitud indica hasta qué distancia del color cuenta, y la suavidad difumina el borde.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "La eliminación automática conserva lo que encuentra el modelo y elimina el resto: una persona, o la única cosa que muestra la imagen. La eliminación manual parte de la misma máscara y acepta correcciones pintadas en el monitor: el pincel inteligente conserva entera la cosa que toca y el borrador inteligente la elimina; los simples hacen exactamente lo que pintas; Aplicar termina el pintado. El croma, en cambio, elimina un color; la similitud indica hasta qué distancia del color cuenta, y la suavidad difumina el borde.",
"Balanced": "Equilibrada",
"Basic": "Básico",
"Bass": "Graves",
@@ -435,6 +435,7 @@
"Slower": "Más lento",
"Small": "Pequeño",
"Smart brush": "Pincel inteligente",
+ "Smart brush: {0}": "Pincel inteligente: {0}",
"Smart eraser": "Borrador inteligente",
"Snap to edges": "Ajustar a los bordes",
"Softness": "Suavidad",
diff --git a/engine/crates/concat/locales/fr.json b/engine/crates/concat/locales/fr.json
index 763f71bc..e21fa6c8 100644
--- a/engine/crates/concat/locales/fr.json
+++ b/engine/crates/concat/locales/fr.json
@@ -45,7 +45,7 @@
"Auto captions": "Sous-titres automatiques",
"Auto captions from this clip's sound": "Sous-titres automatiques à partir du son de ce clip",
"Auto removal": "Suppression automatique",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "La suppression automatique garde la personne que le modèle de détourage trouve et retire le reste. La suppression manuelle part du même masque et accepte des corrections peintes sur le moniteur : le pinceau et la gomme intelligents suivent la confiance du modèle, les simples font exactement ce que vous peignez ; Appliquer termine la peinture. L'incrustation chroma retire une couleur à la place ; la similarité dit jusqu'où la couleur compte encore, la douceur adoucit le bord.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "La suppression automatique garde ce que le modèle trouve et retire le reste : une personne, ou la chose que montre l'image. La suppression manuelle part du même masque et accepte des corrections peintes sur le moniteur : le pinceau intelligent garde entière la chose qu'il touche et la gomme intelligente la retire, les simples font exactement ce que vous peignez ; Appliquer termine la peinture. L'incrustation chroma retire une couleur à la place ; la similarité dit jusqu'où la couleur compte encore, la douceur adoucit le bord.",
"Balanced": "Équilibrée",
"Basic": "Base",
"Bass": "Basses",
@@ -435,6 +435,7 @@
"Slower": "Plus lent",
"Small": "Petit",
"Smart brush": "Pinceau intelligent",
+ "Smart brush: {0}": "Pinceau intelligent : {0}",
"Smart eraser": "Gomme intelligente",
"Snap to edges": "Aimanter aux bords",
"Softness": "Douceur",
diff --git a/engine/crates/concat/locales/it.json b/engine/crates/concat/locales/it.json
index c187b2d4..e72352b2 100644
--- a/engine/crates/concat/locales/it.json
+++ b/engine/crates/concat/locales/it.json
@@ -45,7 +45,7 @@
"Auto captions": "Sottotitoli automatici",
"Auto captions from this clip's sound": "Sottotitoli automatici dal suono di questa clip",
"Auto removal": "Rimozione automatica",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "La rimozione automatica tiene la persona trovata dal modello di ritaglio e toglie il resto. La rimozione manuale parte dalla stessa maschera e accetta correzioni dipinte sul monitor: pennello e gomma intelligenti seguono la sicurezza del modello, quelli semplici fanno esattamente ciò che dipingi; Applica chiude la pittura. Il chroma key toglie invece un colore; la somiglianza indica quanto lontano dal colore conta ancora, la morbidezza sfuma il bordo.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "La rimozione automatica tiene ciò che il modello trova e toglie il resto: una persona, o la cosa che l'immagine mostra. La rimozione manuale parte dalla stessa maschera e accetta correzioni dipinte sul monitor: il pennello intelligente tiene intera la cosa che tocca e la gomma intelligente la toglie, quelli semplici fanno esattamente ciò che dipingi; Applica chiude la pittura. Il chroma key toglie invece un colore; la somiglianza indica quanto lontano dal colore conta ancora, la morbidezza sfuma il bordo.",
"Balanced": "Bilanciata",
"Basic": "Base",
"Bass": "Bassi",
@@ -435,6 +435,7 @@
"Slower": "Più lento",
"Small": "Piccolo",
"Smart brush": "Pennello intelligente",
+ "Smart brush: {0}": "Pennello intelligente: {0}",
"Smart eraser": "Gomma intelligente",
"Snap to edges": "Aggancia ai bordi",
"Softness": "Morbidezza",
diff --git a/engine/crates/concat/locales/ja.json b/engine/crates/concat/locales/ja.json
index 0373456c..a933cbaa 100644
--- a/engine/crates/concat/locales/ja.json
+++ b/engine/crates/concat/locales/ja.json
@@ -45,7 +45,7 @@
"Auto captions": "自動字幕",
"Auto captions from this clip's sound": "このクリップの音声から自動字幕を作成",
"Auto removal": "自動削除",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "「自動削除」は切り抜きモデルが見つけた人物を残し、それ以外を取り除きます。「カスタム削除」は同じマスクから始めて、モニター上で塗った修正を受け付けます。スマートブラシとスマート消しゴムはモデルの確信度に従い、通常のブラシと消しゴムは塗ったとおりに働きます。「適用」で塗りを終えます。「クロマキー」は代わりに一色を抜きます。類似度はその色からどれだけ離れても対象とみなすか、柔らかさは縁のぼかし具合です。",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "「自動削除」はモデルが見つけたもの、つまり人物か画像の主題を残し、それ以外を取り除きます。「カスタム削除」は同じマスクから始めて、モニター上で塗った修正を受け付けます。スマートブラシは触れたものを丸ごと残し、スマート消しゴムはそれを丸ごと取り除きます。通常のブラシと消しゴムは塗ったとおりに働き、「適用」で塗りを終えます。「クロマキー」は代わりに一色を抜きます。類似度はその色からどれだけ離れても対象とみなすか、柔らかさは縁のぼかし具合です。",
"Balanced": "バランス",
"Basic": "基本",
"Bass": "低音",
@@ -435,6 +435,7 @@
"Slower": "遅く",
"Small": "小",
"Smart brush": "スマートブラシ",
+ "Smart brush: {0}": "スマートブラシ: {0}",
"Smart eraser": "スマート消しゴム",
"Snap to edges": "端にスナップ",
"Softness": "柔らかさ",
diff --git a/engine/crates/concat/locales/ko.json b/engine/crates/concat/locales/ko.json
index 91232d2e..68b65da0 100644
--- a/engine/crates/concat/locales/ko.json
+++ b/engine/crates/concat/locales/ko.json
@@ -45,7 +45,7 @@
"Auto captions": "자동 자막",
"Auto captions from this clip's sound": "이 클립의 소리로 자동 자막 만들기",
"Auto removal": "자동 제거",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "자동 제거는 컷아웃 모델이 찾은 인물을 남기고 나머지를 제거합니다. 사용자 지정 제거는 같은 마스크에서 시작해 모니터에 칠한 수정을 받아들입니다. 스마트 브러시와 스마트 지우개는 모델의 확신도를 따르고, 일반 도구는 칠한 대로 작동하며, 적용을 누르면 칠하기가 끝납니다. 크로마 키는 대신 한 가지 색을 제거합니다. 유사도는 그 색에서 얼마나 떨어져도 포함할지, 부드러움은 가장자리를 얼마나 흐릴지를 정합니다.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "자동 제거는 모델이 찾은 것, 즉 인물이나 그림의 주된 사물을 남기고 나머지를 제거합니다. 사용자 지정 제거는 같은 마스크에서 시작해 모니터에 칠한 수정을 받아들입니다. 스마트 브러시는 닿은 것을 통째로 남기고 스마트 지우개는 통째로 지우며, 일반 도구는 칠한 대로 작동하고, 적용을 누르면 칠하기가 끝납니다. 크로마 키는 대신 한 가지 색을 제거합니다. 유사도는 그 색에서 얼마나 떨어져도 포함할지, 부드러움은 가장자리를 얼마나 흐릴지를 정합니다.",
"Balanced": "균형",
"Basic": "기본",
"Bass": "저음",
@@ -435,6 +435,7 @@
"Slower": "느리게",
"Small": "작게",
"Smart brush": "스마트 브러시",
+ "Smart brush: {0}": "스마트 브러시: {0}",
"Smart eraser": "스마트 지우개",
"Snap to edges": "가장자리에 스냅",
"Softness": "부드러움",
diff --git a/engine/crates/concat/locales/pt-BR.json b/engine/crates/concat/locales/pt-BR.json
index 7218e617..3a6452b1 100644
--- a/engine/crates/concat/locales/pt-BR.json
+++ b/engine/crates/concat/locales/pt-BR.json
@@ -45,7 +45,7 @@
"Auto captions": "Legendas automáticas",
"Auto captions from this clip's sound": "Legendas automáticas a partir do som deste clipe",
"Auto removal": "Remoção automática",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "A remoção automática mantém a pessoa que o modelo de recorte encontra e descarta o resto. A remoção manual parte da mesma máscara e aceita correções pintadas no monitor: o pincel e a borracha inteligentes seguem a confiança do modelo; os simples fazem exatamente o que você pinta; Aplicar encerra a pintura. O chroma key, em vez disso, remove uma cor; similaridade define até que distância da cor ainda conta, e suavidade esfuma a borda.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "A remoção automática mantém o que o modelo encontra e descarta o resto: uma pessoa, ou a única coisa que a imagem mostra. A remoção manual parte da mesma máscara e aceita correções pintadas no monitor: o pincel inteligente mantém inteira a coisa que toca e a borracha inteligente a remove; os simples fazem exatamente o que você pinta; Aplicar encerra a pintura. O chroma key, em vez disso, remove uma cor; similaridade define até que distância da cor ainda conta, e suavidade esfuma a borda.",
"Balanced": "Equilibrada",
"Basic": "Básico",
"Bass": "Graves",
@@ -435,6 +435,7 @@
"Slower": "Mais lento",
"Small": "Pequeno",
"Smart brush": "Pincel inteligente",
+ "Smart brush: {0}": "Pincel inteligente: {0}",
"Smart eraser": "Borracha inteligente",
"Snap to edges": "Ajustar às bordas",
"Softness": "Suavidade",
diff --git a/engine/crates/concat/locales/ru.json b/engine/crates/concat/locales/ru.json
index 5d7eb4f6..ef002c4d 100644
--- a/engine/crates/concat/locales/ru.json
+++ b/engine/crates/concat/locales/ru.json
@@ -45,7 +45,7 @@
"Auto captions": "Автосубтитры",
"Auto captions from this clip's sound": "Автосубтитры из звука этого клипа",
"Auto removal": "Автоматическое удаление",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "Автоматическое удаление оставляет человека, которого нашла модель вырезания, и убирает остальное. Ручное удаление берёт ту же маску плюс поправки, нарисованные на мониторе: умная кисть и умный ластик следуют уверенности модели, обычные делают ровно то, что вы рисуете; «Применить» завершает рисование. Хромакей вместо этого убирает один цвет; сходство задаёт, насколько далёкий от цвета оттенок ещё считается, мягкость растушёвывает край.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "Автоматическое удаление оставляет то, что нашла модель, и убирает остальное: человека или тот единственный предмет, который показан в кадре. Ручное удаление берёт ту же маску плюс поправки, нарисованные на мониторе: умная кисть оставляет целиком то, чего касается, умный ластик целиком убирает, обычные делают ровно то, что вы рисуете; «Применить» завершает рисование. Хромакей вместо этого убирает один цвет; сходство задаёт, насколько далёкий от цвета оттенок ещё считается, мягкость растушёвывает край.",
"Balanced": "Сбалансированное",
"Basic": "Основное",
"Bass": "Низкие",
@@ -435,6 +435,7 @@
"Slower": "Медленнее",
"Small": "Малый",
"Smart brush": "Умная кисть",
+ "Smart brush: {0}": "Умная кисть: {0}",
"Smart eraser": "Умный ластик",
"Snap to edges": "Привязка к краям",
"Softness": "Мягкость",
diff --git a/engine/crates/concat/locales/tr.json b/engine/crates/concat/locales/tr.json
index 4596d007..fad15b8d 100644
--- a/engine/crates/concat/locales/tr.json
+++ b/engine/crates/concat/locales/tr.json
@@ -45,7 +45,7 @@
"Auto captions": "Otomatik altyazı",
"Auto captions from this clip's sound": "Bu klibin sesinden otomatik altyazı",
"Auto removal": "Otomatik kaldırma",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "Otomatik kaldırma, kesme modelinin bulduğu kişiyi tutar, gerisini atar. Özel kaldırma aynı maskeden başlar ve monitörde boyanan düzeltmeleri alır: akıllı fırça ve akıllı silgi modelin güvenine uyar, düz olanlar tam boyadığınızı yapar; Uygula boyamayı bitirir. Chroma key ise bir rengi çıkarır; benzerlik, renkten ne kadar uzağın hâlâ sayılacağını, yumuşaklık kenarın ne kadar yumuşayacağını belirler.",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "Otomatik kaldırma modelin bulduğunu tutar, gerisini atar: bir kişiyi ya da görüntünün gösterdiği tek şeyi. Özel kaldırma aynı maskeden başlar ve monitörde boyanan düzeltmeleri alır: akıllı fırça dokunduğu şeyi bütünüyle tutar, akıllı silgi bütünüyle siler, düz olanlar tam boyadığınızı yapar; Uygula boyamayı bitirir. Chroma key ise bir rengi çıkarır; benzerlik, renkten ne kadar uzağın hâlâ sayılacağını, yumuşaklık kenarın ne kadar yumuşayacağını belirler.",
"Balanced": "Dengeli",
"Basic": "Temel",
"Bass": "Bas",
@@ -435,6 +435,7 @@
"Slower": "Daha yavaş",
"Small": "Küçük",
"Smart brush": "Akıllı fırça",
+ "Smart brush: {0}": "Akıllı fırça: {0}",
"Smart eraser": "Akıllı silgi",
"Snap to edges": "Kenarlara yapış",
"Softness": "Yumuşaklık",
diff --git a/engine/crates/concat/locales/zh-Hans.json b/engine/crates/concat/locales/zh-Hans.json
index 50073d1f..afe8b0ea 100644
--- a/engine/crates/concat/locales/zh-Hans.json
+++ b/engine/crates/concat/locales/zh-Hans.json
@@ -45,7 +45,7 @@
"Auto captions": "自动字幕",
"Auto captions from this clip's sound": "根据此片段的声音生成自动字幕",
"Auto removal": "自动移除",
- "Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "“自动移除”保留抠像模型找到的人物并去掉其余部分。“自定义移除”从同一蒙版出发,接受在监视器上涂画的修正:智能画笔和智能橡皮遵循模型的置信度,普通工具则完全按你涂画的执行;点“应用”结束涂画。“色度键”则是抠掉一种颜色;相似度决定与该颜色相差多远仍算在内,柔和度决定边缘羽化程度。",
+ "Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.": "“自动移除”保留模型找到的内容并去掉其余部分:一个人物,或画面所拍的那一样东西。“自定义移除”从同一蒙版出发,接受在监视器上涂画的修正:智能画笔保留它触到的整个物体,智能橡皮则整个去掉,普通工具完全按你涂画的执行;点“应用”结束涂画。“色度键”则是抠掉一种颜色;相似度决定与该颜色相差多远仍算在内,柔和度决定边缘羽化程度。",
"Balanced": "均衡",
"Basic": "基础",
"Bass": "低音",
@@ -435,6 +435,7 @@
"Slower": "更慢",
"Small": "小",
"Smart brush": "智能画笔",
+ "Smart brush: {0}": "智能画笔:{0}",
"Smart eraser": "智能橡皮",
"Snap to edges": "吸附到边缘",
"Softness": "柔和度",
diff --git a/engine/crates/concat/src/host.rs b/engine/crates/concat/src/host.rs
index a8bc507a..67cc9378 100644
--- a/engine/crates/concat/src/host.rs
+++ b/engine/crates/concat/src/host.rs
@@ -41,8 +41,10 @@ pub struct Host {
pub speech: Arc<Speech>,
/// Titles painted to pictures, and the cache of them.
pub titles: concat_host::Titles,
- /// The cutout model, and the masks it finds for the project's media.
+ /// The cutout models, and the masks they find for the project's media.
pub cutouts: Arc<concat_host::Cutouts>,
+ /// The brush model, and the regions it reads under smart strokes.
+ pub brushes: Arc<concat_host::Brushes>,
}
impl Host {
@@ -56,6 +58,7 @@ impl Host {
Ok(Host {
titles: concat_host::Titles::new(&dirs),
cutouts: Arc::new(concat_host::Cutouts::new(&dirs.data)),
+ brushes: Arc::new(concat_host::Brushes::new(&dirs.data)),
dirs,
playback: Playback::start(Arc::new(Events))?,
monitor: match gpu {
diff --git a/engine/crates/concat/src/studio.rs b/engine/crates/concat/src/studio.rs
index 17ac5b7c..177dd3e2 100644
--- a/engine/crates/concat/src/studio.rs
+++ b/engine/crates/concat/src/studio.rs
@@ -28,7 +28,9 @@ use concat_effects::manifest::Kind as PackageKind;
use concat_host::export::{self, ExportSpec};
use concat_host::playback::ClipSpec;
use concat_host::preview::FrameSpec;
-use concat_host::{AnalyseRequest, Cutouts, ProjectInfo, Session, media, projects, templates};
+use concat_host::{
+ AnalyseRequest, Cutouts, ProjectInfo, RegionRequest, Session, media, projects, templates,
+};
use concat_media::Peaks;
use concat_project::commands::{ClipMove, ClipPatch, TrackFlag, TrimEdge};
use concat_project::model::{
@@ -422,6 +424,8 @@ pub enum Gesture {
clip: String,
tool: model::BrushTool,
size: f64,
+ /// The source instant under the playhead when the stroke began.
+ at: f64,
points: Vec<[f64; 2]>,
screen: Vec<(f32, f32)>,
},
@@ -720,6 +724,8 @@ pub struct Studio {
pub painting: bool,
/// The mask analyses running, by media id, and how far each has got.
cutout_jobs: HashMap<String, (bool, f32)>,
+ /// The smart stroke being read, by the same name, while one is.
+ region_job: Option<String>,
}
// ── conversions between the document and the window ─────────────────────
@@ -1142,6 +1148,7 @@ impl Studio {
brush_size: 0.06,
painting: true,
cutout_jobs: HashMap::new(),
+ region_job: None,
host,
};
studio.settings.language = studio
@@ -1349,6 +1356,7 @@ impl Studio {
self.request_media_art();
self.request_preview();
self.ensure_cutouts();
+ self.ensure_regions();
}
pub fn undo(&mut self) {
@@ -3030,6 +3038,7 @@ impl Studio {
clip: clip.id.clone(),
tool: BRUSHES[self.brush.min(BRUSHES.len() - 1)],
size: self.brush_size,
+ at: self.source_at_playhead(&clip),
points: vec![point],
screen: vec![(x, y)],
};
@@ -3389,14 +3398,22 @@ impl Studio {
clip,
tool,
size,
+ at,
points,
..
} => {
- // The stroke becomes one command, and one undo step.
+ // The stroke becomes one command, and one undo step; a
+ // smart stroke then has its thing read from the frame.
self.apply(Command::AddCutoutStroke {
clip_id: clip,
- stroke: model::Stroke { tool, size, points },
+ stroke: model::Stroke {
+ tool,
+ size,
+ points,
+ at: Some(at),
+ },
});
+ self.ensure_regions();
return;
}
other => {
@@ -3915,6 +3932,7 @@ impl Studio {
self.request_media_art();
self.request_preview();
self.ensure_cutouts();
+ self.ensure_regions();
}
Err(error) => {
self.start.busy = false;
@@ -3961,6 +3979,7 @@ impl Studio {
self.pause();
self.host.cutouts.cancel();
self.cutout_jobs.clear();
+ self.region_job = None;
if let Some(session) = self.session.as_mut() {
let (path, document) = session.prepare_save(None);
if let Err(error) = projects::save(&path, &document) {
@@ -5183,6 +5202,12 @@ impl Studio {
}
}
+ /// The source instant of `clip` under the playhead, held to the clip.
+ fn source_at_playhead(&self, clip: &Clip) -> f64 {
+ let along = (f64::from(self.playhead) - clip.start).clamp(0.0, clip.duration);
+ clip.source_start + along * clip.speed
+ }
+
/// Whether the cutout model found nothing at the playhead's frame of
/// `clip`, so the picture is showing as shot. Only an automatic cutout
/// says so: a custom one is whatever was painted.
@@ -5205,9 +5230,85 @@ impl Studio {
&media.path,
cutout.subject,
));
- let along = (f64::from(self.playhead) - clip.start).clamp(0.0, clip.duration);
- let source = clip.source_start + along * clip.speed;
- store.mask_at(source).is_some_and(|mask| mask.is_blank())
+ store
+ .mask_at(self.source_at_playhead(clip))
+ .is_some_and(|mask| mask.is_blank())
+ }
+
+ /// Starts reading the first smart stroke whose region is not there
+ /// yet, unless one is being read. Called after every change, and by
+ /// the finished job for whatever is next.
+ pub fn ensure_regions(&mut self) {
+ if self.region_job.is_some() || self.host.brushes.is_busy() {
+ return;
+ }
+ let Some(session) = self.session.as_ref() else {
+ return;
+ };
+ let project = std::path::PathBuf::from(session.path());
+ let mut next: Option<(String, RegionRequest)> = None;
+ 'clips: for clip in &self.timeline().clips {
+ let Some(cutout) = clip.cutout.as_ref() else {
+ continue;
+ };
+ if cutout.mode != model::CutoutMode::Custom || !clip.kind.is_visual() {
+ continue;
+ }
+ let Some(media) = self.project().media_by_id(&clip.media_id) else {
+ continue;
+ };
+ for stroke in &cutout.strokes {
+ let request = RegionRequest {
+ project: project.clone(),
+ media_path: media.path.clone(),
+ still: media.kind == model::MediaKind::Image,
+ subject: cutout.subject,
+ stroke: stroke.clone(),
+ };
+ if request.outstanding() {
+ next = Some((Self::analysis_key(&media.id, cutout.subject), request));
+ break 'clips;
+ }
+ }
+ }
+ let Some((key, request)) = next else {
+ return;
+ };
+ self.region_job = Some(key.clone());
+ self.cutout_jobs.entry(key.clone()).or_insert((false, 0.0));
+ let brushes = Arc::clone(&self.host.brushes);
+ spawn(
+ move || {
+ let reporting = key.clone();
+ let result = brushes.read(&request, &mut |progress| {
+ let now = match progress {
+ concat_host::cutout::Progress::Fetching { received, total } => {
+ (true, received as f32 / total.max(1) as f32)
+ }
+ concat_host::cutout::Progress::Analysing(fraction) => (false, fraction),
+ };
+ let key = reporting.clone();
+ on_ui(move |studio, _, _| {
+ if let Some(held) = studio.cutout_jobs.get_mut(&key) {
+ *held = now;
+ }
+ });
+ });
+ (key, result)
+ },
+ |studio, _, _, (key, result)| {
+ studio.region_job = None;
+ studio.cutout_jobs.remove(&key);
+ match result {
+ Ok(()) => {
+ studio.request_preview();
+ studio.ensure_regions();
+ }
+ Err(error) if error.contains("cancelled") => {}
+ Err(error) => studio.notify(&tf("Smart brush: {0}", &[&error]), true),
+ }
+ },
+ );
}
/// The menus, the dialogs, the bin and the engine lists.
diff --git a/engine/crates/concat/ui/inspector/clip-inspector.slint b/engine/crates/concat/ui/inspector/clip-inspector.slint
index f8f3b9dc..9e720cd9 100644
--- a/engine/crates/concat/ui/inspector/clip-inspector.slint
+++ b/engine/crates/concat/ui/inspector/clip-inspector.slint
@@ -937,7 +937,7 @@ export component VideoPanel inherits VerticalLayout {
if cutout.open: SectionBody {
if root.cutout-help: Note {
- text: I18n.t("Auto removal keeps the person the cutout model finds and drops the rest. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush and smart eraser follow the model's confidence, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.");
+ text: I18n.t("Auto removal keeps what the model finds and drops the rest: a person, or the one thing the picture is of. Custom removal starts from the same mask and takes corrections painted on the monitor: the smart brush keeps the whole thing it touches and the smart eraser removes it, the plain ones do exactly what you paint; Apply ends the painting. Chroma key drops one colour instead; similarity is how far from the colour still counts, softness feathers the edge.");
}
RemovalBlock {