The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(enhance): Size, Strength and a one-frame preview before the whole run
Jareer03c8fd66706f
diff --git a/src/crates/concat-host/src/enhance.rs b/src/crates/concat-host/src/enhance.rsindex d8330fe3..369bd94d 100644--- a/src/crates/concat-host/src/enhance.rs+++ b/src/crates/concat-host/src/enhance.rs@@ -14,24 +14,50 @@//!//! One job at a time through a [`SingleFlight`], like every long job the//! host runs, with the model fetched on first use and kept loaded. The-//! copy is named after the file's path, size and modification time and-//! the factor it was enlarged by, so a file replaced on disk gets a fresh-//! one and the same file asked for twice gets the one already there.+//! copy is named after the file's path, size and modification time, the+//! factor it was enlarged by and the strength it was blended at, so a file+//! replaced on disk gets a fresh one, the same file asked for twice gets+//! the one already there, and asking again with other settings writes a+//! copy of its own beside the first rather than over it.+//!+//! Before a person spends minutes of their machine on a whole file,+//! [`Enhancers::preview`] runs the model on the middle of one frame - a+//! few tiles, seconds - and says how long a tile took, which is what the+//! window's estimate for the whole run is made of.+//!+//! Enhance only ever runs because the person asked, so a crash inside it+//! does not come back on the next open and needs no tripwire; see+//! [`crate::jobs::Tripwire`].use std::hash::{Hash, Hasher};use std::path::{Path, PathBuf};use std::sync::atomic::AtomicBool;use std::sync::{Arc, Mutex};+use concat_core::Frame;+use concat_core::time::Rational;use concat_media::{- DecodeOptions, Decoder, EncodeOptions, Encoder, FrameSink, FrameSource, RateMode, VideoCodec,+ DecodeOptions, Decoder, EncodeOptions, Encoder, FrameSink, FrameSource, RateMode,+ SeekableSource, VideoCodec,};-use concat_vision::enhance::Enhancer;+use concat_vision::enhance::{self as vision, Enhancer};use concat_vision::{ModelId, models};pub use crate::cutout::Progress;use crate::jobs::SingleFlight;+/// The most a preview shows, in output pixels: about what an inspector a+/// few hundred points wide, on a screen of two pixels a point, shows one+/// to one. Its source is a handful of tiles at most.+pub const PREVIEW_WIDTH: u32 = 640;+/// See [`PREVIEW_WIDTH`]: the same at sixteen by nine.+pub const PREVIEW_HEIGHT: u32 = 360;++/// The strength the inspector starts at: most of the model's restoration,+/// with a quarter of the original's texture left in so faces do not come+/// out waxen - the "plastic" look a full-strength enhancer is known for.+pub const DEFAULT_STRENGTH: f32 = 0.75;+/// What one enhancement covers.#[derive(Clone, Debug)]pub struct EnhanceRequest {@@ -40,12 +66,71 @@ pub struct EnhanceRequest {/// A still: one frame in, one PNG out.pub still: bool,/// How many times larger the copy is; see- /// [`concat_vision::enhance::factor_for`].+ /// [`concat_vision::enhance::factor_of`].pub factor: u32,+ /// How much of the model's answer is kept, 0 to 1; the rest is the+ /// original, enlarged plainly. See [`concat_vision::enhance::mix`].+ pub strength: f32,/// Where the copy goes; see [`target_for`].pub target: PathBuf,}+/// What a preview covers: one frame of a file, at the size it would be+/// enhanced to.+#[derive(Clone, Debug)]+pub struct PreviewRequest {+ /// The media file to read.+ pub media_path: String,+ /// Seconds into the file: the frame under the playhead. 0 for a still.+ pub seconds: f64,+ /// How many times larger, as for [`EnhanceRequest::factor`].+ pub factor: u32,+}++/// One frame's middle, before and after: both at the enhanced size, so+/// the window can blend them at any strength without the model.+#[derive(Clone, Debug)]+pub struct Preview {+ /// The middle of the frame enlarged by an ordinary bicubic filter -+ /// what the clip would be without Enhance, and what Strength blends+ /// towards.+ pub plain: Frame,+ /// The same middle as the model answers it, at full strength.+ pub enhanced: Frame,+ /// Seconds the model took over one tile, warm; see+ /// [`steady_tile_seconds`].+ pub tile_seconds: f64,+ /// Tiles in one whole frame of the file.+ pub tiles_per_frame: usize,+}++/// `strength` as the whole percent it is keyed by: what the inspector+/// shows, and what tells two copies apart, so float noise in a slider+/// never makes a copy of its own.+pub fn strength_percent(strength: f32) -> u32 {+ (strength.clamp(0.0, 1.0) * 100.0).round() as u32+}++/// How long the model takes over one tile once it is warm: the fastest of+/// `times`. The first run of a model in a session compiles it and can take+/// many times longer than the rest, and a preview is often that first run;+/// every tile is one shape, so the rest take as long as each other.+pub fn steady_tile_seconds(times: &[std::time::Duration]) -> f64 {+ times+ .iter()+ .min()+ .map(std::time::Duration::as_secs_f64)+ .unwrap_or(0.0)+}++/// About how long enhancing `frames` frames of `tiles_per_frame` tiles+/// each takes at `tile_seconds` a tile, in seconds: the model's share,+/// which is nearly all of it - decoding and encoding are a rounding error+/// beside it.+pub fn estimate_seconds(tile_seconds: f64, tiles_per_frame: usize, frames: f64) -> f64 {+ tile_seconds * tiles_per_frame as f64 * frames.max(1.0)+}+/// The enhancement service: the model, loaded once, and the one-job slot.pub struct Enhancers {gate: Arc<SingleFlight>,@@ -54,10 +139,19 @@ pub struct Enhancers {model: Mutex<Option<Arc<Enhancer>>>,}-/// Where the enhanced copy of `media` at `factor` lives under `project`:-/// in the cache, named by the file's path, size and modification time.-/// None for a file that cannot be stat'ed, which cannot be read either.-pub fn target_for(project: &Path, media: &str, factor: u32, still: bool) -> Option<PathBuf> {+/// Where the enhanced copy of `media` at `factor` and `strength` lives+/// under `project`: in the cache, named by the file's path, size and+/// modification time and by the settings. Full strength - all Enhance+/// made before it had a strength - keeps the old name, so a copy made+/// then is found again rather than made twice. None for a file that+/// cannot be stat'ed, which cannot be read either.+pub fn target_for(+ project: &Path,+ media: &str,+ factor: u32,+ strength: f32,+ still: bool,+) -> Option<PathBuf> {let meta = std::fs::metadata(media).ok()?;let mut hasher = std::collections::hash_map::DefaultHasher::new();media.hash(&mut hasher);@@ -68,12 +162,38 @@ pub fn target_for(project: &Path, media: &str, factor: u32, still: bool) -> Optisince.as_secs().hash(&mut hasher);}let extension = if still { "png" } else { "mp4" };+ let percent = strength_percent(strength);+ let strength = if percent == 100 {+ String::new()+ } else {+ format!("-{percent}pc")+ };Some(project.join("cache").join(format!(- "enhance-{:016x}-{factor}x.{extension}",+ "enhance-{:016x}-{factor}x{strength}.{extension}",hasher.finish())))}+/// The frame of `path` at `seconds`, at its own size, or the first frame+/// when `seconds` is past the end.+fn frame_at(path: &str, seconds: f64) -> Result<Frame, String> {+ let mut options = DecodeOptions::default().in_software();+ if seconds > 0.0 {+ options = options.starting_at(Rational::approximate(seconds).unwrap_or(Rational::ZERO));+ }+ let mut decoder = Decoder::open(path, &options).map_err(|error| error.to_string())?;+ if let Some(frame) = decoder.next_frame().map_err(|error| error.to_string())? {+ return Ok(frame);+ }+ decoder+ .seek(Rational::ZERO)+ .map_err(|error| error.to_string())?;+ decoder+ .next_frame()+ .map_err(|error| error.to_string())?+ .ok_or_else(|| format!("{path} holds no picture"))+}+impl Enhancers {/// A service with nothing loaded yet, keeping its model under `data`.pub fn new(data: &Path) -> Enhancers {@@ -148,6 +268,41 @@ impl Enhancers {};result.map(|()| request.target.clone())}++ /// The middle of one frame through the model, at most+ /// [`PREVIEW_WIDTH`] by [`PREVIEW_HEIGHT`] once enlarged, beside the+ /// same middle enlarged plainly. Takes the one job slot like a whole+ /// run - the model is one model - and blocks until done; `progress`+ /// reports the model's download, the one slow part a preview can+ /// have. Nothing is written: the clip is not touched.+ pub fn preview(+ &self,+ request: &PreviewRequest,+ progress: &mut dyn FnMut(Progress),+ ) -> Result<Preview, String> {+ let job = self.gate.begin("enhance")?;+ let cancel = job.cancel_handle();+ let enhancer = self.loaded(&cancel, progress)?;+ if job.cancelled() {+ return Err("enhance cancelled".to_owned());+ }+ let frame = frame_at(&request.media_path, request.seconds)?;+ let (width, height) = (frame.width(), frame.height());+ let region =+ vision::centre_region(width, height, request.factor, PREVIEW_WIDTH, PREVIEW_HEIGHT);+ let (enhanced, times) = enhancer.enhance_region(&frame, region, request.factor)?;+ let plain = vision::resize(+ &vision::cut(&frame, region),+ enhanced.width(),+ enhanced.height(),+ );+ Ok(Preview {+ plain,+ enhanced,+ tile_seconds: steady_tile_seconds(×),+ tiles_per_frame: vision::tiles(width, height).len(),+ })+ }}/// One frame through the model, as a PNG: lossless, since the point was@@ -159,7 +314,11 @@ fn still(enhancer: &Enhancer, request: &EnhanceRequest) -> Result<(), String> {.next_frame().map_err(|error| error.to_string())?.ok_or_else(|| format!("{} holds no picture", request.media_path))?;- let enhanced = enhancer.enhance(&frame, request.factor)?;+ let enhanced = vision::mix(+ enhancer.enhance(&frame, request.factor)?,+ &frame,+ request.strength,+ );let partial = request.target.with_extension("part.png");let written = (|| {let file = std::fs::File::create(&partial).map_err(|error| error.to_string())?;@@ -248,7 +407,11 @@ fn footage(if job.cancelled() {return Err("enhance cancelled".to_owned());}- let enhanced = enhancer.enhance(&frame, request.factor)?;+ let enhanced = vision::mix(+ enhancer.enhance(&frame, request.factor)?,+ &frame,+ request.strength,+ );let enhanced = crop(&enhanced, width, height);encoder.write_frame(&enhanced)@@ -319,23 +482,73 @@ mod tests {std::fs::write(&media, b"not really").expect("write");let media = media.to_string_lossy().into_owned();let project = dir.join("project");- let two = target_for(&project, &media, 2, false).expect("named");+ // Full strength keeps the name Enhance gave copies before it had+ // one, so those are found again.+ let two = target_for(&project, &media, 2, 1.0, false).expect("named");assert!(two.starts_with(project.join("cache")));assert!(two.to_string_lossy().ends_with("-2x.mp4"), "{two:?}");- assert_eq!(target_for(&project, &media, 2, false), Some(two.clone()));- assert_ne!(target_for(&project, &media, 1, false), Some(two.clone()));- let still = target_for(&project, &media, 2, true).expect("named");+ assert_eq!(+ target_for(&project, &media, 2, 1.0, false),+ Some(two.clone())+ );+ assert_ne!(+ target_for(&project, &media, 1, 1.0, false),+ Some(two.clone())+ );+ let still = target_for(&project, &media, 2, 1.0, true).expect("named");assert!(still.to_string_lossy().ends_with("-2x.png"), "{still:?}");assert_eq!(- target_for(&project, &dir.join("gone.mp4").to_string_lossy(), 2, false),+ target_for(+ &project,+ &dir.join("gone.mp4").to_string_lossy(),+ 2,+ 1.0,+ false+ ),None);+ // Another strength is another copy, keyed by its whole percent so+ // a slider's float noise is not.+ let blended = target_for(&project, &media, 2, 0.75, false).expect("named");+ assert!(+ blended.to_string_lossy().ends_with("-2x-75pc.mp4"),+ "{blended:?}"+ );+ assert_ne!(blended, two);+ assert_eq!(+ target_for(&project, &media, 2, 0.750_001, false),+ Some(blended.clone())+ );+ assert_ne!(target_for(&project, &media, 4, 0.75, false), Some(blended));// A file with different bytes is a different copy.std::fs::write(&media, b"not really, but longer").expect("write");- assert_ne!(target_for(&project, &media, 2, false), Some(two));+ assert_ne!(target_for(&project, &media, 2, 1.0, false), Some(two));let _ = std::fs::remove_dir_all(&dir);}+ #[test]+ fn the_estimate_is_the_warm_tile_time_over_every_tile_of_every_frame() {+ use std::time::Duration;+ // The first run compiled the model; the warm ones say what a tile+ // costs.+ let times = [+ Duration::from_millis(2400),+ Duration::from_millis(300),+ Duration::from_millis(320),+ ];+ assert!((steady_tile_seconds(×) - 0.3).abs() < 1e-9);+ assert_eq!(steady_tile_seconds(&[]), 0.0);+ // 1080p is 8 by 5 tiles; ten seconds at 30 is 300 frames.+ let tiles = vision::tiles(1920, 1080).len();+ assert_eq!(tiles, 40);+ assert!((estimate_seconds(0.3, tiles, 300.0) - 3600.0).abs() < 1e-6);+ // A still is one frame, however short its duration says it is.+ assert!((estimate_seconds(0.3, tiles, 0.0) - 12.0).abs() < 1e-6);+ assert_eq!(strength_percent(0.754), 75);+ assert_eq!(strength_percent(1.2), 100);+ assert_eq!(strength_percent(-1.0), 0);+ }+#[test]fn a_crop_takes_the_top_left_and_leaves_a_fitting_frame_alone() {let mut frame = concat_core::Frame::black(4, 3);diff --git a/src/crates/concat-vision/src/enhance.rs b/src/crates/concat-vision/src/enhance.rsindex 9d76a767..7b8aa275 100644--- a/src/crates/concat-vision/src/enhance.rs+++ b/src/crates/concat-vision/src/enhance.rs@@ -17,7 +17,16 @@//! plenty at 4K, and 8K is a file nobody can play - so [`factor_for`]//! picks two, or one when the picture is already large, and the four-times//! answer is averaged down to it. Averaging down from the restored picture-//! keeps the restoration and drops only the size.+//! keeps the restoration and drops only the size. The person can ask for a+//! size of their own instead - see [`factor_of`] - but never one past+//! [`LONGEST_SIDE`].+//!+//! The model's answer is also as strong as it is: skin smoothed to wax,+//! grain scrubbed to plastic, which is the complaint every one-button+//! enhancer earns. [`mix`] blends it back towards the original, enlarged+//! by an ordinary bicubic filter to the same size, by a strength the+//! person picks - most of the restoration, with some of the texture the+//! model took away left in.use concat_core::Frame;use concat_core::frame::BYTES_PER_PIXEL;@@ -74,11 +83,177 @@ pub fn tiles(width: u32, height: u32) -> Vec<Tile> {/// unless that would pass [`LONGEST_SIDE`], and then once, restored at/// its own size.pub fn factor_for(width: u32, height: u32) -> u32 {- if width.max(height) * 2 > LONGEST_SIDE {- 1- } else {- 2+ if fits(width, height, 2) { 2 } else { 1 }+}++/// The sizes a person can ask for, in times larger: restored at its own+/// size, twice, and the model's own four times.+pub const SIZES: [u32; 3] = [1, 2, 4];++/// Whether a `width` by `height` picture enlarged `factor` times stays+/// within [`LONGEST_SIDE`]. Once - restored at its own size - always does:+/// a picture already past the limit is still restored, just not enlarged.+pub fn fits(width: u32, height: u32, factor: u32) -> bool {+ factor <= 1 || u64::from(width.max(height)) * u64::from(factor) <= u64::from(LONGEST_SIDE)+}++/// How many times larger a `width` by `height` picture comes out when the+/// person asked for `asked`: 0 is Auto, [`factor_for`]; otherwise the+/// largest of [`SIZES`] no larger than what was asked that [`fits`]. The+/// ask is a setting that outlives the clip it was made on, so 4x picked+/// on a small clip and carried to a 4K one comes out once, not at 16K.+pub fn factor_of(asked: u32, width: u32, height: u32) -> u32 {+ if asked == 0 {+ return factor_for(width, height);+ }+ SIZES+ .iter()+ .copied()+ .filter(|&factor| factor <= asked && fits(width, height, factor))+ .max()+ .unwrap_or(1)+}++/// The rectangle of a `width` by `height` picture, at its centre, that+/// comes out no larger than `most_width` by `most_height` once enlarged+/// `factor` times: what a preview restores, so that it shows the model's+/// work at full size in seconds rather than the whole frame in minutes.+/// The whole picture when it is smaller than that.+pub fn centre_region(+ width: u32,+ height: u32,+ factor: u32,+ most_width: u32,+ most_height: u32,+) -> Tile {+ let factor = factor.max(1);+ let w = (most_width / factor).max(1).min(width);+ let h = (most_height / factor).max(1).min(height);+ Tile {+ x: (width - w) / 2,+ y: (height - h) / 2,+ width: w,+ height: h,+ }+}++/// The pixels of `frame` inside `region`, which must lie inside it.+pub fn cut(frame: &Frame, region: Tile) -> Frame {+ let mut out = Frame::black(region.width, region.height);+ let row = region.width as usize * BYTES_PER_PIXEL;+ let stride = frame.width() as usize * BYTES_PER_PIXEL;+ let source = frame.pixels();+ let pixels = out.pixels_mut();+ for y in 0..region.height as usize {+ let at = (region.y as usize + y) * stride + region.x as usize * BYTES_PER_PIXEL;+ pixels[y * row..(y + 1) * row].copy_from_slice(&source[at..at + row]);+ }+ out+}++/// The Catmull-Rom weights of the four source samples around a point `t`+/// of the way from the second to the third: sharp enough that an+/// enlargement is not mush, with less ringing than a longer filter, and+/// summing to one so a flat picture stays flat.+fn cubic_weights(t: f32) -> [f32; 4] {+ let (t2, t3) = (t * t, t * t * t);+ [+ 0.5 * (-t3 + 2.0 * t2 - t),+ 0.5 * (3.0 * t3 - 5.0 * t2 + 2.0),+ 0.5 * (-3.0 * t3 + 4.0 * t2 + t),+ 0.5 * (t3 - t2),+ ]+}++/// For each of `to` output samples along one side, the four samples it+/// reads of a side `from` long, the edge repeated past the end, and their+/// weights. Pixel centres line up: output sample `i` sits at source+/// position `(i + 0.5) * from / to - 0.5`.+fn taps(from: u32, to: u32) -> Vec<([usize; 4], [f32; 4])> {+ let ratio = from as f32 / to.max(1) as f32;+ let last = i64::from(from.saturating_sub(1));+ (0..to)+ .map(|i| {+ let at = (i as f32 + 0.5) * ratio - 0.5;+ let base = at.floor();+ let weights = cubic_weights(at - base);+ let base = base as i64;+ let index = |k: i64| (base + k).clamp(0, last) as usize;+ ([index(-1), index(0), index(1), index(2)], weights)+ })+ .collect()+}++/// `frame` enlarged to `width` by `height` with a bicubic filter: the+/// picture as an ordinary resize gives it, which is what Strength blends+/// the model's answer towards. Meant for enlarging; shrinking with it+/// skips pixels rather than averaging them.+pub fn resize(frame: &Frame, width: u32, height: u32) -> Frame {+ if frame.width() == width && frame.height() == height {+ return frame.clone();+ }+ let (fw, fh) = (frame.width() as usize, frame.height() as usize);+ if fw == 0 || fh == 0 || width == 0 || height == 0 {+ return Frame::black(width, height);+ }+ let (ow, oh) = (width as usize, height as usize);+ let source = frame.pixels();+ // Across first, into floats a source row at a time, then down.+ let across = taps(frame.width(), width);+ let mut wide = vec![0f32; ow * fh * BYTES_PER_PIXEL];+ for y in 0..fh {+ let row = &source[y * fw * BYTES_PER_PIXEL..(y + 1) * fw * BYTES_PER_PIXEL];+ for (x, (index, weight)) in across.iter().enumerate() {+ let at = (y * ow + x) * BYTES_PER_PIXEL;+ for c in 0..BYTES_PER_PIXEL {+ wide[at + c] = (0..4)+ .map(|k| weight[k] * f32::from(row[index[k] * BYTES_PER_PIXEL + c]))+ .sum();+ }+ }+ }+ let down = taps(frame.height(), height);+ let mut pixels = vec![0u8; ow * oh * BYTES_PER_PIXEL];+ for (y, (index, weight)) in down.iter().enumerate() {+ for x in 0..ow {+ let at = (y * ow + x) * BYTES_PER_PIXEL;+ for c in 0..BYTES_PER_PIXEL {+ let value: f32 = (0..4)+ .map(|k| weight[k] * wide[(index[k] * ow + x) * BYTES_PER_PIXEL + c])+ .sum();+ pixels[at + c] = (value + 0.5).clamp(0.0, 255.0) as u8;+ }+ }+ }+ Frame::from_rgba(width, height, pixels).expect("sides and bytes agree")+}++/// `strength` of the way from `plain` to `enhanced`, pixel by pixel: 0 is+/// `plain`, 1 is `enhanced`. The two must be one size.+pub fn blend(enhanced: &Frame, plain: &Frame, strength: f32) -> Frame {+ debug_assert!(enhanced.same_size_as(plain));+ let strength = strength.clamp(0.0, 1.0);+ let pixels = enhanced+ .pixels()+ .iter()+ .zip(plain.pixels())+ .map(|(&e, &p)| {+ let (e, p) = (f32::from(e), f32::from(p));+ (p + (e - p) * strength + 0.5).clamp(0.0, 255.0) as u8+ })+ .collect();+ Frame::from_rgba(enhanced.width(), enhanced.height(), pixels).expect("sides and bytes agree")+}++/// The model's answer `enhanced` for `original`, at `strength`: itself at+/// full strength, and otherwise blended with `original` enlarged to its+/// size by [`resize`].+pub fn mix(enhanced: Frame, original: &Frame, strength: f32) -> Frame {+ if strength >= 1.0 {+ return enhanced;}+ let plain = resize(original, enhanced.width(), enhanced.height());+ blend(&enhanced, &plain, strength)}/// The model's input for one tile: the [`WINDOW`] square around it as@@ -189,15 +364,39 @@ impl Enhancer {/// `frame` restored at [`SCALE`] times its size.pub fn upscale(&self, frame: &Frame) -> Result<Frame, String> {- let (width, height) = (frame.width(), frame.height());- if width == 0 || height == 0 {+ let whole = Tile {+ x: 0,+ y: 0,+ width: frame.width(),+ height: frame.height(),+ };+ Ok(self.upscale_region(frame, whole)?.0)+ }++ /// `region` of `frame` restored at [`SCALE`] times its size, and how+ /// long each of its tiles took the model. Each tile still reads its+ /// margin from the frame around the region, so the region comes out as+ /// it would inside the whole frame.+ pub fn upscale_region(+ &self,+ frame: &Frame,+ region: Tile,+ ) -> Result<(Frame, Vec<std::time::Duration>), String> {+ if region.width == 0 || region.height == 0 {return Err("enhance: an empty picture".to_owned());}- let mut out = Frame::black(width * SCALE, height * SCALE);+ let mut out = Frame::black(region.width * SCALE, region.height * SCALE);let side = WINDOW as usize;let mut model = self.model.lock().map_err(|_| "enhance: model poisoned")?;- for tile in tiles(width, height) {- let planes = window_planes(frame, tile);+ let mut times = Vec::new();+ for tile in tiles(region.width, region.height) {+ let placed = Tile {+ x: region.x + tile.x,+ y: region.y + tile.y,+ ..tile+ };+ let planes = window_planes(frame, placed);+ let started = std::time::Instant::now();let answer = model.run(vec![crate::runtime::Input {name: "input",@@ -206,9 +405,10 @@ impl Enhancer {}],&["output"],)?;+ times.push(started.elapsed());paste(&mut out, tile, &answer[0].data)?;}- Ok(out)+ Ok((out, times))}/// `frame` restored and enlarged `factor` times, `factor` being 1, 2@@ -218,6 +418,19 @@ impl Enhancer {let large = self.upscale(frame)?;Ok(downscale(&large, SCALE / factor))}++ /// [`Enhancer::enhance`] for `region` of `frame` alone, with the time+ /// each of its tiles took; see [`Enhancer::upscale_region`].+ pub fn enhance_region(+ &self,+ frame: &Frame,+ region: Tile,+ factor: u32,+ ) -> Result<(Frame, Vec<std::time::Duration>), String> {+ let factor = factor.clamp(1, SCALE);+ let (large, times) = self.upscale_region(frame, region)?;+ Ok((downscale(&large, SCALE / factor), times))+ }}#[cfg(test)]@@ -376,4 +589,107 @@ mod tests {let none = downscale(&Frame::black(3, 3), 4);assert_eq!((none.width(), none.height()), (0, 0));}++ #[test]+ fn an_asked_size_is_kept_under_the_limit_and_auto_is_the_old_choice() {+ // Once always fits, even past the limit: that is restore-only.+ assert!(fits(8000, 6000, 1));+ assert!(fits(1024, 768, 4));+ assert!(!fits(1025, 768, 4));+ assert!(fits(2048, 1152, 2));+ assert!(!fits(1920, 1080, 4), "1080p at 4x is 7680 across");+ // Auto is what Enhance always did.+ for (w, h) in [(1920, 1080), (3840, 2160), (640, 480), (2049, 10)] {+ assert_eq!(factor_of(0, w, h), factor_for(w, h), "{w}x{h}");+ }+ // An ask that fits is kept.+ assert_eq!(factor_of(1, 640, 480), 1);+ assert_eq!(factor_of(2, 640, 480), 2);+ assert_eq!(factor_of(4, 640, 480), 4);+ // An ask that does not falls to the largest size that does.+ assert_eq!(factor_of(4, 1920, 1080), 2);+ assert_eq!(factor_of(4, 3840, 2160), 1);+ assert_eq!(factor_of(2, 3840, 2160), 1);+ }++ #[test]+ fn a_preview_region_sits_at_the_centre_and_comes_out_no_larger_than_asked() {+ let region = centre_region(1920, 1080, 2, 640, 360);+ assert_eq!(+ region,+ Tile {+ x: 800,+ y: 450,+ width: 320,+ height: 180+ }+ );+ // At once, more of the picture fits the same output.+ assert_eq!(centre_region(1920, 1080, 1, 640, 360).width, 640);+ // A picture smaller than the region is taken whole.+ assert_eq!(+ centre_region(200, 100, 2, 640, 360),+ Tile {+ x: 0,+ y: 0,+ width: 200,+ height: 100+ }+ );+ let frame = picture(8, 6, |x, y| [x as u8, y as u8, 0, 255]);+ let region = centre_region(8, 6, 1, 4, 2);+ let piece = cut(&frame, region);+ assert_eq!((piece.width(), piece.height()), (4, 2));+ assert_eq!(piece.pixel(0, 0), Some([2, 2, 0, 255]));+ assert_eq!(piece.pixel(3, 1), Some([5, 3, 0, 255]));+ }++ #[test]+ fn a_bicubic_enlargement_keeps_a_flat_picture_flat_and_lines_up_its_pixels() {+ let flat = picture(5, 3, |_, _| [40, 120, 200, 255]);+ let big = resize(&flat, 20, 12);+ assert_eq!((big.width(), big.height()), (20, 12));+ assert!(+ big.pixels()+ .chunks_exact(4)+ .all(|p| p == [40, 120, 200, 255]),+ "weights that sum to one leave a flat field alone"+ );+ // The same size is a copy.+ let ramp = picture(6, 4, |x, y| [(x * 40) as u8, (y * 60) as u8, 9, 255]);+ assert_eq!(resize(&ramp, 6, 4).pixels(), ramp.pixels());+ // Doubled, a ramp stays a ramp: output pixel 5 sits at source 2.25,+ // which on a line of 40 a pixel is 90, and the cubic reproduces a+ // line exactly away from the ends. At the ends, where the edge+ // repeats and the line turns flat, it overshoots by a few levels+ // and no more.+ let big = resize(&ramp, 12, 8);+ let reds: Vec<u8> = (0..12).map(|x| big.pixel(x, 3).unwrap()[0]).collect();+ assert!(reds.windows(2).all(|pair| pair[0] <= pair[1]), "{reds:?}");+ assert_eq!(reds[5], 90, "{reds:?}");+ assert_eq!(reds[0], 0);+ assert!((200..=206).contains(&reds[11]), "{reds:?}");+ }++ #[test]+ fn strength_blends_from_the_plain_enlargement_to_the_models_answer() {+ let enhanced = picture(2, 1, |_, _| [200, 100, 0, 255]);+ let plain = picture(2, 1, |_, _| [100, 100, 255, 255]);+ assert_eq!(blend(&enhanced, &plain, 1.0).pixels(), enhanced.pixels());+ assert_eq!(blend(&enhanced, &plain, 0.0).pixels(), plain.pixels());+ assert_eq!(+ blend(&enhanced, &plain, 0.75).pixel(1, 0),+ Some([175, 100, 64, 255])+ );+ // Out of range is held to it.+ assert_eq!(blend(&enhanced, &plain, 3.0).pixels(), enhanced.pixels());+ // Mixing at full strength is the answer untouched; below it, the+ // original is enlarged to the answer's size first.+ let original = picture(1, 1, |_, _| [100, 100, 255, 255]);+ let full = mix(enhanced.clone(), &original, 1.0);+ assert_eq!(full.pixels(), enhanced.pixels());+ let half = mix(enhanced.clone(), &original, 0.5);+ assert_eq!((half.width(), half.height()), (2, 1));+ assert_eq!(half.pixel(0, 0), Some([150, 100, 128, 255]));+ }}diff --git a/src/crates/concat/locales/en.json b/src/crates/concat/locales/en.jsonindex 5303d31c..bb3e01ba 100644--- a/src/crates/concat/locales/en.json+++ b/src/crates/concat/locales/en.json@@ -38,9 +38,18 @@"clipInspector.darken": "Darken","clipInspector.effectLibrary": "Effect library","clipInspector.enable": "Enable {0}",- "clipInspector.enhanceExplain": "Restores and enlarges the picture with the enhance model, into a copy the clip then shows. Tick to start; it runs in the background.",- "clipInspector.enhanceShowingCopy": "Showing the enhanced copy. Untick to show the original again.",+ "clipInspector.enhanceAfter": "After · {0}% strength",+ "clipInspector.enhanceAgain": "Enhance again",+ "clipInspector.enhanceAuto": "Auto ({0})",+ "clipInspector.enhanceBefore": "Before · a plain enlargement",+ "clipInspector.enhanceExplain": "Restores and enlarges the picture with the enhance model, into a copy the clip then shows. Preview a frame to judge the settings first; tick to run the whole file in the background.",+ "clipInspector.enhancePreview": "Preview this frame",+ "clipInspector.enhancePreviewing": "Running the model on the middle of the frame under the playhead…",+ "clipInspector.enhanceShowingCopy": "Showing the enhanced copy. Untick to show the original again, or change the settings and Enhance again for another copy.","clipInspector.enhanceShowingOriginal": "Showing the original. Tick to show the enhanced copy again.",+ "clipInspector.enhanceSize": "Size",+ "clipInspector.enhanceStrength": "Strength",+ "clipInspector.enhanceStrengthExplain": "How much of the model's work is kept; the rest is the original, enlarged plainly. Lower keeps more real texture, so faces look less waxen.","clipInspector.enhanceVoice": "Enhance voice","clipInspector.enhancing": "Enhancing… {0}%","clipInspector.eraser": "Eraser",@@ -222,11 +231,14 @@"export.toneMapped": "Tone-mapped, plays anywhere","format.almostDone": "almost done","format.daysAgo": "{0} days ago",+ "format.hoursMinutes": "{0} h {1} min","format.justNow": "just now",+ "format.minutes": "{0} min","format.minutesAgo": "{0} minutes ago","format.minutesSecondsLeft": "{0}m {1}s left","format.monthsAgo": "{0} months ago","format.sLeft": "{0}s left",+ "format.seconds": "{0}s","format.today": "today","format.yesterday": "yesterday","keyframesPane.addRemoveKeyPlayhead": "Add or remove a key at the playhead",@@ -523,6 +535,10 @@"studio.duplicate": "Duplicate","studio.enhance": "Enhance: {0}","studio.enhanceAlreadyWorkOne": "Enhance is already at work; one clip at a time",+ "studio.enhanceEstimate": "The whole file would take about {0} here: {1} frames at {2}s each.",+ "studio.enhanceEstimateStill": "The whole picture would take about {0} here.",+ "studio.enhanceNoFourTimes": "4x would make it {0} × {1}, past the {2}-pixel limit.",+ "studio.enhanceOnlyRestore": "At {0} pixels across, this picture would pass the {1}-pixel limit if enlarged; it can only be restored at its own size.","studio.enhancedClipNowShows": "Enhanced; the clip now shows the copy","studio.enhancingWayFrameFrame": "Enhancing {0}: {1}× on its way, frame by frame","studio.exit": "Exit",diff --git a/src/crates/concat/src/format.rs b/src/crates/concat/src/format.rsindex 27a1f04a..dbfcca05 100644--- a/src/crates/concat/src/format.rs+++ b/src/crates/concat/src/format.rs@@ -140,6 +140,20 @@ pub fn eta(seconds: f32) -> String {}}+/// Seconds as a rough length of time, for an estimate given before the+/// work starts: "40s", "12 min", "3 h 20 min". Rounded to the unit it is+/// said in, for the reason [`eta`] is.+pub fn span(seconds: f64) -> String {+ if seconds < 59.5 {+ tf("format.seconds", &[&format!("{:.0}", seconds.max(1.0))])+ } else if seconds < 3570.0 {+ tf("format.minutes", &[&format!("{:.0}", seconds / 60.0)])+ } else {+ let minutes = (seconds / 60.0).round() as u64;+ tf("format.hoursMinutes", &[&(minutes / 60), &(minutes % 60)])+ }+}+pub fn hex_of(colour: slint::Color) -> String {format!("#{:02x}{:02x}{:02x}",@@ -503,4 +517,15 @@ mod tests {assert_eq!(when_phrase(now - 86_400_000 - 1000), "yesterday");assert_eq!(when_phrase(now - 3 * 86_400_000), "3 days ago");}++ #[test]+ fn an_estimate_is_said_in_the_unit_that_fits_it() {+ let _english = crate::i18n::language_lock();+ assert_eq!(span(0.2), "1s");+ assert_eq!(span(42.4), "42s");+ assert_eq!(span(59.6), "1 min");+ assert_eq!(span(12.0 * 60.0 + 10.0), "12 min");+ assert_eq!(span(3600.0), "1 h 0 min");+ assert_eq!(span(3.0 * 3600.0 + 20.0 * 60.0), "3 h 20 min");+ }}diff --git a/src/crates/concat/src/lib.rs b/src/crates/concat/src/lib.rsindex f66090e7..f9b2ba33 100644--- a/src/crates/concat/src/lib.rs+++ b/src/crates/concat/src/lib.rs@@ -1305,6 +1305,18 @@ fn start(project: Option<std::path::PathBuf>, smoke: bool) -> Result<(), slint::editor.on_clip_enhance_set(on_window!(|state, on: bool| {state.clip_enhance_set(on);}));+ editor.on_clip_enhance_size(on_window!(|state, index: i32| {+ state.clip_enhance_size(index);+ }));+ editor.on_clip_enhance_strength(on_window!(|state, strength: f32| {+ state.clip_enhance_strength(strength);+ }));+ editor.on_clip_enhance_preview(on_window!(|state| {+ state.clip_enhance_preview();+ }));+ editor.on_clip_enhance_run(on_window!(|state| {+ state.clip_enhance_run();+ }));editor.on_transition_remove(on_window!(|state| {state.remove_transition();}));diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rsindex 8777ac1f..46fa44c1 100644--- a/src/crates/concat/src/studio.rs+++ b/src/crates/concat/src/studio.rs@@ -110,6 +110,58 @@ pub struct Strip {pub height: i32,}+/// One frame's middle through the Enhance model, before and after, for+/// the inspector to show before a whole file is committed to.+struct EnhancePreview {+ /// The clip it was made for, and the file and size it was made from:+ /// a preview of other settings or another file is not this one.+ clip_id: String,+ media_id: String,+ factor: u32,+ /// The plain enlargement and the model's full-strength answer, kept so+ /// Strength re-blends them at once rather than running the model again.+ plain: concat_core::Frame,+ enhanced: concat_core::Frame,+ before: slint::Image,+ after: slint::Image,+ /// The whole percent `after` is blended at.+ after_percent: u32,+ /// See `concat_host::enhance::Preview`.+ tile_seconds: f64,+ tiles_per_frame: usize,+}++/// The Enhance section's rows for one clip; see `Studio::enhance_readout`.+#[derive(Default)]+struct EnhanceReadout {+ /// The Size row's segment: 0 Auto, 1 once, 2 twice, 3 four times.+ size: i32,+ /// What Auto comes to for this file.+ auto: u32,+ fits_2x: bool,+ fits_4x: bool,+ /// Why a size is greyed, or empty.+ size_note: String,+ /// Before, after and the estimate for a whole run, when a preview of+ /// these settings is on hand.+ preview: Option<(slint::Image, slint::Image, String)>,+}++impl EnhancePreview {+ /// `after` blended at `strength`, unless it already is.+ fn blend_at(&mut self, strength: f32) {+ let percent = concat_host::enhance::strength_percent(strength);+ if percent != self.after_percent {+ self.after = image_of(&concat_vision::enhance::blend(+ &self.enhanced,+ &self.plain,+ strength,+ ));+ self.after_percent = percent;+ }+ }+}+impl From<CachedStrip> for Strip {fn from(strip: CachedStrip) -> Self {Self {@@ -948,6 +1000,19 @@ pub struct Studio {/// Enhance at work, by clip id: whether the model is still/// downloading, and how far along. One at a time, like every long job.enhance_jobs: HashMap<String, (bool, f32)>,+ /// The Enhance section's Size, in times larger, 0 for Auto, and its+ /// Strength, 0 to 1. The window's rather than a clip's: they are what+ /// the next run is made with, carried from clip to clip the way a+ /// tool's settings are, and a copy already made is named by its own;+ /// see `concat_host::enhance::target_for`.+ enhance_size: u32,+ enhance_strength: f32,+ /// The last preview of a frame through the model, while the clip it+ /// was made for is the one in the inspector.+ enhance_preview: Option<EnhancePreview>,+ /// A preview being made, for the clip by this id: whether the model is+ /// still downloading, and how far along.+ enhance_previewing: Option<(String, bool, f32)>,/// Reverse writing a clip's backwards copy, by clip: how far along.reverse_jobs: HashMap<String, f32>,/// A selection being rendered into one clip, with its own cancel flag.@@ -2169,6 +2234,10 @@ impl Studio {painting: true,cutout_jobs: HashMap::new(),enhance_jobs: HashMap::new(),+ enhance_size: 0,+ enhance_strength: concat_host::enhance::DEFAULT_STRENGTH,+ enhance_preview: None,+ enhance_previewing: None,reverse_jobs: HashMap::new(),merge_job: None,region_job: None,@@ -5889,10 +5958,11 @@ impl Studio {// an enhanced copy into the project's cache; the clip is then pointed// at the copy, as one undo step, and shows it wherever it showed the// original. The window's part is to start the job, keep the readout- // moving, and re-point the clip when the copy is whole.+ // moving, and re-point the clip when the copy is whole. Before that,+ // it holds the section's Size and Strength and runs a preview of the+ // frame under the playhead, so the person can judge both - and see+ // what the whole run will cost - before committing minutes to it.- /// Enhances the media of clip `id`: starts the job, unless one is- /// running, and says so./// Writes one clip's sound as it plays - trimmed, at its speed, with its/// level, fades and effects baked in - to a WAV in the project's audio/// folder, and puts the file in the bin under Generated › Processed. A@@ -6081,8 +6151,178 @@ impl Studio {}}+ /// The Size row: 0 Auto, then once, twice and four times. A size this+ /// clip cannot take is greyed in the row, and one carried here from a+ /// smaller clip falls to the largest that fits; see+ /// `concat_vision::enhance::factor_of`.+ pub fn clip_enhance_size(&mut self, index: i32) {+ self.enhance_size = match index {+ 1 => 1,+ 2 => 2,+ 3 => 4,+ _ => 0,+ };+ }++ /// The Strength row, 0 to 1: how much of the model's answer the copy+ /// keeps. A preview on screen is re-blended at once, which is the+ /// point of keeping both its halves.+ pub fn clip_enhance_strength(&mut self, strength: f32) {+ self.enhance_strength = strength.clamp(0.0, 1.0);+ let strength = self.enhance_strength;+ if let Some(preview) = self.enhance_preview.as_mut() {+ preview.blend_at(strength);+ }+ }++ /// "Enhance again": a clip that has a copy, enhanced anew from its+ /// original with the section's settings as they are now. A copy those+ /// settings already made is picked up at once rather than redone.+ pub fn clip_enhance_run(&mut self) {+ let Some(id) = self.sole_selection() else {+ return;+ };+ if self+ .clip(&id)+ .is_some_and(|clip| !self.locked(&clip.track_id))+ {+ self.enhance_clip(&id);+ }+ }++ /// The file Enhance reads for `clip`: its original when it is showing,+ /// or holding, a copy already - a second run restores the source, not+ /// the first run's answer.+ fn enhance_source(&self, clip: &Clip) -> Option<model::MediaItem> {+ let id = clip+ .enhanced+ .as_ref()+ .map_or(&clip.media_id, |link| &link.original);+ self.project()+ .media+ .iter()+ .find(|item| &item.id == id)+ .cloned()+ }++ /// How many times larger `media` comes out at the section's Size.+ fn enhance_factor(&self, media: &model::MediaItem) -> u32 {+ concat_vision::enhance::factor_of(+ self.enhance_size,+ media.width.unwrap_or(0),+ media.height.unwrap_or(0),+ )+ }++ /// "Preview this frame": the middle of the frame of the selected clip+ /// under the playhead, through the model at the section's Size, shown+ /// before and after in the inspector. Seconds rather than minutes - a+ /// few tiles, not a file - and nothing is written or re-pointed, so the+ /// clip is as it was whatever comes of it.+ pub fn clip_enhance_preview(&mut self) {+ let Some(id) = self.sole_selection() else {+ return;+ };+ if self.enhance_previewing.is_some()+ || !self.enhance_jobs.is_empty()+ || self.host.enhancers.is_busy()+ {+ self.notify(&t("studio.enhanceAlreadyWorkOne"), true);+ return;+ }+ let Some(clip) = self.clip(&id).cloned() else {+ return;+ };+ if clip.kind != model::ClipKind::Video && clip.kind != model::ClipKind::Image {+ return;+ }+ let Some(media) = self.enhance_source(&clip) else {+ return;+ };+ if media.width.unwrap_or(0) == 0 || media.height.unwrap_or(0) == 0 {+ self.notify(&t("studio.fileHasNoPicture"), true);+ return;+ }+ let factor = self.enhance_factor(&media);+ let seconds = if clip.kind == model::ClipKind::Image {+ 0.0+ } else {+ self.source_at_playhead(&clip)+ };+ let request = concat_host::enhance::PreviewRequest {+ media_path: media.path.clone(),+ seconds,+ factor,+ };+ self.enhance_previewing = Some((id.clone(), false, 0.0));+ let enhancers = Arc::clone(&self.host.enhancers);+ let epoch = crate::host::project_epoch();+ let media_id = media.id.clone();+ spawn_in_project(+ move || {+ let mut last = -1.0f32;+ let result = enhancers.preview(&request, &mut |progress| {+ // The model's download is the one slow part a preview+ // can have; the tiles themselves take seconds.+ if let concat_host::enhance::Progress::Fetching { received, total } = progress {+ let fraction = received as f32 / total.max(1) as f32;+ if fraction - last >= 0.01 {+ last = fraction;+ on_ui_in_project(epoch, move |studio, _, _| {+ if let Some(held) = studio.enhance_previewing.as_mut() {+ held.1 = true;+ held.2 = fraction;+ }+ });+ }+ }+ });+ (id, media_id, factor, result)+ },+ |studio, _, _, (clip_id, media_id, factor, result)| {+ studio.enhance_previewing = None;+ match result {+ Ok(preview) => {+ let mut made = EnhancePreview {+ clip_id,+ media_id,+ factor,+ before: image_of(&preview.plain),+ after: image_of(&preview.enhanced),+ after_percent: 100,+ plain: preview.plain,+ enhanced: preview.enhanced,+ tile_seconds: preview.tile_seconds,+ tiles_per_frame: preview.tiles_per_frame,+ };+ made.blend_at(studio.enhance_strength);+ studio.enhance_preview = Some(made);+ }+ Err(error) if error.contains("cancelled") => {}+ Err(error) => studio.notify(&tf("studio.enhance", &[&error]), true),+ }+ },+ );+ }++ /// The preview on screen for `clip` read from `media` at `factor`, if+ /// the last one made is that.+ fn enhance_preview_for(+ &self,+ clip: &Clip,+ media: &model::MediaItem,+ factor: u32,+ ) -> Option<&EnhancePreview> {+ self.enhance_preview.as_ref().filter(|preview| {+ preview.clip_id == clip.id && preview.media_id == media.id && preview.factor == factor+ })+ }+pub fn enhance_clip(&mut self, id: &str) {- if !self.enhance_jobs.is_empty() || self.host.enhancers.is_busy() {+ if !self.enhance_jobs.is_empty()+ || self.enhance_previewing.is_some()+ || self.host.enhancers.is_busy()+ {self.notify(&t("studio.enhanceAlreadyWorkOne"), true);return;}@@ -6092,13 +6332,7 @@ impl Studio {if clip.kind != model::ClipKind::Video && clip.kind != model::ClipKind::Image {return;}- let Some(media) = self- .project()- .media- .iter()- .find(|item| item.id == clip.media_id)- .cloned()- else {+ let Some(media) = self.enhance_source(&clip) else {return;};let Some(session) = self.session.as_ref() else {@@ -6110,9 +6344,11 @@ impl Studio {self.notify(&t("studio.fileHasNoPicture"), true);return;}- let factor = concat_vision::enhance::factor_for(width, height);+ let factor = self.enhance_factor(&media);+ let strength = self.enhance_strength;let still = clip.kind == model::ClipKind::Image;- let Some(target) = concat_host::enhance::target_for(&project, &media.path, factor, still)+ let Some(target) =+ concat_host::enhance::target_for(&project, &media.path, factor, strength, still)else {self.notify(&tf("studio.couldNotRead", &[&media.path]), true);return;@@ -6121,8 +6357,10 @@ impl Studio {media_path: media.path.clone(),still,factor,+ strength,target,};+ let percent = concat_host::enhance::strength_percent(strength);let clip_id = clip.id.clone();self.enhance_jobs.insert(clip_id.clone(), (false, 0.0));self.notify(@@ -6154,10 +6392,13 @@ impl Studio {});(clip_id, result)},- |studio, _, _, (clip_id, result)| {+ move |studio, _, _, (clip_id, result)| {studio.enhance_jobs.remove(&clip_id);+ // A cancelled or failed run leaves the clip as it was: the+ // job writes under partial names and the clip is only+ // re-pointed here, on a copy that is whole.match result {- Ok(path) => studio.adopt_enhanced(&clip_id, &path),+ Ok(path) => studio.adopt_enhanced(&clip_id, &path, factor, percent),Err(error) if error.contains("cancelled") => {}Err(error) => studio.notify(&tf("studio.enhance", &[&error]), true),}@@ -6286,7 +6527,7 @@ impl Studio {/// Points clip `id` at the enhanced copy at `path`, probed the way an/// import is, and shows it.- fn adopt_enhanced(&mut self, id: &str, path: &std::path::Path) {+ fn adopt_enhanced(&mut self, id: &str, path: &std::path::Path, factor: u32, percent: u32) {let summary = match media::probe(&path.to_string_lossy()) {Ok(summary) => summary,Err(error) => {@@ -6295,13 +6536,11 @@ impl Studio {}};let mut item = summary.to_new_media();- // The bin names the copy after the original, marked, rather than- // after its cache file.- if let Some(original) = self- .clip(id)- .and_then(|clip| self.project().media.iter().find(|m| m.id == clip.media_id))- {- item.name = format!("{} (enhanced)", original.name);+ // The bin names the copy after the original, marked with what it+ // was made with - one file can have a copy for each setting -+ // rather than after its cache file.+ if let Some(original) = self.clip(id).and_then(|clip| self.enhance_source(clip)) {+ item.name = format!("{} (enhanced {factor}x, {percent}%)", original.name);}self.apply(Command::ReplaceClipMedia {clip_id: id.to_owned(),@@ -7302,6 +7541,8 @@ impl Studio {self.host.enhancers.cancel();self.host.reversers.cancel();self.enhance_jobs.clear();+ self.enhance_preview = None;+ self.enhance_previewing = None;self.reverse_jobs.clear();self.cancel_merge();self.cutout_jobs.clear();@@ -9044,6 +9285,10 @@ impl Studio {.get(&Self::analysis_key(&clip.media_id, cutout.subject)).copied()});+ let mut enhance = self.enhance_readout(clip);+ let enhance_preview = enhance.preview.is_some();+ let (enhance_before, enhance_after, enhance_estimate) =+ enhance.preview.take().unwrap_or_default();SelectedClipData {present: true,batch: targets.len() as i32,@@ -9166,9 +9411,99 @@ impl Studio {enhance_linked: clip.enhanced.is_some(),enhance_on: clip.enhanced.as_ref().is_some_and(|link| link.on)|| self.enhance_jobs.contains_key(&clip.id),+ enhance_busy: !self.enhance_jobs.is_empty() || self.enhance_previewing.is_some(),+ enhance_size: enhance.size,+ enhance_auto: enhance.auto as i32,+ enhance_fits_2x: enhance.fits_2x,+ enhance_fits_4x: enhance.fits_4x,+ enhance_size_note: enhance.size_note.into(),+ enhance_strength: self.enhance_strength,+ enhance_previewing: self+ .enhance_previewing+ .as_ref()+ .is_some_and(|(id, _, _)| *id == clip.id),+ enhance_preview_fetch: self+ .enhance_previewing+ .as_ref()+ .filter(|(id, fetching, _)| *id == clip.id && *fetching)+ .map(|(_, _, fraction)| *fraction)+ .unwrap_or(-1.0),+ enhance_preview,+ enhance_before,+ enhance_after,+ enhance_estimate: enhance_estimate.into(),}}+ /// What the Enhance section shows for `clip` beyond its switch: the+ /// sizes its file can take, the preview made for it at the section's+ /// settings, if there is one, and what a whole run would cost.+ fn enhance_readout(&self, clip: &Clip) -> EnhanceReadout {+ let mut readout = EnhanceReadout::default();+ let Some(media) = self.enhance_source(clip) else {+ return readout;+ };+ let (width, height) = (media.width.unwrap_or(0), media.height.unwrap_or(0));+ let limit = concat_vision::enhance::LONGEST_SIDE;+ let factor = self.enhance_factor(&media);+ readout.auto = concat_vision::enhance::factor_for(width, height);+ readout.fits_2x = concat_vision::enhance::fits(width, height, 2);+ readout.fits_4x = concat_vision::enhance::fits(width, height, 4);+ // The row shows what will happen: Auto as Auto, and an ask this+ // file cannot take as the size it falls to.+ readout.size = match (self.enhance_size, factor) {+ (0, _) => 0,+ (_, 1) => 1,+ (_, 2) => 2,+ _ => 3,+ };+ readout.size_note = if !readout.fits_2x {+ tf("studio.enhanceOnlyRestore", &[&width.max(height), &limit])+ } else if !readout.fits_4x {+ tf(+ "studio.enhanceNoFourTimes",+ &[&(width * 4), &(height * 4), &limit],+ )+ } else {+ String::new()+ };+ readout.preview = self+ .enhance_preview_for(clip, &media, factor)+ .map(|preview| {+ // The model's work on the whole file - every frame of it,+ // not only the span the clip shows - at the warm tile time+ // the preview measured.+ let frames = if clip.kind == model::ClipKind::Image {+ 1.0+ } else {+ media.duration.unwrap_or(0.0) * media.frame_rate.unwrap_or(30.0)+ };+ let per_frame = preview.tile_seconds * preview.tiles_per_frame as f64;+ let total = concat_host::enhance::estimate_seconds(+ preview.tile_seconds,+ preview.tiles_per_frame,+ frames,+ );+ let estimate = if clip.kind == model::ClipKind::Image {+ tf(+ "studio.enhanceEstimateStill",+ &[&crate::format::span(total)],+ )+ } else {+ tf(+ "studio.enhanceEstimate",+ &[+ &crate::format::span(total),+ &format!("{:.0}", frames.max(1.0)),+ &format!("{per_frame:.1}"),+ ],+ )+ };+ (preview.before.clone(), preview.after.clone(), estimate)+ });+ readout+ }+/// The source instant of `clip` under the playhead, held to the clip.fn source_at_playhead(&self, clip: &Clip) -> f64 {let along = (f64::from(self.playhead) - clip.start).clamp(0.0, clip.duration);diff --git a/src/crates/concat/ui/editor.slint b/src/crates/concat/ui/editor.slintindex 2f51291c..79d34d31 100644--- a/src/crates/concat/ui/editor.slint+++ b/src/crates/concat/ui/editor.slint@@ -277,6 +277,12 @@ export global Editor {callback cutout-subject(index: int);callback cutout-clear();callback clip-enhance-set(on: bool);+ /// The Enhance section's settings and its preview; see EnhancePanel.+ /// `index` is 0 Auto, then 1x, 2x and 4x.+ callback clip-enhance-size(index: int);+ callback clip-enhance-strength(strength: float);+ callback clip-enhance-preview();+ callback clip-enhance-run();/// The transition riding into the selected clip; see VideoPanel.callback transition-remove();callback transition-duration-set(seconds: float);diff --git a/src/crates/concat/ui/inspector/clip-inspector.slint b/src/crates/concat/ui/inspector/clip-inspector.slintindex c8320129..e55206f3 100644--- a/src/crates/concat/ui/inspector/clip-inspector.slint+++ b/src/crates/concat/ui/inspector/clip-inspector.slint@@ -1465,14 +1465,71 @@ export component VideoPanel inherits VerticalLayout {}}+/// One half of Enhance's before and after: a label over the picture,+/// fitted whole into a fixed height so the pair reads the same on a phone+/// as in the desktop inspector, and nothing in the layout leans on the+/// picture's own size.+component PreviewHalf inherits VerticalLayout {+ in property <string> label;+ in property <image> source;++ padding-left: Theme.pad;+ padding-right: Theme.pad;+ spacing: 4px;++ Text {+ text: root.label;+ color: Theme.fg-muted;+ font-size: Theme.fs-sm;+ }+ Rectangle {+ height: 150px;+ background: Theme.well;+ border-radius: Theme.r-sm;+ clip: true;++ Image {+ width: parent.width;+ height: parent.height;+ source: root.source;+ image-fit: contain;+ }+ }+}+/// Enhance: the picture restored and enlarged by the model, into a copy the/// clip then shows. The header's tick starts it the first time; once there/// is a copy, it switches between the copy and the original without running/// the model again.+///+/// Below the tick, what the run is made with - Size and Strength - and a+/// preview of the frame under the playhead at those settings, so the+/// person judges the look, and what a whole run will cost, in seconds+/// rather than after minutes of their machine. Strength is the answer to+/// the waxen, "plastic" faces a full-strength enhancer is known for: it+/// blends the model's answer back towards the original.export component EnhancePanel inherits VerticalLayout {in property <SelectedClipData> selected;callback enhance-set(on: bool);+ /// 0 Auto, 1 once, 2 twice, 3 four times.+ callback enhance-size(index: int);+ callback enhance-strength(strength: float);+ callback enhance-preview();+ /// A clip with a copy enhanced anew with the settings as they are.+ callback enhance-run();++ property <float> strength;++ init => {+ root.strength = root.selected.enhance-strength;+ }+ // Put back on every publish: a click on the row takes its binding, and+ // the next clip's file may not take the size this one showed.+ changed selected => {+ root.strength = root.selected.enhance-strength;+ sizes.current = root.selected.enhance-size;+ }Section {first: true;@@ -1495,6 +1552,77 @@ export component EnhancePanel inherits VerticalLayout {: I18n.t("clipInspector.enhanceShowingOriginal")): I18n.t("clipInspector.enhanceExplain");}++ Row {+ label: I18n.t("clipInspector.enhanceSize");+ sizes := SegmentedControl {+ options: [+ I18n.t1("clipInspector.enhanceAuto", "\{root.selected.enhance-auto}x"),+ "1x",+ "2x",+ "4x",+ ];+ disabled: [+ false,+ false,+ !root.selected.enhance-fits-2x,+ !root.selected.enhance-fits-4x,+ ];+ current: root.selected.enhance-size;+ changed(index) => { root.enhance-size(index); }+ }+ }+ if root.selected.enhance-size-note != "": Note {+ text: root.selected.enhance-size-note;+ }++ Param {+ label: I18n.t("clipInspector.enhanceStrength");+ value <=> root.strength;+ minimum: 0;+ maximum: 1;+ step: 0.01;+ default-value: 0.75;+ fmt: ParamFormat.fraction;+ unit: "%";+ unit-scale: 100;+ changed(v) => { root.enhance-strength(v); }+ }+ Note {+ text: I18n.t("clipInspector.enhanceStrengthExplain");+ }++ Row {+ Button {+ label: I18n.t("clipInspector.enhancePreview");+ enabled: !root.selected.enhance-busy;+ clicked => { root.enhance-preview(); }+ }+ if root.selected.enhance-linked: Button {+ label: I18n.t("clipInspector.enhanceAgain");+ enabled: !root.selected.enhance-busy;+ clicked => { root.enhance-run(); }+ }+ Rectangle { horizontal-stretch: 1; }+ }+ if root.selected.enhance-previewing: Note {+ text: root.selected.enhance-preview-fetch >= 0+ ? I18n.t1("clipInspector.fetchingTheModel",+ "\{Math.round(root.selected.enhance-preview-fetch * 100)}")+ : I18n.t("clipInspector.enhancePreviewing");+ }++ if root.selected.enhance-preview: PreviewHalf {+ label: I18n.t("clipInspector.enhanceBefore");+ source: root.selected.enhance-before;+ }+ if root.selected.enhance-preview: PreviewHalf {+ label: I18n.t1("clipInspector.enhanceAfter", "\{Math.round(root.strength * 100)}");+ source: root.selected.enhance-after;+ }+ if root.selected.enhance-preview: Note {+ text: root.selected.enhance-estimate;+ }}}}diff --git a/src/crates/concat/ui/phone/editor.slint b/src/crates/concat/ui/phone/editor.slintindex 1a8d69ea..992a3da4 100644--- a/src/crates/concat/ui/phone/editor.slint+++ b/src/crates/concat/ui/phone/editor.slint@@ -14,8 +14,8 @@ import {CategoryPillBar, MediaCard, MediaItemData, PresetGrid, ShelfGrid, ShelfSearch,} from "../workspace/media-pane.slint";import {- AdjustPanel, AudioPanel, FilterPanel, Library, PresetStack, ProcessorRows, SpeedPanel,- VideoPanel,+ AdjustPanel, AudioPanel, EnhancePanel, FilterPanel, Library, PresetStack, ProcessorRows,+ SpeedPanel, VideoPanel,} from "../inspector/clip-inspector.slint";import { TextPanel } from "../inspector/text-panel.slint";import { RoundButton, ToolItem, ToolStrip } from "controls.slint";@@ -252,6 +252,7 @@ export component PhoneEditor inherits Rectangle {: name == "clip-speed" ? I18n.t("common.speed"): name == "clip-audio" ? I18n.t("common.audio"): name == "clip-adjust" ? I18n.t("common.adjust")+ : name == "clip-enhance" ? I18n.t("common.enhance"): name == "clip-layer" ? I18n.t("configPane.filter"): "";}@@ -761,6 +762,14 @@ export component PhoneEditor inherits Rectangle {current: root.sheet == "clip-adjust";clicked => { root.show("clip-adjust", true); }}+ // The desk's Video > Enhance, on a sheet of its own: the same+ // panel, settings, preview and all.+ if root.is-picture: ToolItem {+ glyph: Glyph.maximize;+ label: I18n.t("common.enhance");+ current: root.sheet == "clip-enhance";+ clicked => { root.show("clip-enhance", true); }+ }if root.is-picture || root.is-text: ToolItem {glyph: Glyph.sparkle;label: I18n.t("common.effects");@@ -1093,6 +1102,15 @@ export component PhoneEditor inherits Rectangle {commit => { Editor.clip-commit(); }}+ if root.sheet == "clip-enhance": EnhancePanel {+ selected: Editor.selected-clip;+ enhance-set(on) => { Editor.clip-enhance-set(on); }+ enhance-size(index) => { Editor.clip-enhance-size(index); }+ enhance-strength(strength) => { Editor.clip-enhance-strength(strength); }+ enhance-preview => { Editor.clip-enhance-preview(); }+ enhance-run => { Editor.clip-enhance-run(); }+ }+if root.sheet == "clip-text": TextPanel {selected: Editor.selected-clip;families: Editor.font-families;diff --git a/src/crates/concat/ui/primitives/segmented-control.slint b/src/crates/concat/ui/primitives/segmented-control.slintindex 0f8bf5fd..5cf16751 100644--- a/src/crates/concat/ui/primitives/segmented-control.slint+++ b/src/crates/concat/ui/primitives/segmented-control.slint@@ -18,6 +18,10 @@ export component SegmentedControl inherits Rectangle {in property <[Glyph]> icons;in-out property <int> current;in property <bool> enabled: true;+ /// Segments that cannot be picked, by index: greyed, and refused+ /// by a click or the arrow keys. Shorter than `options` is fine; a+ /// segment past the end can be picked.+ in property <[bool]> disabled;callback changed(index: int);@@ -39,7 +43,7 @@ export component SegmentedControl inherits Rectangle {opacity: root.enabled ? 1.0 : 0.45;function select(index: int) {- if (index >= 0 && index < root.options.length) {+ if (index >= 0 && index < root.options.length && !root.disabled[index]) {root.current = index;root.changed(index);}@@ -93,7 +97,8 @@ export component SegmentedControl inherits Rectangle {width: parent.width;height: parent.height;text: option;- color: (index == root.current || segment.has-hover) ? Theme.fg : Theme.fg-muted;+ color: root.disabled[index] ? Theme.fg-dim+ : (index == root.current || segment.has-hover) ? Theme.fg : Theme.fg-muted;font-size: Theme.fs;horizontal-alignment: center;vertical-alignment: center;@@ -110,7 +115,7 @@ export component SegmentedControl inherits Rectangle {}segment := TouchArea {- enabled: root.enabled;+ enabled: root.enabled && !root.disabled[index];mouse-cursor: pointer;clicked => {fs.focus();diff --git a/src/crates/concat/ui/timeline/model.slint b/src/crates/concat/ui/timeline/model.slintindex f7f95c02..a2c8f9c9 100644--- a/src/crates/concat/ui/timeline/model.slint+++ b/src/crates/concat/ui/timeline/model.slint@@ -349,6 +349,32 @@ export struct SelectedClipData {enhance-linked: bool,/// The Enhance switch: the copy shown, or a run under wayenhance-on: bool,+ /// Enhance at work on some clip, a run or a preview: the model is one+ /// model, so the section's buttons wait for it+ enhance-busy: bool,+ /// The Size row: 0 Auto, 1 once, 2 twice, 3 four times - what will+ /// happen, so an ask this file cannot take shows as what it falls to.+ /// `enhance-auto` is what Auto comes to here; a size that would pass+ /// the longest-side limit is greyed, `enhance-size-note` saying why+ enhance-size: int,+ enhance-auto: int,+ enhance-fits-2x: bool,+ enhance-fits-4x: bool,+ enhance-size-note: string,+ /// The Strength row, 0 to 1: how much of the model's answer is kept+ enhance-strength: float,+ /// A preview of the frame under the playhead being made for this clip,+ /// and how far the model's download is when that is what it waits on+ /// (-1 when it is not)+ enhance-previewing: bool,+ enhance-preview-fetch: float,+ /// The preview made for this clip at these settings: the middle of+ /// the frame enlarged plainly and enhanced, and what a whole run would+ /// take at the speed it measured+ enhance-preview: bool,+ enhance-before: image,+ enhance-after: image,+ enhance-estimate: string,/// Reverse writing this clip's backwards copy: -1 when it is not,/// else how far alongreverse-progress: float,diff --git a/src/crates/concat/ui/workspace/config-pane.slint b/src/crates/concat/ui/workspace/config-pane.slintindex 68123885..d9c3eb0f 100644--- a/src/crates/concat/ui/workspace/config-pane.slint+++ b/src/crates/concat/ui/workspace/config-pane.slint@@ -205,6 +205,10 @@ export component ConfigPane inherits Rectangle {callback cutout-clear();/// The Enhance switch: start it, or show the copy or the original.callback clip-enhance-set(on: bool);+ callback clip-enhance-size(index: int);+ callback clip-enhance-strength(strength: float);+ callback clip-enhance-preview();+ callback clip-enhance-run();callback transition-remove();callback transition-duration-set(seconds: float);@@ -647,6 +651,10 @@ export component ConfigPane inherits Rectangle {if root.showing == "Enhance": EnhancePanel {selected: root.selected;enhance-set(on) => { root.clip-enhance-set(on); }+ enhance-size(index) => { root.clip-enhance-size(index); }+ enhance-strength(strength) => { root.clip-enhance-strength(strength); }+ enhance-preview => { root.clip-enhance-preview(); }+ enhance-run => { root.clip-enhance-run(); }}if root.showing == "Adjust": AdjustPanel {diff --git a/src/crates/concat/ui/workspace/seat.slint b/src/crates/concat/ui/workspace/seat.slintindex ac8bddb4..19511aa0 100644--- a/src/crates/concat/ui/workspace/seat.slint+++ b/src/crates/concat/ui/workspace/seat.slint@@ -271,6 +271,10 @@ export component Seat inherits Rectangle {cutout-subject(index) => { Editor.cutout-subject(index); }cutout-clear => { Editor.cutout-clear(); }clip-enhance-set(on) => { Editor.clip-enhance-set(on); }+ clip-enhance-size(index) => { Editor.clip-enhance-size(index); }+ clip-enhance-strength(strength) => { Editor.clip-enhance-strength(strength); }+ clip-enhance-preview => { Editor.clip-enhance-preview(); }+ clip-enhance-run => { Editor.clip-enhance-run(); }transition-remove => { Editor.transition-remove(); }transition-duration-set(seconds) => { Editor.transition-duration-set(seconds); }