Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(enhance): Size, Strength and a one-frame preview before the whole run

Jareer03c8fd66706f
diff --git a/src/crates/concat-host/src/enhance.rs b/src/crates/concat-host/src/enhance.rs
index d8330fe3..369bd94d 100644
--- a/src/crates/concat-host/src/enhance.rs
+++ b/src/crates/concat-host/src/enhance.rs
@@ -14,24 +14,50 @@
//!
//! One job at a time through a [`SingleFlight`], like every long job the
//! host runs, with the model fetched on first use and kept loaded. The
-//! copy is named after the file's path, size and modification time and
-//! the factor it was enlarged by, so a file replaced on disk gets a fresh
-//! one and the same file asked for twice gets the one already there.
+//! copy is named after the file's path, size and modification time, the
+//! factor it was enlarged by and the strength it was blended at, so a file
+//! replaced on disk gets a fresh one, the same file asked for twice gets
+//! the one already there, and asking again with other settings writes a
+//! copy of its own beside the first rather than over it.
+//!
+//! Before a person spends minutes of their machine on a whole file,
+//! [`Enhancers::preview`] runs the model on the middle of one frame - a
+//! few tiles, seconds - and says how long a tile took, which is what the
+//! window's estimate for the whole run is made of.
+//!
+//! Enhance only ever runs because the person asked, so a crash inside it
+//! does not come back on the next open and needs no tripwire; see
+//! [`crate::jobs::Tripwire`].
use std::hash::{Hash, Hasher};
use std::path::{Path, PathBuf};
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Mutex};
+use concat_core::Frame;
+use concat_core::time::Rational;
use concat_media::{
- DecodeOptions, Decoder, EncodeOptions, Encoder, FrameSink, FrameSource, RateMode, VideoCodec,
+ DecodeOptions, Decoder, EncodeOptions, Encoder, FrameSink, FrameSource, RateMode,
+ SeekableSource, VideoCodec,
};
-use concat_vision::enhance::Enhancer;
+use concat_vision::enhance::{self as vision, Enhancer};
use concat_vision::{ModelId, models};
pub use crate::cutout::Progress;
use crate::jobs::SingleFlight;
+/// The most a preview shows, in output pixels: about what an inspector a
+/// few hundred points wide, on a screen of two pixels a point, shows one
+/// to one. Its source is a handful of tiles at most.
+pub const PREVIEW_WIDTH: u32 = 640;
+/// See [`PREVIEW_WIDTH`]: the same at sixteen by nine.
+pub const PREVIEW_HEIGHT: u32 = 360;
+
+/// The strength the inspector starts at: most of the model's restoration,
+/// with a quarter of the original's texture left in so faces do not come
+/// out waxen - the "plastic" look a full-strength enhancer is known for.
+pub const DEFAULT_STRENGTH: f32 = 0.75;
+
/// What one enhancement covers.
#[derive(Clone, Debug)]
pub struct EnhanceRequest {
@@ -40,12 +66,71 @@ pub struct EnhanceRequest {
/// A still: one frame in, one PNG out.
pub still: bool,
/// How many times larger the copy is; see
- /// [`concat_vision::enhance::factor_for`].
+ /// [`concat_vision::enhance::factor_of`].
pub factor: u32,
+ /// How much of the model's answer is kept, 0 to 1; the rest is the
+ /// original, enlarged plainly. See [`concat_vision::enhance::mix`].
+ pub strength: f32,
/// Where the copy goes; see [`target_for`].
pub target: PathBuf,
}
+/// What a preview covers: one frame of a file, at the size it would be
+/// enhanced to.
+#[derive(Clone, Debug)]
+pub struct PreviewRequest {
+ /// The media file to read.
+ pub media_path: String,
+ /// Seconds into the file: the frame under the playhead. 0 for a still.
+ pub seconds: f64,
+ /// How many times larger, as for [`EnhanceRequest::factor`].
+ pub factor: u32,
+}
+
+/// One frame's middle, before and after: both at the enhanced size, so
+/// the window can blend them at any strength without the model.
+#[derive(Clone, Debug)]
+pub struct Preview {
+ /// The middle of the frame enlarged by an ordinary bicubic filter -
+ /// what the clip would be without Enhance, and what Strength blends
+ /// towards.
+ pub plain: Frame,
+ /// The same middle as the model answers it, at full strength.
+ pub enhanced: Frame,
+ /// Seconds the model took over one tile, warm; see
+ /// [`steady_tile_seconds`].
+ pub tile_seconds: f64,
+ /// Tiles in one whole frame of the file.
+ pub tiles_per_frame: usize,
+}
+
+/// `strength` as the whole percent it is keyed by: what the inspector
+/// shows, and what tells two copies apart, so float noise in a slider
+/// never makes a copy of its own.
+pub fn strength_percent(strength: f32) -> u32 {
+ (strength.clamp(0.0, 1.0) * 100.0).round() as u32
+}
+
+/// How long the model takes over one tile once it is warm: the fastest of
+/// `times`. The first run of a model in a session compiles it and can take
+/// many times longer than the rest, and a preview is often that first run;
+/// every tile is one shape, so the rest take as long as each other.
+pub fn steady_tile_seconds(times: &[std::time::Duration]) -> f64 {
+ times
+ .iter()
+ .min()
+ .map(std::time::Duration::as_secs_f64)
+ .unwrap_or(0.0)
+}
+
+/// About how long enhancing `frames` frames of `tiles_per_frame` tiles
+/// each takes at `tile_seconds` a tile, in seconds: the model's share,
+/// which is nearly all of it - decoding and encoding are a rounding error
+/// beside it.
+pub fn estimate_seconds(tile_seconds: f64, tiles_per_frame: usize, frames: f64) -> f64 {
+ tile_seconds * tiles_per_frame as f64 * frames.max(1.0)
+}
+
/// The enhancement service: the model, loaded once, and the one-job slot.
pub struct Enhancers {
gate: Arc<SingleFlight>,
@@ -54,10 +139,19 @@ pub struct Enhancers {
model: Mutex<Option<Arc<Enhancer>>>,
}
-/// Where the enhanced copy of `media` at `factor` lives under `project`:
-/// in the cache, named by the file's path, size and modification time.
-/// None for a file that cannot be stat'ed, which cannot be read either.
-pub fn target_for(project: &Path, media: &str, factor: u32, still: bool) -> Option<PathBuf> {
+/// Where the enhanced copy of `media` at `factor` and `strength` lives
+/// under `project`: in the cache, named by the file's path, size and
+/// modification time and by the settings. Full strength - all Enhance
+/// made before it had a strength - keeps the old name, so a copy made
+/// then is found again rather than made twice. None for a file that
+/// cannot be stat'ed, which cannot be read either.
+pub fn target_for(
+ project: &Path,
+ media: &str,
+ factor: u32,
+ strength: f32,
+ still: bool,
+) -> Option<PathBuf> {
let meta = std::fs::metadata(media).ok()?;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
media.hash(&mut hasher);
@@ -68,12 +162,38 @@ pub fn target_for(project: &Path, media: &str, factor: u32, still: bool) -> Opti
since.as_secs().hash(&mut hasher);
}
let extension = if still { "png" } else { "mp4" };
+ let percent = strength_percent(strength);
+ let strength = if percent == 100 {
+ String::new()
+ } else {
+ format!("-{percent}pc")
+ };
Some(project.join("cache").join(format!(
- "enhance-{:016x}-{factor}x.{extension}",
+ "enhance-{:016x}-{factor}x{strength}.{extension}",
hasher.finish()
)))
}
+/// The frame of `path` at `seconds`, at its own size, or the first frame
+/// when `seconds` is past the end.
+fn frame_at(path: &str, seconds: f64) -> Result<Frame, String> {
+ let mut options = DecodeOptions::default().in_software();
+ if seconds > 0.0 {
+ options = options.starting_at(Rational::approximate(seconds).unwrap_or(Rational::ZERO));
+ }
+ let mut decoder = Decoder::open(path, &options).map_err(|error| error.to_string())?;
+ if let Some(frame) = decoder.next_frame().map_err(|error| error.to_string())? {
+ return Ok(frame);
+ }
+ decoder
+ .seek(Rational::ZERO)
+ .map_err(|error| error.to_string())?;
+ decoder
+ .next_frame()
+ .map_err(|error| error.to_string())?
+ .ok_or_else(|| format!("{path} holds no picture"))
+}
+
impl Enhancers {
/// A service with nothing loaded yet, keeping its model under `data`.
pub fn new(data: &Path) -> Enhancers {
@@ -148,6 +268,41 @@ impl Enhancers {
};
result.map(|()| request.target.clone())
}
+
+ /// The middle of one frame through the model, at most
+ /// [`PREVIEW_WIDTH`] by [`PREVIEW_HEIGHT`] once enlarged, beside the
+ /// same middle enlarged plainly. Takes the one job slot like a whole
+ /// run - the model is one model - and blocks until done; `progress`
+ /// reports the model's download, the one slow part a preview can
+ /// have. Nothing is written: the clip is not touched.
+ pub fn preview(
+ &self,
+ request: &PreviewRequest,
+ progress: &mut dyn FnMut(Progress),
+ ) -> Result<Preview, String> {
+ let job = self.gate.begin("enhance")?;
+ let cancel = job.cancel_handle();
+ let enhancer = self.loaded(&cancel, progress)?;
+ if job.cancelled() {
+ return Err("enhance cancelled".to_owned());
+ }
+ let frame = frame_at(&request.media_path, request.seconds)?;
+ let (width, height) = (frame.width(), frame.height());
+ let region =
+ vision::centre_region(width, height, request.factor, PREVIEW_WIDTH, PREVIEW_HEIGHT);
+ let (enhanced, times) = enhancer.enhance_region(&frame, region, request.factor)?;
+ let plain = vision::resize(
+ &vision::cut(&frame, region),
+ enhanced.width(),
+ enhanced.height(),
+ );
+ Ok(Preview {
+ plain,
+ enhanced,
+ tile_seconds: steady_tile_seconds(&times),
+ tiles_per_frame: vision::tiles(width, height).len(),
+ })
+ }
}
/// One frame through the model, as a PNG: lossless, since the point was
@@ -159,7 +314,11 @@ fn still(enhancer: &Enhancer, request: &EnhanceRequest) -> Result<(), String> {
.next_frame()
.map_err(|error| error.to_string())?
.ok_or_else(|| format!("{} holds no picture", request.media_path))?;
- let enhanced = enhancer.enhance(&frame, request.factor)?;
+ let enhanced = vision::mix(
+ enhancer.enhance(&frame, request.factor)?,
+ &frame,
+ request.strength,
+ );
let partial = request.target.with_extension("part.png");
let written = (|| {
let file = std::fs::File::create(&partial).map_err(|error| error.to_string())?;
@@ -248,7 +407,11 @@ fn footage(
if job.cancelled() {
return Err("enhance cancelled".to_owned());
}
- let enhanced = enhancer.enhance(&frame, request.factor)?;
+ let enhanced = vision::mix(
+ enhancer.enhance(&frame, request.factor)?,
+ &frame,
+ request.strength,
+ );
let enhanced = crop(&enhanced, width, height);
encoder
.write_frame(&enhanced)
@@ -319,23 +482,73 @@ mod tests {
std::fs::write(&media, b"not really").expect("write");
let media = media.to_string_lossy().into_owned();
let project = dir.join("project");
- let two = target_for(&project, &media, 2, false).expect("named");
+ // Full strength keeps the name Enhance gave copies before it had
+ // one, so those are found again.
+ let two = target_for(&project, &media, 2, 1.0, false).expect("named");
assert!(two.starts_with(project.join("cache")));
assert!(two.to_string_lossy().ends_with("-2x.mp4"), "{two:?}");
- assert_eq!(target_for(&project, &media, 2, false), Some(two.clone()));
- assert_ne!(target_for(&project, &media, 1, false), Some(two.clone()));
- let still = target_for(&project, &media, 2, true).expect("named");
+ assert_eq!(
+ target_for(&project, &media, 2, 1.0, false),
+ Some(two.clone())
+ );
+ assert_ne!(
+ target_for(&project, &media, 1, 1.0, false),
+ Some(two.clone())
+ );
+ let still = target_for(&project, &media, 2, 1.0, true).expect("named");
assert!(still.to_string_lossy().ends_with("-2x.png"), "{still:?}");
assert_eq!(
- target_for(&project, &dir.join("gone.mp4").to_string_lossy(), 2, false),
+ target_for(
+ &project,
+ &dir.join("gone.mp4").to_string_lossy(),
+ 2,
+ 1.0,
+ false
+ ),
None
);
+ // Another strength is another copy, keyed by its whole percent so
+ // a slider's float noise is not.
+ let blended = target_for(&project, &media, 2, 0.75, false).expect("named");
+ assert!(
+ blended.to_string_lossy().ends_with("-2x-75pc.mp4"),
+ "{blended:?}"
+ );
+ assert_ne!(blended, two);
+ assert_eq!(
+ target_for(&project, &media, 2, 0.750_001, false),
+ Some(blended.clone())
+ );
+ assert_ne!(target_for(&project, &media, 4, 0.75, false), Some(blended));
// A file with different bytes is a different copy.
std::fs::write(&media, b"not really, but longer").expect("write");
- assert_ne!(target_for(&project, &media, 2, false), Some(two));
+ assert_ne!(target_for(&project, &media, 2, 1.0, false), Some(two));
let _ = std::fs::remove_dir_all(&dir);
}
+ #[test]
+ fn the_estimate_is_the_warm_tile_time_over_every_tile_of_every_frame() {
+ use std::time::Duration;
+ // The first run compiled the model; the warm ones say what a tile
+ // costs.
+ let times = [
+ Duration::from_millis(2400),
+ Duration::from_millis(300),
+ Duration::from_millis(320),
+ ];
+ assert!((steady_tile_seconds(&times) - 0.3).abs() < 1e-9);
+ assert_eq!(steady_tile_seconds(&[]), 0.0);
+ // 1080p is 8 by 5 tiles; ten seconds at 30 is 300 frames.
+ let tiles = vision::tiles(1920, 1080).len();
+ assert_eq!(tiles, 40);
+ assert!((estimate_seconds(0.3, tiles, 300.0) - 3600.0).abs() < 1e-6);
+ // A still is one frame, however short its duration says it is.
+ assert!((estimate_seconds(0.3, tiles, 0.0) - 12.0).abs() < 1e-6);
+ assert_eq!(strength_percent(0.754), 75);
+ assert_eq!(strength_percent(1.2), 100);
+ assert_eq!(strength_percent(-1.0), 0);
+ }
+
#[test]
fn a_crop_takes_the_top_left_and_leaves_a_fitting_frame_alone() {
let mut frame = concat_core::Frame::black(4, 3);
diff --git a/src/crates/concat-vision/src/enhance.rs b/src/crates/concat-vision/src/enhance.rs
index 9d76a767..7b8aa275 100644
--- a/src/crates/concat-vision/src/enhance.rs
+++ b/src/crates/concat-vision/src/enhance.rs
@@ -17,7 +17,16 @@
//! plenty at 4K, and 8K is a file nobody can play - so [`factor_for`]
//! picks two, or one when the picture is already large, and the four-times
//! answer is averaged down to it. Averaging down from the restored picture
-//! keeps the restoration and drops only the size.
+//! keeps the restoration and drops only the size. The person can ask for a
+//! size of their own instead - see [`factor_of`] - but never one past
+//! [`LONGEST_SIDE`].
+//!
+//! The model's answer is also as strong as it is: skin smoothed to wax,
+//! grain scrubbed to plastic, which is the complaint every one-button
+//! enhancer earns. [`mix`] blends it back towards the original, enlarged
+//! by an ordinary bicubic filter to the same size, by a strength the
+//! person picks - most of the restoration, with some of the texture the
+//! model took away left in.
use concat_core::Frame;
use concat_core::frame::BYTES_PER_PIXEL;
@@ -74,11 +83,177 @@ pub fn tiles(width: u32, height: u32) -> Vec<Tile> {
/// unless that would pass [`LONGEST_SIDE`], and then once, restored at
/// its own size.
pub fn factor_for(width: u32, height: u32) -> u32 {
- if width.max(height) * 2 > LONGEST_SIDE {
- 1
- } else {
- 2
+ if fits(width, height, 2) { 2 } else { 1 }
+}
+
+/// The sizes a person can ask for, in times larger: restored at its own
+/// size, twice, and the model's own four times.
+pub const SIZES: [u32; 3] = [1, 2, 4];
+
+/// Whether a `width` by `height` picture enlarged `factor` times stays
+/// within [`LONGEST_SIDE`]. Once - restored at its own size - always does:
+/// a picture already past the limit is still restored, just not enlarged.
+pub fn fits(width: u32, height: u32, factor: u32) -> bool {
+ factor <= 1 || u64::from(width.max(height)) * u64::from(factor) <= u64::from(LONGEST_SIDE)
+}
+
+/// How many times larger a `width` by `height` picture comes out when the
+/// person asked for `asked`: 0 is Auto, [`factor_for`]; otherwise the
+/// largest of [`SIZES`] no larger than what was asked that [`fits`]. The
+/// ask is a setting that outlives the clip it was made on, so 4x picked
+/// on a small clip and carried to a 4K one comes out once, not at 16K.
+pub fn factor_of(asked: u32, width: u32, height: u32) -> u32 {
+ if asked == 0 {
+ return factor_for(width, height);
+ }
+ SIZES
+ .iter()
+ .copied()
+ .filter(|&factor| factor <= asked && fits(width, height, factor))
+ .max()
+ .unwrap_or(1)
+}
+
+/// The rectangle of a `width` by `height` picture, at its centre, that
+/// comes out no larger than `most_width` by `most_height` once enlarged
+/// `factor` times: what a preview restores, so that it shows the model's
+/// work at full size in seconds rather than the whole frame in minutes.
+/// The whole picture when it is smaller than that.
+pub fn centre_region(
+ width: u32,
+ height: u32,
+ factor: u32,
+ most_width: u32,
+ most_height: u32,
+) -> Tile {
+ let factor = factor.max(1);
+ let w = (most_width / factor).max(1).min(width);
+ let h = (most_height / factor).max(1).min(height);
+ Tile {
+ x: (width - w) / 2,
+ y: (height - h) / 2,
+ width: w,
+ height: h,
+ }
+}
+
+/// The pixels of `frame` inside `region`, which must lie inside it.
+pub fn cut(frame: &Frame, region: Tile) -> Frame {
+ let mut out = Frame::black(region.width, region.height);
+ let row = region.width as usize * BYTES_PER_PIXEL;
+ let stride = frame.width() as usize * BYTES_PER_PIXEL;
+ let source = frame.pixels();
+ let pixels = out.pixels_mut();
+ for y in 0..region.height as usize {
+ let at = (region.y as usize + y) * stride + region.x as usize * BYTES_PER_PIXEL;
+ pixels[y * row..(y + 1) * row].copy_from_slice(&source[at..at + row]);
+ }
+ out
+}
+
+/// The Catmull-Rom weights of the four source samples around a point `t`
+/// of the way from the second to the third: sharp enough that an
+/// enlargement is not mush, with less ringing than a longer filter, and
+/// summing to one so a flat picture stays flat.
+fn cubic_weights(t: f32) -> [f32; 4] {
+ let (t2, t3) = (t * t, t * t * t);
+ [
+ 0.5 * (-t3 + 2.0 * t2 - t),
+ 0.5 * (3.0 * t3 - 5.0 * t2 + 2.0),
+ 0.5 * (-3.0 * t3 + 4.0 * t2 + t),
+ 0.5 * (t3 - t2),
+ ]
+}
+
+/// For each of `to` output samples along one side, the four samples it
+/// reads of a side `from` long, the edge repeated past the end, and their
+/// weights. Pixel centres line up: output sample `i` sits at source
+/// position `(i + 0.5) * from / to - 0.5`.
+fn taps(from: u32, to: u32) -> Vec<([usize; 4], [f32; 4])> {
+ let ratio = from as f32 / to.max(1) as f32;
+ let last = i64::from(from.saturating_sub(1));
+ (0..to)
+ .map(|i| {
+ let at = (i as f32 + 0.5) * ratio - 0.5;
+ let base = at.floor();
+ let weights = cubic_weights(at - base);
+ let base = base as i64;
+ let index = |k: i64| (base + k).clamp(0, last) as usize;
+ ([index(-1), index(0), index(1), index(2)], weights)
+ })
+ .collect()
+}
+
+/// `frame` enlarged to `width` by `height` with a bicubic filter: the
+/// picture as an ordinary resize gives it, which is what Strength blends
+/// the model's answer towards. Meant for enlarging; shrinking with it
+/// skips pixels rather than averaging them.
+pub fn resize(frame: &Frame, width: u32, height: u32) -> Frame {
+ if frame.width() == width && frame.height() == height {
+ return frame.clone();
+ }
+ let (fw, fh) = (frame.width() as usize, frame.height() as usize);
+ if fw == 0 || fh == 0 || width == 0 || height == 0 {
+ return Frame::black(width, height);
+ }
+ let (ow, oh) = (width as usize, height as usize);
+ let source = frame.pixels();
+ // Across first, into floats a source row at a time, then down.
+ let across = taps(frame.width(), width);
+ let mut wide = vec![0f32; ow * fh * BYTES_PER_PIXEL];
+ for y in 0..fh {
+ let row = &source[y * fw * BYTES_PER_PIXEL..(y + 1) * fw * BYTES_PER_PIXEL];
+ for (x, (index, weight)) in across.iter().enumerate() {
+ let at = (y * ow + x) * BYTES_PER_PIXEL;
+ for c in 0..BYTES_PER_PIXEL {
+ wide[at + c] = (0..4)
+ .map(|k| weight[k] * f32::from(row[index[k] * BYTES_PER_PIXEL + c]))
+ .sum();
+ }
+ }
+ }
+ let down = taps(frame.height(), height);
+ let mut pixels = vec![0u8; ow * oh * BYTES_PER_PIXEL];
+ for (y, (index, weight)) in down.iter().enumerate() {
+ for x in 0..ow {
+ let at = (y * ow + x) * BYTES_PER_PIXEL;
+ for c in 0..BYTES_PER_PIXEL {
+ let value: f32 = (0..4)
+ .map(|k| weight[k] * wide[(index[k] * ow + x) * BYTES_PER_PIXEL + c])
+ .sum();
+ pixels[at + c] = (value + 0.5).clamp(0.0, 255.0) as u8;
+ }
+ }
+ }
+ Frame::from_rgba(width, height, pixels).expect("sides and bytes agree")
+}
+
+/// `strength` of the way from `plain` to `enhanced`, pixel by pixel: 0 is
+/// `plain`, 1 is `enhanced`. The two must be one size.
+pub fn blend(enhanced: &Frame, plain: &Frame, strength: f32) -> Frame {
+ debug_assert!(enhanced.same_size_as(plain));
+ let strength = strength.clamp(0.0, 1.0);
+ let pixels = enhanced
+ .pixels()
+ .iter()
+ .zip(plain.pixels())
+ .map(|(&e, &p)| {
+ let (e, p) = (f32::from(e), f32::from(p));
+ (p + (e - p) * strength + 0.5).clamp(0.0, 255.0) as u8
+ })
+ .collect();
+ Frame::from_rgba(enhanced.width(), enhanced.height(), pixels).expect("sides and bytes agree")
+}
+
+/// The model's answer `enhanced` for `original`, at `strength`: itself at
+/// full strength, and otherwise blended with `original` enlarged to its
+/// size by [`resize`].
+pub fn mix(enhanced: Frame, original: &Frame, strength: f32) -> Frame {
+ if strength >= 1.0 {
+ return enhanced;
}
+ let plain = resize(original, enhanced.width(), enhanced.height());
+ blend(&enhanced, &plain, strength)
}
/// The model's input for one tile: the [`WINDOW`] square around it as
@@ -189,15 +364,39 @@ impl Enhancer {
/// `frame` restored at [`SCALE`] times its size.
pub fn upscale(&self, frame: &Frame) -> Result<Frame, String> {
- let (width, height) = (frame.width(), frame.height());
- if width == 0 || height == 0 {
+ let whole = Tile {
+ x: 0,
+ y: 0,
+ width: frame.width(),
+ height: frame.height(),
+ };
+ Ok(self.upscale_region(frame, whole)?.0)
+ }
+
+ /// `region` of `frame` restored at [`SCALE`] times its size, and how
+ /// long each of its tiles took the model. Each tile still reads its
+ /// margin from the frame around the region, so the region comes out as
+ /// it would inside the whole frame.
+ pub fn upscale_region(
+ &self,
+ frame: &Frame,
+ region: Tile,
+ ) -> Result<(Frame, Vec<std::time::Duration>), String> {
+ if region.width == 0 || region.height == 0 {
return Err("enhance: an empty picture".to_owned());
}
- let mut out = Frame::black(width * SCALE, height * SCALE);
+ let mut out = Frame::black(region.width * SCALE, region.height * SCALE);
let side = WINDOW as usize;
let mut model = self.model.lock().map_err(|_| "enhance: model poisoned")?;
- for tile in tiles(width, height) {
- let planes = window_planes(frame, tile);
+ let mut times = Vec::new();
+ for tile in tiles(region.width, region.height) {
+ let placed = Tile {
+ x: region.x + tile.x,
+ y: region.y + tile.y,
+ ..tile
+ };
+ let planes = window_planes(frame, placed);
+ let started = std::time::Instant::now();
let answer = model.run(
vec![crate::runtime::Input {
name: "input",
@@ -206,9 +405,10 @@ impl Enhancer {
}],
&["output"],
)?;
+ times.push(started.elapsed());
paste(&mut out, tile, &answer[0].data)?;
}
- Ok(out)
+ Ok((out, times))
}
/// `frame` restored and enlarged `factor` times, `factor` being 1, 2
@@ -218,6 +418,19 @@ impl Enhancer {
let large = self.upscale(frame)?;
Ok(downscale(&large, SCALE / factor))
}
+
+ /// [`Enhancer::enhance`] for `region` of `frame` alone, with the time
+ /// each of its tiles took; see [`Enhancer::upscale_region`].
+ pub fn enhance_region(
+ &self,
+ frame: &Frame,
+ region: Tile,
+ factor: u32,
+ ) -> Result<(Frame, Vec<std::time::Duration>), String> {
+ let factor = factor.clamp(1, SCALE);
+ let (large, times) = self.upscale_region(frame, region)?;
+ Ok((downscale(&large, SCALE / factor), times))
+ }
}
#[cfg(test)]
@@ -376,4 +589,107 @@ mod tests {
let none = downscale(&Frame::black(3, 3), 4);
assert_eq!((none.width(), none.height()), (0, 0));
}
+
+ #[test]
+ fn an_asked_size_is_kept_under_the_limit_and_auto_is_the_old_choice() {
+ // Once always fits, even past the limit: that is restore-only.
+ assert!(fits(8000, 6000, 1));
+ assert!(fits(1024, 768, 4));
+ assert!(!fits(1025, 768, 4));
+ assert!(fits(2048, 1152, 2));
+ assert!(!fits(1920, 1080, 4), "1080p at 4x is 7680 across");
+ // Auto is what Enhance always did.
+ for (w, h) in [(1920, 1080), (3840, 2160), (640, 480), (2049, 10)] {
+ assert_eq!(factor_of(0, w, h), factor_for(w, h), "{w}x{h}");
+ }
+ // An ask that fits is kept.
+ assert_eq!(factor_of(1, 640, 480), 1);
+ assert_eq!(factor_of(2, 640, 480), 2);
+ assert_eq!(factor_of(4, 640, 480), 4);
+ // An ask that does not falls to the largest size that does.
+ assert_eq!(factor_of(4, 1920, 1080), 2);
+ assert_eq!(factor_of(4, 3840, 2160), 1);
+ assert_eq!(factor_of(2, 3840, 2160), 1);
+ }
+
+ #[test]
+ fn a_preview_region_sits_at_the_centre_and_comes_out_no_larger_than_asked() {
+ let region = centre_region(1920, 1080, 2, 640, 360);
+ assert_eq!(
+ region,
+ Tile {
+ x: 800,
+ y: 450,
+ width: 320,
+ height: 180
+ }
+ );
+ // At once, more of the picture fits the same output.
+ assert_eq!(centre_region(1920, 1080, 1, 640, 360).width, 640);
+ // A picture smaller than the region is taken whole.
+ assert_eq!(
+ centre_region(200, 100, 2, 640, 360),
+ Tile {
+ x: 0,
+ y: 0,
+ width: 200,
+ height: 100
+ }
+ );
+ let frame = picture(8, 6, |x, y| [x as u8, y as u8, 0, 255]);
+ let region = centre_region(8, 6, 1, 4, 2);
+ let piece = cut(&frame, region);
+ assert_eq!((piece.width(), piece.height()), (4, 2));
+ assert_eq!(piece.pixel(0, 0), Some([2, 2, 0, 255]));
+ assert_eq!(piece.pixel(3, 1), Some([5, 3, 0, 255]));
+ }
+
+ #[test]
+ fn a_bicubic_enlargement_keeps_a_flat_picture_flat_and_lines_up_its_pixels() {
+ let flat = picture(5, 3, |_, _| [40, 120, 200, 255]);
+ let big = resize(&flat, 20, 12);
+ assert_eq!((big.width(), big.height()), (20, 12));
+ assert!(
+ big.pixels()
+ .chunks_exact(4)
+ .all(|p| p == [40, 120, 200, 255]),
+ "weights that sum to one leave a flat field alone"
+ );
+ // The same size is a copy.
+ let ramp = picture(6, 4, |x, y| [(x * 40) as u8, (y * 60) as u8, 9, 255]);
+ assert_eq!(resize(&ramp, 6, 4).pixels(), ramp.pixels());
+ // Doubled, a ramp stays a ramp: output pixel 5 sits at source 2.25,
+ // which on a line of 40 a pixel is 90, and the cubic reproduces a
+ // line exactly away from the ends. At the ends, where the edge
+ // repeats and the line turns flat, it overshoots by a few levels
+ // and no more.
+ let big = resize(&ramp, 12, 8);
+ let reds: Vec<u8> = (0..12).map(|x| big.pixel(x, 3).unwrap()[0]).collect();
+ assert!(reds.windows(2).all(|pair| pair[0] <= pair[1]), "{reds:?}");
+ assert_eq!(reds[5], 90, "{reds:?}");
+ assert_eq!(reds[0], 0);
+ assert!((200..=206).contains(&reds[11]), "{reds:?}");
+ }
+
+ #[test]
+ fn strength_blends_from_the_plain_enlargement_to_the_models_answer() {
+ let enhanced = picture(2, 1, |_, _| [200, 100, 0, 255]);
+ let plain = picture(2, 1, |_, _| [100, 100, 255, 255]);
+ assert_eq!(blend(&enhanced, &plain, 1.0).pixels(), enhanced.pixels());
+ assert_eq!(blend(&enhanced, &plain, 0.0).pixels(), plain.pixels());
+ assert_eq!(
+ blend(&enhanced, &plain, 0.75).pixel(1, 0),
+ Some([175, 100, 64, 255])
+ );
+ // Out of range is held to it.
+ assert_eq!(blend(&enhanced, &plain, 3.0).pixels(), enhanced.pixels());
+ // Mixing at full strength is the answer untouched; below it, the
+ // original is enlarged to the answer's size first.
+ let original = picture(1, 1, |_, _| [100, 100, 255, 255]);
+ let full = mix(enhanced.clone(), &original, 1.0);
+ assert_eq!(full.pixels(), enhanced.pixels());
+ let half = mix(enhanced.clone(), &original, 0.5);
+ assert_eq!((half.width(), half.height()), (2, 1));
+ assert_eq!(half.pixel(0, 0), Some([150, 100, 128, 255]));
+ }
}
diff --git a/src/crates/concat/locales/en.json b/src/crates/concat/locales/en.json
index 5303d31c..bb3e01ba 100644
--- a/src/crates/concat/locales/en.json
+++ b/src/crates/concat/locales/en.json
@@ -38,9 +38,18 @@
"clipInspector.darken": "Darken",
"clipInspector.effectLibrary": "Effect library",
"clipInspector.enable": "Enable {0}",
- "clipInspector.enhanceExplain": "Restores and enlarges the picture with the enhance model, into a copy the clip then shows. Tick to start; it runs in the background.",
- "clipInspector.enhanceShowingCopy": "Showing the enhanced copy. Untick to show the original again.",
+ "clipInspector.enhanceAfter": "After · {0}% strength",
+ "clipInspector.enhanceAgain": "Enhance again",
+ "clipInspector.enhanceAuto": "Auto ({0})",
+ "clipInspector.enhanceBefore": "Before · a plain enlargement",
+ "clipInspector.enhanceExplain": "Restores and enlarges the picture with the enhance model, into a copy the clip then shows. Preview a frame to judge the settings first; tick to run the whole file in the background.",
+ "clipInspector.enhancePreview": "Preview this frame",
+ "clipInspector.enhancePreviewing": "Running the model on the middle of the frame under the playhead…",
+ "clipInspector.enhanceShowingCopy": "Showing the enhanced copy. Untick to show the original again, or change the settings and Enhance again for another copy.",
"clipInspector.enhanceShowingOriginal": "Showing the original. Tick to show the enhanced copy again.",
+ "clipInspector.enhanceSize": "Size",
+ "clipInspector.enhanceStrength": "Strength",
+ "clipInspector.enhanceStrengthExplain": "How much of the model's work is kept; the rest is the original, enlarged plainly. Lower keeps more real texture, so faces look less waxen.",
"clipInspector.enhanceVoice": "Enhance voice",
"clipInspector.enhancing": "Enhancing… {0}%",
"clipInspector.eraser": "Eraser",
@@ -222,11 +231,14 @@
"export.toneMapped": "Tone-mapped, plays anywhere",
"format.almostDone": "almost done",
"format.daysAgo": "{0} days ago",
+ "format.hoursMinutes": "{0} h {1} min",
"format.justNow": "just now",
+ "format.minutes": "{0} min",
"format.minutesAgo": "{0} minutes ago",
"format.minutesSecondsLeft": "{0}m {1}s left",
"format.monthsAgo": "{0} months ago",
"format.sLeft": "{0}s left",
+ "format.seconds": "{0}s",
"format.today": "today",
"format.yesterday": "yesterday",
"keyframesPane.addRemoveKeyPlayhead": "Add or remove a key at the playhead",
@@ -523,6 +535,10 @@
"studio.duplicate": "Duplicate",
"studio.enhance": "Enhance: {0}",
"studio.enhanceAlreadyWorkOne": "Enhance is already at work; one clip at a time",
+ "studio.enhanceEstimate": "The whole file would take about {0} here: {1} frames at {2}s each.",
+ "studio.enhanceEstimateStill": "The whole picture would take about {0} here.",
+ "studio.enhanceNoFourTimes": "4x would make it {0} × {1}, past the {2}-pixel limit.",
+ "studio.enhanceOnlyRestore": "At {0} pixels across, this picture would pass the {1}-pixel limit if enlarged; it can only be restored at its own size.",
"studio.enhancedClipNowShows": "Enhanced; the clip now shows the copy",
"studio.enhancingWayFrameFrame": "Enhancing {0}: {1}× on its way, frame by frame",
"studio.exit": "Exit",
diff --git a/src/crates/concat/src/format.rs b/src/crates/concat/src/format.rs
index 27a1f04a..dbfcca05 100644
--- a/src/crates/concat/src/format.rs
+++ b/src/crates/concat/src/format.rs
@@ -140,6 +140,20 @@ pub fn eta(seconds: f32) -> String {
}
}
+/// Seconds as a rough length of time, for an estimate given before the
+/// work starts: "40s", "12 min", "3 h 20 min". Rounded to the unit it is
+/// said in, for the reason [`eta`] is.
+pub fn span(seconds: f64) -> String {
+ if seconds < 59.5 {
+ tf("format.seconds", &[&format!("{:.0}", seconds.max(1.0))])
+ } else if seconds < 3570.0 {
+ tf("format.minutes", &[&format!("{:.0}", seconds / 60.0)])
+ } else {
+ let minutes = (seconds / 60.0).round() as u64;
+ tf("format.hoursMinutes", &[&(minutes / 60), &(minutes % 60)])
+ }
+}
+
pub fn hex_of(colour: slint::Color) -> String {
format!(
"#{:02x}{:02x}{:02x}",
@@ -503,4 +517,15 @@ mod tests {
assert_eq!(when_phrase(now - 86_400_000 - 1000), "yesterday");
assert_eq!(when_phrase(now - 3 * 86_400_000), "3 days ago");
}
+
+ #[test]
+ fn an_estimate_is_said_in_the_unit_that_fits_it() {
+ let _english = crate::i18n::language_lock();
+ assert_eq!(span(0.2), "1s");
+ assert_eq!(span(42.4), "42s");
+ assert_eq!(span(59.6), "1 min");
+ assert_eq!(span(12.0 * 60.0 + 10.0), "12 min");
+ assert_eq!(span(3600.0), "1 h 0 min");
+ assert_eq!(span(3.0 * 3600.0 + 20.0 * 60.0), "3 h 20 min");
+ }
}
diff --git a/src/crates/concat/src/lib.rs b/src/crates/concat/src/lib.rs
index f66090e7..f9b2ba33 100644
--- a/src/crates/concat/src/lib.rs
+++ b/src/crates/concat/src/lib.rs
@@ -1305,6 +1305,18 @@ fn start(project: Option<std::path::PathBuf>, smoke: bool) -> Result<(), slint::
editor.on_clip_enhance_set(on_window!(|state, on: bool| {
state.clip_enhance_set(on);
}));
+ editor.on_clip_enhance_size(on_window!(|state, index: i32| {
+ state.clip_enhance_size(index);
+ }));
+ editor.on_clip_enhance_strength(on_window!(|state, strength: f32| {
+ state.clip_enhance_strength(strength);
+ }));
+ editor.on_clip_enhance_preview(on_window!(|state| {
+ state.clip_enhance_preview();
+ }));
+ editor.on_clip_enhance_run(on_window!(|state| {
+ state.clip_enhance_run();
+ }));
editor.on_transition_remove(on_window!(|state| {
state.remove_transition();
}));
diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rs
index 8777ac1f..46fa44c1 100644
--- a/src/crates/concat/src/studio.rs
+++ b/src/crates/concat/src/studio.rs
@@ -110,6 +110,58 @@ pub struct Strip {
pub height: i32,
}
+/// One frame's middle through the Enhance model, before and after, for
+/// the inspector to show before a whole file is committed to.
+struct EnhancePreview {
+ /// The clip it was made for, and the file and size it was made from:
+ /// a preview of other settings or another file is not this one.
+ clip_id: String,
+ media_id: String,
+ factor: u32,
+ /// The plain enlargement and the model's full-strength answer, kept so
+ /// Strength re-blends them at once rather than running the model again.
+ plain: concat_core::Frame,
+ enhanced: concat_core::Frame,
+ before: slint::Image,
+ after: slint::Image,
+ /// The whole percent `after` is blended at.
+ after_percent: u32,
+ /// See `concat_host::enhance::Preview`.
+ tile_seconds: f64,
+ tiles_per_frame: usize,
+}
+
+/// The Enhance section's rows for one clip; see `Studio::enhance_readout`.
+#[derive(Default)]
+struct EnhanceReadout {
+ /// The Size row's segment: 0 Auto, 1 once, 2 twice, 3 four times.
+ size: i32,
+ /// What Auto comes to for this file.
+ auto: u32,
+ fits_2x: bool,
+ fits_4x: bool,
+ /// Why a size is greyed, or empty.
+ size_note: String,
+ /// Before, after and the estimate for a whole run, when a preview of
+ /// these settings is on hand.
+ preview: Option<(slint::Image, slint::Image, String)>,
+}
+
+impl EnhancePreview {
+ /// `after` blended at `strength`, unless it already is.
+ fn blend_at(&mut self, strength: f32) {
+ let percent = concat_host::enhance::strength_percent(strength);
+ if percent != self.after_percent {
+ self.after = image_of(&concat_vision::enhance::blend(
+ &self.enhanced,
+ &self.plain,
+ strength,
+ ));
+ self.after_percent = percent;
+ }
+ }
+}
+
impl From<CachedStrip> for Strip {
fn from(strip: CachedStrip) -> Self {
Self {
@@ -948,6 +1000,19 @@ pub struct Studio {
/// Enhance at work, by clip id: whether the model is still
/// downloading, and how far along. One at a time, like every long job.
enhance_jobs: HashMap<String, (bool, f32)>,
+ /// The Enhance section's Size, in times larger, 0 for Auto, and its
+ /// Strength, 0 to 1. The window's rather than a clip's: they are what
+ /// the next run is made with, carried from clip to clip the way a
+ /// tool's settings are, and a copy already made is named by its own;
+ /// see `concat_host::enhance::target_for`.
+ enhance_size: u32,
+ enhance_strength: f32,
+ /// The last preview of a frame through the model, while the clip it
+ /// was made for is the one in the inspector.
+ enhance_preview: Option<EnhancePreview>,
+ /// A preview being made, for the clip by this id: whether the model is
+ /// still downloading, and how far along.
+ enhance_previewing: Option<(String, bool, f32)>,
/// Reverse writing a clip's backwards copy, by clip: how far along.
reverse_jobs: HashMap<String, f32>,
/// A selection being rendered into one clip, with its own cancel flag.
@@ -2169,6 +2234,10 @@ impl Studio {
painting: true,
cutout_jobs: HashMap::new(),
enhance_jobs: HashMap::new(),
+ enhance_size: 0,
+ enhance_strength: concat_host::enhance::DEFAULT_STRENGTH,
+ enhance_preview: None,
+ enhance_previewing: None,
reverse_jobs: HashMap::new(),
merge_job: None,
region_job: None,
@@ -5889,10 +5958,11 @@ impl Studio {
// an enhanced copy into the project's cache; the clip is then pointed
// at the copy, as one undo step, and shows it wherever it showed the
// original. The window's part is to start the job, keep the readout
- // moving, and re-point the clip when the copy is whole.
+ // moving, and re-point the clip when the copy is whole. Before that,
+ // it holds the section's Size and Strength and runs a preview of the
+ // frame under the playhead, so the person can judge both - and see
+ // what the whole run will cost - before committing minutes to it.
- /// Enhances the media of clip `id`: starts the job, unless one is
- /// running, and says so.
/// Writes one clip's sound as it plays - trimmed, at its speed, with its
/// level, fades and effects baked in - to a WAV in the project's audio
/// folder, and puts the file in the bin under Generated › Processed. A
@@ -6081,8 +6151,178 @@ impl Studio {
}
}
+ /// The Size row: 0 Auto, then once, twice and four times. A size this
+ /// clip cannot take is greyed in the row, and one carried here from a
+ /// smaller clip falls to the largest that fits; see
+ /// `concat_vision::enhance::factor_of`.
+ pub fn clip_enhance_size(&mut self, index: i32) {
+ self.enhance_size = match index {
+ 1 => 1,
+ 2 => 2,
+ 3 => 4,
+ _ => 0,
+ };
+ }
+
+ /// The Strength row, 0 to 1: how much of the model's answer the copy
+ /// keeps. A preview on screen is re-blended at once, which is the
+ /// point of keeping both its halves.
+ pub fn clip_enhance_strength(&mut self, strength: f32) {
+ self.enhance_strength = strength.clamp(0.0, 1.0);
+ let strength = self.enhance_strength;
+ if let Some(preview) = self.enhance_preview.as_mut() {
+ preview.blend_at(strength);
+ }
+ }
+
+ /// "Enhance again": a clip that has a copy, enhanced anew from its
+ /// original with the section's settings as they are now. A copy those
+ /// settings already made is picked up at once rather than redone.
+ pub fn clip_enhance_run(&mut self) {
+ let Some(id) = self.sole_selection() else {
+ return;
+ };
+ if self
+ .clip(&id)
+ .is_some_and(|clip| !self.locked(&clip.track_id))
+ {
+ self.enhance_clip(&id);
+ }
+ }
+
+ /// The file Enhance reads for `clip`: its original when it is showing,
+ /// or holding, a copy already - a second run restores the source, not
+ /// the first run's answer.
+ fn enhance_source(&self, clip: &Clip) -> Option<model::MediaItem> {
+ let id = clip
+ .enhanced
+ .as_ref()
+ .map_or(&clip.media_id, |link| &link.original);
+ self.project()
+ .media
+ .iter()
+ .find(|item| &item.id == id)
+ .cloned()
+ }
+
+ /// How many times larger `media` comes out at the section's Size.
+ fn enhance_factor(&self, media: &model::MediaItem) -> u32 {
+ concat_vision::enhance::factor_of(
+ self.enhance_size,
+ media.width.unwrap_or(0),
+ media.height.unwrap_or(0),
+ )
+ }
+
+ /// "Preview this frame": the middle of the frame of the selected clip
+ /// under the playhead, through the model at the section's Size, shown
+ /// before and after in the inspector. Seconds rather than minutes - a
+ /// few tiles, not a file - and nothing is written or re-pointed, so the
+ /// clip is as it was whatever comes of it.
+ pub fn clip_enhance_preview(&mut self) {
+ let Some(id) = self.sole_selection() else {
+ return;
+ };
+ if self.enhance_previewing.is_some()
+ || !self.enhance_jobs.is_empty()
+ || self.host.enhancers.is_busy()
+ {
+ self.notify(&t("studio.enhanceAlreadyWorkOne"), true);
+ return;
+ }
+ let Some(clip) = self.clip(&id).cloned() else {
+ return;
+ };
+ if clip.kind != model::ClipKind::Video && clip.kind != model::ClipKind::Image {
+ return;
+ }
+ let Some(media) = self.enhance_source(&clip) else {
+ return;
+ };
+ if media.width.unwrap_or(0) == 0 || media.height.unwrap_or(0) == 0 {
+ self.notify(&t("studio.fileHasNoPicture"), true);
+ return;
+ }
+ let factor = self.enhance_factor(&media);
+ let seconds = if clip.kind == model::ClipKind::Image {
+ 0.0
+ } else {
+ self.source_at_playhead(&clip)
+ };
+ let request = concat_host::enhance::PreviewRequest {
+ media_path: media.path.clone(),
+ seconds,
+ factor,
+ };
+ self.enhance_previewing = Some((id.clone(), false, 0.0));
+ let enhancers = Arc::clone(&self.host.enhancers);
+ let epoch = crate::host::project_epoch();
+ let media_id = media.id.clone();
+ spawn_in_project(
+ move || {
+ let mut last = -1.0f32;
+ let result = enhancers.preview(&request, &mut |progress| {
+ // The model's download is the one slow part a preview
+ // can have; the tiles themselves take seconds.
+ if let concat_host::enhance::Progress::Fetching { received, total } = progress {
+ let fraction = received as f32 / total.max(1) as f32;
+ if fraction - last >= 0.01 {
+ last = fraction;
+ on_ui_in_project(epoch, move |studio, _, _| {
+ if let Some(held) = studio.enhance_previewing.as_mut() {
+ held.1 = true;
+ held.2 = fraction;
+ }
+ });
+ }
+ }
+ });
+ (id, media_id, factor, result)
+ },
+ |studio, _, _, (clip_id, media_id, factor, result)| {
+ studio.enhance_previewing = None;
+ match result {
+ Ok(preview) => {
+ let mut made = EnhancePreview {
+ clip_id,
+ media_id,
+ factor,
+ before: image_of(&preview.plain),
+ after: image_of(&preview.enhanced),
+ after_percent: 100,
+ plain: preview.plain,
+ enhanced: preview.enhanced,
+ tile_seconds: preview.tile_seconds,
+ tiles_per_frame: preview.tiles_per_frame,
+ };
+ made.blend_at(studio.enhance_strength);
+ studio.enhance_preview = Some(made);
+ }
+ Err(error) if error.contains("cancelled") => {}
+ Err(error) => studio.notify(&tf("studio.enhance", &[&error]), true),
+ }
+ },
+ );
+ }
+
+ /// The preview on screen for `clip` read from `media` at `factor`, if
+ /// the last one made is that.
+ fn enhance_preview_for(
+ &self,
+ clip: &Clip,
+ media: &model::MediaItem,
+ factor: u32,
+ ) -> Option<&EnhancePreview> {
+ self.enhance_preview.as_ref().filter(|preview| {
+ preview.clip_id == clip.id && preview.media_id == media.id && preview.factor == factor
+ })
+ }
+
pub fn enhance_clip(&mut self, id: &str) {
- if !self.enhance_jobs.is_empty() || self.host.enhancers.is_busy() {
+ if !self.enhance_jobs.is_empty()
+ || self.enhance_previewing.is_some()
+ || self.host.enhancers.is_busy()
+ {
self.notify(&t("studio.enhanceAlreadyWorkOne"), true);
return;
}
@@ -6092,13 +6332,7 @@ impl Studio {
if clip.kind != model::ClipKind::Video && clip.kind != model::ClipKind::Image {
return;
}
- let Some(media) = self
- .project()
- .media
- .iter()
- .find(|item| item.id == clip.media_id)
- .cloned()
- else {
+ let Some(media) = self.enhance_source(&clip) else {
return;
};
let Some(session) = self.session.as_ref() else {
@@ -6110,9 +6344,11 @@ impl Studio {
self.notify(&t("studio.fileHasNoPicture"), true);
return;
}
- let factor = concat_vision::enhance::factor_for(width, height);
+ let factor = self.enhance_factor(&media);
+ let strength = self.enhance_strength;
let still = clip.kind == model::ClipKind::Image;
- let Some(target) = concat_host::enhance::target_for(&project, &media.path, factor, still)
+ let Some(target) =
+ concat_host::enhance::target_for(&project, &media.path, factor, strength, still)
else {
self.notify(&tf("studio.couldNotRead", &[&media.path]), true);
return;
@@ -6121,8 +6357,10 @@ impl Studio {
media_path: media.path.clone(),
still,
factor,
+ strength,
target,
};
+ let percent = concat_host::enhance::strength_percent(strength);
let clip_id = clip.id.clone();
self.enhance_jobs.insert(clip_id.clone(), (false, 0.0));
self.notify(
@@ -6154,10 +6392,13 @@ impl Studio {
});
(clip_id, result)
},
- |studio, _, _, (clip_id, result)| {
+ move |studio, _, _, (clip_id, result)| {
studio.enhance_jobs.remove(&clip_id);
+ // A cancelled or failed run leaves the clip as it was: the
+ // job writes under partial names and the clip is only
+ // re-pointed here, on a copy that is whole.
match result {
- Ok(path) => studio.adopt_enhanced(&clip_id, &path),
+ Ok(path) => studio.adopt_enhanced(&clip_id, &path, factor, percent),
Err(error) if error.contains("cancelled") => {}
Err(error) => studio.notify(&tf("studio.enhance", &[&error]), true),
}
@@ -6286,7 +6527,7 @@ impl Studio {
/// Points clip `id` at the enhanced copy at `path`, probed the way an
/// import is, and shows it.
- fn adopt_enhanced(&mut self, id: &str, path: &std::path::Path) {
+ fn adopt_enhanced(&mut self, id: &str, path: &std::path::Path, factor: u32, percent: u32) {
let summary = match media::probe(&path.to_string_lossy()) {
Ok(summary) => summary,
Err(error) => {
@@ -6295,13 +6536,11 @@ impl Studio {
}
};
let mut item = summary.to_new_media();
- // The bin names the copy after the original, marked, rather than
- // after its cache file.
- if let Some(original) = self
- .clip(id)
- .and_then(|clip| self.project().media.iter().find(|m| m.id == clip.media_id))
- {
- item.name = format!("{} (enhanced)", original.name);
+ // The bin names the copy after the original, marked with what it
+ // was made with - one file can have a copy for each setting -
+ // rather than after its cache file.
+ if let Some(original) = self.clip(id).and_then(|clip| self.enhance_source(clip)) {
+ item.name = format!("{} (enhanced {factor}x, {percent}%)", original.name);
}
self.apply(Command::ReplaceClipMedia {
clip_id: id.to_owned(),
@@ -7302,6 +7541,8 @@ impl Studio {
self.host.enhancers.cancel();
self.host.reversers.cancel();
self.enhance_jobs.clear();
+ self.enhance_preview = None;
+ self.enhance_previewing = None;
self.reverse_jobs.clear();
self.cancel_merge();
self.cutout_jobs.clear();
@@ -9044,6 +9285,10 @@ impl Studio {
.get(&Self::analysis_key(&clip.media_id, cutout.subject))
.copied()
});
+ let mut enhance = self.enhance_readout(clip);
+ let enhance_preview = enhance.preview.is_some();
+ let (enhance_before, enhance_after, enhance_estimate) =
+ enhance.preview.take().unwrap_or_default();
SelectedClipData {
present: true,
batch: targets.len() as i32,
@@ -9166,9 +9411,99 @@ impl Studio {
enhance_linked: clip.enhanced.is_some(),
enhance_on: clip.enhanced.as_ref().is_some_and(|link| link.on)
|| self.enhance_jobs.contains_key(&clip.id),
+ enhance_busy: !self.enhance_jobs.is_empty() || self.enhance_previewing.is_some(),
+ enhance_size: enhance.size,
+ enhance_auto: enhance.auto as i32,
+ enhance_fits_2x: enhance.fits_2x,
+ enhance_fits_4x: enhance.fits_4x,
+ enhance_size_note: enhance.size_note.into(),
+ enhance_strength: self.enhance_strength,
+ enhance_previewing: self
+ .enhance_previewing
+ .as_ref()
+ .is_some_and(|(id, _, _)| *id == clip.id),
+ enhance_preview_fetch: self
+ .enhance_previewing
+ .as_ref()
+ .filter(|(id, fetching, _)| *id == clip.id && *fetching)
+ .map(|(_, _, fraction)| *fraction)
+ .unwrap_or(-1.0),
+ enhance_preview,
+ enhance_before,
+ enhance_after,
+ enhance_estimate: enhance_estimate.into(),
}
}
+ /// What the Enhance section shows for `clip` beyond its switch: the
+ /// sizes its file can take, the preview made for it at the section's
+ /// settings, if there is one, and what a whole run would cost.
+ fn enhance_readout(&self, clip: &Clip) -> EnhanceReadout {
+ let mut readout = EnhanceReadout::default();
+ let Some(media) = self.enhance_source(clip) else {
+ return readout;
+ };
+ let (width, height) = (media.width.unwrap_or(0), media.height.unwrap_or(0));
+ let limit = concat_vision::enhance::LONGEST_SIDE;
+ let factor = self.enhance_factor(&media);
+ readout.auto = concat_vision::enhance::factor_for(width, height);
+ readout.fits_2x = concat_vision::enhance::fits(width, height, 2);
+ readout.fits_4x = concat_vision::enhance::fits(width, height, 4);
+ // The row shows what will happen: Auto as Auto, and an ask this
+ // file cannot take as the size it falls to.
+ readout.size = match (self.enhance_size, factor) {
+ (0, _) => 0,
+ (_, 1) => 1,
+ (_, 2) => 2,
+ _ => 3,
+ };
+ readout.size_note = if !readout.fits_2x {
+ tf("studio.enhanceOnlyRestore", &[&width.max(height), &limit])
+ } else if !readout.fits_4x {
+ tf(
+ "studio.enhanceNoFourTimes",
+ &[&(width * 4), &(height * 4), &limit],
+ )
+ } else {
+ String::new()
+ };
+ readout.preview = self
+ .enhance_preview_for(clip, &media, factor)
+ .map(|preview| {
+ // The model's work on the whole file - every frame of it,
+ // not only the span the clip shows - at the warm tile time
+ // the preview measured.
+ let frames = if clip.kind == model::ClipKind::Image {
+ 1.0
+ } else {
+ media.duration.unwrap_or(0.0) * media.frame_rate.unwrap_or(30.0)
+ };
+ let per_frame = preview.tile_seconds * preview.tiles_per_frame as f64;
+ let total = concat_host::enhance::estimate_seconds(
+ preview.tile_seconds,
+ preview.tiles_per_frame,
+ frames,
+ );
+ let estimate = if clip.kind == model::ClipKind::Image {
+ tf(
+ "studio.enhanceEstimateStill",
+ &[&crate::format::span(total)],
+ )
+ } else {
+ tf(
+ "studio.enhanceEstimate",
+ &[
+ &crate::format::span(total),
+ &format!("{:.0}", frames.max(1.0)),
+ &format!("{per_frame:.1}"),
+ ],
+ )
+ };
+ (preview.before.clone(), preview.after.clone(), estimate)
+ });
+ readout
+ }
+
/// The source instant of `clip` under the playhead, held to the clip.
fn source_at_playhead(&self, clip: &Clip) -> f64 {
let along = (f64::from(self.playhead) - clip.start).clamp(0.0, clip.duration);
diff --git a/src/crates/concat/ui/editor.slint b/src/crates/concat/ui/editor.slint
index 2f51291c..79d34d31 100644
--- a/src/crates/concat/ui/editor.slint
+++ b/src/crates/concat/ui/editor.slint
@@ -277,6 +277,12 @@ export global Editor {
callback cutout-subject(index: int);
callback cutout-clear();
callback clip-enhance-set(on: bool);
+ /// The Enhance section's settings and its preview; see EnhancePanel.
+ /// `index` is 0 Auto, then 1x, 2x and 4x.
+ callback clip-enhance-size(index: int);
+ callback clip-enhance-strength(strength: float);
+ callback clip-enhance-preview();
+ callback clip-enhance-run();
/// The transition riding into the selected clip; see VideoPanel.
callback transition-remove();
callback transition-duration-set(seconds: float);
diff --git a/src/crates/concat/ui/inspector/clip-inspector.slint b/src/crates/concat/ui/inspector/clip-inspector.slint
index c8320129..e55206f3 100644
--- a/src/crates/concat/ui/inspector/clip-inspector.slint
+++ b/src/crates/concat/ui/inspector/clip-inspector.slint
@@ -1465,14 +1465,71 @@ export component VideoPanel inherits VerticalLayout {
}
}
+/// One half of Enhance's before and after: a label over the picture,
+/// fitted whole into a fixed height so the pair reads the same on a phone
+/// as in the desktop inspector, and nothing in the layout leans on the
+/// picture's own size.
+component PreviewHalf inherits VerticalLayout {
+ in property <string> label;
+ in property <image> source;
+
+ padding-left: Theme.pad;
+ padding-right: Theme.pad;
+ spacing: 4px;
+
+ Text {
+ text: root.label;
+ color: Theme.fg-muted;
+ font-size: Theme.fs-sm;
+ }
+ Rectangle {
+ height: 150px;
+ background: Theme.well;
+ border-radius: Theme.r-sm;
+ clip: true;
+
+ Image {
+ width: parent.width;
+ height: parent.height;
+ source: root.source;
+ image-fit: contain;
+ }
+ }
+}
+
/// Enhance: the picture restored and enlarged by the model, into a copy the
/// clip then shows. The header's tick starts it the first time; once there
/// is a copy, it switches between the copy and the original without running
/// the model again.
+///
+/// Below the tick, what the run is made with - Size and Strength - and a
+/// preview of the frame under the playhead at those settings, so the
+/// person judges the look, and what a whole run will cost, in seconds
+/// rather than after minutes of their machine. Strength is the answer to
+/// the waxen, "plastic" faces a full-strength enhancer is known for: it
+/// blends the model's answer back towards the original.
export component EnhancePanel inherits VerticalLayout {
in property <SelectedClipData> selected;
callback enhance-set(on: bool);
+ /// 0 Auto, 1 once, 2 twice, 3 four times.
+ callback enhance-size(index: int);
+ callback enhance-strength(strength: float);
+ callback enhance-preview();
+ /// A clip with a copy enhanced anew with the settings as they are.
+ callback enhance-run();
+
+ property <float> strength;
+
+ init => {
+ root.strength = root.selected.enhance-strength;
+ }
+ // Put back on every publish: a click on the row takes its binding, and
+ // the next clip's file may not take the size this one showed.
+ changed selected => {
+ root.strength = root.selected.enhance-strength;
+ sizes.current = root.selected.enhance-size;
+ }
Section {
first: true;
@@ -1495,6 +1552,77 @@ export component EnhancePanel inherits VerticalLayout {
: I18n.t("clipInspector.enhanceShowingOriginal"))
: I18n.t("clipInspector.enhanceExplain");
}
+
+ Row {
+ label: I18n.t("clipInspector.enhanceSize");
+ sizes := SegmentedControl {
+ options: [
+ I18n.t1("clipInspector.enhanceAuto", "\{root.selected.enhance-auto}x"),
+ "1x",
+ "2x",
+ "4x",
+ ];
+ disabled: [
+ false,
+ false,
+ !root.selected.enhance-fits-2x,
+ !root.selected.enhance-fits-4x,
+ ];
+ current: root.selected.enhance-size;
+ changed(index) => { root.enhance-size(index); }
+ }
+ }
+ if root.selected.enhance-size-note != "": Note {
+ text: root.selected.enhance-size-note;
+ }
+
+ Param {
+ label: I18n.t("clipInspector.enhanceStrength");
+ value <=> root.strength;
+ minimum: 0;
+ maximum: 1;
+ step: 0.01;
+ default-value: 0.75;
+ fmt: ParamFormat.fraction;
+ unit: "%";
+ unit-scale: 100;
+ changed(v) => { root.enhance-strength(v); }
+ }
+ Note {
+ text: I18n.t("clipInspector.enhanceStrengthExplain");
+ }
+
+ Row {
+ Button {
+ label: I18n.t("clipInspector.enhancePreview");
+ enabled: !root.selected.enhance-busy;
+ clicked => { root.enhance-preview(); }
+ }
+ if root.selected.enhance-linked: Button {
+ label: I18n.t("clipInspector.enhanceAgain");
+ enabled: !root.selected.enhance-busy;
+ clicked => { root.enhance-run(); }
+ }
+ Rectangle { horizontal-stretch: 1; }
+ }
+ if root.selected.enhance-previewing: Note {
+ text: root.selected.enhance-preview-fetch >= 0
+ ? I18n.t1("clipInspector.fetchingTheModel",
+ "\{Math.round(root.selected.enhance-preview-fetch * 100)}")
+ : I18n.t("clipInspector.enhancePreviewing");
+ }
+
+ if root.selected.enhance-preview: PreviewHalf {
+ label: I18n.t("clipInspector.enhanceBefore");
+ source: root.selected.enhance-before;
+ }
+ if root.selected.enhance-preview: PreviewHalf {
+ label: I18n.t1("clipInspector.enhanceAfter", "\{Math.round(root.strength * 100)}");
+ source: root.selected.enhance-after;
+ }
+ if root.selected.enhance-preview: Note {
+ text: root.selected.enhance-estimate;
+ }
}
}
}
diff --git a/src/crates/concat/ui/phone/editor.slint b/src/crates/concat/ui/phone/editor.slint
index 1a8d69ea..992a3da4 100644
--- a/src/crates/concat/ui/phone/editor.slint
+++ b/src/crates/concat/ui/phone/editor.slint
@@ -14,8 +14,8 @@ import {
CategoryPillBar, MediaCard, MediaItemData, PresetGrid, ShelfGrid, ShelfSearch,
} from "../workspace/media-pane.slint";
import {
- AdjustPanel, AudioPanel, FilterPanel, Library, PresetStack, ProcessorRows, SpeedPanel,
- VideoPanel,
+ AdjustPanel, AudioPanel, EnhancePanel, FilterPanel, Library, PresetStack, ProcessorRows,
+ SpeedPanel, VideoPanel,
} from "../inspector/clip-inspector.slint";
import { TextPanel } from "../inspector/text-panel.slint";
import { RoundButton, ToolItem, ToolStrip } from "controls.slint";
@@ -252,6 +252,7 @@ export component PhoneEditor inherits Rectangle {
: name == "clip-speed" ? I18n.t("common.speed")
: name == "clip-audio" ? I18n.t("common.audio")
: name == "clip-adjust" ? I18n.t("common.adjust")
+ : name == "clip-enhance" ? I18n.t("common.enhance")
: name == "clip-layer" ? I18n.t("configPane.filter")
: "";
}
@@ -761,6 +762,14 @@ export component PhoneEditor inherits Rectangle {
current: root.sheet == "clip-adjust";
clicked => { root.show("clip-adjust", true); }
}
+ // The desk's Video > Enhance, on a sheet of its own: the same
+ // panel, settings, preview and all.
+ if root.is-picture: ToolItem {
+ glyph: Glyph.maximize;
+ label: I18n.t("common.enhance");
+ current: root.sheet == "clip-enhance";
+ clicked => { root.show("clip-enhance", true); }
+ }
if root.is-picture || root.is-text: ToolItem {
glyph: Glyph.sparkle;
label: I18n.t("common.effects");
@@ -1093,6 +1102,15 @@ export component PhoneEditor inherits Rectangle {
commit => { Editor.clip-commit(); }
}
+ if root.sheet == "clip-enhance": EnhancePanel {
+ selected: Editor.selected-clip;
+ enhance-set(on) => { Editor.clip-enhance-set(on); }
+ enhance-size(index) => { Editor.clip-enhance-size(index); }
+ enhance-strength(strength) => { Editor.clip-enhance-strength(strength); }
+ enhance-preview => { Editor.clip-enhance-preview(); }
+ enhance-run => { Editor.clip-enhance-run(); }
+ }
+
if root.sheet == "clip-text": TextPanel {
selected: Editor.selected-clip;
families: Editor.font-families;
diff --git a/src/crates/concat/ui/primitives/segmented-control.slint b/src/crates/concat/ui/primitives/segmented-control.slint
index 0f8bf5fd..5cf16751 100644
--- a/src/crates/concat/ui/primitives/segmented-control.slint
+++ b/src/crates/concat/ui/primitives/segmented-control.slint
@@ -18,6 +18,10 @@ export component SegmentedControl inherits Rectangle {
in property <[Glyph]> icons;
in-out property <int> current;
in property <bool> enabled: true;
+ /// Segments that cannot be picked, by index: greyed, and refused
+ /// by a click or the arrow keys. Shorter than `options` is fine; a
+ /// segment past the end can be picked.
+ in property <[bool]> disabled;
callback changed(index: int);
@@ -39,7 +43,7 @@ export component SegmentedControl inherits Rectangle {
opacity: root.enabled ? 1.0 : 0.45;
function select(index: int) {
- if (index >= 0 && index < root.options.length) {
+ if (index >= 0 && index < root.options.length && !root.disabled[index]) {
root.current = index;
root.changed(index);
}
@@ -93,7 +97,8 @@ export component SegmentedControl inherits Rectangle {
width: parent.width;
height: parent.height;
text: option;
- color: (index == root.current || segment.has-hover) ? Theme.fg : Theme.fg-muted;
+ color: root.disabled[index] ? Theme.fg-dim
+ : (index == root.current || segment.has-hover) ? Theme.fg : Theme.fg-muted;
font-size: Theme.fs;
horizontal-alignment: center;
vertical-alignment: center;
@@ -110,7 +115,7 @@ export component SegmentedControl inherits Rectangle {
}
segment := TouchArea {
- enabled: root.enabled;
+ enabled: root.enabled && !root.disabled[index];
mouse-cursor: pointer;
clicked => {
fs.focus();
diff --git a/src/crates/concat/ui/timeline/model.slint b/src/crates/concat/ui/timeline/model.slint
index f7f95c02..a2c8f9c9 100644
--- a/src/crates/concat/ui/timeline/model.slint
+++ b/src/crates/concat/ui/timeline/model.slint
@@ -349,6 +349,32 @@ export struct SelectedClipData {
enhance-linked: bool,
/// The Enhance switch: the copy shown, or a run under way
enhance-on: bool,
+ /// Enhance at work on some clip, a run or a preview: the model is one
+ /// model, so the section's buttons wait for it
+ enhance-busy: bool,
+ /// The Size row: 0 Auto, 1 once, 2 twice, 3 four times - what will
+ /// happen, so an ask this file cannot take shows as what it falls to.
+ /// `enhance-auto` is what Auto comes to here; a size that would pass
+ /// the longest-side limit is greyed, `enhance-size-note` saying why
+ enhance-size: int,
+ enhance-auto: int,
+ enhance-fits-2x: bool,
+ enhance-fits-4x: bool,
+ enhance-size-note: string,
+ /// The Strength row, 0 to 1: how much of the model's answer is kept
+ enhance-strength: float,
+ /// A preview of the frame under the playhead being made for this clip,
+ /// and how far the model's download is when that is what it waits on
+ /// (-1 when it is not)
+ enhance-previewing: bool,
+ enhance-preview-fetch: float,
+ /// The preview made for this clip at these settings: the middle of
+ /// the frame enlarged plainly and enhanced, and what a whole run would
+ /// take at the speed it measured
+ enhance-preview: bool,
+ enhance-before: image,
+ enhance-after: image,
+ enhance-estimate: string,
/// Reverse writing this clip's backwards copy: -1 when it is not,
/// else how far along
reverse-progress: float,
diff --git a/src/crates/concat/ui/workspace/config-pane.slint b/src/crates/concat/ui/workspace/config-pane.slint
index 68123885..d9c3eb0f 100644
--- a/src/crates/concat/ui/workspace/config-pane.slint
+++ b/src/crates/concat/ui/workspace/config-pane.slint
@@ -205,6 +205,10 @@ export component ConfigPane inherits Rectangle {
callback cutout-clear();
/// The Enhance switch: start it, or show the copy or the original.
callback clip-enhance-set(on: bool);
+ callback clip-enhance-size(index: int);
+ callback clip-enhance-strength(strength: float);
+ callback clip-enhance-preview();
+ callback clip-enhance-run();
callback transition-remove();
callback transition-duration-set(seconds: float);
@@ -647,6 +651,10 @@ export component ConfigPane inherits Rectangle {
if root.showing == "Enhance": EnhancePanel {
selected: root.selected;
enhance-set(on) => { root.clip-enhance-set(on); }
+ enhance-size(index) => { root.clip-enhance-size(index); }
+ enhance-strength(strength) => { root.clip-enhance-strength(strength); }
+ enhance-preview => { root.clip-enhance-preview(); }
+ enhance-run => { root.clip-enhance-run(); }
}
if root.showing == "Adjust": AdjustPanel {
diff --git a/src/crates/concat/ui/workspace/seat.slint b/src/crates/concat/ui/workspace/seat.slint
index ac8bddb4..19511aa0 100644
--- a/src/crates/concat/ui/workspace/seat.slint
+++ b/src/crates/concat/ui/workspace/seat.slint
@@ -271,6 +271,10 @@ export component Seat inherits Rectangle {
cutout-subject(index) => { Editor.cutout-subject(index); }
cutout-clear => { Editor.cutout-clear(); }
clip-enhance-set(on) => { Editor.clip-enhance-set(on); }
+ clip-enhance-size(index) => { Editor.clip-enhance-size(index); }
+ clip-enhance-strength(strength) => { Editor.clip-enhance-strength(strength); }
+ clip-enhance-preview => { Editor.clip-enhance-preview(); }
+ clip-enhance-run => { Editor.clip-enhance-run(); }
transition-remove => { Editor.transition-remove(); }
transition-duration-set(seconds) => { Editor.transition-duration-set(seconds); }