Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(enhance): Enhance on a clip's right-click menu, an ML-restored copy the clip then shows

Jareer6d5b07ed5972
diff --git a/src/crates/concat-host/Cargo.toml b/src/crates/concat-host/Cargo.toml
index 2e3c29dc..bbf5ab8e 100644
--- a/src/crates/concat-host/Cargo.toml
+++ b/src/crates/concat-host/Cargo.toml
@@ -21,6 +21,9 @@ concat-text.workspace = true
# The cutout models and their masks; `infer` because the host is what
# runs them, and ureq because the host is what fetches them.
concat-vision = { workspace = true, features = ["infer"] }
+# An enhanced still is written as a PNG: lossless, since the point was
+# the detail.
+png = "0.17"
ureq = "2"
# A finished download is checked against the digest its table carries
# before anything is allowed to call it installed; see models.rs.
diff --git a/src/crates/concat-host/src/enhance.rs b/src/crates/concat-host/src/enhance.rs
new file mode 100644
index 00000000..abe3996c
--- /dev/null
+++ b/src/crates/concat-host/src/enhance.rs
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Enhance: an enhanced copy of a media file, written once.
+//!
+//! Restoration is not real time - a frame takes the model a good part of
+//! a second on an accelerator and many seconds without one - so a clip is
+//! not enhanced as it plays. Instead the job here reads the file frame by
+//! frame, runs each through [`concat_vision::enhance`], and writes the
+//! result into the project's cache as a file of its own: a PNG for a
+//! still, an MP4 with the original's sound for footage. The window then
+//! points the clip at the copy, the way a freeze frame points at its
+//! still, and the monitor and the export read it like any other file.
+//!
+//! One job at a time through a [`SingleFlight`], like every long job the
+//! host runs, with the model fetched on first use and kept loaded. The
+//! copy is named after the file's path, size and modification time and
+//! the factor it was enlarged by, so a file replaced on disk gets a fresh
+//! one and the same file asked for twice gets the one already there.
+
+use std::hash::{Hash, Hasher};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::AtomicBool;
+use std::sync::{Arc, Mutex};
+
+use concat_media::{
+ DecodeOptions, Decoder, EncodeOptions, Encoder, FrameSink, FrameSource, RateMode, VideoCodec,
+};
+use concat_vision::enhance::Enhancer;
+use concat_vision::{ModelId, models};
+
+pub use crate::cutout::Progress;
+use crate::jobs::SingleFlight;
+
+/// What one enhancement covers.
+#[derive(Clone, Debug)]
+pub struct EnhanceRequest {
+ /// The media file to read.
+ pub media_path: String,
+ /// A still: one frame in, one PNG out.
+ pub still: bool,
+ /// How many times larger the copy is; see
+ /// [`concat_vision::enhance::factor_for`].
+ pub factor: u32,
+ /// Where the copy goes; see [`target_for`].
+ pub target: PathBuf,
+}
+
+/// The enhancement service: the model, loaded once, and the one-job slot.
+pub struct Enhancers {
+ gate: Arc<SingleFlight>,
+ /// The app's data directory, where the downloaded model lives.
+ data: PathBuf,
+ model: Mutex<Option<Arc<Enhancer>>>,
+}
+
+/// Where the enhanced copy of `media` at `factor` lives under `project`:
+/// in the cache, named by the file's path, size and modification time.
+/// None for a file that cannot be stat'ed, which cannot be read either.
+pub fn target_for(project: &Path, media: &str, factor: u32, still: bool) -> Option<PathBuf> {
+ let meta = std::fs::metadata(media).ok()?;
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ media.hash(&mut hasher);
+ meta.len().hash(&mut hasher);
+ if let Ok(modified) = meta.modified()
+ && let Ok(since) = modified.duration_since(std::time::UNIX_EPOCH)
+ {
+ since.as_secs().hash(&mut hasher);
+ }
+ let extension = if still { "png" } else { "mp4" };
+ Some(project.join("cache").join(format!(
+ "enhance-{:016x}-{factor}x.{extension}",
+ hasher.finish()
+ )))
+}
+
+impl Enhancers {
+ /// A service with nothing loaded yet, keeping its model under `data`.
+ pub fn new(data: &Path) -> Enhancers {
+ Enhancers {
+ gate: Arc::new(SingleFlight::new()),
+ data: data.to_path_buf(),
+ model: Mutex::new(None),
+ }
+ }
+
+ /// Whether an enhancement is running.
+ pub fn is_busy(&self) -> bool {
+ self.gate.is_busy()
+ }
+
+ /// Asks the running enhancement to stop after the frame in hand.
+ pub fn cancel(&self) {
+ self.gate.cancel();
+ }
+
+ /// Whether the model is on disk.
+ pub fn installed(&self) -> bool {
+ models::installed(&self.data, ModelId::Enhance)
+ }
+
+ /// The model, fetched first when it is not on disk.
+ fn loaded(
+ &self,
+ cancel: &AtomicBool,
+ progress: &mut dyn FnMut(Progress),
+ ) -> Result<Arc<Enhancer>, String> {
+ if let Some(loaded) = self
+ .model
+ .lock()
+ .map_err(|_| "model slot poisoned")?
+ .as_ref()
+ {
+ return Ok(Arc::clone(loaded));
+ }
+ let file = models::model_file(&self.data, ModelId::Enhance);
+ if !self.installed() {
+ crate::cutout::fetch(ModelId::Enhance, &file, cancel, progress)?;
+ }
+ let enhancer = Arc::new(Enhancer::load(&file)?);
+ *self.model.lock().map_err(|_| "model slot poisoned")? = Some(Arc::clone(&enhancer));
+ Ok(enhancer)
+ }
+
+ /// Writes the copy `request` asks for, or finds it already written.
+ /// Blocks until done, so run it on its own thread; `progress` is
+ /// called as it goes. Returns where the copy is.
+ pub fn enhance(
+ &self,
+ request: &EnhanceRequest,
+ progress: &mut dyn FnMut(Progress),
+ ) -> Result<PathBuf, String> {
+ if request.target.is_file() {
+ return Ok(request.target.clone());
+ }
+ let job = self.gate.begin("enhance")?;
+ let cancel = job.cancel_handle();
+ let enhancer = self.loaded(&cancel, progress)?;
+ if let Some(folder) = request.target.parent() {
+ std::fs::create_dir_all(folder)
+ .map_err(|error| format!("{}: {error}", folder.display()))?;
+ }
+ progress(Progress::Analysing(0.0));
+ let result = if request.still {
+ still(&enhancer, request)
+ } else {
+ footage(&enhancer, request, &job, progress)
+ };
+ result.map(|()| request.target.clone())
+ }
+}
+
+/// One frame through the model, as a PNG: lossless, since the point was
+/// the detail.
+fn still(enhancer: &Enhancer, request: &EnhanceRequest) -> Result<(), String> {
+ let mut decoder = Decoder::open(&request.media_path, &DecodeOptions::default().in_software())
+ .map_err(|error| error.to_string())?;
+ let frame = decoder
+ .next_frame()
+ .map_err(|error| error.to_string())?
+ .ok_or_else(|| format!("{} holds no picture", request.media_path))?;
+ let enhanced = enhancer.enhance(&frame, request.factor)?;
+ let partial = request.target.with_extension("part.png");
+ let written = (|| {
+ let file = std::fs::File::create(&partial).map_err(|error| error.to_string())?;
+ let mut encoder = png::Encoder::new(
+ std::io::BufWriter::new(file),
+ enhanced.width(),
+ enhanced.height(),
+ );
+ encoder.set_color(png::ColorType::Rgba);
+ encoder.set_depth(png::BitDepth::Eight);
+ let mut writer = encoder.write_header().map_err(|error| error.to_string())?;
+ writer
+ .write_image_data(enhanced.pixels())
+ .map_err(|error| error.to_string())
+ })();
+ match written {
+ Ok(()) => std::fs::rename(&partial, &request.target)
+ .map_err(|error| format!("{}: {error}", request.target.display())),
+ Err(error) => {
+ let _ = std::fs::remove_file(&partial);
+ Err(error)
+ }
+ }
+}
+
+/// Every frame through the model, encoded at the enlarged size, then the
+/// original's sound copied in beside the picture. Written under partial
+/// names and moved into place only when whole, so a copy that is there is
+/// a copy that is complete.
+fn footage(
+ enhancer: &Enhancer,
+ request: &EnhanceRequest,
+ job: &crate::jobs::Job,
+ progress: &mut dyn FnMut(Progress),
+) -> Result<(), String> {
+ let info = concat_media::probe(&request.media_path).map_err(|error| error.to_string())?;
+ let video = info
+ .video
+ .as_ref()
+ .ok_or_else(|| "no video stream".to_owned())?;
+ let rate = video.frame_rate;
+ // About how many frames there are, for the readout; the count is not
+ // known until the last one is read.
+ let expected = info
+ .duration
+ .map(|duration| duration.as_f64() * rate.fps().as_f64())
+ .unwrap_or(0.0)
+ .max(1.0);
+ // Even sides, which the encoder wants: a stray pixel at the right or
+ // bottom is cropped rather than the whole copy refused.
+ let width = (video.width * request.factor) & !1;
+ let height = (video.height * request.factor) & !1;
+ if width == 0 || height == 0 {
+ return Err("the picture is too small to enhance".to_owned());
+ }
+
+ let partial_video = request.target.with_extension("part.mp4");
+ let partial_muxed = request.target.with_extension("muxed.part.mp4");
+ let options = DecodeOptions::default().in_software().threaded(2);
+ let mut decoder =
+ Decoder::open(&request.media_path, &options).map_err(|error| error.to_string())?;
+ let mut encoder = Encoder::create(
+ &partial_video,
+ width,
+ height,
+ rate,
+ &EncodeOptions {
+ codec: VideoCodec::H264,
+ // A copy the timeline reads for as long as the project lives:
+ // near-transparent quality, and the time it takes is spent in
+ // the model anyway.
+ preset: "medium".to_owned(),
+ crf: 16,
+ rate_mode: RateMode::Vbr,
+ bitrate_kbps: 0,
+ ten_bit: false,
+ hardware: false,
+ threads: 2,
+ },
+ )
+ .map_err(|error| error.to_string())?;
+ let encoded = (|| {
+ let mut count = 0u64;
+ while let Some(frame) = decoder.next_frame().map_err(|error| error.to_string())? {
+ if job.cancelled() {
+ return Err("enhance cancelled".to_owned());
+ }
+ let enhanced = enhancer.enhance(&frame, request.factor)?;
+ let enhanced = crop(&enhanced, width, height);
+ encoder
+ .write_frame(&enhanced)
+ .map_err(|error| error.to_string())?;
+ count += 1;
+ progress(Progress::Analysing(
+ (count as f64 / expected).min(0.99) as f32
+ ));
+ }
+ encoder.finish().map_err(|error| error.to_string())
+ })();
+ if let Err(error) = encoded {
+ let _ = std::fs::remove_file(&partial_video);
+ return Err(error);
+ }
+ let finished = if info.audio.is_some() {
+ let muxed = concat_media::audio::mux(
+ &partial_video,
+ Path::new(&request.media_path),
+ &partial_muxed,
+ )
+ .map_err(|error| error.to_string());
+ let _ = std::fs::remove_file(&partial_video);
+ muxed.and_then(|()| {
+ std::fs::rename(&partial_muxed, &request.target)
+ .map_err(|error| format!("{}: {error}", request.target.display()))
+ })
+ } else {
+ std::fs::rename(&partial_video, &request.target)
+ .map_err(|error| format!("{}: {error}", request.target.display()))
+ };
+ if finished.is_err() {
+ let _ = std::fs::remove_file(&partial_muxed);
+ }
+ finished
+}
+
+/// `frame` cut to `width` by `height` from its top-left, or the frame
+/// itself when it is that size already.
+fn crop(frame: &concat_core::Frame, width: u32, height: u32) -> concat_core::Frame {
+ if frame.width() == width && frame.height() == height {
+ return frame.clone();
+ }
+ let mut out = concat_core::Frame::black(width, height);
+ let bytes = width as usize * concat_core::frame::BYTES_PER_PIXEL;
+ let stride = frame.width() as usize * concat_core::frame::BYTES_PER_PIXEL;
+ let source = frame.pixels();
+ let pixels = out.pixels_mut();
+ for y in 0..height as usize {
+ pixels[y * bytes..(y + 1) * bytes].copy_from_slice(&source[y * stride..y * stride + bytes]);
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_copy_is_named_after_the_file_and_the_factor() {
+ let dir = std::env::temp_dir().join(format!("concat-enhance-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).expect("mkdir");
+ let media = dir.join("clip.mp4");
+ std::fs::write(&media, b"not really").expect("write");
+ let media = media.to_string_lossy().into_owned();
+ let project = dir.join("project");
+ let two = target_for(&project, &media, 2, false).expect("named");
+ assert!(two.starts_with(project.join("cache")));
+ assert!(two.to_string_lossy().ends_with("-2x.mp4"), "{two:?}");
+ assert_eq!(target_for(&project, &media, 2, false), Some(two.clone()));
+ assert_ne!(target_for(&project, &media, 1, false), Some(two.clone()));
+ let still = target_for(&project, &media, 2, true).expect("named");
+ assert!(still.to_string_lossy().ends_with("-2x.png"), "{still:?}");
+ assert_eq!(
+ target_for(&project, &dir.join("gone.mp4").to_string_lossy(), 2, false),
+ None
+ );
+ // A file with different bytes is a different copy.
+ std::fs::write(&media, b"not really, but longer").expect("write");
+ assert_ne!(target_for(&project, &media, 2, false), Some(two));
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn a_crop_takes_the_top_left_and_leaves_a_fitting_frame_alone() {
+ let mut frame = concat_core::Frame::black(4, 3);
+ frame.set_pixel(3, 2, [1, 2, 3, 255]);
+ frame.set_pixel(0, 0, [9, 9, 9, 255]);
+ let same = crop(&frame, 4, 3);
+ assert_eq!(same.pixels(), frame.pixels());
+ let cut = crop(&frame, 2, 2);
+ assert_eq!((cut.width(), cut.height()), (2, 2));
+ assert_eq!(cut.pixel(0, 0), Some([9, 9, 9, 255]));
+ assert_eq!(cut.pixel(1, 1), Some([0, 0, 0, 255]));
+ }
+}
diff --git a/src/crates/concat-host/src/lib.rs b/src/crates/concat-host/src/lib.rs
index 665dc993..07f1d272 100644
--- a/src/crates/concat-host/src/lib.rs
+++ b/src/crates/concat-host/src/lib.rs
@@ -21,6 +21,7 @@
pub mod brush;
pub mod cutout;
pub mod dirs;
+pub mod enhance;
pub mod export;
pub mod jobs;
pub mod logs;
@@ -37,6 +38,7 @@ pub mod titles;
pub use brush::{Brushes, RegionRequest};
pub use cutout::{AnalyseRequest, Cutouts};
pub use dirs::AppDirs;
+pub use enhance::{EnhanceRequest, Enhancers};
pub use jobs::{Job, SingleFlight};
pub use projects::ProjectInfo;
pub use session::{EditorView, Session, SettingsView};
diff --git a/src/crates/concat-project/src/commands/clips.rs b/src/crates/concat-project/src/commands/clips.rs
index 36d8bcc2..1fba3831 100644
--- a/src/crates/concat-project/src/commands/clips.rs
+++ b/src/crates/concat-project/src/commands/clips.rs
@@ -319,6 +319,50 @@ pub(super) fn apply(
})
}
+ Command::ReplaceClipMedia { clip_id, item } => {
+ if project.active().clip(&clip_id).is_none() {
+ return Ok(Outcome::default());
+ }
+ // The copy's bin entry, or the one already there for its path:
+ // two clips of one file enhanced in turn share one copy.
+ let media_id = match project.media.iter().find(|media| media.path == item.path) {
+ Some(existing) => existing.id.clone(),
+ None => {
+ let id = mint.next("m");
+ project.media.push(MediaItem {
+ id: id.clone(),
+ path: item.path,
+ name: item.name,
+ duration: item.duration,
+ kind: item.kind,
+ width: item.width,
+ height: item.height,
+ frame_rate: item.frame_rate,
+ frame_rate_fraction: item.frame_rate_fraction,
+ video_codec: item.video_codec,
+ audio_codec: item.audio_codec,
+ has_audio: item.has_audio,
+ audio_tracks: item.audio_tracks,
+ placeholder: false,
+ extra: Default::default(),
+ });
+ id
+ }
+ };
+ let timeline = project.active_mut();
+ let Some(index) = timeline.clips.iter().position(|clip| clip.id == clip_id) else {
+ return Ok(Outcome::default());
+ };
+ if timeline.clips[index].media_id == media_id {
+ return Ok(Outcome::default());
+ }
+ timeline.clip_at_mut(index).media_id = media_id.clone();
+ Ok(Outcome {
+ created_id: Some(media_id),
+ applied: true,
+ })
+ }
+
Command::FreezeFrame {
clip_id,
time,
diff --git a/src/crates/concat-project/src/commands/mod.rs b/src/crates/concat-project/src/commands/mod.rs
index dda39cde..33e4e42f 100644
--- a/src/crates/concat-project/src/commands/mod.rs
+++ b/src/crates/concat-project/src/commands/mod.rs
@@ -481,6 +481,18 @@ pub enum Command {
/// The cut point, in timeline seconds.
time: f64,
},
+ /// Points a clip at another file - the enhanced copy of its media -
+ /// adding the file to the bin first when it is not there. The clip's
+ /// in-point, length, looks and name are kept: the copy stands in for
+ /// the original frame for frame, at the same rate and length. The bin
+ /// keeps the original, since other clips may show it and undo may
+ /// want it back. An unknown clip is a no-op.
+ ReplaceClipMedia {
+ /// The clip to re-point.
+ clip_id: String,
+ /// The probed copy, as described by the host.
+ item: NewMedia,
+ },
/// CapCut-style freeze at `time`: splits `clip_id`, inserts a still of
/// `duration` on the same track, and ripples later clips on that track
/// by `duration`. Video needs a probed `still` (host-extracted jpg);
@@ -882,6 +894,7 @@ impl Command {
Command::TrimClip { delta, .. } => bad([*delta]),
Command::SplitClips { time, .. } => bad([*time]),
Command::FreezeFrame { time, duration, .. } => bad([*time]) || bad(*duration),
+ Command::ReplaceClipMedia { item, .. } => bad(item.duration) || bad(item.frame_rate),
Command::UpdateClip { patch, .. } => {
bad(patch.volume)
|| bad(patch.fade_in)
@@ -992,6 +1005,7 @@ pub fn apply(
| Command::TrimClip { .. }
| Command::SplitClips { .. }
| Command::FreezeFrame { .. }
+ | Command::ReplaceClipMedia { .. }
| Command::MergeClips { .. }
| Command::RemoveClips { .. }) => clips::apply(project, mint, command),
command @ (Command::UpdateClip { .. }
diff --git a/src/crates/concat-project/src/lib.rs b/src/crates/concat-project/src/lib.rs
index 1cc980b8..fed73fe3 100644
--- a/src/crates/concat-project/src/lib.rs
+++ b/src/crates/concat-project/src/lib.rs
@@ -617,6 +617,110 @@ mod tests {
assert_eq!(editor.project().media.len(), 1);
}
+ #[test]
+ fn replace_clip_media_points_the_clip_at_the_copy_and_keeps_the_original() {
+ let (mut editor, media_id, clip_id) = fixture();
+ let copy = NewMedia {
+ path: "/project/cache/enhance-1-2x.mp4".into(),
+ name: "a.mp4 (enhanced)".into(),
+ duration: Some(10.0),
+ kind: MediaKind::Video,
+ width: Some(3840),
+ height: Some(2160),
+ frame_rate: Some(30.0),
+ frame_rate_fraction: Some("30/1".into()),
+ video_codec: Some("h264".into()),
+ audio_codec: Some("aac".into()),
+ has_audio: true,
+ audio_tracks: Vec::new(),
+ };
+ let outcome = editor
+ .apply(Command::ReplaceClipMedia {
+ clip_id: clip_id.clone(),
+ item: copy.clone(),
+ })
+ .expect("replaces");
+ assert!(outcome.applied);
+ let copy_id = outcome.created_id.expect("the copy's id");
+ assert_ne!(copy_id, media_id);
+ let clip = editor.project().active().clip(&clip_id).expect("clip");
+ assert_eq!(clip.media_id, copy_id);
+ assert_eq!(clip.source_start, 0.0, "the in-point is kept");
+ assert_eq!(
+ editor.project().media.len(),
+ 2,
+ "the original stays in the bin"
+ );
+ assert_eq!(
+ editor.project().media_by_id(&copy_id).expect("copy").width,
+ Some(3840)
+ );
+
+ // The same copy again changes nothing and mints nothing.
+ let again = editor
+ .apply(Command::ReplaceClipMedia {
+ clip_id: clip_id.clone(),
+ item: copy.clone(),
+ })
+ .expect("no-op");
+ assert!(!again.applied);
+ assert_eq!(editor.project().media.len(), 2);
+
+ // A clip nobody has is a no-op that adds nothing to the bin.
+ let nobody = editor
+ .apply(Command::ReplaceClipMedia {
+ clip_id: "c999".into(),
+ item: NewMedia {
+ path: "/elsewhere.mp4".into(),
+ ..copy
+ },
+ })
+ .expect("no-op");
+ assert!(!nobody.applied);
+ assert_eq!(editor.project().media.len(), 2);
+
+ // A non-finite number in the copy's facts is refused, like an import's.
+ let bad = NewMedia {
+ duration: Some(f64::NAN),
+ path: "/nan.mp4".into(),
+ ..NewMedia {
+ path: String::new(),
+ name: String::new(),
+ duration: None,
+ kind: MediaKind::Video,
+ width: None,
+ height: None,
+ frame_rate: None,
+ frame_rate_fraction: None,
+ video_codec: None,
+ audio_codec: None,
+ has_audio: false,
+ audio_tracks: Vec::new(),
+ }
+ };
+ assert!(
+ editor
+ .apply(Command::ReplaceClipMedia {
+ clip_id: clip_id.clone(),
+ item: bad,
+ })
+ .is_err()
+ );
+
+ // Undo puts the clip back on the original; the copy stays in the
+ // bin, harmless, for redo.
+ editor.undo();
+ assert_eq!(
+ editor
+ .project()
+ .active()
+ .clip(&clip_id)
+ .expect("clip")
+ .media_id,
+ media_id
+ );
+ }
+
#[test]
fn freeze_frame_holds_a_second_and_ripples_the_tail() {
let (mut editor, media_id, clip_id) = fixture();
diff --git a/src/crates/concat-vision/src/enhance.rs b/src/crates/concat-vision/src/enhance.rs
new file mode 100644
index 00000000..9d76a767
--- /dev/null
+++ b/src/crates/concat-vision/src/enhance.rs
@@ -0,0 +1,379 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Enhance: a picture restored and enlarged by a model.
+//!
+//! The model is Real-ESRGAN's compact general network: a frame in, the
+//! same frame four times the size out, with its noise, blocking and
+//! softness gone. It reads any size in principle, but a whole frame at
+//! once is more memory than a phone has and a shape the accelerators
+//! compile for badly, so the frame is cut into tiles of one fixed size,
+//! each run with a margin of its neighbours around it so the seams do
+//! not show, and the answers are pasted back. Everything about the tiling
+//! is a pure function here, tested without a model; only [`Enhancer`]
+//! needs one, and the `infer` feature that brings the runtime.
+//!
+//! Four times is more than a clip usually wants - a phone's 1080p is
+//! plenty at 4K, and 8K is a file nobody can play - so [`factor_for`]
+//! picks two, or one when the picture is already large, and the four-times
+//! answer is averaged down to it. Averaging down from the restored picture
+//! keeps the restoration and drops only the size.
+
+use concat_core::Frame;
+use concat_core::frame::BYTES_PER_PIXEL;
+
+/// How much larger the model's answer is than what it read.
+pub const SCALE: u32 = 4;
+/// The square of source pixels one run restores.
+pub const CORE: u32 = 256;
+/// The margin of neighbouring pixels read around a tile and thrown away
+/// after, so the tile's edge is restored knowing what is past it.
+pub const PAD: u32 = 16;
+/// The square the model actually reads: the tile and its margin.
+pub const WINDOW: u32 = CORE + 2 * PAD;
+/// The longest side Enhance will produce: past this the picture is
+/// restored at its own size rather than enlarged.
+pub const LONGEST_SIDE: u32 = 4096;
+
+/// One tile: a rectangle of source pixels restored in one run. Every tile
+/// but the last in a row or column is [`CORE`] square.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Tile {
+ /// Left edge, in source pixels.
+ pub x: u32,
+ /// Top edge, in source pixels.
+ pub y: u32,
+ /// Width, in source pixels: `CORE`, or what is left of the row.
+ pub width: u32,
+ /// Height, in source pixels: `CORE`, or what is left of the column.
+ pub height: u32,
+}
+
+/// The tiles of a `width` by `height` picture, row by row: they cover it
+/// exactly, and no pixel is in two. Empty for an empty picture.
+pub fn tiles(width: u32, height: u32) -> Vec<Tile> {
+ let mut tiles = Vec::new();
+ let mut y = 0;
+ while y < height {
+ let mut x = 0;
+ while x < width {
+ tiles.push(Tile {
+ x,
+ y,
+ width: CORE.min(width - x),
+ height: CORE.min(height - y),
+ });
+ x += CORE;
+ }
+ y += CORE;
+ }
+ tiles
+}
+
+/// How many times larger a `width` by `height` picture comes out: twice,
+/// unless that would pass [`LONGEST_SIDE`], and then once, restored at
+/// its own size.
+pub fn factor_for(width: u32, height: u32) -> u32 {
+ if width.max(height) * 2 > LONGEST_SIDE {
+ 1
+ } else {
+ 2
+ }
+}
+
+/// The model's input for one tile: the [`WINDOW`] square around it as
+/// three planes of `0..=1`, red then green then blue, row-major. Past the
+/// picture's edge the edge pixel repeats, which is what a restoration
+/// network expects to see there.
+pub fn window_planes(frame: &Frame, tile: Tile) -> Vec<f32> {
+ let (fw, fh) = (frame.width() as i64, frame.height() as i64);
+ let pixels = frame.pixels();
+ let side = WINDOW as usize;
+ let mut planes = vec![0f32; 3 * side * side];
+ for wy in 0..side {
+ let sy = (tile.y as i64 + wy as i64 - PAD as i64).clamp(0, fh - 1) as usize;
+ for wx in 0..side {
+ let sx = (tile.x as i64 + wx as i64 - PAD as i64).clamp(0, fw - 1) as usize;
+ let at = (sy * fw as usize + sx) * BYTES_PER_PIXEL;
+ for c in 0..3 {
+ planes[(c * side + wy) * side + wx] = f32::from(pixels[at + c]) / 255.0;
+ }
+ }
+ }
+ planes
+}
+
+/// The model's answer for one tile - three planes of `WINDOW * SCALE`
+/// square, red then green then blue - pasted into `out`, which is the
+/// source picture's size times [`SCALE`]. The margin is dropped: only the
+/// tile's own pixels, enlarged, land. Values past `0..=1` are clamped.
+pub fn paste(out: &mut Frame, tile: Tile, answer: &[f32]) -> Result<(), String> {
+ let side = (WINDOW * SCALE) as usize;
+ if answer.len() != 3 * side * side {
+ return Err(format!(
+ "enhance: the model answered {} values for a tile, not {}",
+ answer.len(),
+ 3 * side * side
+ ));
+ }
+ let ow = out.width() as usize;
+ let pixels = out.pixels_mut();
+ let (tw, th) = (
+ (tile.width * SCALE) as usize,
+ (tile.height * SCALE) as usize,
+ );
+ let (ox, oy) = ((tile.x * SCALE) as usize, (tile.y * SCALE) as usize);
+ let margin = (PAD * SCALE) as usize;
+ for y in 0..th {
+ let ay = y + margin;
+ for x in 0..tw {
+ let ax = x + margin;
+ let at = ((oy + y) * ow + ox + x) * BYTES_PER_PIXEL;
+ for c in 0..3 {
+ let value = answer[(c * side + ay) * side + ax];
+ pixels[at + c] = (value.clamp(0.0, 1.0) * 255.0 + 0.5) as u8;
+ }
+ pixels[at + 3] = 255;
+ }
+ }
+ Ok(())
+}
+
+/// `frame` averaged down by `by` in each direction: every `by` by `by`
+/// block becomes one pixel. `by` of 1 is a copy. A picture whose sides
+/// are not multiples of `by` loses the remainder at the right and bottom.
+pub fn downscale(frame: &Frame, by: u32) -> Frame {
+ if by <= 1 {
+ return frame.clone();
+ }
+ let (fw, fh) = (frame.width() as usize, frame.height() as usize);
+ let by = by as usize;
+ let (ow, oh) = (fw / by, fh / by);
+ let source = frame.pixels();
+ let mut pixels = vec![0u8; ow * oh * BYTES_PER_PIXEL];
+ let count = (by * by) as u32;
+ for y in 0..oh {
+ for x in 0..ow {
+ let mut sum = [0u32; 4];
+ for dy in 0..by {
+ for dx in 0..by {
+ let at = ((y * by + dy) * fw + x * by + dx) * BYTES_PER_PIXEL;
+ for c in 0..4 {
+ sum[c] += u32::from(source[at + c]);
+ }
+ }
+ }
+ let at = (y * ow + x) * BYTES_PER_PIXEL;
+ for c in 0..4 {
+ pixels[at + c] = ((sum[c] + count / 2) / count) as u8;
+ }
+ }
+ }
+ Frame::from_rgba(ow as u32, oh as u32, pixels).expect("sides and bytes agree")
+}
+
+/// The model, loaded once and run tile by tile.
+#[cfg(feature = "infer")]
+pub struct Enhancer {
+ model: std::sync::Mutex<crate::runtime::Model>,
+}
+
+#[cfg(feature = "infer")]
+impl Enhancer {
+ /// Loads the model from its file; see [`crate::models::ModelId::Enhance`].
+ pub fn load(file: &std::path::Path) -> Result<Enhancer, String> {
+ Ok(Enhancer {
+ model: std::sync::Mutex::new(crate::runtime::Model::from_file(file)?),
+ })
+ }
+
+ /// `frame` restored at [`SCALE`] times its size.
+ pub fn upscale(&self, frame: &Frame) -> Result<Frame, String> {
+ let (width, height) = (frame.width(), frame.height());
+ if width == 0 || height == 0 {
+ return Err("enhance: an empty picture".to_owned());
+ }
+ let mut out = Frame::black(width * SCALE, height * SCALE);
+ let side = WINDOW as usize;
+ let mut model = self.model.lock().map_err(|_| "enhance: model poisoned")?;
+ for tile in tiles(width, height) {
+ let planes = window_planes(frame, tile);
+ let answer = model.run(
+ vec![crate::runtime::Input {
+ name: "input",
+ dims: vec![1, 3, side, side],
+ data: planes.into(),
+ }],
+ &["output"],
+ )?;
+ paste(&mut out, tile, &answer[0].data)?;
+ }
+ Ok(out)
+ }
+
+ /// `frame` restored and enlarged `factor` times, `factor` being 1, 2
+ /// or 4: the model's answer, averaged down to size.
+ pub fn enhance(&self, frame: &Frame, factor: u32) -> Result<Frame, String> {
+ let factor = factor.clamp(1, SCALE);
+ let large = self.upscale(frame)?;
+ Ok(downscale(&large, SCALE / factor))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn picture(width: u32, height: u32, at: impl Fn(u32, u32) -> [u8; 4]) -> Frame {
+ let mut pixels = Vec::with_capacity((width * height) as usize * BYTES_PER_PIXEL);
+ for y in 0..height {
+ for x in 0..width {
+ pixels.extend_from_slice(&at(x, y));
+ }
+ }
+ Frame::from_rgba(width, height, pixels).expect("sized")
+ }
+
+ #[test]
+ fn tiles_cover_the_picture_once_and_exactly() {
+ assert!(tiles(0, 0).is_empty());
+ assert!(tiles(10, 0).is_empty());
+ assert_eq!(
+ tiles(1, 1),
+ vec![Tile {
+ x: 0,
+ y: 0,
+ width: 1,
+ height: 1
+ }]
+ );
+ for (width, height) in [(1920, 1080), (256, 256), (257, 255), (3840, 2160), (7, 300)] {
+ let all = tiles(width, height);
+ let mut seen = vec![0u8; (width * height) as usize];
+ for tile in &all {
+ assert!(tile.width >= 1 && tile.width <= CORE, "{tile:?}");
+ assert!(tile.height >= 1 && tile.height <= CORE, "{tile:?}");
+ assert!(tile.x + tile.width <= width && tile.y + tile.height <= height);
+ for y in tile.y..tile.y + tile.height {
+ for x in tile.x..tile.x + tile.width {
+ seen[(y * width + x) as usize] += 1;
+ }
+ }
+ }
+ assert!(
+ seen.iter().all(|&n| n == 1),
+ "{width}x{height}: a pixel missed or doubled"
+ );
+ let full = all
+ .iter()
+ .filter(|t| t.width == CORE && t.height == CORE)
+ .count();
+ assert_eq!(
+ full,
+ ((width / CORE) * (height / CORE)) as usize,
+ "{width}x{height}"
+ );
+ }
+ }
+
+ #[test]
+ fn twice_unless_that_would_be_more_than_anyone_can_play() {
+ assert_eq!(factor_for(1920, 1080), 2);
+ assert_eq!(factor_for(1080, 1920), 2);
+ assert_eq!(factor_for(2048, 1152), 2);
+ assert_eq!(factor_for(2049, 1152), 1);
+ assert_eq!(factor_for(3840, 2160), 1);
+ assert_eq!(factor_for(0, 0), 2);
+ }
+
+ #[test]
+ fn a_window_is_the_tile_with_its_neighbours_and_the_edge_repeated_past_the_picture() {
+ // A 300 by 300 picture whose red is x and green is y, so any pixel
+ // says where it came from.
+ let frame = picture(300, 300, |x, y| [x as u8, y as u8, 7, 255]);
+ let tile = tiles(300, 300)[0];
+ let planes = window_planes(&frame, tile);
+ let side = WINDOW as usize;
+ assert_eq!(planes.len(), 3 * side * side);
+ let red = |wx: usize, wy: usize| (planes[wy * side + wx] * 255.0).round() as u32;
+ let green = |wx: usize, wy: usize| (planes[(side + wy) * side + wx] * 255.0).round() as u32;
+ let blue =
+ |wx: usize, wy: usize| (planes[(2 * side + wy) * side + wx] * 255.0).round() as u32;
+ // Inside the margin, the edge pixel repeats.
+ assert_eq!((red(0, 0), green(0, 0)), (0, 0));
+ assert_eq!((red(5, 9), green(5, 9)), (0, 0));
+ // The tile's own first pixel sits past the margin.
+ assert_eq!(
+ (
+ red(PAD as usize, PAD as usize),
+ green(PAD as usize, PAD as usize)
+ ),
+ (0, 0)
+ );
+ assert_eq!(red(PAD as usize + 10, PAD as usize), 10);
+ assert_eq!(green(PAD as usize, PAD as usize + 20), 20);
+ // The right margin reads the neighbouring tile, not a repeat.
+ assert_eq!(red(side - 1, PAD as usize), (CORE + PAD - 1) & 0xff);
+ assert_eq!(blue(3, 3), 7);
+
+ // The last tile in the row: its right margin is past the picture.
+ let last = *tiles(300, 300).last().unwrap();
+ assert_eq!(
+ (last.x, last.y, last.width, last.height),
+ (256, 256, 44, 44)
+ );
+ let planes = window_planes(&frame, last);
+ let red = |wx: usize, wy: usize| (planes[wy * side + wx] * 255.0).round() as u32;
+ assert_eq!(red(side - 1, PAD as usize), 299 & 0xff);
+ }
+
+ #[test]
+ fn a_pasted_answer_lands_on_the_tiles_own_pixels_only_and_is_clamped() {
+ let mut out = Frame::black(300 * SCALE, 300 * SCALE);
+ let side = (WINDOW * SCALE) as usize;
+ // An answer that is 1.0 red everywhere, 2.0 (over) green, -1 blue.
+ let mut answer = vec![0f32; 3 * side * side];
+ answer[..side * side].fill(1.0);
+ answer[side * side..2 * side * side].fill(2.0);
+ answer[2 * side * side..].fill(-1.0);
+ let tile = Tile {
+ x: 256,
+ y: 0,
+ width: 44,
+ height: 256,
+ };
+ paste(&mut out, tile, &answer).expect("fits");
+ assert_eq!(out.pixel(256 * 4, 0), Some([255, 255, 0, 255]));
+ assert_eq!(
+ out.pixel(300 * 4 - 1, 256 * 4 - 1),
+ Some([255, 255, 0, 255])
+ );
+ // The neighbouring tile's pixels are untouched.
+ assert_eq!(out.pixel(256 * 4 - 1, 0), Some([0, 0, 0, 255]));
+ assert_eq!(out.pixel(256 * 4, 256 * 4), Some([0, 0, 0, 255]));
+ // A short answer is refused, not read past.
+ assert!(paste(&mut out, tile, &answer[1..]).is_err());
+ }
+
+ #[test]
+ fn averaging_down_takes_the_mean_and_drops_the_remainder() {
+ let frame = picture(5, 3, |x, y| {
+ if (x + y) % 2 == 0 {
+ [200, 0, 100, 255]
+ } else {
+ [0, 200, 100, 255]
+ }
+ });
+ let half = downscale(&frame, 2);
+ assert_eq!((half.width(), half.height()), (2, 1));
+ assert_eq!(half.pixel(0, 0), Some([100, 100, 100, 255]));
+ assert_eq!(half.pixel(1, 0), Some([100, 100, 100, 255]));
+ let same = downscale(&frame, 1);
+ assert_eq!((same.width(), same.height()), (5, 3));
+ assert_eq!(same.pixels(), frame.pixels());
+ let quarter = downscale(&Frame::black(8, 8), 4);
+ assert_eq!((quarter.width(), quarter.height()), (2, 2));
+ let none = downscale(&Frame::black(3, 3), 4);
+ assert_eq!((none.width(), none.height()), (0, 0));
+ }
+}
diff --git a/src/crates/concat-vision/src/lib.rs b/src/crates/concat-vision/src/lib.rs
index a89c0e8e..6708bdbd 100644
--- a/src/crates/concat-vision/src/lib.rs
+++ b/src/crates/concat-vision/src/lib.rs
@@ -42,6 +42,7 @@
pub mod apply;
#[cfg(feature = "infer")]
pub mod brush;
+pub mod enhance;
pub mod mask;
pub mod models;
#[cfg(feature = "infer")]
@@ -54,6 +55,8 @@ pub mod strokes;
pub use apply::{Mapping, cut, highlight};
#[cfg(feature = "infer")]
pub use brush::{Brush, Embedding};
+#[cfg(feature = "infer")]
+pub use enhance::Enhancer;
pub use mask::Mask;
pub use models::ModelId;
#[cfg(feature = "infer")]
diff --git a/src/crates/concat-vision/src/models.rs b/src/crates/concat-vision/src/models.rs
index eabf8e60..4514084f 100644
--- a/src/crates/concat-vision/src/models.rs
+++ b/src/crates/concat-vision/src/models.rs
@@ -34,6 +34,9 @@ pub enum ModelId {
/// SlimSAM's prompt encoder and mask decoder: points on the picture
/// into the mask of what is under them.
BrushDecoder,
+ /// Real-ESRGAN's compact general model: a picture four times its size
+ /// with its noise and compression gone, what Enhance is made of.
+ Enhance,
}
/// One model as the table describes it.
@@ -57,7 +60,7 @@ pub struct ModelSpec {
}
/// Every downloadable model.
-pub const MODELS: [ModelSpec; 4] = [
+pub const MODELS: [ModelSpec; 5] = [
ModelSpec {
id: ModelId::Person,
file: "rvm-mobilenetv3.onnx",
@@ -90,6 +93,14 @@ pub const MODELS: [ModelSpec; 4] = [
sha256: "",
licence: "Apache-2.0",
},
+ ModelSpec {
+ id: ModelId::Enhance,
+ file: "realesr-general-x4v3.onnx",
+ upstream: "https://huggingface.co/CoderViking/realesr-general-x4v3-onnx/resolve/main/realesr-general-x4v3.onnx",
+ bytes: 4_866_417,
+ sha256: "1940a93ee08283a0a7286183186357b1688fe9fa8ede74604b424586aaddf112",
+ licence: "BSD-3-Clause",
+ },
];
impl ModelId {
@@ -108,6 +119,7 @@ impl ModelId {
ModelId::Object => "isnet-1024",
ModelId::BrushEncoder => "slimsam-77-encoder",
ModelId::BrushDecoder => "slimsam-77-decoder",
+ ModelId::Enhance => "realesr-general-x4v3",
}
}
}
diff --git a/src/crates/concat/src/host.rs b/src/crates/concat/src/host.rs
index da01996f..cab5ebfd 100644
--- a/src/crates/concat/src/host.rs
+++ b/src/crates/concat/src/host.rs
@@ -46,6 +46,8 @@ pub struct Host {
pub cutouts: Arc<concat_host::Cutouts>,
/// The brush model, and the regions it reads under smart strokes.
pub brushes: Arc<concat_host::Brushes>,
+ /// The restoration model, and the enhanced copies it writes.
+ pub enhancers: Arc<concat_host::Enhancers>,
/// The Concat API on a socket, while the Remote page has it on. Its
/// own sessions, apart from the window's: a caller edits projects of
/// its own, not the one on screen.
@@ -64,6 +66,7 @@ impl Host {
titles: concat_host::Titles::new(&dirs),
cutouts: Arc::new(concat_host::Cutouts::new(&dirs.data)),
brushes: Arc::new(concat_host::Brushes::new(&dirs.data)),
+ enhancers: Arc::new(concat_host::Enhancers::new(&dirs.data)),
dirs,
playback: Playback::start(Arc::new(Events))?,
monitor: match gpu {
diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rs
index 8db7180c..4fc39170 100644
--- a/src/crates/concat/src/studio.rs
+++ b/src/crates/concat/src/studio.rs
@@ -33,7 +33,8 @@ use concat_effects::Catalogue;
use concat_effects::manifest::Kind as PackageKind;
use concat_host::playback::ClipSpec;
use concat_host::{
- AnalyseRequest, Cutouts, ProjectInfo, RegionRequest, Session, media, projects, templates,
+ AnalyseRequest, Cutouts, EnhanceRequest, ProjectInfo, RegionRequest, Session, media, projects,
+ templates,
};
use concat_media::{DecodeOptions, Decoder, FrameSource, Pyramid, jpeg};
use concat_project::commands::{ClipMove, ClipPatch, TrackFlag, TrimEdge};
@@ -684,6 +685,9 @@ pub struct Studio {
pub painting: bool,
/// The mask analyses running, by media id, and how far each has got.
cutout_jobs: HashMap<String, (bool, f32)>,
+ /// Enhance at work, by clip id: whether the model is still
+ /// downloading, and how far along. One at a time, like every long job.
+ enhance_jobs: HashMap<String, (bool, f32)>,
/// The smart stroke being read, by the same name, while one is.
region_job: Option<String>,
/// The last smart stroke as the stage drew it, kept on screen from the
@@ -1327,6 +1331,7 @@ impl Studio {
brush_size: 0.06,
painting: true,
cutout_jobs: HashMap::new(),
+ enhance_jobs: HashMap::new(),
region_job: None,
pending_stroke: None,
host,
@@ -4366,6 +4371,129 @@ impl Studio {
);
}
+ // ── enhance ──
+ //
+ // The restoration model reads a clip's file frame by frame and writes
+ // an enhanced copy into the project's cache; the clip is then pointed
+ // at the copy, as one undo step, and shows it wherever it showed the
+ // original. The window's part is to start the job, keep the readout
+ // moving, and re-point the clip when the copy is whole.
+
+ /// Enhances the media of clip `id`: starts the job, unless one is
+ /// running, and says so.
+ pub fn enhance_clip(&mut self, id: &str) {
+ if !self.enhance_jobs.is_empty() || self.host.enhancers.is_busy() {
+ self.notify(&t("Enhance is already at work; one clip at a time"), true);
+ return;
+ }
+ let Some(clip) = self.clip(id).cloned() else {
+ return;
+ };
+ if clip.kind != model::ClipKind::Video && clip.kind != model::ClipKind::Image {
+ return;
+ }
+ let Some(media) = self
+ .project()
+ .media
+ .iter()
+ .find(|item| item.id == clip.media_id)
+ .cloned()
+ else {
+ return;
+ };
+ let Some(session) = self.session.as_ref() else {
+ return;
+ };
+ let project = std::path::PathBuf::from(session.path());
+ let (width, height) = (media.width.unwrap_or(0), media.height.unwrap_or(0));
+ if width == 0 || height == 0 {
+ self.notify(&t("This file has no picture to enhance"), true);
+ return;
+ }
+ let factor = concat_vision::enhance::factor_for(width, height);
+ let still = clip.kind == model::ClipKind::Image;
+ let Some(target) = concat_host::enhance::target_for(&project, &media.path, factor, still)
+ else {
+ self.notify(&tf("Could not read {0}", &[&media.path]), true);
+ return;
+ };
+ let request = EnhanceRequest {
+ media_path: media.path.clone(),
+ still,
+ factor,
+ target,
+ };
+ let clip_id = clip.id.clone();
+ self.enhance_jobs.insert(clip_id.clone(), (false, 0.0));
+ self.notify(
+ &tf(
+ "Enhancing {0}: {1}× on its way, frame by frame",
+ &[&media.name, &factor],
+ ),
+ false,
+ );
+ let enhancers = Arc::clone(&self.host.enhancers);
+ spawn(
+ move || {
+ let mut last = (false, -1.0f32);
+ let reporting = clip_id.clone();
+ let result = enhancers.enhance(&request, &mut |progress| {
+ let now = match progress {
+ concat_host::enhance::Progress::Fetching { received, total } => {
+ (true, received as f32 / total.max(1) as f32)
+ }
+ concat_host::enhance::Progress::Analysing(fraction) => (false, fraction),
+ };
+ if now.0 != last.0 || now.1 - last.1 >= 0.01 {
+ last = now;
+ let id = reporting.clone();
+ on_ui(move |studio, _, _| {
+ if let Some(held) = studio.enhance_jobs.get_mut(&id) {
+ *held = now;
+ }
+ });
+ }
+ });
+ (clip_id, result)
+ },
+ |studio, _, _, (clip_id, result)| {
+ studio.enhance_jobs.remove(&clip_id);
+ match result {
+ Ok(path) => studio.adopt_enhanced(&clip_id, &path),
+ Err(error) if error.contains("cancelled") => {}
+ Err(error) => studio.notify(&tf("Enhance: {0}", &[&error]), true),
+ }
+ },
+ );
+ }
+
+ /// Points clip `id` at the enhanced copy at `path`, probed the way an
+ /// import is, and shows it.
+ fn adopt_enhanced(&mut self, id: &str, path: &std::path::Path) {
+ let summary = match media::probe(&path.to_string_lossy()) {
+ Ok(summary) => summary,
+ Err(error) => {
+ self.notify(&tf("Enhance: {0}", &[&error]), true);
+ return;
+ }
+ };
+ let mut item = summary.to_new_media();
+ // The bin names the copy after the original, marked, rather than
+ // after its cache file.
+ if let Some(original) = self
+ .clip(id)
+ .and_then(|clip| self.project().media.iter().find(|m| m.id == clip.media_id))
+ {
+ item.name = format!("{} (enhanced)", original.name);
+ }
+ self.apply(Command::ReplaceClipMedia {
+ clip_id: id.to_owned(),
+ item,
+ });
+ self.request_preview();
+ self.notify(&t("Enhanced; the clip now shows the copy"), false);
+ }
+
/// The Mode row: 0 off, 1 automatic, 2 custom. The chroma rows are the
/// inspector's own business, on the chain.
pub fn cutout_mode(&mut self, mode: i32) {
@@ -4953,6 +5081,8 @@ impl Studio {
self.flush_commit();
self.pause();
self.host.cutouts.cancel();
+ self.host.enhancers.cancel();
+ self.enhance_jobs.clear();
self.cutout_jobs.clear();
self.region_job = None;
if let Some(session) = self.session.as_mut() {
@@ -5911,6 +6041,15 @@ impl Studio {
cutout_progress: analysis.map(|(_, fraction)| fraction).unwrap_or(-1.0),
cutout_fetching: analysis.is_some_and(|(fetching, _)| fetching),
cutout_empty: self.cutout_empty(clip),
+ enhance_progress: self
+ .enhance_jobs
+ .get(&clip.id)
+ .map(|(_, fraction)| *fraction)
+ .unwrap_or(-1.0),
+ enhance_fetching: self
+ .enhance_jobs
+ .get(&clip.id)
+ .is_some_and(|(fetching, _)| *fetching),
}
}
@@ -6323,6 +6462,18 @@ impl Studio {
&& !locked
&& (clip.kind == model::ClipKind::Video || clip.kind == model::ClipKind::Image),
),
+ // The picture restored and enlarged by the model, as a copy
+ // the clip then shows. Greyed while one is being written,
+ // since the job runs one at a time.
+ action(
+ "enhance",
+ t("Enhance"),
+ Glyph::Sparkle,
+ "",
+ !locked
+ && (clip.kind == model::ClipKind::Video || clip.kind == model::ClipKind::Image)
+ && self.enhance_jobs.is_empty(),
+ ),
rule(),
];
// One slot, two verbs: the sound is either on its picture or off it.
@@ -6782,6 +6933,7 @@ impl Studio {
self.split_at(at, true);
}
"freeze" => self.freeze_at_playhead(),
+ "enhance" => self.enhance_clip(id),
"detach" => {
self.apply(Command::DetachAudio {
clip_id: id.to_owned(),
diff --git a/src/crates/concat/ui/inspector/clip-inspector.slint b/src/crates/concat/ui/inspector/clip-inspector.slint
index 45becab4..c3fb144c 100644
--- a/src/crates/concat/ui/inspector/clip-inspector.slint
+++ b/src/crates/concat/ui/inspector/clip-inspector.slint
@@ -786,6 +786,23 @@ component AnalysisRow inherits Row {
}
}
+/// Enhance at work on the clip's media: the model on its way, then the
+/// frames, one by one, into the copy the clip will show.
+component EnhanceRow inherits Row {
+ in property <float> progress;
+ in property <bool> fetching;
+
+ label: I18n.t("Enhance");
+ Text {
+ text: root.fetching
+ ? I18n.t1("Fetching the model… {0}%", "\{Math.round(root.progress * 100)}")
+ : I18n.t1("Enhancing… {0}%", "\{Math.round(root.progress * 100)}");
+ color: Theme.fg-muted;
+ font-size: Theme.fs;
+ vertical-alignment: center;
+ }
+}
+
// A rule between blocks inside one section, inset to the section's padding
// where a Section's own hairline runs edge to edge.
component Hairline inherits HorizontalLayout {
@@ -1424,6 +1441,10 @@ export component AudioPanel inherits VerticalLayout {
? root.audio-tracks[root.selected.audio-track] : "";
}
if source.open: SectionBody {
+ if root.selected.enhance-progress >= 0: EnhanceRow {
+ progress: root.selected.enhance-progress;
+ fetching: root.selected.enhance-fetching;
+ }
Row {
label: I18n.t("Audio track");
Select {
diff --git a/src/crates/concat/ui/timeline/model.slint b/src/crates/concat/ui/timeline/model.slint
index 4b30944c..b011c950 100644
--- a/src/crates/concat/ui/timeline/model.slint
+++ b/src/crates/concat/ui/timeline/model.slint
@@ -325,6 +325,12 @@ export struct SelectedClipData {
cutout-fetching: bool,
cutout-empty: bool,
+ // Enhance writing this clip's enhanced copy: -1 when it is not, else
+ // how far along, and whether that is the model downloading rather
+ // than frames being restored
+ enhance-progress: float,
+ enhance-fetching: bool,
+
// the overlay, when this is a text clip
content: string,
font-family: string,