Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

refactor(lightcraft-engine): split denoise.rs into denoise/

aiongg8942984c3532
diff --git a/crates/engine/src/denoise.rs b/crates/engine/src/denoise.rs
index 89af8463a..44ef3eef7 100644
--- a/crates/engine/src/denoise.rs
+++ b/crates/engine/src/denoise.rs
@@ -36,1611 +36,32 @@ use crate::merge::ByteReader;
use crate::model_download::Downloads;
use crate::{Selection, Session};
-/// Extension of a product file.
-pub const EXT: &str = "lcdn";
-/// Where the products of a library are kept, inside the library's folder (safe to delete, like `thumbs/`).
-pub const DIR: &str = "denoise";
-/// What the cache may take up unless the user says otherwise (GB). A product is about as large as the raw file's
-/// decoded samples, compressed: ~100 MB for 24 megapixels.
-pub const DEFAULT_CACHE_GB: u32 = 20;
-/// Samples at or above this (white = 1) are clipped: the model leaves them as the camera recorded them.
-const CLIP: f32 = 0.99;
-/// Most photos waiting in the queue.
-const MAX_QUEUE: usize = 5000;
-/// Most selected photos the automatic queue looks at in one pump.
-const AUTO_LOOK: usize = 64;
-/// How long a look at the model settings is trusted.
-const SETTINGS_TTL: Duration = Duration::from_secs(2);
-/// Stack of the threads that run the model (the runtime recurses through its graph).
-const STACK: usize = 16 << 20;
-
-// ---------------------------------------------------------------------------------------------------------------------
-// the model
-
-/// Where the model runs (Settings ▸ AI Denoise ▸ Speed; `denoise.settings {runOn}`).
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub enum RunOn {
- /// The graphics card where it is faster than the processor at the pace the work goes at (both are timed on this
- /// computer when the card is set up), else the processor.
- #[default]
- Auto,
- /// The graphics card whenever it can run the model, even where the processor would be faster.
- Gpu,
- /// Always the processor.
- Cpu,
-}
-
-impl RunOn {
- /// `auto`, `gpu` or `cpu`.
- pub fn parse(name: &str) -> Option<RunOn> {
- match name {
- "auto" => Some(RunOn::Auto),
- "gpu" => Some(RunOn::Gpu),
- "cpu" => Some(RunOn::Cpu),
- _ => None,
- }
- }
-
- pub fn name(self) -> &'static str {
- match self {
- RunOn::Auto => "auto",
- RunOn::Gpu => "gpu",
- RunOn::Cpu => "cpu",
- }
- }
-}
-
-/// A loaded denoise model: what runs tiles, and a self-check.
-pub(crate) trait Model: Send + Sync {
- /// What runs a photo's tiles, and how many to run at once, when `threads` may be used and the setting is `run_on`
- /// (a graphics card is kept busy by a few). The first time the card may be used it is set up and checked here.
- fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize);
- /// Check it does something sensible: the test's result, or why not. Unless `run_on` is the processor the graphics
- /// card is set up and checked too, and the result says where the work will run.
- fn self_test(&self, run_on: RunOn) -> Result<Value, String>;
- /// Conservative scratch bound for an unknown runner.
- fn bytes_per_tile(&self) -> usize {
- 512 * 1024 * 1024
- }
- /// Where the tiles run: `{kind: "gpu", adapter}`, `{kind: "cpu", reason?}` or `{kind: "pending"}` (the card is not
- /// set up yet), with the card's and the processor's time for the check tile (`cardMs`, `cpuMs`) once both were
- /// timed. Never starts anything.
- fn device(&self, _run_on: RunOn) -> Value {
- json!({"kind": "cpu"})
- }
-}
-
-/// Loads the model at a path as its manifest describes it.
-pub(crate) type Loader = Arc<dyn Fn(&Path, &DenoiserManifest) -> Result<Arc<dyn Model>, String> + Send + Sync>;
-
-/// Tiles run at once on a graphics card: enough to keep it fed while others are packed and blended.
-#[cfg(feature = "denoise")]
-const GPU_PARALLEL: usize = 4;
-/// How far the GPU's answer on the check tile may be from the CPU's (of the answer's largest value) before it is not used.
-#[cfg(feature = "denoise")]
-const GPU_AGREE: f32 = 1e-3;
-/// Longest the card may take to set up (build its kernels, run the check tile, time it) before the processor does the
-/// work instead.
-#[cfg(feature = "denoise")]
-const GPU_SETUP_LIMIT: Duration = Duration::from_secs(30);
-/// Tiles the card may get wrong (an error, numbers that are not finite) before it stops being used for the model.
-#[cfg(feature = "denoise")]
-const GPU_MAX_FALLBACKS: usize = 3;
-/// The file beside a model while the graphics card is set up for it (see [`set_up_card`]).
-pub(crate) const GPU_SETUP_MARKER: &str = "gpu-setup.marker";
-/// Why the card is not tried when a set-up marker is found.
-#[cfg(feature = "denoise")]
-const GPU_CRASHED: &str = "LightCraft closed while it was setting up the graphics card for AI Denoise last time";
-/// Why the card is not used when its set-up ran out of time (the start of it).
-#[cfg(feature = "denoise")]
-const GPU_SLOW: &str = "setting up the graphics card took longer than";
-/// Why the card is not used while a set-up that ran out of time is still going.
-#[cfg(feature = "denoise")]
-const GPU_STUCK: &str = "the graphics card is still setting up from before (it took too long)";
-
-/// A set-up that may work another time (a crash, the time limit), unlike one the card or the model cannot do.
-#[cfg(feature = "denoise")]
-fn worth_retrying(why: &str) -> bool {
- why == GPU_CRASHED || why == GPU_STUCK || why.starts_with(GPU_SLOW)
-}
-
-/// The model on our pure-Rust CPU runner, and on the graphics card when there is one that can run it: the card is set up for the
-/// first photo that may use it, checked against the CPU's answer on a check tile and timed against it, and used while
-/// it keeps working; a tile it gets wrong is run on the CPU, so a photo is always finished.
-#[cfg(feature = "denoise")]
-struct Network {
- cpu: lightcraft_denoise::runtime::CpuRunner,
- path: PathBuf,
- tile: usize,
- gpu: OnceLock<Result<GpuSide, String>>,
- /// Tiles the card got wrong (an error, numbers that are not finite) and the CPU ran.
- fallbacks: AtomicUsize,
- /// Tiles at once of the photo started last (0: none yet), to say where the work runs.
- last_threads: AtomicUsize,
- /// Another computer's graphics card (`LIGHTCRAFT_DENOISE_URL` + `LIGHTCRAFT_DENOISE_TOKEN`; see
- /// `docs/denoise.md` → Another computer's graphics card), greeted on first use: it takes tiles while it has a free
- /// slot, on top of this computer's runner.
- remote: Option<RemoteSide>,
- /// Whether this computer's share runs on the card (else the CPU), as chosen for the photo started last.
- local_card: AtomicBool,
-}
-
-#[cfg(feature = "denoise")]
-struct RemoteSide {
- runner: lightcraft_denoise::remote::Remote,
- /// The greeting, and the server's time for the check tile there and back (ms; the faster of two runs).
- greeted: OnceLock<Result<(lightcraft_denoise::remote::ServerInfo, f64), String>>,
- /// Whether the photo started last shares tiles with the server, and if not because it is slower, the two times.
- share: AtomicBool,
- slower: Mutex<Option<(f64, f64)>>,
-}
-
-#[cfg(feature = "denoise")]
-impl RemoteSide {
- /// From the environment, for the model whose file has `sha256` and tiles of `tile` cells.
- fn from_env(sha256: &str, tile: usize) -> Option<RemoteSide> {
- let var = |k: &str| std::env::var(k).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty());
- let address = var("LIGHTCRAFT_DENOISE_URL")?;
- let token = var("LIGHTCRAFT_DENOISE_TOKEN").unwrap_or_default();
- Some(RemoteSide::new(lightcraft_denoise::remote::Remote::new(&address, &token, sha256, tile)))
- }
-
- fn new(runner: lightcraft_denoise::remote::Remote) -> RemoteSide {
- RemoteSide { runner, greeted: OnceLock::new(), share: AtomicBool::new(false), slower: Mutex::new(None) }
- }
-
- /// Greet the server and time a check tile there and back: a server behind a slow link can take seconds a tile, and
- /// handing it tiles a fast local card would finish sooner only makes the photo wait for it.
- fn greet(&self) -> Result<(lightcraft_denoise::remote::ServerInfo, f64), String> {
- let info = self.runner.hello().map_err(|e| e.to_string())?;
- let input = lightcraft_denoise::runtime::check_tile(info.tile);
- let mut best = f64::INFINITY;
- for _ in 0..2 {
- let started = web_time::Instant::now();
- match self.runner.try_run(&input) {
- Some(Ok(_)) => best = best.min(started.elapsed().as_secs_f64() * 1000.0),
- Some(Err(e)) => return Err(format!("the check tile failed: {e}")),
- None => return Err("no free slot for the check tile".into()),
- }
- }
- Ok((info, best))
- }
-
- /// The server when it can take tiles now (greeting it the first time), else why not.
- fn usable(&self, start: bool) -> Result<&lightcraft_denoise::remote::Remote, String> {
- let greeted = if start { Some(self.greeted.get_or_init(|| self.greet())) } else { self.greeted.get() };
- match greeted {
- None => Err("not contacted yet".into()),
- Some(Err(why)) => Err(why.clone()),
- Some(Ok(_)) if !self.runner.healthy() => {
- Err(format!("it failed {} tiles ({})", self.runner.failures(), self.runner.last_error().unwrap_or_default()))
- }
- Some(Ok(_)) => Ok(&self.runner),
- }
- }
-
- /// The server's time for a tile, once greeted.
- fn tile_ms(&self) -> Option<f64> {
- self.greeted.get().and_then(|g| g.as_ref().ok()).map(|g| g.1)
- }
-
- fn describe(&self) -> Value {
- match self.greeted.get() {
- None => json!({"address": self.runner.address(), "state": "pending"}),
- Some(Err(why)) => json!({"address": self.runner.address(), "state": "unavailable", "reason": why}),
- Some(Ok((info, ms))) => {
- let slower = *self.slower.lock().unwrap_or_else(PoisonError::into_inner);
- let state = match () {
- _ if !self.runner.healthy() => "given up",
- _ if slower.is_some() => "slower",
- _ if self.share.load(Ordering::Relaxed) => "used",
- _ => "idle",
- };
- let mut v =
- json!({"address": info.address, "device": info.device, "state": state, "remoteMs": ms, "failures": self.runner.failures()});
- if let (Some((_, local)), Some(o)) = (slower, v.as_object_mut()) {
- o.insert("localMs".into(), json!(local));
- }
- if let (Some(e), Some(o)) = (self.runner.last_error(), v.as_object_mut()) {
- o.insert("lastError".into(), json!(e));
- }
- v
- }
- }
- }
-}
-
-#[cfg(feature = "denoise")]
-struct GpuSide {
- runner: Box<dyn TileRunner + Send>,
- adapter: String,
- /// The check tile on the card, the fastest of a few runs (ms).
- card_ms: f64,
- /// The check tile on the CPU, on one thread (ms).
- cpu_ms: f64,
-}
-
-/// Whether the card (`card_ms` a tile) is faster than the CPU running `threads` tiles at once, from the CPU's time for
-/// one tile alone (`cpu_ms`). Tiles at once share the memory and the cores' second threads, so each one more counts as
-/// half a tile's speed: on a 16-core laptop one tile alone took 1.4 s and sixteen at once finished one every 0.2 s.
-#[cfg(feature = "denoise")]
-fn card_is_faster(card_ms: f64, cpu_ms: f64, threads: usize) -> bool {
- let at_once = 1.0 + 0.5 * threads.saturating_sub(1) as f64;
- card_ms.is_finite() && card_ms <= cpu_ms / at_once
-}
-
-/// Card set-ups of this process that have not finished, by marker (their markers are not a crash's).
-#[cfg(feature = "denoise")]
-static SETTING_UP: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
-
-/// Removes a set-up's marker when the set-up ends, however it ends — but not when the process dies inside the driver.
-#[cfg(feature = "denoise")]
-struct SetUpEnded(PathBuf);
-
-#[cfg(feature = "denoise")]
-impl Drop for SetUpEnded {
- fn drop(&mut self) {
- let _ = std::fs::remove_file(&self.0);
- SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner).retain(|m| m != &self.0);
- }
-}
-
-/// Set the graphics card up with `card` on a thread of its own while `meanwhile` runs here, and give up on the card when
-/// it takes longer than `limit` (a driver building kernels for minutes, or stuck). The `marker` file exists while `card`
-/// runs: one found before it starts was left by a LightCraft that closed during a set-up (a driver crash takes the app
-/// with it), so the card is not tried again until the user chooses where denoise runs (`denoise.settings {runOn}`).
-#[cfg(feature = "denoise")]
-fn set_up_card<T: Send + 'static, R>(
- marker: &Path,
- limit: Duration,
- card: impl FnOnce() -> Result<T, String> + Send + 'static,
- meanwhile: impl FnOnce() -> R,
-) -> Result<(T, R), String> {
- let ended = {
- let mut running = SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner);
- if running.iter().any(|m| m == marker) {
- return Err(GPU_STUCK.into());
- }
- if marker.exists() {
- return Err(GPU_CRASHED.into());
- }
- let secs = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map_or(0, |d| d.as_secs());
- // a marker that cannot be written only loses the crash guard
- let _ = std::fs::write(marker, format!("setting up the graphics card for AI Denoise (unix time {secs})\n"));
- running.push(marker.to_path_buf());
- SetUpEnded(marker.to_path_buf())
- };
- let started = web_time::Instant::now();
- let (tx, rx) = std::sync::mpsc::channel();
- std::thread::Builder::new()
- .name("denoise-gpu-setup".into())
- .stack_size(STACK)
- .spawn(move || {
- let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(card)).unwrap_or_else(|_| Err("the GPU runner gave up".into()));
- drop(ended);
- let _ = tx.send(r);
- })
- .map_err(|e| format!("could not start setting up the graphics card: {e}"))?;
- let here = meanwhile();
- match rx.recv_timeout(limit.saturating_sub(started.elapsed())) {
- Ok(r) => r.map(|t| (t, here)),
- Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(format!("{GPU_SLOW} {} s, so it is not used", limit.as_secs_f64())),
- Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err("the GPU runner gave up".into()),
- }
-}
-
-/// The card's time for one tile once it is warm: the fastest of up to three runs, stopping after about two seconds (a
-/// slow card is slow enough to tell at once).
-#[cfg(feature = "denoise")]
-fn card_tile_ms(runner: &dyn TileRunner, input: &[f32]) -> f64 {
- if runner.run(input).is_err() {
- return f64::INFINITY;
- }
- let began = web_time::Instant::now();
- let mut best = f64::INFINITY;
- for _ in 0..3 {
- let one = web_time::Instant::now();
- if runner.run(input).is_err() {
- break;
- }
- best = best.min(one.elapsed().as_secs_f64() * 1000.0);
- if began.elapsed() > Duration::from_secs(2) {
- break;
- }
- }
- best
-}
-
-/// Set the card up for the model at `path`: kernels built, the check tile run on both and compared, both timed.
-#[cfg(feature = "denoise")]
-fn make_gpu(cpu: &lightcraft_denoise::runtime::CpuRunner, path: &Path, tile: usize) -> Result<GpuSide, String> {
- let input = Arc::new(lightcraft_denoise::runtime::check_tile(tile));
- let (on_card, net) = (input.clone(), cpu.net().clone());
- let card = move || -> Result<_, String> {
- let runner = lightcraft_gpu::nn::runner(&net, tile)?;
- let got = runner.run(&on_card).map_err(|e| format!("the GPU could not run the check tile: {e}"))?;
- let card_ms = card_tile_ms(&runner, &on_card);
- Ok((runner, got, card_ms))
- };
- let on_cpu = || {
- let started = web_time::Instant::now();
- (cpu.run(&input), started.elapsed().as_secs_f64() * 1000.0)
- };
- let ((runner, got, card_ms), (want, cpu_ms)) = set_up_card(&path.with_file_name(GPU_SETUP_MARKER), GPU_SETUP_LIMIT, card, on_cpu)?;
- let want = want.map_err(|e| format!("the CPU could not check the GPU: {e}"))?;
- let scale = want.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-12);
- let worst = want.iter().zip(&got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
- if got.len() != want.len() || !got.iter().all(|v| v.is_finite()) || worst > GPU_AGREE {
- return Err(format!("its answer on the check tile is not the CPU's (off by {worst:.1e} of the largest value): not used"));
- }
- Ok(GpuSide { adapter: runner.adapter(), runner: Box::new(runner), card_ms, cpu_ms })
-}
-
-#[cfg(feature = "denoise")]
-impl Network {
- /// The card when it can be used now, else why not. `start` sets it up when that has not been tried yet; without it
- /// nothing is started.
- fn card(&self, start: bool) -> Result<&GpuSide, String> {
- let side = if start {
- Some(self.gpu.get_or_init(|| {
- let made = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_gpu(&self.cpu, &self.path, self.tile)))
- .unwrap_or_else(|_| Err("the GPU runner gave up".into()));
- if let Err(why) = &made {
- log::info!("denoise: running on the CPU ({why})");
- }
- made
- }))
- } else {
- self.gpu.get()
- };
- let g = match side {
- None => return Err("the graphics card is not set up yet".into()),
- Some(Err(why)) => return Err(why.clone()),
- Some(Ok(g)) => g,
- };
- if let Some(why) = lightcraft_gpu::nn::broken_reason() {
- return Err(format!("the graphics card stopped working ({why})"));
- }
- if self.fallbacks.load(Ordering::Relaxed) >= GPU_MAX_FALLBACKS {
- return Err(format!("{} got {GPU_MAX_FALLBACKS} tiles wrong", g.adapter));
- }
- Ok(g)
- }
-
- /// The card, when a photo made `threads` tiles at a time under `run_on` runs on it; else why the CPU does.
- fn choose(&self, run_on: RunOn, threads: usize, start: bool) -> Result<&GpuSide, String> {
- if run_on == RunOn::Cpu {
- return Err("the processor is chosen in Settings".into());
- }
- let g = self.card(start)?;
- if run_on == RunOn::Auto && !card_is_faster(g.card_ms, g.cpu_ms, threads) {
- return Err(format!("it is faster than {} on this computer", g.adapter));
- }
- Ok(g)
- }
-
- fn fell_back(&self, why: &str) {
- let n = self.fallbacks.fetch_add(1, Ordering::Relaxed) + 1;
- if n == 1 {
- log::warn!("denoise: a tile came back wrong from the GPU ({why}); the CPU runs it");
- } else if n == GPU_MAX_FALLBACKS {
- log::warn!("denoise: the GPU got {n} tiles wrong; the CPU does the rest");
- }
- }
-}
-
-/// A remote server's free slot first (when there is one), then the card when this photo runs on it, with the CPU
-/// running any tile either gets wrong. Handed out for photos that run on the card or have a server to share.
-#[cfg(feature = "denoise")]
-impl TileRunner for Network {
- fn run(&self, input: &[f32]) -> Result<Vec<f32>, RunError> {
- if let Some(Ok(remote)) = self.remote.as_ref().filter(|r| r.share.load(Ordering::Relaxed)).map(|r| r.usable(false))
- && let Some(Ok(out)) = remote.try_run(input)
- {
- return Ok(out);
- }
- if !self.local_card.load(Ordering::Relaxed) {
- return self.cpu.run(input);
- }
- if let Ok(g) = self.card(false) {
- match g.runner.run(input) {
- Ok(out) if out.iter().all(|v| v.is_finite()) => return Ok(out),
- Ok(_) => self.fell_back("numbers that are not finite"),
- Err(e) => self.fell_back(&e.to_string()),
- }
- }
- self.cpu.run(input)
- }
-}
-
-#[cfg(feature = "denoise")]
-impl Model for Network {
- fn bytes_per_tile(&self) -> usize {
- self.cpu.bytes_per_tile()
- }
- fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
- let threads = threads.max(1);
- self.last_threads.store(threads, Ordering::Relaxed);
- let (card, local) = match self.choose(run_on, threads, true) {
- Ok(_) => (true, threads.min(GPU_PARALLEL)),
- Err(_) => (false, threads),
- };
- self.local_card.store(card, Ordering::Relaxed);
- if let Some(r) = &self.remote {
- // this computer's time for a tile: the card's, or the CPU's with `threads` tiles sharing it (as
- // `card_is_faster` counts them); unknown when the card was never set up (then the server is used)
- let local_ms = match self.gpu.get() {
- Some(Ok(g)) if card => g.card_ms,
- Some(Ok(g)) => g.cpu_ms / (1.0 + 0.5 * threads.saturating_sub(1) as f64),
- _ => f64::INFINITY,
- };
- let share = match (r.usable(true), r.tile_ms()) {
- (Ok(_), Some(ms)) if ms < local_ms => {
- *r.slower.lock().unwrap_or_else(PoisonError::into_inner) = None;
- true
- }
- (Ok(_), Some(ms)) => {
- *r.slower.lock().unwrap_or_else(PoisonError::into_inner) = Some((ms, local_ms));
- false
- }
- _ => false,
- };
- r.share.store(share, Ordering::Relaxed);
- if let (true, Ok(remote)) = (share, r.usable(false)) {
- // the server's slots come on top of this computer's own tiles at once
- return (self, local + remote.slots());
- }
- }
- if card { (self, local) } else { (&self.cpu, local) }
- }
-
- fn self_test(&self, run_on: RunOn) -> Result<Value, String> {
- let t = self.cpu.self_test();
- if !t.ok {
- let failed: Vec<&str> = t.checks.iter().filter(|(_, ok)| !*ok).map(|(c, _)| c.as_str()).collect();
- return Err(format!("the model failed its self-test: it does not {}", failed.join(", and does not ")));
- }
- let mut v = serde_json::to_value(&t).map_err(|e| e.to_string())?;
- // the card is set up and checked here, so the test says where the work will run
- if run_on != RunOn::Cpu {
- let _ = self.card(true);
- }
- if let Some(o) = v.as_object_mut() {
- o.insert("device".into(), self.device(run_on));
- }
- Ok(v)
- }
-
- fn device(&self, run_on: RunOn) -> Value {
- if run_on != RunOn::Cpu && self.gpu.get().is_none() {
- return json!({"kind": "pending"});
- }
- let threads = match self.last_threads.load(Ordering::Relaxed) {
- 0 => Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None),
- n => n,
- };
- let mut v = match self.choose(run_on, threads, false) {
- Ok(g) => json!({"kind": "gpu", "adapter": g.adapter}),
- // a set-up that crashed or ran out of time can be tried again (`denoise.settings {runOn}`)
- Err(why) => json!({"kind": "cpu", "retry": worth_retrying(&why), "reason": why}),
- };
- if let (Some(Ok(g)), Some(o)) = (self.gpu.get(), v.as_object_mut()) {
- // measured on this computer, on the check tile: the card's best run, and one tile alone on the CPU
- o.insert("cardMs".into(), json!(g.card_ms));
- o.insert("cpuMs".into(), json!(g.cpu_ms));
- o.insert("fellBack".into(), json!(self.fallbacks.load(Ordering::Relaxed)));
- }
- if let (Some(r), Some(o)) = (&self.remote, v.as_object_mut()) {
- o.insert("remote".into(), r.describe());
- }
- v
- }
-}
-
-/// The loader of this build: our pure-Rust executor with the `denoise` feature, else one that says why nothing can run.
-pub(crate) fn default_loader() -> Loader {
- #[cfg(feature = "denoise")]
- {
- Arc::new(|path, manifest| {
- // the server must run this very file: identified by the manifest's hash, else the file's
- let sha256 = match &manifest.sha256 {
- Some(h) => Some(h.clone()),
- None if std::env::var_os("LIGHTCRAFT_DENOISE_URL").is_some() => lightcraft_denoise::hash::sha256_file(path).ok(),
- None => None,
- };
- let remote = sha256.and_then(|h| RemoteSide::from_env(&h, manifest.tile as usize));
- Ok(Arc::new(network(path, manifest, remote)?) as Arc<dyn Model>)
- })
- }
- #[cfg(not(feature = "denoise"))]
- {
- Arc::new(|_, _| Err("this build cannot run denoise models".to_string()))
- }
-}
-
-/// The model at `path` on this computer, sharing tiles with `remote` when there is one.
-#[cfg(feature = "denoise")]
-fn network(path: &Path, manifest: &DenoiserManifest, remote: Option<RemoteSide>) -> Result<Network, String> {
- let cpu = lightcraft_denoise::runtime::CpuRunner::load(path, manifest).map_err(|e| e.to_string())?;
- Ok(Network {
- remote,
- local_card: AtomicBool::new(true),
- cpu,
- path: path.to_path_buf(),
- tile: manifest.tile as usize,
- gpu: OnceLock::new(),
- fallbacks: AtomicUsize::new(0),
- last_threads: AtomicUsize::new(0),
- })
-}
-
-/// A model ready to be served to other computers (`lightcraft-cli denoise-serve`).
-#[cfg(feature = "denoise")]
-pub struct Serving {
- pub model: lightcraft_denoise::remote::ModelId,
- pub tile: usize,
- /// Where its tiles run here: the graphics card's name, or why the CPU does.
- pub device: String,
- pub run: lightcraft_denoise::remote::RunTile,
-}
-
-/// The installed model `id` (default: the one selected in Settings) in the models folder `dir` (default: this user's),
-/// set up to run on `run_on` as the app would: the card checked against the CPU first, the CPU when the card can't.
-#[cfg(feature = "denoise")]
-pub fn serving(dir: Option<&Path>, id: Option<&str>, run_on: RunOn) -> Result<Serving, String> {
- let dir = match dir {
- Some(d) => d.to_path_buf(),
- None => crate::config::default_denoise_models_dir().ok_or("no denoise models folder (set LIGHTCRAFT_DENOISE_MODELS)")?,
- };
- let want = match id {
- Some(i) => i.to_string(),
- None => read_settings(&dir).model.ok_or_else(|| {
- format!("no denoise model is selected in {} (install one in Settings ▸ AI Denoise, or name it with --model)", dir.display())
- })?,
- };
- let installed = installed_models(&dir)
- .into_iter()
- .find(|m| m.manifest.id == want)
- .ok_or_else(|| format!("no installed denoise model `{want}` in {}", dir.display()))?;
- let m = &installed.manifest;
- // the same identity a client derives (see `default_loader`)
- let sha256 = match &m.sha256 {
- Some(h) => h.clone(),
- None => lightcraft_denoise::hash::sha256_file(&installed.onnx).map_err(|e| format!("{}: {e}", installed.onnx.display()))?,
- };
- let net = Arc::new(network(&installed.onnx, m, None)?);
- let tile = m.tile as usize;
- // set the card up now (and say where the tiles run) rather than with the first client's tile
- let _ = net.runner(run_on, GPU_PARALLEL);
- let d = net.device(run_on);
- let device = match d["kind"].as_str() {
- Some("gpu") => format!("GPU: {}", d["adapter"].as_str().unwrap_or("?")),
- _ => format!("CPU ({})", d["reason"].as_str().unwrap_or("chosen")),
- };
- let run: lightcraft_denoise::remote::RunTile = Arc::new(move |input: &[f32]| net.runner(run_on, GPU_PARALLEL).0.run(input));
- Ok(Serving { model: lightcraft_denoise::remote::ModelId { id: m.id.clone(), version: m.version.clone(), sha256 }, tile, device, run })
-}
-
-/// A model that is loaded on first use and kept, shared by the background job and the exports.
-pub(crate) type ModelSlot = Arc<Mutex<Option<Arc<dyn Model>>>>;
-
-// ---------------------------------------------------------------------------------------------------------------------
-// settings and installed models
-
-/// `settings.json` in the models folder.
-#[derive(Clone, Default, Serialize, Deserialize)]
-#[serde(default, rename_all = "camelCase")]
-pub(crate) struct Settings {
- /// The installed model in use; `None`: denoise is off.
- pub model: Option<String>,
- /// Photos with a Denoise amount above 0 that are being looked at (active, selected) get their picture made without
- /// being asked (`None`: yes).
- pub auto: Option<bool>,
- /// Cache size limit (GB; `None`: [`DEFAULT_CACHE_GB`]).
- pub cache_gb: Option<u32>,
- /// Most tiles run at once (`None`: by how hard the pump says to work).
- pub threads: Option<u32>,
- /// Where the model runs: `auto`, `gpu` or `cpu` (`None` or anything else: `auto`; see [`RunOn`]).
- pub run_on: Option<String>,
- /// What came before `runOn`: `false` was the processor only.
- pub gpu: Option<bool>,
-}
-
-impl Settings {
- pub fn auto(&self) -> bool {
- self.auto.unwrap_or(true)
- }
-
- pub fn run_on(&self) -> RunOn {
- match self.run_on.as_deref().and_then(RunOn::parse) {
- Some(r) => r,
- None if self.gpu == Some(false) => RunOn::Cpu,
- None => RunOn::Auto,
- }
- }
-
- pub fn cache_bytes(&self) -> u64 {
- u64::from(self.cache_gb.unwrap_or(DEFAULT_CACHE_GB).clamp(1, 10_000)) << 30
- }
-}
-
-pub(crate) fn read_capped(path: &Path, max: usize) -> Option<Vec<u8>> {
- use std::io::Read;
- let mut buf = Vec::new();
- std::fs::File::open(path).ok()?.take(max as u64 + 1).read_to_end(&mut buf).ok()?;
- (buf.len() <= max).then_some(buf)
-}
-
-pub(crate) fn read_settings(dir: &Path) -> Settings {
- read_capped(&dir.join("settings.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
-}
-
-/// Write a file next to its final name and rename it into place.
-pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> {
- let part = path.with_extension("part");
- std::fs::write(&part, bytes).map_err(|e| format!("could not write {}: {e}", path.display()))?;
- std::fs::rename(&part, path).map_err(|e| {
- let _ = std::fs::remove_file(&part);
- format!("could not write {}: {e}", path.display())
- })
-}
-
-pub(crate) fn write_settings(dir: &Path, st: &Settings) -> Result<(), String> {
- std::fs::create_dir_all(dir).map_err(|e| format!("could not create the denoise models folder: {e}"))?;
- write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| e.to_string())?)
-}
-
-/// An installed model folder: `<id>/model.onnx`, `denoise-model.json`, `installed.json`.
-pub(crate) struct Installed {
- pub manifest: DenoiserManifest,
- pub onnx: PathBuf,
- /// What was recorded when it was installed (terms accepted, hash of the file, self-test).
- pub accepted: Value,
-}
-
-impl Installed {
- /// Names the model file the cache is made with: a hash of the file as installed, so another file with the same id
- /// never finds this one's products.
- pub fn fingerprint(&self) -> String {
- let sha = self.accepted.get("sha256").and_then(Value::as_str).or(self.manifest.sha256.as_deref()).unwrap_or("");
- let len = std::fs::metadata(&self.onnx).map(|m| m.len()).unwrap_or(0);
- format!("{}@{}:{len}", self.manifest.id, sha)
- }
-}
-
-/// The models in the folder. A folder that is not a valid model (wrong name, no model file, a manifest that does not
-/// validate or does not match its folder) is ignored, never trusted.
-pub(crate) fn installed_models(dir: &Path) -> Vec<Installed> {
- let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
- let mut out = Vec::new();
- for e in entries.flatten() {
- let p = e.path();
- let Some(name) = p.file_name().and_then(|n| n.to_str()).map(str::to_string) else { continue };
- let onnx = p.join("model.onnx");
- if !lightcraft_denoise::licence::valid_id(&name) || !p.is_dir() || !onnx.is_file() {
- continue;
- }
- let Some(m) = read_capped(&p.join("denoise-model.json"), lightcraft_denoise::manifest::MAX_MANIFEST_BYTES)
- .and_then(|b| lightcraft_denoise::manifest::parse(&b).ok())
- else {
- continue;
- };
- if m.id != name {
- continue;
- }
- let accepted = read_capped(&p.join("installed.json"), 8192).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null);
- out.push(Installed { manifest: m, onnx, accepted });
- }
- out.sort_by(|a, b| a.manifest.id.cmp(&b.manifest.id));
- out
-}
-
-/// Forget that setting the graphics card up for a model closed LightCraft before (the markers [`set_up_card`] leaves):
-/// the user chose where denoise runs, so the card may be tried again.
-pub(crate) fn forget_gpu_set_ups(dir: &Path) {
- for i in installed_models(dir) {
- let _ = std::fs::remove_file(i.onnx.with_file_name(GPU_SETUP_MARKER));
- }
-}
-
-/// The model in use.
-pub(crate) struct Active {
- pub id: String,
- pub manifest: DenoiserManifest,
- pub onnx: PathBuf,
- pub fingerprint: String,
- pub model: ModelSlot,
-}
-
-// ---------------------------------------------------------------------------------------------------------------------
-// products
-
-/// The photos whose files can be denoised at all: raw files developed from their sensor data and read from disk.
-pub(crate) fn eligible(p: &Photo) -> bool {
- p.develops_raw() && matches!(p.source, Source::File { .. }) && !p.deleted
-}
-
-/// A product's name: what it was made from and with.
-pub(crate) fn product_key(fingerprint: &str, p: &Photo) -> String {
- let mut h = Hasher128::new();
- h.str("lcdn").u64(u64::from(product::ALGORITHM)).str(fingerprint).str(&crate::media::content_key(p));
- h.finish().to_string()
-}
-
-pub(crate) fn product_path(dir: &Path, key: &str) -> PathBuf {
- dir.join(format!("{key}.{EXT}"))
-}
-
-/// The `.lcdn` files in `dir`: path, size, last change.
-fn products_in(dir: &Path) -> Vec<(PathBuf, u64, SystemTime)> {
- let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
- entries
- .flatten()
- .filter(|e| e.path().extension().is_some_and(|x| x == EXT))
- .filter_map(|e| {
- let m = e.metadata().ok()?;
- m.is_file().then(|| (e.path(), m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH)))
- })
- .collect()
-}
-
-/// Make sure only one thread writes a product at a time: a second one waits, then finds it made.
-fn making() -> &'static (Mutex<HashSet<PathBuf>>, Condvar) {
- static MAKING: OnceLock<(Mutex<HashSet<PathBuf>>, Condvar)> = OnceLock::new();
- MAKING.get_or_init(|| (Mutex::new(HashSet::new()), Condvar::new()))
-}
-
-struct Slot(PathBuf);
-
-impl Slot {
- fn enter(path: &Path) -> Slot {
- let (set, cv) = making();
- let mut g = set.lock().unwrap_or_else(PoisonError::into_inner);
- while g.contains(path) {
- g = cv.wait(g).unwrap_or_else(PoisonError::into_inner);
- }
- g.insert(path.to_path_buf());
- Slot(path.to_path_buf())
- }
-}
-
-impl Drop for Slot {
- fn drop(&mut self) {
- let (set, cv) = making();
- set.lock().unwrap_or_else(PoisonError::into_inner).remove(&self.0);
- cv.notify_all();
- }
-}
-
-// ---------------------------------------------------------------------------------------------------------------------
-// making a product
-
-/// Everything a thread needs to make one product, detached from the session.
-#[derive(Clone)]
-pub(crate) struct JobSpec {
- /// The original file.
- pub path: String,
- pub read: Option<ByteReader>,
- pub product: PathBuf,
- pub key: String,
- pub manifest: DenoiserManifest,
- pub onnx: PathBuf,
- pub model: ModelSlot,
- pub loader: Loader,
- /// Tiles run at once.
- pub parallel: usize,
- /// Where the model runs (the setting).
- pub run_on: RunOn,
-}
-
-/// Why a product was not made.
-#[derive(Debug, PartialEq)]
-pub(crate) enum MakeError {
- /// Denoise does not apply to this file (not a Bayer raw): nothing is wrong.
- Unsupported(String),
- Failed(String),
- Cancelled,
-}
-
-impl From<RunError> for MakeError {
- fn from(e: RunError) -> Self {
- match e {
- RunError::Cancelled => MakeError::Cancelled,
- other => MakeError::Failed(other.to_string()),
- }
- }
-}
-
-/// Progress and cancel of a running job.
-#[derive(Default)]
-pub(crate) struct Progress {
- done: AtomicUsize,
- total: AtomicUsize,
- cancel: AtomicBool,
-}
-
-impl Progress {
- /// Ask the job to stop at the next tile.
- pub(crate) fn cancel_now(&self) {
- self.cancel.store(true, Ordering::Relaxed);
- }
-}
-
-/// The loaded model, loading it first when it is not (kept for the next photo).
-fn loaded(spec: &JobSpec) -> Result<Arc<dyn Model>, MakeError> {
- if let Some(m) = spec.model.lock().unwrap_or_else(PoisonError::into_inner).clone() {
- return Ok(m);
- }
- let m = (spec.loader)(&spec.onnx, &spec.manifest).map_err(|e| MakeError::Failed(format!("the denoise model could not be loaded: {e}")))?;
- *spec.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(m.clone());
- Ok(m)
-}
-
-/// The cell `[(0,0), (1,0), (0,1), (1,1)]` colours of a Bayer mosaic and so its layout.
-fn layout_of(cfa: &lightcraft_raw::Cfa) -> Option<Layout> {
- Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)])
-}
-
-/// Make the product of `spec` unless it is there: decode the photo, run the model over its mosaic, write the result.
-/// `Ok(true)` when it was made now, `Ok(false)` when it already existed.
-pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Result<bool, MakeError> {
- let _slot = Slot::enter(&spec.product);
- if product::is_current(&spec.product, &spec.key) {
- return Ok(false);
- }
- let began = web_time::Instant::now();
- let cancelled = || progress.is_some_and(|p| p.cancel.load(Ordering::Relaxed));
- if cancelled() {
- return Err(MakeError::Cancelled);
- }
- let bytes = match &spec.read {
- Some(r) => r(&spec.path).map_err(MakeError::Failed)?,
- None => std::fs::read(&spec.path).map_err(|e| MakeError::Failed(format!("{}: {e}", spec.path)))?,
- };
- // the decode and the normalised mosaic wait their turn at the process-wide memory gate, as every decode does
- let (mosaic, layout) = {
- let _held = crate::memory::work_gate().acquire(bytes.len().saturating_mul(4));
- if lightcraft_raw::probe(&bytes).is_none() {
- return Err(MakeError::Unsupported("not a raw file".into()));
- }
- let raw = match lightcraft_raw::decode(&bytes) {
- Ok(r) => r,
- Err(lightcraft_raw::RawError::Unsupported(why)) => return Err(MakeError::Unsupported(why)),
- Err(e) => return Err(MakeError::Failed(e.to_string())),
- };
- drop(bytes);
- if raw.cpp != 1 || !raw.cfa.as_ref().is_some_and(lightcraft_raw::Cfa::is_bayer) {
- return Err(MakeError::Unsupported(
- "only Bayer raw files can be denoised (this one has another sensor layout or is already demosaiced)".into(),
- ));
- }
- let n = raw.normalized().map_err(|e| MakeError::Failed(e.to_string()))?;
- drop(raw);
- let layout =
- n.cfa.as_ref().and_then(layout_of).ok_or_else(|| MakeError::Unsupported("the colour filter layout is not a Bayer pattern".into()))?;
- (n, layout)
- };
- if cancelled() {
- return Err(MakeError::Cancelled);
- }
- let decoded = began.elapsed();
- let model = loaded(spec)?;
- // Packed input, returned RGB and the preceding group's RGB overlap while tiles run.
- let scratch =
- model.bytes_per_tile().saturating_add(128usize.saturating_mul(spec.manifest.tile as usize).saturating_mul(spec.manifest.tile as usize));
- let (_, budget) = crate::memory::work_gate().usage();
- let parallel = spec.parallel.clamp(1, 4).min((budget / scratch.max(1)).max(1));
- let _held = crate::memory::work_gate().acquire(mosaic.data.len().saturating_mul(20).saturating_add(scratch.saturating_mul(parallel)));
- let (runner, parallel) = model.runner(spec.run_on, parallel);
- let params = Params {
- tile: spec.manifest.tile as usize,
- overlap: spec.manifest.overlap as usize,
- gain: spec.manifest.gain,
- clip: Some(CLIP),
- parallel: parallel.max(1),
- };
- let report = |done: usize, total: usize| {
- if let Some(p) = progress {
- p.total.store(total, Ordering::Relaxed);
- p.done.store(done, Ordering::Relaxed);
- }
- };
- let ctl = Control { cancel: progress.map(|p| &p.cancel), progress: Some(&report) };
- // the tiles run on threads of their own, so a busy window's renders keep theirs
- let pool = rayon::ThreadPoolBuilder::new()
- .num_threads(params.parallel)
- .stack_size(STACK)
- .thread_name(|i| format!("denoise-{i}"))
- .build()
- .map_err(|e| MakeError::Failed(format!("could not start the worker threads: {e}")))?;
- let rgb = pool.install(|| denoise_bayer(&mosaic.data, mosaic.width, mosaic.height, layout, runner, &params, &ctl))?;
- drop(mosaic);
- if cancelled() {
- return Err(MakeError::Cancelled);
- }
- let ran = began.elapsed();
- product::write(&spec.product, &rgb, &spec.key).map_err(|e| MakeError::Failed(format!("could not save the denoised picture: {e}")))?;
- if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
- eprintln!(
- "[denoise] read + decode {:.0} ms, model (set-up, tiles, blend) {:.0} ms, write {:.0} ms",
- decoded.as_secs_f64() * 1e3,
- (ran - decoded).as_secs_f64() * 1e3,
- (began.elapsed() - ran).as_secs_f64() * 1e3
- );
- }
- Ok(true)
-}
-
-// ---------------------------------------------------------------------------------------------------------------------
-// the session's state
-
-/// How hard the background work may go (the app picks it from what the user is doing, like the face scan).
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum Pace {
- /// Start nothing new.
- Pause,
- /// Two tiles at once.
- Light,
- /// Half the machine.
- Normal,
- /// Most of the machine: the user is watching the progress.
- Full,
-}
-
-impl Pace {
- /// `pause`, `light`, `normal` or `full`; anything else is `normal`.
- pub fn parse(name: Option<&str>) -> Pace {
- match name {
- Some("pause") => Pace::Pause,
- Some("light") => Pace::Light,
- Some("full") => Pace::Full,
- _ => Pace::Normal,
- }
- }
-
- /// Tiles to run at once on a machine with `cores` threads (`cap`: the user's own limit).
- pub fn parallel(self, cores: usize, cap: Option<u32>) -> usize {
- let n = match self {
- Pace::Pause | Pace::Light => 2,
- Pace::Normal => cores / 2,
- Pace::Full => cores * 4 / 5,
- }
- .clamp(1, 16);
- cap.map_or(n, |c| n.min(c.max(1) as usize))
- }
-}
-
-/// How a photo stands, for the interface.
-#[derive(Clone, Debug, PartialEq, Serialize)]
-#[serde(tag = "state", rename_all = "camelCase")]
-pub enum PhotoState {
- /// Not a photo denoise applies to (not a raw, offline, a demo).
- NotApplicable,
- /// No model is chosen: nothing can be made.
- NoModel,
- /// Its picture is made: the Amount slider works.
- Ready,
- Queued {
- /// Photos ahead of it (0 = next).
- ahead: usize,
- },
- Running {
- done: usize,
- total: usize,
- },
- /// Making it failed (`why`), or denoise does not apply to this file (`unsupported`).
- Failed {
- why: String,
- unsupported: bool,
- },
- /// Nothing has asked for it yet.
- Idle,
-}
-
-#[derive(Clone)]
-struct Failure {
- key: String,
- why: String,
- unsupported: bool,
-}
-
-enum Outcome {
- Done,
- Failed(String, bool),
- Cancelled,
-}
-
-struct Running {
- photo: PhotoId,
- key: String,
- product: PathBuf,
- progress: Arc<Progress>,
- outcome: Arc<Mutex<Option<Outcome>>>,
- started: web_time::Instant,
-}
-
-pub(crate) struct State {
- /// Where the models are kept (`None` where there is no file system).
- pub models_dir: Option<PathBuf>,
- /// Models being downloaded at the user's request.
- pub downloads: Downloads,
- pub(crate) install: Option<crate::cmd::denoise::InstallJob>,
- /// How a model is loaded (tests put another in).
- pub loader: Loader,
- pub(crate) active: Option<Active>,
- active_seen: Option<web_time::Instant>,
- /// `settings.json` as last read (with the model choice).
- pub(crate) settings: Settings,
- /// The index must be made again (the library, the model or the cache changed).
- dirty: bool,
- /// The catalog revision and time of the last index, to look again when photos changed.
- reindexed: Option<(u64, web_time::Instant)>,
- queue: VecDeque<PhotoId>,
- running: Option<Running>,
- failed: HashMap<PhotoId, Failure>,
- /// Counts every change of what is ready, so the interface knows to draw again.
- pub generation: u64,
- /// Photos made since the session started.
- made: u64,
- /// AI Denoise's row in the activity stack, while a photo is being made or waits its turn.
- task: Option<crate::activity::TaskGuard>,
-}
-
-impl Default for State {
- fn default() -> Self {
- State {
- models_dir: None,
- downloads: Downloads::default(),
- install: None,
- loader: default_loader(),
- active: None,
- active_seen: None,
- settings: Settings::default(),
- dirty: true,
- reindexed: None,
- queue: VecDeque::new(),
- running: None,
- failed: HashMap::new(),
- generation: 0,
- made: 0,
- task: None,
- }
- }
-}
-
-impl Drop for State {
- /// Quitting stops the job instead of leaving it to finish a photo nobody will see.
- fn drop(&mut self) {
- if let Some(r) = &self.running {
- r.progress.cancel_now();
- }
- }
-}
-
-impl State {
- /// The library was replaced: nothing carries over (photo ids are per library).
- pub fn library_changed(&mut self) {
- if let Some(r) = self.running.take() {
- r.progress.cancel_now();
- }
- self.queue.clear();
- self.failed.clear();
- self.dirty = true;
- self.generation += 1;
- }
-
- /// The model choice or the cache changed on disk: look again at the next pump.
- pub fn touch(&mut self) {
- self.active_seen = None;
- self.dirty = true;
- }
-
- pub fn queued(&self) -> usize {
- self.queue.len()
- }
-}
-
-impl Session {
- /// Where the active model's tiles run (see [`Model::device`]); `{kind: "none"}` while no model is loaded.
- pub(crate) fn denoise_device(&self) -> Value {
- let loaded = self.denoise.active.as_ref().and_then(|a| a.model.lock().unwrap_or_else(PoisonError::into_inner).clone());
- match loaded {
- Some(m) => m.device(self.denoise.settings.run_on()),
- None => json!({"kind": "none"}),
- }
- }
-
- /// The folder the products of the open library are kept in; `None` without a library on disk.
- pub(crate) fn denoise_products_dir(&self) -> Option<PathBuf> {
- self.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join(DIR))
- }
-
- /// Read which model is in use (at most every couple of seconds): the manifest and file of the model the settings
- /// name, if it is installed.
- pub(crate) fn denoise_refresh_active(&mut self, force: bool) {
- let fresh = self.denoise.active_seen.is_some_and(|t| t.elapsed() < SETTINGS_TTL);
- if fresh && !force {
- return;
- }
- self.denoise.active_seen = Some(web_time::Instant::now());
- self.denoise.settings = self.denoise.models_dir.as_deref().map(read_settings).unwrap_or_default();
- let chosen = self.denoise.models_dir.as_deref().and_then(|dir| {
- let id = self.denoise.settings.model.clone()?;
- installed_models(dir).into_iter().find(|i| i.manifest.id == id)
- });
- let found = chosen.map(|i| (i.fingerprint(), i));
- let same = match (&self.denoise.active, &found) {
- (None, None) => true,
- (Some(a), Some((fp, _))) => a.fingerprint == *fp,
- _ => false,
- };
- if same {
- return;
- }
- // another model: what was made with the old one is not this one's, and what failed may work now
- self.denoise.active = found.map(|(fingerprint, i)| Active {
- id: i.manifest.id.clone(),
- manifest: i.manifest,
- onnx: i.onnx,
- fingerprint,
- model: Arc::new(Mutex::new(None)),
- });
- if let Some(r) = self.denoise.running.take() {
- r.progress.cancel_now();
- }
- self.denoise.queue.clear();
- self.denoise.failed.clear();
- self.denoise.dirty = true;
- }
-
- fn denoise_spec(&self, p: &Photo) -> Option<(DenoiseSpec, String)> {
- let (dir, active) = (self.denoise_products_dir()?, self.denoise.active.as_ref()?);
- let key = product_key(&active.fingerprint, p);
- Some((DenoiseSpec { product: product_path(&dir, &key), key: key.clone(), make: None }, key))
- }
-
- /// Make the index of ready photos from the products on disk (a directory listing and a hash per photo: no file is
- /// opened). Pictures that appeared or vanished drop the decoded sources they affect. The photo being made is its
- /// job's to report (see [`Self::denoise_check_photo`]).
- pub(crate) fn denoise_reindex(&mut self) {
- self.denoise.dirty = false;
- self.denoise.reindexed = Some((self.catalog.revision, web_time::Instant::now()));
- let running = self.denoise.running.as_ref().map(|r| r.photo);
- let mut now: HashMap<PhotoId, DenoiseSpec> = HashMap::new();
- if let (Some(dir), Some(active)) = (self.denoise_products_dir(), self.denoise.active.as_ref()) {
- let names: HashSet<String> =
- products_in(&dir).into_iter().filter_map(|(p, _, _)| p.file_stem().and_then(|s| s.to_str()).map(str::to_string)).collect();
- if !names.is_empty() {
- for p in self.catalog.photos().filter(|p| eligible(p) && running != Some(p.id)) {
- let key = product_key(&active.fingerprint, p);
- if names.contains(&key) {
- now.insert(p.id, DenoiseSpec { product: product_path(&dir, &key), key, make: None });
- }
- }
- }
- }
- let mut changed = false;
- for id in self.media.denoise.ids() {
- if !now.contains_key(&id) {
- self.media.denoise.remove(id);
- self.media.forget(id);
- changed = true;
- }
- }
- for (id, spec) in now {
- if self.media.denoise.spec(id).is_none_or(|old| old.key != spec.key) {
- self.media.denoise.set(id, spec);
- self.media.forget(id);
- changed = true;
- }
- }
- if changed {
- self.denoise.generation += 1;
- }
- }
-
- /// A photo whose file changed (its content key is not the one its index entry was made for) has no picture.
- ///
- /// The photo whose picture is being made is left to its job: the worker saves the picture before it reports, so the
- /// file is on disk while the job still counts as running. Finding it here would make the photo Ready while the session
- /// is busy with it (#719); [`Self::denoise_poll`] makes it Ready in the pump that takes the job's result instead.
- fn denoise_check_photo(&mut self, id: PhotoId) {
- if self.denoise.running.as_ref().is_some_and(|r| r.photo == id) {
- return;
- }
- let Some(p) = self.catalog.photo(id).cloned() else { return };
- let Some((spec, _)) = self.denoise_spec(&p) else { return };
- match self.media.denoise.spec(id) {
- Some(have) if have.key != spec.key => {
- self.media.denoise.remove(id);
- self.media.forget(id);
- self.denoise.generation += 1;
- }
- None if spec.product.is_file() && product::is_current(&spec.product, &spec.key) => {
- // made by an export, or by another session
- self.media.denoise.set(id, spec);
- self.media.forget(id);
- self.denoise.generation += 1;
- }
- _ => {}
- }
- }
-
- /// How photo `id` stands.
- pub fn denoise_photo_state(&self, id: PhotoId) -> PhotoState {
- let Some(p) = self.catalog.photo(id) else { return PhotoState::NotApplicable };
- if !eligible(p) {
- return PhotoState::NotApplicable;
- }
- if self.denoise.active.is_none() {
- // the model folder has not been looked at yet (just started): not "no model" until it has been
- return if self.denoise.active_seen.is_none() { PhotoState::Idle } else { PhotoState::NoModel };
- }
- if self.denoise_products_dir().is_none() {
- return PhotoState::Failed {
- why: "AI Denoise keeps its pictures in the library folder, and this session has none.".into(),
- unsupported: true,
- };
- }
- if self.media.denoise.spec(id).is_some() {
- return PhotoState::Ready;
- }
- if let Some(r) = self.denoise.running.as_ref().filter(|r| r.photo == id) {
- return PhotoState::Running { done: r.progress.done.load(Ordering::Relaxed), total: r.progress.total.load(Ordering::Relaxed) };
- }
- if let Some(ahead) = self.denoise.queue.iter().position(|q| *q == id) {
- return PhotoState::Queued { ahead: ahead + usize::from(self.denoise.running.is_some()) };
- }
- let key = self.denoise_spec(p).map(|s| s.1);
- match self.denoise.failed.get(&id) {
- Some(f) if Some(&f.key) == key.as_ref() => PhotoState::Failed { why: f.why.clone(), unsupported: f.unsupported },
- _ => PhotoState::Idle,
- }
- }
-
- /// Ask for photos' pictures to be made (the ones that are not made, queued or known to be impossible). With `retry`
- /// a photo that failed before is tried again. A photo that is not raw is skipped. Returns how many were queued.
- pub(crate) fn denoise_enqueue(&mut self, ids: &[PhotoId], front: bool, retry: bool) -> usize {
- self.denoise_refresh_active(false);
- if self.denoise.active.is_none() || self.denoise_products_dir().is_none() {
- return 0;
- }
- let mut added = 0;
- for id in ids {
- let Some(p) = self.catalog.photo(*id).cloned() else { continue };
- if !eligible(&p) || self.media.denoise.spec(*id).is_some() || self.denoise.queue.contains(id) {
- continue;
- }
- if self.denoise.running.as_ref().is_some_and(|r| r.photo == *id) || self.denoise.queue.len() >= MAX_QUEUE {
- continue;
- }
- let Some((_, key)) = self.denoise_spec(&p) else { continue };
- if retry {
- self.denoise.failed.remove(id);
- } else if self.denoise.failed.get(id).is_some_and(|f| f.key == key) {
- continue;
- }
- if front {
- self.denoise.queue.push_front(*id);
- } else {
- self.denoise.queue.push_back(*id);
- }
- added += 1;
- }
- added
- }
-
- /// Stop the job that is running, and with `all` forget everything that waits.
- pub(crate) fn denoise_cancel(&mut self, photo: Option<PhotoId>, all: bool) -> usize {
- let mut n = 0;
- if let Some(r) = &self.denoise.running
- && photo.is_none_or(|p| p == r.photo)
- {
- r.progress.cancel_now();
- n += 1;
- }
- let before = self.denoise.queue.len();
- match photo {
- Some(p) => self.denoise.queue.retain(|q| *q != p),
- None if all => self.denoise.queue.clear(),
- None => {}
- }
- n + before - self.denoise.queue.len()
- }
-
- /// The photos the automatic queue looks at: the one open and the ones selected, those with a Denoise amount.
- fn denoise_wanted(&self) -> Vec<PhotoId> {
- let Selection { active, ids, .. } = &self.selection;
- active
- .iter()
- .chain(ids.iter().take(AUTO_LOOK))
- .copied()
- .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0))
- .collect()
- }
-
- fn denoise_job_spec(&self, p: &Photo, parallel: usize) -> Option<JobSpec> {
- let Source::File { path } = &p.source else { return None };
- let (spec, _) = self.denoise_spec(p)?;
- let active = self.denoise.active.as_ref()?;
- Some(JobSpec {
- path: path.clone(),
- read: self.media.file_bytes.clone(),
- product: spec.product,
- key: spec.key,
- manifest: active.manifest.clone(),
- onnx: active.onnx.clone(),
- model: active.model.clone(),
- loader: self.denoise.loader.clone(),
- parallel,
- run_on: self.denoise.settings.run_on(),
- })
- }
-
- /// Start the next queued photo on its own thread.
- fn denoise_start_next(&mut self, parallel: usize) {
- while let Some(id) = self.denoise.queue.pop_front() {
- let Some(p) = self.catalog.photo(id).cloned() else { continue };
- if !eligible(&p) || self.media.denoise.spec(id).is_some() {
- continue;
- }
- let Some(spec) = self.denoise_job_spec(&p, parallel) else { continue };
- // another session or an export may have made it since it was queued
- if product::is_current(&spec.product, &spec.key) {
- self.denoise_check_photo(id);
- continue;
- }
- let progress = Arc::new(Progress::default());
- let outcome: Arc<Mutex<Option<Outcome>>> = Arc::new(Mutex::new(None));
- let (key, product) = (spec.key.clone(), spec.product.clone());
- let (worker_progress, worker_outcome) = (progress.clone(), outcome.clone());
- let started = std::thread::Builder::new().name("denoise".into()).stack_size(STACK).spawn(move || {
- let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_product(&spec, Some(&worker_progress))));
- let o = match result {
- Ok(Ok(_)) => Outcome::Done,
- Ok(Err(MakeError::Cancelled)) => Outcome::Cancelled,
- Ok(Err(MakeError::Unsupported(why))) => Outcome::Failed(why, true),
- Ok(Err(MakeError::Failed(why))) => Outcome::Failed(why, false),
- Err(_) => Outcome::Failed("the denoiser stopped unexpectedly".into(), false),
- };
- *worker_outcome.lock().unwrap_or_else(PoisonError::into_inner) = Some(o);
- });
- match started {
- Ok(_) => {
- self.denoise.running = Some(Running { photo: id, key, product, progress, outcome, started: web_time::Instant::now() });
- }
- Err(e) => {
- self.denoise.failed.insert(id, Failure { key, why: format!("could not start the denoiser: {e}"), unsupported: false });
- continue;
- }
- }
- return;
- }
- }
-
- /// Take in the running job's result.
- fn denoise_poll(&mut self) {
- let Some(r) = &self.denoise.running else { return };
- let Some(outcome) = r.outcome.lock().unwrap_or_else(PoisonError::into_inner).take() else { return };
- let Some(r) = self.denoise.running.take() else { return };
- match outcome {
- Outcome::Done => {
- self.denoise.made += 1;
- self.denoise_check_photo(r.photo);
- self.denoise_evict(Some(&r.product));
- }
- Outcome::Failed(why, unsupported) => {
- self.denoise.failed.insert(r.photo, Failure { key: r.key, why, unsupported });
- }
- Outcome::Cancelled => {}
- }
- // the picture's state changed: the interface draws it again
- self.denoise.generation += 1;
- }
-
- /// Keep the cache within its limit: the pictures nobody asks for go first, the oldest first; `keep` (the one just
- /// made) never goes. The index forgets what was deleted.
- pub(crate) fn denoise_evict(&mut self, keep: Option<&Path>) {
- let Some(models) = self.denoise.models_dir.clone() else { return };
- self.denoise_evict_to(read_settings(&models).cache_bytes(), keep);
- }
-
- /// [`Self::denoise_evict`] for a limit of `limit` bytes.
- pub(crate) fn denoise_evict_to(&mut self, limit: u64, keep: Option<&Path>) {
- let Some(dir) = self.denoise_products_dir() else { return };
- let mut files = products_in(&dir);
- let mut total: u64 = files.iter().map(|f| f.1).sum();
- if total <= limit {
- return;
- }
- // the pictures photos use now (a Denoise amount above 0) are the last to go
- let wanted: HashSet<PathBuf> = self
- .media
- .denoise
- .ids()
- .into_iter()
- .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0))
- .filter_map(|id| self.media.denoise.spec(id).map(|s| s.product.clone()))
- .collect();
- files.sort_by_key(|(path, _, modified)| (wanted.contains(path), *modified));
- let mut gone: HashSet<PathBuf> = HashSet::new();
- for (path, size, _) in files {
- if total <= limit {
- break;
- }
- if keep == Some(path.as_path()) {
- continue;
- }
- if std::fs::remove_file(&path).is_ok() {
- total = total.saturating_sub(size);
- gone.insert(path);
- }
- }
- if gone.is_empty() {
- return;
- }
- for id in self.media.denoise.ids() {
- if self.media.denoise.spec(id).is_some_and(|s| gone.contains(&s.product)) {
- self.media.denoise.remove(id);
- self.media.forget(id);
- }
- }
- self.denoise.generation += 1;
- }
-
- /// Delete every product of this library (they are made again when wanted).
- pub(crate) fn denoise_clear(&mut self) -> (usize, u64) {
- self.denoise_cancel(None, true);
- let (mut files, mut bytes) = (0, 0);
- if let Some(dir) = self.denoise_products_dir() {
- for (path, size, _) in products_in(&dir) {
- if std::fs::remove_file(&path).is_ok() {
- files += 1;
- bytes += size;
- }
- }
- }
- self.denoise.failed.clear();
- self.denoise.touch();
- (files, bytes)
- }
-
- /// Per frame: take in what finished, look at what the user is looking at, and keep one job going. Cheap when
- /// nothing is happening (no file is opened unless a photo with a Denoise amount lacks its picture).
- pub(crate) fn denoise_pump(&mut self, pace: Pace) {
- crate::cmd::denoise::finish_downloads(self);
- self.denoise_refresh_active(false);
- // photos that changed (a replaced file has another key) must not keep the picture of what they were
- let stale = !self.media.denoise.is_empty()
- && self.denoise.reindexed.is_none_or(|(rev, at)| rev != self.catalog.revision && at.elapsed() > SETTINGS_TTL);
- if self.denoise.dirty || stale {
- self.denoise_reindex();
- }
- self.denoise_poll();
- if self.denoise.active.is_none() || self.denoise_products_dir().is_none() {
- return;
- }
- let (cap, auto) = (self.denoise.settings.threads, self.denoise.settings.auto());
- let wanted = self.denoise_wanted();
- for id in &wanted {
- self.denoise_check_photo(*id);
- }
- if auto {
- // the photo being edited goes first
- let (open, others): (Vec<PhotoId>, Vec<PhotoId>) = wanted.into_iter().partition(|id| self.selection.active == Some(*id));
- self.denoise_enqueue(&others, false, false);
- self.denoise_enqueue(&open, true, false);
- }
- if self.denoise.running.is_none() && pace != Pace::Pause {
- let cores = std::thread::available_parallelism().map_or(1, |n| n.get());
- self.denoise_start_next(pace.parallel(cores, cap));
- }
- }
-
- /// The statistics `denoise.status` reports besides the queue (reads the cache folder).
- pub(crate) fn denoise_cache_usage(&self) -> (usize, u64) {
- match self.denoise_products_dir() {
- Some(dir) => {
- let files = products_in(&dir);
- (files.len(), files.iter().map(|f| f.1).sum())
- }
- None => (0, 0),
- }
- }
-
- /// Are pictures made without being asked for photos being looked at (Settings ▸ AI Denoise)?
- pub fn denoise_auto(&self) -> bool {
- self.denoise.settings.auto()
- }
-
- /// Is a picture being made, or waiting for its turn with a model to make it? (Headless runs wait for this.)
- pub fn denoise_busy(&self) -> bool {
- self.denoise.running.is_some() || (!self.denoise.queue.is_empty() && self.denoise.active.is_some())
- }
-
- /// Keep AI Denoise's row in the activity stack up to date (each `denoise.pump`): there while a photo is being made or
- /// waits its turn, with the running photo's tiles and its file name; gone when the queue is idle. It has no ✕: what is
- /// made follows the photos' Denoise amounts and Settings ▸ AI Denoise.
- pub(crate) fn denoise_track(&mut self) {
- if !self.denoise_busy() {
- self.denoise.task = None;
- return;
- }
- let task = self.denoise.task.get_or_insert_with(|| {
- let t = self.activity.start("denoise", "AI Denoise", crate::activity::Cancel::No);
- t.set_unit(crate::activity::Unit::Percent);
- t
- });
- match &self.denoise.running {
- Some(r) => {
- task.progress(r.progress.done.load(Ordering::Relaxed) as u64, r.progress.total.load(Ordering::Relaxed) as u64);
- task.detail(self.catalog.photo(r.photo).map_or("", |p| p.file_name.as_str()));
- }
- None => {
- task.progress(0, 0);
- task.detail("");
- }
- }
- }
-
- /// Seconds the running job has been going, and its progress.
- pub(crate) fn denoise_running_json(&self) -> Value {
- match &self.denoise.running {
- Some(r) => json!({
- "photo": r.photo.0,
- "done": r.progress.done.load(Ordering::Relaxed),
- "total": r.progress.total.load(Ordering::Relaxed),
- "seconds": r.started.elapsed().as_secs_f64(),
- }),
- None => Value::Null,
- }
- }
-
- pub(crate) fn denoise_made(&self) -> u64 {
- self.denoise.made
- }
-
- /// The running job's product path and key (tests stand in for the worker's save).
- #[cfg(test)]
- pub(crate) fn denoise_running_product(&self) -> Option<(PathBuf, String)> {
- self.denoise.running.as_ref().map(|r| (r.product.clone(), r.key.clone()))
- }
-
- /// What an export of photo `id` with `settings` needs for its Denoise amount: its denoised picture, made now when the
- /// queue has not got to it. `Ok(None)` when denoise does not apply (amount 0, not a Bayer raw), `Err` when it should
- /// have and cannot (no model chosen, nowhere to keep the picture, the model failing).
- pub(crate) fn denoise_for_export(&mut self, id: PhotoId, settings: &DevelopSettings) -> Result<Option<(DenoiseSpec, PairLoader)>, String> {
- if settings.denoise_amount() <= 0.0 {
- return Ok(None);
- }
- let Some(p) = self.catalog.photo(id).cloned() else { return Ok(None) };
- let Some(loader) = self.media.denoise.loader.clone() else { return Ok(None) };
- if !eligible(&p) {
- return Ok(None);
- }
- self.denoise_refresh_active(false);
- let name = &p.file_name;
- if self.denoise.active.is_none() {
- return Err(format!("{name}: Denoise is on for this photo but no denoise model is chosen; choose one in Settings, or set Denoise to 0"));
- }
- if self.denoise_products_dir().is_none() {
- return Err(format!("{name}: Denoise needs a library on disk to keep its pictures in"));
- }
- self.denoise_check_photo(id);
- let Some(job) = self.denoise_job_spec(&p, Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None)) else {
- return Ok(None);
- };
- let label = name.clone();
- let spec = job.clone();
- let make: MakeProduct = Arc::new(move || match make_product(&spec, None) {
- Ok(_) => Ok(true),
- Err(MakeError::Unsupported(_)) => Ok(false),
- Err(MakeError::Failed(why)) => Err(format!("{label}: could not denoise: {why}")),
- Err(MakeError::Cancelled) => Err(format!("{label}: denoise was cancelled")),
- });
- Ok(Some((DenoiseSpec { product: job.product, key: job.key, make: Some(make) }, loader)))
- }
-}
+mod gpu;
+mod model;
+mod products;
+mod session;
+mod settings;
+mod state;
+
+#[cfg(test)]
+pub(crate) use gpu::GPU_SETUP_MARKER;
+#[cfg(all(feature = "denoise", test))]
+use gpu::{GPU_CRASHED, GPU_MAX_FALLBACKS, GPU_PARALLEL, SETTING_UP, card_is_faster, set_up_card};
+pub use model::RunOn;
+#[cfg(all(feature = "denoise", test))]
+pub(crate) use model::default_loader;
+#[cfg(all(feature = "denoise", test))]
+use model::{GpuSide, Network, RemoteSide, network, worth_retrying};
+pub(crate) use model::{Loader, Model};
+#[cfg(feature = "denoise")]
+pub use model::{Serving, serving};
+pub(crate) use products::eligible;
+#[cfg(test)]
+pub(crate) use products::{JobSpec, MakeError, Progress, make_product};
+pub use settings::{DEFAULT_CACHE_GB, DIR, EXT};
+pub(crate) use settings::{Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings};
+pub(crate) use state::State;
+pub use state::{Pace, PhotoState};
#[cfg(all(test, feature = "denoise"))]
mod gpu_tests;
diff --git a/crates/engine/src/denoise/gpu.rs b/crates/engine/src/denoise/gpu.rs
new file mode 100644
index 000000000..1577b4edc
--- /dev/null
+++ b/crates/engine/src/denoise/gpu.rs
@@ -0,0 +1,152 @@
+#[cfg(feature = "denoise")]
+use super::model::{GpuSide, STACK};
+#[cfg(feature = "denoise")]
+use super::*;
+
+/// Tiles run at once on a graphics card: enough to keep it fed while others are packed and blended.
+#[cfg(feature = "denoise")]
+pub(super) const GPU_PARALLEL: usize = 4;
+
+/// How far the GPU's answer on the check tile may be from the CPU's (of the answer's largest value) before it is not used.
+#[cfg(feature = "denoise")]
+const GPU_AGREE: f32 = 1e-3;
+
+/// Longest the card may take to set up (build its kernels, run the check tile, time it) before the processor does the
+/// work instead.
+#[cfg(feature = "denoise")]
+const GPU_SETUP_LIMIT: Duration = Duration::from_secs(30);
+
+/// Tiles the card may get wrong (an error, numbers that are not finite) before it stops being used for the model.
+#[cfg(feature = "denoise")]
+pub(super) const GPU_MAX_FALLBACKS: usize = 3;
+
+/// The file beside a model while the graphics card is set up for it (see [`set_up_card`]).
+pub(crate) const GPU_SETUP_MARKER: &str = "gpu-setup.marker";
+
+/// Why the card is not tried when a set-up marker is found.
+#[cfg(feature = "denoise")]
+pub(super) const GPU_CRASHED: &str = "LightCraft closed while it was setting up the graphics card for AI Denoise last time";
+
+/// Why the card is not used when its set-up ran out of time (the start of it).
+#[cfg(feature = "denoise")]
+pub(super) const GPU_SLOW: &str = "setting up the graphics card took longer than";
+
+/// Why the card is not used while a set-up that ran out of time is still going.
+#[cfg(feature = "denoise")]
+pub(super) const GPU_STUCK: &str = "the graphics card is still setting up from before (it took too long)";
+
+/// Whether the card (`card_ms` a tile) is faster than the CPU running `threads` tiles at once, from the CPU's time for
+/// one tile alone (`cpu_ms`). Tiles at once share the memory and the cores' second threads, so each one more counts as
+/// half a tile's speed: on a 16-core laptop one tile alone took 1.4 s and sixteen at once finished one every 0.2 s.
+#[cfg(feature = "denoise")]
+pub(super) fn card_is_faster(card_ms: f64, cpu_ms: f64, threads: usize) -> bool {
+ let at_once = 1.0 + 0.5 * threads.saturating_sub(1) as f64;
+ card_ms.is_finite() && card_ms <= cpu_ms / at_once
+}
+
+/// Card set-ups of this process that have not finished, by marker (their markers are not a crash's).
+#[cfg(feature = "denoise")]
+pub(super) static SETTING_UP: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
+
+/// Removes a set-up's marker when the set-up ends, however it ends — but not when the process dies inside the driver.
+#[cfg(feature = "denoise")]
+struct SetUpEnded(PathBuf);
+
+#[cfg(feature = "denoise")]
+impl Drop for SetUpEnded {
+ fn drop(&mut self) {
+ let _ = std::fs::remove_file(&self.0);
+ SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner).retain(|m| m != &self.0);
+ }
+}
+
+/// Set the graphics card up with `card` on a thread of its own while `meanwhile` runs here, and give up on the card when
+/// it takes longer than `limit` (a driver building kernels for minutes, or stuck). The `marker` file exists while `card`
+/// runs: one found before it starts was left by a LightCraft that closed during a set-up (a driver crash takes the app
+/// with it), so the card is not tried again until the user chooses where denoise runs (`denoise.settings {runOn}`).
+#[cfg(feature = "denoise")]
+pub(super) fn set_up_card<T: Send + 'static, R>(
+ marker: &Path,
+ limit: Duration,
+ card: impl FnOnce() -> Result<T, String> + Send + 'static,
+ meanwhile: impl FnOnce() -> R,
+) -> Result<(T, R), String> {
+ let ended = {
+ let mut running = SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner);
+ if running.iter().any(|m| m == marker) {
+ return Err(GPU_STUCK.into());
+ }
+ if marker.exists() {
+ return Err(GPU_CRASHED.into());
+ }
+ let secs = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map_or(0, |d| d.as_secs());
+ // a marker that cannot be written only loses the crash guard
+ let _ = std::fs::write(marker, format!("setting up the graphics card for AI Denoise (unix time {secs})\n"));
+ running.push(marker.to_path_buf());
+ SetUpEnded(marker.to_path_buf())
+ };
+ let started = web_time::Instant::now();
+ let (tx, rx) = std::sync::mpsc::channel();
+ std::thread::Builder::new()
+ .name("denoise-gpu-setup".into())
+ .stack_size(STACK)
+ .spawn(move || {
+ let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(card)).unwrap_or_else(|_| Err("the GPU runner gave up".into()));
+ drop(ended);
+ let _ = tx.send(r);
+ })
+ .map_err(|e| format!("could not start setting up the graphics card: {e}"))?;
+ let here = meanwhile();
+ match rx.recv_timeout(limit.saturating_sub(started.elapsed())) {
+ Ok(r) => r.map(|t| (t, here)),
+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(format!("{GPU_SLOW} {} s, so it is not used", limit.as_secs_f64())),
+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err("the GPU runner gave up".into()),
+ }
+}
+
+/// The card's time for one tile once it is warm: the fastest of up to three runs, stopping after about two seconds (a
+/// slow card is slow enough to tell at once).
+#[cfg(feature = "denoise")]
+fn card_tile_ms(runner: &dyn TileRunner, input: &[f32]) -> f64 {
+ if runner.run(input).is_err() {
+ return f64::INFINITY;
+ }
+ let began = web_time::Instant::now();
+ let mut best = f64::INFINITY;
+ for _ in 0..3 {
+ let one = web_time::Instant::now();
+ if runner.run(input).is_err() {
+ break;
+ }
+ best = best.min(one.elapsed().as_secs_f64() * 1000.0);
+ if began.elapsed() > Duration::from_secs(2) {
+ break;
+ }
+ }
+ best
+}
+
+/// Set the card up for the model at `path`: kernels built, the check tile run on both and compared, both timed.
+#[cfg(feature = "denoise")]
+pub(super) fn make_gpu(cpu: &lightcraft_denoise::runtime::CpuRunner, path: &Path, tile: usize) -> Result<GpuSide, String> {
+ let input = Arc::new(lightcraft_denoise::runtime::check_tile(tile));
+ let (on_card, net) = (input.clone(), cpu.net().clone());
+ let card = move || -> Result<_, String> {
+ let runner = lightcraft_gpu::nn::runner(&net, tile)?;
+ let got = runner.run(&on_card).map_err(|e| format!("the GPU could not run the check tile: {e}"))?;
+ let card_ms = card_tile_ms(&runner, &on_card);
+ Ok((runner, got, card_ms))
+ };
+ let on_cpu = || {
+ let started = web_time::Instant::now();
+ (cpu.run(&input), started.elapsed().as_secs_f64() * 1000.0)
+ };
+ let ((runner, got, card_ms), (want, cpu_ms)) = set_up_card(&path.with_file_name(GPU_SETUP_MARKER), GPU_SETUP_LIMIT, card, on_cpu)?;
+ let want = want.map_err(|e| format!("the CPU could not check the GPU: {e}"))?;
+ let scale = want.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-12);
+ let worst = want.iter().zip(&got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
+ if got.len() != want.len() || !got.iter().all(|v| v.is_finite()) || worst > GPU_AGREE {
+ return Err(format!("its answer on the check tile is not the CPU's (off by {worst:.1e} of the largest value): not used"));
+ }
+ Ok(GpuSide { adapter: runner.adapter(), runner: Box::new(runner), card_ms, cpu_ms })
+}
diff --git a/crates/engine/src/denoise/model.rs b/crates/engine/src/denoise/model.rs
new file mode 100644
index 000000000..28ff953bf
--- /dev/null
+++ b/crates/engine/src/denoise/model.rs
@@ -0,0 +1,439 @@
+#[cfg(feature = "denoise")]
+use super::gpu::{GPU_CRASHED, GPU_MAX_FALLBACKS, GPU_PARALLEL, GPU_SLOW, GPU_STUCK, card_is_faster, make_gpu};
+#[cfg(feature = "denoise")]
+use super::settings::{installed_models, read_settings};
+#[cfg(feature = "denoise")]
+use super::state::Pace;
+use super::*;
+
+/// Stack of the threads that run the model (the runtime recurses through its graph).
+pub(super) const STACK: usize = 16 << 20;
+
+// ---------------------------------------------------------------------------------------------------------------------
+// the model
+
+/// Where the model runs (Settings ▸ AI Denoise ▸ Speed; `denoise.settings {runOn}`).
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum RunOn {
+ /// The graphics card where it is faster than the processor at the pace the work goes at (both are timed on this
+ /// computer when the card is set up), else the processor.
+ #[default]
+ Auto,
+ /// The graphics card whenever it can run the model, even where the processor would be faster.
+ Gpu,
+ /// Always the processor.
+ Cpu,
+}
+
+impl RunOn {
+ /// `auto`, `gpu` or `cpu`.
+ pub fn parse(name: &str) -> Option<RunOn> {
+ match name {
+ "auto" => Some(RunOn::Auto),
+ "gpu" => Some(RunOn::Gpu),
+ "cpu" => Some(RunOn::Cpu),
+ _ => None,
+ }
+ }
+
+ pub fn name(self) -> &'static str {
+ match self {
+ RunOn::Auto => "auto",
+ RunOn::Gpu => "gpu",
+ RunOn::Cpu => "cpu",
+ }
+ }
+}
+
+/// A loaded denoise model: what runs tiles, and a self-check.
+pub(crate) trait Model: Send + Sync {
+ /// What runs a photo's tiles, and how many to run at once, when `threads` may be used and the setting is `run_on`
+ /// (a graphics card is kept busy by a few). The first time the card may be used it is set up and checked here.
+ fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize);
+ /// Check it does something sensible: the test's result, or why not. Unless `run_on` is the processor the graphics
+ /// card is set up and checked too, and the result says where the work will run.
+ fn self_test(&self, run_on: RunOn) -> Result<Value, String>;
+ /// Conservative scratch bound for an unknown runner.
+ fn bytes_per_tile(&self) -> usize {
+ 512 * 1024 * 1024
+ }
+ /// Where the tiles run: `{kind: "gpu", adapter}`, `{kind: "cpu", reason?}` or `{kind: "pending"}` (the card is not
+ /// set up yet), with the card's and the processor's time for the check tile (`cardMs`, `cpuMs`) once both were
+ /// timed. Never starts anything.
+ fn device(&self, _run_on: RunOn) -> Value {
+ json!({"kind": "cpu"})
+ }
+}
+
+/// Loads the model at a path as its manifest describes it.
+pub(crate) type Loader = Arc<dyn Fn(&Path, &DenoiserManifest) -> Result<Arc<dyn Model>, String> + Send + Sync>;
+
+/// A set-up that may work another time (a crash, the time limit), unlike one the card or the model cannot do.
+#[cfg(feature = "denoise")]
+pub(super) fn worth_retrying(why: &str) -> bool {
+ why == GPU_CRASHED || why == GPU_STUCK || why.starts_with(GPU_SLOW)
+}
+
+/// The model on our pure-Rust CPU runner, and on the graphics card when there is one that can run it: the card is set up for the
+/// first photo that may use it, checked against the CPU's answer on a check tile and timed against it, and used while
+/// it keeps working; a tile it gets wrong is run on the CPU, so a photo is always finished.
+#[cfg(feature = "denoise")]
+pub(super) struct Network {
+ pub(super) cpu: lightcraft_denoise::runtime::CpuRunner,
+ pub(super) path: PathBuf,
+ pub(super) tile: usize,
+ pub(super) gpu: OnceLock<Result<GpuSide, String>>,
+ /// Tiles the card got wrong (an error, numbers that are not finite) and the CPU ran.
+ pub(super) fallbacks: AtomicUsize,
+ /// Tiles at once of the photo started last (0: none yet), to say where the work runs.
+ pub(super) last_threads: AtomicUsize,
+ /// Another computer's graphics card (`LIGHTCRAFT_DENOISE_URL` + `LIGHTCRAFT_DENOISE_TOKEN`; see
+ /// `docs/denoise.md` → Another computer's graphics card), greeted on first use: it takes tiles while it has a free
+ /// slot, on top of this computer's runner.
+ pub(super) remote: Option<RemoteSide>,
+ /// Whether this computer's share runs on the card (else the CPU), as chosen for the photo started last.
+ pub(super) local_card: AtomicBool,
+}
+
+#[cfg(feature = "denoise")]
+pub(super) struct RemoteSide {
+ runner: lightcraft_denoise::remote::Remote,
+ /// The greeting, and the server's time for the check tile there and back (ms; the faster of two runs).
+ greeted: OnceLock<Result<(lightcraft_denoise::remote::ServerInfo, f64), String>>,
+ /// Whether the photo started last shares tiles with the server, and if not because it is slower, the two times.
+ share: AtomicBool,
+ slower: Mutex<Option<(f64, f64)>>,
+}
+
+#[cfg(feature = "denoise")]
+impl RemoteSide {
+ /// From the environment, for the model whose file has `sha256` and tiles of `tile` cells.
+ fn from_env(sha256: &str, tile: usize) -> Option<RemoteSide> {
+ let var = |k: &str| std::env::var(k).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty());
+ let address = var("LIGHTCRAFT_DENOISE_URL")?;
+ let token = var("LIGHTCRAFT_DENOISE_TOKEN").unwrap_or_default();
+ Some(RemoteSide::new(lightcraft_denoise::remote::Remote::new(&address, &token, sha256, tile)))
+ }
+
+ pub(super) fn new(runner: lightcraft_denoise::remote::Remote) -> RemoteSide {
+ RemoteSide { runner, greeted: OnceLock::new(), share: AtomicBool::new(false), slower: Mutex::new(None) }
+ }
+
+ /// Greet the server and time a check tile there and back: a server behind a slow link can take seconds a tile, and
+ /// handing it tiles a fast local card would finish sooner only makes the photo wait for it.
+ fn greet(&self) -> Result<(lightcraft_denoise::remote::ServerInfo, f64), String> {
+ let info = self.runner.hello().map_err(|e| e.to_string())?;
+ let input = lightcraft_denoise::runtime::check_tile(info.tile);
+ let mut best = f64::INFINITY;
+ for _ in 0..2 {
+ let started = web_time::Instant::now();
+ match self.runner.try_run(&input) {
+ Some(Ok(_)) => best = best.min(started.elapsed().as_secs_f64() * 1000.0),
+ Some(Err(e)) => return Err(format!("the check tile failed: {e}")),
+ None => return Err("no free slot for the check tile".into()),
+ }
+ }
+ Ok((info, best))
+ }
+
+ /// The server when it can take tiles now (greeting it the first time), else why not.
+ fn usable(&self, start: bool) -> Result<&lightcraft_denoise::remote::Remote, String> {
+ let greeted = if start { Some(self.greeted.get_or_init(|| self.greet())) } else { self.greeted.get() };
+ match greeted {
+ None => Err("not contacted yet".into()),
+ Some(Err(why)) => Err(why.clone()),
+ Some(Ok(_)) if !self.runner.healthy() => {
+ Err(format!("it failed {} tiles ({})", self.runner.failures(), self.runner.last_error().unwrap_or_default()))
+ }
+ Some(Ok(_)) => Ok(&self.runner),
+ }
+ }
+
+ /// The server's time for a tile, once greeted.
+ fn tile_ms(&self) -> Option<f64> {
+ self.greeted.get().and_then(|g| g.as_ref().ok()).map(|g| g.1)
+ }
+
+ fn describe(&self) -> Value {
+ match self.greeted.get() {
+ None => json!({"address": self.runner.address(), "state": "pending"}),
+ Some(Err(why)) => json!({"address": self.runner.address(), "state": "unavailable", "reason": why}),
+ Some(Ok((info, ms))) => {
+ let slower = *self.slower.lock().unwrap_or_else(PoisonError::into_inner);
+ let state = match () {
+ _ if !self.runner.healthy() => "given up",
+ _ if slower.is_some() => "slower",
+ _ if self.share.load(Ordering::Relaxed) => "used",
+ _ => "idle",
+ };
+ let mut v =
+ json!({"address": info.address, "device": info.device, "state": state, "remoteMs": ms, "failures": self.runner.failures()});
+ if let (Some((_, local)), Some(o)) = (slower, v.as_object_mut()) {
+ o.insert("localMs".into(), json!(local));
+ }
+ if let (Some(e), Some(o)) = (self.runner.last_error(), v.as_object_mut()) {
+ o.insert("lastError".into(), json!(e));
+ }
+ v
+ }
+ }
+ }
+}
+
+#[cfg(feature = "denoise")]
+pub(super) struct GpuSide {
+ pub(super) runner: Box<dyn TileRunner + Send>,
+ pub(super) adapter: String,
+ /// The check tile on the card, the fastest of a few runs (ms).
+ pub(super) card_ms: f64,
+ /// The check tile on the CPU, on one thread (ms).
+ pub(super) cpu_ms: f64,
+}
+
+#[cfg(feature = "denoise")]
+impl Network {
+ /// The card when it can be used now, else why not. `start` sets it up when that has not been tried yet; without it
+ /// nothing is started.
+ fn card(&self, start: bool) -> Result<&GpuSide, String> {
+ let side = if start {
+ Some(self.gpu.get_or_init(|| {
+ let made = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_gpu(&self.cpu, &self.path, self.tile)))
+ .unwrap_or_else(|_| Err("the GPU runner gave up".into()));
+ if let Err(why) = &made {
+ log::info!("denoise: running on the CPU ({why})");
+ }
+ made
+ }))
+ } else {
+ self.gpu.get()
+ };
+ let g = match side {
+ None => return Err("the graphics card is not set up yet".into()),
+ Some(Err(why)) => return Err(why.clone()),
+ Some(Ok(g)) => g,
+ };
+ if let Some(why) = lightcraft_gpu::nn::broken_reason() {
+ return Err(format!("the graphics card stopped working ({why})"));
+ }
+ if self.fallbacks.load(Ordering::Relaxed) >= GPU_MAX_FALLBACKS {
+ return Err(format!("{} got {GPU_MAX_FALLBACKS} tiles wrong", g.adapter));
+ }
+ Ok(g)
+ }
+
+ /// The card, when a photo made `threads` tiles at a time under `run_on` runs on it; else why the CPU does.
+ fn choose(&self, run_on: RunOn, threads: usize, start: bool) -> Result<&GpuSide, String> {
+ if run_on == RunOn::Cpu {
+ return Err("the processor is chosen in Settings".into());
+ }
+ let g = self.card(start)?;
+ if run_on == RunOn::Auto && !card_is_faster(g.card_ms, g.cpu_ms, threads) {
+ return Err(format!("it is faster than {} on this computer", g.adapter));
+ }
+ Ok(g)
+ }
+
+ fn fell_back(&self, why: &str) {
+ let n = self.fallbacks.fetch_add(1, Ordering::Relaxed) + 1;
+ if n == 1 {
+ log::warn!("denoise: a tile came back wrong from the GPU ({why}); the CPU runs it");
+ } else if n == GPU_MAX_FALLBACKS {
+ log::warn!("denoise: the GPU got {n} tiles wrong; the CPU does the rest");
+ }
+ }
+}
+
+/// A remote server's free slot first (when there is one), then the card when this photo runs on it, with the CPU
+/// running any tile either gets wrong. Handed out for photos that run on the card or have a server to share.
+#[cfg(feature = "denoise")]
+impl TileRunner for Network {
+ fn run(&self, input: &[f32]) -> Result<Vec<f32>, RunError> {
+ if let Some(Ok(remote)) = self.remote.as_ref().filter(|r| r.share.load(Ordering::Relaxed)).map(|r| r.usable(false))
+ && let Some(Ok(out)) = remote.try_run(input)
+ {
+ return Ok(out);
+ }
+ if !self.local_card.load(Ordering::Relaxed) {
+ return self.cpu.run(input);
+ }
+ if let Ok(g) = self.card(false) {
+ match g.runner.run(input) {
+ Ok(out) if out.iter().all(|v| v.is_finite()) => return Ok(out),
+ Ok(_) => self.fell_back("numbers that are not finite"),
+ Err(e) => self.fell_back(&e.to_string()),
+ }
+ }
+ self.cpu.run(input)
+ }
+}
+
+#[cfg(feature = "denoise")]
+impl Model for Network {
+ fn bytes_per_tile(&self) -> usize {
+ self.cpu.bytes_per_tile()
+ }
+ fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
+ let threads = threads.max(1);
+ self.last_threads.store(threads, Ordering::Relaxed);
+ let (card, local) = match self.choose(run_on, threads, true) {
+ Ok(_) => (true, threads.min(GPU_PARALLEL)),
+ Err(_) => (false, threads),
+ };
+ self.local_card.store(card, Ordering::Relaxed);
+ if let Some(r) = &self.remote {
+ // this computer's time for a tile: the card's, or the CPU's with `threads` tiles sharing it (as
+ // `card_is_faster` counts them); unknown when the card was never set up (then the server is used)
+ let local_ms = match self.gpu.get() {
+ Some(Ok(g)) if card => g.card_ms,
+ Some(Ok(g)) => g.cpu_ms / (1.0 + 0.5 * threads.saturating_sub(1) as f64),
+ _ => f64::INFINITY,
+ };
+ let share = match (r.usable(true), r.tile_ms()) {
+ (Ok(_), Some(ms)) if ms < local_ms => {
+ *r.slower.lock().unwrap_or_else(PoisonError::into_inner) = None;
+ true
+ }
+ (Ok(_), Some(ms)) => {
+ *r.slower.lock().unwrap_or_else(PoisonError::into_inner) = Some((ms, local_ms));
+ false
+ }
+ _ => false,
+ };
+ r.share.store(share, Ordering::Relaxed);
+ if let (true, Ok(remote)) = (share, r.usable(false)) {
+ // the server's slots come on top of this computer's own tiles at once
+ return (self, local + remote.slots());
+ }
+ }
+ if card { (self, local) } else { (&self.cpu, local) }
+ }
+
+ fn self_test(&self, run_on: RunOn) -> Result<Value, String> {
+ let t = self.cpu.self_test();
+ if !t.ok {
+ let failed: Vec<&str> = t.checks.iter().filter(|(_, ok)| !*ok).map(|(c, _)| c.as_str()).collect();
+ return Err(format!("the model failed its self-test: it does not {}", failed.join(", and does not ")));
+ }
+ let mut v = serde_json::to_value(&t).map_err(|e| e.to_string())?;
+ // the card is set up and checked here, so the test says where the work will run
+ if run_on != RunOn::Cpu {
+ let _ = self.card(true);
+ }
+ if let Some(o) = v.as_object_mut() {
+ o.insert("device".into(), self.device(run_on));
+ }
+ Ok(v)
+ }
+
+ fn device(&self, run_on: RunOn) -> Value {
+ if run_on != RunOn::Cpu && self.gpu.get().is_none() {
+ return json!({"kind": "pending"});
+ }
+ let threads = match self.last_threads.load(Ordering::Relaxed) {
+ 0 => Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None),
+ n => n,
+ };
+ let mut v = match self.choose(run_on, threads, false) {
+ Ok(g) => json!({"kind": "gpu", "adapter": g.adapter}),
+ // a set-up that crashed or ran out of time can be tried again (`denoise.settings {runOn}`)
+ Err(why) => json!({"kind": "cpu", "retry": worth_retrying(&why), "reason": why}),
+ };
+ if let (Some(Ok(g)), Some(o)) = (self.gpu.get(), v.as_object_mut()) {
+ // measured on this computer, on the check tile: the card's best run, and one tile alone on the CPU
+ o.insert("cardMs".into(), json!(g.card_ms));
+ o.insert("cpuMs".into(), json!(g.cpu_ms));
+ o.insert("fellBack".into(), json!(self.fallbacks.load(Ordering::Relaxed)));
+ }
+ if let (Some(r), Some(o)) = (&self.remote, v.as_object_mut()) {
+ o.insert("remote".into(), r.describe());
+ }
+ v
+ }
+}
+
+/// The loader of this build: our pure-Rust executor with the `denoise` feature, else one that says why nothing can run.
+pub(crate) fn default_loader() -> Loader {
+ #[cfg(feature = "denoise")]
+ {
+ Arc::new(|path, manifest| {
+ // the server must run this very file: identified by the manifest's hash, else the file's
+ let sha256 = match &manifest.sha256 {
+ Some(h) => Some(h.clone()),
+ None if std::env::var_os("LIGHTCRAFT_DENOISE_URL").is_some() => lightcraft_denoise::hash::sha256_file(path).ok(),
+ None => None,
+ };
+ let remote = sha256.and_then(|h| RemoteSide::from_env(&h, manifest.tile as usize));
+ Ok(Arc::new(network(path, manifest, remote)?) as Arc<dyn Model>)
+ })
+ }
+ #[cfg(not(feature = "denoise"))]
+ {
+ Arc::new(|_, _| Err("this build cannot run denoise models".to_string()))
+ }
+}
+
+/// The model at `path` on this computer, sharing tiles with `remote` when there is one.
+#[cfg(feature = "denoise")]
+pub(super) fn network(path: &Path, manifest: &DenoiserManifest, remote: Option<RemoteSide>) -> Result<Network, String> {
+ let cpu = lightcraft_denoise::runtime::CpuRunner::load(path, manifest).map_err(|e| e.to_string())?;
+ Ok(Network {
+ remote,
+ local_card: AtomicBool::new(true),
+ cpu,
+ path: path.to_path_buf(),
+ tile: manifest.tile as usize,
+ gpu: OnceLock::new(),
+ fallbacks: AtomicUsize::new(0),
+ last_threads: AtomicUsize::new(0),
+ })
+}
+
+/// A model ready to be served to other computers (`lightcraft-cli denoise-serve`).
+#[cfg(feature = "denoise")]
+pub struct Serving {
+ pub model: lightcraft_denoise::remote::ModelId,
+ pub tile: usize,
+ /// Where its tiles run here: the graphics card's name, or why the CPU does.
+ pub device: String,
+ pub run: lightcraft_denoise::remote::RunTile,
+}
+
+/// The installed model `id` (default: the one selected in Settings) in the models folder `dir` (default: this user's),
+/// set up to run on `run_on` as the app would: the card checked against the CPU first, the CPU when the card can't.
+#[cfg(feature = "denoise")]
+pub fn serving(dir: Option<&Path>, id: Option<&str>, run_on: RunOn) -> Result<Serving, String> {
+ let dir = match dir {
+ Some(d) => d.to_path_buf(),
+ None => crate::config::default_denoise_models_dir().ok_or("no denoise models folder (set LIGHTCRAFT_DENOISE_MODELS)")?,
+ };
+ let want = match id {
+ Some(i) => i.to_string(),
+ None => read_settings(&dir).model.ok_or_else(|| {
+ format!("no denoise model is selected in {} (install one in Settings ▸ AI Denoise, or name it with --model)", dir.display())
+ })?,
+ };
+ let installed = installed_models(&dir)
+ .into_iter()
+ .find(|m| m.manifest.id == want)
+ .ok_or_else(|| format!("no installed denoise model `{want}` in {}", dir.display()))?;
+ let m = &installed.manifest;
+ // the same identity a client derives (see `default_loader`)
+ let sha256 = match &m.sha256 {
+ Some(h) => h.clone(),
+ None => lightcraft_denoise::hash::sha256_file(&installed.onnx).map_err(|e| format!("{}: {e}", installed.onnx.display()))?,
+ };
+ let net = Arc::new(network(&installed.onnx, m, None)?);
+ let tile = m.tile as usize;
+ // set the card up now (and say where the tiles run) rather than with the first client's tile
+ let _ = net.runner(run_on, GPU_PARALLEL);
+ let d = net.device(run_on);
+ let device = match d["kind"].as_str() {
+ Some("gpu") => format!("GPU: {}", d["adapter"].as_str().unwrap_or("?")),
+ _ => format!("CPU ({})", d["reason"].as_str().unwrap_or("chosen")),
+ };
+ let run: lightcraft_denoise::remote::RunTile = Arc::new(move |input: &[f32]| net.runner(run_on, GPU_PARALLEL).0.run(input));
+ Ok(Serving { model: lightcraft_denoise::remote::ModelId { id: m.id.clone(), version: m.version.clone(), sha256 }, tile, device, run })
+}
+
+/// A model that is loaded on first use and kept, shared by the background job and the exports.
+pub(crate) type ModelSlot = Arc<Mutex<Option<Arc<dyn Model>>>>;
diff --git a/crates/engine/src/denoise/products.rs b/crates/engine/src/denoise/products.rs
new file mode 100644
index 000000000..499e8fd6e
--- /dev/null
+++ b/crates/engine/src/denoise/products.rs
@@ -0,0 +1,234 @@
+use super::model::{Loader, Model, ModelSlot, RunOn, STACK};
+use super::settings::EXT;
+use super::*;
+
+/// Samples at or above this (white = 1) are clipped: the model leaves them as the camera recorded them.
+const CLIP: f32 = 0.99;
+
+/// The model in use.
+pub(crate) struct Active {
+ pub id: String,
+ pub manifest: DenoiserManifest,
+ pub onnx: PathBuf,
+ pub fingerprint: String,
+ pub model: ModelSlot,
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// products
+
+/// The photos whose files can be denoised at all: raw files developed from their sensor data and read from disk.
+pub(crate) fn eligible(p: &Photo) -> bool {
+ p.develops_raw() && matches!(p.source, Source::File { .. }) && !p.deleted
+}
+
+/// A product's name: what it was made from and with.
+pub(crate) fn product_key(fingerprint: &str, p: &Photo) -> String {
+ let mut h = Hasher128::new();
+ h.str("lcdn").u64(u64::from(product::ALGORITHM)).str(fingerprint).str(&crate::media::content_key(p));
+ h.finish().to_string()
+}
+
+pub(crate) fn product_path(dir: &Path, key: &str) -> PathBuf {
+ dir.join(format!("{key}.{EXT}"))
+}
+
+/// The `.lcdn` files in `dir`: path, size, last change.
+pub(super) fn products_in(dir: &Path) -> Vec<(PathBuf, u64, SystemTime)> {
+ let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
+ entries
+ .flatten()
+ .filter(|e| e.path().extension().is_some_and(|x| x == EXT))
+ .filter_map(|e| {
+ let m = e.metadata().ok()?;
+ m.is_file().then(|| (e.path(), m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH)))
+ })
+ .collect()
+}
+
+/// Make sure only one thread writes a product at a time: a second one waits, then finds it made.
+fn making() -> &'static (Mutex<HashSet<PathBuf>>, Condvar) {
+ static MAKING: OnceLock<(Mutex<HashSet<PathBuf>>, Condvar)> = OnceLock::new();
+ MAKING.get_or_init(|| (Mutex::new(HashSet::new()), Condvar::new()))
+}
+
+struct Slot(PathBuf);
+
+impl Slot {
+ fn enter(path: &Path) -> Slot {
+ let (set, cv) = making();
+ let mut g = set.lock().unwrap_or_else(PoisonError::into_inner);
+ while g.contains(path) {
+ g = cv.wait(g).unwrap_or_else(PoisonError::into_inner);
+ }
+ g.insert(path.to_path_buf());
+ Slot(path.to_path_buf())
+ }
+}
+
+impl Drop for Slot {
+ fn drop(&mut self) {
+ let (set, cv) = making();
+ set.lock().unwrap_or_else(PoisonError::into_inner).remove(&self.0);
+ cv.notify_all();
+ }
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// making a product
+
+/// Everything a thread needs to make one product, detached from the session.
+#[derive(Clone)]
+pub(crate) struct JobSpec {
+ /// The original file.
+ pub path: String,
+ pub read: Option<ByteReader>,
+ pub product: PathBuf,
+ pub key: String,
+ pub manifest: DenoiserManifest,
+ pub onnx: PathBuf,
+ pub model: ModelSlot,
+ pub loader: Loader,
+ /// Tiles run at once.
+ pub parallel: usize,
+ /// Where the model runs (the setting).
+ pub run_on: RunOn,
+}
+
+/// Why a product was not made.
+#[derive(Debug, PartialEq)]
+pub(crate) enum MakeError {
+ /// Denoise does not apply to this file (not a Bayer raw): nothing is wrong.
+ Unsupported(String),
+ Failed(String),
+ Cancelled,
+}
+
+impl From<RunError> for MakeError {
+ fn from(e: RunError) -> Self {
+ match e {
+ RunError::Cancelled => MakeError::Cancelled,
+ other => MakeError::Failed(other.to_string()),
+ }
+ }
+}
+
+/// Progress and cancel of a running job.
+#[derive(Default)]
+pub(crate) struct Progress {
+ pub(super) done: AtomicUsize,
+ pub(super) total: AtomicUsize,
+ cancel: AtomicBool,
+}
+
+impl Progress {
+ /// Ask the job to stop at the next tile.
+ pub(crate) fn cancel_now(&self) {
+ self.cancel.store(true, Ordering::Relaxed);
+ }
+}
+
+/// The loaded model, loading it first when it is not (kept for the next photo).
+pub(super) fn loaded(spec: &JobSpec) -> Result<Arc<dyn Model>, MakeError> {
+ if let Some(m) = spec.model.lock().unwrap_or_else(PoisonError::into_inner).clone() {
+ return Ok(m);
+ }
+ let m = (spec.loader)(&spec.onnx, &spec.manifest).map_err(|e| MakeError::Failed(format!("the denoise model could not be loaded: {e}")))?;
+ *spec.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(m.clone());
+ Ok(m)
+}
+
+/// The cell `[(0,0), (1,0), (0,1), (1,1)]` colours of a Bayer mosaic and so its layout.
+fn layout_of(cfa: &lightcraft_raw::Cfa) -> Option<Layout> {
+ Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)])
+}
+
+/// Make the product of `spec` unless it is there: decode the photo, run the model over its mosaic, write the result.
+/// `Ok(true)` when it was made now, `Ok(false)` when it already existed.
+pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Result<bool, MakeError> {
+ let _slot = Slot::enter(&spec.product);
+ if product::is_current(&spec.product, &spec.key) {
+ return Ok(false);
+ }
+ let began = web_time::Instant::now();
+ let cancelled = || progress.is_some_and(|p| p.cancel.load(Ordering::Relaxed));
+ if cancelled() {
+ return Err(MakeError::Cancelled);
+ }
+ let bytes = match &spec.read {
+ Some(r) => r(&spec.path).map_err(MakeError::Failed)?,
+ None => std::fs::read(&spec.path).map_err(|e| MakeError::Failed(format!("{}: {e}", spec.path)))?,
+ };
+ // the decode and the normalised mosaic wait their turn at the process-wide memory gate, as every decode does
+ let (mosaic, layout) = {
+ let _held = crate::memory::work_gate().acquire(bytes.len().saturating_mul(4));
+ if lightcraft_raw::probe(&bytes).is_none() {
+ return Err(MakeError::Unsupported("not a raw file".into()));
+ }
+ let raw = match lightcraft_raw::decode(&bytes) {
+ Ok(r) => r,
+ Err(lightcraft_raw::RawError::Unsupported(why)) => return Err(MakeError::Unsupported(why)),
+ Err(e) => return Err(MakeError::Failed(e.to_string())),
+ };
+ drop(bytes);
+ if raw.cpp != 1 || !raw.cfa.as_ref().is_some_and(lightcraft_raw::Cfa::is_bayer) {
+ return Err(MakeError::Unsupported(
+ "only Bayer raw files can be denoised (this one has another sensor layout or is already demosaiced)".into(),
+ ));
+ }
+ let n = raw.normalized().map_err(|e| MakeError::Failed(e.to_string()))?;
+ drop(raw);
+ let layout =
+ n.cfa.as_ref().and_then(layout_of).ok_or_else(|| MakeError::Unsupported("the colour filter layout is not a Bayer pattern".into()))?;
+ (n, layout)
+ };
+ if cancelled() {
+ return Err(MakeError::Cancelled);
+ }
+ let decoded = began.elapsed();
+ let model = loaded(spec)?;
+ // Packed input, returned RGB and the preceding group's RGB overlap while tiles run.
+ let scratch =
+ model.bytes_per_tile().saturating_add(128usize.saturating_mul(spec.manifest.tile as usize).saturating_mul(spec.manifest.tile as usize));
+ let (_, budget) = crate::memory::work_gate().usage();
+ let parallel = spec.parallel.clamp(1, 4).min((budget / scratch.max(1)).max(1));
+ let _held = crate::memory::work_gate().acquire(mosaic.data.len().saturating_mul(20).saturating_add(scratch.saturating_mul(parallel)));
+ let (runner, parallel) = model.runner(spec.run_on, parallel);
+ let params = Params {
+ tile: spec.manifest.tile as usize,
+ overlap: spec.manifest.overlap as usize,
+ gain: spec.manifest.gain,
+ clip: Some(CLIP),
+ parallel: parallel.max(1),
+ };
+ let report = |done: usize, total: usize| {
+ if let Some(p) = progress {
+ p.total.store(total, Ordering::Relaxed);
+ p.done.store(done, Ordering::Relaxed);
+ }
+ };
+ let ctl = Control { cancel: progress.map(|p| &p.cancel), progress: Some(&report) };
+ // the tiles run on threads of their own, so a busy window's renders keep theirs
+ let pool = rayon::ThreadPoolBuilder::new()
+ .num_threads(params.parallel)
+ .stack_size(STACK)
+ .thread_name(|i| format!("denoise-{i}"))
+ .build()
+ .map_err(|e| MakeError::Failed(format!("could not start the worker threads: {e}")))?;
+ let rgb = pool.install(|| denoise_bayer(&mosaic.data, mosaic.width, mosaic.height, layout, runner, &params, &ctl))?;
+ drop(mosaic);
+ if cancelled() {
+ return Err(MakeError::Cancelled);
+ }
+ let ran = began.elapsed();
+ product::write(&spec.product, &rgb, &spec.key).map_err(|e| MakeError::Failed(format!("could not save the denoised picture: {e}")))?;
+ if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
+ eprintln!(
+ "[denoise] read + decode {:.0} ms, model (set-up, tiles, blend) {:.0} ms, write {:.0} ms",
+ decoded.as_secs_f64() * 1e3,
+ (ran - decoded).as_secs_f64() * 1e3,
+ (began.elapsed() - ran).as_secs_f64() * 1e3
+ );
+ }
+ Ok(true)
+}
diff --git a/crates/engine/src/denoise/session.rs b/crates/engine/src/denoise/session.rs
new file mode 100644
index 000000000..6a452f693
--- /dev/null
+++ b/crates/engine/src/denoise/session.rs
@@ -0,0 +1,509 @@
+#[cfg(doc)]
+use super::model::Model;
+use super::model::STACK;
+use super::products::{Active, JobSpec, MakeError, Progress, eligible, make_product, product_key, product_path, products_in};
+use super::settings::{DIR, SETTINGS_TTL, installed_models, read_settings};
+use super::state::{AUTO_LOOK, Failure, MAX_QUEUE, Outcome, Pace, PhotoState, Running};
+use super::*;
+
+impl Session {
+ /// Where the active model's tiles run (see [`Model::device`]); `{kind: "none"}` while no model is loaded.
+ pub(crate) fn denoise_device(&self) -> Value {
+ let loaded = self.denoise.active.as_ref().and_then(|a| a.model.lock().unwrap_or_else(PoisonError::into_inner).clone());
+ match loaded {
+ Some(m) => m.device(self.denoise.settings.run_on()),
+ None => json!({"kind": "none"}),
+ }
+ }
+
+ /// The folder the products of the open library are kept in; `None` without a library on disk.
+ pub(crate) fn denoise_products_dir(&self) -> Option<PathBuf> {
+ self.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join(DIR))
+ }
+
+ /// Read which model is in use (at most every couple of seconds): the manifest and file of the model the settings
+ /// name, if it is installed.
+ pub(crate) fn denoise_refresh_active(&mut self, force: bool) {
+ let fresh = self.denoise.active_seen.is_some_and(|t| t.elapsed() < SETTINGS_TTL);
+ if fresh && !force {
+ return;
+ }
+ self.denoise.active_seen = Some(web_time::Instant::now());
+ self.denoise.settings = self.denoise.models_dir.as_deref().map(read_settings).unwrap_or_default();
+ let chosen = self.denoise.models_dir.as_deref().and_then(|dir| {
+ let id = self.denoise.settings.model.clone()?;
+ installed_models(dir).into_iter().find(|i| i.manifest.id == id)
+ });
+ let found = chosen.map(|i| (i.fingerprint(), i));
+ let same = match (&self.denoise.active, &found) {
+ (None, None) => true,
+ (Some(a), Some((fp, _))) => a.fingerprint == *fp,
+ _ => false,
+ };
+ if same {
+ return;
+ }
+ // another model: what was made with the old one is not this one's, and what failed may work now
+ self.denoise.active = found.map(|(fingerprint, i)| Active {
+ id: i.manifest.id.clone(),
+ manifest: i.manifest,
+ onnx: i.onnx,
+ fingerprint,
+ model: Arc::new(Mutex::new(None)),
+ });
+ if let Some(r) = self.denoise.running.take() {
+ r.progress.cancel_now();
+ }
+ self.denoise.queue.clear();
+ self.denoise.failed.clear();
+ self.denoise.dirty = true;
+ }
+
+ fn denoise_spec(&self, p: &Photo) -> Option<(DenoiseSpec, String)> {
+ let (dir, active) = (self.denoise_products_dir()?, self.denoise.active.as_ref()?);
+ let key = product_key(&active.fingerprint, p);
+ Some((DenoiseSpec { product: product_path(&dir, &key), key: key.clone(), make: None }, key))
+ }
+
+ /// Make the index of ready photos from the products on disk (a directory listing and a hash per photo: no file is
+ /// opened). Pictures that appeared or vanished drop the decoded sources they affect. The photo being made is its
+ /// job's to report (see [`Self::denoise_check_photo`]).
+ pub(crate) fn denoise_reindex(&mut self) {
+ self.denoise.dirty = false;
+ self.denoise.reindexed = Some((self.catalog.revision, web_time::Instant::now()));
+ let running = self.denoise.running.as_ref().map(|r| r.photo);
+ let mut now: HashMap<PhotoId, DenoiseSpec> = HashMap::new();
+ if let (Some(dir), Some(active)) = (self.denoise_products_dir(), self.denoise.active.as_ref()) {
+ let names: HashSet<String> =
+ products_in(&dir).into_iter().filter_map(|(p, _, _)| p.file_stem().and_then(|s| s.to_str()).map(str::to_string)).collect();
+ if !names.is_empty() {
+ for p in self.catalog.photos().filter(|p| eligible(p) && running != Some(p.id)) {
+ let key = product_key(&active.fingerprint, p);
+ if names.contains(&key) {
+ now.insert(p.id, DenoiseSpec { product: product_path(&dir, &key), key, make: None });
+ }
+ }
+ }
+ }
+ let mut changed = false;
+ for id in self.media.denoise.ids() {
+ if !now.contains_key(&id) {
+ self.media.denoise.remove(id);
+ self.media.forget(id);
+ changed = true;
+ }
+ }
+ for (id, spec) in now {
+ if self.media.denoise.spec(id).is_none_or(|old| old.key != spec.key) {
+ self.media.denoise.set(id, spec);
+ self.media.forget(id);
+ changed = true;
+ }
+ }
+ if changed {
+ self.denoise.generation += 1;
+ }
+ }
+
+ /// A photo whose file changed (its content key is not the one its index entry was made for) has no picture.
+ ///
+ /// The photo whose picture is being made is left to its job: the worker saves the picture before it reports, so the
+ /// file is on disk while the job still counts as running. Finding it here would make the photo Ready while the session
+ /// is busy with it (#719); [`Self::denoise_poll`] makes it Ready in the pump that takes the job's result instead.
+ fn denoise_check_photo(&mut self, id: PhotoId) {
+ if self.denoise.running.as_ref().is_some_and(|r| r.photo == id) {
+ return;
+ }
+ let Some(p) = self.catalog.photo(id).cloned() else { return };
+ let Some((spec, _)) = self.denoise_spec(&p) else { return };
+ match self.media.denoise.spec(id) {
+ Some(have) if have.key != spec.key => {
+ self.media.denoise.remove(id);
+ self.media.forget(id);
+ self.denoise.generation += 1;
+ }
+ None if spec.product.is_file() && product::is_current(&spec.product, &spec.key) => {
+ // made by an export, or by another session
+ self.media.denoise.set(id, spec);
+ self.media.forget(id);
+ self.denoise.generation += 1;
+ }
+ _ => {}
+ }
+ }
+
+ /// How photo `id` stands.
+ pub fn denoise_photo_state(&self, id: PhotoId) -> PhotoState {
+ let Some(p) = self.catalog.photo(id) else { return PhotoState::NotApplicable };
+ if !eligible(p) {
+ return PhotoState::NotApplicable;
+ }
+ if self.denoise.active.is_none() {
+ // the model folder has not been looked at yet (just started): not "no model" until it has been
+ return if self.denoise.active_seen.is_none() { PhotoState::Idle } else { PhotoState::NoModel };
+ }
+ if self.denoise_products_dir().is_none() {
+ return PhotoState::Failed {
+ why: "AI Denoise keeps its pictures in the library folder, and this session has none.".into(),
+ unsupported: true,
+ };
+ }
+ if self.media.denoise.spec(id).is_some() {
+ return PhotoState::Ready;
+ }
+ if let Some(r) = self.denoise.running.as_ref().filter(|r| r.photo == id) {
+ return PhotoState::Running { done: r.progress.done.load(Ordering::Relaxed), total: r.progress.total.load(Ordering::Relaxed) };
+ }
+ if let Some(ahead) = self.denoise.queue.iter().position(|q| *q == id) {
+ return PhotoState::Queued { ahead: ahead + usize::from(self.denoise.running.is_some()) };
+ }
+ let key = self.denoise_spec(p).map(|s| s.1);
+ match self.denoise.failed.get(&id) {
+ Some(f) if Some(&f.key) == key.as_ref() => PhotoState::Failed { why: f.why.clone(), unsupported: f.unsupported },
+ _ => PhotoState::Idle,
+ }
+ }
+
+ /// Ask for photos' pictures to be made (the ones that are not made, queued or known to be impossible). With `retry`
+ /// a photo that failed before is tried again. A photo that is not raw is skipped. Returns how many were queued.
+ pub(crate) fn denoise_enqueue(&mut self, ids: &[PhotoId], front: bool, retry: bool) -> usize {
+ self.denoise_refresh_active(false);
+ if self.denoise.active.is_none() || self.denoise_products_dir().is_none() {
+ return 0;
+ }
+ let mut added = 0;
+ for id in ids {
+ let Some(p) = self.catalog.photo(*id).cloned() else { continue };
+ if !eligible(&p) || self.media.denoise.spec(*id).is_some() || self.denoise.queue.contains(id) {
+ continue;
+ }
+ if self.denoise.running.as_ref().is_some_and(|r| r.photo == *id) || self.denoise.queue.len() >= MAX_QUEUE {
+ continue;
+ }
+ let Some((_, key)) = self.denoise_spec(&p) else { continue };
+ if retry {
+ self.denoise.failed.remove(id);
+ } else if self.denoise.failed.get(id).is_some_and(|f| f.key == key) {
+ continue;
+ }
+ if front {
+ self.denoise.queue.push_front(*id);
+ } else {
+ self.denoise.queue.push_back(*id);
+ }
+ added += 1;
+ }
+ added
+ }
+
+ /// Stop the job that is running, and with `all` forget everything that waits.
+ pub(crate) fn denoise_cancel(&mut self, photo: Option<PhotoId>, all: bool) -> usize {
+ let mut n = 0;
+ if let Some(r) = &self.denoise.running
+ && photo.is_none_or(|p| p == r.photo)
+ {
+ r.progress.cancel_now();
+ n += 1;
+ }
+ let before = self.denoise.queue.len();
+ match photo {
+ Some(p) => self.denoise.queue.retain(|q| *q != p),
+ None if all => self.denoise.queue.clear(),
+ None => {}
+ }
+ n + before - self.denoise.queue.len()
+ }
+
+ /// The photos the automatic queue looks at: the one open and the ones selected, those with a Denoise amount.
+ fn denoise_wanted(&self) -> Vec<PhotoId> {
+ let Selection { active, ids, .. } = &self.selection;
+ active
+ .iter()
+ .chain(ids.iter().take(AUTO_LOOK))
+ .copied()
+ .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0))
+ .collect()
+ }
+
+ fn denoise_job_spec(&self, p: &Photo, parallel: usize) -> Option<JobSpec> {
+ let Source::File { path } = &p.source else { return None };
+ let (spec, _) = self.denoise_spec(p)?;
+ let active = self.denoise.active.as_ref()?;
+ Some(JobSpec {
+ path: path.clone(),
+ read: self.media.file_bytes.clone(),
+ product: spec.product,
+ key: spec.key,
+ manifest: active.manifest.clone(),
+ onnx: active.onnx.clone(),
+ model: active.model.clone(),
+ loader: self.denoise.loader.clone(),
+ parallel,
+ run_on: self.denoise.settings.run_on(),
+ })
+ }
+
+ /// Start the next queued photo on its own thread.
+ fn denoise_start_next(&mut self, parallel: usize) {
+ while let Some(id) = self.denoise.queue.pop_front() {
+ let Some(p) = self.catalog.photo(id).cloned() else { continue };
+ if !eligible(&p) || self.media.denoise.spec(id).is_some() {
+ continue;
+ }
+ let Some(spec) = self.denoise_job_spec(&p, parallel) else { continue };
+ // another session or an export may have made it since it was queued
+ if product::is_current(&spec.product, &spec.key) {
+ self.denoise_check_photo(id);
+ continue;
+ }
+ let progress = Arc::new(Progress::default());
+ let outcome: Arc<Mutex<Option<Outcome>>> = Arc::new(Mutex::new(None));
+ let (key, product) = (spec.key.clone(), spec.product.clone());
+ let (worker_progress, worker_outcome) = (progress.clone(), outcome.clone());
+ let started = std::thread::Builder::new().name("denoise".into()).stack_size(STACK).spawn(move || {
+ let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_product(&spec, Some(&worker_progress))));
+ let o = match result {
+ Ok(Ok(_)) => Outcome::Done,
+ Ok(Err(MakeError::Cancelled)) => Outcome::Cancelled,
+ Ok(Err(MakeError::Unsupported(why))) => Outcome::Failed(why, true),
+ Ok(Err(MakeError::Failed(why))) => Outcome::Failed(why, false),
+ Err(_) => Outcome::Failed("the denoiser stopped unexpectedly".into(), false),
+ };
+ *worker_outcome.lock().unwrap_or_else(PoisonError::into_inner) = Some(o);
+ });
+ match started {
+ Ok(_) => {
+ self.denoise.running = Some(Running { photo: id, key, product, progress, outcome, started: web_time::Instant::now() });
+ }
+ Err(e) => {
+ self.denoise.failed.insert(id, Failure { key, why: format!("could not start the denoiser: {e}"), unsupported: false });
+ continue;
+ }
+ }
+ return;
+ }
+ }
+
+ /// Take in the running job's result.
+ fn denoise_poll(&mut self) {
+ let Some(r) = &self.denoise.running else { return };
+ let Some(outcome) = r.outcome.lock().unwrap_or_else(PoisonError::into_inner).take() else { return };
+ let Some(r) = self.denoise.running.take() else { return };
+ match outcome {
+ Outcome::Done => {
+ self.denoise.made += 1;
+ self.denoise_check_photo(r.photo);
+ self.denoise_evict(Some(&r.product));
+ }
+ Outcome::Failed(why, unsupported) => {
+ self.denoise.failed.insert(r.photo, Failure { key: r.key, why, unsupported });
+ }
+ Outcome::Cancelled => {}
+ }
+ // the picture's state changed: the interface draws it again
+ self.denoise.generation += 1;
+ }
+
+ /// Keep the cache within its limit: the pictures nobody asks for go first, the oldest first; `keep` (the one just
+ /// made) never goes. The index forgets what was deleted.
+ pub(crate) fn denoise_evict(&mut self, keep: Option<&Path>) {
+ let Some(models) = self.denoise.models_dir.clone() else { return };
+ self.denoise_evict_to(read_settings(&models).cache_bytes(), keep);
+ }
+
+ /// [`Self::denoise_evict`] for a limit of `limit` bytes.
+ pub(crate) fn denoise_evict_to(&mut self, limit: u64, keep: Option<&Path>) {
+ let Some(dir) = self.denoise_products_dir() else { return };
+ let mut files = products_in(&dir);
+ let mut total: u64 = files.iter().map(|f| f.1).sum();
+ if total <= limit {
+ return;
+ }
+ // the pictures photos use now (a Denoise amount above 0) are the last to go
+ let wanted: HashSet<PathBuf> = self
+ .media
+ .denoise
+ .ids()
+ .into_iter()
+ .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0))
+ .filter_map(|id| self.media.denoise.spec(id).map(|s| s.product.clone()))
+ .collect();
+ files.sort_by_key(|(path, _, modified)| (wanted.contains(path), *modified));
+ let mut gone: HashSet<PathBuf> = HashSet::new();
+ for (path, size, _) in files {
+ if total <= limit {
+ break;
+ }
+ if keep == Some(path.as_path()) {
+ continue;
+ }
+ if std::fs::remove_file(&path).is_ok() {
+ total = total.saturating_sub(size);
+ gone.insert(path);
+ }
+ }
+ if gone.is_empty() {
+ return;
+ }
+ for id in self.media.denoise.ids() {
+ if self.media.denoise.spec(id).is_some_and(|s| gone.contains(&s.product)) {
+ self.media.denoise.remove(id);
+ self.media.forget(id);
+ }
+ }
+ self.denoise.generation += 1;
+ }
+
+ /// Delete every product of this library (they are made again when wanted).
+ pub(crate) fn denoise_clear(&mut self) -> (usize, u64) {
+ self.denoise_cancel(None, true);
+ let (mut files, mut bytes) = (0, 0);
+ if let Some(dir) = self.denoise_products_dir() {
+ for (path, size, _) in products_in(&dir) {
+ if std::fs::remove_file(&path).is_ok() {
+ files += 1;
+ bytes += size;
+ }
+ }
+ }
+ self.denoise.failed.clear();
+ self.denoise.touch();
+ (files, bytes)
+ }
+
+ /// Per frame: take in what finished, look at what the user is looking at, and keep one job going. Cheap when
+ /// nothing is happening (no file is opened unless a photo with a Denoise amount lacks its picture).
+ pub(crate) fn denoise_pump(&mut self, pace: Pace) {
+ crate::cmd::denoise::finish_downloads(self);
+ self.denoise_refresh_active(false);
+ // photos that changed (a replaced file has another key) must not keep the picture of what they were
+ let stale = !self.media.denoise.is_empty()
+ && self.denoise.reindexed.is_none_or(|(rev, at)| rev != self.catalog.revision && at.elapsed() > SETTINGS_TTL);
+ if self.denoise.dirty || stale {
+ self.denoise_reindex();
+ }
+ self.denoise_poll();
+ if self.denoise.active.is_none() || self.denoise_products_dir().is_none() {
+ return;
+ }
+ let (cap, auto) = (self.denoise.settings.threads, self.denoise.settings.auto());
+ let wanted = self.denoise_wanted();
+ for id in &wanted {
+ self.denoise_check_photo(*id);
+ }
+ if auto {
+ // the photo being edited goes first
+ let (open, others): (Vec<PhotoId>, Vec<PhotoId>) = wanted.into_iter().partition(|id| self.selection.active == Some(*id));
+ self.denoise_enqueue(&others, false, false);
+ self.denoise_enqueue(&open, true, false);
+ }
+ if self.denoise.running.is_none() && pace != Pace::Pause {
+ let cores = std::thread::available_parallelism().map_or(1, |n| n.get());
+ self.denoise_start_next(pace.parallel(cores, cap));
+ }
+ }
+
+ /// The statistics `denoise.status` reports besides the queue (reads the cache folder).
+ pub(crate) fn denoise_cache_usage(&self) -> (usize, u64) {
+ match self.denoise_products_dir() {
+ Some(dir) => {
+ let files = products_in(&dir);
+ (files.len(), files.iter().map(|f| f.1).sum())
+ }
+ None => (0, 0),
+ }
+ }
+
+ /// Are pictures made without being asked for photos being looked at (Settings ▸ AI Denoise)?
+ pub fn denoise_auto(&self) -> bool {
+ self.denoise.settings.auto()
+ }
+
+ /// Is a picture being made, or waiting for its turn with a model to make it? (Headless runs wait for this.)
+ pub fn denoise_busy(&self) -> bool {
+ self.denoise.running.is_some() || (!self.denoise.queue.is_empty() && self.denoise.active.is_some())
+ }
+
+ /// Keep AI Denoise's row in the activity stack up to date (each `denoise.pump`): there while a photo is being made or
+ /// waits its turn, with the running photo's tiles and its file name; gone when the queue is idle. It has no ✕: what is
+ /// made follows the photos' Denoise amounts and Settings ▸ AI Denoise.
+ pub(crate) fn denoise_track(&mut self) {
+ if !self.denoise_busy() {
+ self.denoise.task = None;
+ return;
+ }
+ let task = self.denoise.task.get_or_insert_with(|| {
+ let t = self.activity.start("denoise", "AI Denoise", crate::activity::Cancel::No);
+ t.set_unit(crate::activity::Unit::Percent);
+ t
+ });
+ match &self.denoise.running {
+ Some(r) => {
+ task.progress(r.progress.done.load(Ordering::Relaxed) as u64, r.progress.total.load(Ordering::Relaxed) as u64);
+ task.detail(self.catalog.photo(r.photo).map_or("", |p| p.file_name.as_str()));
+ }
+ None => {
+ task.progress(0, 0);
+ task.detail("");
+ }
+ }
+ }
+
+ /// Seconds the running job has been going, and its progress.
+ pub(crate) fn denoise_running_json(&self) -> Value {
+ match &self.denoise.running {
+ Some(r) => json!({
+ "photo": r.photo.0,
+ "done": r.progress.done.load(Ordering::Relaxed),
+ "total": r.progress.total.load(Ordering::Relaxed),
+ "seconds": r.started.elapsed().as_secs_f64(),
+ }),
+ None => Value::Null,
+ }
+ }
+
+ pub(crate) fn denoise_made(&self) -> u64 {
+ self.denoise.made
+ }
+
+ /// The running job's product path and key (tests stand in for the worker's save).
+ #[cfg(test)]
+ pub(crate) fn denoise_running_product(&self) -> Option<(PathBuf, String)> {
+ self.denoise.running.as_ref().map(|r| (r.product.clone(), r.key.clone()))
+ }
+
+ /// What an export of photo `id` with `settings` needs for its Denoise amount: its denoised picture, made now when the
+ /// queue has not got to it. `Ok(None)` when denoise does not apply (amount 0, not a Bayer raw), `Err` when it should
+ /// have and cannot (no model chosen, nowhere to keep the picture, the model failing).
+ pub(crate) fn denoise_for_export(&mut self, id: PhotoId, settings: &DevelopSettings) -> Result<Option<(DenoiseSpec, PairLoader)>, String> {
+ if settings.denoise_amount() <= 0.0 {
+ return Ok(None);
+ }
+ let Some(p) = self.catalog.photo(id).cloned() else { return Ok(None) };
+ let Some(loader) = self.media.denoise.loader.clone() else { return Ok(None) };
+ if !eligible(&p) {
+ return Ok(None);
+ }
+ self.denoise_refresh_active(false);
+ let name = &p.file_name;
+ if self.denoise.active.is_none() {
+ return Err(format!("{name}: Denoise is on for this photo but no denoise model is chosen; choose one in Settings, or set Denoise to 0"));
+ }
+ if self.denoise_products_dir().is_none() {
+ return Err(format!("{name}: Denoise needs a library on disk to keep its pictures in"));
+ }
+ self.denoise_check_photo(id);
+ let Some(job) = self.denoise_job_spec(&p, Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None)) else {
+ return Ok(None);
+ };
+ let label = name.clone();
+ let spec = job.clone();
+ let make: MakeProduct = Arc::new(move || match make_product(&spec, None) {
+ Ok(_) => Ok(true),
+ Err(MakeError::Unsupported(_)) => Ok(false),
+ Err(MakeError::Failed(why)) => Err(format!("{label}: could not denoise: {why}")),
+ Err(MakeError::Cancelled) => Err(format!("{label}: denoise was cancelled")),
+ });
+ Ok(Some((DenoiseSpec { product: job.product, key: job.key, make: Some(make) }, loader)))
+ }
+}
diff --git a/crates/engine/src/denoise/settings.rs b/crates/engine/src/denoise/settings.rs
new file mode 100644
index 000000000..f8acd5320
--- /dev/null
+++ b/crates/engine/src/denoise/settings.rs
@@ -0,0 +1,135 @@
+use super::gpu::GPU_SETUP_MARKER;
+use super::model::RunOn;
+use super::*;
+
+/// Extension of a product file.
+pub const EXT: &str = "lcdn";
+
+/// Where the products of a library are kept, inside the library's folder (safe to delete, like `thumbs/`).
+pub const DIR: &str = "denoise";
+
+/// What the cache may take up unless the user says otherwise (GB). A product is about as large as the raw file's
+/// decoded samples, compressed: ~100 MB for 24 megapixels.
+pub const DEFAULT_CACHE_GB: u32 = 20;
+
+/// How long a look at the model settings is trusted.
+pub(super) const SETTINGS_TTL: Duration = Duration::from_secs(2);
+
+// ---------------------------------------------------------------------------------------------------------------------
+// settings and installed models
+
+/// `settings.json` in the models folder.
+#[derive(Clone, Default, Serialize, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+pub(crate) struct Settings {
+ /// The installed model in use; `None`: denoise is off.
+ pub model: Option<String>,
+ /// Photos with a Denoise amount above 0 that are being looked at (active, selected) get their picture made without
+ /// being asked (`None`: yes).
+ pub auto: Option<bool>,
+ /// Cache size limit (GB; `None`: [`DEFAULT_CACHE_GB`]).
+ pub cache_gb: Option<u32>,
+ /// Most tiles run at once (`None`: by how hard the pump says to work).
+ pub threads: Option<u32>,
+ /// Where the model runs: `auto`, `gpu` or `cpu` (`None` or anything else: `auto`; see [`RunOn`]).
+ pub run_on: Option<String>,
+ /// What came before `runOn`: `false` was the processor only.
+ pub gpu: Option<bool>,
+}
+
+impl Settings {
+ pub fn auto(&self) -> bool {
+ self.auto.unwrap_or(true)
+ }
+
+ pub fn run_on(&self) -> RunOn {
+ match self.run_on.as_deref().and_then(RunOn::parse) {
+ Some(r) => r,
+ None if self.gpu == Some(false) => RunOn::Cpu,
+ None => RunOn::Auto,
+ }
+ }
+
+ pub fn cache_bytes(&self) -> u64 {
+ u64::from(self.cache_gb.unwrap_or(DEFAULT_CACHE_GB).clamp(1, 10_000)) << 30
+ }
+}
+
+pub(crate) fn read_capped(path: &Path, max: usize) -> Option<Vec<u8>> {
+ use std::io::Read;
+ let mut buf = Vec::new();
+ std::fs::File::open(path).ok()?.take(max as u64 + 1).read_to_end(&mut buf).ok()?;
+ (buf.len() <= max).then_some(buf)
+}
+
+pub(crate) fn read_settings(dir: &Path) -> Settings {
+ read_capped(&dir.join("settings.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
+}
+
+/// Write a file next to its final name and rename it into place.
+pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> {
+ let part = path.with_extension("part");
+ std::fs::write(&part, bytes).map_err(|e| format!("could not write {}: {e}", path.display()))?;
+ std::fs::rename(&part, path).map_err(|e| {
+ let _ = std::fs::remove_file(&part);
+ format!("could not write {}: {e}", path.display())
+ })
+}
+
+pub(crate) fn write_settings(dir: &Path, st: &Settings) -> Result<(), String> {
+ std::fs::create_dir_all(dir).map_err(|e| format!("could not create the denoise models folder: {e}"))?;
+ write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| e.to_string())?)
+}
+
+/// An installed model folder: `<id>/model.onnx`, `denoise-model.json`, `installed.json`.
+pub(crate) struct Installed {
+ pub manifest: DenoiserManifest,
+ pub onnx: PathBuf,
+ /// What was recorded when it was installed (terms accepted, hash of the file, self-test).
+ pub accepted: Value,
+}
+
+impl Installed {
+ /// Names the model file the cache is made with: a hash of the file as installed, so another file with the same id
+ /// never finds this one's products.
+ pub fn fingerprint(&self) -> String {
+ let sha = self.accepted.get("sha256").and_then(Value::as_str).or(self.manifest.sha256.as_deref()).unwrap_or("");
+ let len = std::fs::metadata(&self.onnx).map(|m| m.len()).unwrap_or(0);
+ format!("{}@{}:{len}", self.manifest.id, sha)
+ }
+}
+
+/// The models in the folder. A folder that is not a valid model (wrong name, no model file, a manifest that does not
+/// validate or does not match its folder) is ignored, never trusted.
+pub(crate) fn installed_models(dir: &Path) -> Vec<Installed> {
+ let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };
+ let mut out = Vec::new();
+ for e in entries.flatten() {
+ let p = e.path();
+ let Some(name) = p.file_name().and_then(|n| n.to_str()).map(str::to_string) else { continue };
+ let onnx = p.join("model.onnx");
+ if !lightcraft_denoise::licence::valid_id(&name) || !p.is_dir() || !onnx.is_file() {
+ continue;
+ }
+ let Some(m) = read_capped(&p.join("denoise-model.json"), lightcraft_denoise::manifest::MAX_MANIFEST_BYTES)
+ .and_then(|b| lightcraft_denoise::manifest::parse(&b).ok())
+ else {
+ continue;
+ };
+ if m.id != name {
+ continue;
+ }
+ let accepted = read_capped(&p.join("installed.json"), 8192).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null);
+ out.push(Installed { manifest: m, onnx, accepted });
+ }
+ out.sort_by(|a, b| a.manifest.id.cmp(&b.manifest.id));
+ out
+}
+
+/// Forget that setting the graphics card up for a model closed LightCraft before (the markers [`set_up_card`] leaves):
+/// the user chose where denoise runs, so the card may be tried again.
+pub(crate) fn forget_gpu_set_ups(dir: &Path) {
+ for i in installed_models(dir) {
+ let _ = std::fs::remove_file(i.onnx.with_file_name(GPU_SETUP_MARKER));
+ }
+}
diff --git a/crates/engine/src/denoise/state.rs b/crates/engine/src/denoise/state.rs
new file mode 100644
index 000000000..76a4b6130
--- /dev/null
+++ b/crates/engine/src/denoise/state.rs
@@ -0,0 +1,179 @@
+use super::model::{Loader, default_loader};
+use super::products::{Active, Progress};
+use super::settings::Settings;
+use super::*;
+
+/// Most photos waiting in the queue.
+pub(super) const MAX_QUEUE: usize = 5000;
+
+/// Most selected photos the automatic queue looks at in one pump.
+pub(super) const AUTO_LOOK: usize = 64;
+
+// ---------------------------------------------------------------------------------------------------------------------
+// the session's state
+
+/// How hard the background work may go (the app picks it from what the user is doing, like the face scan).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Pace {
+ /// Start nothing new.
+ Pause,
+ /// Two tiles at once.
+ Light,
+ /// Half the machine.
+ Normal,
+ /// Most of the machine: the user is watching the progress.
+ Full,
+}
+
+impl Pace {
+ /// `pause`, `light`, `normal` or `full`; anything else is `normal`.
+ pub fn parse(name: Option<&str>) -> Pace {
+ match name {
+ Some("pause") => Pace::Pause,
+ Some("light") => Pace::Light,
+ Some("full") => Pace::Full,
+ _ => Pace::Normal,
+ }
+ }
+
+ /// Tiles to run at once on a machine with `cores` threads (`cap`: the user's own limit).
+ pub fn parallel(self, cores: usize, cap: Option<u32>) -> usize {
+ let n = match self {
+ Pace::Pause | Pace::Light => 2,
+ Pace::Normal => cores / 2,
+ Pace::Full => cores * 4 / 5,
+ }
+ .clamp(1, 16);
+ cap.map_or(n, |c| n.min(c.max(1) as usize))
+ }
+}
+
+/// How a photo stands, for the interface.
+#[derive(Clone, Debug, PartialEq, Serialize)]
+#[serde(tag = "state", rename_all = "camelCase")]
+pub enum PhotoState {
+ /// Not a photo denoise applies to (not a raw, offline, a demo).
+ NotApplicable,
+ /// No model is chosen: nothing can be made.
+ NoModel,
+ /// Its picture is made: the Amount slider works.
+ Ready,
+ Queued {
+ /// Photos ahead of it (0 = next).
+ ahead: usize,
+ },
+ Running {
+ done: usize,
+ total: usize,
+ },
+ /// Making it failed (`why`), or denoise does not apply to this file (`unsupported`).
+ Failed {
+ why: String,
+ unsupported: bool,
+ },
+ /// Nothing has asked for it yet.
+ Idle,
+}
+
+#[derive(Clone)]
+pub(super) struct Failure {
+ pub(super) key: String,
+ pub(super) why: String,
+ pub(super) unsupported: bool,
+}
+
+pub(super) enum Outcome {
+ Done,
+ Failed(String, bool),
+ Cancelled,
+}
+
+pub(super) struct Running {
+ pub(super) photo: PhotoId,
+ pub(super) key: String,
+ pub(super) product: PathBuf,
+ pub(super) progress: Arc<Progress>,
+ pub(super) outcome: Arc<Mutex<Option<Outcome>>>,
+ pub(super) started: web_time::Instant,
+}
+
+pub(crate) struct State {
+ /// Where the models are kept (`None` where there is no file system).
+ pub models_dir: Option<PathBuf>,
+ /// Models being downloaded at the user's request.
+ pub downloads: Downloads,
+ pub(crate) install: Option<crate::cmd::denoise::InstallJob>,
+ /// How a model is loaded (tests put another in).
+ pub loader: Loader,
+ pub(crate) active: Option<Active>,
+ pub(super) active_seen: Option<web_time::Instant>,
+ /// `settings.json` as last read (with the model choice).
+ pub(crate) settings: Settings,
+ /// The index must be made again (the library, the model or the cache changed).
+ pub(super) dirty: bool,
+ /// The catalog revision and time of the last index, to look again when photos changed.
+ pub(super) reindexed: Option<(u64, web_time::Instant)>,
+ pub(super) queue: VecDeque<PhotoId>,
+ pub(super) running: Option<Running>,
+ pub(super) failed: HashMap<PhotoId, Failure>,
+ /// Counts every change of what is ready, so the interface knows to draw again.
+ pub generation: u64,
+ /// Photos made since the session started.
+ pub(super) made: u64,
+ /// AI Denoise's row in the activity stack, while a photo is being made or waits its turn.
+ pub(super) task: Option<crate::activity::TaskGuard>,
+}
+
+impl Default for State {
+ fn default() -> Self {
+ State {
+ models_dir: None,
+ downloads: Downloads::default(),
+ install: None,
+ loader: default_loader(),
+ active: None,
+ active_seen: None,
+ settings: Settings::default(),
+ dirty: true,
+ reindexed: None,
+ queue: VecDeque::new(),
+ running: None,
+ failed: HashMap::new(),
+ generation: 0,
+ made: 0,
+ task: None,
+ }
+ }
+}
+
+impl Drop for State {
+ /// Quitting stops the job instead of leaving it to finish a photo nobody will see.
+ fn drop(&mut self) {
+ if let Some(r) = &self.running {
+ r.progress.cancel_now();
+ }
+ }
+}
+
+impl State {
+ /// The library was replaced: nothing carries over (photo ids are per library).
+ pub fn library_changed(&mut self) {
+ if let Some(r) = self.running.take() {
+ r.progress.cancel_now();
+ }
+ self.queue.clear();
+ self.failed.clear();
+ self.dirty = true;
+ self.generation += 1;
+ }
+
+ /// The model choice or the cache changed on disk: look again at the next pump.
+ pub fn touch(&mut self) {
+ self.active_seen = None;
+ self.dirty = true;
+ }
+
+ pub fn queued(&self) -> usize {
+ self.queue.len()
+ }
+}