LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
LR-MASK-OBJECTS: SAM 3 falls back to the CPU when the GPU can't run it (#1974)
helgehelge12353f57b9404b1
diff --git a/crates/engine/src/segment/mod.rs b/crates/engine/src/segment/mod.rsindex c955dcc01..b69a6dca2 100644--- a/crates/engine/src/segment/mod.rs+++ b/crates/engine/src/segment/mod.rs@@ -75,6 +75,7 @@ pub(crate) struct Input {}/// What a detail pass re-runs.+#[derive(Clone)]#[cfg_attr(not(feature = "sam"), allow(dead_code))]pub(crate) enum Prompt {Clicks(Vec<Click>),@@ -440,7 +441,11 @@ impl Session {let job = self.segment_job(id, kind, Tag::Wait, tx, region)?;self.segmenter.worker.submit(job)?;match rx.recv_timeout(WAIT) {- Ok(o) => o.result,+ Ok(o) => {+ // (shown by the next poll in an app; logged anyway)+ self.segmenter.messages.extend(o.notice);+ o.result+ }Err(_) => Err("the AI model took too long; try again".into()),}}@@ -579,7 +584,7 @@ impl Session {self.segment_queue(id, kind, tag, Some(region))?;} else {let patch = self.segment_wait(id, kind, Some(region))?;- let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false };+ let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false, notice: None };let mut polled = Polled::default();self.apply_outcome(outcome, &mut polled);}@@ -618,6 +623,7 @@ impl Session {if o.superseded {return;}+ polled.messages.extend(o.notice);match o.tag {Tag::Wait => {}Tag::Prepare => {@@ -711,6 +717,18 @@ impl Session {mod tests {use super::*;+ #[cfg(feature = "sam")]+ #[test]+ fn a_worker_notice_reaches_the_user_with_the_result() {+ // the job after which the model runs on the CPU (the GPU failed) says so+ let mut s = Session::with_demo();+ let notice = Some(worker::GPU_FAILED.to_string());+ s.segmenter.results.0.send(worker::Outcome { tag: Tag::Prepare, result: Ok(None), superseded: false, notice }).unwrap();+ let polled = s.segment_poll();+ assert_eq!(polled.messages, vec![worker::GPU_FAILED.to_string()]);+ assert!(s.segment_poll().messages.is_empty(), "once");+ }+fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask {let l: Vec<f32> =(0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect();diff --git a/crates/engine/src/segment/worker.rs b/crates/engine/src/segment/worker.rsindex 5769fd2ae..efbb1c21c 100644--- a/crates/engine/src/segment/worker.rs+++ b/crates/engine/src/segment/worker.rs@@ -7,6 +7,13 @@//! the (possibly inconsistent) model and leaves the worker ready for the next job. The busy//! counters are decremented by drop guards, so they can't stay stuck. The model is unloaded//! after [`IDLE_UNLOAD`] without requests (it holds several GB).+//!+//! Where the model can run on the GPU (Metal on macOS), a job that fails there (a panic or an+//! error from candle, e.g. a kernel the GPU's compiler can't build, issue #1974) frees the+//! model and runs once more on the CPU. When that works, the model stays on the CPU for the rest+//! of the session and the job's outcome says so; when it fails too, the GPU wasn't the cause and+//! the next job uses it again. `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off` keep the model on+//! the CPU from the start.use std::collections::VecDeque;use std::panic::AssertUnwindSafe;@@ -29,6 +36,7 @@ use crate::media::RenderJob;pub const IDLE_UNLOAD: Duration = Duration::from_secs(10 * 60);/// What to compute.+#[derive(Clone)]pub enum Kind {/// Load the model and encode the photo (so the first click is fast).Prepare,@@ -42,6 +50,12 @@ pub enum Kind {/// Panics (tests of the worker's recovery).#[cfg(test)]Panic,+ /// Fails like the model on a GPU that can't run it (a panic or a candle error), and+ /// succeeds on the CPU (tests of the CPU fallback).+ #[cfg(test)]+ FailOnGpu {+ panic: bool,+ },}/// One request.@@ -65,6 +79,70 @@ pub struct Outcome {pub result: Result<Option<SegMask>, String>,/// A newer request for the same selection came in; this one was skipped.pub superseded: bool,+ /// Something the user should know besides the result ([`GPU_FAILED`]).+ pub notice: Option<String>,+}++/// The notice of the job after which the model runs on the CPU.+pub const GPU_FAILED: &str =+ "The GPU couldn't run the AI mask model, so it runs on the CPU until LightCraft restarts (slower). Details are in the log.";++/// Where the model is loaded.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum Placement {+ /// On the GPU when there is one (Metal on macOS); a job that fails there is retried on the CPU.+ Gpu,+ /// On the CPU: this build has no GPU path, the environment switches the GPU off, or the model+ /// failed on it.+ Cpu,+}++impl Placement {+ fn initial() -> Self {+ if !lightcraft_segment::GPU_BUILD {+ return Placement::Cpu;+ }+ if let Some(why) = lightcraft_gpu::env_off_reason() {+ log::info!("SAM 3 runs on the CPU ({why})");+ return Placement::Cpu;+ }+ Placement::Gpu+ }+}++/// Why a job failed.+#[derive(Debug)]+enum Failure {+ /// The model failed (a candle error, or a panic): on the GPU, the CPU may run it.+ Model(String),+ /// Anything else (no model files, the photo, a remote worker): another device won't help.+ Other(String),+}++impl Failure {+ fn message(self) -> String {+ match self {+ Failure::Model(m) | Failure::Other(m) => m,+ }+ }+}++impl From<String> for Failure {+ fn from(m: String) -> Self {+ Failure::Other(m)+ }+}++impl From<&str> for Failure {+ fn from(m: &str) -> Self {+ Failure::Other(m.into())+ }+}++impl From<lightcraft_segment::Error> for Failure {+ fn from(e: lightcraft_segment::Error) -> Self {+ if matches!(e, lightcraft_segment::Error::Candle(_)) { Failure::Model(e.to_string()) } else { Failure::Other(e.to_string()) }+ }}/// Counters the session reads without locking.@@ -159,7 +237,7 @@ impl Worker {// (re)start the threadlet (tx, rx) = mpsc::channel();let shared = self.shared.clone();- match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD)) {+ match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD, Placement::initial())) {Ok(t) => {self.thread = Some(t);self.tx = Some(tx.clone());@@ -191,7 +269,7 @@ fn supersedes(later: &Job, job: &Job) -> bool {}}-fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {+fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration, mut placement: Placement) {let mut state = State::default();let mut queue: VecDeque<Job> = VecDeque::new();loop {@@ -219,41 +297,78 @@ fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {let done = Done(if is_detail { &shared.detail } else { &shared.pending });if queue.iter().any(|later| supersedes(later, &job)) {drop(done);- let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true });+ let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true, notice: None });continue;}- let result = match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(&mut state, shared, &mut job))) {- Ok(r) => r,- Err(p) => {- let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_default();- log::error!("SAM 3 panicked: {why}");- // don't trust what the model kept: load it again next time- state = State::default();- Err("the AI model failed unexpectedly (it will be reloaded on the next try)".to_string())- }+ // what a retry on the CPU needs (running the job takes them)+ let retry = (placement == Placement::Gpu && remote::configured().is_none()).then(|| (job.kind.clone(), job.render.clone()));+ let mut result = attempt(&mut state, shared, &mut job, placement);+ let mut notice = None;+ let gpu_failed = match &result {+ Err(Failure::Model(why)) => Some(why.clone()),+ _ => None,};+ if let (Some(why), Some((kind, render))) = (gpu_failed, retry) {+ // the GPU may be the cause (e.g. a kernel its compiler can't build, issue #1974): free+ // the model there and try once on the CPU+ log::warn!("SAM 3 failed on the GPU ({why}); trying the CPU");+ state = State::default();+ (job.kind, job.render) = (kind, render);+ match attempt(&mut state, shared, &mut job, Placement::Cpu) {+ Ok(r) => {+ log::warn!("SAM 3 runs on the CPU from now on: it failed on the GPU and works on the CPU");+ placement = Placement::Cpu;+ notice = Some(GPU_FAILED.to_string());+ result = Ok(r);+ }+ Err(e) => {+ // the CPU fails too, so the GPU isn't the cause: the next job uses it again+ log::warn!("SAM 3 failed on the CPU too: {}", e.message());+ state = State::default();+ }+ }+ }+ let result = result.map_err(Failure::message);if result.is_err() && !is_detail {state.cache = None;}shared.loaded.store(state.model.is_some() || state.remote.is_some(), Ordering::SeqCst);drop(done);- let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false });+ let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false, notice });+ }+}++/// Run `job` with the model on `placement`. A panic is a [`Failure::Model`] and drops what the+/// model kept.+fn attempt(state: &mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<Option<SegMask>, Failure> {+ match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(state, shared, job, placement))) {+ Ok(r) => r,+ Err(p) => {+ let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_default();+ log::error!("SAM 3 panicked: {why}");+ // don't trust what the model kept: load it again next time+ *state = State::default();+ Err(Failure::Model("the AI model failed unexpectedly (it will be reloaded on the next try)".into()))+ }}}-/// The model for `dir`, loading it when needed.-fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path) -> Result<&'a mut Sam3, String> {+/// The model for `dir`, loading it on `placement` when needed.+fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path, placement: Placement) -> Result<&'a mut Sam3, Failure> {if state.model.as_ref().is_none_or(|(d, _)| d != dir) {state.model = None;state.cache = None;let _busy = Flag::raise(&shared.encoding);let t = web_time::Instant::now();- let m = Sam3::load(dir).map_err(|e| e.to_string())?;- log::info!("SAM 3 loaded in {:?}", t.elapsed());+ let m = match placement {+ Placement::Gpu => Sam3::load(dir)?,+ Placement::Cpu => Sam3::load_cpu(dir)?,+ };+ log::info!("SAM 3 loaded in {:?} ({})", t.elapsed(), if m.device().is_cpu() { "CPU" } else { "GPU" });state.model = Some((dir.to_path_buf(), m));shared.loaded.store(true, Ordering::SeqCst);}- state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".to_string())+ state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".into())}/// Render the model input.@@ -275,16 +390,16 @@ fn render(job: &mut Job) -> Result<Input, String> {}/// The model and the encoded photo `job.key` (rendering and encoding it when needed).-fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(&'a mut Sam3, &'a mut Encoded), String> {- model(state, shared, &job.dir)?;+fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<(&'a mut Sam3, &'a mut Encoded), Failure> {+ model(state, shared, &job.dir, placement)?;if state.cache.as_ref().is_none_or(|(k, _)| *k != job.key) {state.cache = None;let input = render(job)?;let _busy = Flag::raise(&shared.encoding);let m = state.model.as_mut().map(|(_, m)| m).ok_or("the model did not load")?;let t = web_time::Instant::now();- let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;- m.prepare_clicks(&mut enc).map_err(|e| e.to_string())?;+ let mut enc = m.encode(&input.rgb, input.w, input.h)?;+ m.prepare_clicks(&mut enc)?;log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());state.cache = Some((job.key, enc));}@@ -328,7 +443,7 @@ fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str)}}#[cfg(test)]- Kind::Panic => return Err("test panic request".into()),+ Kind::Panic | Kind::FailOnGpu { .. } => return Err("test panic request".into()),};let input = if region.is_some() || client.key != Some(job.key) { render(job)? } else { Input { rgb: Vec::new(), w: 0, h: 0 } };let request = Request { width: input.w, height: input.h, temporary: region.is_some(), operation };@@ -342,37 +457,44 @@ fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str)}))}-fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<SegMask>, String> {+fn run_job(state: &mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<Option<SegMask>, Failure> {if let Some(endpoint) = remote::configured() {let result = remote_job(state, shared, job, &endpoint);if result.is_err() {state.remote = None;}- return result.map_err(|e| format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry."));+ return result.map_err(|e| Failure::Other(format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry.")));}let side = lightcraft_segment::MASK_SIDE;match std::mem::replace(&mut job.kind, Kind::Prepare) {Kind::Prepare => {- encoded(state, shared, job)?;+ encoded(state, shared, job, placement)?;Ok(None)}Kind::Clicks(clicks) => {- let (m, enc) = encoded(state, shared, job)?;- let p = m.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?;+ let (m, enc) = encoded(state, shared, job, placement)?;+ let p = m.segment_clicks(enc, &clicks)?;Ok(Some(SegMask::from_logits(side, &p.logits)))}Kind::Text(text) => {- let (m, enc) = encoded(state, shared, job)?;+ let (m, enc) = encoded(state, shared, job, placement)?;Ok(text_logits(m, enc, &text)?.map(|l| SegMask::from_logits(side, &l)))}#[cfg(test)]#[allow(clippy::panic)]Kind::Panic => panic!("a test panic"),+ #[cfg(test)]+ #[allow(clippy::panic)]+ Kind::FailOnGpu { panic } => match placement {+ Placement::Gpu if panic => panic!("a test panic on the GPU"),+ Placement::Gpu => Err(Failure::Model("a test error on the GPU".into())),+ Placement::Cpu => Ok(None),+ },Kind::Detail { prompt, region } => {let input = render(job)?;- let m = model(state, shared, &job.dir)?;+ let m = model(state, shared, &job.dir, placement)?;let t = web_time::Instant::now();- let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;+ let mut enc = m.encode(&input.rgb, input.w, input.h)?;let (rw, rh) = (region[2] - region[0], region[3] - region[1]);if !(rw > 0.0 && rh > 0.0) {return Ok(None);@@ -388,7 +510,7 @@ fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<Sif !inside.iter().any(|c| c.positive) {return Ok(None);}- Some(m.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits)+ Some(m.segment_clicks(&mut enc, &inside)?.logits)}Prompt::Text(text) => text_logits(m, &mut enc, &text)?,};@@ -438,7 +560,7 @@ mod tests {jtx.send(job(Tag::Wait, &tx)).unwrap();shared.pending.store(2, Ordering::SeqCst);drop(jtx);- run(&jrx, &shared, Duration::from_secs(60));+ run(&jrx, &shared, Duration::from_secs(60), Placement::Cpu);let first = rx.recv().unwrap();assert!(first.result.unwrap_err().contains("failed unexpectedly"));let second = rx.recv().unwrap();@@ -447,6 +569,72 @@ mod tests {assert!(!shared.encoding.load(Ordering::SeqCst));}+ /// Run `kinds` in turn on a worker that starts on `placement`; their outcomes.+ fn run_kinds(placement: Placement, kinds: Vec<Kind>) -> Vec<Outcome> {+ let (tx, rx) = mpsc::channel();+ let (jtx, jrx) = mpsc::channel();+ let shared = Shared::default();+ shared.pending.store(kinds.len(), Ordering::SeqCst);+ for kind in kinds {+ let mut j = job(Tag::Wait, &tx);+ j.kind = kind;+ jtx.send(j).unwrap();+ }+ drop(jtx);+ run(&jrx, &shared, Duration::from_secs(60), placement);+ assert_eq!(shared.pending.load(Ordering::SeqCst), 0, "counters reset");+ assert!(!shared.encoding.load(Ordering::SeqCst));+ rx.try_iter().collect()+ }++ #[test]+ fn a_job_that_panics_on_the_gpu_runs_on_the_cpu_and_so_do_the_next_ones() {+ // issue #1974: candle panics compiling a Metal kernel the GPU's compiler can't build+ let out = run_kinds(Placement::Gpu, vec![Kind::FailOnGpu { panic: true }, Kind::FailOnGpu { panic: true }]);+ assert_eq!(out.len(), 2);+ assert!(matches!(out[0].result, Ok(None)), "retried on the CPU: {:?}", out[0].result.as_ref().err());+ assert_eq!(out[0].notice.as_deref(), Some(GPU_FAILED));+ assert!(matches!(out[1].result, Ok(None)), "the next job starts on the CPU");+ assert_eq!(out[1].notice, None, "said once");+ }++ #[test]+ fn a_model_error_on_the_gpu_is_retried_on_the_cpu_too() {+ let out = run_kinds(Placement::Gpu, vec![Kind::FailOnGpu { panic: false }]);+ assert!(matches!(out[0].result, Ok(None)));+ assert_eq!(out[0].notice.as_deref(), Some(GPU_FAILED));+ }++ #[test]+ fn a_job_that_fails_on_the_cpu_too_keeps_the_gpu() {+ let out = run_kinds(Placement::Gpu, vec![Kind::Panic, Kind::FailOnGpu { panic: true }]);+ assert!(out[0].result.as_ref().unwrap_err().contains("failed unexpectedly"));+ assert_eq!(out[0].notice, None);+ // not the GPU's fault: the next job tries the GPU first again+ assert!(matches!(out[1].result, Ok(None)));+ assert_eq!(out[1].notice.as_deref(), Some(GPU_FAILED));+ }++ #[test]+ fn a_worker_on_the_cpu_never_tries_the_gpu() {+ // no GPU in this build, or LIGHTCRAFT_GPU=0 / LIGHTCRAFT_GPU_BACKEND=off+ let out = run_kinds(Placement::Cpu, vec![Kind::FailOnGpu { panic: true }]);+ assert!(matches!(out[0].result, Ok(None)));+ assert_eq!(out[0].notice, None);+ if !lightcraft_segment::GPU_BUILD {+ assert_eq!(Placement::initial(), Placement::Cpu);+ }+ }++ #[test]+ fn missing_model_files_are_not_retried_on_the_cpu() {+ let out = run_kinds(Placement::Gpu, vec![Kind::Prepare]);+ assert!(out[0].result.as_ref().unwrap_err().contains("not found"));+ assert_eq!(out[0].notice, None);+ let missing = lightcraft_segment::Error::Missing(PathBuf::from("models"));+ assert!(matches!(Failure::from(missing), Failure::Other(_)));+ }+#[test]fn a_newer_request_for_the_same_selection_supersedes_a_queued_one() {let (tx, rx) = mpsc::channel();@@ -464,7 +652,7 @@ mod tests {drop(jtx);let shared = Shared::default();shared.pending.store(3, Ordering::SeqCst);- run(&jrx, &shared, Duration::from_secs(60));+ run(&jrx, &shared, Duration::from_secs(60), Placement::Cpu);let out: Vec<Outcome> = rx.try_iter().collect();assert_eq!(out.len(), 3);assert!(out[0].superseded && out[1].superseded && !out[2].superseded);diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rsindex b0409962d..7efae6a57 100644--- a/crates/gpu/src/lib.rs+++ b/crates/gpu/src/lib.rs@@ -75,12 +75,8 @@ fn record_fallback(reason: String) {/// E.g. "disabled by LIGHTCRAFT_GPU=0", "software adapter (llvmpipe …) skipped", "device lost …"./// Never blocks.pub fn unavailable_reason() -> Option<String> {- if env_disabled() {- #[cfg(not(target_arch = "wasm32"))]- if backend::env_off() {- return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off".into());- }- return Some("disabled by LIGHTCRAFT_GPU=0".into());+ if let Some(why) = env_off_reason() {+ return Some(why.into());}if !ENABLED.load(Ordering::Relaxed) {return Some("disabled by the GPU rendering preference (app.gpu)".into());@@ -245,12 +241,26 @@ pub fn wait_idle(timeout: std::time::Duration) -> bool {/// environment switches and the rendering preference, but not a render-device failure (the devices are separate).#[cfg(not(target_arch = "wasm32"))]pub(crate) fn switched_off() -> Option<String> {- if env_disabled() {- return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());+ if let Some(why) = env_off_reason() {+ return Some(why.into());}(!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into())}+/// Why the environment switches the GPU off for the whole process (`LIGHTCRAFT_GPU=0` or+/// `LIGHTCRAFT_GPU_BACKEND=off`), or `None`. Code with a GPU path of its own (the SAM 3 model on+/// Metal) follows it too.+pub fn env_off_reason() -> Option<&'static str> {+ if !env_disabled() {+ return None;+ }+ #[cfg(not(target_arch = "wasm32"))]+ if backend::env_off() {+ return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off");+ }+ Some("disabled by LIGHTCRAFT_GPU=0")+}+/// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process.fn env_disabled() -> bool {static OFF: std::sync::OnceLock<bool> = std::sync::OnceLock::new();diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rsindex 6d488a919..6c24d0836 100644--- a/crates/segment/src/lib.rs+++ b/crates/segment/src/lib.rs@@ -87,6 +87,11 @@ pub fn is_model_dir(dir: &Path) -> bool {dir.join(WEIGHTS_FILE).is_file() && dir.join("vocab.json").is_file() && dir.join("merges.txt").is_file()}+#[cfg(not(target_arch = "wasm32"))]+/// Whether [`best_device`] can be a GPU in this build (Metal on macOS). Where it can, a caller that+/// sees the model fail there can load it again with [`Sam3::load_cpu`].+pub const GPU_BUILD: bool = cfg!(target_os = "macos");+#[cfg(not(target_arch = "wasm32"))]/// The best device here: Metal on macOS (when a GPU is available), else the CPU.pub fn best_device() -> Device {@@ -134,6 +139,11 @@ impl Sam3 {Self::load_on(dir, best_device())}+ /// [`Sam3::load`] on the CPU, whatever GPU there is.+ pub fn load_cpu(dir: &Path) -> Result<Self> {+ Self::load_on(dir, Device::Cpu)+ }+pub fn load_on(dir: &Path, device: Device) -> Result<Self> {if !is_model_dir(dir) {return Err(Error::Missing(dir.to_path_buf()));diff --git a/crates/segment/src/remote.rs b/crates/segment/src/remote.rsindex cdee096fe..10f7b2931 100644--- a/crates/segment/src/remote.rs+++ b/crates/segment/src/remote.rs@@ -108,10 +108,10 @@ impl Client {}}-pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result<Option<Vec<f32>>, String> {+pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> crate::Result<Option<Vec<f32>>> {let mut merged: Option<Vec<f32>> = None;for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {- if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {+ if let Some(p) = model.segment_text(enc, phrase, 0.5)? {if let Some(m) = &mut merged {for (a, b) in m.iter_mut().zip(p.logits) {*a = a.max(b);@@ -130,7 +130,7 @@ fn predict(model: &mut Sam3, enc: &mut Encoded, operation: Operation) -> Result<model.prepare_clicks(enc).map_err(|e| e.to_string())?;Ok(None)}- Operation::Text(text) => text_logits(model, enc, &text),+ Operation::Text(text) => text_logits(model, enc, &text).map_err(|e| e.to_string()),Operation::Clicks(points) => {let clicks: Vec<_> = points.into_iter().map(|p| Click { x: p[0], y: p[1], positive: p[2] == 1.0 }).collect();Ok(Some(model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?.logits))diff --git a/docs/ai-masks.md b/docs/ai-masks.mdindex b02912abc..cd803b388 100644--- a/docs/ai-masks.md+++ b/docs/ai-masks.md@@ -2,7 +2,8 @@LightCraft selects objects with **SAM 3** (Segment Anything with Concepts, Meta 2025), run inpure Rust by `crates/segment` on [candle](https://github.com/huggingface/candle): on the GPU-through Metal on macOS, on the CPU elsewhere (for now).+through Metal on macOS, on the CPU elsewhere (for now). If the Mac's GPU can't run it, it runs on+the CPU (see the worker below).**The model is optional.** It is not part of LightCraft, and nothing requires it: without it,Object and Describe offer to download it, every other feature (and every existing mask, see@@ -225,6 +226,13 @@ wait and return the result (when built with the `sam` feature; the default CLI hencoded photo; the session sends it jobs and receives results over channels, never holding alock the model holds. Each job runs under `catch_unwind`: a panic inside candle becomes anerror and the model is reloaded on the next request; busy counters are reset by drop guards.+- GPU fallback (issue #1974): a job that fails on the GPU (a panic or a candle error, e.g. an+ AMD GPU in an Intel Mac whose Metal compiler crashes on a matmul kernel) frees the model and runs+ once more on the CPU. If that works, the model stays on the CPU until LightCraft restarts and+ the user is told once ("The GPU couldn't run the AI mask model…"); if the CPU fails too, the+ error is reported and the next job tries the GPU again. Missing or damaged model files and+ remote-worker errors are not retried. `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off` keep+ the model on the CPU from the start.- Hostile documents: a selection keeps at most 4 detail patches (when read and when rendered),each at most 1024² logits; more than 64 clicks are never sent to the model.- Accuracy: `crates/segment/tests/reference.rs` compares every stage against the Pythondiff --git a/docs/gpu-pipeline.md b/docs/gpu-pipeline.mdindex b74af85a2..5bd0119f9 100644--- a/docs/gpu-pipeline.md+++ b/docs/gpu-pipeline.md@@ -43,6 +43,8 @@ Overrides (read once at launch):- else `WGPU_BACKEND` (wgpu's own variable, same names) — before #136 only the window honoured it,the compute device always used Vulkan + DX12 (+ Metal).- `LIGHTCRAFT_GPU=0`: GPU rendering off for the process (the window is unaffected).+- Both `LIGHTCRAFT_GPU=0` and `LIGHTCRAFT_GPU_BACKEND=off` also keep AI Denoise, the face models+ and the AI mask model (SAM 3 on Metal) on the CPU.**DX12 shader compiler (issue #471).** Every instance (window and compute) compiles DX12 shaders withFXC (`d3dcompiler_47.dll`, part of Windows; `lightcraft_gpu::backend::backend_options`). wgpu'sdiff --git a/docs/parity-checklist.md b/docs/parity-checklist.mdindex 95dcd2aef..a78ff0f7c 100644--- a/docs/parity-checklist.md+++ b/docs/parity-checklist.md@@ -293,7 +293,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts).| LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model; a photo that carries person mattes (DNG semantic masks: iPhone ProRAW portrait / skin / hair / teeth / glasses mattes) uses them instead (synthetic files only: no corpus file carries person mattes) || LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`), `crates/pipeline/src/masks.rs` (`Mattes`), `crates/raw/src/semantic.rs` | heuristic; a photo that carries a sky matte (DNG semantic mask, e.g. iPhone ProRAW) uses it instead, through orientation / crop / lens corrections, preview = export (checked on one CC0 iPhone 12 Pro ProRAW and synthetic files) || LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject mask (heuristic, or the photo's person mattes) |-| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) |+| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors; a job that fails on the Mac's GPU runs again on the CPU, which then serves the rest of the session, #1974), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) || LR-MASK-PEOPLE | People parts | P2 | ⬜ | | shape exists (no UI): uses the photo's person / skin / hair / teeth / glasses mattes when it carries them (DNG semantic masks), else the subject heuristic || LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty || LR-MASK-BRUSH | Brush mask | P0 | 🟡 | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs`, `crates/ui-egui/src/brush_live.rs` | size/feather/flow/density/erase; Create New Mask starts a separate brush in paint mode; the stroke shows while the button is held with the committed stroke's dab spacing, feather and flow, so fast strokes have no gaps (#517); coverage builds up by passes, not hold time (holding the brush still adds nothing, live or committed, #518); Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); ⇧-click draws a straight stroke from where the last stroke ended (`connect`), tapering size, feather and flow from that stroke's values when they changed; no A/B brushes; pen pressure (Brush ▸ Pen Pressure, on by default; `pressure` per point in `mask.brushStroke`) scales each dab's size (a tenth to all of it) and flow, live and committed alike, where winit reports a pen force (Windows; not macOS / Linux tablets yet); no tilt |