Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

LR-MASK-OBJECTS: SAM 3 falls back to the CPU when the GPU can't run it (#1974)

helgehelge12353f57b9404b1
diff --git a/crates/engine/src/segment/mod.rs b/crates/engine/src/segment/mod.rs
index c955dcc01..b69a6dca2 100644
--- a/crates/engine/src/segment/mod.rs
+++ b/crates/engine/src/segment/mod.rs
@@ -75,6 +75,7 @@ pub(crate) struct Input {
}
/// What a detail pass re-runs.
+#[derive(Clone)]
#[cfg_attr(not(feature = "sam"), allow(dead_code))]
pub(crate) enum Prompt {
Clicks(Vec<Click>),
@@ -440,7 +441,11 @@ impl Session {
let job = self.segment_job(id, kind, Tag::Wait, tx, region)?;
self.segmenter.worker.submit(job)?;
match rx.recv_timeout(WAIT) {
- Ok(o) => o.result,
+ Ok(o) => {
+ // (shown by the next poll in an app; logged anyway)
+ self.segmenter.messages.extend(o.notice);
+ o.result
+ }
Err(_) => Err("the AI model took too long; try again".into()),
}
}
@@ -579,7 +584,7 @@ impl Session {
self.segment_queue(id, kind, tag, Some(region))?;
} else {
let patch = self.segment_wait(id, kind, Some(region))?;
- let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false };
+ let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false, notice: None };
let mut polled = Polled::default();
self.apply_outcome(outcome, &mut polled);
}
@@ -618,6 +623,7 @@ impl Session {
if o.superseded {
return;
}
+ polled.messages.extend(o.notice);
match o.tag {
Tag::Wait => {}
Tag::Prepare => {
@@ -711,6 +717,18 @@ impl Session {
mod tests {
use super::*;
+ #[cfg(feature = "sam")]
+ #[test]
+ fn a_worker_notice_reaches_the_user_with_the_result() {
+ // the job after which the model runs on the CPU (the GPU failed) says so
+ let mut s = Session::with_demo();
+ let notice = Some(worker::GPU_FAILED.to_string());
+ s.segmenter.results.0.send(worker::Outcome { tag: Tag::Prepare, result: Ok(None), superseded: false, notice }).unwrap();
+ let polled = s.segment_poll();
+ assert_eq!(polled.messages, vec![worker::GPU_FAILED.to_string()]);
+ assert!(s.segment_poll().messages.is_empty(), "once");
+ }
+
fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask {
let l: Vec<f32> =
(0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect();
diff --git a/crates/engine/src/segment/worker.rs b/crates/engine/src/segment/worker.rs
index 5769fd2ae..efbb1c21c 100644
--- a/crates/engine/src/segment/worker.rs
+++ b/crates/engine/src/segment/worker.rs
@@ -7,6 +7,13 @@
//! the (possibly inconsistent) model and leaves the worker ready for the next job. The busy
//! counters are decremented by drop guards, so they can't stay stuck. The model is unloaded
//! after [`IDLE_UNLOAD`] without requests (it holds several GB).
+//!
+//! Where the model can run on the GPU (Metal on macOS), a job that fails there (a panic or an
+//! error from candle, e.g. a kernel the GPU's compiler can't build, issue #1974) frees the
+//! model and runs once more on the CPU. When that works, the model stays on the CPU for the rest
+//! of the session and the job's outcome says so; when it fails too, the GPU wasn't the cause and
+//! the next job uses it again. `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off` keep the model on
+//! the CPU from the start.
use std::collections::VecDeque;
use std::panic::AssertUnwindSafe;
@@ -29,6 +36,7 @@ use crate::media::RenderJob;
pub const IDLE_UNLOAD: Duration = Duration::from_secs(10 * 60);
/// What to compute.
+#[derive(Clone)]
pub enum Kind {
/// Load the model and encode the photo (so the first click is fast).
Prepare,
@@ -42,6 +50,12 @@ pub enum Kind {
/// Panics (tests of the worker's recovery).
#[cfg(test)]
Panic,
+ /// Fails like the model on a GPU that can't run it (a panic or a candle error), and
+ /// succeeds on the CPU (tests of the CPU fallback).
+ #[cfg(test)]
+ FailOnGpu {
+ panic: bool,
+ },
}
/// One request.
@@ -65,6 +79,70 @@ pub struct Outcome {
pub result: Result<Option<SegMask>, String>,
/// A newer request for the same selection came in; this one was skipped.
pub superseded: bool,
+ /// Something the user should know besides the result ([`GPU_FAILED`]).
+ pub notice: Option<String>,
+}
+
+/// The notice of the job after which the model runs on the CPU.
+pub const GPU_FAILED: &str =
+ "The GPU couldn't run the AI mask model, so it runs on the CPU until LightCraft restarts (slower). Details are in the log.";
+
+/// Where the model is loaded.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum Placement {
+ /// On the GPU when there is one (Metal on macOS); a job that fails there is retried on the CPU.
+ Gpu,
+ /// On the CPU: this build has no GPU path, the environment switches the GPU off, or the model
+ /// failed on it.
+ Cpu,
+}
+
+impl Placement {
+ fn initial() -> Self {
+ if !lightcraft_segment::GPU_BUILD {
+ return Placement::Cpu;
+ }
+ if let Some(why) = lightcraft_gpu::env_off_reason() {
+ log::info!("SAM 3 runs on the CPU ({why})");
+ return Placement::Cpu;
+ }
+ Placement::Gpu
+ }
+}
+
+/// Why a job failed.
+#[derive(Debug)]
+enum Failure {
+ /// The model failed (a candle error, or a panic): on the GPU, the CPU may run it.
+ Model(String),
+ /// Anything else (no model files, the photo, a remote worker): another device won't help.
+ Other(String),
+}
+
+impl Failure {
+ fn message(self) -> String {
+ match self {
+ Failure::Model(m) | Failure::Other(m) => m,
+ }
+ }
+}
+
+impl From<String> for Failure {
+ fn from(m: String) -> Self {
+ Failure::Other(m)
+ }
+}
+
+impl From<&str> for Failure {
+ fn from(m: &str) -> Self {
+ Failure::Other(m.into())
+ }
+}
+
+impl From<lightcraft_segment::Error> for Failure {
+ fn from(e: lightcraft_segment::Error) -> Self {
+ if matches!(e, lightcraft_segment::Error::Candle(_)) { Failure::Model(e.to_string()) } else { Failure::Other(e.to_string()) }
+ }
}
/// Counters the session reads without locking.
@@ -159,7 +237,7 @@ impl Worker {
// (re)start the thread
let (tx, rx) = mpsc::channel();
let shared = self.shared.clone();
- match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD)) {
+ match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD, Placement::initial())) {
Ok(t) => {
self.thread = Some(t);
self.tx = Some(tx.clone());
@@ -191,7 +269,7 @@ fn supersedes(later: &Job, job: &Job) -> bool {
}
}
-fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
+fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration, mut placement: Placement) {
let mut state = State::default();
let mut queue: VecDeque<Job> = VecDeque::new();
loop {
@@ -219,41 +297,78 @@ fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
let done = Done(if is_detail { &shared.detail } else { &shared.pending });
if queue.iter().any(|later| supersedes(later, &job)) {
drop(done);
- let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true });
+ let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true, notice: None });
continue;
}
- let result = match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(&mut state, shared, &mut job))) {
- Ok(r) => r,
- Err(p) => {
- let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_default();
- log::error!("SAM 3 panicked: {why}");
- // don't trust what the model kept: load it again next time
- state = State::default();
- Err("the AI model failed unexpectedly (it will be reloaded on the next try)".to_string())
- }
+ // what a retry on the CPU needs (running the job takes them)
+ let retry = (placement == Placement::Gpu && remote::configured().is_none()).then(|| (job.kind.clone(), job.render.clone()));
+ let mut result = attempt(&mut state, shared, &mut job, placement);
+ let mut notice = None;
+ let gpu_failed = match &result {
+ Err(Failure::Model(why)) => Some(why.clone()),
+ _ => None,
};
+ if let (Some(why), Some((kind, render))) = (gpu_failed, retry) {
+ // the GPU may be the cause (e.g. a kernel its compiler can't build, issue #1974): free
+ // the model there and try once on the CPU
+ log::warn!("SAM 3 failed on the GPU ({why}); trying the CPU");
+ state = State::default();
+ (job.kind, job.render) = (kind, render);
+ match attempt(&mut state, shared, &mut job, Placement::Cpu) {
+ Ok(r) => {
+ log::warn!("SAM 3 runs on the CPU from now on: it failed on the GPU and works on the CPU");
+ placement = Placement::Cpu;
+ notice = Some(GPU_FAILED.to_string());
+ result = Ok(r);
+ }
+ Err(e) => {
+ // the CPU fails too, so the GPU isn't the cause: the next job uses it again
+ log::warn!("SAM 3 failed on the CPU too: {}", e.message());
+ state = State::default();
+ }
+ }
+ }
+ let result = result.map_err(Failure::message);
if result.is_err() && !is_detail {
state.cache = None;
}
shared.loaded.store(state.model.is_some() || state.remote.is_some(), Ordering::SeqCst);
drop(done);
- let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false });
+ let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false, notice });
+ }
+}
+
+/// Run `job` with the model on `placement`. A panic is a [`Failure::Model`] and drops what the
+/// model kept.
+fn attempt(state: &mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<Option<SegMask>, Failure> {
+ match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(state, shared, job, placement))) {
+ Ok(r) => r,
+ Err(p) => {
+ let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_default();
+ log::error!("SAM 3 panicked: {why}");
+ // don't trust what the model kept: load it again next time
+ *state = State::default();
+ Err(Failure::Model("the AI model failed unexpectedly (it will be reloaded on the next try)".into()))
+ }
}
}
-/// The model for `dir`, loading it when needed.
-fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path) -> Result<&'a mut Sam3, String> {
+/// The model for `dir`, loading it on `placement` when needed.
+fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path, placement: Placement) -> Result<&'a mut Sam3, Failure> {
if state.model.as_ref().is_none_or(|(d, _)| d != dir) {
state.model = None;
state.cache = None;
let _busy = Flag::raise(&shared.encoding);
let t = web_time::Instant::now();
- let m = Sam3::load(dir).map_err(|e| e.to_string())?;
- log::info!("SAM 3 loaded in {:?}", t.elapsed());
+ let m = match placement {
+ Placement::Gpu => Sam3::load(dir)?,
+ Placement::Cpu => Sam3::load_cpu(dir)?,
+ };
+ log::info!("SAM 3 loaded in {:?} ({})", t.elapsed(), if m.device().is_cpu() { "CPU" } else { "GPU" });
state.model = Some((dir.to_path_buf(), m));
shared.loaded.store(true, Ordering::SeqCst);
}
- state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".to_string())
+ state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".into())
}
/// Render the model input.
@@ -275,16 +390,16 @@ fn render(job: &mut Job) -> Result<Input, String> {
}
/// The model and the encoded photo `job.key` (rendering and encoding it when needed).
-fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(&'a mut Sam3, &'a mut Encoded), String> {
- model(state, shared, &job.dir)?;
+fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<(&'a mut Sam3, &'a mut Encoded), Failure> {
+ model(state, shared, &job.dir, placement)?;
if state.cache.as_ref().is_none_or(|(k, _)| *k != job.key) {
state.cache = None;
let input = render(job)?;
let _busy = Flag::raise(&shared.encoding);
let m = state.model.as_mut().map(|(_, m)| m).ok_or("the model did not load")?;
let t = web_time::Instant::now();
- let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
- m.prepare_clicks(&mut enc).map_err(|e| e.to_string())?;
+ let mut enc = m.encode(&input.rgb, input.w, input.h)?;
+ m.prepare_clicks(&mut enc)?;
log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());
state.cache = Some((job.key, enc));
}
@@ -328,7 +443,7 @@ fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str)
}
}
#[cfg(test)]
- Kind::Panic => return Err("test panic request".into()),
+ Kind::Panic | Kind::FailOnGpu { .. } => return Err("test panic request".into()),
};
let input = if region.is_some() || client.key != Some(job.key) { render(job)? } else { Input { rgb: Vec::new(), w: 0, h: 0 } };
let request = Request { width: input.w, height: input.h, temporary: region.is_some(), operation };
@@ -342,37 +457,44 @@ fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str)
}))
}
-fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<SegMask>, String> {
+fn run_job(state: &mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<Option<SegMask>, Failure> {
if let Some(endpoint) = remote::configured() {
let result = remote_job(state, shared, job, &endpoint);
if result.is_err() {
state.remote = None;
}
- return result.map_err(|e| format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry."));
+ return result.map_err(|e| Failure::Other(format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry.")));
}
let side = lightcraft_segment::MASK_SIDE;
match std::mem::replace(&mut job.kind, Kind::Prepare) {
Kind::Prepare => {
- encoded(state, shared, job)?;
+ encoded(state, shared, job, placement)?;
Ok(None)
}
Kind::Clicks(clicks) => {
- let (m, enc) = encoded(state, shared, job)?;
- let p = m.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?;
+ let (m, enc) = encoded(state, shared, job, placement)?;
+ let p = m.segment_clicks(enc, &clicks)?;
Ok(Some(SegMask::from_logits(side, &p.logits)))
}
Kind::Text(text) => {
- let (m, enc) = encoded(state, shared, job)?;
+ let (m, enc) = encoded(state, shared, job, placement)?;
Ok(text_logits(m, enc, &text)?.map(|l| SegMask::from_logits(side, &l)))
}
#[cfg(test)]
#[allow(clippy::panic)]
Kind::Panic => panic!("a test panic"),
+ #[cfg(test)]
+ #[allow(clippy::panic)]
+ Kind::FailOnGpu { panic } => match placement {
+ Placement::Gpu if panic => panic!("a test panic on the GPU"),
+ Placement::Gpu => Err(Failure::Model("a test error on the GPU".into())),
+ Placement::Cpu => Ok(None),
+ },
Kind::Detail { prompt, region } => {
let input = render(job)?;
- let m = model(state, shared, &job.dir)?;
+ let m = model(state, shared, &job.dir, placement)?;
let t = web_time::Instant::now();
- let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
+ let mut enc = m.encode(&input.rgb, input.w, input.h)?;
let (rw, rh) = (region[2] - region[0], region[3] - region[1]);
if !(rw > 0.0 && rh > 0.0) {
return Ok(None);
@@ -388,7 +510,7 @@ fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<S
if !inside.iter().any(|c| c.positive) {
return Ok(None);
}
- Some(m.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits)
+ Some(m.segment_clicks(&mut enc, &inside)?.logits)
}
Prompt::Text(text) => text_logits(m, &mut enc, &text)?,
};
@@ -438,7 +560,7 @@ mod tests {
jtx.send(job(Tag::Wait, &tx)).unwrap();
shared.pending.store(2, Ordering::SeqCst);
drop(jtx);
- run(&jrx, &shared, Duration::from_secs(60));
+ run(&jrx, &shared, Duration::from_secs(60), Placement::Cpu);
let first = rx.recv().unwrap();
assert!(first.result.unwrap_err().contains("failed unexpectedly"));
let second = rx.recv().unwrap();
@@ -447,6 +569,72 @@ mod tests {
assert!(!shared.encoding.load(Ordering::SeqCst));
}
+ /// Run `kinds` in turn on a worker that starts on `placement`; their outcomes.
+ fn run_kinds(placement: Placement, kinds: Vec<Kind>) -> Vec<Outcome> {
+ let (tx, rx) = mpsc::channel();
+ let (jtx, jrx) = mpsc::channel();
+ let shared = Shared::default();
+ shared.pending.store(kinds.len(), Ordering::SeqCst);
+ for kind in kinds {
+ let mut j = job(Tag::Wait, &tx);
+ j.kind = kind;
+ jtx.send(j).unwrap();
+ }
+ drop(jtx);
+ run(&jrx, &shared, Duration::from_secs(60), placement);
+ assert_eq!(shared.pending.load(Ordering::SeqCst), 0, "counters reset");
+ assert!(!shared.encoding.load(Ordering::SeqCst));
+ rx.try_iter().collect()
+ }
+
+ #[test]
+ fn a_job_that_panics_on_the_gpu_runs_on_the_cpu_and_so_do_the_next_ones() {
+ // issue #1974: candle panics compiling a Metal kernel the GPU's compiler can't build
+ let out = run_kinds(Placement::Gpu, vec![Kind::FailOnGpu { panic: true }, Kind::FailOnGpu { panic: true }]);
+ assert_eq!(out.len(), 2);
+ assert!(matches!(out[0].result, Ok(None)), "retried on the CPU: {:?}", out[0].result.as_ref().err());
+ assert_eq!(out[0].notice.as_deref(), Some(GPU_FAILED));
+ assert!(matches!(out[1].result, Ok(None)), "the next job starts on the CPU");
+ assert_eq!(out[1].notice, None, "said once");
+ }
+
+ #[test]
+ fn a_model_error_on_the_gpu_is_retried_on_the_cpu_too() {
+ let out = run_kinds(Placement::Gpu, vec![Kind::FailOnGpu { panic: false }]);
+ assert!(matches!(out[0].result, Ok(None)));
+ assert_eq!(out[0].notice.as_deref(), Some(GPU_FAILED));
+ }
+
+ #[test]
+ fn a_job_that_fails_on_the_cpu_too_keeps_the_gpu() {
+ let out = run_kinds(Placement::Gpu, vec![Kind::Panic, Kind::FailOnGpu { panic: true }]);
+ assert!(out[0].result.as_ref().unwrap_err().contains("failed unexpectedly"));
+ assert_eq!(out[0].notice, None);
+ // not the GPU's fault: the next job tries the GPU first again
+ assert!(matches!(out[1].result, Ok(None)));
+ assert_eq!(out[1].notice.as_deref(), Some(GPU_FAILED));
+ }
+
+ #[test]
+ fn a_worker_on_the_cpu_never_tries_the_gpu() {
+ // no GPU in this build, or LIGHTCRAFT_GPU=0 / LIGHTCRAFT_GPU_BACKEND=off
+ let out = run_kinds(Placement::Cpu, vec![Kind::FailOnGpu { panic: true }]);
+ assert!(matches!(out[0].result, Ok(None)));
+ assert_eq!(out[0].notice, None);
+ if !lightcraft_segment::GPU_BUILD {
+ assert_eq!(Placement::initial(), Placement::Cpu);
+ }
+ }
+
+ #[test]
+ fn missing_model_files_are_not_retried_on_the_cpu() {
+ let out = run_kinds(Placement::Gpu, vec![Kind::Prepare]);
+ assert!(out[0].result.as_ref().unwrap_err().contains("not found"));
+ assert_eq!(out[0].notice, None);
+ let missing = lightcraft_segment::Error::Missing(PathBuf::from("models"));
+ assert!(matches!(Failure::from(missing), Failure::Other(_)));
+ }
+
#[test]
fn a_newer_request_for_the_same_selection_supersedes_a_queued_one() {
let (tx, rx) = mpsc::channel();
@@ -464,7 +652,7 @@ mod tests {
drop(jtx);
let shared = Shared::default();
shared.pending.store(3, Ordering::SeqCst);
- run(&jrx, &shared, Duration::from_secs(60));
+ run(&jrx, &shared, Duration::from_secs(60), Placement::Cpu);
let out: Vec<Outcome> = rx.try_iter().collect();
assert_eq!(out.len(), 3);
assert!(out[0].superseded && out[1].superseded && !out[2].superseded);
diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs
index b0409962d..7efae6a57 100644
--- a/crates/gpu/src/lib.rs
+++ b/crates/gpu/src/lib.rs
@@ -75,12 +75,8 @@ fn record_fallback(reason: String) {
/// E.g. "disabled by LIGHTCRAFT_GPU=0", "software adapter (llvmpipe …) skipped", "device lost …".
/// Never blocks.
pub fn unavailable_reason() -> Option<String> {
- if env_disabled() {
- #[cfg(not(target_arch = "wasm32"))]
- if backend::env_off() {
- return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off".into());
- }
- return Some("disabled by LIGHTCRAFT_GPU=0".into());
+ if let Some(why) = env_off_reason() {
+ return Some(why.into());
}
if !ENABLED.load(Ordering::Relaxed) {
return Some("disabled by the GPU rendering preference (app.gpu)".into());
@@ -245,12 +241,26 @@ pub fn wait_idle(timeout: std::time::Duration) -> bool {
/// environment switches and the rendering preference, but not a render-device failure (the devices are separate).
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn switched_off() -> Option<String> {
- if env_disabled() {
- return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());
+ if let Some(why) = env_off_reason() {
+ return Some(why.into());
}
(!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into())
}
+/// Why the environment switches the GPU off for the whole process (`LIGHTCRAFT_GPU=0` or
+/// `LIGHTCRAFT_GPU_BACKEND=off`), or `None`. Code with a GPU path of its own (the SAM 3 model on
+/// Metal) follows it too.
+pub fn env_off_reason() -> Option<&'static str> {
+ if !env_disabled() {
+ return None;
+ }
+ #[cfg(not(target_arch = "wasm32"))]
+ if backend::env_off() {
+ return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off");
+ }
+ Some("disabled by LIGHTCRAFT_GPU=0")
+}
+
/// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process.
fn env_disabled() -> bool {
static OFF: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs
index 6d488a919..6c24d0836 100644
--- a/crates/segment/src/lib.rs
+++ b/crates/segment/src/lib.rs
@@ -87,6 +87,11 @@ pub fn is_model_dir(dir: &Path) -> bool {
dir.join(WEIGHTS_FILE).is_file() && dir.join("vocab.json").is_file() && dir.join("merges.txt").is_file()
}
+#[cfg(not(target_arch = "wasm32"))]
+/// Whether [`best_device`] can be a GPU in this build (Metal on macOS). Where it can, a caller that
+/// sees the model fail there can load it again with [`Sam3::load_cpu`].
+pub const GPU_BUILD: bool = cfg!(target_os = "macos");
+
#[cfg(not(target_arch = "wasm32"))]
/// The best device here: Metal on macOS (when a GPU is available), else the CPU.
pub fn best_device() -> Device {
@@ -134,6 +139,11 @@ impl Sam3 {
Self::load_on(dir, best_device())
}
+ /// [`Sam3::load`] on the CPU, whatever GPU there is.
+ pub fn load_cpu(dir: &Path) -> Result<Self> {
+ Self::load_on(dir, Device::Cpu)
+ }
+
pub fn load_on(dir: &Path, device: Device) -> Result<Self> {
if !is_model_dir(dir) {
return Err(Error::Missing(dir.to_path_buf()));
diff --git a/crates/segment/src/remote.rs b/crates/segment/src/remote.rs
index cdee096fe..10f7b2931 100644
--- a/crates/segment/src/remote.rs
+++ b/crates/segment/src/remote.rs
@@ -108,10 +108,10 @@ impl Client {
}
}
-pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result<Option<Vec<f32>>, String> {
+pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> crate::Result<Option<Vec<f32>>> {
let mut merged: Option<Vec<f32>> = None;
for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
- if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {
+ if let Some(p) = model.segment_text(enc, phrase, 0.5)? {
if let Some(m) = &mut merged {
for (a, b) in m.iter_mut().zip(p.logits) {
*a = a.max(b);
@@ -130,7 +130,7 @@ fn predict(model: &mut Sam3, enc: &mut Encoded, operation: Operation) -> Result<
model.prepare_clicks(enc).map_err(|e| e.to_string())?;
Ok(None)
}
- Operation::Text(text) => text_logits(model, enc, &text),
+ Operation::Text(text) => text_logits(model, enc, &text).map_err(|e| e.to_string()),
Operation::Clicks(points) => {
let clicks: Vec<_> = points.into_iter().map(|p| Click { x: p[0], y: p[1], positive: p[2] == 1.0 }).collect();
Ok(Some(model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?.logits))
diff --git a/docs/ai-masks.md b/docs/ai-masks.md
index b02912abc..cd803b388 100644
--- a/docs/ai-masks.md
+++ b/docs/ai-masks.md
@@ -2,7 +2,8 @@
LightCraft selects objects with **SAM 3** (Segment Anything with Concepts, Meta 2025), run in
pure Rust by `crates/segment` on [candle](https://github.com/huggingface/candle): on the GPU
-through Metal on macOS, on the CPU elsewhere (for now).
+through Metal on macOS, on the CPU elsewhere (for now). If the Mac's GPU can't run it, it runs on
+the CPU (see the worker below).
**The model is optional.** It is not part of LightCraft, and nothing requires it: without it,
Object and Describe offer to download it, every other feature (and every existing mask, see
@@ -225,6 +226,13 @@ wait and return the result (when built with the `sam` feature; the default CLI h
encoded photo; the session sends it jobs and receives results over channels, never holding a
lock the model holds. Each job runs under `catch_unwind`: a panic inside candle becomes an
error and the model is reloaded on the next request; busy counters are reset by drop guards.
+- GPU fallback (issue #1974): a job that fails on the GPU (a panic or a candle error, e.g. an
+ AMD GPU in an Intel Mac whose Metal compiler crashes on a matmul kernel) frees the model and runs
+ once more on the CPU. If that works, the model stays on the CPU until LightCraft restarts and
+ the user is told once ("The GPU couldn't run the AI mask model…"); if the CPU fails too, the
+ error is reported and the next job tries the GPU again. Missing or damaged model files and
+ remote-worker errors are not retried. `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off` keep
+ the model on the CPU from the start.
- Hostile documents: a selection keeps at most 4 detail patches (when read and when rendered),
each at most 1024² logits; more than 64 clicks are never sent to the model.
- Accuracy: `crates/segment/tests/reference.rs` compares every stage against the Python
diff --git a/docs/gpu-pipeline.md b/docs/gpu-pipeline.md
index b74af85a2..5bd0119f9 100644
--- a/docs/gpu-pipeline.md
+++ b/docs/gpu-pipeline.md
@@ -43,6 +43,8 @@ Overrides (read once at launch):
- else `WGPU_BACKEND` (wgpu's own variable, same names) — before #136 only the window honoured it,
the compute device always used Vulkan + DX12 (+ Metal).
- `LIGHTCRAFT_GPU=0`: GPU rendering off for the process (the window is unaffected).
+- Both `LIGHTCRAFT_GPU=0` and `LIGHTCRAFT_GPU_BACKEND=off` also keep AI Denoise, the face models
+ and the AI mask model (SAM 3 on Metal) on the CPU.
**DX12 shader compiler (issue #471).** Every instance (window and compute) compiles DX12 shaders with
FXC (`d3dcompiler_47.dll`, part of Windows; `lightcraft_gpu::backend::backend_options`). wgpu's
diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md
index 95dcd2aef..a78ff0f7c 100644
--- a/docs/parity-checklist.md
+++ b/docs/parity-checklist.md
@@ -293,7 +293,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts).
| LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model; a photo that carries person mattes (DNG semantic masks: iPhone ProRAW portrait / skin / hair / teeth / glasses mattes) uses them instead (synthetic files only: no corpus file carries person mattes) |
| LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`), `crates/pipeline/src/masks.rs` (`Mattes`), `crates/raw/src/semantic.rs` | heuristic; a photo that carries a sky matte (DNG semantic mask, e.g. iPhone ProRAW) uses it instead, through orientation / crop / lens corrections, preview = export (checked on one CC0 iPhone 12 Pro ProRAW and synthetic files) |
| LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject mask (heuristic, or the photo's person mattes) |
-| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) |
+| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors; a job that fails on the Mac's GPU runs again on the CPU, which then serves the rest of the session, #1974), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) |
| LR-MASK-PEOPLE | People parts | P2 | ⬜ | | shape exists (no UI): uses the photo's person / skin / hair / teeth / glasses mattes when it carries them (DNG semantic masks), else the subject heuristic |
| LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty |
| LR-MASK-BRUSH | Brush mask | P0 | 🟡 | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs`, `crates/ui-egui/src/brush_live.rs` | size/feather/flow/density/erase; Create New Mask starts a separate brush in paint mode; the stroke shows while the button is held with the committed stroke's dab spacing, feather and flow, so fast strokes have no gaps (#517); coverage builds up by passes, not hold time (holding the brush still adds nothing, live or committed, #518); Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); ⇧-click draws a straight stroke from where the last stroke ended (`connect`), tapering size, feather and flow from that stroke's values when they changed; no A/B brushes; pen pressure (Brush ▸ Pen Pressure, on by default; `pressure` per point in `mask.brushStroke`) scales each dab's size (a tenth to all of it) and flow, live and committed alike, where winit reports a pen force (Windows; not macOS / Linux tablets yet); no tilt |