Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

Faces: run the networks' convolutions on the GPU (wgpu) when it agrees with the CPU and is faster

Andrew Frahn0242a75b5a8b
diff --git a/Cargo.lock b/Cargo.lock
index bf2ac824f..db28a530d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3299,6 +3299,7 @@ dependencies = [
"lightcraft-color",
"lightcraft-denoise",
"lightcraft-develop",
+ "lightcraft-faces",
"lightcraft-geom",
"lightcraft-pipeline",
"lightcraft-raster",
diff --git a/crates/engine/src/cmd/face_detect.rs b/crates/engine/src/cmd/face_detect.rs
index ffdedbef1..8e706c367 100644
--- a/crates/engine/src/cmd/face_detect.rs
+++ b/crates/engine/src/cmd/face_detect.rs
@@ -56,10 +56,38 @@ pub(crate) fn detector(s: &Session) -> std::result::Result<Arc<Detector>, String
}
let bytes = std::fs::read(&path).map_err(|e| format!("could not read the detector: {e}"))?;
let d = Arc::new(Detector::new(&bytes).map_err(|e| format!("{e}: remove the detector in Settings > Faces and download it again"))?);
+ #[cfg(not(target_arch = "wasm32"))]
+ {
+ let net = d.clone();
+ offer_gpu("detection", move |gpu| net.accelerate(gpu, true));
+ }
*loaded = Some((path, d.clone()));
Ok(d)
}
+/// Check in the background whether the GPU runs a face model faster than the CPU and gives its results; the model
+/// runs on the CPU meanwhile, and stays there when the GPU is off, missing, wrong or slower.
+#[cfg(not(target_arch = "wasm32"))]
+pub(crate) fn offer_gpu(
+ what: &'static str,
+ attach: impl FnOnce(Arc<dyn lightcraft_faces::net::Accelerator>) -> std::result::Result<(), String> + Send + 'static,
+) {
+ let spawned = std::thread::Builder::new().name(format!("faces-gpu-{what}")).spawn(move || {
+ let gpu = match lightcraft_gpu::faces::accelerator() {
+ Ok(gpu) => gpu,
+ Err(why) => return log::info!("faces: {what} runs on the CPU ({why})"),
+ };
+ let device = lightcraft_faces::net::Accelerator::device(&*gpu);
+ match attach(gpu) {
+ Ok(()) => log::info!("faces: {what} runs on {device}"),
+ Err(why) => log::info!("faces: {what} stays on the CPU: {why}"),
+ }
+ });
+ if let Err(e) = spawned {
+ log::warn!("faces: could not check the GPU for {what}: {e}");
+ }
+}
+
fn face_json(f: &Face) -> Value {
json!({
"rect": {"x0": f.x0, "y0": f.y0, "x1": f.x1, "y1": f.y1},
diff --git a/crates/engine/src/cmd/face_recognize.rs b/crates/engine/src/cmd/face_recognize.rs
index e099f13ec..094735c6d 100644
--- a/crates/engine/src/cmd/face_recognize.rs
+++ b/crates/engine/src/cmd/face_recognize.rs
@@ -234,6 +234,8 @@ mod imp {
return Ok(e.clone());
}
let embedder = Arc::new(Embedder::load(&dir.join(&id).join("model.onnx"), &installed.manifest).map_err(|e| bad(cmd, e.to_string()))?);
+ let net = embedder.clone();
+ super::super::face_detect::offer_gpu("recognition", move |gpu| net.accelerate(gpu, true).map_err(|e| e.to_string()));
let dim = match installed.manifest.output {
lightcraft_faces::OutputSpec::Embedding { dim } => dim as usize,
_ => return Err(bad(cmd, "the chosen model is not a recognition model")),
diff --git a/crates/faces/src/lib.rs b/crates/faces/src/lib.rs
index b4b7066d5..030cda088 100644
--- a/crates/faces/src/lib.rs
+++ b/crates/faces/src/lib.rs
@@ -5,7 +5,8 @@
//!
//! Everything here treats model files and manifests as hostile input: sizes are capped, numbers must be
//! finite and sane, and a malformed file is an error, never a panic. Detection and recognition run
-//! on the same checked, single-thread pure-Rust CPU interpreter.
+//! on the same checked, single-thread pure-Rust CPU interpreter; an [`net::Accelerator`] (the GPU, in `lightcraft-gpu`) can
+//! take its convolutions once it agrees with it.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
diff --git a/crates/faces/src/net.rs b/crates/faces/src/net.rs
index 6222314f2..3983404c5 100644
--- a/crates/faces/src/net.rs
+++ b/crates/faces/src/net.rs
@@ -9,6 +9,7 @@
use std::collections::HashMap;
use std::rc::Rc;
+use std::sync::{Arc, PoisonError, RwLock};
mod kernels;
mod shape;
@@ -64,9 +65,58 @@ impl Tensor {
}
}
+/// One convolution, as handed to an [`Accelerator`]: batch 1, planar (NCHW) float32, already checked.
+///
+/// `out[co][oy][ox] = bias[co] + Σ weights[co][ci][ky][kx] · input[g·cg + ci][oy·sh + ky − pt][ox·sw + kx − pl]`
+/// with `g = co / (cout / groups)` and zero outside the input.
+#[derive(Clone, Copy, Debug)]
+pub struct ConvJob<'a> {
+ /// The node's index in the graph: the same for every run of a net, so its weights can stay on the device.
+ pub layer: usize,
+ pub cin: usize,
+ pub h: usize,
+ pub w: usize,
+ pub cout: usize,
+ pub groups: usize,
+ pub kh: usize,
+ pub kw: usize,
+ pub sh: usize,
+ pub sw: usize,
+ pub pt: usize,
+ pub pl: usize,
+ pub oh: usize,
+ pub ow: usize,
+ /// `[cout][cin / groups][kh][kw]`.
+ pub weights: &'a [f32],
+ /// `[cout]`, if the node has one.
+ pub bias: Option<&'a [f32]>,
+ /// `[cin][h][w]`.
+ pub input: &'a [f32],
+}
+
+impl ConvJob<'_> {
+ /// Multiply-adds the convolution takes.
+ pub fn macs(&self) -> usize {
+ let per_group = (self.cin / self.groups.max(1)).saturating_mul(self.kh).saturating_mul(self.kw);
+ self.cout.saturating_mul(per_group).saturating_mul(self.oh).saturating_mul(self.ow)
+ }
+}
+
+/// Something that can run a network's convolutions faster than the CPU kernels (the GPU, in `lightcraft-gpu`).
+///
+/// The CPU interpreter stays the reference: a net only uses an accelerator that agreed with it on a test input
+/// ([`Net::attach_checked`]), and the first error makes the net drop it and run on the CPU again.
+pub trait Accelerator: Send + Sync {
+ /// What it runs on, for the user ("AMD Radeon RX 7900 (Vulkan)").
+ fn device(&self) -> String;
+ /// `[cout][oh][ow]`, or `Ok(None)` to leave this one to the CPU (too small to be worth a dispatch, say).
+ fn conv(&self, job: &ConvJob) -> std::result::Result<Option<Vec<f32>>, String>;
+}
+
/// A loaded network, ready to run.
pub struct Net {
graph: Graph,
+ accelerator: RwLock<Option<Arc<dyn Accelerator>>>,
}
/// Validate semantics rather than silently ignoring unknown or mistyped attributes.
@@ -80,7 +130,75 @@ impl Net {
for n in &graph.nodes {
check(n, &graph)?;
}
- Ok(Net { graph })
+ Ok(Net { graph, accelerator: RwLock::new(None) })
+ }
+
+ /// The accelerator in use, if any.
+ pub fn accelerator(&self) -> Option<Arc<dyn Accelerator>> {
+ self.accelerator.read().unwrap_or_else(PoisonError::into_inner).clone()
+ }
+
+ /// Use `accelerator` for this net's convolutions, but only after it gave the CPU's outputs on `probe` (every
+ /// output value within `tolerance` of the CPU's, relative to that output's largest magnitude) and, with
+ /// `require_faster`, ran it faster than the CPU did (a GPU's per-layer round trip can cost more than a small
+ /// model's layers). Returns why it was not used, so the caller can say so; the net then stays on the CPU.
+ pub fn attach_checked(
+ &self,
+ accelerator: Arc<dyn Accelerator>,
+ probe: &Tensor,
+ tolerance: f32,
+ require_faster: bool,
+ ) -> std::result::Result<(), String> {
+ self.detach();
+ let started = web_time::Instant::now();
+ let cpu = self.run(probe.clone()).map_err(|e| e.to_string())?;
+ let cpu_time = started.elapsed();
+ *self.accelerator.write().unwrap_or_else(PoisonError::into_inner) = Some(accelerator);
+ // the first run also uploads the weights and builds the kernels: time a second one
+ let fast = self.run(probe.clone()).map_err(|e| e.to_string());
+ let mut verdict = match fast {
+ Err(e) => Err(e),
+ Ok(_) if self.accelerator().is_none() => Err("the device failed on the test input".into()),
+ Ok(fast) => agree(&cpu, &fast, tolerance),
+ };
+ if verdict.is_ok() && require_faster {
+ let started = web_time::Instant::now();
+ let again = self.run(probe.clone()).map_err(|e| e.to_string());
+ let fast_time = started.elapsed();
+ verdict = match again {
+ Err(e) => Err(e),
+ Ok(_) if self.accelerator().is_none() => Err("the device failed on the test input".into()),
+ Ok(_) if fast_time >= cpu_time => Err(format!(
+ "the device is not faster for this model ({:.0} ms against {:.0} ms on the CPU)",
+ fast_time.as_secs_f64() * 1000.0,
+ cpu_time.as_secs_f64() * 1000.0
+ )),
+ Ok(_) => Ok(()),
+ };
+ }
+ if verdict.is_err() {
+ self.detach();
+ }
+ verdict
+ }
+
+ /// Run on the CPU only from now on.
+ pub fn detach(&self) {
+ *self.accelerator.write().unwrap_or_else(PoisonError::into_inner) = None;
+ }
+
+ /// A convolution on the accelerator, if one is attached and takes it. A failing accelerator is dropped.
+ fn accelerated_conv(&self, layer: usize, node: &Node, x: &Tensor) -> Result<Option<Tensor>> {
+ let Some(accelerator) = self.accelerator() else { return Ok(None) };
+ let job = kernels::conv_job(layer, node, x, &self.graph)?;
+ match accelerator.conv(&job) {
+ Ok(Some(out)) if out.len() == job.cout * job.oh * job.ow => Ok(Some(Tensor::new(vec![1, job.cout, job.oh, job.ow], out)?)),
+ Ok(None) => Ok(None),
+ Ok(Some(_)) | Err(_) => {
+ self.detach();
+ Ok(None)
+ }
+ }
}
/// The shape the model declares for its input (0 for a dynamic size).
@@ -127,7 +245,13 @@ impl Net {
};
let started = web_time::Instant::now();
let out = match node.op.as_str() {
- "Conv" => kernels::conv(node, &*get(0)?, g, &mut scratch)?,
+ "Conv" => {
+ let x = get(0)?;
+ match self.accelerated_conv(i, node, &x)? {
+ Some(out) => out,
+ None => kernels::conv(node, &x, g, &mut scratch)?,
+ }
+ }
"Relu" => map(&*get(0)?, |v| v.max(0.0)),
"Sigmoid" => map(&*get(0)?, |v| 1.0 / (1.0 + (-v).exp())),
"Add" | "Sub" | "Mul" => kernels::binary(node.op.as_str(), &*get(0)?, &*get(1)?)?,
@@ -168,6 +292,25 @@ impl Net {
}
}
+/// Whether two runs' outputs agree: same names and shapes, values within `tolerance` of each other relative to the
+/// output's largest magnitude.
+fn agree(a: &[(String, Tensor)], b: &[(String, Tensor)], tolerance: f32) -> std::result::Result<(), String> {
+ if a.len() != b.len() {
+ return Err("the device gave a different number of outputs".into());
+ }
+ for ((name, x), (other, y)) in a.iter().zip(b) {
+ if name != other || x.shape != y.shape {
+ return Err(format!("the device gave `{other}` a different shape"));
+ }
+ let scale = x.data.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-6);
+ let worst = x.data.iter().zip(&y.data).fold(0f32, |m, (p, q)| m.max((p - q).abs()));
+ if worst.is_nan() || worst > tolerance * scale {
+ return Err(format!("the device's `{name}` differs from the CPU's by {:.2e} (of {scale:.2e})", worst));
+ }
+ }
+ Ok(())
+}
+
fn map(x: &Tensor, f: impl Fn(f32) -> f32) -> Tensor {
Tensor { shape: x.shape.clone(), data: x.data.iter().map(|v| f(*v)).collect() }
}
@@ -534,6 +677,74 @@ mod tests {
assert_eq!(rs.shape, vec![1, 6, 2]);
}
+ /// An accelerator built on the slow reference convolution, optionally wrong or failing.
+ struct Fake {
+ skew: f32,
+ fail_after: std::sync::atomic::AtomicUsize,
+ }
+ impl Accelerator for Fake {
+ fn device(&self) -> String {
+ "test".into()
+ }
+ fn conv(&self, j: &ConvJob) -> std::result::Result<Option<Vec<f32>>, String> {
+ if self.fail_after.fetch_sub(1, std::sync::atomic::Ordering::Relaxed) == 0 {
+ return Err("device lost".into());
+ }
+ assert!(j.pt == j.pl && j.sh == j.sw);
+ let x = Tensor::new(vec![1, j.cin, j.h, j.w], j.input.to_vec()).unwrap();
+ let zero = vec![0.0; j.cout];
+ let dims = [j.cout, j.cin / j.groups, j.kh, j.kw];
+ let out = reference_conv(&x, j.weights, dims, j.bias.unwrap_or(&zero), j.sh, j.pt, j.groups);
+ Ok(Some(out.data.iter().map(|v| v + self.skew).collect()))
+ }
+ }
+
+ fn two_convs() -> (Graph, Tensor) {
+ let attrs = [("strides", Attr::Ints(vec![2, 2])), ("pads", Attr::Ints(vec![1; 4]))];
+ let mut a = node("Conv", &["x", "w1", "b1"], &attrs);
+ a.outputs = vec!["h".into()];
+ let b = node("Conv", &["h", "w2"], &[("group", Attr::Int(2)), ("pads", Attr::Ints(vec![1; 4]))]);
+ let mut g = graph(
+ vec![a, b],
+ vec![
+ ("w1", w(&[4, 3, 3, 3], (0..108).map(|i| ((i * 7 % 13) as f32 - 6.0) * 0.05).collect())),
+ ("b1", w(&[4], vec![0.1, -0.2, 0.3, 0.0])),
+ ("w2", w(&[4, 2, 3, 3], (0..72).map(|i| ((i * 5 % 11) as f32 - 5.0) * 0.05).collect())),
+ ],
+ );
+ g.inputs = vec![("x".into(), vec![1, 3, 9, 8])];
+ (g, input(3, 9, 8, |i| ((i * 13 % 17) as f32 - 8.0) * 0.1))
+ }
+
+ #[test]
+ fn an_accelerator_is_used_only_when_it_agrees_with_the_cpu() {
+ let (g, x) = two_convs();
+ let net = Net::new(g).unwrap();
+ let cpu = net.run(x.clone()).unwrap();
+ let good = Arc::new(Fake { skew: 0.0, fail_after: usize::MAX.into() });
+ net.attach_checked(good, &x, 1e-4, false).unwrap();
+ assert!(net.accelerator().is_some());
+ assert_eq!(net.run(x.clone()).unwrap()[0].1.shape, cpu[0].1.shape);
+ let wrong = Arc::new(Fake { skew: 0.5, fail_after: usize::MAX.into() });
+ assert!(net.attach_checked(wrong, &x, 1e-4, false).is_err());
+ assert!(net.accelerator().is_none(), "a device that disagrees is not used");
+ assert_eq!(net.run(x).unwrap(), cpu);
+ }
+
+ #[test]
+ fn a_failing_accelerator_falls_back_to_the_cpu_mid_run() {
+ let (g, x) = two_convs();
+ let net = Net::new(g).unwrap();
+ let cpu = net.run(x.clone()).unwrap();
+ // agrees on the probe (two convs), then fails on the second conv of the next run
+ net.attach_checked(Arc::new(Fake { skew: 0.0, fail_after: 3.into() }), &x, 1e-4, false).unwrap();
+ let out = net.run(x).unwrap();
+ assert!(net.accelerator().is_none());
+ for (a, b) in out[0].1.data.iter().zip(&cpu[0].1.data) {
+ assert!((a - b).abs() < 1e-4);
+ }
+ }
+
#[test]
fn unsupported_and_inconsistent_models_are_errors() {
let x = input(1, 3, 3, |i| i as f32);
diff --git a/crates/faces/src/net/kernels.rs b/crates/faces/src/net/kernels.rs
index 73cc72b93..6d7c1dbb9 100644
--- a/crates/faces/src/net/kernels.rs
+++ b/crates/faces/src/net/kernels.rs
@@ -59,6 +59,33 @@ fn shifted_plane(plane: &[f32], width: usize, ky: usize, kx: usize, pt: usize, p
}
Ok(())
}
+/// The checked geometry of a convolution node, for an accelerator.
+pub(super) fn conv_job<'a>(layer: usize, n: &Node, x: &'a Tensor, g: &'a Graph) -> Result<ConvJob<'a>> {
+ let (cin, h, w) = chw(x)?;
+ let wt = shape::weight(n, g, 1)?;
+ let weights = shape::floats(n, g, 1)?;
+ let [cout, cg, kh, kw] = wt.dims.as_slice() else {
+ return shape_err("invalid convolution weights");
+ };
+ let (cout, cg, kh, kw) = (*cout, *cg, *kh, *kw);
+ let groups = usize::try_from(n.int("group").unwrap_or(1)).map_err(|_| NetError::Shape("invalid group".into()))?;
+ let (sh, sw) = pair(n, "strides", 1)?;
+ let (pt, pl, pb, pr) = pads(n)?;
+ let ph = h.checked_add(pt).and_then(|v| v.checked_add(pb)).ok_or_else(|| NetError::Shape("padding overflow".into()))?;
+ let pw = w.checked_add(pl).and_then(|v| v.checked_add(pr)).ok_or_else(|| NetError::Shape("padding overflow".into()))?;
+ if ph < kh || pw < kw || groups == 0 || cg == 0 || kh == 0 || kw == 0 || cin != groups * cg || cout % groups != 0 {
+ return shape_err("invalid convolution shape");
+ }
+ if cout.checked_mul(cg).and_then(|v| v.checked_mul(kh * kw)) != Some(weights.len()) {
+ return shape_err("short convolution weights");
+ }
+ let bias = if n.inputs.get(2).is_some_and(|s| !s.is_empty()) { Some(shape::floats(n, g, 2)?) } else { None };
+ if bias.is_some_and(|b| b.len() != cout) {
+ return shape_err("the convolution bias does not match");
+ }
+ let (oh, ow) = ((ph - kh) / sh + 1, (pw - kw) / sw + 1);
+ Ok(ConvJob { layer, cin, h, w, cout, groups, kh, kw, sh, sw, pt, pl, oh, ow, weights, bias, input: &x.data })
+}
pub(super) fn conv(n: &Node, x: &Tensor, g: &Graph, scratch: &mut Scratch) -> Result<Tensor> {
let (cin, h, w) = chw(x)?;
let wt = shape::weight(n, g, 1)?;
diff --git a/crates/faces/src/runtime.rs b/crates/faces/src/runtime.rs
index bbb1457eb..7a8ab948a 100644
--- a/crates/faces/src/runtime.rs
+++ b/crates/faces/src/runtime.rs
@@ -99,6 +99,31 @@ impl Embedder {
})
}
+ /// The network, e.g. to see which device it runs on.
+ pub fn net(&self) -> &Net {
+ &self.plan
+ }
+
+ /// Run the model's convolutions on `accelerator` if it gives the CPU's embedding on a test face (and, with
+ /// `require_faster`, runs it faster than the CPU); otherwise (or on
+ /// its first error) recognition stays on the CPU. Returns why it was not used.
+ pub fn accelerate(&self, accelerator: Arc<dyn crate::net::Accelerator>, require_faster: bool) -> Result<(), RuntimeError> {
+ let (mean, std) = (self.manifest.input.mean, self.manifest.input.std);
+ let plane = self.width * self.height;
+ let data: Vec<f32> = crate::yunet::probe_pixels(3 * plane)
+ .into_iter()
+ .enumerate()
+ .map(|(i, v)| {
+ let c = (i / plane.max(1)).min(2);
+ (v - mean.get(c).copied().unwrap_or(0.0)) / std.get(c).copied().filter(|s| *s != 0.0).unwrap_or(1.0)
+ })
+ .collect();
+ let probe = Tensor::new(vec![1, 3, self.height, self.width], data).map_err(|e| RuntimeError::Input(e.to_string()))?;
+ guarded("checking the device", || {
+ self.plan.attach_checked(accelerator, &probe, crate::yunet::ACCELERATOR_TOLERANCE, require_faster).map_err(RuntimeError::Run)
+ })
+ }
+
/// The size of the aligned face picture the model wants (width, height).
pub fn input_size(&self) -> (usize, usize) {
(self.width, self.height)
diff --git a/crates/faces/src/yunet.rs b/crates/faces/src/yunet.rs
index b77f318bf..d1c62f75c 100644
--- a/crates/faces/src/yunet.rs
+++ b/crates/faces/src/yunet.rs
@@ -60,6 +60,14 @@ pub struct Face {
pub landmarks: [(f32, f32); 5],
}
+/// How far an accelerator's outputs may stray from the CPU's on the test input, relative to each output's largest value.
+pub const ACCELERATOR_TOLERANCE: f32 = 1e-3;
+
+/// A deterministic, photo-like test input (smooth gradients plus texture, in 0..255).
+pub fn probe_pixels(n: usize) -> Vec<f32> {
+ (0..n).map(|i| ((i % 640) as f32 * 0.3 + ((i * 7919) % 97) as f32 + (i / 640 % 480) as f32 * 0.2) % 255.0).collect()
+}
+
pub struct Detector {
net: Net,
/// Side of the square network input.
@@ -90,6 +98,20 @@ impl Detector {
Ok(Detector { net, side })
}
+ /// The network, e.g. to see which device it runs on.
+ pub fn net(&self) -> &Net {
+ &self.net
+ }
+
+ /// Run the network's convolutions on `accelerator` if it gives the CPU's outputs on a test picture (and, with `require_faster`,
+ /// runs it faster than the CPU); otherwise
+ /// (or on its first error) detection stays on the CPU. Returns why it was not used.
+ pub fn accelerate(&self, accelerator: std::sync::Arc<dyn crate::net::Accelerator>, require_faster: bool) -> Result<(), String> {
+ let side = self.side;
+ let probe = Tensor::new(vec![1, 3, side, side], probe_pixels(3 * side * side)).map_err(|e| e.to_string())?;
+ self.net.attach_checked(accelerator, &probe, ACCELERATOR_TOLERANCE, require_faster)
+ }
+
/// The square input side, used to check an installed manifest.
pub fn input_side(&self) -> usize {
self.side
diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml
index dc348ee88..060abb112 100644
--- a/crates/gpu/Cargo.toml
+++ b/crates/gpu/Cargo.toml
@@ -14,6 +14,7 @@ lightcraft-geom = { workspace = true }
lightcraft-color = { workspace = true }
lightcraft-raster = { workspace = true }
lightcraft-develop = { workspace = true }
+lightcraft-faces = { workspace = true }
lightcraft-pipeline = { workspace = true }
log = { workspace = true }
diff --git a/crates/gpu/src/faces.rs b/crates/gpu/src/faces.rs
new file mode 100644
index 000000000..ae982d30b
--- /dev/null
+++ b/crates/gpu/src/faces.rs
@@ -0,0 +1,593 @@
+//! Face networks on the GPU: an [`lightcraft_faces::net::Accelerator`] that runs the interpreter's convolutions (most of
+//! YuNet's, SFace's and AuraFace's time) with a WGSL kernel (`wgsl/faces_conv.wgsl`). Everything else in the graph
+//! stays on the CPU interpreter, which is the reference: a net attaches this only after it agreed with the CPU on a
+//! test input (`Net::attach_checked`), and drops it on the first error.
+//!
+//! The device is a separate one (like denoise's), so a face scan and the interactive renders never share a queue, and
+//! it respects the same switches as GPU rendering (`LIGHTCRAFT_GPU`, `LIGHTCRAFT_GPU_BACKEND`, the GPU preference)
+//! and the crash sentinel around device creation. A model's weights are uploaded once, on first use, and stay on the
+//! device while the accelerator (one per loaded model) lives. Every failure is an `Err`, never a panic.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex, OnceLock, PoisonError};
+use std::time::Duration;
+
+use bytemuck::{Pod, Zeroable};
+use lightcraft_faces::net::{Accelerator, ConvJob};
+use wgpu::util::DeviceExt;
+
+/// Longest wait for one convolution before the device is given up on.
+const TIMEOUT: Duration = Duration::from_secs(30);
+/// Below this many multiply-adds a convolution stays on the CPU: the upload, dispatch and read-back cost more.
+const MIN_MACS: usize = 1 << 20;
+/// Workgroups a convolution should make at least, cutting its K into slices when its tiles are too few.
+const TARGET_WORKGROUPS: usize = 256;
+/// Weights one model may keep on the device (AuraFace's are about 250 MB).
+const MAX_RESIDENT: u64 = 768 << 20;
+
+struct Dev {
+ device: wgpu::Device,
+ queue: wgpu::Queue,
+ info: wgpu::AdapterInfo,
+ limits: wgpu::Limits,
+ layout: wgpu::BindGroupLayout,
+ tiled: wgpu::ComputePipeline,
+ direct: wgpu::ComputePipeline,
+ reduce: wgpu::ComputePipeline,
+}
+
+static DEV: OnceLock<Result<Dev, String>> = OnceLock::new();
+static BROKEN: AtomicBool = AtomicBool::new(false);
+static BROKEN_WHY: Mutex<Option<String>> = Mutex::new(None);
+
+fn mark_broken(why: &str) {
+ let mut w = BROKEN_WHY.lock().unwrap_or_else(PoisonError::into_inner);
+ if w.is_none() {
+ *w = Some(why.to_string());
+ }
+ BROKEN.store(true, Ordering::Relaxed);
+}
+
+/// Why the GPU stopped being used for faces (a device error), if it did.
+pub fn broken_reason() -> Option<String> {
+ BROKEN.load(Ordering::Relaxed).then(|| BROKEN_WHY.lock().unwrap_or_else(PoisonError::into_inner).clone().unwrap_or_else(|| "device error".into()))
+}
+
+fn dev() -> Result<&'static Dev, String> {
+ if let Some(why) = broken_reason() {
+ return Err(format!("stopped after a GPU failure: {why}"));
+ }
+ if let Some(why) = crate::switched_off() {
+ return Err(why);
+ }
+ let _work = match DEV.get() {
+ Some(_) => None,
+ None => Some(crate::exit::enter().ok_or("the app is closing")?),
+ };
+ DEV.get_or_init(create_device).as_ref().map_err(|e| e.clone())
+}
+
+fn create_device() -> Result<Dev, String> {
+ let Some(backends) = crate::backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) };
+ crate::backend::with_init_marker(backends, || {
+ std::panic::catch_unwind(|| make_device(backends)).unwrap_or_else(|_| Err("device creation panicked".into()))
+ })
+}
+
+fn entry(binding: u32, ty: wgpu::BufferBindingType) -> wgpu::BindGroupLayoutEntry {
+ wgpu::BindGroupLayoutEntry {
+ binding,
+ visibility: wgpu::ShaderStages::COMPUTE,
+ ty: wgpu::BindingType::Buffer { ty, has_dynamic_offset: false, min_binding_size: None },
+ count: None,
+ }
+}
+
+fn make_device(backends: wgpu::Backends) -> Result<Dev, String> {
+ let mut desc = wgpu::InstanceDescriptor::new_without_display_handle();
+ desc.backends = backends;
+ desc.backend_options = crate::backend::backend_options();
+ desc.flags = crate::backend::instance_flags_from_env(desc.flags);
+ let instance = wgpu::Instance::new(desc);
+ let wanted = std::env::var("LIGHTCRAFT_GPU_ADAPTER").ok().map(|s| s.trim().to_lowercase()).filter(|s| !s.is_empty());
+ let adapter = match &wanted {
+ Some(w) => pollster::block_on(instance.enumerate_adapters(backends))
+ .into_iter()
+ .find(|a| {
+ let i = a.get_info();
+ i.device_type != wgpu::DeviceType::Cpu && i.name.to_lowercase().contains(w.as_str())
+ })
+ .ok_or_else(|| format!("no GPU adapter named like `{w}`"))?,
+ None => pollster::block_on(
+ instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }),
+ )
+ .map_err(|e| format!("no GPU adapter among {backends:?} ({e})"))?,
+ };
+ let info = adapter.get_info();
+ if info.device_type == wgpu::DeviceType::Cpu {
+ return Err(format!("software adapter ({}) skipped: the CPU is faster", info.name));
+ }
+ let limits = adapter.limits();
+ if limits.max_storage_buffers_per_shader_stage < 4
+ || limits.max_compute_invocations_per_workgroup < 256
+ || limits.max_compute_workgroup_size_x < 256
+ || limits.max_compute_workgroup_storage_size < 8 * 1024
+ {
+ return Err(format!("{} is too limited for the face kernels", info.name));
+ }
+ let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {
+ label: Some("lightcraft-faces"),
+ required_limits: limits.clone(),
+ ..Default::default()
+ }))
+ .map_err(|e| format!("{}: device creation failed ({e})", info.name))?;
+ device.on_uncaptured_error(Arc::new(|e| {
+ log::error!("gpu (faces): {e}");
+ mark_broken(&format!("device error: {e}"));
+ }));
+ device.set_device_lost_callback(|reason, msg| {
+ log::error!("gpu (faces): device lost ({reason:?}): {msg}");
+ mark_broken(&format!("device lost ({reason:?}): {msg}"));
+ });
+ use wgpu::BufferBindingType::{Storage, Uniform};
+ let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+ label: Some("faces conv"),
+ entries: &[
+ entry(0, Uniform),
+ entry(1, Storage { read_only: true }),
+ entry(2, Storage { read_only: true }),
+ entry(3, Storage { read_only: true }),
+ entry(4, Storage { read_only: false }),
+ ],
+ });
+ let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
+ label: Some("faces conv"),
+ source: wgpu::ShaderSource::Wgsl(include_str!("wgsl/faces_conv.wgsl").into()),
+ });
+ let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: None, bind_group_layouts: &[Some(&layout)], immediate_size: 0 });
+ let pipe = |entry: &str| {
+ device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
+ label: Some(entry),
+ layout: Some(&pl),
+ module: &module,
+ entry_point: Some(entry),
+ // both kernels write all of their workgroup memory before reading it
+ compilation_options: wgpu::PipelineCompilationOptions { zero_initialize_workgroup_memory: false, ..Default::default() },
+ cache: None,
+ })
+ };
+ let (tiled, direct, reduce) = (pipe("tiled"), pipe("direct"), pipe("reduce"));
+ log::info!("gpu (faces): {} ({:?})", info.name, info.backend);
+ Ok(Dev { device, queue, info, limits, layout, tiled, direct, reduce })
+}
+
+/// The adapter face scans would use ("AMD Radeon RX 7900 XTX (Vulkan)"), or why they cannot use one. Creates the
+/// device on first use.
+pub fn adapter() -> Result<String, String> {
+ dev().map(|d| format!("{} ({:?})", d.info.name, d.info.backend))
+}
+
+#[repr(C)]
+#[derive(Clone, Copy, Pod, Zeroable)]
+struct Params {
+ m: u32,
+ n: u32,
+ k: u32,
+ cg: u32,
+ h: u32,
+ w: u32,
+ ow: u32,
+ kh: u32,
+ kw: u32,
+ sh: u32,
+ sw: u32,
+ pt: u32,
+ pl: u32,
+ bias: u32,
+ total: u32,
+ row: u32,
+ splits: u32,
+ kchunk: u32,
+ _a: u32,
+ _b: u32,
+}
+
+/// A layer's weights and bias on the device.
+struct Resident {
+ weights: wgpu::Buffer,
+ bias: wgpu::Buffer,
+}
+
+/// The GPU accelerator for one loaded face model: its weights stay on the device while it lives.
+pub struct FaceGpu {
+ dev: &'static Dev,
+ layers: Mutex<HashMap<usize, Arc<Resident>>>,
+ resident: AtomicU64,
+}
+
+/// A GPU accelerator for one face model, or why there is none (GPU off, missing or broken).
+pub fn accelerator() -> Result<Arc<FaceGpu>, String> {
+ Ok(Arc::new(FaceGpu { dev: dev()?, layers: Mutex::new(HashMap::new()), resident: AtomicU64::new(0) }))
+}
+
+fn u32_of(v: usize) -> Result<u32, String> {
+ u32::try_from(v).ok().filter(|v| *v < (1 << 31)).ok_or_else(|| "a layer too large for the GPU kernel".to_string())
+}
+
+impl FaceGpu {
+ /// Bytes of weights this model keeps on the device.
+ pub fn resident_bytes(&self) -> u64 {
+ self.resident.load(Ordering::Relaxed)
+ }
+
+ fn resident(&self, job: &ConvJob) -> Result<Option<Arc<Resident>>, String> {
+ let mut layers = self.layers.lock().unwrap_or_else(PoisonError::into_inner);
+ if let Some(r) = layers.get(&job.layer) {
+ return Ok(Some(r.clone()));
+ }
+ let bytes = (job.weights.len() as u64 + job.cout as u64) * 4;
+ if self.resident_bytes().saturating_add(bytes) > MAX_RESIDENT {
+ return Ok(None);
+ }
+ let d = self.dev;
+ let usage = wgpu::BufferUsages::STORAGE;
+ let weights = d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("faces weights"),
+ contents: bytemuck::cast_slice(job.weights),
+ usage,
+ });
+ let zero = [0f32; 1];
+ let bias = d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("faces bias"),
+ contents: bytemuck::cast_slice(job.bias.unwrap_or(&zero)),
+ usage,
+ });
+ let r = Arc::new(Resident { weights, bias });
+ self.resident.fetch_add(bytes, Ordering::Relaxed);
+ layers.insert(job.layer, r.clone());
+ Ok(Some(r))
+ }
+
+ fn run(&self, job: &ConvJob) -> Result<Option<Vec<f32>>, String> {
+ let d = self.dev;
+ let groups = job.groups.max(1);
+ let (m, cg) = (job.cout / groups, job.cin / groups);
+ let n = job.oh.checked_mul(job.ow).ok_or("a layer too large")?;
+ let k = cg.checked_mul(job.kh).and_then(|v| v.checked_mul(job.kw)).ok_or("a layer too large")?;
+ let total = job.cout.checked_mul(n).ok_or("a layer too large")?;
+ let direct = m < 8;
+ // Too few 64 x 64 tiles to keep the GPU busy (a 7 x 7 map of 512 channels makes 8): cut K into slices that
+ // run side by side, and sum them in a second pass.
+ let tiles = n.div_ceil(64) * m.div_ceil(64) * groups;
+ let splits = if direct { 1 } else { (TARGET_WORKGROUPS / tiles.max(1)).clamp(1, (k / 128).max(1)) };
+ let kchunk = k.div_ceil(splits).div_ceil(16) * 16;
+ let splits = k.div_ceil(kchunk.max(1)).max(1);
+ let partial = total.checked_mul(splits).ok_or("a layer too large")?;
+ let largest = [job.input.len(), job.weights.len(), partial].into_iter().max().unwrap_or(0) as u64 * 4;
+ if largest > d.limits.max_storage_buffer_binding_size.min(d.limits.max_buffer_size) {
+ return Ok(None);
+ }
+ let flat = |count: usize| {
+ let blocks = count.div_ceil(256);
+ let row = blocks.clamp(1, 65_535);
+ (row, blocks.div_ceil(row), 1, row)
+ };
+ let (wx, wy, wz, row) = if direct { flat(total) } else { (n.div_ceil(64), m.div_ceil(64), groups * splits, flat(total).3) };
+ let (rx, ry, _, _) = flat(total);
+ let limit = d.limits.max_compute_workgroups_per_dimension as usize;
+ if [wx, wy, wz, rx, ry].into_iter().any(|v| v > limit) {
+ return Ok(None);
+ }
+ let Some(res) = self.resident(job)? else { return Ok(None) };
+ let params = Params {
+ m: u32_of(m)?,
+ n: u32_of(n)?,
+ k: u32_of(k)?,
+ cg: u32_of(cg)?,
+ h: u32_of(job.h)?,
+ w: u32_of(job.w)?,
+ ow: u32_of(job.ow)?,
+ kh: u32_of(job.kh)?,
+ kw: u32_of(job.kw)?,
+ sh: u32_of(job.sh)?,
+ sw: u32_of(job.sw)?,
+ pt: u32_of(job.pt)?,
+ pl: u32_of(job.pl)?,
+ bias: u32::from(job.bias.is_some()),
+ total: u32_of(total)?,
+ row: u32_of(row)?,
+ splits: u32_of(splits)?,
+ kchunk: u32_of(kchunk)?,
+ _a: 0,
+ _b: 0,
+ };
+ let Some(_work) = crate::exit::enter() else { return Err("the app is closing".into()) };
+ let scopes = [d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), d.device.push_error_scope(wgpu::ErrorFilter::Validation)];
+ let uniform = d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("faces params"),
+ contents: bytemuck::bytes_of(&params),
+ usage: wgpu::BufferUsages::UNIFORM,
+ });
+ let input = d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("faces input"),
+ contents: bytemuck::cast_slice(job.input),
+ usage: wgpu::BufferUsages::STORAGE,
+ });
+ let bytes = total as u64 * 4;
+ let output = d.device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("faces output"),
+ size: bytes,
+ usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
+ mapped_at_creation: false,
+ });
+ let partials = (splits > 1).then(|| {
+ d.device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("faces partial sums"),
+ size: partial as u64 * 4,
+ usage: wgpu::BufferUsages::STORAGE,
+ mapped_at_creation: false,
+ })
+ });
+ let staging = d.device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("faces read-back"),
+ size: bytes,
+ usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
+ mapped_at_creation: false,
+ });
+ let bind = d.device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: None,
+ layout: &d.layout,
+ entries: &[
+ wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 1, resource: input.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 2, resource: res.weights.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 3, resource: res.bias.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 4, resource: partials.as_ref().unwrap_or(&output).as_entire_binding() },
+ ],
+ });
+ let summing = partials.as_ref().map(|partials| {
+ d.device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: None,
+ layout: &d.layout,
+ entries: &[
+ wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 1, resource: partials.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 2, resource: res.weights.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 3, resource: res.bias.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 4, resource: output.as_entire_binding() },
+ ],
+ })
+ });
+ let mut enc = d.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None });
+ {
+ let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("faces conv"), timestamp_writes: None });
+ pass.set_pipeline(if direct { &d.direct } else { &d.tiled });
+ pass.set_bind_group(0, &bind, &[]);
+ pass.dispatch_workgroups(u32_of(wx)?, u32_of(wy)?, u32_of(wz)?);
+ if let Some(summing) = &summing {
+ pass.set_pipeline(&d.reduce);
+ pass.set_bind_group(0, summing, &[]);
+ pass.dispatch_workgroups(u32_of(rx)?, u32_of(ry)?, 1);
+ }
+ }
+ enc.copy_buffer_to_buffer(&output, 0, &staging, 0, bytes);
+ let index = d.queue.submit([enc.finish()]);
+ let slice = staging.slice(..);
+ let (tx, rx) = std::sync::mpsc::channel();
+ slice.map_async(wgpu::MapMode::Read, move |r| {
+ let _ = tx.send(r);
+ });
+ let waited = d.device.poll(wgpu::PollType::Wait { submission_index: Some(index), timeout: Some(TIMEOUT) });
+ let [oom, validation] = scopes;
+ let (validation, oom) = (pollster::block_on(validation.pop()), pollster::block_on(oom.pop()));
+ if let Some(e) = oom {
+ return Err(format!("the GPU ran out of memory: {e}"));
+ }
+ if let Some(e) = validation {
+ mark_broken(&format!("device error: {e}"));
+ return Err(format!("the GPU reported an error: {e}"));
+ }
+ if let Err(e) = waited {
+ mark_broken(&format!("the GPU did not finish a convolution: {e}"));
+ return Err(format!("the GPU did not finish: {e}"));
+ }
+ match rx.recv_timeout(Duration::from_secs(10)) {
+ Ok(Ok(())) => {}
+ Ok(Err(e)) => return Err(format!("reading the result back failed: {e}")),
+ Err(e) => return Err(format!("reading the result back failed: {e}")),
+ }
+ let out = {
+ let data = slice.get_mapped_range().map_err(|e| format!("reading the result back failed: {e:?}"))?;
+ data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect::<Vec<f32>>()
+ };
+ staging.unmap();
+ Ok(Some(out))
+ }
+}
+
+impl Accelerator for FaceGpu {
+ fn device(&self) -> String {
+ format!("{} ({:?})", self.dev.info.name, self.dev.info.backend)
+ }
+
+ fn conv(&self, job: &ConvJob) -> Result<Option<Vec<f32>>, String> {
+ if let Some(why) = broken_reason() {
+ return Err(format!("the GPU is not used any more: {why}"));
+ }
+ if crate::switched_off().is_some() || job.macs() < MIN_MACS {
+ return Ok(None);
+ }
+ std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.run(job))).unwrap_or_else(|_| {
+ mark_broken("a face convolution panicked");
+ Err("the GPU runner gave up".into())
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn gpu() -> Option<Arc<FaceGpu>> {
+ match accelerator() {
+ Ok(a) => Some(a),
+ Err(e) => {
+ eprintln!("no GPU for the face kernels, skipping: {e}");
+ None
+ }
+ }
+ }
+
+ /// The plain definition of a convolution, to check the kernels against.
+ fn reference(j: &ConvJob) -> Vec<f32> {
+ let (m, cg) = (j.cout / j.groups, j.cin / j.groups);
+ let mut out = vec![0f32; j.cout * j.oh * j.ow];
+ for co in 0..j.cout {
+ let g = co / m;
+ for oy in 0..j.oh {
+ for ox in 0..j.ow {
+ let mut v = j.bias.map_or(0.0, |b| b[co]);
+ for ci in 0..cg {
+ for ky in 0..j.kh {
+ for kx in 0..j.kw {
+ let (iy, ix) = ((oy * j.sh + ky) as isize - j.pt as isize, (ox * j.sw + kx) as isize - j.pl as isize);
+ if iy >= 0 && ix >= 0 && (iy as usize) < j.h && (ix as usize) < j.w {
+ v += j.weights[((co * cg + ci) * j.kh + ky) * j.kw + kx]
+ * j.input[((g * cg + ci) * j.h + iy as usize) * j.w + ix as usize];
+ }
+ }
+ }
+ }
+ out[(co * j.oh + oy) * j.ow + ox] = v;
+ }
+ }
+ }
+ out
+ }
+
+ #[test]
+ fn the_gpu_convolution_gives_what_the_definition_gives() {
+ let Some(gpu) = gpu() else { return };
+ // (cin, cout, groups, k, stride, pad, h, w): dense, strided, depthwise, grouped, pointwise, odd sizes
+ for (layer, (cin, cout, groups, k, s, p, h, w)) in [
+ (3, 16, 1, 3, 2, 1, 41, 37),
+ (64, 64, 1, 3, 1, 1, 28, 28),
+ (32, 32, 32, 3, 1, 1, 30, 31),
+ (16, 32, 4, 3, 2, 1, 33, 20),
+ (48, 72, 1, 1, 1, 0, 50, 45),
+ (8, 130, 1, 5, 1, 2, 23, 19),
+ ]
+ .into_iter()
+ .enumerate()
+ {
+ let cg = cin / groups;
+ let weights: Vec<f32> = (0..cout * cg * k * k).map(|i| ((i * 37 % 23) as f32 - 11.0) * 0.03).collect();
+ let bias: Vec<f32> = (0..cout).map(|i| i as f32 * 0.01 - 0.3).collect();
+ let input: Vec<f32> = (0..cin * h * w).map(|i| ((i * 13 % 29) as f32 - 14.0) * 0.07).collect();
+ let (oh, ow) = ((h + 2 * p - k) / s + 1, (w + 2 * p - k) / s + 1);
+ let job = ConvJob {
+ layer,
+ cin,
+ h,
+ w,
+ cout,
+ groups,
+ kh: k,
+ kw: k,
+ sh: s,
+ sw: s,
+ pt: p,
+ pl: p,
+ oh,
+ ow,
+ weights: &weights,
+ bias: Some(&bias),
+ input: &input,
+ };
+ let got = gpu.run(&job).unwrap().unwrap();
+ let want = reference(&job);
+ let worst = got.iter().zip(&want).fold(0f32, |m, (a, b)| m.max((a - b).abs()));
+ assert!(worst < 1e-3, "case {layer}: off by {worst}");
+ }
+ assert!(gpu.resident_bytes() > 0, "the weights stay on the device");
+ }
+
+ /// The real models on the GPU agree with the CPU interpreter: `LC_FACE_MODELS=<folder>` holding
+ /// `face_detection_yunet_2023mar.onnx`, `face_recognition_sface_2021dec.onnx` and (optionally) `glintr100.onnx`.
+ #[test]
+ fn real_face_models_on_the_gpu_agree_with_the_cpu() {
+ let Some(dir) = std::env::var_os("LC_FACE_MODELS").map(std::path::PathBuf::from) else {
+ eprintln!("LC_FACE_MODELS is not set: skipping");
+ return;
+ };
+ if gpu().is_none() {
+ return;
+ }
+ let det = lightcraft_faces::yunet::Detector::new(&std::fs::read(dir.join("face_detection_yunet_2023mar.onnx")).unwrap()).unwrap();
+ let (w, h) = (900, 700);
+ let rgb: Vec<u8> = (0..w * h * 3).map(|i| ((i / 3 % w) as f32 * 0.25 + (i / 3 / w) as f32 * 0.3 + (i * 7919 % 41) as f32) as u8).collect();
+ let opts = lightcraft_faces::yunet::Options { score: 0.0, ..Default::default() };
+ let cpu = det.detect(&rgb, w, h, &opts).unwrap();
+ det.accelerate(accelerator().unwrap(), false).unwrap();
+ let fast = det.detect(&rgb, w, h, &opts).unwrap();
+ assert!(det.net().accelerator().is_some());
+ assert_eq!(cpu.len(), fast.len());
+ for (a, b) in cpu.iter().zip(&fast) {
+ assert!((a.score - b.score).abs() < 1e-3, "{} vs {}", a.score, b.score);
+ }
+ for (file, sha) in [
+ ("face_recognition_sface_2021dec.onnx", lightcraft_faces::known::SFACE_SHA256),
+ ("glintr100.onnx", lightcraft_faces::known::AURAFACE_SHA256),
+ ] {
+ let path = dir.join(file);
+ if !path.exists() {
+ continue;
+ }
+ let e = lightcraft_faces::runtime::Embedder::load(&path, &lightcraft_faces::known::lookup(sha).unwrap()).unwrap();
+ let (ew, eh) = e.input_size();
+ let face: Vec<u8> = (0..ew * eh * 3).map(|i| ((i * 13) % 251) as u8).collect();
+ let cpu = e.embed(&face).unwrap();
+ e.accelerate(accelerator().unwrap(), false).unwrap();
+ let fast = e.embed(&face).unwrap();
+ let cosine: f32 = cpu.iter().zip(&fast).map(|(a, b)| a * b).sum();
+ assert!(cosine > 0.9999, "{file}: cosine {cosine}");
+ }
+ }
+
+ /// Timings, CPU against GPU: `LC_FACE_MODELS=<folder> cargo test --release -p lightcraft-gpu faces::tests::timings -- --ignored --nocapture`.
+ #[test]
+ #[ignore = "a benchmark: needs LC_FACE_MODELS and a GPU"]
+ fn timings() {
+ let Some(dir) = std::env::var_os("LC_FACE_MODELS").map(std::path::PathBuf::from) else { return };
+ let Some(_) = gpu() else { return };
+ let time = |runs: usize, f: &mut dyn FnMut()| {
+ f();
+ let t = std::time::Instant::now();
+ for _ in 0..runs {
+ f();
+ }
+ t.elapsed().as_secs_f64() * 1000.0 / runs as f64
+ };
+ let det = lightcraft_faces::yunet::Detector::new(&std::fs::read(dir.join("face_detection_yunet_2023mar.onnx")).unwrap()).unwrap();
+ let (w, h) = (1024, 768);
+ let rgb: Vec<u8> = (0..w * h * 3).map(|i| (i * 7 % 251) as u8).collect();
+ let opts = lightcraft_faces::yunet::Options::default();
+ let cpu = time(5, &mut || drop(det.detect(&rgb, w, h, &opts).unwrap()));
+ det.accelerate(accelerator().unwrap(), false).unwrap();
+ let fast = time(5, &mut || drop(det.detect(&rgb, w, h, &opts).unwrap()));
+ eprintln!("YuNet 640: CPU {cpu:.1} ms, GPU {fast:.1} ms");
+ for (file, sha) in [
+ ("face_recognition_sface_2021dec.onnx", lightcraft_faces::known::SFACE_SHA256),
+ ("glintr100.onnx", lightcraft_faces::known::AURAFACE_SHA256),
+ ] {
+ let e = lightcraft_faces::runtime::Embedder::load(&dir.join(file), &lightcraft_faces::known::lookup(sha).unwrap()).unwrap();
+ let (ew, eh) = e.input_size();
+ let face: Vec<u8> = (0..ew * eh * 3).map(|i| (i * 13 % 251) as u8).collect();
+ let cpu = time(3, &mut || drop(e.embed(&face).unwrap()));
+ e.accelerate(accelerator().unwrap(), false).unwrap();
+ let fast = time(3, &mut || drop(e.embed(&face).unwrap()));
+ eprintln!("{file}: CPU {cpu:.1} ms, GPU {fast:.1} ms");
+ }
+ }
+}
diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs
index fb9785ba6..9c1de0f99 100644
--- a/crates/gpu/src/lib.rs
+++ b/crates/gpu/src/lib.rs
@@ -35,6 +35,8 @@ pub mod backend;
mod ctx;
#[cfg(not(target_arch = "wasm32"))]
mod exit;
+#[cfg(not(target_arch = "wasm32"))]
+pub mod faces;
#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))]
pub mod nn;
#[cfg(not(target_arch = "wasm32"))]
@@ -239,9 +241,9 @@ pub fn wait_idle(timeout: std::time::Duration) -> bool {
}
}
-/// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the
+/// Why GPU compute that has a device of its own (denoise, faces) is switched off by the user's settings, or `None`: the
/// environment switches and the rendering preference, but not a render-device failure (the devices are separate).
-#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))]
+#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn switched_off() -> Option<String> {
if env_disabled() {
return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());
diff --git a/crates/gpu/src/wgsl/faces_conv.wgsl b/crates/gpu/src/wgsl/faces_conv.wgsl
new file mode 100644
index 000000000..8384394e9
--- /dev/null
+++ b/crates/gpu/src/wgsl/faces_conv.wgsl
@@ -0,0 +1,171 @@
+// Convolution for the face networks (lightcraft-faces), planar (NCHW) float32, batch 1, any stride, padding and grouping.
+//
+// `tiled` is an implicit matrix product per group: rows are the group's output channels (M), columns its output
+// pixels (N), and K runs over input channel x kernel tap. The input is gathered (im2col) straight into workgroup
+// memory, so the lowered matrix is never made. 256 threads make a 64 x 64 block, 4 x 4 values each.
+// `direct` gives one output value per thread, for depthwise convolutions (one output channel per group), where a
+// 64-row tile would be almost all waste.
+//
+// out[co][p] = bias[co] + sum_k W[co][k] * X[k][p] X[k][p] = input[g*cg + k/taps][oy*sh + ky - pt][ox*sw + kx - pl]
+
+struct P {
+ m: u32, // output channels per group
+ n: u32, // output pixels (oh * ow)
+ k: u32, // cg * kh * kw
+ cg: u32, // input channels per group
+ h: u32,
+ w: u32,
+ ow: u32,
+ kh: u32,
+ kw: u32,
+ sh: u32,
+ sw: u32,
+ pt: u32,
+ pl: u32,
+ bias: u32, // 1 when there is a bias
+ total: u32, // cout * n
+ row: u32, // workgroups along x, for `direct` and `reduce` (so a large layer can use y too)
+ splits: u32, // K is cut into this many slices (z = group * splits + slice), summed by `reduce`
+ kchunk: u32, // K values per slice, a multiple of BK
+ _a: u32,
+ _b: u32,
+}
+
+@group(0) @binding(0) var<uniform> p: P;
+@group(0) @binding(1) var<storage, read> src: array<f32>;
+@group(0) @binding(2) var<storage, read> wgt: array<f32>;
+@group(0) @binding(3) var<storage, read> bias: array<f32>;
+@group(0) @binding(4) var<storage, read_write> dst: array<f32>;
+
+const BM: u32 = 64u;
+const BN: u32 = 64u;
+const BK: u32 = 16u;
+
+var<workgroup> As: array<f32, 1024>; // [BK][BM]
+var<workgroup> Bs: array<f32, 1024>; // [BK][BN]
+
+fn gather(g: u32, k: u32, pix: u32) -> f32 {
+ let taps = p.kh * p.kw;
+ let ci = g * p.cg + k / taps;
+ let tap = k % taps;
+ let oy = pix / p.ow;
+ let ox = pix % p.ow;
+ let iy = i32(oy * p.sh + tap / p.kw) - i32(p.pt);
+ let ix = i32(ox * p.sw + tap % p.kw) - i32(p.pl);
+ if (iy < 0 || ix < 0 || iy >= i32(p.h) || ix >= i32(p.w)) {
+ return 0.0;
+ }
+ return src[(ci * p.h + u32(iy)) * p.w + u32(ix)];
+}
+
+@compute @workgroup_size(256)
+fn tiled(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
+ let n0 = wg.x * BN;
+ let m0 = wg.y * BM;
+ let g = wg.z / p.splits;
+ let slice = wg.z % p.splits;
+ let k_lo = slice * p.kchunk;
+ let k_hi = min(p.k, k_lo + p.kchunk);
+ let tx = tid % 16u;
+ let ty = tid / 16u;
+ var acc: array<array<f32, 4>, 4>;
+ for (var i = 0u; i < 4u; i++) {
+ for (var j = 0u; j < 4u; j++) {
+ acc[i][j] = 0.0;
+ }
+ }
+ for (var k0 = k_lo; k0 < k_hi; k0 += BK) {
+ // 1024 values of each tile, 4 per thread; every slot has exactly one writer
+ for (var e = tid; e < 1024u; e += 256u) {
+ // weights: neighbouring threads read neighbouring k of one row
+ let mm = e / BK;
+ let ka = k0 + e % BK;
+ let m = m0 + mm;
+ var a = 0.0;
+ if (ka < k_hi && m < p.m) {
+ a = wgt[(g * p.m + m) * p.k + ka];
+ }
+ As[(e % BK) * BM + mm] = a;
+ // input: neighbouring threads read neighbouring pixels
+ let kk = e / BN;
+ let k = k0 + kk;
+ let pix = n0 + e % BN;
+ var b = 0.0;
+ if (k < k_hi && pix < p.n) {
+ b = gather(g, k, pix);
+ }
+ Bs[e] = b;
+ }
+ workgroupBarrier();
+ for (var kk = 0u; kk < BK; kk++) {
+ var a: array<f32, 4>;
+ var b: array<f32, 4>;
+ for (var i = 0u; i < 4u; i++) {
+ a[i] = As[kk * BM + ty * 4u + i];
+ b[i] = Bs[kk * BN + tx + 16u * i];
+ }
+ for (var i = 0u; i < 4u; i++) {
+ for (var j = 0u; j < 4u; j++) {
+ acc[i][j] += a[i] * b[j];
+ }
+ }
+ }
+ workgroupBarrier();
+ }
+ for (var i = 0u; i < 4u; i++) {
+ let m = m0 + ty * 4u + i;
+ if (m >= p.m) {
+ continue;
+ }
+ let co = g * p.m + m;
+ var b0 = 0.0;
+ if (p.bias != 0u && p.splits == 1u) {
+ b0 = bias[co];
+ }
+ // with slices, each writes a partial sum of its own: [slice][cout][n]
+ let base = slice * p.total + co * p.n;
+ for (var j = 0u; j < 4u; j++) {
+ // columns tx, tx + 16, tx + 32, tx + 48: neighbouring threads write neighbouring pixels
+ let pix = n0 + tx + 16u * j;
+ if (pix < p.n) {
+ dst[base + pix] = acc[i][j] + b0;
+ }
+ }
+ }
+}
+
+@compute @workgroup_size(256)
+fn direct(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
+ let at = (wg.y * p.row + wg.x) * 256u + tid;
+ if (at >= p.total) {
+ return;
+ }
+ let co = at / p.n;
+ let pix = at % p.n;
+ let g = co / p.m;
+ var v = 0.0;
+ if (p.bias != 0u) {
+ v = bias[co];
+ }
+ for (var k = 0u; k < p.k; k++) {
+ v += wgt[co * p.k + k] * gather(g, k, pix);
+ }
+ dst[at] = v;
+}
+
+// The sum of the K slices' partial results, plus the bias. `src` holds [splits][cout][n].
+@compute @workgroup_size(256)
+fn reduce(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
+ let at = (wg.y * p.row + wg.x) * 256u + tid;
+ if (at >= p.total) {
+ return;
+ }
+ var v = 0.0;
+ if (p.bias != 0u) {
+ v = bias[at / p.n];
+ }
+ for (var s = 0u; s < p.splits; s++) {
+ v += src[s * p.total + at];
+ }
+ dst[at] = v;
+}
diff --git a/docs/faces.md b/docs/faces.md
index 5c8e54317..f4f4a4757 100644
--- a/docs/faces.md
+++ b/docs/faces.md
@@ -143,6 +143,34 @@ SFace's builtin speed metadata now uses the measured 6.9× ratio. User `catalog.
Older published tables for MobileFaceNet/R50/R100 came from a different runtime and are not measurements of this runner; architecture
names alone do not guarantee the same speed, and those additional model files were not verified in this task.
+### On the GPU
+
+On a machine with a usable GPU, the convolutions of the detector and the recogniser (most of their time) run on it
+through wgpu (`crates/gpu/src/faces.rs`, `wgsl/faces_conv.wgsl`: an implicit matrix product that gathers the input
+straight into workgroup memory, cuts K into slices when a layer's map is small, and a direct kernel for depthwise
+layers). Everything else in the graph stays on the CPU interpreter, which remains the reference. After a model loads,
+a background check runs it once on the CPU and twice through the GPU on a fixed test input: the GPU is used only when
+every output is within 1e-3 of the CPU's (relative to the output's largest value) **and** it was faster; until then,
+and whenever the GPU is switched off (`LIGHTCRAFT_GPU=0`, `LIGHTCRAFT_GPU_BACKEND=off`, the GPU preference), missing,
+wrong or slower, the model runs on the CPU. The first GPU error drops it for that model. The weights stay on a device of
+the scan's own (separate from the editor's) while the model is loaded, at most 768 MB per model. The log says which
+device each model runs on.
+
+Measured on Linux with a Radeon (Vulkan) and a 32-thread CPU, one call at a time, machine under heavy load from other
+jobs (so the absolute numbers are noisy; the ratios held across three runs):
+
+| Model | CPU | GPU |
+| --- | ---: | ---: |
+| YuNet (640², one photo) | 33–34 ms | 35–39 ms |
+| SFace 2021dec | 25–28 ms | 22–27 ms |
+| AuraFace v1 | 204–208 ms | 81–103 ms |
+
+Each layer still goes to the GPU and back: for YuNet and SFace that round trip costs about what the layer saves, so on
+most machines they stay on the CPU by the speed check, while AuraFace runs 2–2.5× faster. Keeping activations on the GPU
+between layers (the remaining operators as kernels) and batching faces per dispatch are the next steps.
+`LC_FACE_MODELS=<folder> cargo test --release -p lightcraft-gpu faces::tests -- --include-ignored --nocapture` checks
+agreement on the real models and prints these timings.
+
### Agreement and regression coverage
The public-crop embeddings match the reference runtime's outputs: