Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

M12: AI masks never require SAM 3 — background worker, consented download, review fixes

Brandon Thomasd379e780cc75
diff --git a/NOTICE b/NOTICE
index 1fcf11e74..f3f6499f6 100644
--- a/NOTICE
+++ b/NOTICE
@@ -15,11 +15,17 @@ list (author, source, license) is in assets/ATTRIBUTION.md.
Inter (assets/fonts/Inter-*.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-Inter.txt.
Showcase photographs by Ansel Adams, Dorothea Lange and NASA (docs/images/) — public domain. See assets/ATTRIBUTION.md.
-crates/segment ports the SAM 3 model code of Hugging Face Transformers
+crates/segment (lightcraft-segment) ports the SAM 3 model code of Hugging Face Transformers
(https://github.com/huggingface/transformers, src/transformers/models/sam3 and sam3_tracker),
Copyright The HuggingFace Team and Meta Platforms, Inc., licensed under the Apache License 2.0.
-The SAM 3 weights are not part of LightCraft: users download the facebook/sam3 checkpoint
-themselves, under Meta's SAM License.
+The port was modified by the LightCraft contributors (2026): translated from Python/PyTorch to
+Rust on candle and restructured; each ported file says so. Because of this, crates/segment is
+licensed under the Apache License 2.0 only (LICENSE-APACHE), not under the MIT option.
+
+The SAM 3 weights are not part of LightCraft and are never distributed with it. A user downloads
+the facebook/sam3 checkpoint only when they choose to (in the app, or by hand), under Meta's SAM
+License (https://github.com/facebookresearch/sam3/blob/main/LICENSE), a separate, non-OSI
+licence. See docs/ai-masks.md.
Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases)
also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md
diff --git a/ROADMAP.md b/ROADMAP.md
index fdfbec0cb..f2dcb9bc5 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -39,7 +39,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
| Nikon / Sony / older-Canon raw shooter | ~65% | Camera colour fidelity and coverage (ARW and NEF preview estimates are only a starting point) |
| Canon CR3 / Fujifilm / Olympus shooter | ~35% | Their raws open as embedded previews only (CR3: the full-size JPEG with full metadata) |
| Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering |
-| Relies on AI (masks, denoise) | ~25% | No segmentation or AI denoise models |
+| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise |
## Where we're going
@@ -57,7 +57,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par
(references stay in the local `plan/`), then tune against the numbers.
5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1).
6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for
- segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance.
+ segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Object / Describe masks now
+ run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still
+ to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open.
7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation
and accessibility.
diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs
index 7617cba12..e46fd2ee2 100644
--- a/apps/lightcraft/src/main.rs
+++ b/apps/lightcraft/src/main.rs
@@ -406,7 +406,9 @@ OPTIONS:
ENVIRONMENT:
LIGHTCRAFT_GPU_BACKEND=dx12|vulkan|metal|auto|off graphics backend (default: DX12 on Windows, Metal on macOS,
Vulkan on Linux; off = render on the CPU); else WGPU_BACKEND. LIGHTCRAFT_GPU=0: CPU rendering.
- LIGHTCRAFT_SAM3_DIR=DIR the SAM 3 model for Object / Describe masks (default: <settings folder>/models/sam3)
+ LIGHTCRAFT_SAM3_DIR=DIR the SAM 3 model for Object / Describe masks (default: <settings folder>/models/sam3;
+ optional: LightCraft offers to download it when first needed)
+ LIGHTCRAFT_SAM3_MIRRORS=URL,… where to download the SAM 3 model from (base URLs, tried in order)
";
/// Warnings and errors (failed commands, AI mask analysis) on stderr; `LIGHTCRAFT_LOG=info`
@@ -504,8 +506,11 @@ fn main() -> eframe::Result {
Box::new(move |cc| {
let (mut session, problem) = open_session(in_memory, library_dir, seed_demo && files.is_empty());
// AI masks: the SAM 3 checkpoint (facebook/sam3) in <config>/models/sam3, or LIGHTCRAFT_SAM3_DIR
+ // (never required: without it, AI masks offer to download it; see docs/ai-masks.md)
session.segmenter.dir =
std::env::var_os("LIGHTCRAFT_SAM3_DIR").map(std::path::PathBuf::from).or_else(|| config_dir().map(|d| d.join("models").join("sam3")));
+ // the user's own download locations, one base URL per line (LIGHTCRAFT_SAM3_MIRRORS too)
+ session.segmenter.mirrors_file = config_dir().map(|d| d.join("models").join("sam3-mirrors.txt"));
let mut app = LightcraftApp::new(session, services());
if let Some(ui) = prefs {
app.ui = ui;
diff --git a/crates/develop/src/segmask.rs b/crates/develop/src/segmask.rs
index 724ebdf56..172d53fce 100644
--- a/crates/develop/src/segmask.rs
+++ b/crates/develop/src/segmask.rs
@@ -55,6 +55,34 @@ impl SegMask {
}
}
+/// Most zoomed-in detail patches one AI mask component keeps (the app makes one; each costs a
+/// decode per render, so a hostile document can't make rendering explode).
+pub const MAX_DETAIL: usize = 4;
+
+/// Deserialize a component's detail patches, keeping at most [`MAX_DETAIL`] (the rest are
+/// skipped without being decoded).
+pub(crate) fn de_detail<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Vec<SegMask>, D::Error> {
+ struct Capped;
+ impl<'de> serde::de::Visitor<'de> for Capped {
+ type Value = Vec<SegMask>;
+ fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ f.write_str("a list of segmentations")
+ }
+ fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<SegMask>, A::Error> {
+ let mut out = Vec::new();
+ while out.len() < MAX_DETAIL {
+ match seq.next_element::<SegMask>()? {
+ Some(m) => out.push(m),
+ None => return Ok(out),
+ }
+ }
+ while seq.next_element::<serde::de::IgnoredAny>()?.is_some() {}
+ Ok(out)
+ }
+ }
+ d.deserialize_seq(Capped)
+}
+
const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
fn base64_encode(bytes: &[u8]) -> String {
@@ -118,6 +146,30 @@ mod tests {
}
}
+ #[test]
+ fn a_hostile_document_keeps_at_most_max_detail_patches() {
+ let patch = serde_json::to_value(SegMask::from_logits(4, &[1.0; 16])).unwrap();
+ let shape = serde_json::json!({"kind": "object", "hint": [{"x": 0.5, "y": 0.5}], "detail": vec![patch; 10_000]});
+ let s: crate::MaskShape = serde_json::from_value(shape).unwrap();
+ let crate::MaskShape::Object { detail, .. } = s else { panic!("an Object") };
+ assert_eq!(detail.len(), MAX_DETAIL);
+ // and a whole develop document with them still loads
+ let mut d = crate::DevelopSettings::default();
+ d.masks.push(crate::Mask {
+ id: 1,
+ components: vec![crate::MaskComponent {
+ name: None,
+ op: crate::MaskOp::Add,
+ invert: false,
+ shape: crate::MaskShape::Prompt { text: "x".into(), seg: None, detail: vec![SegMask::from_logits(4, &[1.0; 16]); 50], edge: 0.0 },
+ }],
+ ..Default::default()
+ });
+ let back: crate::DevelopSettings = serde_json::from_str(&serde_json::to_string(&d).unwrap()).unwrap();
+ let crate::MaskShape::Prompt { detail, .. } = &back.masks[0].components[0].shape else { panic!("a Prompt") };
+ assert_eq!(detail.len(), MAX_DETAIL);
+ }
+
#[test]
fn damaged_data_is_none() {
let mut m = SegMask::from_logits(8, &[1.0; 64]);
diff --git a/crates/develop/src/settings.rs b/crates/develop/src/settings.rs
index 736586fe9..800f229b5 100644
--- a/crates/develop/src/settings.rs
+++ b/crates/develop/src/settings.rs
@@ -782,7 +782,7 @@ pub enum MaskShape {
seg: Option<crate::SegMask>,
/// Zoomed-in passes over parts of the image (higher resolution than `seg`), used
/// inside their rectangles.
- #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ #[serde(default, skip_serializing_if = "Vec::is_empty", deserialize_with = "crate::segmask::de_detail")]
detail: Vec<crate::SegMask>,
/// Edge −100..100: below 0 harder (a steeper transition), above 0 softer (feathered).
#[serde(default, skip_serializing_if = "is_zero")]
@@ -794,7 +794,7 @@ pub enum MaskShape {
text: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
seg: Option<crate::SegMask>,
- #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ #[serde(default, skip_serializing_if = "Vec::is_empty", deserialize_with = "crate::segmask::de_detail")]
detail: Vec<crate::SegMask>,
/// Edge −100..100, as for `Object`.
#[serde(default, skip_serializing_if = "is_zero")]
diff --git a/crates/engine/src/cmd/masks.rs b/crates/engine/src/cmd/masks.rs
index da96f967b..1615fb06b 100644
--- a/crates/engine/src/cmd/masks.rs
+++ b/crates/engine/src/cmd/masks.rs
@@ -5,7 +5,7 @@ use lightcraft_geom::Point;
use serde_json::{Value, json};
use super::{CommandSpec, bad, bool_or, cmd, f64_or, has_active, ok, point, str_param};
-use crate::segment::Click;
+use crate::segment::{Click, MAX_CLICKS};
use crate::{Result, Session};
fn shape_from(kind: &str, p: &Value, c: &str) -> Result<MaskShape> {
@@ -39,8 +39,10 @@ fn shape_from(kind: &str, p: &Value, c: &str) -> Result<MaskShape> {
.unwrap_or_default();
MaskShape::ColorRange { samples, refine: f64_or(p, "refine", 50.0) }
}
- "object" => MaskShape::Object { hint: points(p, "points"), exclude: points(p, "exclude"), seg: None, detail: vec![], edge: 0.0 },
- "prompt" => MaskShape::Prompt { text: str_param(p, "text").unwrap_or_default().trim().to_string(), seg: None, detail: vec![], edge: 0.0 },
+ "object" => MaskShape::Object { hint: points(p, "points"), exclude: points(p, "exclude"), seg: seg_param(p), detail: vec![], edge: 0.0 },
+ "prompt" => {
+ MaskShape::Prompt { text: str_param(p, "text").unwrap_or_default().trim().to_string(), seg: seg_param(p), detail: vec![], edge: 0.0 }
+ }
other => {
return Err(bad(
c,
@@ -58,19 +60,50 @@ fn points(p: &Value, key: &str) -> Vec<Point> {
.unwrap_or_default()
}
-/// Compute the segmentation of a new AI shape (Object with clicks, Prompt): an Object
-/// without clicks stays empty until the photo is clicked; a Prompt that matches nothing is
-/// an error (nothing to select).
-fn resolve_ai(s: &mut Session, shape: &mut MaskShape, c: &str) -> Result<()> {
+/// A segmentation passed in (`seg`: a stored or replayed AI mask; the model isn't needed).
+fn seg_param(p: &Value) -> Option<lightcraft_develop::SegMask> {
+ p.get("seg").and_then(|v| serde_json::from_value(v.clone()).ok())
+}
+
+/// What a new AI shape still needs once it is added.
+enum Later {
+ Nothing,
+ /// Background mode: these Object clicks are segmented on the worker.
+ Clicks(Vec<Point>, Vec<Point>),
+ /// Background mode: the Describe selection is computed first; the mask is added when found.
+ Text(String),
+}
+
+/// Compute the segmentation of a new AI shape (Object with clicks, Prompt) — or, in background
+/// mode, say what to queue. An Object without clicks stays empty until the photo is clicked
+/// (the photo is analyzed meanwhile); a Prompt that matches nothing is an error. Shapes given
+/// with their `seg` need no model.
+fn resolve_ai(s: &mut Session, shape: &mut MaskShape, c: &str) -> Result<Later> {
let id = s.active().ok_or_else(|| bad(c, "no active photo"))?;
+ let background = s.segmenter.background;
match shape {
+ MaskShape::Object { seg: Some(_), .. } | MaskShape::Prompt { seg: Some(_), .. } => {}
+ MaskShape::Object { hint, exclude, .. } if !hint.is_empty() && background => {
+ s.segmenter.model_dir().map_err(ai_err)?;
+ return Ok(Later::Clicks(std::mem::take(hint), std::mem::take(exclude)));
+ }
MaskShape::Object { hint, exclude, seg, .. } if !hint.is_empty() => {
*seg = Some(s.segment_clicks(id, &clicks_of(hint, exclude)).map_err(ai_err)?);
}
MaskShape::Object { .. } => {
- // the clicks come later: start analyzing the photo now, and refuse an Object mask that
- // could never be computed (no model, missing file) rather than leave an empty one
- s.segment_prepare(id).map_err(ai_err)?;
+ // the clicks come later: refuse an Object mask that could never be computed (no
+ // model) rather than leave an empty one; in the app, analyze the photo meanwhile
+ s.segmenter.model_dir().map_err(ai_err)?;
+ if background {
+ s.segment_prepare(id).map_err(ai_err)?;
+ }
+ }
+ MaskShape::Prompt { text, .. } if background => {
+ s.segmenter.model_dir().map_err(ai_err)?;
+ if text.trim().is_empty() {
+ return Err(ai_err("describe what to select"));
+ }
+ return Ok(Later::Text(text.clone()));
}
MaskShape::Prompt { text, seg, .. } => {
let found = s.segment_text(id, text).map_err(ai_err)?;
@@ -78,7 +111,17 @@ fn resolve_ai(s: &mut Session, shape: &mut MaskShape, c: &str) -> Result<()> {
}
_ => {}
}
- Ok(())
+ Ok(Later::Nothing)
+}
+
+/// Queue what a new AI component (component `comp` of mask `mask`) still needs.
+fn queue_later(s: &mut Session, later: Later, mask: u32, comp: usize, mut out: Value) -> Result<Value> {
+ if let Later::Clicks(hint, exclude) = later {
+ let id = s.active().ok_or_else(|| ai_err("no active photo"))?;
+ s.segment_clicks_later(id, mask, comp, hint, exclude).map_err(ai_err)?;
+ out["pending"] = json!(true);
+ }
+ Ok(out)
}
/// AI mask failures are shown as they are (not as "invalid parameters").
@@ -86,7 +129,7 @@ fn ai_err(msg: impl Into<String>) -> crate::EngineError {
crate::EngineError::Other(msg.into())
}
-fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec<Click> {
+pub(crate) fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec<Click> {
include.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect()
}
@@ -115,15 +158,20 @@ pub fn specs() -> Vec<CommandSpec> {
"Create New Mask",
[],
None,
- "{kind: brush|linear|radial|sky|subject|background|luminanceRange|colorRange|object|prompt, ...shape params (start/end, center/rx/ry/angle/feather, lo/hi…; object: points/exclude [[x,y],…]; prompt: text), name?}",
+ "{kind: brush|linear|radial|sky|subject|background|luminanceRange|colorRange|object|prompt, ...shape params (start/end, center/rx/ry/angle/feather, lo/hi…; object: points/exclude [[x,y],…]; prompt: text; object/prompt: seg? a stored segmentation, no model needed), name?} — AI kinds need the SAM 3 model; in the app a prompt returns {pending} and the mask appears when found",
has_active,
|s, p| {
let kind = str_param(p, "kind").unwrap_or("radial").to_string();
let mut shape = shape_from(&kind, p, "mask.add")?;
- resolve_ai(s, &mut shape, "mask.add")?;
+ let later = resolve_ai(s, &mut shape, "mask.add")?;
let name = str_param(p, "name").map(str::to_string);
+ if let Later::Text(text) = &later {
+ let id = s.active().ok_or_else(|| bad("mask.add", "no active photo"))?;
+ s.segment_text_later(id, None, "add", name, text).map_err(ai_err)?;
+ return Ok(json!({"pending": true}));
+ }
let next = s.active().and_then(|id| s.develop_of(id)).map(|d| d.next_mask_id()).unwrap_or(1);
- masks_edit(s, "mask.add", "Add Mask", |masks, active| {
+ let out = masks_edit(s, "mask.add", "Add Mask", |masks, active| {
masks.push(Mask {
id: next,
name: name.unwrap_or_else(|| format!("Mask {next}")),
@@ -132,7 +180,8 @@ pub fn specs() -> Vec<CommandSpec> {
});
*active = Some(next);
Ok(())
- })
+ })?;
+ queue_later(s, later, next, 0, out)
}
),
cmd!(
@@ -144,17 +193,29 @@ pub fn specs() -> Vec<CommandSpec> {
has_active,
|s, p| {
let kind = str_param(p, "kind").unwrap_or("brush").to_string();
- let mut shape = shape_from(&kind, p, "mask.addComponent")?;
- resolve_ai(s, &mut shape, "mask.addComponent")?;
let op: MaskOp =
serde_json::from_value(p.get("op").cloned().unwrap_or(json!("add"))).map_err(|e| bad("mask.addComponent", e.to_string()))?;
let active = s.active_mask;
let mid = mask_id(p, active, "mask.addComponent")?;
- masks_edit(s, "mask.addComponent", "Edit Mask", |masks, _| {
+ let id = s.active().ok_or_else(|| bad("mask.addComponent", "no active photo"))?;
+ if !s.develop_of(id).is_some_and(|d| d.masks.iter().any(|m| m.id == mid)) {
+ return Err(bad("mask.addComponent", format!("no mask {mid}")));
+ }
+ let mut shape = shape_from(&kind, p, "mask.addComponent")?;
+ let later = resolve_ai(s, &mut shape, "mask.addComponent")?;
+ if let Later::Text(text) = &later {
+ let op = p.get("op").and_then(Value::as_str).unwrap_or("add");
+ s.segment_text_later(id, Some(mid), op, None, text).map_err(ai_err)?;
+ return Ok(json!({"pending": true}));
+ }
+ let mut comp = 0;
+ let out = masks_edit(s, "mask.addComponent", "Edit Mask", |masks, _| {
let i = find(masks, mid, "mask.addComponent")?;
masks[i].components.push(MaskComponent { name: None, op, invert: bool_or(p, "invert", false), shape });
+ comp = masks[i].components.len() - 1;
Ok(())
- })
+ })?;
+ queue_later(s, later, mid, comp, out)
}
),
cmd!(
@@ -290,11 +351,23 @@ pub fn specs() -> Vec<CommandSpec> {
return Err(bad(c, "not an Object component"));
};
let (mut hint, mut ex, edge) = (hint.clone(), ex.clone(), *edge);
+ // clicks still on their way to the model count too
+ if let Some(pending) = s.segmenter.pending_clicks().filter(|q| (q.photo, q.mask, q.comp) == (id, mid, k)) {
+ (hint, ex) = (pending.hint.clone(), pending.exclude.clone());
+ }
+ if hint.len() + ex.len() >= MAX_CLICKS {
+ return Err(ai_err(format!("an Object selection takes at most {MAX_CLICKS} clicks; start a new one")));
+ }
if exclude {
ex.push(at)
} else {
hint.push(at)
}
+ if s.segmenter.background && !hint.is_empty() {
+ let out = json!({"include": hint.len(), "exclude": ex.len(), "pending": true});
+ s.segment_clicks_later(id, mid, k, hint, ex).map_err(ai_err)?;
+ return Ok(out);
+ }
let seg = if hint.is_empty() { None } else { Some(s.segment_clicks(id, &clicks_of(&hint, &ex)).map_err(ai_err)?) };
let out = json!({"include": hint.len(), "exclude": ex.len()});
masks_edit(s, c, "Object Mask", |masks, _| {
@@ -346,6 +419,63 @@ pub fn specs() -> Vec<CommandSpec> {
Ok(json!({"busy": s.segmenter.busy()}))
}
),
+ cmd!(
+ query "segment.model.status",
+ "AI Mask Model Status",
+ [],
+ None,
+ "{} — whether this build has AI masks, whether the SAM 3 model is installed (and where), loaded, busy, and the download's progress → {available, installed, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download: {running, done, total, file, error, finished}}",
+ super::always,
+ |s, _| {
+ let g = &s.segmenter;
+ Ok(json!({
+ "available": crate::segment::Segmenter::AVAILABLE,
+ "installed": g.installed(),
+ "dir": g.dir.as_ref().map(|d| d.display().to_string()),
+ "loaded": g.loaded(),
+ "busy": g.busy(),
+ "analyzing": g.analyzing(),
+ "sizeBytes": crate::segment::MODEL_BYTES,
+ "license": crate::segment::LICENSE_NAME,
+ "licenseUrl": crate::segment::LICENSE_URL,
+ "mirrors": g.mirrors().len(),
+ "download": g.download_status(),
+ }))
+ }
+ ),
+ cmd!(
+ query "segment.model.download",
+ "Download AI Mask Model",
+ [],
+ None,
+ "{acknowledged: true} — download the SAM 3 model (about 3.4 GB, Meta's SAM License, not LightCraft's) in the background, from the configured mirrors; only after the user agreed to it. Watch segment.model.status; segment.model.cancel stops it (it resumes later) → {started, installed, downloading}",
+ super::always,
+ |s, p| {
+ let c = "segment.model.download";
+ if p.get("acknowledged").and_then(Value::as_bool) != Some(true) {
+ return Err(bad(
+ c,
+ format!(
+ "the SAM 3 model is a {:.1} GB download under Meta's {} ({}): pass `acknowledged: true` once the user has agreed to download it",
+ crate::segment::MODEL_BYTES as f64 / 1e9,
+ crate::segment::LICENSE_NAME,
+ crate::segment::LICENSE_URL
+ ),
+ ));
+ }
+ let started = s.segmenter.start_download().map_err(ai_err)?;
+ Ok(json!({"started": started, "installed": s.segmenter.installed(), "downloading": s.segmenter.download_status().running}))
+ }
+ ),
+ cmd!(
+ query "segment.model.cancel",
+ "Cancel AI Mask Model Download",
+ [],
+ None,
+ "{} — stop the SAM 3 download (what has arrived is kept, and a new download resumes from it) → {cancelled}",
+ super::always,
+ |s, _| Ok(json!({"cancelled": s.segmenter.cancel_download()}))
+ ),
cmd!(
"mask.refine",
"Refine Mask Edges",
diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs
index 5edcad679..ec39222a4 100644
--- a/crates/engine/src/lib.rs
+++ b/crates/engine/src/lib.rs
@@ -727,6 +727,8 @@ mod tests_persist;
#[cfg(test)]
mod tests_prefs;
#[cfg(test)]
+mod tests_segment;
+#[cfg(test)]
mod tests_settings_files;
#[cfg(test)]
mod tests_spots;
diff --git a/crates/engine/src/segment.rs b/crates/engine/src/segment.rs
deleted file mode 100644
index 41aed09e9..000000000
--- a/crates/engine/src/segment.rs
+++ /dev/null
@@ -1,588 +0,0 @@
-//! AI masks: Object (clicks) and Describe (text) selections computed with SAM 3
-//! (`lightcraft-segment`, cargo feature `sam`; the desktop app enables it).
-//!
-//! The model sees the photo as developed, uncropped and without masks, at 1008 px — the frame
-//! mask coordinates live in — so a stored segmentation lines up with the photo at any size.
-//! Encoding the image is the slow part (a few seconds on a laptop GPU); it runs once per photo
-//! and look, on a worker thread when [`Session::segment_prepare`] is called (the Masking panel
-//! does this when an AI mask is started), and each click then takes milliseconds.
-//!
-//! Without the feature (web, CLI) every request reports that this build has no model; stored
-//! segmentations still render (they are plain data in the develop settings).
-
-use std::path::PathBuf;
-
-use lightcraft_catalog::PhotoId;
-use lightcraft_develop::{MaskShape, SegMask};
-use lightcraft_geom::Point;
-
-use crate::Session;
-
-/// Long edge of the image the model sees.
-pub const INPUT_EDGE: usize = 1008;
-
-/// One click of an Object selection (normalized image coordinates).
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub struct Click {
- pub at: Point,
- pub include: bool,
-}
-
-/// What a request runs on: the photo, its look (settings without masks), the model input.
-#[cfg_attr(not(feature = "sam"), allow(dead_code))]
-struct Input {
- key: u64,
- rgb: Vec<u8>,
- w: usize,
- h: usize,
-}
-
-#[cfg(feature = "sam")]
-mod model {
- use std::sync::atomic::{AtomicBool, Ordering};
- use std::sync::{Arc, Mutex, PoisonError};
-
- use lightcraft_segment::{Encoded, Sam3};
-
- #[derive(Default)]
- pub struct Inner {
- pub model: Option<Sam3>,
- /// The last encoded image and its key.
- pub cache: Option<(u64, Encoded)>,
- }
-
- #[derive(Clone, Default)]
- pub struct Shared {
- pub inner: Arc<Mutex<Inner>>,
- pub busy: Arc<AtomicBool>,
- /// A zoomed-in detail pass is running.
- pub detail_busy: Arc<AtomicBool>,
- /// Finished detail passes, waiting for [`crate::Session::segment_poll`].
- pub results: Arc<Mutex<Vec<super::DetailResult>>>,
- }
-
- impl Shared {
- pub fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
- self.inner.lock().unwrap_or_else(PoisonError::into_inner)
- }
-
- pub fn busy(&self) -> bool {
- self.busy.load(Ordering::Relaxed)
- }
-
- pub fn set_busy(&self, v: bool) {
- self.busy.store(v, Ordering::Relaxed);
- }
- }
-}
-
-/// The model (loaded on first use) and the last encoded photo.
-#[derive(Default)]
-pub struct Segmenter {
- /// Where the `facebook/sam3` checkpoint lives (set by the app).
- pub dir: Option<PathBuf>,
- #[cfg(feature = "sam")]
- shared: model::Shared,
-}
-
-impl Segmenter {
- /// Whether this build can compute AI masks at all.
- pub const AVAILABLE: bool = cfg!(feature = "sam");
-
- /// Whether a worker is preparing a photo right now.
- pub fn busy(&self) -> bool {
- #[cfg(feature = "sam")]
- return self.shared.busy();
- #[cfg(not(feature = "sam"))]
- false
- }
-
- /// Whether a zoomed-in detail pass is running.
- pub fn detail_busy(&self) -> bool {
- #[cfg(feature = "sam")]
- return self.shared.detail_busy.load(std::sync::atomic::Ordering::Relaxed);
- #[cfg(not(feature = "sam"))]
- false
- }
-
- /// The model directory, if its files are there.
- pub fn model_dir(&self) -> Result<PathBuf, String> {
- if !Self::AVAILABLE {
- return Err("AI masks are not available in this build".into());
- }
- let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?;
- #[cfg(feature = "sam")]
- if !lightcraft_segment::is_model_dir(&dir) {
- return Err(format!(
- "the SAM 3 model is not installed: put the facebook/sam3 files (model.safetensors, vocab.json, merges.txt) in {}",
- dir.display()
- ));
- }
- Ok(dir)
- }
-}
-
-/// Mix `v` into a 64-bit hash.
-fn mix(h: u64, v: u64) -> u64 {
- (h ^ v).wrapping_mul(0x0100_0000_01b3).rotate_left(29)
-}
-
-/// A finished zoomed-in pass: for which component (and the shape it was computed from, so a
-/// component changed meanwhile is left alone), and the patch (`None`: nothing found there).
-#[cfg_attr(not(feature = "sam"), allow(dead_code))]
-pub struct DetailResult {
- photo: PhotoId,
- mask: u32,
- comp: usize,
- shape: MaskShape,
- patch: Option<SegMask>,
-}
-
-/// What a detail pass re-runs.
-#[cfg_attr(not(feature = "sam"), allow(dead_code))]
-enum Prompt {
- Clicks(Vec<Click>),
- Text(String),
-}
-
-/// Objects covering more than this much of the photo gain little from a zoomed pass.
-const DETAIL_MAX_AREA: f64 = 0.4;
-
-/// The part of the photo to zoom into for `seg`'s selection: its bounding box with a margin of
-/// a quarter of its size, at least 12 % of the long edge, at most 2:1, inside the photo
-/// (`aspect` = width / height). `None` when nothing is selected or it is most of the photo.
-pub fn detail_region(seg: &SegMask, aspect: f64) -> Option<[f64; 4]> {
- let logits = seg.logits()?;
- let r = seg.bounds()?;
- let side = seg.side as usize;
- let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0, 0);
- for (i, l) in logits.iter().enumerate() {
- if *l > 0.0 {
- let (x, y) = (i % side, i / side);
- (x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x + 1), y1.max(y + 1));
- }
- }
- if x0 == usize::MAX {
- return None;
- }
- let (rw, rh) = (r[2] - r[0], r[3] - r[1]);
- let nx = |c: usize| r[0] + c as f64 / side as f64 * rw;
- let ny = |c: usize| r[1] + c as f64 / side as f64 * rh;
- let (bx0, by0, bx1, by1) = (nx(x0), ny(y0), nx(x1), ny(y1));
- if (bx1 - bx0) * (by1 - by0) > DETAIL_MAX_AREA {
- return None;
- }
- // in units of the photo's height: x × aspect
- let a = aspect.clamp(0.1, 10.0);
- let (cx, cy) = ((bx0 + bx1) / 2.0 * a, (by0 + by1) / 2.0);
- let size = ((bx1 - bx0) * a).max(by1 - by0);
- let m = 0.25 * size;
- let mut w = ((bx1 - bx0) * a + 2.0 * m).max(0.12 * a.max(1.0));
- let mut h = ((by1 - by0) + 2.0 * m).max(0.12 * a.max(1.0));
- w = w.max(h / 2.0).min(a);
- h = h.max(w / 2.0).min(1.0);
- let x = (cx - w / 2.0).clamp(0.0, (a - w).max(0.0));
- let y = (cy - h / 2.0).clamp(0.0, (1.0 - h).max(0.0));
- let out = [x / a, y, (x + w) / a, y + h];
- // a region nearly the whole photo isn't worth a second pass
- ((out[2] - out[0]) * (out[3] - out[1]) < 0.8).then_some(out)
-}
-
-impl Session {
- /// The key and settings of photo `id` as the model sees it: its look without masks, uncropped.
- fn segment_key(&self, id: PhotoId) -> Option<(u64, lightcraft_develop::DevelopSettings)> {
- let p = self.catalog.photo(id)?;
- let mut d = (*p.develop).clone();
- d.masks.clear();
- let content = crate::media::content_key(p);
- let mut h = mix(0xcbf2_9ce4_8422_2325, d.hash64());
- h = mix(h, id.0);
- for b in content.bytes() {
- h = mix(h, u64::from(b));
- }
- Some((h, d))
- }
-
- /// Render the model input for `id`.
- #[cfg_attr(not(feature = "sam"), allow(dead_code))]
- fn segment_input(&mut self, id: PhotoId) -> Result<Input, String> {
- let (key, d) = self.segment_key(id).ok_or("no such photo")?;
- let job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, false, &d).ok_or("no such photo")?;
- let r = job.run();
- self.accept(&r);
- let img = match r.rendered {
- Ok(r) => r.image,
- Err(e) => {
- // the usual reason: the file was moved or deleted (and there's no smart preview)
- if let Some(lightcraft_catalog::Source::File { path }) = self.catalog.photo(id).map(|p| &p.source)
- && !std::path::Path::new(path).exists()
- {
- return Err(format!(
- "This photo's file is missing (moved or deleted), so it can't be analyzed for AI masks: {path}. Library ▸ Find Missing Photos can relink it."
- ));
- }
- return Err(format!("couldn't render the photo for AI masks: {e}"));
- }
- };
- let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect();
- Ok(Input { key, rgb, w: img.width, h: img.height })
- }
-
- /// Start preparing photo `id` for AI masks on a worker thread (load the model, encode the
- /// image). Returns at once; does nothing when it is ready or being prepared.
- pub fn segment_prepare(&mut self, id: PhotoId) -> Result<(), String> {
- let dir = self.segmenter.model_dir()?;
- #[cfg(feature = "sam")]
- {
- let (key, _) = self.segment_key(id).ok_or("no such photo")?;
- let shared = self.segmenter.shared.clone();
- if shared.busy() {
- return Ok(());
- }
- if let Ok(inner) = shared.inner.try_lock()
- && inner.cache.as_ref().is_some_and(|c| c.0 == key)
- {
- return Ok(());
- }
- let input = self.segment_input(id)?;
- shared.set_busy(true);
- let spawned = std::thread::Builder::new().name("sam3-prepare".into()).spawn(move || {
- let mut inner = shared.lock();
- if let Err(e) = prepare(&mut inner, &dir, &input) {
- log::warn!("SAM 3: {e}");
- }
- shared.set_busy(false);
- });
- if let Err(e) = spawned {
- self.segmenter.shared.set_busy(false);
- return Err(format!("could not start the model: {e}"));
- }
- }
- #[cfg(not(feature = "sam"))]
- let _ = (dir, id);
- Ok(())
- }
-
- /// Segment the object `clicks` pick on photo `id` (waits for a running prepare).
- pub fn segment_clicks(&mut self, id: PhotoId, clicks: &[Click]) -> Result<SegMask, String> {
- let dir = self.segmenter.model_dir()?;
- if !clicks.iter().any(|c| c.include) {
- return Err("click the object to include it first".into());
- }
- #[cfg(feature = "sam")]
- {
- let input = self.segment_input_if_needed(id)?;
- let shared = self.segmenter.shared.clone();
- let mut inner = shared.lock();
- let enc = encoded(&mut inner, &dir, input)?;
- let clicks: Vec<lightcraft_segment::Click> =
- clicks.iter().map(|c| lightcraft_segment::Click { x: c.at.x as f32, y: c.at.y as f32, positive: c.include }).collect();
- let (model, enc) = enc;
- let p = model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?;
- Ok(SegMask::from_logits(lightcraft_segment::MASK_SIDE, &p.logits))
- }
- #[cfg(not(feature = "sam"))]
- {
- let _ = (dir, id);
- Err("AI masks are not available in this build".into())
- }
- }
-
- /// Segment everything `text` describes on photo `id`; `Ok(None)` when nothing matches.
- pub fn segment_text(&mut self, id: PhotoId, text: &str) -> Result<Option<SegMask>, String> {
- let dir = self.segmenter.model_dir()?;
- let text = text.trim();
- if text.is_empty() {
- return Err("describe what to select".into());
- }
- #[cfg(feature = "sam")]
- {
- let input = self.segment_input_if_needed(id)?;
- let shared = self.segmenter.shared.clone();
- let mut inner = shared.lock();
- let (model, enc) = encoded(&mut inner, &dir, input)?;
- // "car, road": each phrase on its own, merged (the per-pixel maximum)
- let mut merged: Option<Vec<f32>> = None;
- for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
- if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {
- merged = Some(match merged {
- None => p.logits,
- Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(),
- });
- }
- }
- Ok(merged.map(|l| SegMask::from_logits(lightcraft_segment::MASK_SIDE, &l)))
- }
- #[cfg(not(feature = "sam"))]
- {
- let _ = (dir, id);
- Err("AI masks are not available in this build".into())
- }
- }
-
- /// Render the part `region` (normalized) of photo `id` for the model, from a source sharp
- /// enough for it.
- #[cfg_attr(not(feature = "sam"), allow(dead_code))]
- fn segment_region_input(&mut self, id: PhotoId, region: [f64; 4]) -> Result<Input, String> {
- let (_, mut d) = self.segment_key(id).ok_or("no such photo")?;
- d.crop.geometry = lightcraft_geom::CropGeometry { rect: lightcraft_geom::Rect::new(region[0], region[1], region[2], region[3]), angle: 0.0 };
- d.crop.flip_h = false;
- d.crop.flip_v = false;
- d.geometry.constrain_crop = false;
- let mut job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, true, &d).ok_or("no such photo")?;
- let frac = (region[2] - region[0]).max(region[3] - region[1]).max(0.01);
- let p = self.catalog.photo(id).ok_or("no such photo")?.clone();
- job.level = crate::media::SourceLevel::for_size((INPUT_EDGE as f64 / frac) as usize);
- job.source = self.media.source_ref(&p, job.level);
- let r = job.run();
- self.accept(&r);
- let img = r.rendered.map_err(|e| format!("couldn't render the photo for AI masks: {e}"))?.image;
- let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect();
- Ok(Input { key: 0, rgb, w: img.width, h: img.height })
- }
-
- /// Start a zoomed-in pass for component `comp` of mask `mask` on photo `id` (an Object or
- /// Describe selection): the photo around the selection is rendered at a higher resolution
- /// and segmented again, on a worker thread; [`Session::segment_poll`] applies the result.
- /// `Ok(false)` when there is nothing to refine (no selection yet, or it is most of the photo).
- pub fn segment_detail(&mut self, id: PhotoId, mask: u32, comp: usize) -> Result<bool, String> {
- let dir = self.segmenter.model_dir()?;
- let d = self.develop_of(id).ok_or("no such photo")?;
- let shape = d.masks.iter().find(|m| m.id == mask).and_then(|m| m.components.get(comp)).map(|c| c.shape.clone()).ok_or("no such mask")?;
- let (seg, prompt) = match &shape {
- MaskShape::Object { hint, exclude, seg: Some(seg), .. } if !hint.is_empty() => (
- seg.clone(),
- Prompt::Clicks(
- hint.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect(),
- ),
- ),
- MaskShape::Prompt { text, seg: Some(seg), .. } => (seg.clone(), Prompt::Text(text.clone())),
- _ => return Ok(false),
- };
- let p = self.catalog.photo(id).ok_or("no such photo")?;
- let (pw, ph) = (f64::from(p.width.max(1)), f64::from(p.height.max(1)));
- let aspect = if d.orientation.swaps_axes() { ph / pw } else { pw / ph };
- let Some(region) = detail_region(&seg, aspect) else { return Ok(false) };
- #[cfg(feature = "sam")]
- {
- let shared = self.segmenter.shared.clone();
- if shared.detail_busy.swap(true, std::sync::atomic::Ordering::Relaxed) {
- return Ok(false);
- }
- let input = match self.segment_region_input(id, region) {
- Ok(i) => i,
- Err(e) => {
- shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed);
- return Err(e);
- }
- };
- let shape = strip_detail(shape);
- let spawned = std::thread::Builder::new().name("sam3-detail".into()).spawn({
- let shared = shared.clone();
- move || {
- let patch = {
- let mut inner = shared.lock();
- detail_pass(&mut inner, &dir, &input, &prompt, region)
- };
- match patch {
- Ok(patch) => {
- let r = DetailResult { photo: id, mask, comp, shape, patch };
- shared.results.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(r);
- }
- Err(e) => log::warn!("SAM 3 detail pass: {e}"),
- }
- shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed);
- }
- });
- if let Err(e) = spawned {
- shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed);
- return Err(format!("could not start the detail pass: {e}"));
- }
- Ok(true)
- }
- #[cfg(not(feature = "sam"))]
- {
- let _ = (dir, prompt, region, mask);
- Ok(false)
- }
- }
-
- /// Apply finished detail passes (to components that haven't changed since). True when a
- /// mask changed.
- pub fn segment_poll(&mut self) -> bool {
- #[cfg(feature = "sam")]
- {
- let done: Vec<DetailResult> =
- std::mem::take(&mut *self.segmenter.shared.results.lock().unwrap_or_else(std::sync::PoisonError::into_inner));
- let mut changed = false;
- for r in done {
- let Some(patch) = r.patch else { continue };
- let Some(d) = self.develop_of(r.photo) else { continue };
- let mut d = (*d).clone();
- let Some(c) = d.masks.iter_mut().find(|m| m.id == r.mask).and_then(|m| m.components.get_mut(r.comp)) else { continue };
- if strip_detail(c.shape.clone()) != r.shape {
- continue; // clicked again meanwhile: a newer pass will follow
- }
- match &mut c.shape {
- MaskShape::Object { detail, .. } | MaskShape::Prompt { detail, .. } => *detail = vec![patch],
- _ => continue,
- }
- match self.set_develop(r.photo, d, "AI Mask Detail") {
- Ok(()) => changed = true,
- Err(e) => log::warn!("AI mask detail: {e}"),
- }
- }
- changed
- }
- #[cfg(not(feature = "sam"))]
- false
- }
-
- /// The model input for `id`, unless the cache already holds it (or a prepare is running,
- /// which will have it).
- #[cfg(feature = "sam")]
- fn segment_input_if_needed(&mut self, id: PhotoId) -> Result<Either, String> {
- let (key, _) = self.segment_key(id).ok_or("no such photo")?;
- if self.segmenter.shared.busy() {
- return Ok(Either::Key(key));
- }
- let cached = self.segmenter.shared.lock().cache.as_ref().is_some_and(|c| c.0 == key);
- if cached { Ok(Either::Key(key)) } else { Ok(Either::Input(self.segment_input(id)?)) }
- }
-}
-
-#[cfg(feature = "sam")]
-enum Either {
- /// Already encoded (or being encoded) under this key.
- Key(u64),
- Input(Input),
-}
-
-#[cfg(feature = "sam")]
-fn load(inner: &mut model::Inner, dir: &std::path::Path) -> Result<(), String> {
- if inner.model.is_none() {
- let t = web_time::Instant::now();
- inner.model = Some(lightcraft_segment::Sam3::load(dir).map_err(|e| e.to_string())?);
- log::info!("SAM 3 loaded in {:?}", t.elapsed());
- }
- Ok(())
-}
-
-/// Load the model and encode `input` into the cache (with the click features ready).
-#[cfg(feature = "sam")]
-fn prepare(inner: &mut model::Inner, dir: &std::path::Path, input: &Input) -> Result<(), String> {
- load(inner, dir)?;
- let model = inner.model.as_mut().ok_or("the model did not load")?;
- let t = web_time::Instant::now();
- let mut enc = model.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
- model.prepare_clicks(&mut enc).map_err(|e| e.to_string())?;
- log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());
- inner.cache = Some((input.key, enc));
- Ok(())
-}
-
-/// The model and the encoded image for `want` (encoding it now when needed).
-#[cfg(feature = "sam")]
-fn encoded<'a>(
- inner: &'a mut model::Inner,
- dir: &std::path::Path,
- want: Either,
-) -> Result<(&'a mut lightcraft_segment::Sam3, &'a mut lightcraft_segment::Encoded), String> {
- match want {
- Either::Input(input) => prepare(inner, dir, &input)?,
- Either::Key(key) => {
- if !inner.cache.as_ref().is_some_and(|c| c.0 == key) {
- return Err("the photo changed while it was being prepared; try again".into());
- }
- }
- }
- let model::Inner { model, cache } = inner;
- match (model.as_mut(), cache.as_mut()) {
- (Some(m), Some((_, e))) => Ok((m, e)),
- _ => Err("the model is not ready".into()),
- }
-}
-
-/// `shape` without its detail patches (what a detail pass was computed from)
-#[cfg_attr(not(feature = "sam"), allow(dead_code))]
-fn strip_detail(mut shape: MaskShape) -> MaskShape {
- // (nor its edge setting: moving the slider meanwhile keeps the pass)
- if let MaskShape::Object { detail, edge, .. } | MaskShape::Prompt { detail, edge, .. } = &mut shape {
- detail.clear();
- *edge = 0.0;
- }
- shape
-}
-
-/// Segment `input` (the photo's part `region`) with `prompt`, as a patch over `region`.
-#[cfg(feature = "sam")]
-fn detail_pass(inner: &mut model::Inner, dir: &std::path::Path, input: &Input, prompt: &Prompt, region: [f64; 4]) -> Result<Option<SegMask>, String> {
- load(inner, dir)?;
- let model = inner.model.as_mut().ok_or("the model did not load")?;
- let t = web_time::Instant::now();
- let mut enc = model.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
- let (rw, rh) = (region[2] - region[0], region[3] - region[1]);
- let logits = match prompt {
- Prompt::Clicks(clicks) => {
- let inside: Vec<lightcraft_segment::Click> = clicks
- .iter()
- .map(|c| ((c.at.x - region[0]) / rw, (c.at.y - region[1]) / rh, c.include))
- .filter(|(x, y, _)| (0.0..=1.0).contains(x) && (0.0..=1.0).contains(y))
- .map(|(x, y, positive)| lightcraft_segment::Click { x: x as f32, y: y as f32, positive })
- .collect();
- if !inside.iter().any(|c| c.positive) {
- return Ok(None);
- }
- Some(model.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits)
- }
- Prompt::Text(text) => {
- let mut merged: Option<Vec<f32>> = None;
- for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
- if let Some(p) = model.segment_text(&mut enc, phrase, 0.5).map_err(|e| e.to_string())? {
- merged = Some(match merged {
- None => p.logits,
- Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(),
- });
- }
- }
- merged
- }
- };
- log::info!("SAM 3 detail pass ({}×{}) in {:?}", input.w, input.h, t.elapsed());
- Ok(logits.map(|l| SegMask::from_logits_in(lightcraft_segment::MASK_SIDE, &l, region)))
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask {
- let l: Vec<f32> =
- (0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect();
- SegMask::from_logits(side, &l)
- }
-
- #[test]
- fn detail_region_frames_the_selection_with_a_margin() {
- // a small object in the middle of a square photo
- let r = detail_region(&seg_box(100, 40, 45, 50, 55), 1.0).unwrap();
- assert!(r[0] < 0.4 && r[2] > 0.5 && r[1] < 0.45 && r[3] > 0.55, "contains it: {r:?}");
- assert!(r[2] - r[0] < 0.25 && r[3] - r[1] < 0.25, "zoomed in: {r:?}");
- assert!(r.iter().all(|v| (0.0..=1.0).contains(v)));
- // at the corner: shifted inside the photo, not cut
- let r = detail_region(&seg_box(100, 0, 0, 5, 5), 1.5).unwrap();
- assert!(r[0] >= 0.0 && r[1] >= 0.0 && r[0] < 1e-9 && r[1] < 1e-9);
- // a thin object stays at most 2:1 (in photo pixels, 1.5:1 photo)
- let r = detail_region(&seg_box(100, 10, 50, 90, 52), 1.5).unwrap();
- let (w, h) = ((r[2] - r[0]) * 1.5, r[3] - r[1]);
- assert!(w / h <= 2.0 + 1e-9, "{w} × {h}");
- }
-
- #[test]
- fn no_region_for_nothing_or_most_of_the_photo() {
- assert!(detail_region(&seg_box(50, 0, 0, 0, 0), 1.0).is_none());
- assert!(detail_region(&seg_box(50, 2, 2, 48, 48), 1.0).is_none());
- let damaged = SegMask { side: 50, data: "?".into(), rect: None };
- assert!(detail_region(&damaged, 1.0).is_none());
- }
-}
diff --git a/crates/engine/src/segment/download.rs b/crates/engine/src/segment/download.rs
new file mode 100644
index 000000000..875d652f8
--- /dev/null
+++ b/crates/engine/src/segment/download.rs
@@ -0,0 +1,146 @@
+//! The model download as a background task the session starts, watches and cancels without
+//! ever blocking: progress lives in atomics, messages behind a mutex only ever `try_lock`ed
+//! by the session (the download thread holds it for a few instructions at a time).
+
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+
+use lightcraft_segment::fetch::{self, Options, Progress};
+
+/// What the session sees of a download.
+#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
+pub struct Status {
+ pub running: bool,
+ pub done: u64,
+ pub total: u64,
+ /// The file being fetched.
+ pub file: String,
+ /// The last download's error (cleared when a new one starts).
+ pub error: Option<String>,
+ /// The last download finished and the files are in place.
+ pub finished: bool,
+}
+
+#[derive(Default)]
+struct Text {
+ file: String,
+ error: Option<String>,
+}
+
+#[derive(Default)]
+struct Shared {
+ running: AtomicBool,
+ cancel: AtomicBool,
+ finished: AtomicBool,
+ done: AtomicU64,
+ total: AtomicU64,
+ text: Mutex<Text>,
+}
+
+/// Clears `running` when the download thread ends, however it ends.
+struct Running(Arc<Shared>);
+
+impl Drop for Running {
+ fn drop(&mut self) {
+ self.0.running.store(false, Ordering::SeqCst);
+ }
+}
+
+#[derive(Default)]
+pub struct Downloader {
+ shared: Arc<Shared>,
+ /// The last text seen (when the download thread held the lock at that moment).
+ last: Mutex<Text>,
+}
+
+fn lock(m: &Mutex<Text>) -> std::sync::MutexGuard<'_, Text> {
+ m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
+}
+
+impl Downloader {
+ pub fn status(&self) -> Status {
+ let s = &self.shared;
+ if let Ok(t) = s.text.try_lock()
+ && let Ok(mut last) = self.last.try_lock()
+ {
+ last.file.clone_from(&t.file);
+ last.error.clone_from(&t.error);
+ }
+ // (only the session's thread locks `last`)
+ let (file, error) = self.last.try_lock().map(|l| (l.file.clone(), l.error.clone())).unwrap_or_default();
+ Status {
+ running: s.running.load(Ordering::SeqCst),
+ done: s.done.load(Ordering::SeqCst),
+ total: s.total.load(Ordering::SeqCst),
+ file,
+ error,
+ finished: s.finished.load(Ordering::SeqCst),
+ }
+ }
+
+ pub fn running(&self) -> bool {
+ self.shared.running.load(Ordering::SeqCst)
+ }
+
+ /// Ask a running download to stop (it keeps its partial files to resume later).
+ pub fn cancel(&self) -> bool {
+ let running = self.running();
+ if running {
+ self.shared.cancel.store(true, Ordering::SeqCst);
+ }
+ running
+ }
+
+ /// Start downloading `files` from `mirrors` into `dir` on a background thread. False when
+ /// one is already running.
+ pub fn start(&self, files: &'static [fetch::FileSpec], mirrors: Vec<String>, dir: PathBuf, opts: Options) -> Result<bool, String> {
+ let s = self.shared.clone();
+ if s.running.swap(true, Ordering::SeqCst) {
+ return Ok(false);
+ }
+ s.cancel.store(false, Ordering::SeqCst);
+ s.finished.store(false, Ordering::SeqCst);
+ s.done.store(0, Ordering::SeqCst);
+ s.total.store(files.iter().filter_map(|f| f.size).sum(), Ordering::SeqCst);
+ // nothing else runs while `running` was false
+ *lock(&s.text) = Text::default();
+ *lock(&self.last) = Text::default();
+ let guard = Running(s.clone());
+ let spawned = std::thread::Builder::new().name("sam3-download".into()).spawn(move || {
+ let s = guard.0.clone();
+ let mut last_file = String::new();
+ let mut on_progress = |p: &Progress| {
+ s.done.store(p.done, Ordering::SeqCst);
+ s.total.store(p.total, Ordering::SeqCst);
+ if p.file != last_file {
+ last_file.clone_from(&p.file);
+ lock(&s.text).file.clone_from(&p.file);
+ }
+ };
+ let r =
+ std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| fetch::download(files, &mirrors, &dir, &opts, &s.cancel, &mut on_progress)));
+ let error = match r {
+ Ok(Ok(())) => {
+ s.finished.store(true, Ordering::SeqCst);
+ log::info!("SAM 3 model downloaded to {}", dir.display());
+ None
+ }
+ Ok(Err(e)) => Some(e.to_string()),
+ Err(_) => Some("the download failed unexpectedly".to_string()),
+ };
+ if let Some(e) = &error {
+ log::warn!("SAM 3 download: {e}");
+ }
+ lock(&s.text).error = error;
+ drop(guard);
+ });
+ match spawned {
+ Ok(_) => Ok(true),
+ Err(e) => {
+ // the closure (and its guard) was dropped: `running` is false again
+ Err(format!("could not start the download: {e}"))
+ }
+ }
+ }
+}
diff --git a/crates/engine/src/segment/mod.rs b/crates/engine/src/segment/mod.rs
new file mode 100644
index 000000000..84397042d
--- /dev/null
+++ b/crates/engine/src/segment/mod.rs
@@ -0,0 +1,715 @@
+//! AI masks: Object (clicks) and Describe (text) selections computed with SAM 3
+//! (`lightcraft-segment`, cargo feature `sam`; the desktop app enables it).
+//!
+//! **Nothing requires the model.** It is not part of LightCraft (SAM License); the user
+//! downloads it when they first use an AI mask and agree to (`segment.model.download`), or
+//! puts the files in the model folder themselves. Without it, AI mask requests fail with a
+//! clear "not installed" error; renders, exports and every other feature never touch it
+//! (stored segmentations are plain data in the develop settings).
+//!
+//! The model sees the photo as developed, uncropped and without masks, at 1008 px — the frame
+//! mask coordinates live in — so a stored segmentation lines up with the photo at any size.
+//! Everything that touches the model (loading it, rendering and encoding the photo, clicks,
+//! descriptions, detail passes) runs on one worker thread ([`worker`]); the session talks to it
+//! over channels and never waits on a lock. With [`Segmenter::background`] (the desktop app)
+//! commands return at once and [`Session::segment_poll`] applies the results; without it (CLI,
+//! MCP, tests) they wait for the worker.
+
+#[cfg(feature = "sam")]
+mod download;
+#[cfg(feature = "sam")]
+mod worker;
+
+use std::path::PathBuf;
+
+use lightcraft_catalog::PhotoId;
+use lightcraft_develop::{MaskShape, SegMask};
+use lightcraft_geom::Point;
+
+use crate::Session;
+
+/// Long edge of the image the model sees.
+pub const INPUT_EDGE: usize = 1008;
+/// Download size of the model (`model.safetensors`; the tokenizer files add ~2 MB).
+pub const MODEL_BYTES: u64 = 3_439_938_512;
+/// The model's licence (not LightCraft's): shown before downloading.
+pub const LICENSE_NAME: &str = "SAM License (Meta)";
+pub const LICENSE_URL: &str = "https://github.com/facebookresearch/sam3/blob/main/LICENSE";
+/// How errors about a missing model start (the UI offers the download on it).
+pub const NOT_INSTALLED: &str = "The SAM 3 model is not installed";
+/// Longest a waiting command (CLI, MCP) waits for the model (loading on a CPU can take minutes).
+#[cfg_attr(not(feature = "sam"), allow(dead_code))]
+const WAIT: std::time::Duration = std::time::Duration::from_secs(15 * 60);
+/// Most clicks one Object selection takes (the model's prompt grows with each).
+pub const MAX_CLICKS: usize = 64;
+
+/// One click of an Object selection (normalized image coordinates).
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct Click {
+ pub at: Point,
+ pub include: bool,
+}
+
+/// What a request's result is for (how [`Session::segment_poll`] applies it).
+#[derive(Clone, Debug, PartialEq)]
+#[cfg_attr(not(feature = "sam"), allow(dead_code))]
+pub(crate) enum Tag {
+ /// A command waiting for the result itself.
+ Wait,
+ /// Load the model and encode the photo ahead of the first click.
+ Prepare,
+ /// Object clicks on component `comp` of mask `mask`.
+ Clicks { photo: PhotoId, mask: u32, comp: usize, hint: Vec<Point>, exclude: Vec<Point> },
+ /// A Describe selection: a new mask (`mask: None`) or a component combined by `op`.
+ NewPrompt { photo: PhotoId, mask: Option<u32>, op: String, name: Option<String>, text: String },
+ /// A zoomed-in pass for a component, computed from `shape` (without its detail).
+ Detail { photo: PhotoId, mask: u32, comp: usize, shape: MaskShape },
+}
+
+/// The model input: 8-bit RGB.
+#[cfg_attr(not(feature = "sam"), allow(dead_code))]
+pub(crate) struct Input {
+ rgb: Vec<u8>,
+ w: usize,
+ h: usize,
+}
+
+/// What a detail pass re-runs.
+#[cfg_attr(not(feature = "sam"), allow(dead_code))]
+pub(crate) enum Prompt {
+ Clicks(Vec<Click>),
+ Text(String),
+}
+
+/// Object clicks sent to the worker and not applied yet (background mode): further clicks add
+/// to these, and the loupe shows them.
+#[derive(Clone, Debug, PartialEq)]
+pub struct PendingClicks {
+ pub photo: PhotoId,
+ pub mask: u32,
+ pub comp: usize,
+ pub hint: Vec<Point>,
+ pub exclude: Vec<Point>,
+}
+
+/// What [`Session::segment_poll`] did.
+#[derive(Debug, Default)]
+pub struct Polled {
+ /// A mask changed.
+ pub changed: bool,
+ /// Errors and notices from background work, for the user.
+ pub messages: Vec<String>,
+ /// A mask that got a new selection (worth a detail pass).
+ pub refine: Option<u32>,
+}
+
+/// The model download (`segment.model.*`).
+#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
+pub struct DownloadStatus {
+ pub running: bool,
+ pub done: u64,
+ pub total: u64,
+ pub file: String,
+ pub error: Option<String>,
+ pub finished: bool,
+}
+
+/// The model (on its worker thread), the download, and AI mask requests in flight.
+pub struct Segmenter {
+ /// Where the `facebook/sam3` files live (set by the app; `None`: no AI masks).
+ pub dir: Option<PathBuf>,
+ /// The user's list of download mirrors (one base URL per line), if any.
+ pub mirrors_file: Option<PathBuf>,
+ /// Run requests in the background and apply them in [`Session::segment_poll`] (the
+ /// desktop app); otherwise commands wait for their result (CLI, MCP, tests).
+ pub background: bool,
+ pending: Option<PendingClicks>,
+ messages: Vec<String>,
+ #[cfg(feature = "sam")]
+ worker: worker::Worker,
+ #[cfg(feature = "sam")]
+ download: download::Downloader,
+ #[cfg(feature = "sam")]
+ results: (std::sync::mpsc::Sender<worker::Outcome>, std::sync::mpsc::Receiver<worker::Outcome>),
+}
+
+// (derivable only without the `sam` feature: the results channel has no Default)
+#[allow(clippy::derivable_impls)]
+impl Default for Segmenter {
+ fn default() -> Self {
+ Segmenter {
+ dir: None,
+ mirrors_file: None,
+ background: false,
+ pending: None,
+ messages: Vec::new(),
+ #[cfg(feature = "sam")]
+ worker: worker::Worker::default(),
+ #[cfg(feature = "sam")]
+ download: download::Downloader::default(),
+ #[cfg(feature = "sam")]
+ results: std::sync::mpsc::channel(),
+ }
+ }
+}
+
+impl Segmenter {
+ /// Whether this build can compute AI masks at all.
+ pub const AVAILABLE: bool = cfg!(feature = "sam");
+
+ /// Whether the model's files are in place.
+ pub fn installed(&self) -> bool {
+ #[cfg(feature = "sam")]
+ if let Some(dir) = &self.dir {
+ return lightcraft_segment::is_model_dir(dir);
+ }
+ false
+ }
+
+ /// Whether requests are queued or running (loading, analyzing the photo, a click…).
+ pub fn busy(&self) -> bool {
+ #[cfg(feature = "sam")]
+ return self.worker.pending() > 0;
+ #[cfg(not(feature = "sam"))]
+ false
+ }
+
+ /// Whether the slow part runs now: loading the model or analyzing a photo.
+ pub fn analyzing(&self) -> bool {
+ #[cfg(feature = "sam")]
+ return self.worker.encoding();
+ #[cfg(not(feature = "sam"))]
+ false
+ }
+
+ /// Whether a zoomed-in detail pass is queued or running.
+ pub fn detail_busy(&self) -> bool {
+ #[cfg(feature = "sam")]
+ return self.worker.detail() > 0;
+ #[cfg(not(feature = "sam"))]
+ false
+ }
+
+ /// Whether the model is in memory (it is unloaded after a while without use).
+ pub fn loaded(&self) -> bool {
+ #[cfg(feature = "sam")]
+ return self.worker.loaded();
+ #[cfg(not(feature = "sam"))]
+ false
+ }
+
+ /// Clicks waiting for their segmentation.
+ pub fn pending_clicks(&self) -> Option<&PendingClicks> {
+ self.pending.as_ref()
+ }
+
+ /// The model download's state.
+ pub fn download_status(&self) -> DownloadStatus {
+ #[cfg(feature = "sam")]
+ {
+ let s = self.download.status();
+ DownloadStatus { running: s.running, done: s.done, total: s.total, file: s.file, error: s.error, finished: s.finished }
+ }
+ #[cfg(not(feature = "sam"))]
+ DownloadStatus::default()
+ }
+
+ /// Where the model is downloaded from, in order (the user's mirrors first).
+ pub fn mirrors(&self) -> Vec<String> {
+ #[cfg(feature = "sam")]
+ {
+ let env = std::env::var(lightcraft_segment::fetch::MIRRORS_ENV).ok();
+ lightcraft_segment::fetch::mirrors(env.as_deref(), self.mirrors_file.as_deref())
+ }
+ #[cfg(not(feature = "sam"))]
+ Vec::new()
+ }
+
+ /// The model folder, if the model is there; otherwise an error that says what to do.
+ pub fn model_dir(&self) -> Result<PathBuf, String> {
+ if !Self::AVAILABLE {
+ return Err("AI masks are not available in this build".into());
+ }
+ let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?;
+ if self.installed() {
+ return Ok(dir);
+ }
+ let d = self.download_status();
+ if d.running {
+ let pct = d.done.saturating_mul(100).checked_div(d.total).unwrap_or(0);
+ return Err(format!("{NOT_INSTALLED} yet: it is downloading ({pct} %)."));
+ }
+ Err(format!(
+ "{NOT_INSTALLED}. Download it (about {:.1} GB, {LICENSE_NAME}) when LightCraft offers it, with `segment.model.download {{\"acknowledged\": true}}`, or put model.safetensors, vocab.json and merges.txt in {}.",
+ MODEL_BYTES as f64 / 1e9,
+ dir.display()
+ ))
+ }
+
+ /// Start downloading the model (on a background thread; see `segment.model.download`).
+ /// `Ok(false)` when it is installed or downloading already.
+ pub fn start_download(&self) -> Result<bool, String> {
+ if !Self::AVAILABLE {
+ return Err("AI masks are not available in this build".into());
+ }
+ let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?;
+ if self.installed() {
+ return Ok(false);
+ }
+ #[cfg(feature = "sam")]
+ {
+ let mirrors = self.mirrors();
+ if mirrors.is_empty() {
+ return Err(lightcraft_segment::fetch::DownloadError::NoMirrors.to_string());
+ }
+ self.download.start(lightcraft_segment::fetch::SAM3_FILES, mirrors, dir, lightcraft_segment::fetch::Options::default())
+ }
+ #[cfg(not(feature = "sam"))]
+ {
+ let _ = dir;
+ Ok(false)
+ }
+ }
+
+ /// Stop a running download (its partial files stay, to resume). False when none runs.
+ pub fn cancel_download(&self) -> bool {
+ #[cfg(feature = "sam")]
+ return self.download.cancel();
+ #[cfg(not(feature = "sam"))]
+ false
+ }
+}
+
+/// Mix `v` into a 64-bit hash.
+#[cfg_attr(not(feature = "sam"), allow(dead_code))]
+fn mix(h: u64, v: u64) -> u64 {
+ (h ^ v).wrapping_mul(0x0100_0000_01b3).rotate_left(29)
+}
+
+/// Objects covering more than this much of the photo gain little from a zoomed pass.
+const DETAIL_MAX_AREA: f64 = 0.4;
+
+/// The part of the photo to zoom into for `seg`'s selection: its bounding box with a margin of
+/// a quarter of its size, at least 12 % of the long edge, at most 2:1, inside the photo
+/// (`aspect` = width / height). `None` when nothing is selected or it is most of the photo.
+pub fn detail_region(seg: &SegMask, aspect: f64) -> Option<[f64; 4]> {
+ let logits = seg.logits()?;
+ let r = seg.bounds()?;
+ let side = seg.side as usize;
+ let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0, 0);
+ for (i, l) in logits.iter().enumerate() {
+ if *l > 0.0 {
+ let (x, y) = (i % side, i / side);
+ (x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x + 1), y1.max(y + 1));
+ }
+ }
+ if x0 == usize::MAX {
+ return None;
+ }
+ let (rw, rh) = (r[2] - r[0], r[3] - r[1]);
+ let nx = |c: usize| r[0] + c as f64 / side as f64 * rw;
+ let ny = |c: usize| r[1] + c as f64 / side as f64 * rh;
+ let (bx0, by0, bx1, by1) = (nx(x0), ny(y0), nx(x1), ny(y1));
+ if (bx1 - bx0) * (by1 - by0) > DETAIL_MAX_AREA {
+ return None;
+ }
+ // in units of the photo's height: x × aspect
+ let a = if aspect.is_finite() { aspect.clamp(0.1, 10.0) } else { 1.0 };
+ let (cx, cy) = ((bx0 + bx1) / 2.0 * a, (by0 + by1) / 2.0);
+ let size = ((bx1 - bx0) * a).max(by1 - by0);
+ let m = 0.25 * size;
+ let mut w = ((bx1 - bx0) * a + 2.0 * m).max(0.12 * a.max(1.0));
+ let mut h = ((by1 - by0) + 2.0 * m).max(0.12 * a.max(1.0));
+ w = w.max(h / 2.0).min(a);
+ h = h.max(w / 2.0).min(1.0);
+ let x = (cx - w / 2.0).clamp(0.0, (a - w).max(0.0));
+ let y = (cy - h / 2.0).clamp(0.0, (1.0 - h).max(0.0));
+ let out = [x / a, y, (x + w) / a, y + h];
+ // a region nearly the whole photo isn't worth a second pass
+ ((out[2] - out[0]) * (out[3] - out[1]) < 0.8).then_some(out)
+}
+
+/// `shape` without its detail patches (what a detail pass was computed from)
+#[cfg_attr(not(feature = "sam"), allow(dead_code))]
+fn strip_detail(mut shape: MaskShape) -> MaskShape {
+ // (nor its edge setting: moving the slider meanwhile keeps the pass)
+ if let MaskShape::Object { detail, edge, .. } | MaskShape::Prompt { detail, edge, .. } = &mut shape {
+ detail.clear();
+ *edge = 0.0;
+ }
+ shape
+}
+
+fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec<Click> {
+ include.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect()
+}
+
+impl Session {
+ /// The key and settings of photo `id` as the model sees it: its look without masks, uncropped.
+ #[cfg_attr(not(feature = "sam"), allow(dead_code))]
+ fn segment_key(&self, id: PhotoId) -> Option<(u64, lightcraft_develop::DevelopSettings)> {
+ let p = self.catalog.photo(id)?;
+ let mut d = (*p.develop).clone();
+ d.masks.clear();
+ let content = crate::media::content_key(p);
+ let mut h = mix(0xcbf2_9ce4_8422_2325, d.hash64());
+ h = mix(h, id.0);
+ for b in content.bytes() {
+ h = mix(h, u64::from(b));
+ }
+ Some((h, d))
+ }
+
+ /// Clicks (in normalized photo coordinates) for the model.
+ #[cfg(feature = "sam")]
+ fn model_clicks(clicks: &[Click]) -> Vec<lightcraft_segment::Click> {
+ clicks.iter().map(|c| lightcraft_segment::Click { x: c.at.x as f32, y: c.at.y as f32, positive: c.include }).collect()
+ }
+
+ /// A worker request about photo `id` (the render job for its model input goes along; the
+ /// worker runs it only when its cache doesn't hold the photo). `region` renders that part
+ /// of the photo, from a source sharp enough for it (detail passes).
+ #[cfg(feature = "sam")]
+ fn segment_job(
+ &mut self,
+ id: PhotoId,
+ kind: worker::Kind,
+ tag: Tag,
+ reply: std::sync::mpsc::Sender<worker::Outcome>,
+ region: Option<[f64; 4]>,
+ ) -> Result<worker::Job, String> {
+ let dir = self.segmenter.model_dir()?;
+ let (key, mut d) = self.segment_key(id).ok_or("no such photo")?;
+ let p = self.catalog.photo(id).ok_or("no such photo")?.clone();
+ let missing = match &p.source {
+ lightcraft_catalog::Source::File { path } if !std::path::Path::new(path).exists() => Some(path.clone()),
+ _ => None,
+ };
+ let render = match region {
+ None => self.preview_job(id, INPUT_EDGE, INPUT_EDGE, false, &d).ok_or("no such photo")?,
+ Some(region) => {
+ d.crop.geometry =
+ lightcraft_geom::CropGeometry { rect: lightcraft_geom::Rect::new(region[0], region[1], region[2], region[3]), angle: 0.0 };
+ d.crop.flip_h = false;
+ d.crop.flip_v = false;
+ d.geometry.constrain_crop = false;
+ let mut job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, true, &d).ok_or("no such photo")?;
+ let frac = (region[2] - region[0]).max(region[3] - region[1]).max(0.01);
+ job.level = crate::media::SourceLevel::for_size((INPUT_EDGE as f64 / frac) as usize);
+ job.source = self.media.source_ref(&p, job.level);
+ job
+ }
+ };
+ Ok(worker::Job { dir, key: if region.is_some() { 0 } else { key }, render: Some(render), missing, kind, tag, reply })
+ }
+
+ /// Send a request and wait for its result (CLI, MCP, tests; or when not in background mode).
+ #[cfg(feature = "sam")]
+ fn segment_wait(&mut self, id: PhotoId, kind: worker::Kind, region: Option<[f64; 4]>) -> Result<Option<SegMask>, String> {
+ let (tx, rx) = std::sync::mpsc::channel();
+ let job = self.segment_job(id, kind, Tag::Wait, tx, region)?;
+ self.segmenter.worker.submit(job)?;
+ match rx.recv_timeout(WAIT) {
+ Ok(o) => o.result,
+ Err(_) => Err("the AI model took too long; try again".into()),
+ }
+ }
+
+ /// Send a request whose result [`Session::segment_poll`] applies.
+ #[cfg(feature = "sam")]
+ fn segment_queue(&mut self, id: PhotoId, kind: worker::Kind, tag: Tag, region: Option<[f64; 4]>) -> Result<(), String> {
+ let reply = self.segmenter.results.0.clone();
+ let job = self.segment_job(id, kind, tag, reply, region)?;
+ self.segmenter.worker.submit(job)
+ }
+
+ /// Start preparing photo `id` for AI masks (load the model, encode the image), so the first
+ /// click is fast. In background mode it returns at once.
+ pub fn segment_prepare(&mut self, id: PhotoId) -> Result<(), String> {
+ self.segmenter.model_dir()?;
+ #[cfg(feature = "sam")]
+ {
+ if self.segmenter.background {
+ return self.segment_queue(id, worker::Kind::Prepare, Tag::Prepare, None);
+ }
+ self.segment_wait(id, worker::Kind::Prepare, None)?;
+ }
+ #[cfg(not(feature = "sam"))]
+ let _ = id;
+ Ok(())
+ }
+
+ /// Segment the object `clicks` pick on photo `id`, waiting for the model.
+ pub fn segment_clicks(&mut self, id: PhotoId, clicks: &[Click]) -> Result<SegMask, String> {
+ self.segmenter.model_dir()?;
+ if !clicks.iter().any(|c| c.include) {
+ return Err("click the object to include it first".into());
+ }
+ #[cfg(feature = "sam")]
+ {
+ self.segment_wait(id, worker::Kind::Clicks(Self::model_clicks(clicks)), None)?.ok_or_else(|| "nothing was selected".to_string())
+ }
+ #[cfg(not(feature = "sam"))]
+ {
+ let _ = id;
+ Err("AI masks are not available in this build".into())
+ }
+ }
+
+ /// Segment everything `text` describes on photo `id`, waiting for the model; `Ok(None)`
+ /// when nothing matches.
+ pub fn segment_text(&mut self, id: PhotoId, text: &str) -> Result<Option<SegMask>, String> {
+ self.segmenter.model_dir()?;
+ let text = text.trim();
+ if text.is_empty() {
+ return Err("describe what to select".into());
+ }
+ #[cfg(feature = "sam")]
+ {
+ self.segment_wait(id, worker::Kind::Text(text.to_string()), None)
+ }
+ #[cfg(not(feature = "sam"))]
+ {
+ let _ = id;
+ Err("AI masks are not available in this build".into())
+ }
+ }
+
+ /// Background mode: queue Object clicks for component `comp` of mask `mask`; the result
+ /// (clicks and segmentation, one undo step) is applied by [`Session::segment_poll`].
+ pub(crate) fn segment_clicks_later(&mut self, id: PhotoId, mask: u32, comp: usize, hint: Vec<Point>, exclude: Vec<Point>) -> Result<(), String> {
+ self.segmenter.model_dir()?;
+ if hint.is_empty() {
+ return Err("click the object to include it first".into());
+ }
+ #[cfg(feature = "sam")]
+ {
+ let clicks = Self::model_clicks(&clicks_of(&hint, &exclude));
+ let tag = Tag::Clicks { photo: id, mask, comp, hint: hint.clone(), exclude: exclude.clone() };
+ self.segment_queue(id, worker::Kind::Clicks(clicks), tag, None)?;
+ self.segmenter.pending = Some(PendingClicks { photo: id, mask, comp, hint, exclude });
+ Ok(())
+ }
+ #[cfg(not(feature = "sam"))]
+ {
+ let _ = (id, mask, comp, exclude);
+ Err("AI masks are not available in this build".into())
+ }
+ }
+
+ /// Background mode: queue a Describe selection; the mask (or component, combined by `op`
+ /// into mask `mask`) is created by [`Session::segment_poll`] when something matches.
+ pub(crate) fn segment_text_later(&mut self, id: PhotoId, mask: Option<u32>, op: &str, name: Option<String>, text: &str) -> Result<(), String> {
+ self.segmenter.model_dir()?;
+ let text = text.trim();
+ if text.is_empty() {
+ return Err("describe what to select".into());
+ }
+ #[cfg(feature = "sam")]
+ {
+ let tag = Tag::NewPrompt { photo: id, mask, op: op.to_string(), name, text: text.to_string() };
+ self.segment_queue(id, worker::Kind::Text(text.to_string()), tag, None)
+ }
+ #[cfg(not(feature = "sam"))]
+ {
+ let _ = (id, mask, op, name);
+ Err("AI masks are not available in this build".into())
+ }
+ }
+
+ /// Start a zoomed-in pass for component `comp` of mask `mask` on photo `id` (an Object or
+ /// Describe selection): the photo around the selection is rendered at a higher resolution
+ /// and segmented again. In background mode [`Session::segment_poll`] applies the result;
+ /// otherwise it is applied before this returns. `Ok(false)` when there is nothing to
+ /// refine (no selection yet, or it is most of the photo) or a pass is running already.
+ pub fn segment_detail(&mut self, id: PhotoId, mask: u32, comp: usize) -> Result<bool, String> {
+ self.segmenter.model_dir()?;
+ let d = self.develop_of(id).ok_or("no such photo")?;
+ let shape = d.masks.iter().find(|m| m.id == mask).and_then(|m| m.components.get(comp)).map(|c| c.shape.clone()).ok_or("no such mask")?;
+ let (seg, prompt) = match &shape {
+ // (a document with absurdly many clicks isn't run through the model)
+ MaskShape::Object { hint, exclude, seg: Some(seg), .. } if !hint.is_empty() && hint.len() + exclude.len() <= MAX_CLICKS => {
+ (seg.clone(), Prompt::Clicks(clicks_of(hint, exclude)))
+ }
+ MaskShape::Prompt { text, seg: Some(seg), .. } => (seg.clone(), Prompt::Text(text.clone())),
+ _ => return Ok(false),
+ };
+ let p = self.catalog.photo(id).ok_or("no such photo")?;
+ let (pw, ph) = (f64::from(p.width.max(1)), f64::from(p.height.max(1)));
+ let aspect = if d.orientation.swaps_axes() { ph / pw } else { pw / ph };
+ let Some(region) = detail_region(&seg, aspect) else { return Ok(false) };
+ #[cfg(feature = "sam")]
+ {
+ let kind = worker::Kind::Detail { prompt, region };
+ let tag = Tag::Detail { photo: id, mask, comp, shape: strip_detail(shape) };
+ if self.segmenter.background {
+ if self.segmenter.detail_busy() {
+ return Ok(false);
+ }
+ self.segment_queue(id, kind, tag, Some(region))?;
+ } else {
+ let patch = self.segment_wait(id, kind, Some(region))?;
+ let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false };
+ let mut polled = Polled::default();
+ self.apply_outcome(outcome, &mut polled);
+ }
+ Ok(true)
+ }
+ #[cfg(not(feature = "sam"))]
+ {
+ let _ = (prompt, region, mask);
+ Ok(false)
+ }
+ }
+
+ /// Apply finished background requests (to masks that haven't changed since) and collect
+ /// their messages. Cheap when nothing is pending: call it every frame.
+ pub fn segment_poll(&mut self) -> Polled {
+ #[cfg_attr(not(feature = "sam"), allow(unused_mut))]
+ let mut polled = Polled { messages: std::mem::take(&mut self.segmenter.messages), ..Default::default() };
+ #[cfg(feature = "sam")]
+ {
+ // (before draining: a worker that is idle now has sent everything)
+ let idle = !self.segmenter.busy();
+ let done: Vec<worker::Outcome> = self.segmenter.results.1.try_iter().collect();
+ for o in done {
+ self.apply_outcome(o, &mut polled);
+ }
+ // clicks whose request was lost (the worker restarted) aren't pending forever
+ if idle {
+ self.segmenter.pending = None;
+ }
+ }
+ polled
+ }
+
+ #[cfg(feature = "sam")]
+ fn apply_outcome(&mut self, o: worker::Outcome, polled: &mut Polled) {
+ if o.superseded {
+ return;
+ }
+ match o.tag {
+ Tag::Wait => {}
+ Tag::Prepare => {
+ if let Err(e) = o.result {
+ polled.messages.push(e);
+ }
+ }
+ Tag::Clicks { photo, mask, comp, hint, exclude } => {
+ let latest = PendingClicks { photo, mask, comp, hint, exclude };
+ if self.segmenter.pending.as_ref() != Some(&latest) {
+ return; // newer clicks are on their way
+ }
+ self.segmenter.pending = None;
+ let seg = match o.result {
+ Ok(Some(seg)) => seg,
+ Ok(None) => return,
+ Err(e) => return polled.messages.push(e),
+ };
+ let Some(d) = self.develop_of(photo) else { return };
+ let mut d = (*d).clone();
+ let Some(c) = d.masks.iter_mut().find(|m| m.id == mask).and_then(|m| m.components.get_mut(comp)) else { return };
+ let MaskShape::Object { edge, .. } = c.shape else { return };
+ c.shape = MaskShape::Object { hint: latest.hint, exclude: latest.exclude, seg: Some(seg), detail: vec![], edge };
+ match self.set_develop(photo, d, "Object Mask") {
+ Ok(()) => {
+ polled.changed = true;
+ polled.refine = Some(mask);
+ }
+ Err(e) => polled.messages.push(e.to_string()),
+ }
+ }
+ Tag::NewPrompt { photo, mask, op, name, text } => {
+ let seg = match o.result {
+ Ok(Some(seg)) => seg,
+ Ok(None) => return polled.messages.push(format!("Nothing matching “{text}” was found in this photo.")),
+ Err(e) => return polled.messages.push(e),
+ };
+ if self.active() != Some(photo) {
+ return polled.messages.push(format!("The photo changed before “{text}” was found; try again."));
+ }
+ let Ok(seg) = serde_json::to_value(seg) else { return };
+ let mut p = serde_json::json!({"kind": "prompt", "text": text, "seg": seg});
+ if let Some(n) = name {
+ p["name"] = serde_json::json!(n);
+ }
+ let r = match mask {
+ None => self.execute("mask.add", &p),
+ Some(m) => {
+ if !self.develop_of(photo).is_some_and(|d| d.masks.iter().any(|x| x.id == m)) {
+ return polled.messages.push("The mask was removed before the selection was found.".into());
+ }
+ self.active_mask = Some(m);
+ p["op"] = serde_json::json!(op);
+ self.execute("mask.addComponent", &p)
+ }
+ };
+ match r {
+ Ok(_) => {
+ polled.changed = true;
+ polled.refine = self.active_mask;
+ }
+ Err(e) => polled.messages.push(e.to_string()),
+ }
+ }
+ Tag::Detail { photo, mask, comp, shape } => {
+ let patch = match o.result {
+ Ok(Some(p)) => p,
+ Ok(None) => return,
+ Err(e) => return log::warn!("SAM 3 detail pass: {e}"),
+ };
+ let Some(d) = self.develop_of(photo) else { return };
+ let mut d = (*d).clone();
+ let Some(c) = d.masks.iter_mut().find(|m| m.id == mask).and_then(|m| m.components.get_mut(comp)) else { return };
+ if strip_detail(c.shape.clone()) != shape {
+ return; // clicked again meanwhile: a newer pass will follow
+ }
+ match &mut c.shape {
+ MaskShape::Object { detail, .. } | MaskShape::Prompt { detail, .. } => *detail = vec![patch],
+ _ => return,
+ }
+ match self.set_develop(photo, d, "AI Mask Detail") {
+ Ok(()) => polled.changed = true,
+ Err(e) => log::warn!("AI mask detail: {e}"),
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask {
+ let l: Vec<f32> =
+ (0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect();
+ SegMask::from_logits(side, &l)
+ }
+
+ #[test]
+ fn detail_region_frames_the_selection_with_a_margin() {
+ // a small object in the middle of a square photo
+ let r = detail_region(&seg_box(100, 40, 45, 50, 55), 1.0).unwrap();
+ assert!(r[0] < 0.4 && r[2] > 0.5 && r[1] < 0.45 && r[3] > 0.55, "contains it: {r:?}");
+ assert!(r[2] - r[0] < 0.25 && r[3] - r[1] < 0.25, "zoomed in: {r:?}");
+ assert!(r.iter().all(|v| (0.0..=1.0).contains(v)));
+ // at the corner: shifted inside the photo, not cut
+ let r = detail_region(&seg_box(100, 0, 0, 5, 5), 1.5).unwrap();
+ assert!(r[0] >= 0.0 && r[1] >= 0.0 && r[0] < 1e-9 && r[1] < 1e-9);
+ // a thin object stays at most 2:1 (in photo pixels, 1.5:1 photo)
+ let r = detail_region(&seg_box(100, 10, 50, 90, 52), 1.5).unwrap();
+ let (w, h) = ((r[2] - r[0]) * 1.5, r[3] - r[1]);
+ assert!(w / h <= 2.0 + 1e-9, "{w} × {h}");
+ // a hostile aspect ratio stays sane
+ assert!(detail_region(&seg_box(100, 40, 45, 50, 55), f64::NAN).is_some_and(|r| r.iter().all(|v| v.is_finite())));
+ }
+
+ #[test]
+ fn no_region_for_nothing_or_most_of_the_photo() {
+ assert!(detail_region(&seg_box(50, 0, 0, 0, 0), 1.0).is_none());
+ assert!(detail_region(&seg_box(50, 2, 2, 48, 48), 1.0).is_none());
+ let damaged = SegMask { side: 50, data: "?".into(), rect: None };
+ assert!(detail_region(&damaged, 1.0).is_none());
+ }
+}
diff --git a/crates/engine/src/segment/worker.rs b/crates/engine/src/segment/worker.rs
new file mode 100644
index 000000000..9ded0fbe0
--- /dev/null
+++ b/crates/engine/src/segment/worker.rs
@@ -0,0 +1,426 @@
+//! The SAM 3 worker thread: it owns the model and the last encoded photo, and runs every
+//! request (load, render the model input, encode, clicks, descriptions, detail passes) in
+//! turn. The session only sends jobs and receives outcomes over channels, so no thread ever
+//! waits on a lock the model holds, and the UI thread never runs the model.
+//!
+//! Every job runs under `catch_unwind`: a panic inside candle becomes an error outcome, drops
+//! the (possibly inconsistent) model and leaves the worker ready for the next job. The busy
+//! counters are decremented by drop guards, so they can't stay stuck. The model is unloaded
+//! after [`IDLE_UNLOAD`] without requests (it holds several GB).
+
+use std::collections::VecDeque;
+use std::panic::AssertUnwindSafe;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::mpsc::{self, RecvTimeoutError};
+use std::time::Duration;
+
+use lightcraft_develop::SegMask;
+use lightcraft_segment::{Encoded, Sam3};
+
+use super::{Input, Prompt, Tag};
+use crate::media::RenderJob;
+
+/// Unload the model after this long without a request.
+pub const IDLE_UNLOAD: Duration = Duration::from_secs(10 * 60);
+
+/// What to compute.
+pub enum Kind {
+ /// Load the model and encode the photo (so the first click is fast).
+ Prepare,
+ Clicks(Vec<lightcraft_segment::Click>),
+ Text(String),
+ /// A zoomed-in pass over `region` (normalized) of the photo, rendered by the job's `render`.
+ Detail {
+ prompt: Prompt,
+ region: [f64; 4],
+ },
+ /// Panics (tests of the worker's recovery).
+ #[cfg(test)]
+ Panic,
+}
+
+/// One request.
+pub struct Job {
+ /// The model folder.
+ pub dir: PathBuf,
+ /// Which photo and look the request is about (the encode cache's key).
+ pub key: u64,
+ /// Renders the model input when the cache doesn't hold `key` (always for a detail pass).
+ pub render: Option<RenderJob>,
+ /// The photo's file, when it is missing on disk (for a clearer error if rendering fails).
+ pub missing: Option<String>,
+ pub kind: Kind,
+ pub tag: Tag,
+ pub reply: mpsc::Sender<Outcome>,
+}
+
+/// A finished request: the segmentation (`None`: nothing selected / nothing to report).
+pub struct Outcome {
+ pub tag: Tag,
+ pub result: Result<Option<SegMask>, String>,
+ /// A newer request for the same selection came in; this one was skipped.
+ pub superseded: bool,
+}
+
+/// Counters the session reads without locking.
+#[derive(Default)]
+pub struct Shared {
+ /// Requests other than detail passes, queued or running.
+ pub pending: AtomicUsize,
+ /// Detail passes queued or running.
+ pub detail: AtomicUsize,
+ /// The model is in memory.
+ pub loaded: AtomicBool,
+ /// Loading the model or encoding a photo right now (the slow parts).
+ pub encoding: AtomicBool,
+}
+
+/// Decrements a counter when dropped (also when unwinding).
+struct Done<'a>(&'a AtomicUsize);
+
+impl Drop for Done<'_> {
+ fn drop(&mut self) {
+ let _ = self.0.try_update(Ordering::SeqCst, Ordering::SeqCst, |n| Some(n.saturating_sub(1)));
+ }
+}
+
+/// Sets a flag while alive.
+struct Flag<'a>(&'a AtomicBool);
+
+impl<'a> Flag<'a> {
+ fn raise(f: &'a AtomicBool) -> Self {
+ f.store(true, Ordering::SeqCst);
+ Flag(f)
+ }
+}
+
+impl Drop for Flag<'_> {
+ fn drop(&mut self) {
+ self.0.store(false, Ordering::SeqCst);
+ }
+}
+
+/// The session's handle on the worker thread (started on the first request, restarted if it
+/// ever ends).
+#[derive(Default)]
+pub struct Worker {
+ tx: Option<mpsc::Sender<Job>>,
+ thread: Option<std::thread::JoinHandle<()>>,
+ pub shared: Arc<Shared>,
+}
+
+impl Worker {
+ fn counter(&self, job: &Job) -> &AtomicUsize {
+ if matches!(job.kind, Kind::Detail { .. }) { &self.shared.detail } else { &self.shared.pending }
+ }
+
+ /// Whether the thread has ended (it never should); its counters are then meaningless.
+ fn dead(&self) -> bool {
+ self.thread.as_ref().is_some_and(|t| t.is_finished())
+ }
+
+ pub fn pending(&self) -> usize {
+ if self.dead() { 0 } else { self.shared.pending.load(Ordering::SeqCst) }
+ }
+
+ pub fn detail(&self) -> usize {
+ if self.dead() { 0 } else { self.shared.detail.load(Ordering::SeqCst) }
+ }
+
+ pub fn encoding(&self) -> bool {
+ !self.dead() && self.shared.encoding.load(Ordering::SeqCst)
+ }
+
+ pub fn loaded(&self) -> bool {
+ !self.dead() && self.shared.loaded.load(Ordering::SeqCst)
+ }
+
+ /// Queue `job`; returns at once.
+ pub fn submit(&mut self, job: Job) -> Result<(), String> {
+ if self.dead() {
+ // a thread that ended took its counters' meaning with it
+ self.tx = None;
+ self.thread = None;
+ self.shared = Arc::new(Shared::default());
+ }
+ self.counter(&job).fetch_add(1, Ordering::SeqCst);
+ let job = match &self.tx {
+ Some(tx) => match tx.send(job) {
+ Ok(()) => return Ok(()),
+ Err(mpsc::SendError(job)) => job,
+ },
+ None => job,
+ };
+ // (re)start the thread
+ let (tx, rx) = mpsc::channel();
+ let shared = self.shared.clone();
+ match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD)) {
+ Ok(t) => {
+ self.thread = Some(t);
+ self.tx = Some(tx.clone());
+ tx.send(job).map_err(|_| "the AI mask worker stopped".to_string())
+ }
+ Err(e) => {
+ let _ = self.counter(&job).try_update(Ordering::SeqCst, Ordering::SeqCst, |n| Some(n.saturating_sub(1)));
+ Err(format!("could not start the AI mask worker: {e}"))
+ }
+ }
+ }
+}
+
+/// The model and the last encoded photo.
+#[derive(Default)]
+struct State {
+ model: Option<(PathBuf, Sam3)>,
+ cache: Option<(u64, Encoded)>,
+}
+
+/// Whether `later` makes `job` pointless (it asks for the same selection again).
+fn supersedes(later: &Job, job: &Job) -> bool {
+ match (&later.tag, &job.tag) {
+ (Tag::Prepare, Tag::Prepare) => later.key == job.key,
+ (Tag::Clicks { photo: a, mask: am, comp: ac, .. }, Tag::Clicks { photo: b, mask: bm, comp: bc, .. }) => (a, am, ac) == (b, bm, bc),
+ (Tag::Detail { photo: a, mask: am, comp: ac, .. }, Tag::Detail { photo: b, mask: bm, comp: bc, .. }) => (a, am, ac) == (b, bm, bc),
+ _ => false,
+ }
+}
+
+fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
+ let mut state = State::default();
+ let mut queue: VecDeque<Job> = VecDeque::new();
+ loop {
+ if queue.is_empty() {
+ match rx.recv_timeout(idle) {
+ Ok(j) => queue.push_back(j),
+ Err(RecvTimeoutError::Timeout) => {
+ if state.model.is_some() {
+ state = State::default();
+ shared.loaded.store(false, Ordering::SeqCst);
+ log::info!("SAM 3 unloaded after {} minutes without use", idle.as_secs() / 60);
+ }
+ continue;
+ }
+ Err(RecvTimeoutError::Disconnected) => return,
+ }
+ }
+ while let Ok(j) = rx.try_recv() {
+ queue.push_back(j);
+ }
+ let Some(mut job) = queue.pop_front() else { continue };
+ let is_detail = matches!(job.kind, Kind::Detail { .. });
+ let _done = Done(if is_detail { &shared.detail } else { &shared.pending });
+ if queue.iter().any(|later| supersedes(later, &job)) {
+ let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true });
+ continue;
+ }
+ let result = match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(&mut state, shared, &mut job))) {
+ Ok(r) => r,
+ Err(p) => {
+ let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_default();
+ log::error!("SAM 3 panicked: {why}");
+ // don't trust what the model kept: load it again next time
+ state = State::default();
+ Err("the AI model failed unexpectedly (it will be reloaded on the next try)".to_string())
+ }
+ };
+ if result.is_err() && !is_detail {
+ state.cache = None;
+ }
+ shared.loaded.store(state.model.is_some(), Ordering::SeqCst);
+ let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false });
+ }
+}
+
+/// The model for `dir`, loading it when needed.
+fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path) -> Result<&'a mut Sam3, String> {
+ if state.model.as_ref().is_none_or(|(d, _)| d != dir) {
+ state.model = None;
+ state.cache = None;
+ let _busy = Flag::raise(&shared.encoding);
+ let t = web_time::Instant::now();
+ let m = Sam3::load(dir).map_err(|e| e.to_string())?;
+ log::info!("SAM 3 loaded in {:?}", t.elapsed());
+ state.model = Some((dir.to_path_buf(), m));
+ shared.loaded.store(true, Ordering::SeqCst);
+ }
+ state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".to_string())
+}
+
+/// Render the model input.
+fn render(job: &mut Job) -> Result<Input, String> {
+ let r = job.render.take().ok_or("the photo is not available")?.run();
+ let img = match r.rendered {
+ Ok(r) => r.image,
+ Err(e) => {
+ return Err(match &job.missing {
+ Some(path) => format!(
+ "This photo's file is missing (moved or deleted), so it can't be analyzed for AI masks: {path}. Library ▸ Find Missing Photos can relink it."
+ ),
+ None => format!("couldn't render the photo for AI masks: {e}"),
+ });
+ }
+ };
+ let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect();
+ Ok(Input { rgb, w: img.width, h: img.height })
+}
+
+/// The model and the encoded photo `job.key` (rendering and encoding it when needed).
+fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(&'a mut Sam3, &'a mut Encoded), String> {
+ model(state, shared, &job.dir)?;
+ if state.cache.as_ref().is_none_or(|(k, _)| *k != job.key) {
+ state.cache = None;
+ let input = render(job)?;
+ let _busy = Flag::raise(&shared.encoding);
+ let m = state.model.as_mut().map(|(_, m)| m).ok_or("the model did not load")?;
+ let t = web_time::Instant::now();
+ let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
+ m.prepare_clicks(&mut enc).map_err(|e| e.to_string())?;
+ log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());
+ state.cache = Some((job.key, enc));
+ }
+ let State { model, cache } = state;
+ match (model.as_mut(), cache.as_mut()) {
+ (Some((_, m)), Some((_, e))) => Ok((m, e)),
+ _ => Err("the model is not ready".into()),
+ }
+}
+
+/// Every phrase of `text` ("car, road"), merged (the per-pixel maximum); `None` when nothing
+/// matches.
+fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result<Option<Vec<f32>>, String> {
+ let mut merged: Option<Vec<f32>> = None;
+ for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
+ if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {
+ merged = Some(match merged {
+ None => p.logits,
+ Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(),
+ });
+ }
+ }
+ Ok(merged)
+}
+
+fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<SegMask>, String> {
+ let side = lightcraft_segment::MASK_SIDE;
+ match std::mem::replace(&mut job.kind, Kind::Prepare) {
+ Kind::Prepare => {
+ encoded(state, shared, job)?;
+ Ok(None)
+ }
+ Kind::Clicks(clicks) => {
+ let (m, enc) = encoded(state, shared, job)?;
+ let p = m.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?;
+ Ok(Some(SegMask::from_logits(side, &p.logits)))
+ }
+ Kind::Text(text) => {
+ let (m, enc) = encoded(state, shared, job)?;
+ Ok(text_logits(m, enc, &text)?.map(|l| SegMask::from_logits(side, &l)))
+ }
+ #[cfg(test)]
+ #[allow(clippy::panic)]
+ Kind::Panic => panic!("a test panic"),
+ Kind::Detail { prompt, region } => {
+ let input = render(job)?;
+ let m = model(state, shared, &job.dir)?;
+ let t = web_time::Instant::now();
+ let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
+ let (rw, rh) = (region[2] - region[0], region[3] - region[1]);
+ if !(rw > 0.0 && rh > 0.0) {
+ return Ok(None);
+ }
+ let logits = match prompt {
+ Prompt::Clicks(clicks) => {
+ let inside: Vec<lightcraft_segment::Click> = clicks
+ .iter()
+ .map(|c| ((c.at.x - region[0]) / rw, (c.at.y - region[1]) / rh, c.include))
+ .filter(|(x, y, _)| (0.0..=1.0).contains(x) && (0.0..=1.0).contains(y))
+ .map(|(x, y, positive)| lightcraft_segment::Click { x: x as f32, y: y as f32, positive })
+ .collect();
+ if !inside.iter().any(|c| c.positive) {
+ return Ok(None);
+ }
+ Some(m.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits)
+ }
+ Prompt::Text(text) => text_logits(m, &mut enc, &text)?,
+ };
+ log::info!("SAM 3 detail pass ({}×{}) in {:?}", input.w, input.h, t.elapsed());
+ Ok(logits.map(|l| SegMask::from_logits_in(side, &l, region)))
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn job(tag: Tag, reply: &mpsc::Sender<Outcome>) -> Job {
+ // an empty folder: loading fails with an error (no model files)
+ let dir = std::env::temp_dir().join(format!("lc-sam3-worker-none-{}", std::process::id()));
+ Job { dir, key: 1, render: None, missing: None, kind: Kind::Prepare, tag, reply: reply.clone() }
+ }
+
+ #[test]
+ fn errors_come_back_and_the_counters_return_to_zero() {
+ let mut w = Worker::default();
+ let (tx, rx) = mpsc::channel();
+ for _ in 0..3 {
+ w.submit(job(Tag::Wait, &tx)).unwrap();
+ }
+ for _ in 0..3 {
+ let o = rx.recv_timeout(Duration::from_secs(30)).unwrap();
+ assert!(o.result.unwrap_err().contains("not found"));
+ }
+ let t = web_time::Instant::now();
+ while w.pending() > 0 && t.elapsed() < Duration::from_secs(5) {
+ std::thread::yield_now();
+ }
+ assert_eq!(w.pending(), 0);
+ assert!(!w.encoding() && !w.loaded());
+ }
+
+ #[test]
+ fn a_panic_in_a_job_is_an_error_and_the_worker_carries_on() {
+ let shared = Shared::default();
+ let (tx, rx) = mpsc::channel();
+ let (jtx, jrx) = mpsc::channel();
+ let mut j = job(Tag::Wait, &tx);
+ j.kind = Kind::Panic;
+ jtx.send(j).unwrap();
+ jtx.send(job(Tag::Wait, &tx)).unwrap();
+ shared.pending.store(2, Ordering::SeqCst);
+ drop(jtx);
+ run(&jrx, &shared, Duration::from_secs(60));
+ let first = rx.recv().unwrap();
+ assert!(first.result.unwrap_err().contains("failed unexpectedly"));
+ let second = rx.recv().unwrap();
+ assert!(second.result.is_err(), "the next job still runs");
+ assert_eq!(shared.pending.load(Ordering::SeqCst), 0, "counters reset");
+ assert!(!shared.encoding.load(Ordering::SeqCst));
+ }
+
+ #[test]
+ fn a_newer_request_for_the_same_selection_supersedes_a_queued_one() {
+ let (tx, rx) = mpsc::channel();
+ let (jtx, jrx) = mpsc::channel();
+ let clicks = |n: usize| Tag::Clicks {
+ photo: lightcraft_catalog::PhotoId(1),
+ mask: 1,
+ comp: 0,
+ hint: vec![lightcraft_geom::Point::new(0.5, 0.5); n],
+ exclude: vec![],
+ };
+ for n in 1..=3 {
+ jtx.send(job(clicks(n), &tx)).unwrap();
+ }
+ drop(jtx);
+ let shared = Shared::default();
+ shared.pending.store(3, Ordering::SeqCst);
+ run(&jrx, &shared, Duration::from_secs(60));
+ let out: Vec<Outcome> = rx.try_iter().collect();
+ assert_eq!(out.len(), 3);
+ assert!(out[0].superseded && out[1].superseded && !out[2].superseded);
+ assert_eq!(shared.pending.load(Ordering::SeqCst), 0);
+ }
+}
diff --git a/crates/engine/src/tests_segment.rs b/crates/engine/src/tests_segment.rs
new file mode 100644
index 000000000..59aee18c3
--- /dev/null
+++ b/crates/engine/src/tests_segment.rs
@@ -0,0 +1,232 @@
+//! AI masks without the SAM 3 model (it is never required): every request is a clear error that
+//! comes back at once, nothing waits on the network or the model, a damaged model is an error
+//! (not a crash or a stuck "busy"), and the download command needs consent and fails cleanly.
+//! No test touches the internet: downloads go to a local server.
+
+use std::time::{Duration, Instant};
+
+use serde_json::json;
+
+use crate::Session;
+use crate::segment::{NOT_INSTALLED, Segmenter};
+
+fn tmp(name: &str) -> std::path::PathBuf {
+ let d = std::env::temp_dir().join(format!("lc-sam3-engine-{name}-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&d);
+ d
+}
+
+/// A folder that looks installed but holds no usable model.
+fn damaged_model(name: &str) -> std::path::PathBuf {
+ let d = tmp(name);
+ std::fs::create_dir_all(&d).unwrap();
+ std::fs::write(d.join("model.safetensors"), b"\x10\x00\x00\x00\x00\x00\x00\x00{not json at all}").unwrap();
+ std::fs::write(d.join("vocab.json"), b"{}").unwrap();
+ std::fs::write(d.join("merges.txt"), b"#version: 0.2\n").unwrap();
+ d
+}
+
+/// Poll until the worker is idle (or `limit` passes), collecting messages.
+fn settle(s: &mut Session, limit: Duration) -> Vec<String> {
+ let t = Instant::now();
+ let mut messages = Vec::new();
+ loop {
+ let p = s.segment_poll();
+ messages.extend(p.messages);
+ if !s.segmenter.busy() && !s.segmenter.detail_busy() {
+ messages.extend(s.segment_poll().messages);
+ return messages;
+ }
+ assert!(t.elapsed() < limit, "the worker never finished");
+ std::thread::sleep(Duration::from_millis(10));
+ }
+}
+
+#[test]
+fn status_and_model_commands_without_the_model() {
+ let mut s = Session::with_demo();
+ let st = s.execute("segment.model.status", &json!({})).unwrap();
+ assert_eq!(st["available"], Segmenter::AVAILABLE);
+ assert_eq!(st["installed"], false);
+ assert_eq!(st["download"]["running"], false);
+ assert!(st["license"].as_str().unwrap().contains("SAM License"));
+ // downloading needs the user's consent, and names the size and the licence
+ let e = s.execute("segment.model.download", &json!({})).unwrap_err().to_string();
+ assert!(e.contains("acknowledged") && e.contains("SAM License") && e.contains("GB"), "{e}");
+ let e = s.execute("segment.model.download", &json!({"acknowledged": "yes"})).unwrap_err().to_string();
+ assert!(e.contains("acknowledged"), "{e}");
+ assert_eq!(s.execute("segment.model.cancel", &json!({})).unwrap()["cancelled"], false);
+ if !Segmenter::AVAILABLE {
+ assert!(s.execute("segment.model.download", &json!({"acknowledged": true})).is_err());
+ return;
+ }
+ // no folder, then no download location: clear errors, nothing started
+ let e = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap_err().to_string();
+ assert!(e.contains("no folder"), "{e}");
+ let dir = tmp("nomirrors");
+ s.segmenter.dir = Some(dir.clone());
+ s.segmenter.mirrors_file = Some(dir.join("none.txt"));
+ if s.segmenter.mirrors().is_empty() {
+ let e = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap_err().to_string();
+ assert!(e.contains("LIGHTCRAFT_SAM3_MIRRORS"), "{e}");
+ }
+ assert!(!s.segmenter.download_status().running);
+}
+
+#[cfg(feature = "sam")]
+#[test]
+fn a_failing_download_ends_with_an_error_and_never_blocks() {
+ use std::io::{BufRead, BufReader, Write};
+ // a local mirror that has nothing
+ let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let base = format!("http://{}/sam3", l.local_addr().unwrap());
+ std::thread::spawn(move || {
+ for mut c in l.incoming().flatten() {
+ let mut r = BufReader::new(c.try_clone().unwrap());
+ let mut line = String::new();
+ while r.read_line(&mut line).unwrap_or(0) > 2 {
+ line.clear();
+ }
+ let _ = c.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 13\r\nConnection: close\r\n\r\n<h1>404</h1>\n");
+ }
+ });
+ let dir = tmp("download404");
+ let mirrors = dir.with_extension("mirrors.txt");
+ std::fs::write(&mirrors, format!("# test mirror\n{base}\n")).unwrap();
+ let mut s = Session::with_demo();
+ s.segmenter.dir = Some(dir.clone());
+ s.segmenter.mirrors_file = Some(mirrors.clone());
+ let t = Instant::now();
+ let r = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap();
+ assert_eq!(r["started"], true);
+ assert!(t.elapsed() < Duration::from_secs(1), "starting a download returns at once");
+ // a second request while it runs starts nothing
+ let again = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap();
+ assert!(again["started"] == false || again["downloading"] == false);
+ let t = Instant::now();
+ while s.segmenter.download_status().running {
+ assert!(t.elapsed() < Duration::from_secs(30), "the download never ended");
+ // AI mask requests meanwhile: an error at once, never a wait
+ let q = Instant::now();
+ let e = s.execute("mask.add", &json!({"kind": "prompt", "text": "sky"})).unwrap_err().to_string();
+ assert!(e.contains(NOT_INSTALLED), "{e}");
+ assert!(q.elapsed() < Duration::from_millis(500));
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ let st = s.segmenter.download_status();
+ assert!(!st.finished);
+ let e = st.error.unwrap();
+ assert!(e.contains("not found"), "{e}");
+ assert!(!s.segmenter.installed());
+ // nothing was left behind under a real name, and no error page was saved
+ for f in ["model.safetensors", "vocab.json", "merges.txt", "vocab.json.part"] {
+ assert!(!dir.join(f).exists(), "{f}");
+ }
+ let _ = std::fs::remove_dir_all(&dir);
+ let _ = std::fs::remove_file(&mirrors);
+}
+
+/// The app (background mode) never waits: with the model missing every request fails at once;
+/// with a damaged model the requests are queued, fail on the worker, and come back as messages.
+#[test]
+fn background_requests_never_wait_and_failures_come_back_as_messages() {
+ let mut s = Session::with_demo();
+ s.segmenter.background = true;
+ let id = s.active().unwrap();
+ s.segmenter.dir = Some(tmp("missing"));
+ for (cmd, p) in [("mask.add", json!({"kind": "object"})), ("mask.add", json!({"kind": "prompt", "text": "sky"})), ("segment.prepare", json!({}))]
+ {
+ let t = Instant::now();
+ let e = s.execute(cmd, &p).unwrap_err().to_string();
+ assert!(t.elapsed() < Duration::from_millis(500), "{cmd}");
+ if Segmenter::AVAILABLE {
+ assert!(e.starts_with(NOT_INSTALLED), "{cmd}: {e}");
+ } else {
+ assert!(e.contains("not available"), "{cmd}: {e}");
+ }
+ }
+ assert!(s.develop_of(id).unwrap().masks.is_empty());
+ if !Segmenter::AVAILABLE {
+ return;
+ }
+ // a model folder whose weights are damaged
+ let dir = damaged_model("damaged");
+ s.segmenter.dir = Some(dir.clone());
+ assert!(s.segmenter.installed());
+ let t = Instant::now();
+ s.execute("mask.add", &json!({"kind": "object"})).unwrap();
+ let r = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap();
+ assert_eq!(r["pending"], true);
+ assert!(s.segmenter.pending_clicks().is_some_and(|p| p.hint.len() == 1));
+ // a second click while the first is on its way builds on it
+ let r = s.execute("mask.objectPoint", &json!({"x": 0.6, "y": 0.5, "exclude": true})).unwrap();
+ assert_eq!((r["include"].as_u64(), r["exclude"].as_u64()), (Some(1), Some(1)));
+ let r = s.execute("mask.add", &json!({"kind": "prompt", "text": "the sky"})).unwrap();
+ assert_eq!(r["pending"], true);
+ assert!(t.elapsed() < Duration::from_secs(1), "nothing waited for the model: {:?}", t.elapsed());
+ let messages = settle(&mut s, Duration::from_secs(60));
+ assert!(!messages.is_empty() && messages.iter().all(|m| !m.is_empty()), "{messages:?}");
+ assert!(messages.iter().any(|m| m.contains("model.safetensors") || m.contains("SAM 3")), "{messages:?}");
+ assert!(s.segmenter.pending_clicks().is_none());
+ // the Object mask is there, without a selection; no Describe mask was made from nothing
+ let d = s.develop_of(id).unwrap();
+ assert_eq!(d.masks.len(), 1);
+ assert!(matches!(&d.masks[0].components[0].shape, lightcraft_develop::MaskShape::Object { seg: None, .. }));
+ assert!(!s.segmenter.busy() && !s.segmenter.loaded());
+ let _ = std::fs::remove_dir_all(&dir);
+}
+
+/// CLI / MCP (waiting mode): a damaged model is an error from the command, quickly.
+#[test]
+fn waiting_requests_with_a_damaged_model_are_errors() {
+ if !Segmenter::AVAILABLE {
+ return;
+ }
+ let mut s = Session::with_demo();
+ let id = s.active().unwrap();
+ let dir = damaged_model("damaged-wait");
+ s.segmenter.dir = Some(dir.clone());
+ let e = s.execute("mask.add", &json!({"kind": "prompt", "text": "sky"})).unwrap_err().to_string();
+ assert!(e.contains("model.safetensors") || e.contains("SAM 3"), "{e}");
+ let e = s.execute("mask.add", &json!({"kind": "object", "points": [[0.5, 0.5]]})).unwrap_err().to_string();
+ assert!(!e.is_empty());
+ assert!(s.develop_of(id).unwrap().masks.is_empty());
+ assert!(!s.segmenter.busy());
+ // a stored segmentation needs no model at all (replayed or pasted masks)
+ let seg = serde_json::to_value(lightcraft_develop::SegMask::from_logits(4, &[5.0; 16])).unwrap();
+ s.segmenter.dir = None;
+ s.execute("mask.add", &json!({"kind": "prompt", "text": "sky", "seg": seg})).unwrap();
+ assert_eq!(s.develop_of(id).unwrap().masks.len(), 1);
+ let _ = std::fs::remove_dir_all(&dir);
+}
+
+/// Too many clicks on one Object selection are refused, not sent to the model.
+#[test]
+fn object_clicks_are_capped() {
+ if !Segmenter::AVAILABLE {
+ return;
+ }
+ let mut s = Session::with_demo();
+ s.segmenter.background = true;
+ let dir = damaged_model("clicks");
+ s.segmenter.dir = Some(dir.clone());
+ let id = s.active().unwrap();
+ let mut d = (*s.develop_of(id).unwrap()).clone();
+ let many = vec![lightcraft_geom::Point::new(0.5, 0.5); crate::segment::MAX_CLICKS];
+ d.masks.push(lightcraft_develop::Mask {
+ id: 1,
+ components: vec![lightcraft_develop::MaskComponent {
+ name: None,
+ op: lightcraft_develop::MaskOp::Add,
+ invert: false,
+ shape: lightcraft_develop::MaskShape::Object { hint: many, exclude: vec![], seg: None, detail: vec![], edge: 0.0 },
+ }],
+ ..Default::default()
+ });
+ s.set_develop(id, d, "test").unwrap();
+ s.active_mask = Some(1);
+ let e = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap_err().to_string();
+ assert!(e.contains("at most"), "{e}");
+ assert!(!s.segmenter.busy());
+ let _ = std::fs::remove_dir_all(&dir);
+}
diff --git a/crates/pipeline/src/masks.rs b/crates/pipeline/src/masks.rs
index 01985b5b2..a71667bc0 100644
--- a/crates/pipeline/src/masks.rs
+++ b/crates/pipeline/src/masks.rs
@@ -228,7 +228,8 @@ impl SegGrid {
/// data selects nothing.
fn sample_seg(seg: Option<&SegMask>, detail: &[SegMask], gain: f32, frame: &Frame, w: usize, h: usize, out: &mut Plane) {
let coarse = seg.and_then(SegGrid::new);
- let patches: Vec<SegGrid> = detail.iter().filter_map(SegGrid::new).collect();
+ // (capped: a hostile document can hold many)
+ let patches: Vec<SegGrid> = detail.iter().take(lightcraft_develop::segmask::MAX_DETAIL).filter_map(SegGrid::new).collect();
if coarse.is_none() && patches.is_empty() {
return;
}
@@ -466,6 +467,21 @@ mod tests {
assert!(a.get(190, 50) < 0.01, "inside the patch, its unselected half");
}
+ #[test]
+ fn a_hostile_number_of_detail_patches_is_capped() {
+ // built in memory (a deserialized document is capped already): only the first
+ // MAX_DETAIL are decoded and sampled, so this needs 4 × 4 MB, not 20 000 × 4 MB
+ let side = 1024;
+ let l: Vec<f32> = vec![12.0; side * side];
+ let patch = SegMask::from_logits_in(side, &l, [0.0, 0.0, 0.5, 1.0]);
+ let shape = MaskShape::Object { hint: vec![Point::new(0.2, 0.5)], exclude: vec![], seg: None, detail: vec![patch; 20_000], edge: 0.0 };
+ let (w, h) = (64, 32);
+ let t = std::time::Instant::now();
+ let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0);
+ assert!(a.get(5, 16) > 0.99 && a.get(60, 16) < 0.01);
+ assert!(t.elapsed() < std::time::Duration::from_secs(20), "{:?}", t.elapsed());
+ }
+
#[test]
fn the_edge_setting_hardens_or_feathers_the_transition() {
let side = 16;
diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs
index b592557a7..1b9d22bbb 100644
--- a/crates/ui-egui/src/control.rs
+++ b/crates/ui-egui/src/control.rs
@@ -278,7 +278,9 @@ pub fn handle(app: &mut LightcraftApp, ctx: &egui::Context, req: &ControlRequest
Some(d) => {
let r = crate::panels::dialogs::confirm_dialog(app, &d);
// the import review stays open on an error, as with its button
- if r.is_err() && matches!(d, crate::state::Dialog::Import { .. }) {
+ if (r.is_err() && matches!(d, crate::state::Dialog::Import { .. } | crate::state::Dialog::SamModel { .. }))
+ || (r.is_ok() && crate::panels::dialogs::keeps_open(app, &d))
+ {
app.ui.dialog = Some(d);
}
wrap(r)
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index bdc2780f3..49ea140f9 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -203,7 +203,9 @@ pub struct LightcraftApp {
}
impl LightcraftApp {
- pub fn new(session: Session, services: Services) -> Self {
+ pub fn new(mut session: Session, services: Services) -> Self {
+ // AI mask requests run on the model's worker; frames apply their results (never wait)
+ session.segmenter.background = true;
Self {
session,
ui: UiState::default(),
@@ -358,13 +360,42 @@ impl LightcraftApp {
self.ui.toast = Some((text.into(), t + secs));
}
- /// AI masks: start a zoomed-in detail pass once the clicking stops (`ui.detail_due`), and
- /// apply finished passes.
+ /// AI masks: apply finished background requests (clicks, descriptions, detail passes) and
+ /// show their errors; start a zoomed-in detail pass once the clicking stops
+ /// (`ui.detail_due`); watch the model download. Never waits for the model.
fn ai_mask_detail(&mut self, ctx: &egui::Context) {
- if self.session.segment_poll() {
+ let polled = self.session.segment_poll();
+ let now = ctx.input(|i| i.time);
+ if polled.changed {
ctx.request_repaint();
}
- let now = ctx.input(|i| i.time);
+ if let Some(mask) = polled.refine {
+ self.ui.detail_due = Some((now + 0.3, mask));
+ }
+ if let Some(e) = polled.messages.into_iter().last() {
+ self.ai_error(ctx, e, None);
+ }
+ let seg = &self.session.segmenter;
+ if seg.busy() || seg.pending_clicks().is_some() {
+ ctx.request_repaint_after(std::time::Duration::from_millis(50));
+ }
+ let download = seg.download_status();
+ if download.running {
+ ctx.request_repaint_after(std::time::Duration::from_millis(250));
+ }
+ // a download finishing while its dialog is closed: say so once
+ if self.ui.sam_downloading && !download.running {
+ self.ui.sam_downloading = false;
+ let open = matches!(self.ui.dialog, Some(state::Dialog::SamModel { .. }));
+ match (&download.error, open) {
+ (_, true) => {}
+ (Some(e), false) => self.toast_error(ctx, format!("The SAM 3 download failed: {e}")),
+ (None, false) if download.finished => {
+ self.toast_error(ctx, crate::i18n::tr("The SAM 3 model is installed: Object and Describe masks are ready."))
+ }
+ (None, false) => {}
+ }
+ }
if let Some((due, mask)) = self.ui.detail_due {
if now >= due && !self.session.segmenter.busy() && !self.session.segmenter.detail_busy() {
self.ui.detail_due = None;
@@ -380,6 +411,24 @@ impl LightcraftApp {
}
}
+ /// An AI mask error: when the model isn't installed, the dialog that offers to download it
+ /// (`then`: the AI mask to start afterwards); otherwise a toast.
+ pub fn ai_error(&mut self, ctx: &egui::Context, e: impl Into<String>, then: Option<(&str, &str)>) {
+ let e = e.into();
+ if lightcraft_engine::segment::Segmenter::AVAILABLE && e.starts_with(lightcraft_engine::segment::NOT_INSTALLED) {
+ self.offer_sam_download(then);
+ } else {
+ self.toast_error(ctx, e);
+ }
+ }
+
+ /// Open the dialog offering the SAM 3 download (never downloads by itself).
+ pub fn offer_sam_download(&mut self, then: Option<(&str, &str)>) {
+ if self.ui.dialog.is_none() || matches!(self.ui.dialog, Some(state::Dialog::SamModel { .. })) {
+ self.ui.dialog = Some(state::Dialog::SamModel { then: then.map(|(k, o)| (k.to_string(), o.to_string())), error: None });
+ }
+ }
+
/// A toast for an error the user has to read and act on (stays 6 s).
pub fn toast_error(&mut self, ctx: &egui::Context, text: impl Into<String>) {
let t = ctx.input(|i| i.time);
diff --git a/crates/ui-egui/src/panels/detail.rs b/crates/ui-egui/src/panels/detail.rs
index 67c01f478..3b3622bed 100644
--- a/crates/ui-egui/src/panels/detail.rs
+++ b/crates/ui-egui/src/panels/detail.rs
@@ -698,9 +698,6 @@ fn targeted_drag(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respon
}
}
-/// The colour of a mask hovered in the Masks list.
-const HOVER_RED: [u8; 3] = [230, 30, 40];
-
/// The diagnostic overlay the loupe shows (the selected mask, Point Color's visualized range,
/// Visualize Spots).
pub(crate) fn view_overlay(app: &LightcraftApp, d: &DevelopSettings) -> lightcraft_pipeline::Overlay {
@@ -708,13 +705,6 @@ pub(crate) fn view_overlay(app: &LightcraftApp, d: &DevelopSettings) -> lightcra
if app.ui.fullscreen {
return Overlay::None;
}
- // a mask hovered in the Masks list shows in red; the selected one only with the overlay on (O)
- if app.ui.right == RightPanel::Masking
- && let Some(m) = app.ui.hover_mask.and_then(|id| d.masks.iter().find(|m| m.id == id))
- && !m.components.is_empty()
- {
- return Overlay::Mask { id: m.id.min(u16::MAX as u32) as u16, view: MaskView::default(), color: HOVER_RED, opacity: 55 };
- }
if app.ui.right == RightPanel::Masking
&& app.ui.mask_overlay
&& let Some(m) = app.session.active_mask.and_then(|id| d.masks.iter().find(|m| m.id == id))
@@ -1183,9 +1173,15 @@ fn mask_overlay(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respons
}
}
// Object selections (SAM 3) of the selected mask: each click, green to include, red to exclude
+ // (clicks still on their way to the model included)
+ let pending = app.session.segmenter.pending_clicks().cloned();
for m in d.masks.iter().filter(|m| Some(m.id) == active) {
- for c in &m.components {
+ for (k, c) in m.components.iter().enumerate() {
if let MaskShape::Object { hint, exclude, .. } = &c.shape {
+ let (hint, exclude) = match pending.as_ref().filter(|q| (q.mask, q.comp) == (m.id, k)) {
+ Some(q) => (&q.hint, &q.exclude),
+ None => (hint, exclude),
+ };
for (pts, col) in [(hint, Color32::from_rgb(40, 200, 90)), (exclude, Color32::from_rgb(230, 60, 60))] {
for q in pts {
let q = map.screen(*q);
@@ -1212,7 +1208,7 @@ fn mask_overlay(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respons
app.ui.detail_due = Some((ui.input(|i| i.time) + 1.0, mid));
}
}
- Err(e) => app.toast_error(ui.ctx(), e),
+ Err(e) => app.ai_error(ui.ctx(), e, Some(("object", "new"))),
}
}
return;
diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs
index a9ecdcfe4..67d495deb 100644
--- a/crates/ui-egui/src/panels/dialogs.rs
+++ b/crates/ui-egui/src/panels/dialogs.rs
@@ -94,6 +94,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
Dialog::Merge { opts } => opts.title(),
Dialog::Settings { .. } => "Settings",
Dialog::ConfirmDelete { .. } => "Delete Photos",
+ Dialog::SamModel { .. } => "Download the SAM 3 Model?",
Dialog::About => "About LightCraft",
Dialog::Shortcuts => "Keyboard Shortcuts",
}
@@ -700,6 +701,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
Dialog::Merge { opts } => crate::merge::body(app, ui, opts),
Dialog::Import { opts } => crate::import::body(app, ui, opts),
Dialog::Settings { tab } => crate::panels::settings::body(app, ui, tab),
+ Dialog::SamModel { error, .. } => sam_model_body(app, ui, error.as_deref()),
Dialog::ConfirmDelete { count } => {
let what = if *count == 1 { crate::i18n::tr("this photo").to_string() } else { crate::i18n::tr_format!("these {count} photos", count = count) };
ui.label(crate::i18n::tr_format!("Move {what} to Recently Deleted?", what = what));
@@ -767,8 +769,19 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
ui.add_space(4.0);
ui.horizontal(|ui| {
let informational = matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. });
- if !informational && ui.button(crate::i18n::tr("Cancel")).clicked() {
- close = true;
+ let sam = &app.session.segmenter;
+ let (sam_installed, sam_running, sam_failed) = (sam.installed(), sam.download_status().running, sam.download_status().error.is_some());
+ let cancel = match &dlg {
+ Dialog::SamModel { .. } if sam_running || sam_installed => "Close",
+ Dialog::SamModel { .. } => "Not Now",
+ _ => "Cancel",
+ };
+ if !informational {
+ let r = ui.button(crate::i18n::tr(cancel));
+ crate::widgets::register(ui.ctx(), "button:dialogCancel", r.rect);
+ if r.clicked() {
+ close = true;
+ }
}
let add_label;
let ok = match &dlg {
@@ -780,10 +793,18 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
}
Dialog::Merge { .. } => "Merge",
Dialog::ConfirmDelete { .. } => "Delete",
+ Dialog::SamModel { then: Some(_), .. } if sam_installed => "Continue",
+ Dialog::SamModel { .. } if sam_installed || sam_running => "",
+ Dialog::SamModel { error, .. } if error.is_some() || sam_failed => "Try Again",
+ Dialog::SamModel { .. } => "Download",
_ if informational => "Close",
_ => "OK",
};
- if ui.button(crate::i18n::tr(ok)).clicked() {
+ let r = (!ok.is_empty()).then(|| ui.button(crate::i18n::tr(ok)));
+ if let Some(r) = &r {
+ crate::widgets::register(ui.ctx(), "button:dialogOk", r.rect);
+ }
+ if r.is_some_and(|r| r.clicked()) {
if informational {
close = true;
} else {
@@ -803,6 +824,17 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) {
match confirm_dialog(app, &dlg) {
// the import review stays open on an error (e.g. an unusable folder template)
Err(e) if matches!(dlg, Dialog::Import { .. }) => app.toast(ctx, e),
+ // the SAM 3 dialog stays open to show the download (or why it can't start)
+ Err(e) if matches!(dlg, Dialog::SamModel { .. }) => {
+ if let Dialog::SamModel { error, .. } = &mut dlg {
+ *error = Some(e);
+ }
+ }
+ Ok(_) if keeps_open(app, &dlg) => {
+ if let Dialog::SamModel { error, .. } = &mut dlg {
+ *error = None;
+ }
+ }
_ => close = true,
}
}
@@ -823,8 +855,86 @@ pub fn fmt_gap(v: f64) -> String {
}
}
+/// Whether a dialog stays open after its action succeeded (the SAM 3 dialog while the model
+/// downloads).
+pub fn keeps_open(app: &LightcraftApp, dlg: &Dialog) -> bool {
+ matches!(dlg, Dialog::SamModel { .. }) && !app.session.segmenter.installed()
+}
+
+/// The SAM 3 dialog: what the model is, its size and licence, and the download's progress.
+fn sam_model_body(app: &mut LightcraftApp, ui: &mut egui::Ui, error: Option<&str>) {
+ use lightcraft_engine::segment::{LICENSE_NAME, LICENSE_URL, MODEL_BYTES};
+ let t = Tokens::get(ui.ctx());
+ let seg = &app.session.segmenter;
+ let d = seg.download_status();
+ if seg.installed() {
+ ui.label(crate::i18n::tr("The SAM 3 model is installed: Object and Describe masks are ready."));
+ return;
+ }
+ let gb = |b: u64| b as f64 / 1e9;
+ ui.label(crate::i18n::tr(
+ "Object and Describe masks use SAM 3, Meta's segmentation model. It isn't part of LightCraft, and everything else works without it.",
+ ));
+ let dir = seg.dir.as_ref().map(|d| d.display().to_string()).unwrap_or_default();
+ ui.label(format!("{} {:.1} GB · {dir}", crate::i18n::tr("Download it once (about):"), gb(MODEL_BYTES)));
+ ui.label(
+ egui::RichText::new(format!(
+ "{} {LICENSE_NAME} — {}",
+ crate::i18n::tr("Licence:"),
+ crate::i18n::tr("Meta's terms, not LightCraft's. Downloading it means accepting them.")
+ ))
+ .color(t.text_label),
+ );
+ let r = ui.link(crate::i18n::tr("Read the SAM License")).on_hover_text(LICENSE_URL);
+ crate::widgets::register(ui.ctx(), "link:samLicense", r.rect);
+ if r.clicked() {
+ let _ = crate::links::open(app, LICENSE_URL);
+ }
+ let seg = &app.session.segmenter;
+ if d.running {
+ ui.add_space(4.0);
+ let frac = if d.total > 0 { d.done as f64 / d.total as f64 } else { 0.0 };
+ let text = format!("{:.2} / {:.2} GB · {}", gb(d.done), gb(d.total), d.file);
+ let r = ui.add(egui::ProgressBar::new(frac as f32).text(text));
+ crate::widgets::register(ui.ctx(), "progress:samDownload", r.rect);
+ let r = ui.button(crate::i18n::tr("Cancel Download"));
+ crate::widgets::register(ui.ctx(), "button:samCancel", r.rect);
+ if r.clicked() {
+ app.session.segmenter.cancel_download();
+ }
+ ui.label(egui::RichText::new(crate::i18n::tr("You can close this: the download continues, and resumes if interrupted.")).color(t.text_dim));
+ ui.ctx().request_repaint_after(std::time::Duration::from_millis(250));
+ return;
+ }
+ if seg.mirrors().is_empty() {
+ ui.label(
+ egui::RichText::new(crate::i18n::tr(
+ "This build has no download location for the model yet: put the files in the folder above yourself (see docs/ai-masks.md).",
+ ))
+ .color(t.text_dim),
+ );
+ }
+ if let Some(e) = error.map(str::to_string).or(d.error) {
+ ui.label(egui::RichText::new(format!("{} {e}", crate::i18n::tr("The download didn't work:"))).color(egui::Color32::from_rgb(230, 90, 80)));
+ }
+}
+
pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result<serde_json::Value, String> {
match dlg {
+ Dialog::SamModel { then, .. } => {
+ if app.session.segmenter.installed() {
+ // installed: start what the user was doing
+ if let Some((kind, op)) = then {
+ crate::panels::masking::begin_ai(app, kind, op)?;
+ }
+ return Ok(serde_json::Value::Null);
+ }
+ let r = app.run("segment.model.download", json!({"acknowledged": true}));
+ if r.is_ok() {
+ app.ui.sam_downloading = true;
+ }
+ r
+ }
Dialog::NewAlbum { name, folder } => app.run("album.create", json!({"name": name, "folder": folder, "addSelected": !folder})),
Dialog::RenameAlbum { id, name } => app.run("album.rename", json!({"id": id, "name": name})),
Dialog::TextPrompt { value, command, params, key, .. } => {
diff --git a/crates/ui-egui/src/panels/masking.rs b/crates/ui-egui/src/panels/masking.rs
index 86fa7aad7..2e150a605 100644
--- a/crates/ui-egui/src/panels/masking.rs
+++ b/crates/ui-egui/src/panels/masking.rs
@@ -106,7 +106,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
app.toast(ui.ctx(), "Click the photo to pick a colour · ⇧-click adds more");
}
"object" => start_object(app, ui.ctx(), "new"),
- "prompt" => app.ui.describe = Some(("new".into(), String::new())),
+ "prompt" => start_describe(app, "new"),
"brush" | "linear" | "radial" => {
app.ui.tool = kind.to_string();
if *kind != "brush" {
@@ -124,9 +124,12 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
}
});
describe_field(app, ui, true);
- let status = if app.session.segmenter.busy() {
+ let seg = &app.session.segmenter;
+ let status = if seg.analyzing() {
Some("Analyzing the photo for AI masks…")
- } else if app.session.segmenter.detail_busy() || app.ui.detail_due.is_some() {
+ } else if seg.busy() {
+ Some("Selecting…")
+ } else if seg.detail_busy() || app.ui.detail_due.is_some() {
Some("Refining the mask's detail…")
} else {
None
@@ -136,6 +139,21 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
ui.label(egui::RichText::new(crate::i18n::tr(status)).color(t.text_dim));
ui.ctx().request_repaint_after(std::time::Duration::from_millis(250));
}
+ // the model download, while its dialog is closed
+ let download = seg.download_status();
+ if download.running && app.ui.dialog.is_none() {
+ ui.add_space(4.0);
+ ui.horizontal(|ui| {
+ let pct = if download.total > 0 { download.done as f64 / download.total as f64 } else { 0.0 };
+ let r = ui.add(
+ egui::ProgressBar::new(pct as f32).desired_width(ui.available_width() - 70.0).text(format!("SAM 3: {} %", (pct * 100.0).floor())),
+ );
+ register(ui.ctx(), "maskSamProgress", r.rect);
+ if text_button(ui, "maskSamDetails", crate::i18n::tr("Details"), false).clicked() {
+ app.offer_sam_download(None);
+ }
+ });
+ }
});
divider(ui);
// mask list
@@ -144,17 +162,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
ui.label(egui::RichText::new(crate::i18n::tr("Masks")).font(t.semibold(13.0)).color(t.text_label));
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
let on = app.ui.mask_overlay;
- if icon_button(
- ui,
- "maskOverlay",
- Icon::Eye,
- vec2(24.0, 24.0),
- on,
- true,
- "Always show the selected mask (O); hovering a mask shows it",
- )
- .clicked()
- {
+ if icon_button(ui, "maskOverlay", Icon::Eye, vec2(24.0, 24.0), on, true, "Show overlay (O)").clicked() {
app.ui.mask_overlay = !on;
}
});
@@ -163,7 +171,6 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
ui.label(egui::RichText::new(crate::i18n::tr("No masks yet. Choose a mask type above.")).color(t.text_dim));
}
let count = d.masks.len();
- app.ui.hover_mask = None;
for (index, m) in d.masks.iter().enumerate() {
let sel = app.session.active_mask == Some(m.id);
if let Some((rid, name)) = app.ui.renaming_mask.as_mut().filter(|(rid, _)| *rid == m.id) {
@@ -187,10 +194,6 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
}
let (r, resp) = ui.allocate_exact_size(vec2(ui.available_width(), 30.0), Sense::click());
register(ui.ctx(), format!("mask:{}", m.id), r);
- // the mask under the pointer shows on the photo (red overlay)
- if ui.rect_contains_pointer(r) {
- app.ui.hover_mask = Some(m.id);
- }
ui.painter().rect_filled(
r,
4.0,
@@ -297,9 +300,6 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
});
let resp = resp.inner;
register(ui.ctx(), format!("component:{i}"), resp.rect);
- if ui.rect_contains_pointer(resp.rect) {
- app.ui.hover_mask = Some(m.id);
- }
if resp.double_clicked() {
app.ui.renaming_component = Some((m.id, i, label.clone()));
}
@@ -635,21 +635,62 @@ fn component_row_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, mask: u32, k:
}
}
+/// Whether the SAM 3 model is missing in a build that could use it: then the download is
+/// offered (with `then` to start afterwards) instead of starting an AI mask.
+fn needs_model(app: &mut LightcraftApp, kind: &str, op: &str) -> bool {
+ let seg = &app.session.segmenter;
+ let missing = lightcraft_engine::segment::Segmenter::AVAILABLE && seg.dir.is_some() && !seg.installed();
+ if missing {
+ app.offer_sam_download(Some((kind, op)));
+ }
+ missing
+}
+
/// Start an Object selection (SAM 3 clicks): a new mask, or a component of the selected one
-/// combined by `op`; the photo is analyzed meanwhile.
-fn start_object(app: &mut LightcraftApp, ctx: &egui::Context, op: &str) {
+/// combined by `op`; the photo is analyzed meanwhile (in the background).
+pub(crate) fn start_object(app: &mut LightcraftApp, ctx: &egui::Context, op: &str) {
+ if needs_model(app, "object", op) {
+ return;
+ }
let r =
if op == "new" { app.run("mask.add", json!({"kind": "object"})) } else { app.run("mask.addComponent", json!({"op": op, "kind": "object"})) };
match r {
Ok(_) => {
app.ui.tool = "object".into();
- // a missing model is reported now rather than at the first click
app.toast(ctx, "Click the object to select it · ⌥-click leaves a part out");
}
- Err(e) => app.toast_error(ctx, e),
+ Err(e) => app.ai_error(ctx, e, Some(("object", op))),
}
}
+/// Start an AI mask of `kind` (object|prompt) combined by `op` (new|add|subtract|intersect),
+/// after the model was installed.
+pub(crate) fn begin_ai(app: &mut LightcraftApp, kind: &str, op: &str) -> Result<serde_json::Value, String> {
+ match kind {
+ "object" => {
+ let r = if op == "new" {
+ app.run("mask.add", json!({"kind": "object"}))
+ } else {
+ app.run("mask.addComponent", json!({"op": op, "kind": "object"}))
+ }?;
+ app.ui.tool = "object".into();
+ Ok(r)
+ }
+ _ => {
+ app.ui.describe = Some((op.to_string(), String::new()));
+ Ok(serde_json::Value::Null)
+ }
+ }
+}
+
+/// Open the Describe field (a new mask, or a component combined by `op`).
+pub(crate) fn start_describe(app: &mut LightcraftApp, op: &str) {
+ if needs_model(app, "prompt", op) {
+ return;
+ }
+ app.ui.describe = Some((op.to_string(), String::new()));
+}
+
/// The Describe field: type what to select ("sky", "the red car") and press Return.
fn describe_field(app: &mut LightcraftApp, ui: &mut egui::Ui, new_mask: bool) {
let Some((op, mut text)) = app.ui.describe.clone() else { return };
@@ -691,13 +732,9 @@ fn describe_field(app: &mut LightcraftApp, ui: &mut egui::Ui, new_mask: bool) {
app.run("mask.addComponent", json!({"op": op, "kind": "prompt", "text": text}))
};
match r {
- Ok(_) => {
- app.ui.describe = None;
- if let Some(mid) = app.session.active_mask {
- app.ui.detail_due = Some((ui.input(|i| i.time) + 0.3, mid));
- }
- }
- Err(e) => app.toast_error(ui.ctx(), e),
+ // the mask appears when the model has found it (a detail pass follows)
+ Ok(_) => app.ui.describe = None,
+ Err(e) => app.ai_error(ui.ctx(), e, Some(("prompt", op.as_str()))),
}
}
}
@@ -711,7 +748,7 @@ fn component_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, op: &str) {
let b = ui.button(crate::i18n::tr("Describe…"));
register(ui.ctx(), format!("maskComp:{op}:prompt"), b.rect);
if b.clicked() {
- app.ui.describe = Some((op.to_string(), String::new()));
+ start_describe(app, op);
}
for (kind, label) in [
("brush", "Brush"),
diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs
index 5b8b21d2f..bdbbcf294 100644
--- a/crates/ui-egui/src/state.rs
+++ b/crates/ui-egui/src/state.rs
@@ -291,9 +291,9 @@ pub struct UiState {
/// (`new` mask, or `add`/`subtract`/`intersect` on the selected one) and the text typed.
#[serde(skip)]
pub describe: Option<(String, String)>,
- /// The mask under the pointer in the Masks list (shown on the photo in red).
+ /// A SAM 3 download was started from the app (to report its end once).
#[serde(skip)]
- pub hover_mask: Option<u32>,
+ pub sam_downloading: bool,
/// When to start the zoomed-in detail pass of an AI mask (app time) and which mask: set by
/// each click or description, so the pass runs once the clicking stops.
#[serde(skip)]
@@ -517,6 +517,15 @@ pub enum Dialog {
Settings {
tab: String,
},
+ /// Object and Describe masks need the SAM 3 model, which isn't installed: offer to download
+ /// it (size, licence, progress). `then`: the AI mask to start once it is there (`kind`
+ /// object|prompt, `op` new|add|subtract|intersect).
+ SamModel {
+ then: Option<(String, String)>,
+ /// Why the download couldn't start (shown in the dialog).
+ #[serde(default)]
+ error: Option<String>,
+ },
/// Confirm moving photos to Recently Deleted.
ConfirmDelete {
count: usize,
@@ -553,7 +562,7 @@ impl Default for UiState {
histogram: true,
soft_proof: false,
proof: lightcraft_engine::pipeline::Proof { dest_warning: false, ..Default::default() },
- mask_overlay: false,
+ mask_overlay: true,
mask_overlay_mode: "color".into(),
mask_overlay_color: [230, 30, 40],
mask_overlay_opacity: 50.0,
@@ -573,8 +582,8 @@ impl Default for UiState {
focus_search: false,
renaming_mask: None,
describe: None,
- hover_mask: None,
detail_due: None,
+ sam_downloading: false,
renaming_component: None,
quit: false,
dragging_photos: None,
diff --git a/crates/ui-egui/src/tests_masking.rs b/crates/ui-egui/src/tests_masking.rs
index 5be82af44..c3aac95c1 100644
--- a/crates/ui-egui/src/tests_masking.rs
+++ b/crates/ui-egui/src/tests_masking.rs
@@ -47,20 +47,14 @@ fn mask_overlay_keys_and_pins() {
exec(&mut h, "mask.add", json!({"kind": "radial", "center": [0.3, 0.4], "rx": 0.1, "ry": 0.1}));
exec(&mut h, "mask.add", json!({"kind": "linear", "start": [0.7, 0.2], "end": [0.7, 0.6]}));
assert_eq!(h.app.session.active_mask, Some(2));
- // no overlay until a mask is hovered in the Masks list; then that mask, in red
+ // the loupe asks the renderer for the selected mask's overlay
let d = develop(&h);
- assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::None);
- h.app.ui.hover_mask = Some(1);
- assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::Mask { id: 1, view: MaskView::Color, color: [230, 30, 40], opacity: 55 });
- h.app.ui.hover_mask = None;
- // O shows the selected mask all the time (and toggles it off again)
- h.request("ui.key", json!({"key": "o"}), T);
- assert!(h.app.ui.mask_overlay);
let o = crate::panels::detail::view_overlay(&h.app, &d);
assert_eq!(o, Overlay::Mask { id: 2, view: MaskView::Color, color: [230, 30, 40], opacity: 50 });
+ // O toggles it, Shift+O cycles the colour (and leaves the crop overlay alone)
h.request("ui.key", json!({"key": "o"}), T);
+ assert!(!h.app.ui.mask_overlay);
assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::None);
- // Shift+O cycles the colour (and leaves the crop overlay alone)
h.request("ui.key", json!({"key": "o"}), T);
let crop = h.app.ui.crop_overlay;
let colour = h.app.ui.mask_overlay_color;
@@ -626,3 +620,53 @@ fn soft_proofing_flags_out_of_gamut_colours_and_makes_proof_copies() {
assert!(!h.app.ui.soft_proof);
assert_ne!(h.app.session.catalog.stack_of(copy.id).unwrap().collapsed, stack.collapsed, "S toggled the stack");
}
+
+/// Object / Describe without the SAM 3 model: the download is offered (never started without a
+/// yes), the dialog says why it can't start when no mirror is configured, nothing freezes and no
+/// empty mask is left behind.
+#[test]
+fn ai_masks_without_the_model_offer_the_download() {
+ use crate::state::Dialog;
+ use lightcraft_engine::segment::Segmenter;
+ let mut h = detail("panel.masking");
+ let dir = std::env::temp_dir().join(format!("lc-ui-no-sam3-{}", std::process::id()));
+ h.app.session.segmenter.dir = Some(dir.clone());
+ h.app.session.segmenter.mirrors_file = Some(dir.join("none.txt"));
+ let t = std::time::Instant::now();
+ let r = h.request("ui.clickWidget", json!({"id": "maskNew:object"}), T);
+ assert_eq!(r["ok"], true, "{r}");
+ assert!(develop(&h).masks.is_empty());
+ if !Segmenter::AVAILABLE {
+ // a build without AI masks says so (a toast), no dialog
+ assert_eq!(h.app.ui.dialog, None);
+ return;
+ }
+ assert_eq!(h.app.ui.dialog, Some(Dialog::SamModel { then: Some(("object".into(), "new".into())), error: None }));
+ assert!(!h.app.session.segmenter.download_status().running, "nothing downloads without a yes");
+ if h.app.session.segmenter.mirrors().is_empty() {
+ // Download: no location configured in this build → the reason, in the dialog
+ let r = h.request("ui.dialog.confirm", json!({}), T);
+ assert_eq!(r["ok"], false, "{r}");
+ assert!(matches!(h.app.ui.dialog, Some(Dialog::SamModel { .. })), "stays open");
+ let r = h.request("ui.clickWidget", json!({"id": "button:dialogOk"}), T);
+ assert_eq!(r["ok"], true, "{r}");
+ let Some(Dialog::SamModel { error: Some(e), .. }) = &h.app.ui.dialog else { panic!("{:?}", h.app.ui.dialog) };
+ assert!(e.contains("LIGHTCRAFT_SAM3_MIRRORS"), "{e}");
+ }
+ // (a frame with the message laid out, so the buttons are where they are drawn)
+ h.step();
+ h.step();
+ let r = h.request("ui.clickWidget", json!({"id": "button:dialogCancel"}), T);
+ assert_eq!(r["ok"], true, "{r}");
+ assert_eq!(h.app.ui.dialog, None);
+ // Describe asks too
+ let r = h.request("ui.clickWidget", json!({"id": "maskNew:prompt"}), T);
+ assert_eq!(r["ok"], true, "{r}");
+ assert_eq!(h.app.ui.dialog, Some(Dialog::SamModel { then: Some(("prompt".into(), "new".into())), error: None }));
+ assert_eq!(h.app.ui.describe, None);
+ // a click command from an agent gets the not-installed error at once, and the app offers the model
+ let e = h.app.run("mask.add", json!({"kind": "prompt", "text": "sky"})).unwrap_err();
+ assert!(e.starts_with(lightcraft_engine::segment::NOT_INSTALLED), "{e}");
+ assert!(develop(&h).masks.is_empty());
+ assert!(t.elapsed() < SETTLE, "{:?}", t.elapsed());
+}
diff --git a/docs/ai-masks.md b/docs/ai-masks.md
index dc3e6726e..26d4f717f 100644
--- a/docs/ai-masks.md
+++ b/docs/ai-masks.md
@@ -4,6 +4,10 @@ LightCraft selects objects with **SAM 3** (Segment Anything with Concepts, Meta
pure Rust by `crates/segment` on [candle](https://github.com/huggingface/candle): on the GPU
through Metal on macOS, on the CPU elsewhere (for now).
+**The model is optional.** It is not part of LightCraft, and nothing requires it: without it,
+Object and Describe offer to download it, every other feature (and every existing mask, see
+below) works, and no render, export or command ever waits for it.
+
## Using them
In the Masking panel (M):
@@ -19,85 +23,176 @@ In the Masking panel (M):
the people out of a sky mask; − ▸ Object then clicks remove one object.
- **Edge** (per selection, −100…100): below 0 a crisper border, above 0 a feathered one (up to 2 %
of the long edge, so previews and exports match).
-- Hover a mask in the Masks list to see it in red on the photo; O shows the selected one all the
- time.
- **Detail:** about a second after the last click (or right after a description), the photo
around the selection is analyzed again, zoomed in, from a sharper source (~5 s on an M4 Pro,
- in the background); the result is kept as a high-resolution patch, so small objects get 5–10×
- finer edges than one pass over the whole photo gives.
+ in the background); the result is kept as a high-resolution patch (at most 4 per selection),
+ so small objects get 5–10× finer edges than one pass over the whole photo gives.
The first Object or Describe on a photo analyzes it (the image encoder runs once per photo and
look: ~3 s on an Apple M4 Pro, plus a one-time ~6 s for compiling GPU kernels and loading the
model after launch). That starts in the background as soon as you pick Object; after it, each
-click takes ~35 ms and each description ~0.6 s.
+click takes ~35 ms and each description ~0.6 s. Everything the model does runs on its own worker
+thread: the window stays responsive, a click shows its dot at once and the selection follows
+when it's computed ("Selecting…" / "Analyzing the photo…" under the mask tiles). The model is
+unloaded after 10 minutes without use (it takes several GB of memory) and loaded again when
+needed.
The result is stored with the mask: the model's 288 × 288 mask logits over the uncropped photo
(quantized, compressed, ~10–30 KB). Renders and exports sample them at any resolution and never
need the model, so masks survive moving the library to a machine without it, and the web build
renders them. Editing the photo's look later does not recompute a mask (click again to update).
-## Installing the model
+## Getting the model
+
+### In the app (recommended)
+
+The first time you pick Object or Describe without the model, LightCraft asks:
+
+> **Download the SAM 3 model?** Object and Describe masks use SAM 3, Meta's segmentation model.
+> It isn't part of LightCraft, and everything else works without it. Download it once (about
+> 3.4 GB) · *folder*. Licence: SAM License (Meta) — Meta's terms, not LightCraft's. Downloading
+> it means accepting them. [Read the SAM License] — **Not Now** / **Download**
+
+Nothing is downloaded until you choose **Download**. The download runs in the background (close
+the dialog and keep working; the Masking panel shows its progress), can be cancelled, resumes
+where it stopped if interrupted (also after quitting), and tries the next download location when
+one fails or stalls. Each file is written as `<name>.part` and moved into the model folder only
+when complete and verified — `model.safetensors` must have the exact size and SHA-256 of Meta's
+official checkpoint; a file that doesn't match is deleted, never used. If no location works, the
+dialog says why and LightCraft carries on without the model.
+
+Agents and scripts use the same commands (control channel, `lightcraft-cli mcp --connect`):
+`segment.model.status`, `segment.model.download {"acknowledged": true}` (refused without the
+explicit `acknowledged`: pass it only after the user agreed to the download and the licence),
+`segment.model.cancel`.
+
+### Download locations
+
+The model is fetched from an ordered list of mirrors: base URLs where `<base>/model.safetensors`,
+`<base>/vocab.json` and `<base>/merges.txt` live. The list is, in order:
+
+1. `LIGHTCRAFT_SAM3_MIRRORS` (environment variable; URLs separated by commas or spaces);
+2. the file `models/sam3-mirrors.txt` in LightCraft's settings folder (one URL per line, `#`
+ comments) — `~/Library/Application Support/LightCraft/` (macOS), `%APPDATA%\LightCraft\`
+ (Windows), `~/.config/lightcraft/` (Linux);
+3. the built-in list, `DEFAULT_MIRRORS` in `crates/segment/src/fetch/mod.rs`.
-The weights are not part of LightCraft: they are Meta's `facebook/sam3` checkpoint on Hugging
-Face, under the SAM License, and access is gated. Once:
+> **Maintainers:** the built-in list is **empty** until LightCraft's own CDN locations exist
+> (see the `TODO(maintainer)` there): add them in order of preference, host the three files
+> unchanged, and pin `vocab.json` / `merges.txt` (size + SHA-256) in `SAM3_FILES` at the same
+> time. Until then the in-app download needs a user-configured mirror, and the dialog says so.
-1. Open <https://huggingface.co/facebook/sam3>, accept the license and wait for approval.
-2. Create a read token (Hugging Face ▸ Settings ▸ Access Tokens), or log in with `hf auth login`.
-3. Run the installer from the repository:
+Hugging Face's `facebook/sam3` can't be a default: it is **gated** (each person must accept the
+licence there, wait for approval and download with their own token). Someone with access can
+use it as their mirror:
```sh
-HF_TOKEN=hf_... tools/install-sam3.sh # macOS, Linux
-tools/install-sam3.sh --check # verify an installation
+LIGHTCRAFT_SAM3_MIRRORS=https://huggingface.co/facebook/sam3/resolve/main \
+LIGHTCRAFT_SAM3_TOKEN=hf_... lightcraft
```
-```powershell
-$env:HF_TOKEN = "hf_..."; .\tools\install-sam3.ps1 # Windows
-```
+(`LIGHTCRAFT_SAM3_TOKEN` is sent as a bearer token only over https, and only to the mirror's own
+host — not to the storage host it redirects to.)
-It downloads `model.safetensors` (3.4 GB, fp32; resumes when interrupted, checked against the
-official SHA-256), `vocab.json`, `merges.txt` and the small config files into the folder
-LightCraft looks in: `~/Library/Application Support/LightCraft/models/sam3/` (macOS),
-`%APPDATA%\LightCraft\models\sam3` (Windows), `~/.config/lightcraft/models/sam3` (Linux).
-`--dir` (or `LIGHTCRAFT_SAM3_DIR`, which LightCraft also reads) picks another folder; `--repo`
-another Hugging Face repository with the same files. Restart LightCraft afterwards.
+The download is plain HTTP/1.1 over TLS in pure Rust (rustls with the RustCrypto provider and
+the Mozilla root certificates; no OpenSSL, `ring` or `aws-lc`, nothing compiled from C), with a
+15 s connect timeout and a 30 s stall timeout; it follows redirects but doesn't use HTTP proxies.
-Nothing else is needed at run time: the model runs in LightCraft itself (pure Rust, no Python).
-Without the files, Object and Describe say where they are expected.
+### By hand
-**Developers:** the accuracy test against the Python reference needs `torch` and
-`transformers` (`tools/requirements-sam3-reference.txt`):
+Put `model.safetensors`, `vocab.json` and `merges.txt` from `facebook/sam3` in the model folder:
+`models/sam3/` in the settings folder above, or the folder `LIGHTCRAFT_SAM3_DIR` names. Developers
+can use the installer scripts, which download from Hugging Face with your token and verify the
+weights:
```sh
-python3 -m venv .venv-sam3 && .venv-sam3/bin/pip install -r tools/requirements-sam3-reference.txt
-.venv-sam3/bin/python tools/sam3_reference.py photo.jpg ref.safetensors
-LIGHTCRAFT_SAM3_DIR=<model dir> LIGHTCRAFT_SAM3_REF=ref.safetensors cargo test -p lightcraft-segment --release -- --nocapture
+HF_TOKEN=hf_... tools/install-sam3.sh # macOS, Linux
+tools/install-sam3.sh --check # verify an installation
```
+```powershell
+$env:HF_TOKEN = "hf_..."; .\tools\install-sam3.ps1 # Windows
+```
+
+## Licence of the model
+
+LightCraft's code is MIT OR Apache-2.0; `crates/segment` (a port of Apache-2.0 code) is
+Apache-2.0. **The SAM 3 weights are neither**: they are Meta's, released under the **SAM
+License** (<https://github.com/facebookresearch/sam3/blob/main/LICENSE>), a custom licence that
+is not an OSI open-source licence. It grants a non-exclusive, worldwide, non-transferable,
+royalty-free licence to use, reproduce, distribute, copy, modify and create derivative works of
+the "SAM Materials", subject to its terms, including: anyone redistributing them must provide a
+copy of the licence with them, and derivative works are distributed under its terms; uses
+subject to ITAR, military or warfare purposes, nuclear industries, espionage, guns or illegal
+weapons, and reverse engineering are prohibited, and trade controls (sanctions, export rules)
+apply; the materials come "as is", without warranty; Meta may terminate the licence on a breach
+(then the materials must be deleted); California law governs. Read the licence itself — this
+summary is not legal advice.
+
+Therefore:
+
+- **The weights are never part of this repository, its releases or installers**, and must not be
+ committed.
+- They reach a user's computer only when that user asks for them, after the dialog names the
+ licence (or when they install them by hand).
+- Whoever hosts a mirror redistributes the SAM Materials and must follow the licence (ship it
+ next to the files).
+
## Commands (control channel, MCP)
| Command | Params | |
|---|---|---|
-| `mask.add` | `{kind: "object", points?: [[x,y],…], exclude?: [[x,y],…]}` | an Object mask (empty until clicked) |
-| `mask.add` | `{kind: "prompt", text}` | a Describe mask |
+| `mask.add` | `{kind: "object", points?: [[x,y],…], exclude?: [[x,y],…], seg?}` | an Object mask (empty until clicked) |
+| `mask.add` | `{kind: "prompt", text, seg?}` | a Describe mask |
| `mask.addComponent` | `{op: add\|subtract\|intersect, kind: "object"\|"prompt", …}` | the same as a component |
-| `mask.objectPoint` | `{x, y, exclude?: bool, id?}` | one click on the selected mask's Object selection |
-| `mask.refineDetail` | `{id?, component?}` | the zoomed-in detail pass, in the background → `{started}` |
-| `segment.prepare` | `{}` | analyze the active photo in the background → `{busy}` |
-
-Coordinates are normalized to the uncropped, oriented photo, like every mask shape.
+| `mask.objectPoint` | `{x, y, exclude?: bool, id?}` | one click on the selected mask's Object selection (≤ 64 clicks) |
+| `mask.refineDetail` | `{id?, component?}` | the zoomed-in detail pass → `{started}` |
+| `segment.prepare` | `{}` | load the model and analyze the active photo → `{busy}` |
+| `segment.model.status` | `{}` | `{available, installed, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download}` |
+| `segment.model.download` | `{acknowledged: true}` | start the download in the background → `{started}` |
+| `segment.model.cancel` | `{}` | stop it (it resumes next time) → `{cancelled}` |
+
+Coordinates are normalized to the uncropped, oriented photo, like every mask shape. `seg` passes
+a stored segmentation (as `mask.list` / the develop settings hold it): the mask is made without
+the model. Without the model, the AI kinds fail with an error starting "The SAM 3 model is not
+installed" that says how to get it.
+
+In the desktop app the commands don't wait for the model: `mask.objectPoint` returns
+`{pending: true}` and the selection is applied when it's computed (one undo step per settled
+selection); `mask.add {kind: "prompt"}` returns `{pending: true}` and the mask appears when
+something is found (or a message says nothing was). `lightcraft-cli` and the headless MCP server
+wait and return the result (when built with the `sam` feature; the default CLI has no AI masks).
## Implementation notes
- `crates/segment` ports the Hugging Face `transformers` implementation (Apache-2.0; see
- `NOTICE`): the 32-layer ViT backbone with 2-D RoPE and windowed attention, the feature
- pyramids, the SAM 2-style prompt encoder and two-way mask decoder for clicks, and the CLIP text
- encoder, DETR encoder/decoder (box relative-position bias, presence token) and pixel decoder
- for text. The CLIP tokenizer is a small BPE in `tokenizer.rs`.
+ `NOTICE` — the ported files say so and that they were modified): the 32-layer ViT backbone
+ with 2-D RoPE and windowed attention, the feature pyramids, the SAM 2-style prompt encoder and
+ two-way mask decoder for clicks, and the CLIP text encoder, DETR encoder/decoder (box
+ relative-position bias, presence token) and pixel decoder for text. The CLIP tokenizer is a
+ small BPE in `tokenizer.rs`. `fetch/` is the downloader (original code).
- candle is pinned at 0.9.2: later releases make `candle-core` depend on `tokenizers` with the
Oniguruma C library, and the product is pure Rust. Weights are read with positional reads (no
- memory map: `unsafe` stays in `lightcraft-sysmem`), and only the tensors a path needs.
+ memory map: `unsafe` stays in `lightcraft-sysmem`), only the tensors a path needs, and every
+ tensor's byte range is checked against the file's length when it is opened (a truncated or
+ damaged file is an error, never a crash).
+- The engine (`crates/engine/src/segment/`) owns one worker thread with the model and the last
+ encoded photo; the session sends it jobs and receives results over channels, never holding a
+ lock the model holds. Each job runs under `catch_unwind`: a panic inside candle becomes an
+ error and the model is reloaded on the next request; busy counters are reset by drop guards.
+- Hostile documents: a selection keeps at most 4 detail patches (when read and when rendered),
+ each at most 1024² logits; more than 64 clicks are never sent to the model.
- Accuracy: `crates/segment/tests/reference.rs` compares every stage against the Python
reference (`tools/sam3_reference.py`); on Metal the backbone matches to 7e-5 relative error
and masks to ~5e-5. f16 was tried and rejected (overflow, ~7 % faster).
- The engine feature `sam` (on in the desktop app, off for web and CLI) gates the model; without
it the commands report that the build has no AI masks.
+
+**Developers:** the accuracy test against the Python reference needs `torch` and
+`transformers` (`tools/requirements-sam3-reference.txt`):
+
+```sh
+python3 -m venv .venv-sam3 && .venv-sam3/bin/pip install -r tools/requirements-sam3-reference.txt
+.venv-sam3/bin/python tools/sam3_reference.py photo.jpg ref.safetensors
+LIGHTCRAFT_SAM3_DIR=<model dir> LIGHTCRAFT_SAM3_REF=ref.safetensors cargo test -p lightcraft-segment --release -- --nocapture
+```
diff --git a/docs/parity.md b/docs/parity.md
index c15d1c098..827299711 100644
--- a/docs/parity.md
+++ b/docs/parity.md
@@ -69,9 +69,10 @@ Take the first one nobody is working on.
then tune tone, highlights, texture/clarity/dehaze, NR and sharpening against it.
5. **LR-EDIT-OPTICS-PROFILE** (P1): a lens-profile database of our own (embedded DNG/maker corrections work today).
6. **AI masks and Enhance** (LR-MASK-SUBJECT / SKY / PEOPLE / OBJECTS, LR-EDIT-DETAIL-DENOISE, SUPERRES, LENSBLUR):
- Object and Describe masks run SAM 3 in pure Rust (`crates/segment`) with user-installed weights; Subject / Sky /
- People could use the same model with fixed prompts. Bundling or recommending weights (SAM License) and denoise /
- super-resolution models remain a maintainer decision.
+ Object and Describe masks run SAM 3 in pure Rust (`crates/segment`); the weights (SAM License) are never bundled:
+ the app offers a consented, verified download, but **LightCraft's CDN mirrors are not configured yet** (the
+ built-in list is empty, so today users need their own mirror or a manual install). Subject / Sky / People could
+ use the same model with fixed prompts; denoise / super-resolution models remain a maintainer decision.
7. **HDR** (Q. HDR, LR-EXP-HDR), **video** (R. Video), **Classic output modules** (Map view, Book, Slideshow module,
Print, publish): large, well understood, lower priority than 1–5.
@@ -329,7 +330,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
| LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model |
| LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`) | heuristic |
| LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject heuristic |
-| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:segment.prepare`, `crates/segment`, `crates/engine/src/segment.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components. The segmentation is stored with the mask (288² logits), so renders and exports never need the model. Weights are not bundled: the user installs the `facebook/sam3` checkpoint (SAM License) — see docs/ai-masks.md. + / − under the selected mask combine selections; comma lists (`car, road`); a zoomed-in detail pass (`cmd:mask.refineDetail`, background) for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft); hovering a mask shows it in red. No brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro) |
+| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand; see docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) |
| LR-MASK-PEOPLE | People parts | P2 | ⬜ | | |
| LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty |
| LR-MASK-BRUSH | Brush mask | P0 | ✅ | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs` | size/feather/flow/density/erase; Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); no A/B brushes, no pressure |
diff --git a/docs/whats-new.md b/docs/whats-new.md
index 12984b9a0..004801326 100644
--- a/docs/whats-new.md
+++ b/docs/whats-new.md
@@ -131,6 +131,12 @@
lost everything at quit; a temporary session shows a banner the whole time and never writes to your library.
### Editing
+- AI masks with SAM 3 (Object and Describe in the Masking panel): click an object to select it (⌥-click leaves a
+ part out), or type what to select ("sky", "the red car", "car, road"); both combine with other masks, have an
+ Edge setting, and get a sharper zoomed-in pass in the background. The model runs inside LightCraft in pure Rust
+ and never freezes the window. It is optional and not part of LightCraft (Meta's SAM License, about 3.4 GB): the
+ first time you use an AI mask, LightCraft asks before downloading it, shows the progress, can cancel and resume,
+ and checks the file before using it. Masks keep their selection, so they render and export without the model.
- Auto Sync: edits apply to every selected photo. Auto B&W mix. Automatic versions.
- Colour-range masks: click the photo to sample. Luminance ranges: range bar, smoothness, luminance map.
- ⌘-drag to straighten, ⇧G Guided Upright, a grid while transforming.
diff --git a/tools/install-sam3.ps1 b/tools/install-sam3.ps1
index bd43fd98b..0036c5616 100644
--- a/tools/install-sam3.ps1
+++ b/tools/install-sam3.ps1
@@ -1,5 +1,6 @@
# Install the SAM 3 model for LightCraft's Object and Describe masks on Windows
-# (see docs/ai-masks.md; tools/install-sam3.sh is the macOS/Linux version).
+# (see docs/ai-masks.md; tools/install-sam3.sh is the macOS/Linux version). The desktop app
+# offers to download the model itself when an AI mask first needs it; this is for developers.
#
# facebook/sam3 is gated: accept the SAM License at https://huggingface.co/facebook/sam3, wait
# for approval, then: $env:HF_TOKEN = "hf_..." ; .\tools\install-sam3.ps1
@@ -34,7 +35,26 @@ if ($Token) { $Headers["Authorization"] = "Bearer $Token" }
$Base = "https://huggingface.co/$Repo/resolve/main"
foreach ($f in $Files + "model.safetensors") {
Write-Host "fetching $f"
- Invoke-WebRequest -Uri "$Base/$f" -Headers $Headers -OutFile (Join-Path $Dir $f)
+ # into a .part file, moved into place only when complete (and, for the weights, verified):
+ # a failed or damaged download never ends up under the real name
+ $final = Join-Path $Dir $f
+ $part = "$final.part"
+ Remove-Item -Force -ErrorAction SilentlyContinue $part
+ try {
+ Invoke-WebRequest -Uri "$Base/$f" -Headers $Headers -OutFile $part
+ } catch {
+ Remove-Item -Force -ErrorAction SilentlyContinue $part
+ throw "download of $Repo/$f failed: $($_.Exception.Message)"
+ }
+ if ($f -eq "model.safetensors") {
+ Write-Host "checking model.safetensors (SHA-256 of 3.4 GB)..."
+ $got = (Get-FileHash $part -Algorithm SHA256).Hash.ToLower()
+ if ($got -ne $Sha256) {
+ Remove-Item -Force $part
+ throw "the download is damaged or not the official checkpoint (got $got); deleted it"
+ }
+ }
+ Move-Item -Force $part $final
}
Test-Install
Write-Host "Restart LightCraft; Object and Describe in the Masking panel now use it."
diff --git a/tools/install-sam3.sh b/tools/install-sam3.sh
index 86554c39d..4ec9d0a9c 100755
--- a/tools/install-sam3.sh
+++ b/tools/install-sam3.sh
@@ -1,6 +1,9 @@
#!/bin/sh
# Install the SAM 3 model for LightCraft's Object and Describe masks (see docs/ai-masks.md).
#
+# The desktop app offers to download the model itself when an AI mask first needs it; this script
+# is for developers (reference tests, benchmarks) and for installing from Hugging Face directly.
+#
# The weights are not part of LightCraft: they are Meta's facebook/sam3 checkpoint on Hugging
# Face, under the SAM License. Access is gated: open https://huggingface.co/facebook/sam3, accept
# the license, wait for approval, then run this with a token that can read it:
@@ -75,20 +78,29 @@ fi
mkdir -p "$DIR"
BASE="https://huggingface.co/$REPO/resolve/main"
-fetch() { # fetch FILE DEST [resume]
+# fetch FILE DEST [resume]: HTTP errors never reach DEST (--fail writes no error page), and a
+# resume the server refuses (416) starts over instead of counting as done.
+fetch() {
set -- "$1" "$2" "${3:-}"
if [ -n "$TOKEN" ]; then auth="Authorization: Bearer $TOKEN"; else auth="X-No-Auth: 1"; fi
- code=$(curl -sSL ${3:+-C -} -H "$auth" -w '%{http_code}' -o "$2" "$BASE/$1") || code=$?
+ code=$(curl -fsSL ${3:+-C -} -H "$auth" -w '%{http_code}' -o "$2" "$BASE/$1") || true
case "$code" in
- 200|206|416) return 0 ;;
+ 200|206) return 0 ;;
+ 416)
+ echo "the partial download of $1 doesn't match the server's; starting over" >&2
+ rm -f "$2"
+ code=$(curl -fsSL -H "$auth" -w '%{http_code}' -o "$2" "$BASE/$1") || true
+ [ "$code" = 200 ] && return 0
+ echo "download of $REPO/$1 failed ($code)" >&2; return 1 ;;
401|403) echo "access denied to $REPO/$1 (has your request for facebook/sam3 been approved?)" >&2; return 1 ;;
- *) echo "download of $REPO/$1 failed ($code)" >&2; return 1 ;;
+ *) echo "download of $REPO/$1 failed (${code:-no response})" >&2; return 1 ;;
esac
}
for f in $FILES; do
echo "fetching $f"
- fetch "$f" "$DIR/$f"
+ # small files: whole, into place only when complete
+ fetch "$f" "$DIR/$f.part" && mv -f "$DIR/$f.part" "$DIR/$f" || { rm -f "$DIR/$f.part"; exit 1; }
done
if [ -s "$DIR/model.safetensors" ] && [ "$(sha256 "$DIR/model.safetensors")" = "$SHA256" ]; then
@@ -96,13 +108,21 @@ if [ -s "$DIR/model.safetensors" ] && [ "$(sha256 "$DIR/model.safetensors")" = "
else
echo "fetching model.safetensors (3.4 GB; resumes if interrupted)"
part="$DIR/model.safetensors.part"
- [ -s "$DIR/model.safetensors" ] && mv -f "$DIR/model.safetensors" "$part"
- fetch model.safetensors "$part" resume
+ # a wrong model.safetensors is never resumed from: it goes
+ rm -f "$DIR/model.safetensors"
+ if [ -f "$part" ] && [ "$(wc -c < "$part" | tr -d ' ')" -ge "$SIZE" ]; then rm -f "$part"; fi
+ fetch model.safetensors "$part" resume || exit 1
have=$(wc -c < "$part" | tr -d ' ')
- if [ "$have" != "$SIZE" ]; then
+ if [ "$have" -lt "$SIZE" ]; then
echo "incomplete download ($have of $SIZE bytes); run the script again to resume" >&2
exit 1
fi
+ echo "checking model.safetensors (SHA-256 of 3.4 GB, takes a few seconds)…"
+ if [ "$have" != "$SIZE" ] || [ "$(sha256 "$part")" != "$SHA256" ]; then
+ rm -f "$part"
+ echo "the download is damaged or not the official checkpoint; deleted it, run the script again" >&2
+ exit 1
+ fi
mv -f "$part" "$DIR/model.safetensors"
fi